1 //
    2 // Copyright (c) 2003, 2023, Oracle and/or its affiliates. All rights reserved.
    3 // Copyright (c) 2014, 2021, Red Hat, Inc. All rights reserved.
    4 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
    5 //
    6 // This code is free software; you can redistribute it and/or modify it
    7 // under the terms of the GNU General Public License version 2 only, as
    8 // published by the Free Software Foundation.
    9 //
   10 // This code is distributed in the hope that it will be useful, but WITHOUT
   11 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
   12 // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
   13 // version 2 for more details (a copy is included in the LICENSE file that
   14 // accompanied this code).
   15 //
   16 // You should have received a copy of the GNU General Public License version
   17 // 2 along with this work; if not, write to the Free Software Foundation,
   18 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
   19 //
   20 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
   21 // or visit www.oracle.com if you need additional information or have any
   22 // questions.
   23 //
   24 //
   25 
   26 // AArch64 Architecture Description File
   27 
   28 //----------REGISTER DEFINITION BLOCK------------------------------------------
   29 // This information is used by the matcher and the register allocator to
   30 // describe individual registers and classes of registers within the target
   31 // architecture.
   32 
   33 register %{
   34 //----------Architecture Description Register Definitions----------------------
   35 // General Registers
   36 // "reg_def"  name ( register save type, C convention save type,
   37 //                   ideal register type, encoding );
   38 // Register Save Types:
   39 //
   40 // NS  = No-Save:       The register allocator assumes that these registers
   41 //                      can be used without saving upon entry to the method, &
   42 //                      that they do not need to be saved at call sites.
   43 //
   44 // SOC = Save-On-Call:  The register allocator assumes that these registers
   45 //                      can be used without saving upon entry to the method,
   46 //                      but that they must be saved at call sites.
   47 //
   48 // SOE = Save-On-Entry: The register allocator assumes that these registers
   49 //                      must be saved before using them upon entry to the
   50 //                      method, but they do not need to be saved at call
   51 //                      sites.
   52 //
   53 // AS  = Always-Save:   The register allocator assumes that these registers
   54 //                      must be saved before using them upon entry to the
   55 //                      method, & that they must be saved at call sites.
   56 //
   57 // Ideal Register Type is used to determine how to save & restore a
   58 // register.  Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
   59 // spilled with LoadP/StoreP.  If the register supports both, use Op_RegI.
   60 //
   61 // The encoding number is the actual bit-pattern placed into the opcodes.
   62 
   63 // We must define the 64 bit int registers in two 32 bit halves, the
   64 // real lower register and a virtual upper half register. upper halves
   65 // are used by the register allocator but are not actually supplied as
   66 // operands to memory ops.
   67 //
   68 // follow the C1 compiler in making registers
   69 //
   70 //   r0-r7,r10-r26 volatile (caller save)
   71 //   r27-r32 system (no save, no allocate)
   72 //   r8-r9 non-allocatable (so we can use them as scratch regs)
   73 //
   74 // as regards Java usage. we don't use any callee save registers
   75 // because this makes it difficult to de-optimise a frame (see comment
   76 // in x86 implementation of Deoptimization::unwind_callee_save_values)
   77 //
   78 
   79 // General Registers
   80 
   81 reg_def R0      ( SOC, SOC, Op_RegI,  0, r0->as_VMReg()         );
   82 reg_def R0_H    ( SOC, SOC, Op_RegI,  0, r0->as_VMReg()->next() );
   83 reg_def R1      ( SOC, SOC, Op_RegI,  1, r1->as_VMReg()         );
   84 reg_def R1_H    ( SOC, SOC, Op_RegI,  1, r1->as_VMReg()->next() );
   85 reg_def R2      ( SOC, SOC, Op_RegI,  2, r2->as_VMReg()         );
   86 reg_def R2_H    ( SOC, SOC, Op_RegI,  2, r2->as_VMReg()->next() );
   87 reg_def R3      ( SOC, SOC, Op_RegI,  3, r3->as_VMReg()         );
   88 reg_def R3_H    ( SOC, SOC, Op_RegI,  3, r3->as_VMReg()->next() );
   89 reg_def R4      ( SOC, SOC, Op_RegI,  4, r4->as_VMReg()         );
   90 reg_def R4_H    ( SOC, SOC, Op_RegI,  4, r4->as_VMReg()->next() );
   91 reg_def R5      ( SOC, SOC, Op_RegI,  5, r5->as_VMReg()         );
   92 reg_def R5_H    ( SOC, SOC, Op_RegI,  5, r5->as_VMReg()->next() );
   93 reg_def R6      ( SOC, SOC, Op_RegI,  6, r6->as_VMReg()         );
   94 reg_def R6_H    ( SOC, SOC, Op_RegI,  6, r6->as_VMReg()->next() );
   95 reg_def R7      ( SOC, SOC, Op_RegI,  7, r7->as_VMReg()         );
   96 reg_def R7_H    ( SOC, SOC, Op_RegI,  7, r7->as_VMReg()->next() );
   97 reg_def R8      ( NS,  SOC, Op_RegI,  8, r8->as_VMReg()         ); // rscratch1, non-allocatable
   98 reg_def R8_H    ( NS,  SOC, Op_RegI,  8, r8->as_VMReg()->next() );
   99 reg_def R9      ( NS,  SOC, Op_RegI,  9, r9->as_VMReg()         ); // rscratch2, non-allocatable
  100 reg_def R9_H    ( NS,  SOC, Op_RegI,  9, r9->as_VMReg()->next() );
  101 reg_def R10     ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()        );
  102 reg_def R10_H   ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
  103 reg_def R11     ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()        );
  104 reg_def R11_H   ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
  105 reg_def R12     ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()        );
  106 reg_def R12_H   ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()->next());
  107 reg_def R13     ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()        );
  108 reg_def R13_H   ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()->next());
  109 reg_def R14     ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()        );
  110 reg_def R14_H   ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()->next());
  111 reg_def R15     ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()        );
  112 reg_def R15_H   ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()->next());
  113 reg_def R16     ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()        );
  114 reg_def R16_H   ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
  115 reg_def R17     ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()        );
  116 reg_def R17_H   ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
  117 reg_def R18     ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()        );
  118 reg_def R18_H   ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()->next());
  119 reg_def R19     ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()        );
  120 reg_def R19_H   ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()->next());
  121 reg_def R20     ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()        ); // caller esp
  122 reg_def R20_H   ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()->next());
  123 reg_def R21     ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()        );
  124 reg_def R21_H   ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()->next());
  125 reg_def R22     ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()        );
  126 reg_def R22_H   ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()->next());
  127 reg_def R23     ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()        );
  128 reg_def R23_H   ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()->next());
  129 reg_def R24     ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()        );
  130 reg_def R24_H   ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()->next());
  131 reg_def R25     ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()        );
  132 reg_def R25_H   ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()->next());
  133 reg_def R26     ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()        );
  134 reg_def R26_H   ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()->next());
  135 reg_def R27     ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()        ); // heapbase
  136 reg_def R27_H   ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()->next());
  137 reg_def R28     (  NS, SOE, Op_RegI, 28, r28->as_VMReg()        ); // thread
  138 reg_def R28_H   (  NS, SOE, Op_RegI, 28, r28->as_VMReg()->next());
  139 reg_def R29     (  NS,  NS, Op_RegI, 29, r29->as_VMReg()        ); // fp
  140 reg_def R29_H   (  NS,  NS, Op_RegI, 29, r29->as_VMReg()->next());
  141 reg_def R30     (  NS,  NS, Op_RegI, 30, r30->as_VMReg()        ); // lr
  142 reg_def R30_H   (  NS,  NS, Op_RegI, 30, r30->as_VMReg()->next());
  143 reg_def R31     (  NS,  NS, Op_RegI, 31, r31_sp->as_VMReg()     ); // sp
  144 reg_def R31_H   (  NS,  NS, Op_RegI, 31, r31_sp->as_VMReg()->next());
  145 
  146 // ----------------------------
  147 // Float/Double/Vector Registers
  148 // ----------------------------
  149 
  150 // Double Registers
  151 
  152 // The rules of ADL require that double registers be defined in pairs.
  153 // Each pair must be two 32-bit values, but not necessarily a pair of
  154 // single float registers. In each pair, ADLC-assigned register numbers
  155 // must be adjacent, with the lower number even. Finally, when the
  156 // CPU stores such a register pair to memory, the word associated with
  157 // the lower ADLC-assigned number must be stored to the lower address.
  158 
  159 // AArch64 has 32 floating-point registers. Each can store a vector of
  160 // single or double precision floating-point values up to 8 * 32
  161 // floats, 4 * 64 bit floats or 2 * 128 bit floats.  We currently only
  162 // use the first float or double element of the vector.
  163 
  164 // for Java use float registers v0-v15 are always save on call whereas
  165 // the platform ABI treats v8-v15 as callee save). float registers
  166 // v16-v31 are SOC as per the platform spec
  167 
  168 // For SVE vector registers, we simply extend vector register size to 8
  169 // 'logical' slots. This is nominally 256 bits but it actually covers
  170 // all possible 'physical' SVE vector register lengths from 128 ~ 2048
  171 // bits. The 'physical' SVE vector register length is detected during
  172 // startup, so the register allocator is able to identify the correct
  173 // number of bytes needed for an SVE spill/unspill.
  174 // Note that a vector register with 4 slots denotes a 128-bit NEON
  175 // register allowing it to be distinguished from the corresponding SVE
  176 // vector register when the SVE vector length is 128 bits.
  177 
  178   reg_def V0   ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()          );
  179   reg_def V0_H ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next()  );
  180   reg_def V0_J ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(2) );
  181   reg_def V0_K ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(3) );
  182 
  183   reg_def V1   ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()          );
  184   reg_def V1_H ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next()  );
  185   reg_def V1_J ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(2) );
  186   reg_def V1_K ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(3) );
  187 
  188   reg_def V2   ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()          );
  189   reg_def V2_H ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next()  );
  190   reg_def V2_J ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(2) );
  191   reg_def V2_K ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(3) );
  192 
  193   reg_def V3   ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()          );
  194   reg_def V3_H ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next()  );
  195   reg_def V3_J ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(2) );
  196   reg_def V3_K ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(3) );
  197 
  198   reg_def V4   ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()          );
  199   reg_def V4_H ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next()  );
  200   reg_def V4_J ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(2) );
  201   reg_def V4_K ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(3) );
  202 
  203   reg_def V5   ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()          );
  204   reg_def V5_H ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next()  );
  205   reg_def V5_J ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(2) );
  206   reg_def V5_K ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(3) );
  207 
  208   reg_def V6   ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()          );
  209   reg_def V6_H ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next()  );
  210   reg_def V6_J ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(2) );
  211   reg_def V6_K ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(3) );
  212 
  213   reg_def V7   ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()          );
  214   reg_def V7_H ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next()  );
  215   reg_def V7_J ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(2) );
  216   reg_def V7_K ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(3) );
  217 
  218   reg_def V8   ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()          );
  219   reg_def V8_H ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()->next()  );
  220   reg_def V8_J ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(2) );
  221   reg_def V8_K ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(3) );
  222 
  223   reg_def V9   ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()          );
  224   reg_def V9_H ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()->next()  );
  225   reg_def V9_J ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(2) );
  226   reg_def V9_K ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(3) );
  227 
  228   reg_def V10   ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()          );
  229   reg_def V10_H ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()->next()  );
  230   reg_def V10_J ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(2) );
  231   reg_def V10_K ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(3) );
  232 
  233   reg_def V11   ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()          );
  234   reg_def V11_H ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()->next()  );
  235   reg_def V11_J ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(2) );
  236   reg_def V11_K ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(3) );
  237 
  238   reg_def V12   ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()          );
  239   reg_def V12_H ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()->next()  );
  240   reg_def V12_J ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(2) );
  241   reg_def V12_K ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(3) );
  242 
  243   reg_def V13   ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()          );
  244   reg_def V13_H ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()->next()  );
  245   reg_def V13_J ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(2) );
  246   reg_def V13_K ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(3) );
  247 
  248   reg_def V14   ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()          );
  249   reg_def V14_H ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()->next()  );
  250   reg_def V14_J ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(2) );
  251   reg_def V14_K ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(3) );
  252 
  253   reg_def V15   ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()          );
  254   reg_def V15_H ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()->next()  );
  255   reg_def V15_J ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(2) );
  256   reg_def V15_K ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(3) );
  257 
  258   reg_def V16   ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()          );
  259   reg_def V16_H ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next()  );
  260   reg_def V16_J ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(2) );
  261   reg_def V16_K ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(3) );
  262 
  263   reg_def V17   ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()          );
  264   reg_def V17_H ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next()  );
  265   reg_def V17_J ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(2) );
  266   reg_def V17_K ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(3) );
  267 
  268   reg_def V18   ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()          );
  269   reg_def V18_H ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next()  );
  270   reg_def V18_J ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(2) );
  271   reg_def V18_K ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(3) );
  272 
  273   reg_def V19   ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()          );
  274   reg_def V19_H ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next()  );
  275   reg_def V19_J ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(2) );
  276   reg_def V19_K ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(3) );
  277 
  278   reg_def V20   ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()          );
  279   reg_def V20_H ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next()  );
  280   reg_def V20_J ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(2) );
  281   reg_def V20_K ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(3) );
  282 
  283   reg_def V21   ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()          );
  284   reg_def V21_H ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next()  );
  285   reg_def V21_J ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(2) );
  286   reg_def V21_K ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(3) );
  287 
  288   reg_def V22   ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()          );
  289   reg_def V22_H ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next()  );
  290   reg_def V22_J ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(2) );
  291   reg_def V22_K ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(3) );
  292 
  293   reg_def V23   ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()          );
  294   reg_def V23_H ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next()  );
  295   reg_def V23_J ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(2) );
  296   reg_def V23_K ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(3) );
  297 
  298   reg_def V24   ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()          );
  299   reg_def V24_H ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next()  );
  300   reg_def V24_J ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(2) );
  301   reg_def V24_K ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(3) );
  302 
  303   reg_def V25   ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()          );
  304   reg_def V25_H ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next()  );
  305   reg_def V25_J ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(2) );
  306   reg_def V25_K ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(3) );
  307 
  308   reg_def V26   ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()          );
  309   reg_def V26_H ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next()  );
  310   reg_def V26_J ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(2) );
  311   reg_def V26_K ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(3) );
  312 
  313   reg_def V27   ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()          );
  314   reg_def V27_H ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next()  );
  315   reg_def V27_J ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(2) );
  316   reg_def V27_K ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(3) );
  317 
  318   reg_def V28   ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()          );
  319   reg_def V28_H ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next()  );
  320   reg_def V28_J ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(2) );
  321   reg_def V28_K ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(3) );
  322 
  323   reg_def V29   ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()          );
  324   reg_def V29_H ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next()  );
  325   reg_def V29_J ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(2) );
  326   reg_def V29_K ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(3) );
  327 
  328   reg_def V30   ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()          );
  329   reg_def V30_H ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next()  );
  330   reg_def V30_J ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(2) );
  331   reg_def V30_K ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(3) );
  332 
  333   reg_def V31   ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()          );
  334   reg_def V31_H ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next()  );
  335   reg_def V31_J ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(2) );
  336   reg_def V31_K ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(3) );
  337 
  338 // ----------------------------
  339 // SVE Predicate Registers
  340 // ----------------------------
  341   reg_def P0 (SOC, SOC, Op_RegVectMask, 0, p0->as_VMReg());
  342   reg_def P1 (SOC, SOC, Op_RegVectMask, 1, p1->as_VMReg());
  343   reg_def P2 (SOC, SOC, Op_RegVectMask, 2, p2->as_VMReg());
  344   reg_def P3 (SOC, SOC, Op_RegVectMask, 3, p3->as_VMReg());
  345   reg_def P4 (SOC, SOC, Op_RegVectMask, 4, p4->as_VMReg());
  346   reg_def P5 (SOC, SOC, Op_RegVectMask, 5, p5->as_VMReg());
  347   reg_def P6 (SOC, SOC, Op_RegVectMask, 6, p6->as_VMReg());
  348   reg_def P7 (SOC, SOC, Op_RegVectMask, 7, p7->as_VMReg());
  349   reg_def P8 (SOC, SOC, Op_RegVectMask, 8, p8->as_VMReg());
  350   reg_def P9 (SOC, SOC, Op_RegVectMask, 9, p9->as_VMReg());
  351   reg_def P10 (SOC, SOC, Op_RegVectMask, 10, p10->as_VMReg());
  352   reg_def P11 (SOC, SOC, Op_RegVectMask, 11, p11->as_VMReg());
  353   reg_def P12 (SOC, SOC, Op_RegVectMask, 12, p12->as_VMReg());
  354   reg_def P13 (SOC, SOC, Op_RegVectMask, 13, p13->as_VMReg());
  355   reg_def P14 (SOC, SOC, Op_RegVectMask, 14, p14->as_VMReg());
  356   reg_def P15 (SOC, SOC, Op_RegVectMask, 15, p15->as_VMReg());
  357 
  358 // ----------------------------
  359 // Special Registers
  360 // ----------------------------
  361 
  362 // the AArch64 CSPR status flag register is not directly accessible as
  363 // instruction operand. the FPSR status flag register is a system
  364 // register which can be written/read using MSR/MRS but again does not
  365 // appear as an operand (a code identifying the FSPR occurs as an
  366 // immediate value in the instruction).
  367 
  368 reg_def RFLAGS(SOC, SOC, 0, 32, VMRegImpl::Bad());
  369 
  370 // Specify priority of register selection within phases of register
  371 // allocation.  Highest priority is first.  A useful heuristic is to
  372 // give registers a low priority when they are required by machine
  373 // instructions, like EAX and EDX on I486, and choose no-save registers
  374 // before save-on-call, & save-on-call before save-on-entry.  Registers
  375 // which participate in fixed calling sequences should come last.
  376 // Registers which are used as pairs must fall on an even boundary.
  377 
  378 alloc_class chunk0(
  379     // volatiles
  380     R10, R10_H,
  381     R11, R11_H,
  382     R12, R12_H,
  383     R13, R13_H,
  384     R14, R14_H,
  385     R15, R15_H,
  386     R16, R16_H,
  387     R17, R17_H,
  388     R18, R18_H,
  389 
  390     // arg registers
  391     R0, R0_H,
  392     R1, R1_H,
  393     R2, R2_H,
  394     R3, R3_H,
  395     R4, R4_H,
  396     R5, R5_H,
  397     R6, R6_H,
  398     R7, R7_H,
  399 
  400     // non-volatiles
  401     R19, R19_H,
  402     R20, R20_H,
  403     R21, R21_H,
  404     R22, R22_H,
  405     R23, R23_H,
  406     R24, R24_H,
  407     R25, R25_H,
  408     R26, R26_H,
  409 
  410     // non-allocatable registers
  411 
  412     R27, R27_H, // heapbase
  413     R28, R28_H, // thread
  414     R29, R29_H, // fp
  415     R30, R30_H, // lr
  416     R31, R31_H, // sp
  417     R8, R8_H,   // rscratch1
  418     R9, R9_H,   // rscratch2
  419 );
  420 
  421 alloc_class chunk1(
  422 
  423     // no save
  424     V16, V16_H, V16_J, V16_K,
  425     V17, V17_H, V17_J, V17_K,
  426     V18, V18_H, V18_J, V18_K,
  427     V19, V19_H, V19_J, V19_K,
  428     V20, V20_H, V20_J, V20_K,
  429     V21, V21_H, V21_J, V21_K,
  430     V22, V22_H, V22_J, V22_K,
  431     V23, V23_H, V23_J, V23_K,
  432     V24, V24_H, V24_J, V24_K,
  433     V25, V25_H, V25_J, V25_K,
  434     V26, V26_H, V26_J, V26_K,
  435     V27, V27_H, V27_J, V27_K,
  436     V28, V28_H, V28_J, V28_K,
  437     V29, V29_H, V29_J, V29_K,
  438     V30, V30_H, V30_J, V30_K,
  439     V31, V31_H, V31_J, V31_K,
  440 
  441     // arg registers
  442     V0, V0_H, V0_J, V0_K,
  443     V1, V1_H, V1_J, V1_K,
  444     V2, V2_H, V2_J, V2_K,
  445     V3, V3_H, V3_J, V3_K,
  446     V4, V4_H, V4_J, V4_K,
  447     V5, V5_H, V5_J, V5_K,
  448     V6, V6_H, V6_J, V6_K,
  449     V7, V7_H, V7_J, V7_K,
  450 
  451     // non-volatiles
  452     V8, V8_H, V8_J, V8_K,
  453     V9, V9_H, V9_J, V9_K,
  454     V10, V10_H, V10_J, V10_K,
  455     V11, V11_H, V11_J, V11_K,
  456     V12, V12_H, V12_J, V12_K,
  457     V13, V13_H, V13_J, V13_K,
  458     V14, V14_H, V14_J, V14_K,
  459     V15, V15_H, V15_J, V15_K,
  460 );
  461 
  462 alloc_class chunk2 (
  463     // Governing predicates for load/store and arithmetic
  464     P0,
  465     P1,
  466     P2,
  467     P3,
  468     P4,
  469     P5,
  470     P6,
  471 
  472     // Extra predicates
  473     P8,
  474     P9,
  475     P10,
  476     P11,
  477     P12,
  478     P13,
  479     P14,
  480     P15,
  481 
  482     // Preserved for all-true predicate
  483     P7,
  484 );
  485 
  486 alloc_class chunk3(RFLAGS);
  487 
  488 //----------Architecture Description Register Classes--------------------------
  489 // Several register classes are automatically defined based upon information in
  490 // this architecture description.
  491 // 1) reg_class inline_cache_reg           ( /* as def'd in frame section */ )
  492 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
  493 //
  494 
  495 // Class for all 32 bit general purpose registers
  496 reg_class all_reg32(
  497     R0,
  498     R1,
  499     R2,
  500     R3,
  501     R4,
  502     R5,
  503     R6,
  504     R7,
  505     R10,
  506     R11,
  507     R12,
  508     R13,
  509     R14,
  510     R15,
  511     R16,
  512     R17,
  513     R18,
  514     R19,
  515     R20,
  516     R21,
  517     R22,
  518     R23,
  519     R24,
  520     R25,
  521     R26,
  522     R27,
  523     R28,
  524     R29,
  525     R30,
  526     R31
  527 );
  528 
  529 
  530 // Class for all 32 bit integer registers (excluding SP which
  531 // will never be used as an integer register)
  532 reg_class any_reg32 %{
  533   return _ANY_REG32_mask;
  534 %}
  535 
  536 // Singleton class for R0 int register
  537 reg_class int_r0_reg(R0);
  538 
  539 // Singleton class for R2 int register
  540 reg_class int_r2_reg(R2);
  541 
  542 // Singleton class for R3 int register
  543 reg_class int_r3_reg(R3);
  544 
  545 // Singleton class for R4 int register
  546 reg_class int_r4_reg(R4);
  547 
  548 // Singleton class for R31 int register
  549 reg_class int_r31_reg(R31);
  550 
  551 // Class for all 64 bit general purpose registers
  552 reg_class all_reg(
  553     R0, R0_H,
  554     R1, R1_H,
  555     R2, R2_H,
  556     R3, R3_H,
  557     R4, R4_H,
  558     R5, R5_H,
  559     R6, R6_H,
  560     R7, R7_H,
  561     R10, R10_H,
  562     R11, R11_H,
  563     R12, R12_H,
  564     R13, R13_H,
  565     R14, R14_H,
  566     R15, R15_H,
  567     R16, R16_H,
  568     R17, R17_H,
  569     R18, R18_H,
  570     R19, R19_H,
  571     R20, R20_H,
  572     R21, R21_H,
  573     R22, R22_H,
  574     R23, R23_H,
  575     R24, R24_H,
  576     R25, R25_H,
  577     R26, R26_H,
  578     R27, R27_H,
  579     R28, R28_H,
  580     R29, R29_H,
  581     R30, R30_H,
  582     R31, R31_H
  583 );
  584 
  585 // Class for all long integer registers (including SP)
  586 reg_class any_reg %{
  587   return _ANY_REG_mask;
  588 %}
  589 
  590 // Class for non-allocatable 32 bit registers
  591 reg_class non_allocatable_reg32(
  592 #ifdef R18_RESERVED
  593     // See comment in register_aarch64.hpp
  594     R18,                        // tls on Windows
  595 #endif
  596     R28,                        // thread
  597     R30,                        // lr
  598     R31                         // sp
  599 );
  600 
  601 // Class for non-allocatable 64 bit registers
  602 reg_class non_allocatable_reg(
  603 #ifdef R18_RESERVED
  604     // See comment in register_aarch64.hpp
  605     R18, R18_H,                 // tls on Windows, platform register on macOS
  606 #endif
  607     R28, R28_H,                 // thread
  608     R30, R30_H,                 // lr
  609     R31, R31_H                  // sp
  610 );
  611 
  612 // Class for all non-special integer registers
  613 reg_class no_special_reg32 %{
  614   return _NO_SPECIAL_REG32_mask;
  615 %}
  616 
  617 // Class for all non-special long integer registers
  618 reg_class no_special_reg %{
  619   return _NO_SPECIAL_REG_mask;
  620 %}
  621 
  622 // Class for 64 bit register r0
  623 reg_class r0_reg(
  624     R0, R0_H
  625 );
  626 
  627 // Class for 64 bit register r1
  628 reg_class r1_reg(
  629     R1, R1_H
  630 );
  631 
  632 // Class for 64 bit register r2
  633 reg_class r2_reg(
  634     R2, R2_H
  635 );
  636 
  637 // Class for 64 bit register r3
  638 reg_class r3_reg(
  639     R3, R3_H
  640 );
  641 
  642 // Class for 64 bit register r4
  643 reg_class r4_reg(
  644     R4, R4_H
  645 );
  646 
  647 // Class for 64 bit register r5
  648 reg_class r5_reg(
  649     R5, R5_H
  650 );
  651 
  652 // Class for 64 bit register r10
  653 reg_class r10_reg(
  654     R10, R10_H
  655 );
  656 
  657 // Class for 64 bit register r11
  658 reg_class r11_reg(
  659     R11, R11_H
  660 );
  661 
  662 // Class for method register
  663 reg_class method_reg(
  664     R12, R12_H
  665 );
  666 
  667 // Class for heapbase register
  668 reg_class heapbase_reg(
  669     R27, R27_H
  670 );
  671 
  672 // Class for thread register
  673 reg_class thread_reg(
  674     R28, R28_H
  675 );
  676 
  677 // Class for frame pointer register
  678 reg_class fp_reg(
  679     R29, R29_H
  680 );
  681 
  682 // Class for link register
  683 reg_class lr_reg(
  684     R30, R30_H
  685 );
  686 
  687 // Class for long sp register
  688 reg_class sp_reg(
  689   R31, R31_H
  690 );
  691 
  692 // Class for all pointer registers
  693 reg_class ptr_reg %{
  694   return _PTR_REG_mask;
  695 %}
  696 
  697 // Class for all non_special pointer registers
  698 reg_class no_special_ptr_reg %{
  699   return _NO_SPECIAL_PTR_REG_mask;
  700 %}
  701 
  702 // Class for all float registers
  703 reg_class float_reg(
  704     V0,
  705     V1,
  706     V2,
  707     V3,
  708     V4,
  709     V5,
  710     V6,
  711     V7,
  712     V8,
  713     V9,
  714     V10,
  715     V11,
  716     V12,
  717     V13,
  718     V14,
  719     V15,
  720     V16,
  721     V17,
  722     V18,
  723     V19,
  724     V20,
  725     V21,
  726     V22,
  727     V23,
  728     V24,
  729     V25,
  730     V26,
  731     V27,
  732     V28,
  733     V29,
  734     V30,
  735     V31
  736 );
  737 
  738 // Double precision float registers have virtual `high halves' that
  739 // are needed by the allocator.
  740 // Class for all double registers
  741 reg_class double_reg(
  742     V0, V0_H,
  743     V1, V1_H,
  744     V2, V2_H,
  745     V3, V3_H,
  746     V4, V4_H,
  747     V5, V5_H,
  748     V6, V6_H,
  749     V7, V7_H,
  750     V8, V8_H,
  751     V9, V9_H,
  752     V10, V10_H,
  753     V11, V11_H,
  754     V12, V12_H,
  755     V13, V13_H,
  756     V14, V14_H,
  757     V15, V15_H,
  758     V16, V16_H,
  759     V17, V17_H,
  760     V18, V18_H,
  761     V19, V19_H,
  762     V20, V20_H,
  763     V21, V21_H,
  764     V22, V22_H,
  765     V23, V23_H,
  766     V24, V24_H,
  767     V25, V25_H,
  768     V26, V26_H,
  769     V27, V27_H,
  770     V28, V28_H,
  771     V29, V29_H,
  772     V30, V30_H,
  773     V31, V31_H
  774 );
  775 
  776 // Class for all SVE vector registers.
  777 reg_class vectora_reg (
  778     V0, V0_H, V0_J, V0_K,
  779     V1, V1_H, V1_J, V1_K,
  780     V2, V2_H, V2_J, V2_K,
  781     V3, V3_H, V3_J, V3_K,
  782     V4, V4_H, V4_J, V4_K,
  783     V5, V5_H, V5_J, V5_K,
  784     V6, V6_H, V6_J, V6_K,
  785     V7, V7_H, V7_J, V7_K,
  786     V8, V8_H, V8_J, V8_K,
  787     V9, V9_H, V9_J, V9_K,
  788     V10, V10_H, V10_J, V10_K,
  789     V11, V11_H, V11_J, V11_K,
  790     V12, V12_H, V12_J, V12_K,
  791     V13, V13_H, V13_J, V13_K,
  792     V14, V14_H, V14_J, V14_K,
  793     V15, V15_H, V15_J, V15_K,
  794     V16, V16_H, V16_J, V16_K,
  795     V17, V17_H, V17_J, V17_K,
  796     V18, V18_H, V18_J, V18_K,
  797     V19, V19_H, V19_J, V19_K,
  798     V20, V20_H, V20_J, V20_K,
  799     V21, V21_H, V21_J, V21_K,
  800     V22, V22_H, V22_J, V22_K,
  801     V23, V23_H, V23_J, V23_K,
  802     V24, V24_H, V24_J, V24_K,
  803     V25, V25_H, V25_J, V25_K,
  804     V26, V26_H, V26_J, V26_K,
  805     V27, V27_H, V27_J, V27_K,
  806     V28, V28_H, V28_J, V28_K,
  807     V29, V29_H, V29_J, V29_K,
  808     V30, V30_H, V30_J, V30_K,
  809     V31, V31_H, V31_J, V31_K,
  810 );
  811 
  812 // Class for all 64bit vector registers
  813 reg_class vectord_reg(
  814     V0, V0_H,
  815     V1, V1_H,
  816     V2, V2_H,
  817     V3, V3_H,
  818     V4, V4_H,
  819     V5, V5_H,
  820     V6, V6_H,
  821     V7, V7_H,
  822     V8, V8_H,
  823     V9, V9_H,
  824     V10, V10_H,
  825     V11, V11_H,
  826     V12, V12_H,
  827     V13, V13_H,
  828     V14, V14_H,
  829     V15, V15_H,
  830     V16, V16_H,
  831     V17, V17_H,
  832     V18, V18_H,
  833     V19, V19_H,
  834     V20, V20_H,
  835     V21, V21_H,
  836     V22, V22_H,
  837     V23, V23_H,
  838     V24, V24_H,
  839     V25, V25_H,
  840     V26, V26_H,
  841     V27, V27_H,
  842     V28, V28_H,
  843     V29, V29_H,
  844     V30, V30_H,
  845     V31, V31_H
  846 );
  847 
  848 // Class for all 128bit vector registers
  849 reg_class vectorx_reg(
  850     V0, V0_H, V0_J, V0_K,
  851     V1, V1_H, V1_J, V1_K,
  852     V2, V2_H, V2_J, V2_K,
  853     V3, V3_H, V3_J, V3_K,
  854     V4, V4_H, V4_J, V4_K,
  855     V5, V5_H, V5_J, V5_K,
  856     V6, V6_H, V6_J, V6_K,
  857     V7, V7_H, V7_J, V7_K,
  858     V8, V8_H, V8_J, V8_K,
  859     V9, V9_H, V9_J, V9_K,
  860     V10, V10_H, V10_J, V10_K,
  861     V11, V11_H, V11_J, V11_K,
  862     V12, V12_H, V12_J, V12_K,
  863     V13, V13_H, V13_J, V13_K,
  864     V14, V14_H, V14_J, V14_K,
  865     V15, V15_H, V15_J, V15_K,
  866     V16, V16_H, V16_J, V16_K,
  867     V17, V17_H, V17_J, V17_K,
  868     V18, V18_H, V18_J, V18_K,
  869     V19, V19_H, V19_J, V19_K,
  870     V20, V20_H, V20_J, V20_K,
  871     V21, V21_H, V21_J, V21_K,
  872     V22, V22_H, V22_J, V22_K,
  873     V23, V23_H, V23_J, V23_K,
  874     V24, V24_H, V24_J, V24_K,
  875     V25, V25_H, V25_J, V25_K,
  876     V26, V26_H, V26_J, V26_K,
  877     V27, V27_H, V27_J, V27_K,
  878     V28, V28_H, V28_J, V28_K,
  879     V29, V29_H, V29_J, V29_K,
  880     V30, V30_H, V30_J, V30_K,
  881     V31, V31_H, V31_J, V31_K
  882 );
  883 
  884 // Class for 128 bit register v0
  885 reg_class v0_reg(
  886     V0, V0_H
  887 );
  888 
  889 // Class for 128 bit register v1
  890 reg_class v1_reg(
  891     V1, V1_H
  892 );
  893 
  894 // Class for 128 bit register v2
  895 reg_class v2_reg(
  896     V2, V2_H
  897 );
  898 
  899 // Class for 128 bit register v3
  900 reg_class v3_reg(
  901     V3, V3_H
  902 );
  903 
  904 // Class for 128 bit register v4
  905 reg_class v4_reg(
  906     V4, V4_H
  907 );
  908 
  909 // Class for 128 bit register v5
  910 reg_class v5_reg(
  911     V5, V5_H
  912 );
  913 
  914 // Class for 128 bit register v6
  915 reg_class v6_reg(
  916     V6, V6_H
  917 );
  918 
  919 // Class for 128 bit register v7
  920 reg_class v7_reg(
  921     V7, V7_H
  922 );
  923 
  924 // Class for 128 bit register v8
  925 reg_class v8_reg(
  926     V8, V8_H
  927 );
  928 
  929 // Class for 128 bit register v9
  930 reg_class v9_reg(
  931     V9, V9_H
  932 );
  933 
  934 // Class for 128 bit register v10
  935 reg_class v10_reg(
  936     V10, V10_H
  937 );
  938 
  939 // Class for 128 bit register v11
  940 reg_class v11_reg(
  941     V11, V11_H
  942 );
  943 
  944 // Class for 128 bit register v12
  945 reg_class v12_reg(
  946     V12, V12_H
  947 );
  948 
  949 // Class for 128 bit register v13
  950 reg_class v13_reg(
  951     V13, V13_H
  952 );
  953 
  954 // Class for 128 bit register v14
  955 reg_class v14_reg(
  956     V14, V14_H
  957 );
  958 
  959 // Class for 128 bit register v15
  960 reg_class v15_reg(
  961     V15, V15_H
  962 );
  963 
  964 // Class for 128 bit register v16
  965 reg_class v16_reg(
  966     V16, V16_H
  967 );
  968 
  969 // Class for 128 bit register v17
  970 reg_class v17_reg(
  971     V17, V17_H
  972 );
  973 
  974 // Class for 128 bit register v18
  975 reg_class v18_reg(
  976     V18, V18_H
  977 );
  978 
  979 // Class for 128 bit register v19
  980 reg_class v19_reg(
  981     V19, V19_H
  982 );
  983 
  984 // Class for 128 bit register v20
  985 reg_class v20_reg(
  986     V20, V20_H
  987 );
  988 
  989 // Class for 128 bit register v21
  990 reg_class v21_reg(
  991     V21, V21_H
  992 );
  993 
  994 // Class for 128 bit register v22
  995 reg_class v22_reg(
  996     V22, V22_H
  997 );
  998 
  999 // Class for 128 bit register v23
 1000 reg_class v23_reg(
 1001     V23, V23_H
 1002 );
 1003 
 1004 // Class for 128 bit register v24
 1005 reg_class v24_reg(
 1006     V24, V24_H
 1007 );
 1008 
 1009 // Class for 128 bit register v25
 1010 reg_class v25_reg(
 1011     V25, V25_H
 1012 );
 1013 
 1014 // Class for 128 bit register v26
 1015 reg_class v26_reg(
 1016     V26, V26_H
 1017 );
 1018 
 1019 // Class for 128 bit register v27
 1020 reg_class v27_reg(
 1021     V27, V27_H
 1022 );
 1023 
 1024 // Class for 128 bit register v28
 1025 reg_class v28_reg(
 1026     V28, V28_H
 1027 );
 1028 
 1029 // Class for 128 bit register v29
 1030 reg_class v29_reg(
 1031     V29, V29_H
 1032 );
 1033 
 1034 // Class for 128 bit register v30
 1035 reg_class v30_reg(
 1036     V30, V30_H
 1037 );
 1038 
 1039 // Class for 128 bit register v31
 1040 reg_class v31_reg(
 1041     V31, V31_H
 1042 );
 1043 
 1044 // Class for all SVE predicate registers.
 1045 reg_class pr_reg (
 1046     P0,
 1047     P1,
 1048     P2,
 1049     P3,
 1050     P4,
 1051     P5,
 1052     P6,
 1053     // P7, non-allocatable, preserved with all elements preset to TRUE.
 1054     P8,
 1055     P9,
 1056     P10,
 1057     P11,
 1058     P12,
 1059     P13,
 1060     P14,
 1061     P15
 1062 );
 1063 
 1064 // Class for SVE governing predicate registers, which are used
 1065 // to determine the active elements of a predicated instruction.
 1066 reg_class gov_pr (
 1067     P0,
 1068     P1,
 1069     P2,
 1070     P3,
 1071     P4,
 1072     P5,
 1073     P6,
 1074     // P7, non-allocatable, preserved with all elements preset to TRUE.
 1075 );
 1076 
 1077 reg_class p0_reg(P0);
 1078 reg_class p1_reg(P1);
 1079 
 1080 // Singleton class for condition codes
 1081 reg_class int_flags(RFLAGS);
 1082 
 1083 %}
 1084 
 1085 //----------DEFINITION BLOCK---------------------------------------------------
 1086 // Define name --> value mappings to inform the ADLC of an integer valued name
 1087 // Current support includes integer values in the range [0, 0x7FFFFFFF]
 1088 // Format:
 1089 //        int_def  <name>         ( <int_value>, <expression>);
 1090 // Generated Code in ad_<arch>.hpp
 1091 //        #define  <name>   (<expression>)
 1092 //        // value == <int_value>
 1093 // Generated code in ad_<arch>.cpp adlc_verification()
 1094 //        assert( <name> == <int_value>, "Expect (<expression>) to equal <int_value>");
 1095 //
 1096 
 1097 // we follow the ppc-aix port in using a simple cost model which ranks
 1098 // register operations as cheap, memory ops as more expensive and
 1099 // branches as most expensive. the first two have a low as well as a
 1100 // normal cost. huge cost appears to be a way of saying don't do
 1101 // something
 1102 
 1103 definitions %{
 1104   // The default cost (of a register move instruction).
 1105   int_def INSN_COST            (    100,     100);
 1106   int_def BRANCH_COST          (    200,     2 * INSN_COST);
 1107   int_def CALL_COST            (    200,     2 * INSN_COST);
 1108   int_def VOLATILE_REF_COST    (   1000,     10 * INSN_COST);
 1109 %}
 1110 
 1111 
 1112 //----------SOURCE BLOCK-------------------------------------------------------
 1113 // This is a block of C++ code which provides values, functions, and
 1114 // definitions necessary in the rest of the architecture description
 1115 
 1116 source_hpp %{
 1117 
 1118 #include "asm/macroAssembler.hpp"
 1119 #include "gc/shared/barrierSetAssembler.hpp"
 1120 #include "gc/shared/cardTable.hpp"
 1121 #include "gc/shared/cardTableBarrierSet.hpp"
 1122 #include "gc/shared/collectedHeap.hpp"
 1123 #include "opto/addnode.hpp"
 1124 #include "opto/convertnode.hpp"
 1125 #include "runtime/objectMonitor.hpp"
 1126 
 1127 extern RegMask _ANY_REG32_mask;
 1128 extern RegMask _ANY_REG_mask;
 1129 extern RegMask _PTR_REG_mask;
 1130 extern RegMask _NO_SPECIAL_REG32_mask;
 1131 extern RegMask _NO_SPECIAL_REG_mask;
 1132 extern RegMask _NO_SPECIAL_PTR_REG_mask;
 1133 
 1134 class CallStubImpl {
 1135 
 1136   //--------------------------------------------------------------
 1137   //---<  Used for optimization in Compile::shorten_branches  >---
 1138   //--------------------------------------------------------------
 1139 
 1140  public:
 1141   // Size of call trampoline stub.
 1142   static uint size_call_trampoline() {
 1143     return 0; // no call trampolines on this platform
 1144   }
 1145 
 1146   // number of relocations needed by a call trampoline stub
 1147   static uint reloc_call_trampoline() {
 1148     return 0; // no call trampolines on this platform
 1149   }
 1150 };
 1151 
 1152 class HandlerImpl {
 1153 
 1154  public:
 1155 
 1156   static int emit_exception_handler(CodeBuffer &cbuf);
 1157   static int emit_deopt_handler(CodeBuffer& cbuf);
 1158 
 1159   static uint size_exception_handler() {
 1160     return MacroAssembler::far_codestub_branch_size();
 1161   }
 1162 
 1163   static uint size_deopt_handler() {
 1164     // count one adr and one far branch instruction
 1165     return NativeInstruction::instruction_size + MacroAssembler::far_codestub_branch_size();
 1166   }
 1167 };
 1168 
 1169 class Node::PD {
 1170 public:
 1171   enum NodeFlags {
 1172     _last_flag = Node::_last_flag
 1173   };
 1174 };
 1175 
 1176   bool is_CAS(int opcode, bool maybe_volatile);
 1177 
 1178   // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
 1179 
 1180   bool unnecessary_acquire(const Node *barrier);
 1181   bool needs_acquiring_load(const Node *load);
 1182 
 1183   // predicates controlling emit of str<x>/stlr<x> and associated dmbs
 1184 
 1185   bool unnecessary_release(const Node *barrier);
 1186   bool unnecessary_volatile(const Node *barrier);
 1187   bool needs_releasing_store(const Node *store);
 1188 
 1189   // predicate controlling translation of CompareAndSwapX
 1190   bool needs_acquiring_load_exclusive(const Node *load);
 1191 
 1192   // predicate controlling addressing modes
 1193   bool size_fits_all_mem_uses(AddPNode* addp, int shift);
 1194 %}
 1195 
 1196 source %{
 1197 
 1198   // Derived RegMask with conditionally allocatable registers
 1199 
 1200   void PhaseOutput::pd_perform_mach_node_analysis() {
 1201   }
 1202 
 1203   int MachNode::pd_alignment_required() const {
 1204     return 1;
 1205   }
 1206 
 1207   int MachNode::compute_padding(int current_offset) const {
 1208     return 0;
 1209   }
 1210 
 1211   RegMask _ANY_REG32_mask;
 1212   RegMask _ANY_REG_mask;
 1213   RegMask _PTR_REG_mask;
 1214   RegMask _NO_SPECIAL_REG32_mask;
 1215   RegMask _NO_SPECIAL_REG_mask;
 1216   RegMask _NO_SPECIAL_PTR_REG_mask;
 1217 
 1218   void reg_mask_init() {
 1219     // We derive below RegMask(s) from the ones which are auto-generated from
 1220     // adlc register classes to make AArch64 rheapbase (r27) and rfp (r29)
 1221     // registers conditionally reserved.
 1222 
 1223     _ANY_REG32_mask = _ALL_REG32_mask;
 1224     _ANY_REG32_mask.Remove(OptoReg::as_OptoReg(r31_sp->as_VMReg()));
 1225 
 1226     _ANY_REG_mask = _ALL_REG_mask;
 1227 
 1228     _PTR_REG_mask = _ALL_REG_mask;
 1229 
 1230     _NO_SPECIAL_REG32_mask = _ALL_REG32_mask;
 1231     _NO_SPECIAL_REG32_mask.SUBTRACT(_NON_ALLOCATABLE_REG32_mask);
 1232 
 1233     _NO_SPECIAL_REG_mask = _ALL_REG_mask;
 1234     _NO_SPECIAL_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
 1235 
 1236     _NO_SPECIAL_PTR_REG_mask = _ALL_REG_mask;
 1237     _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
 1238 
 1239     // r27 is not allocatable when compressed oops is on and heapbase is not
 1240     // zero, compressed klass pointers doesn't use r27 after JDK-8234794
 1241     if (UseCompressedOops && (CompressedOops::ptrs_base() != NULL)) {
 1242       _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
 1243       _NO_SPECIAL_REG_mask.SUBTRACT(_HEAPBASE_REG_mask);
 1244       _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_HEAPBASE_REG_mask);
 1245     }
 1246 
 1247     // r29 is not allocatable when PreserveFramePointer is on
 1248     if (PreserveFramePointer) {
 1249       _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
 1250       _NO_SPECIAL_REG_mask.SUBTRACT(_FP_REG_mask);
 1251       _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_FP_REG_mask);
 1252     }
 1253   }
 1254 
 1255   // Optimizaton of volatile gets and puts
 1256   // -------------------------------------
 1257   //
 1258   // AArch64 has ldar<x> and stlr<x> instructions which we can safely
 1259   // use to implement volatile reads and writes. For a volatile read
 1260   // we simply need
 1261   //
 1262   //   ldar<x>
 1263   //
 1264   // and for a volatile write we need
 1265   //
 1266   //   stlr<x>
 1267   //
 1268   // Alternatively, we can implement them by pairing a normal
 1269   // load/store with a memory barrier. For a volatile read we need
 1270   //
 1271   //   ldr<x>
 1272   //   dmb ishld
 1273   //
 1274   // for a volatile write
 1275   //
 1276   //   dmb ish
 1277   //   str<x>
 1278   //   dmb ish
 1279   //
 1280   // We can also use ldaxr and stlxr to implement compare and swap CAS
 1281   // sequences. These are normally translated to an instruction
 1282   // sequence like the following
 1283   //
 1284   //   dmb      ish
 1285   // retry:
 1286   //   ldxr<x>   rval raddr
 1287   //   cmp       rval rold
 1288   //   b.ne done
 1289   //   stlxr<x>  rval, rnew, rold
 1290   //   cbnz      rval retry
 1291   // done:
 1292   //   cset      r0, eq
 1293   //   dmb ishld
 1294   //
 1295   // Note that the exclusive store is already using an stlxr
 1296   // instruction. That is required to ensure visibility to other
 1297   // threads of the exclusive write (assuming it succeeds) before that
 1298   // of any subsequent writes.
 1299   //
 1300   // The following instruction sequence is an improvement on the above
 1301   //
 1302   // retry:
 1303   //   ldaxr<x>  rval raddr
 1304   //   cmp       rval rold
 1305   //   b.ne done
 1306   //   stlxr<x>  rval, rnew, rold
 1307   //   cbnz      rval retry
 1308   // done:
 1309   //   cset      r0, eq
 1310   //
 1311   // We don't need the leading dmb ish since the stlxr guarantees
 1312   // visibility of prior writes in the case that the swap is
 1313   // successful. Crucially we don't have to worry about the case where
 1314   // the swap is not successful since no valid program should be
 1315   // relying on visibility of prior changes by the attempting thread
 1316   // in the case where the CAS fails.
 1317   //
 1318   // Similarly, we don't need the trailing dmb ishld if we substitute
 1319   // an ldaxr instruction since that will provide all the guarantees we
 1320   // require regarding observation of changes made by other threads
 1321   // before any change to the CAS address observed by the load.
 1322   //
 1323   // In order to generate the desired instruction sequence we need to
 1324   // be able to identify specific 'signature' ideal graph node
 1325   // sequences which i) occur as a translation of a volatile reads or
 1326   // writes or CAS operations and ii) do not occur through any other
 1327   // translation or graph transformation. We can then provide
 1328   // alternative aldc matching rules which translate these node
 1329   // sequences to the desired machine code sequences. Selection of the
 1330   // alternative rules can be implemented by predicates which identify
 1331   // the relevant node sequences.
 1332   //
 1333   // The ideal graph generator translates a volatile read to the node
 1334   // sequence
 1335   //
 1336   //   LoadX[mo_acquire]
 1337   //   MemBarAcquire
 1338   //
 1339   // As a special case when using the compressed oops optimization we
 1340   // may also see this variant
 1341   //
 1342   //   LoadN[mo_acquire]
 1343   //   DecodeN
 1344   //   MemBarAcquire
 1345   //
 1346   // A volatile write is translated to the node sequence
 1347   //
 1348   //   MemBarRelease
 1349   //   StoreX[mo_release] {CardMark}-optional
 1350   //   MemBarVolatile
 1351   //
 1352   // n.b. the above node patterns are generated with a strict
 1353   // 'signature' configuration of input and output dependencies (see
 1354   // the predicates below for exact details). The card mark may be as
 1355   // simple as a few extra nodes or, in a few GC configurations, may
 1356   // include more complex control flow between the leading and
 1357   // trailing memory barriers. However, whatever the card mark
 1358   // configuration these signatures are unique to translated volatile
 1359   // reads/stores -- they will not appear as a result of any other
 1360   // bytecode translation or inlining nor as a consequence of
 1361   // optimizing transforms.
 1362   //
 1363   // We also want to catch inlined unsafe volatile gets and puts and
 1364   // be able to implement them using either ldar<x>/stlr<x> or some
 1365   // combination of ldr<x>/stlr<x> and dmb instructions.
 1366   //
 1367   // Inlined unsafe volatiles puts manifest as a minor variant of the
 1368   // normal volatile put node sequence containing an extra cpuorder
 1369   // membar
 1370   //
 1371   //   MemBarRelease
 1372   //   MemBarCPUOrder
 1373   //   StoreX[mo_release] {CardMark}-optional
 1374   //   MemBarCPUOrder
 1375   //   MemBarVolatile
 1376   //
 1377   // n.b. as an aside, a cpuorder membar is not itself subject to
 1378   // matching and translation by adlc rules.  However, the rule
 1379   // predicates need to detect its presence in order to correctly
 1380   // select the desired adlc rules.
 1381   //
 1382   // Inlined unsafe volatile gets manifest as a slightly different
 1383   // node sequence to a normal volatile get because of the
 1384   // introduction of some CPUOrder memory barriers to bracket the
 1385   // Load. However, but the same basic skeleton of a LoadX feeding a
 1386   // MemBarAcquire, possibly through an optional DecodeN, is still
 1387   // present
 1388   //
 1389   //   MemBarCPUOrder
 1390   //        ||       \\
 1391   //   MemBarCPUOrder LoadX[mo_acquire]
 1392   //        ||            |
 1393   //        ||       {DecodeN} optional
 1394   //        ||       /
 1395   //     MemBarAcquire
 1396   //
 1397   // In this case the acquire membar does not directly depend on the
 1398   // load. However, we can be sure that the load is generated from an
 1399   // inlined unsafe volatile get if we see it dependent on this unique
 1400   // sequence of membar nodes. Similarly, given an acquire membar we
 1401   // can know that it was added because of an inlined unsafe volatile
 1402   // get if it is fed and feeds a cpuorder membar and if its feed
 1403   // membar also feeds an acquiring load.
 1404   //
 1405   // Finally an inlined (Unsafe) CAS operation is translated to the
 1406   // following ideal graph
 1407   //
 1408   //   MemBarRelease
 1409   //   MemBarCPUOrder
 1410   //   CompareAndSwapX {CardMark}-optional
 1411   //   MemBarCPUOrder
 1412   //   MemBarAcquire
 1413   //
 1414   // So, where we can identify these volatile read and write
 1415   // signatures we can choose to plant either of the above two code
 1416   // sequences. For a volatile read we can simply plant a normal
 1417   // ldr<x> and translate the MemBarAcquire to a dmb. However, we can
 1418   // also choose to inhibit translation of the MemBarAcquire and
 1419   // inhibit planting of the ldr<x>, instead planting an ldar<x>.
 1420   //
 1421   // When we recognise a volatile store signature we can choose to
 1422   // plant at a dmb ish as a translation for the MemBarRelease, a
 1423   // normal str<x> and then a dmb ish for the MemBarVolatile.
 1424   // Alternatively, we can inhibit translation of the MemBarRelease
 1425   // and MemBarVolatile and instead plant a simple stlr<x>
 1426   // instruction.
 1427   //
 1428   // when we recognise a CAS signature we can choose to plant a dmb
 1429   // ish as a translation for the MemBarRelease, the conventional
 1430   // macro-instruction sequence for the CompareAndSwap node (which
 1431   // uses ldxr<x>) and then a dmb ishld for the MemBarAcquire.
 1432   // Alternatively, we can elide generation of the dmb instructions
 1433   // and plant the alternative CompareAndSwap macro-instruction
 1434   // sequence (which uses ldaxr<x>).
 1435   //
 1436   // Of course, the above only applies when we see these signature
 1437   // configurations. We still want to plant dmb instructions in any
 1438   // other cases where we may see a MemBarAcquire, MemBarRelease or
 1439   // MemBarVolatile. For example, at the end of a constructor which
 1440   // writes final/volatile fields we will see a MemBarRelease
 1441   // instruction and this needs a 'dmb ish' lest we risk the
 1442   // constructed object being visible without making the
 1443   // final/volatile field writes visible.
 1444   //
 1445   // n.b. the translation rules below which rely on detection of the
 1446   // volatile signatures and insert ldar<x> or stlr<x> are failsafe.
 1447   // If we see anything other than the signature configurations we
 1448   // always just translate the loads and stores to ldr<x> and str<x>
 1449   // and translate acquire, release and volatile membars to the
 1450   // relevant dmb instructions.
 1451   //
 1452 
 1453   // is_CAS(int opcode, bool maybe_volatile)
 1454   //
 1455   // return true if opcode is one of the possible CompareAndSwapX
 1456   // values otherwise false.
 1457 
 1458   bool is_CAS(int opcode, bool maybe_volatile)
 1459   {
 1460     switch(opcode) {
 1461       // We handle these
 1462     case Op_CompareAndSwapI:
 1463     case Op_CompareAndSwapL:
 1464     case Op_CompareAndSwapP:
 1465     case Op_CompareAndSwapN:
 1466     case Op_ShenandoahCompareAndSwapP:
 1467     case Op_ShenandoahCompareAndSwapN:
 1468     case Op_CompareAndSwapB:
 1469     case Op_CompareAndSwapS:
 1470     case Op_GetAndSetI:
 1471     case Op_GetAndSetL:
 1472     case Op_GetAndSetP:
 1473     case Op_GetAndSetN:
 1474     case Op_GetAndAddI:
 1475     case Op_GetAndAddL:
 1476       return true;
 1477     case Op_CompareAndExchangeI:
 1478     case Op_CompareAndExchangeN:
 1479     case Op_CompareAndExchangeB:
 1480     case Op_CompareAndExchangeS:
 1481     case Op_CompareAndExchangeL:
 1482     case Op_CompareAndExchangeP:
 1483     case Op_WeakCompareAndSwapB:
 1484     case Op_WeakCompareAndSwapS:
 1485     case Op_WeakCompareAndSwapI:
 1486     case Op_WeakCompareAndSwapL:
 1487     case Op_WeakCompareAndSwapP:
 1488     case Op_WeakCompareAndSwapN:
 1489     case Op_ShenandoahWeakCompareAndSwapP:
 1490     case Op_ShenandoahWeakCompareAndSwapN:
 1491     case Op_ShenandoahCompareAndExchangeP:
 1492     case Op_ShenandoahCompareAndExchangeN:
 1493       return maybe_volatile;
 1494     default:
 1495       return false;
 1496     }
 1497   }
 1498 
 1499   // helper to determine the maximum number of Phi nodes we may need to
 1500   // traverse when searching from a card mark membar for the merge mem
 1501   // feeding a trailing membar or vice versa
 1502 
 1503 // predicates controlling emit of ldr<x>/ldar<x>
 1504 
 1505 bool unnecessary_acquire(const Node *barrier)
 1506 {
 1507   assert(barrier->is_MemBar(), "expecting a membar");
 1508 
 1509   MemBarNode* mb = barrier->as_MemBar();
 1510 
 1511   if (mb->trailing_load()) {
 1512     return true;
 1513   }
 1514 
 1515   if (mb->trailing_load_store()) {
 1516     Node* load_store = mb->in(MemBarNode::Precedent);
 1517     assert(load_store->is_LoadStore(), "unexpected graph shape");
 1518     return is_CAS(load_store->Opcode(), true);
 1519   }
 1520 
 1521   return false;
 1522 }
 1523 
 1524 bool needs_acquiring_load(const Node *n)
 1525 {
 1526   assert(n->is_Load(), "expecting a load");
 1527   LoadNode *ld = n->as_Load();
 1528   return ld->is_acquire();
 1529 }
 1530 
 1531 bool unnecessary_release(const Node *n)
 1532 {
 1533   assert((n->is_MemBar() &&
 1534           n->Opcode() == Op_MemBarRelease),
 1535          "expecting a release membar");
 1536 
 1537   MemBarNode *barrier = n->as_MemBar();
 1538   if (!barrier->leading()) {
 1539     return false;
 1540   } else {
 1541     Node* trailing = barrier->trailing_membar();
 1542     MemBarNode* trailing_mb = trailing->as_MemBar();
 1543     assert(trailing_mb->trailing(), "Not a trailing membar?");
 1544     assert(trailing_mb->leading_membar() == n, "inconsistent leading/trailing membars");
 1545 
 1546     Node* mem = trailing_mb->in(MemBarNode::Precedent);
 1547     if (mem->is_Store()) {
 1548       assert(mem->as_Store()->is_release(), "");
 1549       assert(trailing_mb->Opcode() == Op_MemBarVolatile, "");
 1550       return true;
 1551     } else {
 1552       assert(mem->is_LoadStore(), "");
 1553       assert(trailing_mb->Opcode() == Op_MemBarAcquire, "");
 1554       return is_CAS(mem->Opcode(), true);
 1555     }
 1556   }
 1557   return false;
 1558 }
 1559 
 1560 bool unnecessary_volatile(const Node *n)
 1561 {
 1562   // assert n->is_MemBar();
 1563   MemBarNode *mbvol = n->as_MemBar();
 1564 
 1565   bool release = mbvol->trailing_store();
 1566   assert(!release || (mbvol->in(MemBarNode::Precedent)->is_Store() && mbvol->in(MemBarNode::Precedent)->as_Store()->is_release()), "");
 1567 #ifdef ASSERT
 1568   if (release) {
 1569     Node* leading = mbvol->leading_membar();
 1570     assert(leading->Opcode() == Op_MemBarRelease, "");
 1571     assert(leading->as_MemBar()->leading_store(), "");
 1572     assert(leading->as_MemBar()->trailing_membar() == mbvol, "");
 1573   }
 1574 #endif
 1575 
 1576   return release;
 1577 }
 1578 
 1579 // predicates controlling emit of str<x>/stlr<x>
 1580 
 1581 bool needs_releasing_store(const Node *n)
 1582 {
 1583   // assert n->is_Store();
 1584   StoreNode *st = n->as_Store();
 1585   return st->trailing_membar() != NULL;
 1586 }
 1587 
 1588 // predicate controlling translation of CAS
 1589 //
 1590 // returns true if CAS needs to use an acquiring load otherwise false
 1591 
 1592 bool needs_acquiring_load_exclusive(const Node *n)
 1593 {
 1594   assert(is_CAS(n->Opcode(), true), "expecting a compare and swap");
 1595   LoadStoreNode* ldst = n->as_LoadStore();
 1596   if (is_CAS(n->Opcode(), false)) {
 1597     assert(ldst->trailing_membar() != NULL, "expected trailing membar");
 1598   } else {
 1599     return ldst->trailing_membar() != NULL;
 1600   }
 1601 
 1602   // so we can just return true here
 1603   return true;
 1604 }
 1605 
 1606 #define __ _masm.
 1607 
 1608 // advance declarations for helper functions to convert register
 1609 // indices to register objects
 1610 
 1611 // the ad file has to provide implementations of certain methods
 1612 // expected by the generic code
 1613 //
 1614 // REQUIRED FUNCTIONALITY
 1615 
 1616 //=============================================================================
 1617 
 1618 // !!!!! Special hack to get all types of calls to specify the byte offset
 1619 //       from the start of the call to the point where the return address
 1620 //       will point.
 1621 
 1622 int MachCallStaticJavaNode::ret_addr_offset()
 1623 {
 1624   // call should be a simple bl
 1625   int off = 4;
 1626   return off;
 1627 }
 1628 
 1629 int MachCallDynamicJavaNode::ret_addr_offset()
 1630 {
 1631   return 16; // movz, movk, movk, bl
 1632 }
 1633 
 1634 int MachCallRuntimeNode::ret_addr_offset() {
 1635   // for generated stubs the call will be
 1636   //   bl(addr)
 1637   // or with far branches
 1638   //   bl(trampoline_stub)
 1639   // for real runtime callouts it will be six instructions
 1640   // see aarch64_enc_java_to_runtime
 1641   //   adr(rscratch2, retaddr)
 1642   //   lea(rscratch1, RuntimeAddress(addr)
 1643   //   stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)))
 1644   //   blr(rscratch1)
 1645   CodeBlob *cb = CodeCache::find_blob(_entry_point);
 1646   if (cb) {
 1647     return 1 * NativeInstruction::instruction_size;
 1648   } else if (_entry_point == NULL) {
 1649     // See CallLeafNoFPIndirect
 1650     return 1 * NativeInstruction::instruction_size;
 1651   } else {
 1652     return 6 * NativeInstruction::instruction_size;
 1653   }
 1654 }
 1655 
 1656 //=============================================================================
 1657 
 1658 #ifndef PRODUCT
 1659 void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 1660   st->print("BREAKPOINT");
 1661 }
 1662 #endif
 1663 
 1664 void MachBreakpointNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 1665   C2_MacroAssembler _masm(&cbuf);
 1666   __ brk(0);
 1667 }
 1668 
 1669 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
 1670   return MachNode::size(ra_);
 1671 }
 1672 
 1673 //=============================================================================
 1674 
 1675 #ifndef PRODUCT
 1676   void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
 1677     st->print("nop \t# %d bytes pad for loops and calls", _count);
 1678   }
 1679 #endif
 1680 
 1681   void MachNopNode::emit(CodeBuffer &cbuf, PhaseRegAlloc*) const {
 1682     C2_MacroAssembler _masm(&cbuf);
 1683     for (int i = 0; i < _count; i++) {
 1684       __ nop();
 1685     }
 1686   }
 1687 
 1688   uint MachNopNode::size(PhaseRegAlloc*) const {
 1689     return _count * NativeInstruction::instruction_size;
 1690   }
 1691 
 1692 //=============================================================================
 1693 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
 1694 
 1695 int ConstantTable::calculate_table_base_offset() const {
 1696   return 0;  // absolute addressing, no offset
 1697 }
 1698 
 1699 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
 1700 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
 1701   ShouldNotReachHere();
 1702 }
 1703 
 1704 void MachConstantBaseNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const {
 1705   // Empty encoding
 1706 }
 1707 
 1708 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
 1709   return 0;
 1710 }
 1711 
 1712 #ifndef PRODUCT
 1713 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
 1714   st->print("-- \t// MachConstantBaseNode (empty encoding)");
 1715 }
 1716 #endif
 1717 
 1718 #ifndef PRODUCT
 1719 void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 1720   Compile* C = ra_->C;
 1721 
 1722   int framesize = C->output()->frame_slots() << LogBytesPerInt;
 1723 
 1724   if (C->output()->need_stack_bang(framesize))
 1725     st->print("# stack bang size=%d\n\t", framesize);
 1726 
 1727   if (VM_Version::use_rop_protection()) {
 1728     st->print("ldr zr, [lr]\n\t");
 1729     st->print("pacia  lr, rfp\n\t");
 1730   }
 1731   if (framesize < ((1 << 9) + 2 * wordSize)) {
 1732     st->print("sub  sp, sp, #%d\n\t", framesize);
 1733     st->print("stp  rfp, lr, [sp, #%d]", framesize - 2 * wordSize);
 1734     if (PreserveFramePointer) st->print("\n\tadd  rfp, sp, #%d", framesize - 2 * wordSize);
 1735   } else {
 1736     st->print("stp  lr, rfp, [sp, #%d]!\n\t", -(2 * wordSize));
 1737     if (PreserveFramePointer) st->print("mov  rfp, sp\n\t");
 1738     st->print("mov  rscratch1, #%d\n\t", framesize - 2 * wordSize);
 1739     st->print("sub  sp, sp, rscratch1");
 1740   }
 1741   if (C->stub_function() == NULL && BarrierSet::barrier_set()->barrier_set_nmethod() != NULL) {
 1742     st->print("\n\t");
 1743     st->print("ldr  rscratch1, [guard]\n\t");
 1744     st->print("dmb ishld\n\t");
 1745     st->print("ldr  rscratch2, [rthread, #thread_disarmed_guard_value_offset]\n\t");
 1746     st->print("cmp  rscratch1, rscratch2\n\t");
 1747     st->print("b.eq skip");
 1748     st->print("\n\t");
 1749     st->print("blr #nmethod_entry_barrier_stub\n\t");
 1750     st->print("b skip\n\t");
 1751     st->print("guard: int\n\t");
 1752     st->print("\n\t");
 1753     st->print("skip:\n\t");
 1754   }
 1755 }
 1756 #endif
 1757 
 1758 void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 1759   Compile* C = ra_->C;
 1760   C2_MacroAssembler _masm(&cbuf);
 1761 
 1762   // insert a nop at the start of the prolog so we can patch in a
 1763   // branch if we need to invalidate the method later
 1764   __ nop();
 1765 
 1766   __ verified_entry(C, 0);
 1767 
 1768   if (C->stub_function() == NULL) {
 1769     __ entry_barrier();
 1770   }
 1771 
 1772   if (!Compile::current()->output()->in_scratch_emit_size()) {
 1773     __ bind(*_verified_entry);
 1774   }
 1775 
 1776   if (VerifyStackAtCalls) {
 1777     Unimplemented();
 1778   }
 1779 
 1780   C->output()->set_frame_complete(cbuf.insts_size());
 1781 
 1782   if (C->has_mach_constant_base_node()) {
 1783     // NOTE: We set the table base offset here because users might be
 1784     // emitted before MachConstantBaseNode.
 1785     ConstantTable& constant_table = C->output()->constant_table();
 1786     constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
 1787   }
 1788 }
 1789 
 1790 int MachPrologNode::reloc() const
 1791 {
 1792   return 0;
 1793 }
 1794 
 1795 //=============================================================================
 1796 
 1797 #ifndef PRODUCT
 1798 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 1799   Compile* C = ra_->C;
 1800   int framesize = C->output()->frame_slots() << LogBytesPerInt;
 1801 
 1802   st->print("# pop frame %d\n\t",framesize);
 1803 
 1804   if (framesize == 0) {
 1805     st->print("ldp  lr, rfp, [sp],#%d\n\t", (2 * wordSize));
 1806   } else if (framesize < ((1 << 9) + 2 * wordSize)) {
 1807     st->print("ldp  lr, rfp, [sp,#%d]\n\t", framesize - 2 * wordSize);
 1808     st->print("add  sp, sp, #%d\n\t", framesize);
 1809   } else {
 1810     st->print("mov  rscratch1, #%d\n\t", framesize - 2 * wordSize);
 1811     st->print("add  sp, sp, rscratch1\n\t");
 1812     st->print("ldp  lr, rfp, [sp],#%d\n\t", (2 * wordSize));
 1813   }
 1814   if (VM_Version::use_rop_protection()) {
 1815     st->print("autia lr, rfp\n\t");
 1816     st->print("ldr zr, [lr]\n\t");
 1817   }
 1818 
 1819   if (do_polling() && C->is_method_compilation()) {
 1820     st->print("# test polling word\n\t");
 1821     st->print("ldr  rscratch1, [rthread],#%d\n\t", in_bytes(JavaThread::polling_word_offset()));
 1822     st->print("cmp  sp, rscratch1\n\t");
 1823     st->print("bhi #slow_path");
 1824   }
 1825 }
 1826 #endif
 1827 
 1828 void MachEpilogNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 1829   Compile* C = ra_->C;
 1830   C2_MacroAssembler _masm(&cbuf);
 1831   int framesize = C->output()->frame_slots() << LogBytesPerInt;
 1832 
 1833   __ remove_frame(framesize, C->needs_stack_repair());
 1834 
 1835   if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
 1836     __ reserved_stack_check();
 1837   }
 1838 
 1839   if (do_polling() && C->is_method_compilation()) {
 1840     Label dummy_label;
 1841     Label* code_stub = &dummy_label;
 1842     if (!C->output()->in_scratch_emit_size()) {
 1843       C2SafepointPollStub* stub = new (C->comp_arena()) C2SafepointPollStub(__ offset());
 1844       C->output()->add_stub(stub);
 1845       code_stub = &stub->entry();
 1846     }
 1847     __ relocate(relocInfo::poll_return_type);
 1848     __ safepoint_poll(*code_stub, true /* at_return */, false /* acquire */, true /* in_nmethod */);
 1849   }
 1850 }
 1851 
 1852 int MachEpilogNode::reloc() const {
 1853   // Return number of relocatable values contained in this instruction.
 1854   return 1; // 1 for polling page.
 1855 }
 1856 
 1857 const Pipeline * MachEpilogNode::pipeline() const {
 1858   return MachNode::pipeline_class();
 1859 }
 1860 
 1861 //=============================================================================
 1862 
 1863 // Figure out which register class each belongs in: rc_int, rc_float or
 1864 // rc_stack.
 1865 enum RC { rc_bad, rc_int, rc_float, rc_predicate, rc_stack };
 1866 
 1867 static enum RC rc_class(OptoReg::Name reg) {
 1868 
 1869   if (reg == OptoReg::Bad) {
 1870     return rc_bad;
 1871   }
 1872 
 1873   // we have 32 int registers * 2 halves
 1874   int slots_of_int_registers = Register::number_of_registers * Register::max_slots_per_register;
 1875 
 1876   if (reg < slots_of_int_registers) {
 1877     return rc_int;
 1878   }
 1879 
 1880   // we have 32 float register * 8 halves
 1881   int slots_of_float_registers = FloatRegister::number_of_registers * FloatRegister::max_slots_per_register;
 1882   if (reg < slots_of_int_registers + slots_of_float_registers) {
 1883     return rc_float;
 1884   }
 1885 
 1886   int slots_of_predicate_registers = PRegister::number_of_registers * PRegister::max_slots_per_register;
 1887   if (reg < slots_of_int_registers + slots_of_float_registers + slots_of_predicate_registers) {
 1888     return rc_predicate;
 1889   }
 1890 
 1891   // Between predicate regs & stack is the flags.
 1892   assert(OptoReg::is_stack(reg), "blow up if spilling flags");
 1893 
 1894   return rc_stack;
 1895 }
 1896 
 1897 uint MachSpillCopyNode::implementation(CodeBuffer *cbuf, PhaseRegAlloc *ra_, bool do_size, outputStream *st) const {
 1898   Compile* C = ra_->C;
 1899 
 1900   // Get registers to move.
 1901   OptoReg::Name src_hi = ra_->get_reg_second(in(1));
 1902   OptoReg::Name src_lo = ra_->get_reg_first(in(1));
 1903   OptoReg::Name dst_hi = ra_->get_reg_second(this);
 1904   OptoReg::Name dst_lo = ra_->get_reg_first(this);
 1905 
 1906   enum RC src_hi_rc = rc_class(src_hi);
 1907   enum RC src_lo_rc = rc_class(src_lo);
 1908   enum RC dst_hi_rc = rc_class(dst_hi);
 1909   enum RC dst_lo_rc = rc_class(dst_lo);
 1910 
 1911   assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
 1912 
 1913   if (src_hi != OptoReg::Bad && !bottom_type()->isa_vectmask()) {
 1914     assert((src_lo&1)==0 && src_lo+1==src_hi &&
 1915            (dst_lo&1)==0 && dst_lo+1==dst_hi,
 1916            "expected aligned-adjacent pairs");
 1917   }
 1918 
 1919   if (src_lo == dst_lo && src_hi == dst_hi) {
 1920     return 0;            // Self copy, no move.
 1921   }
 1922 
 1923   bool is64 = (src_lo & 1) == 0 && src_lo + 1 == src_hi &&
 1924               (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi;
 1925   int src_offset = ra_->reg2offset(src_lo);
 1926   int dst_offset = ra_->reg2offset(dst_lo);
 1927 
 1928   if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
 1929     uint ireg = ideal_reg();
 1930     if (ireg == Op_VecA && cbuf) {
 1931       C2_MacroAssembler _masm(cbuf);
 1932       int sve_vector_reg_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
 1933       if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
 1934         // stack->stack
 1935         __ spill_copy_sve_vector_stack_to_stack(src_offset, dst_offset,
 1936                                                 sve_vector_reg_size_in_bytes);
 1937       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
 1938         __ spill_sve_vector(as_FloatRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
 1939                             sve_vector_reg_size_in_bytes);
 1940       } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
 1941         __ unspill_sve_vector(as_FloatRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
 1942                               sve_vector_reg_size_in_bytes);
 1943       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
 1944         __ sve_orr(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 1945                    as_FloatRegister(Matcher::_regEncode[src_lo]),
 1946                    as_FloatRegister(Matcher::_regEncode[src_lo]));
 1947       } else {
 1948         ShouldNotReachHere();
 1949       }
 1950     } else if (cbuf) {
 1951       assert(ireg == Op_VecD || ireg == Op_VecX, "must be 64 bit or 128 bit vector");
 1952       C2_MacroAssembler _masm(cbuf);
 1953       assert((src_lo_rc != rc_int && dst_lo_rc != rc_int), "sanity");
 1954       if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
 1955         // stack->stack
 1956         assert((src_offset & 7) == 0 && (dst_offset & 7) == 0, "unaligned stack offset");
 1957         if (ireg == Op_VecD) {
 1958           __ unspill(rscratch1, true, src_offset);
 1959           __ spill(rscratch1, true, dst_offset);
 1960         } else {
 1961           __ spill_copy128(src_offset, dst_offset);
 1962         }
 1963       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
 1964         __ mov(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 1965                ireg == Op_VecD ? __ T8B : __ T16B,
 1966                as_FloatRegister(Matcher::_regEncode[src_lo]));
 1967       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
 1968         __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
 1969                  ireg == Op_VecD ? __ D : __ Q,
 1970                  ra_->reg2offset(dst_lo));
 1971       } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
 1972         __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 1973                    ireg == Op_VecD ? __ D : __ Q,
 1974                    ra_->reg2offset(src_lo));
 1975       } else {
 1976         ShouldNotReachHere();
 1977       }
 1978     }
 1979   } else if (cbuf) {
 1980     C2_MacroAssembler _masm(cbuf);
 1981     switch (src_lo_rc) {
 1982     case rc_int:
 1983       if (dst_lo_rc == rc_int) {  // gpr --> gpr copy
 1984         if (is64) {
 1985             __ mov(as_Register(Matcher::_regEncode[dst_lo]),
 1986                    as_Register(Matcher::_regEncode[src_lo]));
 1987         } else {
 1988             C2_MacroAssembler _masm(cbuf);
 1989             __ movw(as_Register(Matcher::_regEncode[dst_lo]),
 1990                     as_Register(Matcher::_regEncode[src_lo]));
 1991         }
 1992       } else if (dst_lo_rc == rc_float) { // gpr --> fpr copy
 1993         if (is64) {
 1994             __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 1995                      as_Register(Matcher::_regEncode[src_lo]));
 1996         } else {
 1997             __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 1998                      as_Register(Matcher::_regEncode[src_lo]));
 1999         }
 2000       } else {                    // gpr --> stack spill
 2001         assert(dst_lo_rc == rc_stack, "spill to bad register class");
 2002         __ spill(as_Register(Matcher::_regEncode[src_lo]), is64, dst_offset);
 2003       }
 2004       break;
 2005     case rc_float:
 2006       if (dst_lo_rc == rc_int) {  // fpr --> gpr copy
 2007         if (is64) {
 2008             __ fmovd(as_Register(Matcher::_regEncode[dst_lo]),
 2009                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2010         } else {
 2011             __ fmovs(as_Register(Matcher::_regEncode[dst_lo]),
 2012                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2013         }
 2014       } else if (dst_lo_rc == rc_float) { // fpr --> fpr copy
 2015         if (is64) {
 2016             __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2017                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2018         } else {
 2019             __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2020                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2021         }
 2022       } else {                    // fpr --> stack spill
 2023         assert(dst_lo_rc == rc_stack, "spill to bad register class");
 2024         __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
 2025                  is64 ? __ D : __ S, dst_offset);
 2026       }
 2027       break;
 2028     case rc_stack:
 2029       if (dst_lo_rc == rc_int) {  // stack --> gpr load
 2030         __ unspill(as_Register(Matcher::_regEncode[dst_lo]), is64, src_offset);
 2031       } else if (dst_lo_rc == rc_float) { // stack --> fpr load
 2032         __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2033                    is64 ? __ D : __ S, src_offset);
 2034       } else if (dst_lo_rc == rc_predicate) {
 2035         __ unspill_sve_predicate(as_PRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
 2036                                  Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
 2037       } else {                    // stack --> stack copy
 2038         assert(dst_lo_rc == rc_stack, "spill to bad register class");
 2039         if (ideal_reg() == Op_RegVectMask) {
 2040           __ spill_copy_sve_predicate_stack_to_stack(src_offset, dst_offset,
 2041                                                      Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
 2042         } else {
 2043           __ unspill(rscratch1, is64, src_offset);
 2044           __ spill(rscratch1, is64, dst_offset);
 2045         }
 2046       }
 2047       break;
 2048     case rc_predicate:
 2049       if (dst_lo_rc == rc_predicate) {
 2050         __ sve_mov(as_PRegister(Matcher::_regEncode[dst_lo]), as_PRegister(Matcher::_regEncode[src_lo]));
 2051       } else if (dst_lo_rc == rc_stack) {
 2052         __ spill_sve_predicate(as_PRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
 2053                                Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
 2054       } else {
 2055         assert(false, "bad src and dst rc_class combination.");
 2056         ShouldNotReachHere();
 2057       }
 2058       break;
 2059     default:
 2060       assert(false, "bad rc_class for spill");
 2061       ShouldNotReachHere();
 2062     }
 2063   }
 2064 
 2065   if (st) {
 2066     st->print("spill ");
 2067     if (src_lo_rc == rc_stack) {
 2068       st->print("[sp, #%d] -> ", ra_->reg2offset(src_lo));
 2069     } else {
 2070       st->print("%s -> ", Matcher::regName[src_lo]);
 2071     }
 2072     if (dst_lo_rc == rc_stack) {
 2073       st->print("[sp, #%d]", ra_->reg2offset(dst_lo));
 2074     } else {
 2075       st->print("%s", Matcher::regName[dst_lo]);
 2076     }
 2077     if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
 2078       int vsize = 0;
 2079       switch (ideal_reg()) {
 2080       case Op_VecD:
 2081         vsize = 64;
 2082         break;
 2083       case Op_VecX:
 2084         vsize = 128;
 2085         break;
 2086       case Op_VecA:
 2087         vsize = Matcher::scalable_vector_reg_size(T_BYTE) * 8;
 2088         break;
 2089       default:
 2090         assert(false, "bad register type for spill");
 2091         ShouldNotReachHere();
 2092       }
 2093       st->print("\t# vector spill size = %d", vsize);
 2094     } else if (ideal_reg() == Op_RegVectMask) {
 2095       assert(Matcher::supports_scalable_vector(), "bad register type for spill");
 2096       int vsize = Matcher::scalable_predicate_reg_slots() * 32;
 2097       st->print("\t# predicate spill size = %d", vsize);
 2098     } else {
 2099       st->print("\t# spill size = %d", is64 ? 64 : 32);
 2100     }
 2101   }
 2102 
 2103   return 0;
 2104 
 2105 }
 2106 
 2107 #ifndef PRODUCT
 2108 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 2109   if (!ra_)
 2110     st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
 2111   else
 2112     implementation(NULL, ra_, false, st);
 2113 }
 2114 #endif
 2115 
 2116 void MachSpillCopyNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 2117   implementation(&cbuf, ra_, false, NULL);
 2118 }
 2119 
 2120 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
 2121   return MachNode::size(ra_);
 2122 }
 2123 
 2124 //=============================================================================
 2125 
 2126 #ifndef PRODUCT
 2127 void BoxLockNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 2128   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 2129   int reg = ra_->get_reg_first(this);
 2130   st->print("add %s, rsp, #%d]\t# box lock",
 2131             Matcher::regName[reg], offset);
 2132 }
 2133 #endif
 2134 
 2135 void BoxLockNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 2136   C2_MacroAssembler _masm(&cbuf);
 2137 
 2138   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 2139   int reg    = ra_->get_encode(this);
 2140 
 2141   // This add will handle any 24-bit signed offset. 24 bits allows an
 2142   // 8 megabyte stack frame.
 2143   __ add(as_Register(reg), sp, offset);
 2144 }
 2145 
 2146 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
 2147   // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_).
 2148   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 2149 
 2150   if (Assembler::operand_valid_for_add_sub_immediate(offset)) {
 2151     return NativeInstruction::instruction_size;
 2152   } else {
 2153     return 2 * NativeInstruction::instruction_size;
 2154   }
 2155 }
 2156 
 2157 ///=============================================================================
 2158 #ifndef PRODUCT
 2159 void MachVEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 2160 {
 2161   st->print_cr("# MachVEPNode");
 2162   if (!_verified) {
 2163     st->print_cr("\t load_class");
 2164   } else {
 2165     st->print_cr("\t unpack_inline_arg");
 2166   }
 2167 }
 2168 #endif
 2169 
 2170 void MachVEPNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
 2171 {
 2172   C2_MacroAssembler _masm(&cbuf);
 2173 
 2174   if (!_verified) {
 2175     Label skip;
 2176     __ cmp_klass(j_rarg0, rscratch2, rscratch1);
 2177     __ br(Assembler::EQ, skip);
 2178       __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
 2179     __ bind(skip);
 2180 
 2181   } else {
 2182     // insert a nop at the start of the prolog so we can patch in a
 2183     // branch if we need to invalidate the method later
 2184     __ nop();
 2185 
 2186     // TODO 8284443 Avoid creation of temporary frame
 2187     if (ra_->C->stub_function() == NULL) {
 2188       __ verified_entry(ra_->C, 0);
 2189       __ entry_barrier();
 2190       int framesize = ra_->C->output()->frame_slots() << LogBytesPerInt;
 2191       __ remove_frame(framesize, false);
 2192     }
 2193     // Unpack inline type args passed as oop and then jump to
 2194     // the verified entry point (skipping the unverified entry).
 2195     int sp_inc = __ unpack_inline_args(ra_->C, _receiver_only);
 2196     // Emit code for verified entry and save increment for stack repair on return
 2197     __ verified_entry(ra_->C, sp_inc);
 2198     if (Compile::current()->output()->in_scratch_emit_size()) {
 2199       Label dummy_verified_entry;
 2200       __ b(dummy_verified_entry);
 2201     } else {
 2202       __ b(*_verified_entry);
 2203     }
 2204   }
 2205 }
 2206 
 2207 //=============================================================================
 2208 #ifndef PRODUCT
 2209 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 2210 {
 2211   st->print_cr("# MachUEPNode");
 2212   if (UseCompressedClassPointers) {
 2213     st->print_cr("\tldrw rscratch1, j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
 2214     if (CompressedKlassPointers::shift() != 0) {
 2215       st->print_cr("\tdecode_klass_not_null rscratch1, rscratch1");
 2216     }
 2217   } else {
 2218    st->print_cr("\tldr rscratch1, j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
 2219   }
 2220   st->print_cr("\tcmp r0, rscratch1\t # Inline cache check");
 2221   st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
 2222 }
 2223 #endif
 2224 
 2225 void MachUEPNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
 2226 {
 2227   // This is the unverified entry point.
 2228   C2_MacroAssembler _masm(&cbuf);
 2229   Label skip;
 2230 
 2231   // UseCompressedClassPointers logic are inside cmp_klass
 2232   __ cmp_klass(j_rarg0, rscratch2, rscratch1);
 2233 
 2234   // TODO
 2235   // can we avoid this skip and still use a reloc?
 2236   __ br(Assembler::EQ, skip);
 2237   __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
 2238   __ bind(skip);
 2239 }
 2240 
 2241 // REQUIRED EMIT CODE
 2242 
 2243 //=============================================================================
 2244 
 2245 // Emit exception handler code.
 2246 int HandlerImpl::emit_exception_handler(CodeBuffer& cbuf)
 2247 {
 2248   // mov rscratch1 #exception_blob_entry_point
 2249   // br rscratch1
 2250   // Note that the code buffer's insts_mark is always relative to insts.
 2251   // That's why we must use the macroassembler to generate a handler.
 2252   C2_MacroAssembler _masm(&cbuf);
 2253   address base = __ start_a_stub(size_exception_handler());
 2254   if (base == NULL) {
 2255     ciEnv::current()->record_failure("CodeCache is full");
 2256     return 0;  // CodeBuffer::expand failed
 2257   }
 2258   int offset = __ offset();
 2259   __ far_jump(RuntimeAddress(OptoRuntime::exception_blob()->entry_point()));
 2260   assert(__ offset() - offset <= (int) size_exception_handler(), "overflow");
 2261   __ end_a_stub();
 2262   return offset;
 2263 }
 2264 
 2265 // Emit deopt handler code.
 2266 int HandlerImpl::emit_deopt_handler(CodeBuffer& cbuf)
 2267 {
 2268   // Note that the code buffer's insts_mark is always relative to insts.
 2269   // That's why we must use the macroassembler to generate a handler.
 2270   C2_MacroAssembler _masm(&cbuf);
 2271   address base = __ start_a_stub(size_deopt_handler());
 2272   if (base == NULL) {
 2273     ciEnv::current()->record_failure("CodeCache is full");
 2274     return 0;  // CodeBuffer::expand failed
 2275   }
 2276   int offset = __ offset();
 2277 
 2278   __ adr(lr, __ pc());
 2279   __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
 2280 
 2281   assert(__ offset() - offset == (int) size_deopt_handler(), "overflow");
 2282   __ end_a_stub();
 2283   return offset;
 2284 }
 2285 
 2286 // REQUIRED MATCHER CODE
 2287 
 2288 //=============================================================================
 2289 
 2290 const bool Matcher::match_rule_supported(int opcode) {
 2291   if (!has_match_rule(opcode))
 2292     return false;
 2293 
 2294   bool ret_value = true;
 2295   switch (opcode) {
 2296     case Op_OnSpinWait:
 2297       return VM_Version::supports_on_spin_wait();
 2298     case Op_CacheWB:
 2299     case Op_CacheWBPreSync:
 2300     case Op_CacheWBPostSync:
 2301       if (!VM_Version::supports_data_cache_line_flush()) {
 2302         ret_value = false;
 2303       }
 2304       break;
 2305   }
 2306 
 2307   return ret_value; // Per default match rules are supported.
 2308 }
 2309 
 2310 const RegMask* Matcher::predicate_reg_mask(void) {
 2311   return &_PR_REG_mask;
 2312 }
 2313 
 2314 const TypeVectMask* Matcher::predicate_reg_type(const Type* elemTy, int length) {
 2315   return new TypeVectMask(elemTy, length);
 2316 }
 2317 
 2318 // Vector calling convention not yet implemented.
 2319 const bool Matcher::supports_vector_calling_convention(void) {
 2320   return false;
 2321 }
 2322 
 2323 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
 2324   Unimplemented();
 2325   return OptoRegPair(0, 0);
 2326 }
 2327 
 2328 // Is this branch offset short enough that a short branch can be used?
 2329 //
 2330 // NOTE: If the platform does not provide any short branch variants, then
 2331 //       this method should return false for offset 0.
 2332 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
 2333   // The passed offset is relative to address of the branch.
 2334 
 2335   return (-32768 <= offset && offset < 32768);
 2336 }
 2337 
 2338 // Vector width in bytes.
 2339 const int Matcher::vector_width_in_bytes(BasicType bt) {
 2340   // The MaxVectorSize should have been set by detecting SVE max vector register size.
 2341   int size = MIN2((UseSVE > 0) ? 256 : 16, (int)MaxVectorSize);
 2342   // Minimum 2 values in vector
 2343   if (size < 2*type2aelembytes(bt)) size = 0;
 2344   // But never < 4
 2345   if (size < 4) size = 0;
 2346   return size;
 2347 }
 2348 
 2349 // Limits on vector size (number of elements) loaded into vector.
 2350 const int Matcher::max_vector_size(const BasicType bt) {
 2351   return vector_width_in_bytes(bt)/type2aelembytes(bt);
 2352 }
 2353 
 2354 const int Matcher::min_vector_size(const BasicType bt) {
 2355   int max_size = max_vector_size(bt);
 2356   // Limit the min vector size to 8 bytes.
 2357   int size = 8 / type2aelembytes(bt);
 2358   if (bt == T_BYTE) {
 2359     // To support vector api shuffle/rearrange.
 2360     size = 4;
 2361   } else if (bt == T_BOOLEAN) {
 2362     // To support vector api load/store mask.
 2363     size = 2;
 2364   }
 2365   if (size < 2) size = 2;
 2366   return MIN2(size, max_size);
 2367 }
 2368 
 2369 // Actual max scalable vector register length.
 2370 const int Matcher::scalable_vector_reg_size(const BasicType bt) {
 2371   return Matcher::max_vector_size(bt);
 2372 }
 2373 
 2374 // Vector ideal reg.
 2375 const uint Matcher::vector_ideal_reg(int len) {
 2376   if (UseSVE > 0 && 16 < len && len <= 256) {
 2377     return Op_VecA;
 2378   }
 2379   switch(len) {
 2380     // For 16-bit/32-bit mask vector, reuse VecD.
 2381     case  2:
 2382     case  4:
 2383     case  8: return Op_VecD;
 2384     case 16: return Op_VecX;
 2385   }
 2386   ShouldNotReachHere();
 2387   return 0;
 2388 }
 2389 
 2390 MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* generic_opnd, uint ideal_reg, bool is_temp) {
 2391   assert(Matcher::is_generic_vector(generic_opnd), "not generic");
 2392   switch (ideal_reg) {
 2393     case Op_VecA: return new vecAOper();
 2394     case Op_VecD: return new vecDOper();
 2395     case Op_VecX: return new vecXOper();
 2396   }
 2397   ShouldNotReachHere();
 2398   return NULL;
 2399 }
 2400 
 2401 bool Matcher::is_reg2reg_move(MachNode* m) {
 2402   return false;
 2403 }
 2404 
 2405 bool Matcher::is_generic_vector(MachOper* opnd)  {
 2406   return opnd->opcode() == VREG;
 2407 }
 2408 
 2409 // Return whether or not this register is ever used as an argument.
 2410 // This function is used on startup to build the trampoline stubs in
 2411 // generateOptoStub.  Registers not mentioned will be killed by the VM
 2412 // call in the trampoline, and arguments in those registers not be
 2413 // available to the callee.
 2414 bool Matcher::can_be_java_arg(int reg)
 2415 {
 2416   return
 2417     reg ==  R0_num || reg == R0_H_num ||
 2418     reg ==  R1_num || reg == R1_H_num ||
 2419     reg ==  R2_num || reg == R2_H_num ||
 2420     reg ==  R3_num || reg == R3_H_num ||
 2421     reg ==  R4_num || reg == R4_H_num ||
 2422     reg ==  R5_num || reg == R5_H_num ||
 2423     reg ==  R6_num || reg == R6_H_num ||
 2424     reg ==  R7_num || reg == R7_H_num ||
 2425     reg ==  V0_num || reg == V0_H_num ||
 2426     reg ==  V1_num || reg == V1_H_num ||
 2427     reg ==  V2_num || reg == V2_H_num ||
 2428     reg ==  V3_num || reg == V3_H_num ||
 2429     reg ==  V4_num || reg == V4_H_num ||
 2430     reg ==  V5_num || reg == V5_H_num ||
 2431     reg ==  V6_num || reg == V6_H_num ||
 2432     reg ==  V7_num || reg == V7_H_num;
 2433 }
 2434 
 2435 bool Matcher::is_spillable_arg(int reg)
 2436 {
 2437   return can_be_java_arg(reg);
 2438 }
 2439 
 2440 uint Matcher::int_pressure_limit()
 2441 {
 2442   // JDK-8183543: When taking the number of available registers as int
 2443   // register pressure threshold, the jtreg test:
 2444   // test/hotspot/jtreg/compiler/regalloc/TestC2IntPressure.java
 2445   // failed due to C2 compilation failure with
 2446   // "COMPILE SKIPPED: failed spill-split-recycle sanity check".
 2447   //
 2448   // A derived pointer is live at CallNode and then is flagged by RA
 2449   // as a spilled LRG. Spilling heuristics(Spill-USE) explicitly skip
 2450   // derived pointers and lastly fail to spill after reaching maximum
 2451   // number of iterations. Lowering the default pressure threshold to
 2452   // (_NO_SPECIAL_REG32_mask.Size() minus 1) forces CallNode to become
 2453   // a high register pressure area of the code so that split_DEF can
 2454   // generate DefinitionSpillCopy for the derived pointer.
 2455   uint default_int_pressure_threshold = _NO_SPECIAL_REG32_mask.Size() - 1;
 2456   if (!PreserveFramePointer) {
 2457     // When PreserveFramePointer is off, frame pointer is allocatable,
 2458     // but different from other SOC registers, it is excluded from
 2459     // fatproj's mask because its save type is No-Save. Decrease 1 to
 2460     // ensure high pressure at fatproj when PreserveFramePointer is off.
 2461     // See check_pressure_at_fatproj().
 2462     default_int_pressure_threshold--;
 2463   }
 2464   return (INTPRESSURE == -1) ? default_int_pressure_threshold : INTPRESSURE;
 2465 }
 2466 
 2467 uint Matcher::float_pressure_limit()
 2468 {
 2469   // _FLOAT_REG_mask is generated by adlc from the float_reg register class.
 2470   return (FLOATPRESSURE == -1) ? _FLOAT_REG_mask.Size() : FLOATPRESSURE;
 2471 }
 2472 
 2473 bool Matcher::use_asm_for_ldiv_by_con(jlong divisor) {
 2474   return false;
 2475 }
 2476 
 2477 RegMask Matcher::divI_proj_mask() {
 2478   ShouldNotReachHere();
 2479   return RegMask();
 2480 }
 2481 
 2482 // Register for MODI projection of divmodI.
 2483 RegMask Matcher::modI_proj_mask() {
 2484   ShouldNotReachHere();
 2485   return RegMask();
 2486 }
 2487 
 2488 // Register for DIVL projection of divmodL.
 2489 RegMask Matcher::divL_proj_mask() {
 2490   ShouldNotReachHere();
 2491   return RegMask();
 2492 }
 2493 
 2494 // Register for MODL projection of divmodL.
 2495 RegMask Matcher::modL_proj_mask() {
 2496   ShouldNotReachHere();
 2497   return RegMask();
 2498 }
 2499 
 2500 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
 2501   return FP_REG_mask();
 2502 }
 2503 
 2504 bool size_fits_all_mem_uses(AddPNode* addp, int shift) {
 2505   for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) {
 2506     Node* u = addp->fast_out(i);
 2507     if (u->is_LoadStore()) {
 2508       // On AArch64, LoadStoreNodes (i.e. compare and swap
 2509       // instructions) only take register indirect as an operand, so
 2510       // any attempt to use an AddPNode as an input to a LoadStoreNode
 2511       // must fail.
 2512       return false;
 2513     }
 2514     if (u->is_Mem()) {
 2515       int opsize = u->as_Mem()->memory_size();
 2516       assert(opsize > 0, "unexpected memory operand size");
 2517       if (u->as_Mem()->memory_size() != (1<<shift)) {
 2518         return false;
 2519       }
 2520     }
 2521   }
 2522   return true;
 2523 }
 2524 
 2525 // Binary src (Replicate con)
 2526 bool is_valid_sve_arith_imm_pattern(Node* n, Node* m) {
 2527   if (n == NULL || m == NULL) {
 2528     return false;
 2529   }
 2530 
 2531   if (UseSVE == 0 || !VectorNode::is_invariant_vector(m)) {
 2532     return false;
 2533   }
 2534 
 2535   Node* imm_node = m->in(1);
 2536   if (!imm_node->is_Con()) {
 2537     return false;
 2538   }
 2539 
 2540   const Type* t = imm_node->bottom_type();
 2541   if (!(t->isa_int() || t->isa_long())) {
 2542     return false;
 2543   }
 2544 
 2545   switch (n->Opcode()) {
 2546   case Op_AndV:
 2547   case Op_OrV:
 2548   case Op_XorV: {
 2549     Assembler::SIMD_RegVariant T = Assembler::elemType_to_regVariant(Matcher::vector_element_basic_type(n));
 2550     uint64_t value = t->isa_long() ? (uint64_t)imm_node->get_long() : (uint64_t)imm_node->get_int();
 2551     return Assembler::operand_valid_for_sve_logical_immediate(Assembler::regVariant_to_elemBits(T), value);
 2552   }
 2553   case Op_AddVB:
 2554     return (imm_node->get_int() <= 255 && imm_node->get_int() >= -255);
 2555   case Op_AddVS:
 2556   case Op_AddVI:
 2557     return Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)imm_node->get_int());
 2558   case Op_AddVL:
 2559     return Assembler::operand_valid_for_sve_add_sub_immediate(imm_node->get_long());
 2560   default:
 2561     return false;
 2562   }
 2563 }
 2564 
 2565 // (XorV src (Replicate m1))
 2566 // (XorVMask src (MaskAll m1))
 2567 bool is_vector_bitwise_not_pattern(Node* n, Node* m) {
 2568   if (n != NULL && m != NULL) {
 2569     return (n->Opcode() == Op_XorV || n->Opcode() == Op_XorVMask) &&
 2570            VectorNode::is_all_ones_vector(m);
 2571   }
 2572   return false;
 2573 }
 2574 
 2575 // Should the matcher clone input 'm' of node 'n'?
 2576 bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
 2577   if (is_vshift_con_pattern(n, m) ||
 2578       is_vector_bitwise_not_pattern(n, m) ||
 2579       is_valid_sve_arith_imm_pattern(n, m)) {
 2580     mstack.push(m, Visit);
 2581     return true;
 2582   }
 2583   return false;
 2584 }
 2585 
 2586 // Should the Matcher clone shifts on addressing modes, expecting them
 2587 // to be subsumed into complex addressing expressions or compute them
 2588 // into registers?
 2589 bool Matcher::pd_clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
 2590   if (clone_base_plus_offset_address(m, mstack, address_visited)) {
 2591     return true;
 2592   }
 2593 
 2594   Node *off = m->in(AddPNode::Offset);
 2595   if (off->Opcode() == Op_LShiftL && off->in(2)->is_Con() &&
 2596       size_fits_all_mem_uses(m, off->in(2)->get_int()) &&
 2597       // Are there other uses besides address expressions?
 2598       !is_visited(off)) {
 2599     address_visited.set(off->_idx); // Flag as address_visited
 2600     mstack.push(off->in(2), Visit);
 2601     Node *conv = off->in(1);
 2602     if (conv->Opcode() == Op_ConvI2L &&
 2603         // Are there other uses besides address expressions?
 2604         !is_visited(conv)) {
 2605       address_visited.set(conv->_idx); // Flag as address_visited
 2606       mstack.push(conv->in(1), Pre_Visit);
 2607     } else {
 2608       mstack.push(conv, Pre_Visit);
 2609     }
 2610     address_visited.test_set(m->_idx); // Flag as address_visited
 2611     mstack.push(m->in(AddPNode::Address), Pre_Visit);
 2612     mstack.push(m->in(AddPNode::Base), Pre_Visit);
 2613     return true;
 2614   } else if (off->Opcode() == Op_ConvI2L &&
 2615              // Are there other uses besides address expressions?
 2616              !is_visited(off)) {
 2617     address_visited.test_set(m->_idx); // Flag as address_visited
 2618     address_visited.set(off->_idx); // Flag as address_visited
 2619     mstack.push(off->in(1), Pre_Visit);
 2620     mstack.push(m->in(AddPNode::Address), Pre_Visit);
 2621     mstack.push(m->in(AddPNode::Base), Pre_Visit);
 2622     return true;
 2623   }
 2624   return false;
 2625 }
 2626 
 2627 #define MOV_VOLATILE(REG, BASE, INDEX, SCALE, DISP, SCRATCH, INSN)      \
 2628   C2_MacroAssembler _masm(&cbuf);                                       \
 2629   {                                                                     \
 2630     guarantee(INDEX == -1, "mode not permitted for volatile");          \
 2631     guarantee(DISP == 0, "mode not permitted for volatile");            \
 2632     guarantee(SCALE == 0, "mode not permitted for volatile");           \
 2633     __ INSN(REG, as_Register(BASE));                                    \
 2634   }
 2635 
 2636 
 2637 static Address mem2address(int opcode, Register base, int index, int size, int disp)
 2638   {
 2639     Address::extend scale;
 2640 
 2641     // Hooboy, this is fugly.  We need a way to communicate to the
 2642     // encoder that the index needs to be sign extended, so we have to
 2643     // enumerate all the cases.
 2644     switch (opcode) {
 2645     case INDINDEXSCALEDI2L:
 2646     case INDINDEXSCALEDI2LN:
 2647     case INDINDEXI2L:
 2648     case INDINDEXI2LN:
 2649       scale = Address::sxtw(size);
 2650       break;
 2651     default:
 2652       scale = Address::lsl(size);
 2653     }
 2654 
 2655     if (index == -1) {
 2656       return Address(base, disp);
 2657     } else {
 2658       assert(disp == 0, "unsupported address mode: disp = %d", disp);
 2659       return Address(base, as_Register(index), scale);
 2660     }
 2661   }
 2662 
 2663 
 2664 typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr);
 2665 typedef void (MacroAssembler::* mem_insn2)(Register Rt, Register adr);
 2666 typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr);
 2667 typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt,
 2668                                   MacroAssembler::SIMD_RegVariant T, const Address &adr);
 2669 
 2670   // Used for all non-volatile memory accesses.  The use of
 2671   // $mem->opcode() to discover whether this pattern uses sign-extended
 2672   // offsets is something of a kludge.
 2673   static void loadStore(C2_MacroAssembler masm, mem_insn insn,
 2674                         Register reg, int opcode,
 2675                         Register base, int index, int scale, int disp,
 2676                         int size_in_memory)
 2677   {
 2678     Address addr = mem2address(opcode, base, index, scale, disp);
 2679     if (addr.getMode() == Address::base_plus_offset) {
 2680       /* If we get an out-of-range offset it is a bug in the compiler,
 2681          so we assert here. */
 2682       assert(Address::offset_ok_for_immed(addr.offset(), exact_log2(size_in_memory)),
 2683              "c2 compiler bug");
 2684       /* Fix up any out-of-range offsets. */
 2685       assert_different_registers(rscratch1, base);
 2686       assert_different_registers(rscratch1, reg);
 2687       addr = masm.legitimize_address(addr, size_in_memory, rscratch1);
 2688     }
 2689     (masm.*insn)(reg, addr);
 2690   }
 2691 
 2692   static void loadStore(C2_MacroAssembler masm, mem_float_insn insn,
 2693                         FloatRegister reg, int opcode,
 2694                         Register base, int index, int size, int disp,
 2695                         int size_in_memory)
 2696   {
 2697     Address::extend scale;
 2698 
 2699     switch (opcode) {
 2700     case INDINDEXSCALEDI2L:
 2701     case INDINDEXSCALEDI2LN:
 2702       scale = Address::sxtw(size);
 2703       break;
 2704     default:
 2705       scale = Address::lsl(size);
 2706     }
 2707 
 2708     if (index == -1) {
 2709       /* If we get an out-of-range offset it is a bug in the compiler,
 2710          so we assert here. */
 2711       assert(Address::offset_ok_for_immed(disp, exact_log2(size_in_memory)), "c2 compiler bug");
 2712       /* Fix up any out-of-range offsets. */
 2713       assert_different_registers(rscratch1, base);
 2714       Address addr = Address(base, disp);
 2715       addr = masm.legitimize_address(addr, size_in_memory, rscratch1);
 2716       (masm.*insn)(reg, addr);
 2717     } else {
 2718       assert(disp == 0, "unsupported address mode: disp = %d", disp);
 2719       (masm.*insn)(reg, Address(base, as_Register(index), scale));
 2720     }
 2721   }
 2722 
 2723   static void loadStore(C2_MacroAssembler masm, mem_vector_insn insn,
 2724                         FloatRegister reg, MacroAssembler::SIMD_RegVariant T,
 2725                         int opcode, Register base, int index, int size, int disp)
 2726   {
 2727     if (index == -1) {
 2728       (masm.*insn)(reg, T, Address(base, disp));
 2729     } else {
 2730       assert(disp == 0, "unsupported address mode");
 2731       (masm.*insn)(reg, T, Address(base, as_Register(index), Address::lsl(size)));
 2732     }
 2733   }
 2734 
 2735 %}
 2736 
 2737 
 2738 
 2739 //----------ENCODING BLOCK-----------------------------------------------------
 2740 // This block specifies the encoding classes used by the compiler to
 2741 // output byte streams.  Encoding classes are parameterized macros
 2742 // used by Machine Instruction Nodes in order to generate the bit
 2743 // encoding of the instruction.  Operands specify their base encoding
 2744 // interface with the interface keyword.  There are currently
 2745 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
 2746 // COND_INTER.  REG_INTER causes an operand to generate a function
 2747 // which returns its register number when queried.  CONST_INTER causes
 2748 // an operand to generate a function which returns the value of the
 2749 // constant when queried.  MEMORY_INTER causes an operand to generate
 2750 // four functions which return the Base Register, the Index Register,
 2751 // the Scale Value, and the Offset Value of the operand when queried.
 2752 // COND_INTER causes an operand to generate six functions which return
 2753 // the encoding code (ie - encoding bits for the instruction)
 2754 // associated with each basic boolean condition for a conditional
 2755 // instruction.
 2756 //
 2757 // Instructions specify two basic values for encoding.  Again, a
 2758 // function is available to check if the constant displacement is an
 2759 // oop. They use the ins_encode keyword to specify their encoding
 2760 // classes (which must be a sequence of enc_class names, and their
 2761 // parameters, specified in the encoding block), and they use the
 2762 // opcode keyword to specify, in order, their primary, secondary, and
 2763 // tertiary opcode.  Only the opcode sections which a particular
 2764 // instruction needs for encoding need to be specified.
 2765 encode %{
 2766   // Build emit functions for each basic byte or larger field in the
 2767   // intel encoding scheme (opcode, rm, sib, immediate), and call them
 2768   // from C++ code in the enc_class source block.  Emit functions will
 2769   // live in the main source block for now.  In future, we can
 2770   // generalize this by adding a syntax that specifies the sizes of
 2771   // fields in an order, so that the adlc can build the emit functions
 2772   // automagically
 2773 
 2774   // catch all for unimplemented encodings
 2775   enc_class enc_unimplemented %{
 2776     C2_MacroAssembler _masm(&cbuf);
 2777     __ unimplemented("C2 catch all");
 2778   %}
 2779 
 2780   // BEGIN Non-volatile memory access
 2781 
 2782   // This encoding class is generated automatically from ad_encode.m4.
 2783   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2784   enc_class aarch64_enc_ldrsbw(iRegI dst, memory1 mem) %{
 2785     Register dst_reg = as_Register($dst$$reg);
 2786     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsbw, dst_reg, $mem->opcode(),
 2787                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2788   %}
 2789 
 2790   // This encoding class is generated automatically from ad_encode.m4.
 2791   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2792   enc_class aarch64_enc_ldrsb(iRegI dst, memory1 mem) %{
 2793     Register dst_reg = as_Register($dst$$reg);
 2794     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsb, dst_reg, $mem->opcode(),
 2795                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2796   %}
 2797 
 2798   // This encoding class is generated automatically from ad_encode.m4.
 2799   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2800   enc_class aarch64_enc_ldrb(iRegI dst, memory1 mem) %{
 2801     Register dst_reg = as_Register($dst$$reg);
 2802     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
 2803                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2804   %}
 2805 
 2806   // This encoding class is generated automatically from ad_encode.m4.
 2807   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2808   enc_class aarch64_enc_ldrb(iRegL dst, memory1 mem) %{
 2809     Register dst_reg = as_Register($dst$$reg);
 2810     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
 2811                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2812   %}
 2813 
 2814   // This encoding class is generated automatically from ad_encode.m4.
 2815   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2816   enc_class aarch64_enc_ldrshw(iRegI dst, memory2 mem) %{
 2817     Register dst_reg = as_Register($dst$$reg);
 2818     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrshw, dst_reg, $mem->opcode(),
 2819                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2820   %}
 2821 
 2822   // This encoding class is generated automatically from ad_encode.m4.
 2823   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2824   enc_class aarch64_enc_ldrsh(iRegI dst, memory2 mem) %{
 2825     Register dst_reg = as_Register($dst$$reg);
 2826     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsh, dst_reg, $mem->opcode(),
 2827                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2828   %}
 2829 
 2830   // This encoding class is generated automatically from ad_encode.m4.
 2831   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2832   enc_class aarch64_enc_ldrh(iRegI dst, memory2 mem) %{
 2833     Register dst_reg = as_Register($dst$$reg);
 2834     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
 2835                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2836   %}
 2837 
 2838   // This encoding class is generated automatically from ad_encode.m4.
 2839   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2840   enc_class aarch64_enc_ldrh(iRegL dst, memory2 mem) %{
 2841     Register dst_reg = as_Register($dst$$reg);
 2842     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
 2843                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2844   %}
 2845 
 2846   // This encoding class is generated automatically from ad_encode.m4.
 2847   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2848   enc_class aarch64_enc_ldrw(iRegI dst, memory4 mem) %{
 2849     Register dst_reg = as_Register($dst$$reg);
 2850     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
 2851                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2852   %}
 2853 
 2854   // This encoding class is generated automatically from ad_encode.m4.
 2855   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2856   enc_class aarch64_enc_ldrw(iRegL dst, memory4 mem) %{
 2857     Register dst_reg = as_Register($dst$$reg);
 2858     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
 2859                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2860   %}
 2861 
 2862   // This encoding class is generated automatically from ad_encode.m4.
 2863   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2864   enc_class aarch64_enc_ldrsw(iRegL dst, memory4 mem) %{
 2865     Register dst_reg = as_Register($dst$$reg);
 2866     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsw, dst_reg, $mem->opcode(),
 2867                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2868   %}
 2869 
 2870   // This encoding class is generated automatically from ad_encode.m4.
 2871   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2872   enc_class aarch64_enc_ldr(iRegL dst, memory8 mem) %{
 2873     Register dst_reg = as_Register($dst$$reg);
 2874     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, $mem->opcode(),
 2875                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 2876   %}
 2877 
 2878   // This encoding class is generated automatically from ad_encode.m4.
 2879   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2880   enc_class aarch64_enc_ldrs(vRegF dst, memory4 mem) %{
 2881     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 2882     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrs, dst_reg, $mem->opcode(),
 2883                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2884   %}
 2885 
 2886   // This encoding class is generated automatically from ad_encode.m4.
 2887   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2888   enc_class aarch64_enc_ldrd(vRegD dst, memory8 mem) %{
 2889     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 2890     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrd, dst_reg, $mem->opcode(),
 2891                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 2892   %}
 2893 
 2894   // This encoding class is generated automatically from ad_encode.m4.
 2895   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2896   enc_class aarch64_enc_strb(iRegI src, memory1 mem) %{
 2897     Register src_reg = as_Register($src$$reg);
 2898     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strb, src_reg, $mem->opcode(),
 2899                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2900   %}
 2901 
 2902   // This encoding class is generated automatically from ad_encode.m4.
 2903   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2904   enc_class aarch64_enc_strb0(memory1 mem) %{
 2905     C2_MacroAssembler _masm(&cbuf);
 2906     loadStore(_masm, &MacroAssembler::strb, zr, $mem->opcode(),
 2907                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2908   %}
 2909 
 2910   // This encoding class is generated automatically from ad_encode.m4.
 2911   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2912   enc_class aarch64_enc_strh(iRegI src, memory2 mem) %{
 2913     Register src_reg = as_Register($src$$reg);
 2914     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strh, src_reg, $mem->opcode(),
 2915                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2916   %}
 2917 
 2918   // This encoding class is generated automatically from ad_encode.m4.
 2919   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2920   enc_class aarch64_enc_strh0(memory2 mem) %{
 2921     C2_MacroAssembler _masm(&cbuf);
 2922     loadStore(_masm, &MacroAssembler::strh, zr, $mem->opcode(),
 2923                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2924   %}
 2925 
 2926   // This encoding class is generated automatically from ad_encode.m4.
 2927   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2928   enc_class aarch64_enc_strw(iRegI src, memory4 mem) %{
 2929     Register src_reg = as_Register($src$$reg);
 2930     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strw, src_reg, $mem->opcode(),
 2931                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2932   %}
 2933 
 2934   // This encoding class is generated automatically from ad_encode.m4.
 2935   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2936   enc_class aarch64_enc_strw0(memory4 mem) %{
 2937     C2_MacroAssembler _masm(&cbuf);
 2938     loadStore(_masm, &MacroAssembler::strw, zr, $mem->opcode(),
 2939                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2940   %}
 2941 
 2942   // This encoding class is generated automatically from ad_encode.m4.
 2943   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2944   enc_class aarch64_enc_str(iRegL src, memory8 mem) %{
 2945     Register src_reg = as_Register($src$$reg);
 2946     // we sometimes get asked to store the stack pointer into the
 2947     // current thread -- we cannot do that directly on AArch64
 2948     if (src_reg == r31_sp) {
 2949       C2_MacroAssembler _masm(&cbuf);
 2950       assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
 2951       __ mov(rscratch2, sp);
 2952       src_reg = rscratch2;
 2953     }
 2954     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, $mem->opcode(),
 2955                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 2956   %}
 2957 
 2958   // This encoding class is generated automatically from ad_encode.m4.
 2959   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2960   enc_class aarch64_enc_str0(memory8 mem) %{
 2961     C2_MacroAssembler _masm(&cbuf);
 2962     loadStore(_masm, &MacroAssembler::str, zr, $mem->opcode(),
 2963                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 2964   %}
 2965 
 2966   // This encoding class is generated automatically from ad_encode.m4.
 2967   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2968   enc_class aarch64_enc_strs(vRegF src, memory4 mem) %{
 2969     FloatRegister src_reg = as_FloatRegister($src$$reg);
 2970     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strs, src_reg, $mem->opcode(),
 2971                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2972   %}
 2973 
 2974   // This encoding class is generated automatically from ad_encode.m4.
 2975   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2976   enc_class aarch64_enc_strd(vRegD src, memory8 mem) %{
 2977     FloatRegister src_reg = as_FloatRegister($src$$reg);
 2978     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strd, src_reg, $mem->opcode(),
 2979                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 2980   %}
 2981 
 2982   // This encoding class is generated automatically from ad_encode.m4.
 2983   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2984   enc_class aarch64_enc_strb0_ordered(memory4 mem) %{
 2985       C2_MacroAssembler _masm(&cbuf);
 2986       __ membar(Assembler::StoreStore);
 2987       loadStore(_masm, &MacroAssembler::strb, zr, $mem->opcode(),
 2988                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2989   %}
 2990 
 2991   // END Non-volatile memory access
 2992 
 2993   // Vector loads and stores
 2994   enc_class aarch64_enc_ldrvH(vReg dst, memory mem) %{
 2995     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 2996     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::H,
 2997        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 2998   %}
 2999 
 3000   enc_class aarch64_enc_ldrvS(vReg dst, memory mem) %{
 3001     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3002     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::S,
 3003        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3004   %}
 3005 
 3006   enc_class aarch64_enc_ldrvD(vReg dst, memory mem) %{
 3007     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3008     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::D,
 3009        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3010   %}
 3011 
 3012   enc_class aarch64_enc_ldrvQ(vReg dst, memory mem) %{
 3013     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3014     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::Q,
 3015        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3016   %}
 3017 
 3018   enc_class aarch64_enc_strvH(vReg src, memory mem) %{
 3019     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3020     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::H,
 3021        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3022   %}
 3023 
 3024   enc_class aarch64_enc_strvS(vReg src, memory mem) %{
 3025     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3026     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::S,
 3027        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3028   %}
 3029 
 3030   enc_class aarch64_enc_strvD(vReg src, memory mem) %{
 3031     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3032     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::D,
 3033        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3034   %}
 3035 
 3036   enc_class aarch64_enc_strvQ(vReg src, memory mem) %{
 3037     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3038     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::Q,
 3039        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3040   %}
 3041 
 3042   // volatile loads and stores
 3043 
 3044   enc_class aarch64_enc_stlrb(iRegI src, memory mem) %{
 3045     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3046                  rscratch1, stlrb);
 3047   %}
 3048 
 3049   enc_class aarch64_enc_stlrb0(memory mem) %{
 3050     MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3051                  rscratch1, stlrb);
 3052   %}
 3053 
 3054   enc_class aarch64_enc_stlrh(iRegI src, memory mem) %{
 3055     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3056                  rscratch1, stlrh);
 3057   %}
 3058 
 3059   enc_class aarch64_enc_stlrh0(memory mem) %{
 3060     MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3061                  rscratch1, stlrh);
 3062   %}
 3063 
 3064   enc_class aarch64_enc_stlrw(iRegI src, memory mem) %{
 3065     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3066                  rscratch1, stlrw);
 3067   %}
 3068 
 3069   enc_class aarch64_enc_stlrw0(memory mem) %{
 3070     MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3071                  rscratch1, stlrw);
 3072   %}
 3073 
 3074   enc_class aarch64_enc_ldarsbw(iRegI dst, memory mem) %{
 3075     Register dst_reg = as_Register($dst$$reg);
 3076     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3077              rscratch1, ldarb);
 3078     __ sxtbw(dst_reg, dst_reg);
 3079   %}
 3080 
 3081   enc_class aarch64_enc_ldarsb(iRegL dst, memory mem) %{
 3082     Register dst_reg = as_Register($dst$$reg);
 3083     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3084              rscratch1, ldarb);
 3085     __ sxtb(dst_reg, dst_reg);
 3086   %}
 3087 
 3088   enc_class aarch64_enc_ldarbw(iRegI dst, memory mem) %{
 3089     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3090              rscratch1, ldarb);
 3091   %}
 3092 
 3093   enc_class aarch64_enc_ldarb(iRegL dst, memory mem) %{
 3094     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3095              rscratch1, ldarb);
 3096   %}
 3097 
 3098   enc_class aarch64_enc_ldarshw(iRegI dst, memory mem) %{
 3099     Register dst_reg = as_Register($dst$$reg);
 3100     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3101              rscratch1, ldarh);
 3102     __ sxthw(dst_reg, dst_reg);
 3103   %}
 3104 
 3105   enc_class aarch64_enc_ldarsh(iRegL dst, memory mem) %{
 3106     Register dst_reg = as_Register($dst$$reg);
 3107     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3108              rscratch1, ldarh);
 3109     __ sxth(dst_reg, dst_reg);
 3110   %}
 3111 
 3112   enc_class aarch64_enc_ldarhw(iRegI dst, memory mem) %{
 3113     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3114              rscratch1, ldarh);
 3115   %}
 3116 
 3117   enc_class aarch64_enc_ldarh(iRegL dst, memory mem) %{
 3118     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3119              rscratch1, ldarh);
 3120   %}
 3121 
 3122   enc_class aarch64_enc_ldarw(iRegI dst, memory mem) %{
 3123     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3124              rscratch1, ldarw);
 3125   %}
 3126 
 3127   enc_class aarch64_enc_ldarw(iRegL dst, memory mem) %{
 3128     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3129              rscratch1, ldarw);
 3130   %}
 3131 
 3132   enc_class aarch64_enc_ldar(iRegL dst, memory mem) %{
 3133     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3134              rscratch1, ldar);
 3135   %}
 3136 
 3137   enc_class aarch64_enc_fldars(vRegF dst, memory mem) %{
 3138     MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3139              rscratch1, ldarw);
 3140     __ fmovs(as_FloatRegister($dst$$reg), rscratch1);
 3141   %}
 3142 
 3143   enc_class aarch64_enc_fldard(vRegD dst, memory mem) %{
 3144     MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3145              rscratch1, ldar);
 3146     __ fmovd(as_FloatRegister($dst$$reg), rscratch1);
 3147   %}
 3148 
 3149   enc_class aarch64_enc_stlr(iRegL src, memory mem) %{
 3150     Register src_reg = as_Register($src$$reg);
 3151     // we sometimes get asked to store the stack pointer into the
 3152     // current thread -- we cannot do that directly on AArch64
 3153     if (src_reg == r31_sp) {
 3154       C2_MacroAssembler _masm(&cbuf);
 3155       assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
 3156       __ mov(rscratch2, sp);
 3157       src_reg = rscratch2;
 3158     }
 3159     MOV_VOLATILE(src_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3160                  rscratch1, stlr);
 3161   %}
 3162 
 3163   enc_class aarch64_enc_stlr0(memory mem) %{
 3164     MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3165                  rscratch1, stlr);
 3166   %}
 3167 
 3168   enc_class aarch64_enc_fstlrs(vRegF src, memory mem) %{
 3169     {
 3170       C2_MacroAssembler _masm(&cbuf);
 3171       FloatRegister src_reg = as_FloatRegister($src$$reg);
 3172       __ fmovs(rscratch2, src_reg);
 3173     }
 3174     MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3175                  rscratch1, stlrw);
 3176   %}
 3177 
 3178   enc_class aarch64_enc_fstlrd(vRegD src, memory mem) %{
 3179     {
 3180       C2_MacroAssembler _masm(&cbuf);
 3181       FloatRegister src_reg = as_FloatRegister($src$$reg);
 3182       __ fmovd(rscratch2, src_reg);
 3183     }
 3184     MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3185                  rscratch1, stlr);
 3186   %}
 3187 
 3188   // synchronized read/update encodings
 3189 
 3190   enc_class aarch64_enc_ldaxr(iRegL dst, memory8 mem) %{
 3191     C2_MacroAssembler _masm(&cbuf);
 3192     Register dst_reg = as_Register($dst$$reg);
 3193     Register base = as_Register($mem$$base);
 3194     int index = $mem$$index;
 3195     int scale = $mem$$scale;
 3196     int disp = $mem$$disp;
 3197     if (index == -1) {
 3198        if (disp != 0) {
 3199         __ lea(rscratch1, Address(base, disp));
 3200         __ ldaxr(dst_reg, rscratch1);
 3201       } else {
 3202         // TODO
 3203         // should we ever get anything other than this case?
 3204         __ ldaxr(dst_reg, base);
 3205       }
 3206     } else {
 3207       Register index_reg = as_Register(index);
 3208       if (disp == 0) {
 3209         __ lea(rscratch1, Address(base, index_reg, Address::lsl(scale)));
 3210         __ ldaxr(dst_reg, rscratch1);
 3211       } else {
 3212         __ lea(rscratch1, Address(base, disp));
 3213         __ lea(rscratch1, Address(rscratch1, index_reg, Address::lsl(scale)));
 3214         __ ldaxr(dst_reg, rscratch1);
 3215       }
 3216     }
 3217   %}
 3218 
 3219   enc_class aarch64_enc_stlxr(iRegLNoSp src, memory8 mem) %{
 3220     C2_MacroAssembler _masm(&cbuf);
 3221     Register src_reg = as_Register($src$$reg);
 3222     Register base = as_Register($mem$$base);
 3223     int index = $mem$$index;
 3224     int scale = $mem$$scale;
 3225     int disp = $mem$$disp;
 3226     if (index == -1) {
 3227        if (disp != 0) {
 3228         __ lea(rscratch2, Address(base, disp));
 3229         __ stlxr(rscratch1, src_reg, rscratch2);
 3230       } else {
 3231         // TODO
 3232         // should we ever get anything other than this case?
 3233         __ stlxr(rscratch1, src_reg, base);
 3234       }
 3235     } else {
 3236       Register index_reg = as_Register(index);
 3237       if (disp == 0) {
 3238         __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
 3239         __ stlxr(rscratch1, src_reg, rscratch2);
 3240       } else {
 3241         __ lea(rscratch2, Address(base, disp));
 3242         __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
 3243         __ stlxr(rscratch1, src_reg, rscratch2);
 3244       }
 3245     }
 3246     __ cmpw(rscratch1, zr);
 3247   %}
 3248 
 3249   enc_class aarch64_enc_cmpxchg(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
 3250     C2_MacroAssembler _masm(&cbuf);
 3251     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3252     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3253                Assembler::xword, /*acquire*/ false, /*release*/ true,
 3254                /*weak*/ false, noreg);
 3255   %}
 3256 
 3257   enc_class aarch64_enc_cmpxchgw(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3258     C2_MacroAssembler _masm(&cbuf);
 3259     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3260     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3261                Assembler::word, /*acquire*/ false, /*release*/ true,
 3262                /*weak*/ false, noreg);
 3263   %}
 3264 
 3265   enc_class aarch64_enc_cmpxchgs(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3266     C2_MacroAssembler _masm(&cbuf);
 3267     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3268     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3269                Assembler::halfword, /*acquire*/ false, /*release*/ true,
 3270                /*weak*/ false, noreg);
 3271   %}
 3272 
 3273   enc_class aarch64_enc_cmpxchgb(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3274     C2_MacroAssembler _masm(&cbuf);
 3275     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3276     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3277                Assembler::byte, /*acquire*/ false, /*release*/ true,
 3278                /*weak*/ false, noreg);
 3279   %}
 3280 
 3281 
 3282   // The only difference between aarch64_enc_cmpxchg and
 3283   // aarch64_enc_cmpxchg_acq is that we use load-acquire in the
 3284   // CompareAndSwap sequence to serve as a barrier on acquiring a
 3285   // lock.
 3286   enc_class aarch64_enc_cmpxchg_acq(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
 3287     C2_MacroAssembler _masm(&cbuf);
 3288     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3289     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3290                Assembler::xword, /*acquire*/ true, /*release*/ true,
 3291                /*weak*/ false, noreg);
 3292   %}
 3293 
 3294   enc_class aarch64_enc_cmpxchgw_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3295     C2_MacroAssembler _masm(&cbuf);
 3296     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3297     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3298                Assembler::word, /*acquire*/ true, /*release*/ true,
 3299                /*weak*/ false, noreg);
 3300   %}
 3301 
 3302   enc_class aarch64_enc_cmpxchgs_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3303     C2_MacroAssembler _masm(&cbuf);
 3304     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3305     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3306                Assembler::halfword, /*acquire*/ true, /*release*/ true,
 3307                /*weak*/ false, noreg);
 3308   %}
 3309 
 3310   enc_class aarch64_enc_cmpxchgb_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3311     C2_MacroAssembler _masm(&cbuf);
 3312     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3313     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3314                Assembler::byte, /*acquire*/ true, /*release*/ true,
 3315                /*weak*/ false, noreg);
 3316   %}
 3317 
 3318   // auxiliary used for CompareAndSwapX to set result register
 3319   enc_class aarch64_enc_cset_eq(iRegINoSp res) %{
 3320     C2_MacroAssembler _masm(&cbuf);
 3321     Register res_reg = as_Register($res$$reg);
 3322     __ cset(res_reg, Assembler::EQ);
 3323   %}
 3324 
 3325   // prefetch encodings
 3326 
 3327   enc_class aarch64_enc_prefetchw(memory mem) %{
 3328     C2_MacroAssembler _masm(&cbuf);
 3329     Register base = as_Register($mem$$base);
 3330     int index = $mem$$index;
 3331     int scale = $mem$$scale;
 3332     int disp = $mem$$disp;
 3333     if (index == -1) {
 3334       __ prfm(Address(base, disp), PSTL1KEEP);
 3335     } else {
 3336       Register index_reg = as_Register(index);
 3337       if (disp == 0) {
 3338         __ prfm(Address(base, index_reg, Address::lsl(scale)), PSTL1KEEP);
 3339       } else {
 3340         __ lea(rscratch1, Address(base, disp));
 3341 	__ prfm(Address(rscratch1, index_reg, Address::lsl(scale)), PSTL1KEEP);
 3342       }
 3343     }
 3344   %}
 3345 
 3346   /// mov envcodings
 3347 
 3348   enc_class aarch64_enc_movw_imm(iRegI dst, immI src) %{
 3349     C2_MacroAssembler _masm(&cbuf);
 3350     uint32_t con = (uint32_t)$src$$constant;
 3351     Register dst_reg = as_Register($dst$$reg);
 3352     if (con == 0) {
 3353       __ movw(dst_reg, zr);
 3354     } else {
 3355       __ movw(dst_reg, con);
 3356     }
 3357   %}
 3358 
 3359   enc_class aarch64_enc_mov_imm(iRegL dst, immL src) %{
 3360     C2_MacroAssembler _masm(&cbuf);
 3361     Register dst_reg = as_Register($dst$$reg);
 3362     uint64_t con = (uint64_t)$src$$constant;
 3363     if (con == 0) {
 3364       __ mov(dst_reg, zr);
 3365     } else {
 3366       __ mov(dst_reg, con);
 3367     }
 3368   %}
 3369 
 3370   enc_class aarch64_enc_mov_p(iRegP dst, immP src) %{
 3371     C2_MacroAssembler _masm(&cbuf);
 3372     Register dst_reg = as_Register($dst$$reg);
 3373     address con = (address)$src$$constant;
 3374     if (con == NULL || con == (address)1) {
 3375       ShouldNotReachHere();
 3376     } else {
 3377       relocInfo::relocType rtype = $src->constant_reloc();
 3378       if (rtype == relocInfo::oop_type) {
 3379         __ movoop(dst_reg, (jobject)con);
 3380       } else if (rtype == relocInfo::metadata_type) {
 3381         __ mov_metadata(dst_reg, (Metadata*)con);
 3382       } else {
 3383         assert(rtype == relocInfo::none, "unexpected reloc type");
 3384         if (! __ is_valid_AArch64_address(con) ||
 3385             con < (address)(uintptr_t)os::vm_page_size()) {
 3386           __ mov(dst_reg, con);
 3387         } else {
 3388           uint64_t offset;
 3389           __ adrp(dst_reg, con, offset);
 3390           __ add(dst_reg, dst_reg, offset);
 3391         }
 3392       }
 3393     }
 3394   %}
 3395 
 3396   enc_class aarch64_enc_mov_p0(iRegP dst, immP0 src) %{
 3397     C2_MacroAssembler _masm(&cbuf);
 3398     Register dst_reg = as_Register($dst$$reg);
 3399     __ mov(dst_reg, zr);
 3400   %}
 3401 
 3402   enc_class aarch64_enc_mov_p1(iRegP dst, immP_1 src) %{
 3403     C2_MacroAssembler _masm(&cbuf);
 3404     Register dst_reg = as_Register($dst$$reg);
 3405     __ mov(dst_reg, (uint64_t)1);
 3406   %}
 3407 
 3408   enc_class aarch64_enc_mov_byte_map_base(iRegP dst, immByteMapBase src) %{
 3409     C2_MacroAssembler _masm(&cbuf);
 3410     __ load_byte_map_base($dst$$Register);
 3411   %}
 3412 
 3413   enc_class aarch64_enc_mov_n(iRegN dst, immN src) %{
 3414     C2_MacroAssembler _masm(&cbuf);
 3415     Register dst_reg = as_Register($dst$$reg);
 3416     address con = (address)$src$$constant;
 3417     if (con == NULL) {
 3418       ShouldNotReachHere();
 3419     } else {
 3420       relocInfo::relocType rtype = $src->constant_reloc();
 3421       assert(rtype == relocInfo::oop_type, "unexpected reloc type");
 3422       __ set_narrow_oop(dst_reg, (jobject)con);
 3423     }
 3424   %}
 3425 
 3426   enc_class aarch64_enc_mov_n0(iRegN dst, immN0 src) %{
 3427     C2_MacroAssembler _masm(&cbuf);
 3428     Register dst_reg = as_Register($dst$$reg);
 3429     __ mov(dst_reg, zr);
 3430   %}
 3431 
 3432   enc_class aarch64_enc_mov_nk(iRegN dst, immNKlass src) %{
 3433     C2_MacroAssembler _masm(&cbuf);
 3434     Register dst_reg = as_Register($dst$$reg);
 3435     address con = (address)$src$$constant;
 3436     if (con == NULL) {
 3437       ShouldNotReachHere();
 3438     } else {
 3439       relocInfo::relocType rtype = $src->constant_reloc();
 3440       assert(rtype == relocInfo::metadata_type, "unexpected reloc type");
 3441       __ set_narrow_klass(dst_reg, (Klass *)con);
 3442     }
 3443   %}
 3444 
 3445   // arithmetic encodings
 3446 
 3447   enc_class aarch64_enc_addsubw_imm(iRegI dst, iRegI src1, immIAddSub src2) %{
 3448     C2_MacroAssembler _masm(&cbuf);
 3449     Register dst_reg = as_Register($dst$$reg);
 3450     Register src_reg = as_Register($src1$$reg);
 3451     int32_t con = (int32_t)$src2$$constant;
 3452     // add has primary == 0, subtract has primary == 1
 3453     if ($primary) { con = -con; }
 3454     if (con < 0) {
 3455       __ subw(dst_reg, src_reg, -con);
 3456     } else {
 3457       __ addw(dst_reg, src_reg, con);
 3458     }
 3459   %}
 3460 
 3461   enc_class aarch64_enc_addsub_imm(iRegL dst, iRegL src1, immLAddSub src2) %{
 3462     C2_MacroAssembler _masm(&cbuf);
 3463     Register dst_reg = as_Register($dst$$reg);
 3464     Register src_reg = as_Register($src1$$reg);
 3465     int32_t con = (int32_t)$src2$$constant;
 3466     // add has primary == 0, subtract has primary == 1
 3467     if ($primary) { con = -con; }
 3468     if (con < 0) {
 3469       __ sub(dst_reg, src_reg, -con);
 3470     } else {
 3471       __ add(dst_reg, src_reg, con);
 3472     }
 3473   %}
 3474 
 3475   enc_class aarch64_enc_divw(iRegI dst, iRegI src1, iRegI src2) %{
 3476     C2_MacroAssembler _masm(&cbuf);
 3477    Register dst_reg = as_Register($dst$$reg);
 3478    Register src1_reg = as_Register($src1$$reg);
 3479    Register src2_reg = as_Register($src2$$reg);
 3480     __ corrected_idivl(dst_reg, src1_reg, src2_reg, false, rscratch1);
 3481   %}
 3482 
 3483   enc_class aarch64_enc_div(iRegI dst, iRegI src1, iRegI src2) %{
 3484     C2_MacroAssembler _masm(&cbuf);
 3485    Register dst_reg = as_Register($dst$$reg);
 3486    Register src1_reg = as_Register($src1$$reg);
 3487    Register src2_reg = as_Register($src2$$reg);
 3488     __ corrected_idivq(dst_reg, src1_reg, src2_reg, false, rscratch1);
 3489   %}
 3490 
 3491   enc_class aarch64_enc_modw(iRegI dst, iRegI src1, iRegI src2) %{
 3492     C2_MacroAssembler _masm(&cbuf);
 3493    Register dst_reg = as_Register($dst$$reg);
 3494    Register src1_reg = as_Register($src1$$reg);
 3495    Register src2_reg = as_Register($src2$$reg);
 3496     __ corrected_idivl(dst_reg, src1_reg, src2_reg, true, rscratch1);
 3497   %}
 3498 
 3499   enc_class aarch64_enc_mod(iRegI dst, iRegI src1, iRegI src2) %{
 3500     C2_MacroAssembler _masm(&cbuf);
 3501    Register dst_reg = as_Register($dst$$reg);
 3502    Register src1_reg = as_Register($src1$$reg);
 3503    Register src2_reg = as_Register($src2$$reg);
 3504     __ corrected_idivq(dst_reg, src1_reg, src2_reg, true, rscratch1);
 3505   %}
 3506 
 3507   // compare instruction encodings
 3508 
 3509   enc_class aarch64_enc_cmpw(iRegI src1, iRegI src2) %{
 3510     C2_MacroAssembler _masm(&cbuf);
 3511     Register reg1 = as_Register($src1$$reg);
 3512     Register reg2 = as_Register($src2$$reg);
 3513     __ cmpw(reg1, reg2);
 3514   %}
 3515 
 3516   enc_class aarch64_enc_cmpw_imm_addsub(iRegI src1, immIAddSub src2) %{
 3517     C2_MacroAssembler _masm(&cbuf);
 3518     Register reg = as_Register($src1$$reg);
 3519     int32_t val = $src2$$constant;
 3520     if (val >= 0) {
 3521       __ subsw(zr, reg, val);
 3522     } else {
 3523       __ addsw(zr, reg, -val);
 3524     }
 3525   %}
 3526 
 3527   enc_class aarch64_enc_cmpw_imm(iRegI src1, immI src2) %{
 3528     C2_MacroAssembler _masm(&cbuf);
 3529     Register reg1 = as_Register($src1$$reg);
 3530     uint32_t val = (uint32_t)$src2$$constant;
 3531     __ movw(rscratch1, val);
 3532     __ cmpw(reg1, rscratch1);
 3533   %}
 3534 
 3535   enc_class aarch64_enc_cmp(iRegL src1, iRegL src2) %{
 3536     C2_MacroAssembler _masm(&cbuf);
 3537     Register reg1 = as_Register($src1$$reg);
 3538     Register reg2 = as_Register($src2$$reg);
 3539     __ cmp(reg1, reg2);
 3540   %}
 3541 
 3542   enc_class aarch64_enc_cmp_imm_addsub(iRegL src1, immL12 src2) %{
 3543     C2_MacroAssembler _masm(&cbuf);
 3544     Register reg = as_Register($src1$$reg);
 3545     int64_t val = $src2$$constant;
 3546     if (val >= 0) {
 3547       __ subs(zr, reg, val);
 3548     } else if (val != -val) {
 3549       __ adds(zr, reg, -val);
 3550     } else {
 3551     // aargh, Long.MIN_VALUE is a special case
 3552       __ orr(rscratch1, zr, (uint64_t)val);
 3553       __ subs(zr, reg, rscratch1);
 3554     }
 3555   %}
 3556 
 3557   enc_class aarch64_enc_cmp_imm(iRegL src1, immL src2) %{
 3558     C2_MacroAssembler _masm(&cbuf);
 3559     Register reg1 = as_Register($src1$$reg);
 3560     uint64_t val = (uint64_t)$src2$$constant;
 3561     __ mov(rscratch1, val);
 3562     __ cmp(reg1, rscratch1);
 3563   %}
 3564 
 3565   enc_class aarch64_enc_cmpp(iRegP src1, iRegP src2) %{
 3566     C2_MacroAssembler _masm(&cbuf);
 3567     Register reg1 = as_Register($src1$$reg);
 3568     Register reg2 = as_Register($src2$$reg);
 3569     __ cmp(reg1, reg2);
 3570   %}
 3571 
 3572   enc_class aarch64_enc_cmpn(iRegN src1, iRegN src2) %{
 3573     C2_MacroAssembler _masm(&cbuf);
 3574     Register reg1 = as_Register($src1$$reg);
 3575     Register reg2 = as_Register($src2$$reg);
 3576     __ cmpw(reg1, reg2);
 3577   %}
 3578 
 3579   enc_class aarch64_enc_testp(iRegP src) %{
 3580     C2_MacroAssembler _masm(&cbuf);
 3581     Register reg = as_Register($src$$reg);
 3582     __ cmp(reg, zr);
 3583   %}
 3584 
 3585   enc_class aarch64_enc_testn(iRegN src) %{
 3586     C2_MacroAssembler _masm(&cbuf);
 3587     Register reg = as_Register($src$$reg);
 3588     __ cmpw(reg, zr);
 3589   %}
 3590 
 3591   enc_class aarch64_enc_b(label lbl) %{
 3592     C2_MacroAssembler _masm(&cbuf);
 3593     Label *L = $lbl$$label;
 3594     __ b(*L);
 3595   %}
 3596 
 3597   enc_class aarch64_enc_br_con(cmpOp cmp, label lbl) %{
 3598     C2_MacroAssembler _masm(&cbuf);
 3599     Label *L = $lbl$$label;
 3600     __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
 3601   %}
 3602 
 3603   enc_class aarch64_enc_br_conU(cmpOpU cmp, label lbl) %{
 3604     C2_MacroAssembler _masm(&cbuf);
 3605     Label *L = $lbl$$label;
 3606     __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
 3607   %}
 3608 
 3609   enc_class aarch64_enc_partial_subtype_check(iRegP sub, iRegP super, iRegP temp, iRegP result)
 3610   %{
 3611      Register sub_reg = as_Register($sub$$reg);
 3612      Register super_reg = as_Register($super$$reg);
 3613      Register temp_reg = as_Register($temp$$reg);
 3614      Register result_reg = as_Register($result$$reg);
 3615 
 3616      Label miss;
 3617      C2_MacroAssembler _masm(&cbuf);
 3618      __ check_klass_subtype_slow_path(sub_reg, super_reg, temp_reg, result_reg,
 3619                                      NULL, &miss,
 3620                                      /*set_cond_codes:*/ true);
 3621      if ($primary) {
 3622        __ mov(result_reg, zr);
 3623      }
 3624      __ bind(miss);
 3625   %}
 3626 
 3627   enc_class aarch64_enc_java_static_call(method meth) %{
 3628     C2_MacroAssembler _masm(&cbuf);
 3629 
 3630     address addr = (address)$meth$$method;
 3631     address call;
 3632     if (!_method) {
 3633       // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
 3634       call = __ trampoline_call(Address(addr, relocInfo::runtime_call_type));
 3635       if (call == NULL) {
 3636         ciEnv::current()->record_failure("CodeCache is full");
 3637         return;
 3638       }
 3639     } else if (_method->intrinsic_id() == vmIntrinsicID::_ensureMaterializedForStackWalk) {
 3640       // The NOP here is purely to ensure that eliding a call to
 3641       // JVM_EnsureMaterializedForStackWalk doesn't change the code size.
 3642       __ nop();
 3643       __ block_comment("call JVM_EnsureMaterializedForStackWalk (elided)");
 3644     } else {
 3645       int method_index = resolved_method_index(cbuf);
 3646       RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
 3647                                                   : static_call_Relocation::spec(method_index);
 3648       call = __ trampoline_call(Address(addr, rspec));
 3649       if (call == NULL) {
 3650         ciEnv::current()->record_failure("CodeCache is full");
 3651         return;
 3652       }
 3653       if (CodeBuffer::supports_shared_stubs() && _method->can_be_statically_bound()) {
 3654         // Calls of the same statically bound method can share
 3655         // a stub to the interpreter.
 3656         cbuf.shared_stub_to_interp_for(_method, call - cbuf.insts_begin());
 3657       } else {
 3658         // Emit stub for static call
 3659         address stub = CompiledStaticCall::emit_to_interp_stub(cbuf, call);
 3660         if (stub == NULL) {
 3661           ciEnv::current()->record_failure("CodeCache is full");
 3662           return;
 3663         }
 3664       }
 3665     }
 3666 
 3667     __ post_call_nop();
 3668 
 3669     // Only non uncommon_trap calls need to reinitialize ptrue.
 3670     if (Compile::current()->max_vector_size() > 0 && uncommon_trap_request() == 0) {
 3671       __ reinitialize_ptrue();
 3672     }
 3673   %}
 3674 
 3675   enc_class aarch64_enc_java_dynamic_call(method meth) %{
 3676     C2_MacroAssembler _masm(&cbuf);
 3677     int method_index = resolved_method_index(cbuf);
 3678     address call = __ ic_call((address)$meth$$method, method_index);
 3679     if (call == NULL) {
 3680       ciEnv::current()->record_failure("CodeCache is full");
 3681       return;
 3682     }
 3683     __ post_call_nop();
 3684     if (Compile::current()->max_vector_size() > 0) {
 3685       __ reinitialize_ptrue();
 3686     }
 3687   %}
 3688 
 3689   enc_class aarch64_enc_call_epilog() %{
 3690     C2_MacroAssembler _masm(&cbuf);
 3691     if (VerifyStackAtCalls) {
 3692       // Check that stack depth is unchanged: find majik cookie on stack
 3693       __ call_Unimplemented();
 3694     }
 3695     if (tf()->returns_inline_type_as_fields() && !_method->is_method_handle_intrinsic()) {
 3696       if (!_method->signature()->returns_null_free_inline_type()) {
 3697         // The last return value is not set by the callee but used to pass IsInit information to compiled code.
 3698         // Search for the corresponding projection, get the register and emit code that initialized it.
 3699         uint con = (tf()->range_cc()->cnt() - 1);
 3700         for (DUIterator_Fast imax, i = fast_outs(imax); i < imax; i++) {
 3701           ProjNode* proj = fast_out(i)->as_Proj();
 3702           if (proj->_con == con) {
 3703             // Set IsInit if r0 is non-null (a non-null value is returned buffered or scalarized)
 3704             OptoReg::Name optoReg = ra_->get_reg_first(proj);
 3705             VMReg reg = OptoReg::as_VMReg(optoReg, ra_->_framesize, OptoReg::reg2stack(ra_->_matcher._new_SP));
 3706             Register toReg = reg->is_reg() ? reg->as_Register() : rscratch1;
 3707             __ cmp(r0, zr);
 3708             __ cset(toReg, Assembler::NE);
 3709             if (reg->is_stack()) {
 3710               int st_off = reg->reg2stack() * VMRegImpl::stack_slot_size;
 3711               __ str(toReg, Address(sp, st_off));
 3712             }
 3713             break;
 3714           }
 3715         }
 3716       }
 3717       if (return_value_is_used()) {
 3718         // An inline type is returned as fields in multiple registers.
 3719         // R0 either contains an oop if the inline type is buffered or a pointer
 3720         // to the corresponding InlineKlass with the lowest bit set to 1. Zero r0
 3721         // if the lowest bit is set to allow C2 to use the oop after null checking.
 3722         // r0 &= (r0 & 1) - 1
 3723         __ andr(rscratch1, r0, 0x1);
 3724         __ sub(rscratch1, rscratch1, 0x1);
 3725         __ andr(r0, r0, rscratch1);
 3726       }
 3727     }
 3728   %}
 3729 
 3730   enc_class aarch64_enc_java_to_runtime(method meth) %{
 3731     C2_MacroAssembler _masm(&cbuf);
 3732 
 3733     // some calls to generated routines (arraycopy code) are scheduled
 3734     // by C2 as runtime calls. if so we can call them using a br (they
 3735     // will be in a reachable segment) otherwise we have to use a blr
 3736     // which loads the absolute address into a register.
 3737     address entry = (address)$meth$$method;
 3738     CodeBlob *cb = CodeCache::find_blob(entry);
 3739     if (cb) {
 3740       address call = __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
 3741       if (call == NULL) {
 3742         ciEnv::current()->record_failure("CodeCache is full");
 3743         return;
 3744       }
 3745       __ post_call_nop();
 3746     } else {
 3747       Label retaddr;
 3748       __ adr(rscratch2, retaddr);
 3749       __ lea(rscratch1, RuntimeAddress(entry));
 3750       // Leave a breadcrumb for JavaFrameAnchor::capture_last_Java_pc()
 3751       __ stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)));
 3752       __ blr(rscratch1);
 3753       __ bind(retaddr);
 3754       __ post_call_nop();
 3755       __ add(sp, sp, 2 * wordSize);
 3756     }
 3757     if (Compile::current()->max_vector_size() > 0) {
 3758       __ reinitialize_ptrue();
 3759     }
 3760   %}
 3761 
 3762   enc_class aarch64_enc_rethrow() %{
 3763     C2_MacroAssembler _masm(&cbuf);
 3764     __ far_jump(RuntimeAddress(OptoRuntime::rethrow_stub()));
 3765   %}
 3766 
 3767   enc_class aarch64_enc_ret() %{
 3768     C2_MacroAssembler _masm(&cbuf);
 3769 #ifdef ASSERT
 3770     if (Compile::current()->max_vector_size() > 0) {
 3771       __ verify_ptrue();
 3772     }
 3773 #endif
 3774     __ ret(lr);
 3775   %}
 3776 
 3777   enc_class aarch64_enc_tail_call(iRegP jump_target) %{
 3778     C2_MacroAssembler _masm(&cbuf);
 3779     Register target_reg = as_Register($jump_target$$reg);
 3780     __ br(target_reg);
 3781   %}
 3782 
 3783   enc_class aarch64_enc_tail_jmp(iRegP jump_target) %{
 3784     C2_MacroAssembler _masm(&cbuf);
 3785     Register target_reg = as_Register($jump_target$$reg);
 3786     // exception oop should be in r0
 3787     // ret addr has been popped into lr
 3788     // callee expects it in r3
 3789     __ mov(r3, lr);
 3790     __ br(target_reg);
 3791   %}
 3792 
 3793   enc_class aarch64_enc_fast_lock(iRegP object, iRegP box, iRegP tmp, iRegP tmp2) %{
 3794     C2_MacroAssembler _masm(&cbuf);
 3795     Register oop = as_Register($object$$reg);
 3796     Register box = as_Register($box$$reg);
 3797     Register disp_hdr = as_Register($tmp$$reg);
 3798     Register tmp = as_Register($tmp2$$reg);
 3799     Label cont;
 3800     Label object_has_monitor;
 3801     Label no_count;
 3802 
 3803     assert_different_registers(oop, box, tmp, disp_hdr);
 3804 
 3805     // Load markWord from object into displaced_header.
 3806     __ ldr(disp_hdr, Address(oop, oopDesc::mark_offset_in_bytes()));
 3807 
 3808     if (DiagnoseSyncOnValueBasedClasses != 0) {
 3809       __ load_klass(tmp, oop);
 3810       __ ldrw(tmp, Address(tmp, Klass::access_flags_offset()));
 3811       __ tstw(tmp, JVM_ACC_IS_VALUE_BASED_CLASS);
 3812       __ br(Assembler::NE, cont);
 3813     }
 3814 
 3815     // Check for existing monitor
 3816     __ tbnz(disp_hdr, exact_log2(markWord::monitor_value), object_has_monitor);
 3817 
 3818     if (!UseHeavyMonitors) {
 3819       // Set tmp to be (markWord of object | UNLOCK_VALUE).
 3820       __ orr(tmp, disp_hdr, markWord::unlocked_value);
 3821 
 3822       if (EnableValhalla) {
 3823         // Mask inline_type bit such that we go to the slow path if object is an inline type
 3824         __ andr(tmp, tmp, ~((int) markWord::inline_type_bit_in_place));
 3825       }
 3826 
 3827       // Initialize the box. (Must happen before we update the object mark!)
 3828       __ str(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));
 3829 
 3830       // Compare object markWord with an unlocked value (tmp) and if
 3831       // equal exchange the stack address of our box with object markWord.
 3832       // On failure disp_hdr contains the possibly locked markWord.
 3833       __ cmpxchg(oop, tmp, box, Assembler::xword, /*acquire*/ true,
 3834                  /*release*/ true, /*weak*/ false, disp_hdr);
 3835       __ br(Assembler::EQ, cont);
 3836 
 3837       assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
 3838 
 3839       // If the compare-and-exchange succeeded, then we found an unlocked
 3840       // object, will have now locked it will continue at label cont
 3841 
 3842       // Check if the owner is self by comparing the value in the
 3843       // markWord of object (disp_hdr) with the stack pointer.
 3844       __ mov(rscratch1, sp);
 3845       __ sub(disp_hdr, disp_hdr, rscratch1);
 3846       __ mov(tmp, (address) (~(os::vm_page_size()-1) | markWord::lock_mask_in_place));
 3847       // If condition is true we are cont and hence we can store 0 as the
 3848       // displaced header in the box, which indicates that it is a recursive lock.
 3849       __ ands(tmp/*==0?*/, disp_hdr, tmp);   // Sets flags for result
 3850       __ str(tmp/*==0, perhaps*/, Address(box, BasicLock::displaced_header_offset_in_bytes()));
 3851     } else {
 3852       __ tst(oop, oop); // Set NE to indicate 'failure' -> take slow-path. We know that oop != 0.
 3853     }
 3854     __ b(cont);
 3855 
 3856     // Handle existing monitor.
 3857     __ bind(object_has_monitor);
 3858 
 3859     // The object's monitor m is unlocked iff m->owner == NULL,
 3860     // otherwise m->owner may contain a thread or a stack address.
 3861     //
 3862     // Try to CAS m->owner from NULL to current thread.
 3863     __ add(tmp, disp_hdr, (ObjectMonitor::owner_offset_in_bytes()-markWord::monitor_value));
 3864     __ cmpxchg(tmp, zr, rthread, Assembler::xword, /*acquire*/ true,
 3865                /*release*/ true, /*weak*/ false, rscratch1); // Sets flags for result
 3866 
 3867     // Store a non-null value into the box to avoid looking like a re-entrant
 3868     // lock. The fast-path monitor unlock code checks for
 3869     // markWord::monitor_value so use markWord::unused_mark which has the
 3870     // relevant bit set, and also matches ObjectSynchronizer::enter.
 3871     __ mov(tmp, (address)markWord::unused_mark().value());
 3872     __ str(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));
 3873 
 3874     __ br(Assembler::EQ, cont); // CAS success means locking succeeded
 3875 
 3876     __ cmp(rscratch1, rthread);
 3877     __ br(Assembler::NE, cont); // Check for recursive locking
 3878 
 3879     // Recursive lock case
 3880     __ increment(Address(disp_hdr, ObjectMonitor::recursions_offset_in_bytes() - markWord::monitor_value), 1);
 3881     // flag == EQ still from the cmp above, checking if this is a reentrant lock
 3882 
 3883     __ bind(cont);
 3884     // flag == EQ indicates success
 3885     // flag == NE indicates failure
 3886     __ br(Assembler::NE, no_count);
 3887 
 3888     __ increment(Address(rthread, JavaThread::held_monitor_count_offset()));
 3889 
 3890     __ bind(no_count);
 3891   %}
 3892 
 3893   enc_class aarch64_enc_fast_unlock(iRegP object, iRegP box, iRegP tmp, iRegP tmp2) %{
 3894     C2_MacroAssembler _masm(&cbuf);
 3895     Register oop = as_Register($object$$reg);
 3896     Register box = as_Register($box$$reg);
 3897     Register disp_hdr = as_Register($tmp$$reg);
 3898     Register tmp = as_Register($tmp2$$reg);
 3899     Label cont;
 3900     Label object_has_monitor;
 3901     Label no_count;
 3902 
 3903     assert_different_registers(oop, box, tmp, disp_hdr);
 3904 
 3905     if (!UseHeavyMonitors) {
 3906       // Find the lock address and load the displaced header from the stack.
 3907       __ ldr(disp_hdr, Address(box, BasicLock::displaced_header_offset_in_bytes()));
 3908 
 3909       // If the displaced header is 0, we have a recursive unlock.
 3910       __ cmp(disp_hdr, zr);
 3911       __ br(Assembler::EQ, cont);
 3912     }
 3913 
 3914     // Handle existing monitor.
 3915     __ ldr(tmp, Address(oop, oopDesc::mark_offset_in_bytes()));
 3916     __ tbnz(tmp, exact_log2(markWord::monitor_value), object_has_monitor);
 3917 
 3918     if (!UseHeavyMonitors) {
 3919       // Check if it is still a light weight lock, this is is true if we
 3920       // see the stack address of the basicLock in the markWord of the
 3921       // object.
 3922 
 3923       __ cmpxchg(oop, box, disp_hdr, Assembler::xword, /*acquire*/ false,
 3924                  /*release*/ true, /*weak*/ false, tmp);
 3925     } else {
 3926       __ tst(oop, oop); // Set NE to indicate 'failure' -> take slow-path. We know that oop != 0.
 3927     }
 3928     __ b(cont);
 3929 
 3930     assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
 3931 
 3932     // Handle existing monitor.
 3933     __ bind(object_has_monitor);
 3934     STATIC_ASSERT(markWord::monitor_value <= INT_MAX);
 3935     __ add(tmp, tmp, -(int)markWord::monitor_value); // monitor
 3936     __ ldr(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset_in_bytes()));
 3937 
 3938     Label notRecursive;
 3939     __ cbz(disp_hdr, notRecursive);
 3940 
 3941     // Recursive lock
 3942     __ sub(disp_hdr, disp_hdr, 1u);
 3943     __ str(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset_in_bytes()));
 3944     __ cmp(disp_hdr, disp_hdr); // Sets flags for result
 3945     __ b(cont);
 3946 
 3947     __ bind(notRecursive);
 3948     __ ldr(rscratch1, Address(tmp, ObjectMonitor::EntryList_offset_in_bytes()));
 3949     __ ldr(disp_hdr, Address(tmp, ObjectMonitor::cxq_offset_in_bytes()));
 3950     __ orr(rscratch1, rscratch1, disp_hdr); // Will be 0 if both are 0.
 3951     __ cmp(rscratch1, zr); // Sets flags for result
 3952     __ cbnz(rscratch1, cont);
 3953     // need a release store here
 3954     __ lea(tmp, Address(tmp, ObjectMonitor::owner_offset_in_bytes()));
 3955     __ stlr(zr, tmp); // set unowned
 3956 
 3957     __ bind(cont);
 3958     // flag == EQ indicates success
 3959     // flag == NE indicates failure
 3960     __ br(Assembler::NE, no_count);
 3961 
 3962     __ decrement(Address(rthread, JavaThread::held_monitor_count_offset()));
 3963 
 3964     __ bind(no_count);
 3965   %}
 3966 
 3967 %}
 3968 
 3969 //----------FRAME--------------------------------------------------------------
 3970 // Definition of frame structure and management information.
 3971 //
 3972 //  S T A C K   L A Y O U T    Allocators stack-slot number
 3973 //                             |   (to get allocators register number
 3974 //  G  Owned by    |        |  v    add OptoReg::stack0())
 3975 //  r   CALLER     |        |
 3976 //  o     |        +--------+      pad to even-align allocators stack-slot
 3977 //  w     V        |  pad0  |        numbers; owned by CALLER
 3978 //  t   -----------+--------+----> Matcher::_in_arg_limit, unaligned
 3979 //  h     ^        |   in   |  5
 3980 //        |        |  args  |  4   Holes in incoming args owned by SELF
 3981 //  |     |        |        |  3
 3982 //  |     |        +--------+
 3983 //  V     |        | old out|      Empty on Intel, window on Sparc
 3984 //        |    old |preserve|      Must be even aligned.
 3985 //        |     SP-+--------+----> Matcher::_old_SP, even aligned
 3986 //        |        |   in   |  3   area for Intel ret address
 3987 //     Owned by    |preserve|      Empty on Sparc.
 3988 //       SELF      +--------+
 3989 //        |        |  pad2  |  2   pad to align old SP
 3990 //        |        +--------+  1
 3991 //        |        | locks  |  0
 3992 //        |        +--------+----> OptoReg::stack0(), even aligned
 3993 //        |        |  pad1  | 11   pad to align new SP
 3994 //        |        +--------+
 3995 //        |        |        | 10
 3996 //        |        | spills |  9   spills
 3997 //        V        |        |  8   (pad0 slot for callee)
 3998 //      -----------+--------+----> Matcher::_out_arg_limit, unaligned
 3999 //        ^        |  out   |  7
 4000 //        |        |  args  |  6   Holes in outgoing args owned by CALLEE
 4001 //     Owned by    +--------+
 4002 //      CALLEE     | new out|  6   Empty on Intel, window on Sparc
 4003 //        |    new |preserve|      Must be even-aligned.
 4004 //        |     SP-+--------+----> Matcher::_new_SP, even aligned
 4005 //        |        |        |
 4006 //
 4007 // Note 1: Only region 8-11 is determined by the allocator.  Region 0-5 is
 4008 //         known from SELF's arguments and the Java calling convention.
 4009 //         Region 6-7 is determined per call site.
 4010 // Note 2: If the calling convention leaves holes in the incoming argument
 4011 //         area, those holes are owned by SELF.  Holes in the outgoing area
 4012 //         are owned by the CALLEE.  Holes should not be necessary in the
 4013 //         incoming area, as the Java calling convention is completely under
 4014 //         the control of the AD file.  Doubles can be sorted and packed to
 4015 //         avoid holes.  Holes in the outgoing arguments may be necessary for
 4016 //         varargs C calling conventions.
 4017 // Note 3: Region 0-3 is even aligned, with pad2 as needed.  Region 3-5 is
 4018 //         even aligned with pad0 as needed.
 4019 //         Region 6 is even aligned.  Region 6-7 is NOT even aligned;
 4020 //           (the latter is true on Intel but is it false on AArch64?)
 4021 //         region 6-11 is even aligned; it may be padded out more so that
 4022 //         the region from SP to FP meets the minimum stack alignment.
 4023 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
 4024 //         alignment.  Region 11, pad1, may be dynamically extended so that
 4025 //         SP meets the minimum alignment.
 4026 
 4027 frame %{
 4028   // These three registers define part of the calling convention
 4029   // between compiled code and the interpreter.
 4030 
 4031   // Inline Cache Register or Method for I2C.
 4032   inline_cache_reg(R12);
 4033 
 4034   // Number of stack slots consumed by locking an object
 4035   sync_stack_slots(2);
 4036 
 4037   // Compiled code's Frame Pointer
 4038   frame_pointer(R31);
 4039 
 4040   // Interpreter stores its frame pointer in a register which is
 4041   // stored to the stack by I2CAdaptors.
 4042   // I2CAdaptors convert from interpreted java to compiled java.
 4043   interpreter_frame_pointer(R29);
 4044 
 4045   // Stack alignment requirement
 4046   stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
 4047 
 4048   // Number of outgoing stack slots killed above the out_preserve_stack_slots
 4049   // for calls to C.  Supports the var-args backing area for register parms.
 4050   varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
 4051 
 4052   // The after-PROLOG location of the return address.  Location of
 4053   // return address specifies a type (REG or STACK) and a number
 4054   // representing the register number (i.e. - use a register name) or
 4055   // stack slot.
 4056   // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
 4057   // Otherwise, it is above the locks and verification slot and alignment word
 4058   // TODO this may well be correct but need to check why that - 2 is there
 4059   // ppc port uses 0 but we definitely need to allow for fixed_slots
 4060   // which folds in the space used for monitors
 4061   return_addr(STACK - 2 +
 4062               align_up((Compile::current()->in_preserve_stack_slots() +
 4063                         Compile::current()->fixed_slots()),
 4064                        stack_alignment_in_slots()));
 4065 
 4066   // Location of compiled Java return values.  Same as C for now.
 4067   return_value
 4068   %{
 4069     // TODO do we allow ideal_reg == Op_RegN???
 4070     assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
 4071            "only return normal values");
 4072 
 4073     static const int lo[Op_RegL + 1] = { // enum name
 4074       0,                                 // Op_Node
 4075       0,                                 // Op_Set
 4076       R0_num,                            // Op_RegN
 4077       R0_num,                            // Op_RegI
 4078       R0_num,                            // Op_RegP
 4079       V0_num,                            // Op_RegF
 4080       V0_num,                            // Op_RegD
 4081       R0_num                             // Op_RegL
 4082     };
 4083 
 4084     static const int hi[Op_RegL + 1] = { // enum name
 4085       0,                                 // Op_Node
 4086       0,                                 // Op_Set
 4087       OptoReg::Bad,                      // Op_RegN
 4088       OptoReg::Bad,                      // Op_RegI
 4089       R0_H_num,                          // Op_RegP
 4090       OptoReg::Bad,                      // Op_RegF
 4091       V0_H_num,                          // Op_RegD
 4092       R0_H_num                           // Op_RegL
 4093     };
 4094 
 4095     return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
 4096   %}
 4097 %}
 4098 
 4099 //----------ATTRIBUTES---------------------------------------------------------
 4100 //----------Operand Attributes-------------------------------------------------
 4101 op_attrib op_cost(1);        // Required cost attribute
 4102 
 4103 //----------Instruction Attributes---------------------------------------------
 4104 ins_attrib ins_cost(INSN_COST); // Required cost attribute
 4105 ins_attrib ins_size(32);        // Required size attribute (in bits)
 4106 ins_attrib ins_short_branch(0); // Required flag: is this instruction
 4107                                 // a non-matching short branch variant
 4108                                 // of some long branch?
 4109 ins_attrib ins_alignment(4);    // Required alignment attribute (must
 4110                                 // be a power of 2) specifies the
 4111                                 // alignment that some part of the
 4112                                 // instruction (not necessarily the
 4113                                 // start) requires.  If > 1, a
 4114                                 // compute_padding() function must be
 4115                                 // provided for the instruction
 4116 
 4117 //----------OPERANDS-----------------------------------------------------------
 4118 // Operand definitions must precede instruction definitions for correct parsing
 4119 // in the ADLC because operands constitute user defined types which are used in
 4120 // instruction definitions.
 4121 
 4122 //----------Simple Operands----------------------------------------------------
 4123 
 4124 // Integer operands 32 bit
 4125 // 32 bit immediate
 4126 operand immI()
 4127 %{
 4128   match(ConI);
 4129 
 4130   op_cost(0);
 4131   format %{ %}
 4132   interface(CONST_INTER);
 4133 %}
 4134 
 4135 // 32 bit zero
 4136 operand immI0()
 4137 %{
 4138   predicate(n->get_int() == 0);
 4139   match(ConI);
 4140 
 4141   op_cost(0);
 4142   format %{ %}
 4143   interface(CONST_INTER);
 4144 %}
 4145 
 4146 // 32 bit unit increment
 4147 operand immI_1()
 4148 %{
 4149   predicate(n->get_int() == 1);
 4150   match(ConI);
 4151 
 4152   op_cost(0);
 4153   format %{ %}
 4154   interface(CONST_INTER);
 4155 %}
 4156 
 4157 // 32 bit unit decrement
 4158 operand immI_M1()
 4159 %{
 4160   predicate(n->get_int() == -1);
 4161   match(ConI);
 4162 
 4163   op_cost(0);
 4164   format %{ %}
 4165   interface(CONST_INTER);
 4166 %}
 4167 
 4168 // Shift values for add/sub extension shift
 4169 operand immIExt()
 4170 %{
 4171   predicate(0 <= n->get_int() && (n->get_int() <= 4));
 4172   match(ConI);
 4173 
 4174   op_cost(0);
 4175   format %{ %}
 4176   interface(CONST_INTER);
 4177 %}
 4178 
 4179 operand immI_gt_1()
 4180 %{
 4181   predicate(n->get_int() > 1);
 4182   match(ConI);
 4183 
 4184   op_cost(0);
 4185   format %{ %}
 4186   interface(CONST_INTER);
 4187 %}
 4188 
 4189 operand immI_le_4()
 4190 %{
 4191   predicate(n->get_int() <= 4);
 4192   match(ConI);
 4193 
 4194   op_cost(0);
 4195   format %{ %}
 4196   interface(CONST_INTER);
 4197 %}
 4198 
 4199 operand immI_16()
 4200 %{
 4201   predicate(n->get_int() == 16);
 4202   match(ConI);
 4203 
 4204   op_cost(0);
 4205   format %{ %}
 4206   interface(CONST_INTER);
 4207 %}
 4208 
 4209 operand immI_24()
 4210 %{
 4211   predicate(n->get_int() == 24);
 4212   match(ConI);
 4213 
 4214   op_cost(0);
 4215   format %{ %}
 4216   interface(CONST_INTER);
 4217 %}
 4218 
 4219 operand immI_32()
 4220 %{
 4221   predicate(n->get_int() == 32);
 4222   match(ConI);
 4223 
 4224   op_cost(0);
 4225   format %{ %}
 4226   interface(CONST_INTER);
 4227 %}
 4228 
 4229 operand immI_48()
 4230 %{
 4231   predicate(n->get_int() == 48);
 4232   match(ConI);
 4233 
 4234   op_cost(0);
 4235   format %{ %}
 4236   interface(CONST_INTER);
 4237 %}
 4238 
 4239 operand immI_56()
 4240 %{
 4241   predicate(n->get_int() == 56);
 4242   match(ConI);
 4243 
 4244   op_cost(0);
 4245   format %{ %}
 4246   interface(CONST_INTER);
 4247 %}
 4248 
 4249 operand immI_63()
 4250 %{
 4251   predicate(n->get_int() == 63);
 4252   match(ConI);
 4253 
 4254   op_cost(0);
 4255   format %{ %}
 4256   interface(CONST_INTER);
 4257 %}
 4258 
 4259 operand immI_64()
 4260 %{
 4261   predicate(n->get_int() == 64);
 4262   match(ConI);
 4263 
 4264   op_cost(0);
 4265   format %{ %}
 4266   interface(CONST_INTER);
 4267 %}
 4268 
 4269 operand immI_255()
 4270 %{
 4271   predicate(n->get_int() == 255);
 4272   match(ConI);
 4273 
 4274   op_cost(0);
 4275   format %{ %}
 4276   interface(CONST_INTER);
 4277 %}
 4278 
 4279 operand immI_65535()
 4280 %{
 4281   predicate(n->get_int() == 65535);
 4282   match(ConI);
 4283 
 4284   op_cost(0);
 4285   format %{ %}
 4286   interface(CONST_INTER);
 4287 %}
 4288 
 4289 operand immI_positive()
 4290 %{
 4291   predicate(n->get_int() > 0);
 4292   match(ConI);
 4293 
 4294   op_cost(0);
 4295   format %{ %}
 4296   interface(CONST_INTER);
 4297 %}
 4298 
 4299 operand immL_255()
 4300 %{
 4301   predicate(n->get_long() == 255L);
 4302   match(ConL);
 4303 
 4304   op_cost(0);
 4305   format %{ %}
 4306   interface(CONST_INTER);
 4307 %}
 4308 
 4309 operand immL_65535()
 4310 %{
 4311   predicate(n->get_long() == 65535L);
 4312   match(ConL);
 4313 
 4314   op_cost(0);
 4315   format %{ %}
 4316   interface(CONST_INTER);
 4317 %}
 4318 
 4319 operand immL_4294967295()
 4320 %{
 4321   predicate(n->get_long() == 4294967295L);
 4322   match(ConL);
 4323 
 4324   op_cost(0);
 4325   format %{ %}
 4326   interface(CONST_INTER);
 4327 %}
 4328 
 4329 operand immL_bitmask()
 4330 %{
 4331   predicate((n->get_long() != 0)
 4332             && ((n->get_long() & 0xc000000000000000l) == 0)
 4333             && is_power_of_2(n->get_long() + 1));
 4334   match(ConL);
 4335 
 4336   op_cost(0);
 4337   format %{ %}
 4338   interface(CONST_INTER);
 4339 %}
 4340 
 4341 operand immI_bitmask()
 4342 %{
 4343   predicate((n->get_int() != 0)
 4344             && ((n->get_int() & 0xc0000000) == 0)
 4345             && is_power_of_2(n->get_int() + 1));
 4346   match(ConI);
 4347 
 4348   op_cost(0);
 4349   format %{ %}
 4350   interface(CONST_INTER);
 4351 %}
 4352 
 4353 operand immL_positive_bitmaskI()
 4354 %{
 4355   predicate((n->get_long() != 0)
 4356             && ((julong)n->get_long() < 0x80000000ULL)
 4357             && is_power_of_2(n->get_long() + 1));
 4358   match(ConL);
 4359 
 4360   op_cost(0);
 4361   format %{ %}
 4362   interface(CONST_INTER);
 4363 %}
 4364 
 4365 // Scale values for scaled offset addressing modes (up to long but not quad)
 4366 operand immIScale()
 4367 %{
 4368   predicate(0 <= n->get_int() && (n->get_int() <= 3));
 4369   match(ConI);
 4370 
 4371   op_cost(0);
 4372   format %{ %}
 4373   interface(CONST_INTER);
 4374 %}
 4375 
 4376 // 26 bit signed offset -- for pc-relative branches
 4377 operand immI26()
 4378 %{
 4379   predicate(((-(1 << 25)) <= n->get_int()) && (n->get_int() < (1 << 25)));
 4380   match(ConI);
 4381 
 4382   op_cost(0);
 4383   format %{ %}
 4384   interface(CONST_INTER);
 4385 %}
 4386 
 4387 // 19 bit signed offset -- for pc-relative loads
 4388 operand immI19()
 4389 %{
 4390   predicate(((-(1 << 18)) <= n->get_int()) && (n->get_int() < (1 << 18)));
 4391   match(ConI);
 4392 
 4393   op_cost(0);
 4394   format %{ %}
 4395   interface(CONST_INTER);
 4396 %}
 4397 
 4398 // 12 bit unsigned offset -- for base plus immediate loads
 4399 operand immIU12()
 4400 %{
 4401   predicate((0 <= n->get_int()) && (n->get_int() < (1 << 12)));
 4402   match(ConI);
 4403 
 4404   op_cost(0);
 4405   format %{ %}
 4406   interface(CONST_INTER);
 4407 %}
 4408 
 4409 operand immLU12()
 4410 %{
 4411   predicate((0 <= n->get_long()) && (n->get_long() < (1 << 12)));
 4412   match(ConL);
 4413 
 4414   op_cost(0);
 4415   format %{ %}
 4416   interface(CONST_INTER);
 4417 %}
 4418 
 4419 // Offset for scaled or unscaled immediate loads and stores
 4420 operand immIOffset()
 4421 %{
 4422   predicate(Address::offset_ok_for_immed(n->get_int(), 0));
 4423   match(ConI);
 4424 
 4425   op_cost(0);
 4426   format %{ %}
 4427   interface(CONST_INTER);
 4428 %}
 4429 
 4430 operand immIOffset1()
 4431 %{
 4432   predicate(Address::offset_ok_for_immed(n->get_int(), 0));
 4433   match(ConI);
 4434 
 4435   op_cost(0);
 4436   format %{ %}
 4437   interface(CONST_INTER);
 4438 %}
 4439 
 4440 operand immIOffset2()
 4441 %{
 4442   predicate(Address::offset_ok_for_immed(n->get_int(), 1));
 4443   match(ConI);
 4444 
 4445   op_cost(0);
 4446   format %{ %}
 4447   interface(CONST_INTER);
 4448 %}
 4449 
 4450 operand immIOffset4()
 4451 %{
 4452   predicate(Address::offset_ok_for_immed(n->get_int(), 2));
 4453   match(ConI);
 4454 
 4455   op_cost(0);
 4456   format %{ %}
 4457   interface(CONST_INTER);
 4458 %}
 4459 
 4460 operand immIOffset8()
 4461 %{
 4462   predicate(Address::offset_ok_for_immed(n->get_int(), 3));
 4463   match(ConI);
 4464 
 4465   op_cost(0);
 4466   format %{ %}
 4467   interface(CONST_INTER);
 4468 %}
 4469 
 4470 operand immIOffset16()
 4471 %{
 4472   predicate(Address::offset_ok_for_immed(n->get_int(), 4));
 4473   match(ConI);
 4474 
 4475   op_cost(0);
 4476   format %{ %}
 4477   interface(CONST_INTER);
 4478 %}
 4479 
 4480 operand immLoffset()
 4481 %{
 4482   predicate(Address::offset_ok_for_immed(n->get_long(), 0));
 4483   match(ConL);
 4484 
 4485   op_cost(0);
 4486   format %{ %}
 4487   interface(CONST_INTER);
 4488 %}
 4489 
 4490 operand immLoffset1()
 4491 %{
 4492   predicate(Address::offset_ok_for_immed(n->get_long(), 0));
 4493   match(ConL);
 4494 
 4495   op_cost(0);
 4496   format %{ %}
 4497   interface(CONST_INTER);
 4498 %}
 4499 
 4500 operand immLoffset2()
 4501 %{
 4502   predicate(Address::offset_ok_for_immed(n->get_long(), 1));
 4503   match(ConL);
 4504 
 4505   op_cost(0);
 4506   format %{ %}
 4507   interface(CONST_INTER);
 4508 %}
 4509 
 4510 operand immLoffset4()
 4511 %{
 4512   predicate(Address::offset_ok_for_immed(n->get_long(), 2));
 4513   match(ConL);
 4514 
 4515   op_cost(0);
 4516   format %{ %}
 4517   interface(CONST_INTER);
 4518 %}
 4519 
 4520 operand immLoffset8()
 4521 %{
 4522   predicate(Address::offset_ok_for_immed(n->get_long(), 3));
 4523   match(ConL);
 4524 
 4525   op_cost(0);
 4526   format %{ %}
 4527   interface(CONST_INTER);
 4528 %}
 4529 
 4530 operand immLoffset16()
 4531 %{
 4532   predicate(Address::offset_ok_for_immed(n->get_long(), 4));
 4533   match(ConL);
 4534 
 4535   op_cost(0);
 4536   format %{ %}
 4537   interface(CONST_INTER);
 4538 %}
 4539 
 4540 // 8 bit signed value.
 4541 operand immI8()
 4542 %{
 4543   predicate(n->get_int() <= 127 && n->get_int() >= -128);
 4544   match(ConI);
 4545 
 4546   op_cost(0);
 4547   format %{ %}
 4548   interface(CONST_INTER);
 4549 %}
 4550 
 4551 // 8 bit signed value (simm8), or #simm8 LSL 8.
 4552 operand immI8_shift8()
 4553 %{
 4554   predicate((n->get_int() <= 127 && n->get_int() >= -128) ||
 4555             (n->get_int() <= 32512 && n->get_int() >= -32768 && (n->get_int() & 0xff) == 0));
 4556   match(ConI);
 4557 
 4558   op_cost(0);
 4559   format %{ %}
 4560   interface(CONST_INTER);
 4561 %}
 4562 
 4563 // 8 bit signed value (simm8), or #simm8 LSL 8.
 4564 operand immL8_shift8()
 4565 %{
 4566   predicate((n->get_long() <= 127 && n->get_long() >= -128) ||
 4567             (n->get_long() <= 32512 && n->get_long() >= -32768 && (n->get_long() & 0xff) == 0));
 4568   match(ConL);
 4569 
 4570   op_cost(0);
 4571   format %{ %}
 4572   interface(CONST_INTER);
 4573 %}
 4574 
 4575 // 8 bit integer valid for vector add sub immediate
 4576 operand immBAddSubV()
 4577 %{
 4578   predicate(n->get_int() <= 255 && n->get_int() >= -255);
 4579   match(ConI);
 4580 
 4581   op_cost(0);
 4582   format %{ %}
 4583   interface(CONST_INTER);
 4584 %}
 4585 
 4586 // 32 bit integer valid for add sub immediate
 4587 operand immIAddSub()
 4588 %{
 4589   predicate(Assembler::operand_valid_for_add_sub_immediate((int64_t)n->get_int()));
 4590   match(ConI);
 4591   op_cost(0);
 4592   format %{ %}
 4593   interface(CONST_INTER);
 4594 %}
 4595 
 4596 // 32 bit integer valid for vector add sub immediate
 4597 operand immIAddSubV()
 4598 %{
 4599   predicate(Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)n->get_int()));
 4600   match(ConI);
 4601 
 4602   op_cost(0);
 4603   format %{ %}
 4604   interface(CONST_INTER);
 4605 %}
 4606 
 4607 // 32 bit unsigned integer valid for logical immediate
 4608 
 4609 operand immBLog()
 4610 %{
 4611   predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerByte, (uint64_t)n->get_int()));
 4612   match(ConI);
 4613 
 4614   op_cost(0);
 4615   format %{ %}
 4616   interface(CONST_INTER);
 4617 %}
 4618 
 4619 operand immSLog()
 4620 %{
 4621   predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerShort, (uint64_t)n->get_int()));
 4622   match(ConI);
 4623 
 4624   op_cost(0);
 4625   format %{ %}
 4626   interface(CONST_INTER);
 4627 %}
 4628 
 4629 operand immILog()
 4630 %{
 4631   predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/true, (uint64_t)n->get_int()));
 4632   match(ConI);
 4633 
 4634   op_cost(0);
 4635   format %{ %}
 4636   interface(CONST_INTER);
 4637 %}
 4638 
 4639 // Integer operands 64 bit
 4640 // 64 bit immediate
 4641 operand immL()
 4642 %{
 4643   match(ConL);
 4644 
 4645   op_cost(0);
 4646   format %{ %}
 4647   interface(CONST_INTER);
 4648 %}
 4649 
 4650 // 64 bit zero
 4651 operand immL0()
 4652 %{
 4653   predicate(n->get_long() == 0);
 4654   match(ConL);
 4655 
 4656   op_cost(0);
 4657   format %{ %}
 4658   interface(CONST_INTER);
 4659 %}
 4660 
 4661 // 64 bit unit increment
 4662 operand immL_1()
 4663 %{
 4664   predicate(n->get_long() == 1);
 4665   match(ConL);
 4666 
 4667   op_cost(0);
 4668   format %{ %}
 4669   interface(CONST_INTER);
 4670 %}
 4671 
 4672 // 64 bit unit decrement
 4673 operand immL_M1()
 4674 %{
 4675   predicate(n->get_long() == -1);
 4676   match(ConL);
 4677 
 4678   op_cost(0);
 4679   format %{ %}
 4680   interface(CONST_INTER);
 4681 %}
 4682 
 4683 // 32 bit offset of pc in thread anchor
 4684 
 4685 operand immL_pc_off()
 4686 %{
 4687   predicate(n->get_long() == in_bytes(JavaThread::frame_anchor_offset()) +
 4688                              in_bytes(JavaFrameAnchor::last_Java_pc_offset()));
 4689   match(ConL);
 4690 
 4691   op_cost(0);
 4692   format %{ %}
 4693   interface(CONST_INTER);
 4694 %}
 4695 
 4696 // 64 bit integer valid for add sub immediate
 4697 operand immLAddSub()
 4698 %{
 4699   predicate(Assembler::operand_valid_for_add_sub_immediate(n->get_long()));
 4700   match(ConL);
 4701   op_cost(0);
 4702   format %{ %}
 4703   interface(CONST_INTER);
 4704 %}
 4705 
 4706 // 64 bit integer valid for addv subv immediate
 4707 operand immLAddSubV()
 4708 %{
 4709   predicate(Assembler::operand_valid_for_sve_add_sub_immediate(n->get_long()));
 4710   match(ConL);
 4711 
 4712   op_cost(0);
 4713   format %{ %}
 4714   interface(CONST_INTER);
 4715 %}
 4716 
 4717 // 64 bit integer valid for logical immediate
 4718 operand immLLog()
 4719 %{
 4720   predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/false, (uint64_t)n->get_long()));
 4721   match(ConL);
 4722   op_cost(0);
 4723   format %{ %}
 4724   interface(CONST_INTER);
 4725 %}
 4726 
 4727 // Long Immediate: low 32-bit mask
 4728 operand immL_32bits()
 4729 %{
 4730   predicate(n->get_long() == 0xFFFFFFFFL);
 4731   match(ConL);
 4732   op_cost(0);
 4733   format %{ %}
 4734   interface(CONST_INTER);
 4735 %}
 4736 
 4737 // Pointer operands
 4738 // Pointer Immediate
 4739 operand immP()
 4740 %{
 4741   match(ConP);
 4742 
 4743   op_cost(0);
 4744   format %{ %}
 4745   interface(CONST_INTER);
 4746 %}
 4747 
 4748 // NULL Pointer Immediate
 4749 operand immP0()
 4750 %{
 4751   predicate(n->get_ptr() == 0);
 4752   match(ConP);
 4753 
 4754   op_cost(0);
 4755   format %{ %}
 4756   interface(CONST_INTER);
 4757 %}
 4758 
 4759 // Pointer Immediate One
 4760 // this is used in object initialization (initial object header)
 4761 operand immP_1()
 4762 %{
 4763   predicate(n->get_ptr() == 1);
 4764   match(ConP);
 4765 
 4766   op_cost(0);
 4767   format %{ %}
 4768   interface(CONST_INTER);
 4769 %}
 4770 
 4771 // Card Table Byte Map Base
 4772 operand immByteMapBase()
 4773 %{
 4774   // Get base of card map
 4775   predicate(BarrierSet::barrier_set()->is_a(BarrierSet::CardTableBarrierSet) &&
 4776             (CardTable::CardValue*)n->get_ptr() == ((CardTableBarrierSet*)(BarrierSet::barrier_set()))->card_table()->byte_map_base());
 4777   match(ConP);
 4778 
 4779   op_cost(0);
 4780   format %{ %}
 4781   interface(CONST_INTER);
 4782 %}
 4783 
 4784 // Pointer Immediate Minus One
 4785 // this is used when we want to write the current PC to the thread anchor
 4786 operand immP_M1()
 4787 %{
 4788   predicate(n->get_ptr() == -1);
 4789   match(ConP);
 4790 
 4791   op_cost(0);
 4792   format %{ %}
 4793   interface(CONST_INTER);
 4794 %}
 4795 
 4796 // Pointer Immediate Minus Two
 4797 // this is used when we want to write the current PC to the thread anchor
 4798 operand immP_M2()
 4799 %{
 4800   predicate(n->get_ptr() == -2);
 4801   match(ConP);
 4802 
 4803   op_cost(0);
 4804   format %{ %}
 4805   interface(CONST_INTER);
 4806 %}
 4807 
 4808 // Float and Double operands
 4809 // Double Immediate
 4810 operand immD()
 4811 %{
 4812   match(ConD);
 4813   op_cost(0);
 4814   format %{ %}
 4815   interface(CONST_INTER);
 4816 %}
 4817 
 4818 // Double Immediate: +0.0d
 4819 operand immD0()
 4820 %{
 4821   predicate(jlong_cast(n->getd()) == 0);
 4822   match(ConD);
 4823 
 4824   op_cost(0);
 4825   format %{ %}
 4826   interface(CONST_INTER);
 4827 %}
 4828 
 4829 // constant 'double +0.0'.
 4830 operand immDPacked()
 4831 %{
 4832   predicate(Assembler::operand_valid_for_float_immediate(n->getd()));
 4833   match(ConD);
 4834   op_cost(0);
 4835   format %{ %}
 4836   interface(CONST_INTER);
 4837 %}
 4838 
 4839 // Float Immediate
 4840 operand immF()
 4841 %{
 4842   match(ConF);
 4843   op_cost(0);
 4844   format %{ %}
 4845   interface(CONST_INTER);
 4846 %}
 4847 
 4848 // Float Immediate: +0.0f.
 4849 operand immF0()
 4850 %{
 4851   predicate(jint_cast(n->getf()) == 0);
 4852   match(ConF);
 4853 
 4854   op_cost(0);
 4855   format %{ %}
 4856   interface(CONST_INTER);
 4857 %}
 4858 
 4859 //
 4860 operand immFPacked()
 4861 %{
 4862   predicate(Assembler::operand_valid_for_float_immediate((double)n->getf()));
 4863   match(ConF);
 4864   op_cost(0);
 4865   format %{ %}
 4866   interface(CONST_INTER);
 4867 %}
 4868 
 4869 // Narrow pointer operands
 4870 // Narrow Pointer Immediate
 4871 operand immN()
 4872 %{
 4873   match(ConN);
 4874 
 4875   op_cost(0);
 4876   format %{ %}
 4877   interface(CONST_INTER);
 4878 %}
 4879 
 4880 // Narrow NULL Pointer Immediate
 4881 operand immN0()
 4882 %{
 4883   predicate(n->get_narrowcon() == 0);
 4884   match(ConN);
 4885 
 4886   op_cost(0);
 4887   format %{ %}
 4888   interface(CONST_INTER);
 4889 %}
 4890 
 4891 operand immNKlass()
 4892 %{
 4893   match(ConNKlass);
 4894 
 4895   op_cost(0);
 4896   format %{ %}
 4897   interface(CONST_INTER);
 4898 %}
 4899 
 4900 // Integer 32 bit Register Operands
 4901 // Integer 32 bitRegister (excludes SP)
 4902 operand iRegI()
 4903 %{
 4904   constraint(ALLOC_IN_RC(any_reg32));
 4905   match(RegI);
 4906   match(iRegINoSp);
 4907   op_cost(0);
 4908   format %{ %}
 4909   interface(REG_INTER);
 4910 %}
 4911 
 4912 // Integer 32 bit Register not Special
 4913 operand iRegINoSp()
 4914 %{
 4915   constraint(ALLOC_IN_RC(no_special_reg32));
 4916   match(RegI);
 4917   op_cost(0);
 4918   format %{ %}
 4919   interface(REG_INTER);
 4920 %}
 4921 
 4922 // Integer 64 bit Register Operands
 4923 // Integer 64 bit Register (includes SP)
 4924 operand iRegL()
 4925 %{
 4926   constraint(ALLOC_IN_RC(any_reg));
 4927   match(RegL);
 4928   match(iRegLNoSp);
 4929   op_cost(0);
 4930   format %{ %}
 4931   interface(REG_INTER);
 4932 %}
 4933 
 4934 // Integer 64 bit Register not Special
 4935 operand iRegLNoSp()
 4936 %{
 4937   constraint(ALLOC_IN_RC(no_special_reg));
 4938   match(RegL);
 4939   match(iRegL_R0);
 4940   format %{ %}
 4941   interface(REG_INTER);
 4942 %}
 4943 
 4944 // Pointer Register Operands
 4945 // Pointer Register
 4946 operand iRegP()
 4947 %{
 4948   constraint(ALLOC_IN_RC(ptr_reg));
 4949   match(RegP);
 4950   match(iRegPNoSp);
 4951   match(iRegP_R0);
 4952   //match(iRegP_R2);
 4953   //match(iRegP_R4);
 4954   match(iRegP_R5);
 4955   match(thread_RegP);
 4956   op_cost(0);
 4957   format %{ %}
 4958   interface(REG_INTER);
 4959 %}
 4960 
 4961 // Pointer 64 bit Register not Special
 4962 operand iRegPNoSp()
 4963 %{
 4964   constraint(ALLOC_IN_RC(no_special_ptr_reg));
 4965   match(RegP);
 4966   // match(iRegP);
 4967   // match(iRegP_R0);
 4968   // match(iRegP_R2);
 4969   // match(iRegP_R4);
 4970   // match(iRegP_R5);
 4971   // match(thread_RegP);
 4972   op_cost(0);
 4973   format %{ %}
 4974   interface(REG_INTER);
 4975 %}
 4976 
 4977 // Pointer 64 bit Register R0 only
 4978 operand iRegP_R0()
 4979 %{
 4980   constraint(ALLOC_IN_RC(r0_reg));
 4981   match(RegP);
 4982   // match(iRegP);
 4983   match(iRegPNoSp);
 4984   op_cost(0);
 4985   format %{ %}
 4986   interface(REG_INTER);
 4987 %}
 4988 
 4989 // Pointer 64 bit Register R1 only
 4990 operand iRegP_R1()
 4991 %{
 4992   constraint(ALLOC_IN_RC(r1_reg));
 4993   match(RegP);
 4994   // match(iRegP);
 4995   match(iRegPNoSp);
 4996   op_cost(0);
 4997   format %{ %}
 4998   interface(REG_INTER);
 4999 %}
 5000 
 5001 // Pointer 64 bit Register R2 only
 5002 operand iRegP_R2()
 5003 %{
 5004   constraint(ALLOC_IN_RC(r2_reg));
 5005   match(RegP);
 5006   // match(iRegP);
 5007   match(iRegPNoSp);
 5008   op_cost(0);
 5009   format %{ %}
 5010   interface(REG_INTER);
 5011 %}
 5012 
 5013 // Pointer 64 bit Register R3 only
 5014 operand iRegP_R3()
 5015 %{
 5016   constraint(ALLOC_IN_RC(r3_reg));
 5017   match(RegP);
 5018   // match(iRegP);
 5019   match(iRegPNoSp);
 5020   op_cost(0);
 5021   format %{ %}
 5022   interface(REG_INTER);
 5023 %}
 5024 
 5025 // Pointer 64 bit Register R4 only
 5026 operand iRegP_R4()
 5027 %{
 5028   constraint(ALLOC_IN_RC(r4_reg));
 5029   match(RegP);
 5030   // match(iRegP);
 5031   match(iRegPNoSp);
 5032   op_cost(0);
 5033   format %{ %}
 5034   interface(REG_INTER);
 5035 %}
 5036 
 5037 // Pointer 64 bit Register R5 only
 5038 operand iRegP_R5()
 5039 %{
 5040   constraint(ALLOC_IN_RC(r5_reg));
 5041   match(RegP);
 5042   // match(iRegP);
 5043   match(iRegPNoSp);
 5044   op_cost(0);
 5045   format %{ %}
 5046   interface(REG_INTER);
 5047 %}
 5048 
 5049 // Pointer 64 bit Register R10 only
 5050 operand iRegP_R10()
 5051 %{
 5052   constraint(ALLOC_IN_RC(r10_reg));
 5053   match(RegP);
 5054   // match(iRegP);
 5055   match(iRegPNoSp);
 5056   op_cost(0);
 5057   format %{ %}
 5058   interface(REG_INTER);
 5059 %}
 5060 
 5061 // Long 64 bit Register R0 only
 5062 operand iRegL_R0()
 5063 %{
 5064   constraint(ALLOC_IN_RC(r0_reg));
 5065   match(RegL);
 5066   match(iRegLNoSp);
 5067   op_cost(0);
 5068   format %{ %}
 5069   interface(REG_INTER);
 5070 %}
 5071 
 5072 // Long 64 bit Register R2 only
 5073 operand iRegL_R2()
 5074 %{
 5075   constraint(ALLOC_IN_RC(r2_reg));
 5076   match(RegL);
 5077   match(iRegLNoSp);
 5078   op_cost(0);
 5079   format %{ %}
 5080   interface(REG_INTER);
 5081 %}
 5082 
 5083 // Long 64 bit Register R3 only
 5084 operand iRegL_R3()
 5085 %{
 5086   constraint(ALLOC_IN_RC(r3_reg));
 5087   match(RegL);
 5088   match(iRegLNoSp);
 5089   op_cost(0);
 5090   format %{ %}
 5091   interface(REG_INTER);
 5092 %}
 5093 
 5094 // Long 64 bit Register R11 only
 5095 operand iRegL_R11()
 5096 %{
 5097   constraint(ALLOC_IN_RC(r11_reg));
 5098   match(RegL);
 5099   match(iRegLNoSp);
 5100   op_cost(0);
 5101   format %{ %}
 5102   interface(REG_INTER);
 5103 %}
 5104 
 5105 // Pointer 64 bit Register FP only
 5106 operand iRegP_FP()
 5107 %{
 5108   constraint(ALLOC_IN_RC(fp_reg));
 5109   match(RegP);
 5110   // match(iRegP);
 5111   op_cost(0);
 5112   format %{ %}
 5113   interface(REG_INTER);
 5114 %}
 5115 
 5116 // Register R0 only
 5117 operand iRegI_R0()
 5118 %{
 5119   constraint(ALLOC_IN_RC(int_r0_reg));
 5120   match(RegI);
 5121   match(iRegINoSp);
 5122   op_cost(0);
 5123   format %{ %}
 5124   interface(REG_INTER);
 5125 %}
 5126 
 5127 // Register R2 only
 5128 operand iRegI_R2()
 5129 %{
 5130   constraint(ALLOC_IN_RC(int_r2_reg));
 5131   match(RegI);
 5132   match(iRegINoSp);
 5133   op_cost(0);
 5134   format %{ %}
 5135   interface(REG_INTER);
 5136 %}
 5137 
 5138 // Register R3 only
 5139 operand iRegI_R3()
 5140 %{
 5141   constraint(ALLOC_IN_RC(int_r3_reg));
 5142   match(RegI);
 5143   match(iRegINoSp);
 5144   op_cost(0);
 5145   format %{ %}
 5146   interface(REG_INTER);
 5147 %}
 5148 
 5149 
 5150 // Register R4 only
 5151 operand iRegI_R4()
 5152 %{
 5153   constraint(ALLOC_IN_RC(int_r4_reg));
 5154   match(RegI);
 5155   match(iRegINoSp);
 5156   op_cost(0);
 5157   format %{ %}
 5158   interface(REG_INTER);
 5159 %}
 5160 
 5161 
 5162 // Pointer Register Operands
 5163 // Narrow Pointer Register
 5164 operand iRegN()
 5165 %{
 5166   constraint(ALLOC_IN_RC(any_reg32));
 5167   match(RegN);
 5168   match(iRegNNoSp);
 5169   op_cost(0);
 5170   format %{ %}
 5171   interface(REG_INTER);
 5172 %}
 5173 
 5174 operand iRegN_R0()
 5175 %{
 5176   constraint(ALLOC_IN_RC(r0_reg));
 5177   match(iRegN);
 5178   op_cost(0);
 5179   format %{ %}
 5180   interface(REG_INTER);
 5181 %}
 5182 
 5183 operand iRegN_R2()
 5184 %{
 5185   constraint(ALLOC_IN_RC(r2_reg));
 5186   match(iRegN);
 5187   op_cost(0);
 5188   format %{ %}
 5189   interface(REG_INTER);
 5190 %}
 5191 
 5192 operand iRegN_R3()
 5193 %{
 5194   constraint(ALLOC_IN_RC(r3_reg));
 5195   match(iRegN);
 5196   op_cost(0);
 5197   format %{ %}
 5198   interface(REG_INTER);
 5199 %}
 5200 
 5201 // Integer 64 bit Register not Special
 5202 operand iRegNNoSp()
 5203 %{
 5204   constraint(ALLOC_IN_RC(no_special_reg32));
 5205   match(RegN);
 5206   op_cost(0);
 5207   format %{ %}
 5208   interface(REG_INTER);
 5209 %}
 5210 
 5211 // heap base register -- used for encoding immN0
 5212 
 5213 operand iRegIHeapbase()
 5214 %{
 5215   constraint(ALLOC_IN_RC(heapbase_reg));
 5216   match(RegI);
 5217   op_cost(0);
 5218   format %{ %}
 5219   interface(REG_INTER);
 5220 %}
 5221 
 5222 // Float Register
 5223 // Float register operands
 5224 operand vRegF()
 5225 %{
 5226   constraint(ALLOC_IN_RC(float_reg));
 5227   match(RegF);
 5228 
 5229   op_cost(0);
 5230   format %{ %}
 5231   interface(REG_INTER);
 5232 %}
 5233 
 5234 // Double Register
 5235 // Double register operands
 5236 operand vRegD()
 5237 %{
 5238   constraint(ALLOC_IN_RC(double_reg));
 5239   match(RegD);
 5240 
 5241   op_cost(0);
 5242   format %{ %}
 5243   interface(REG_INTER);
 5244 %}
 5245 
 5246 // Generic vector class. This will be used for
 5247 // all vector operands, including NEON and SVE.
 5248 operand vReg()
 5249 %{
 5250   constraint(ALLOC_IN_RC(dynamic));
 5251   match(VecA);
 5252   match(VecD);
 5253   match(VecX);
 5254 
 5255   op_cost(0);
 5256   format %{ %}
 5257   interface(REG_INTER);
 5258 %}
 5259 
 5260 operand vecA()
 5261 %{
 5262   constraint(ALLOC_IN_RC(vectora_reg));
 5263   match(VecA);
 5264 
 5265   op_cost(0);
 5266   format %{ %}
 5267   interface(REG_INTER);
 5268 %}
 5269 
 5270 operand vecD()
 5271 %{
 5272   constraint(ALLOC_IN_RC(vectord_reg));
 5273   match(VecD);
 5274 
 5275   op_cost(0);
 5276   format %{ %}
 5277   interface(REG_INTER);
 5278 %}
 5279 
 5280 operand vecX()
 5281 %{
 5282   constraint(ALLOC_IN_RC(vectorx_reg));
 5283   match(VecX);
 5284 
 5285   op_cost(0);
 5286   format %{ %}
 5287   interface(REG_INTER);
 5288 %}
 5289 
 5290 operand vRegD_V0()
 5291 %{
 5292   constraint(ALLOC_IN_RC(v0_reg));
 5293   match(RegD);
 5294   op_cost(0);
 5295   format %{ %}
 5296   interface(REG_INTER);
 5297 %}
 5298 
 5299 operand vRegD_V1()
 5300 %{
 5301   constraint(ALLOC_IN_RC(v1_reg));
 5302   match(RegD);
 5303   op_cost(0);
 5304   format %{ %}
 5305   interface(REG_INTER);
 5306 %}
 5307 
 5308 operand vRegD_V2()
 5309 %{
 5310   constraint(ALLOC_IN_RC(v2_reg));
 5311   match(RegD);
 5312   op_cost(0);
 5313   format %{ %}
 5314   interface(REG_INTER);
 5315 %}
 5316 
 5317 operand vRegD_V3()
 5318 %{
 5319   constraint(ALLOC_IN_RC(v3_reg));
 5320   match(RegD);
 5321   op_cost(0);
 5322   format %{ %}
 5323   interface(REG_INTER);
 5324 %}
 5325 
 5326 operand vRegD_V4()
 5327 %{
 5328   constraint(ALLOC_IN_RC(v4_reg));
 5329   match(RegD);
 5330   op_cost(0);
 5331   format %{ %}
 5332   interface(REG_INTER);
 5333 %}
 5334 
 5335 operand vRegD_V5()
 5336 %{
 5337   constraint(ALLOC_IN_RC(v5_reg));
 5338   match(RegD);
 5339   op_cost(0);
 5340   format %{ %}
 5341   interface(REG_INTER);
 5342 %}
 5343 
 5344 operand vRegD_V6()
 5345 %{
 5346   constraint(ALLOC_IN_RC(v6_reg));
 5347   match(RegD);
 5348   op_cost(0);
 5349   format %{ %}
 5350   interface(REG_INTER);
 5351 %}
 5352 
 5353 operand vRegD_V7()
 5354 %{
 5355   constraint(ALLOC_IN_RC(v7_reg));
 5356   match(RegD);
 5357   op_cost(0);
 5358   format %{ %}
 5359   interface(REG_INTER);
 5360 %}
 5361 
 5362 operand vRegD_V8()
 5363 %{
 5364   constraint(ALLOC_IN_RC(v8_reg));
 5365   match(RegD);
 5366   op_cost(0);
 5367   format %{ %}
 5368   interface(REG_INTER);
 5369 %}
 5370 
 5371 operand vRegD_V9()
 5372 %{
 5373   constraint(ALLOC_IN_RC(v9_reg));
 5374   match(RegD);
 5375   op_cost(0);
 5376   format %{ %}
 5377   interface(REG_INTER);
 5378 %}
 5379 
 5380 operand vRegD_V10()
 5381 %{
 5382   constraint(ALLOC_IN_RC(v10_reg));
 5383   match(RegD);
 5384   op_cost(0);
 5385   format %{ %}
 5386   interface(REG_INTER);
 5387 %}
 5388 
 5389 operand vRegD_V11()
 5390 %{
 5391   constraint(ALLOC_IN_RC(v11_reg));
 5392   match(RegD);
 5393   op_cost(0);
 5394   format %{ %}
 5395   interface(REG_INTER);
 5396 %}
 5397 
 5398 operand vRegD_V12()
 5399 %{
 5400   constraint(ALLOC_IN_RC(v12_reg));
 5401   match(RegD);
 5402   op_cost(0);
 5403   format %{ %}
 5404   interface(REG_INTER);
 5405 %}
 5406 
 5407 operand vRegD_V13()
 5408 %{
 5409   constraint(ALLOC_IN_RC(v13_reg));
 5410   match(RegD);
 5411   op_cost(0);
 5412   format %{ %}
 5413   interface(REG_INTER);
 5414 %}
 5415 
 5416 operand vRegD_V14()
 5417 %{
 5418   constraint(ALLOC_IN_RC(v14_reg));
 5419   match(RegD);
 5420   op_cost(0);
 5421   format %{ %}
 5422   interface(REG_INTER);
 5423 %}
 5424 
 5425 operand vRegD_V15()
 5426 %{
 5427   constraint(ALLOC_IN_RC(v15_reg));
 5428   match(RegD);
 5429   op_cost(0);
 5430   format %{ %}
 5431   interface(REG_INTER);
 5432 %}
 5433 
 5434 operand vRegD_V16()
 5435 %{
 5436   constraint(ALLOC_IN_RC(v16_reg));
 5437   match(RegD);
 5438   op_cost(0);
 5439   format %{ %}
 5440   interface(REG_INTER);
 5441 %}
 5442 
 5443 operand vRegD_V17()
 5444 %{
 5445   constraint(ALLOC_IN_RC(v17_reg));
 5446   match(RegD);
 5447   op_cost(0);
 5448   format %{ %}
 5449   interface(REG_INTER);
 5450 %}
 5451 
 5452 operand vRegD_V18()
 5453 %{
 5454   constraint(ALLOC_IN_RC(v18_reg));
 5455   match(RegD);
 5456   op_cost(0);
 5457   format %{ %}
 5458   interface(REG_INTER);
 5459 %}
 5460 
 5461 operand vRegD_V19()
 5462 %{
 5463   constraint(ALLOC_IN_RC(v19_reg));
 5464   match(RegD);
 5465   op_cost(0);
 5466   format %{ %}
 5467   interface(REG_INTER);
 5468 %}
 5469 
 5470 operand vRegD_V20()
 5471 %{
 5472   constraint(ALLOC_IN_RC(v20_reg));
 5473   match(RegD);
 5474   op_cost(0);
 5475   format %{ %}
 5476   interface(REG_INTER);
 5477 %}
 5478 
 5479 operand vRegD_V21()
 5480 %{
 5481   constraint(ALLOC_IN_RC(v21_reg));
 5482   match(RegD);
 5483   op_cost(0);
 5484   format %{ %}
 5485   interface(REG_INTER);
 5486 %}
 5487 
 5488 operand vRegD_V22()
 5489 %{
 5490   constraint(ALLOC_IN_RC(v22_reg));
 5491   match(RegD);
 5492   op_cost(0);
 5493   format %{ %}
 5494   interface(REG_INTER);
 5495 %}
 5496 
 5497 operand vRegD_V23()
 5498 %{
 5499   constraint(ALLOC_IN_RC(v23_reg));
 5500   match(RegD);
 5501   op_cost(0);
 5502   format %{ %}
 5503   interface(REG_INTER);
 5504 %}
 5505 
 5506 operand vRegD_V24()
 5507 %{
 5508   constraint(ALLOC_IN_RC(v24_reg));
 5509   match(RegD);
 5510   op_cost(0);
 5511   format %{ %}
 5512   interface(REG_INTER);
 5513 %}
 5514 
 5515 operand vRegD_V25()
 5516 %{
 5517   constraint(ALLOC_IN_RC(v25_reg));
 5518   match(RegD);
 5519   op_cost(0);
 5520   format %{ %}
 5521   interface(REG_INTER);
 5522 %}
 5523 
 5524 operand vRegD_V26()
 5525 %{
 5526   constraint(ALLOC_IN_RC(v26_reg));
 5527   match(RegD);
 5528   op_cost(0);
 5529   format %{ %}
 5530   interface(REG_INTER);
 5531 %}
 5532 
 5533 operand vRegD_V27()
 5534 %{
 5535   constraint(ALLOC_IN_RC(v27_reg));
 5536   match(RegD);
 5537   op_cost(0);
 5538   format %{ %}
 5539   interface(REG_INTER);
 5540 %}
 5541 
 5542 operand vRegD_V28()
 5543 %{
 5544   constraint(ALLOC_IN_RC(v28_reg));
 5545   match(RegD);
 5546   op_cost(0);
 5547   format %{ %}
 5548   interface(REG_INTER);
 5549 %}
 5550 
 5551 operand vRegD_V29()
 5552 %{
 5553   constraint(ALLOC_IN_RC(v29_reg));
 5554   match(RegD);
 5555   op_cost(0);
 5556   format %{ %}
 5557   interface(REG_INTER);
 5558 %}
 5559 
 5560 operand vRegD_V30()
 5561 %{
 5562   constraint(ALLOC_IN_RC(v30_reg));
 5563   match(RegD);
 5564   op_cost(0);
 5565   format %{ %}
 5566   interface(REG_INTER);
 5567 %}
 5568 
 5569 operand vRegD_V31()
 5570 %{
 5571   constraint(ALLOC_IN_RC(v31_reg));
 5572   match(RegD);
 5573   op_cost(0);
 5574   format %{ %}
 5575   interface(REG_INTER);
 5576 %}
 5577 
 5578 operand pReg()
 5579 %{
 5580   constraint(ALLOC_IN_RC(pr_reg));
 5581   match(RegVectMask);
 5582   match(pRegGov);
 5583   op_cost(0);
 5584   format %{ %}
 5585   interface(REG_INTER);
 5586 %}
 5587 
 5588 operand pRegGov()
 5589 %{
 5590   constraint(ALLOC_IN_RC(gov_pr));
 5591   match(RegVectMask);
 5592   match(pReg);
 5593   op_cost(0);
 5594   format %{ %}
 5595   interface(REG_INTER);
 5596 %}
 5597 
 5598 operand pRegGov_P0()
 5599 %{
 5600   constraint(ALLOC_IN_RC(p0_reg));
 5601   match(RegVectMask);
 5602   op_cost(0);
 5603   format %{ %}
 5604   interface(REG_INTER);
 5605 %}
 5606 
 5607 operand pRegGov_P1()
 5608 %{
 5609   constraint(ALLOC_IN_RC(p1_reg));
 5610   match(RegVectMask);
 5611   op_cost(0);
 5612   format %{ %}
 5613   interface(REG_INTER);
 5614 %}
 5615 
 5616 // Flags register, used as output of signed compare instructions
 5617 
 5618 // note that on AArch64 we also use this register as the output for
 5619 // for floating point compare instructions (CmpF CmpD). this ensures
 5620 // that ordered inequality tests use GT, GE, LT or LE none of which
 5621 // pass through cases where the result is unordered i.e. one or both
 5622 // inputs to the compare is a NaN. this means that the ideal code can
 5623 // replace e.g. a GT with an LE and not end up capturing the NaN case
 5624 // (where the comparison should always fail). EQ and NE tests are
 5625 // always generated in ideal code so that unordered folds into the NE
 5626 // case, matching the behaviour of AArch64 NE.
 5627 //
 5628 // This differs from x86 where the outputs of FP compares use a
 5629 // special FP flags registers and where compares based on this
 5630 // register are distinguished into ordered inequalities (cmpOpUCF) and
 5631 // EQ/NEQ tests (cmpOpUCF2). x86 has to special case the latter tests
 5632 // to explicitly handle the unordered case in branches. x86 also has
 5633 // to include extra CMoveX rules to accept a cmpOpUCF input.
 5634 
 5635 operand rFlagsReg()
 5636 %{
 5637   constraint(ALLOC_IN_RC(int_flags));
 5638   match(RegFlags);
 5639 
 5640   op_cost(0);
 5641   format %{ "RFLAGS" %}
 5642   interface(REG_INTER);
 5643 %}
 5644 
 5645 // Flags register, used as output of unsigned compare instructions
 5646 operand rFlagsRegU()
 5647 %{
 5648   constraint(ALLOC_IN_RC(int_flags));
 5649   match(RegFlags);
 5650 
 5651   op_cost(0);
 5652   format %{ "RFLAGSU" %}
 5653   interface(REG_INTER);
 5654 %}
 5655 
 5656 // Special Registers
 5657 
 5658 // Method Register
 5659 operand inline_cache_RegP(iRegP reg)
 5660 %{
 5661   constraint(ALLOC_IN_RC(method_reg)); // inline_cache_reg
 5662   match(reg);
 5663   match(iRegPNoSp);
 5664   op_cost(0);
 5665   format %{ %}
 5666   interface(REG_INTER);
 5667 %}
 5668 
 5669 // Thread Register
 5670 operand thread_RegP(iRegP reg)
 5671 %{
 5672   constraint(ALLOC_IN_RC(thread_reg)); // link_reg
 5673   match(reg);
 5674   op_cost(0);
 5675   format %{ %}
 5676   interface(REG_INTER);
 5677 %}
 5678 
 5679 operand lr_RegP(iRegP reg)
 5680 %{
 5681   constraint(ALLOC_IN_RC(lr_reg)); // link_reg
 5682   match(reg);
 5683   op_cost(0);
 5684   format %{ %}
 5685   interface(REG_INTER);
 5686 %}
 5687 
 5688 //----------Memory Operands----------------------------------------------------
 5689 
 5690 operand indirect(iRegP reg)
 5691 %{
 5692   constraint(ALLOC_IN_RC(ptr_reg));
 5693   match(reg);
 5694   op_cost(0);
 5695   format %{ "[$reg]" %}
 5696   interface(MEMORY_INTER) %{
 5697     base($reg);
 5698     index(0xffffffff);
 5699     scale(0x0);
 5700     disp(0x0);
 5701   %}
 5702 %}
 5703 
 5704 operand indIndexScaledI2L(iRegP reg, iRegI ireg, immIScale scale)
 5705 %{
 5706   constraint(ALLOC_IN_RC(ptr_reg));
 5707   predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 5708   match(AddP reg (LShiftL (ConvI2L ireg) scale));
 5709   op_cost(0);
 5710   format %{ "$reg, $ireg sxtw($scale), 0, I2L" %}
 5711   interface(MEMORY_INTER) %{
 5712     base($reg);
 5713     index($ireg);
 5714     scale($scale);
 5715     disp(0x0);
 5716   %}
 5717 %}
 5718 
 5719 operand indIndexScaled(iRegP reg, iRegL lreg, immIScale scale)
 5720 %{
 5721   constraint(ALLOC_IN_RC(ptr_reg));
 5722   predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 5723   match(AddP reg (LShiftL lreg scale));
 5724   op_cost(0);
 5725   format %{ "$reg, $lreg lsl($scale)" %}
 5726   interface(MEMORY_INTER) %{
 5727     base($reg);
 5728     index($lreg);
 5729     scale($scale);
 5730     disp(0x0);
 5731   %}
 5732 %}
 5733 
 5734 operand indIndexI2L(iRegP reg, iRegI ireg)
 5735 %{
 5736   constraint(ALLOC_IN_RC(ptr_reg));
 5737   match(AddP reg (ConvI2L ireg));
 5738   op_cost(0);
 5739   format %{ "$reg, $ireg, 0, I2L" %}
 5740   interface(MEMORY_INTER) %{
 5741     base($reg);
 5742     index($ireg);
 5743     scale(0x0);
 5744     disp(0x0);
 5745   %}
 5746 %}
 5747 
 5748 operand indIndex(iRegP reg, iRegL lreg)
 5749 %{
 5750   constraint(ALLOC_IN_RC(ptr_reg));
 5751   match(AddP reg lreg);
 5752   op_cost(0);
 5753   format %{ "$reg, $lreg" %}
 5754   interface(MEMORY_INTER) %{
 5755     base($reg);
 5756     index($lreg);
 5757     scale(0x0);
 5758     disp(0x0);
 5759   %}
 5760 %}
 5761 
 5762 operand indOffI(iRegP reg, immIOffset off)
 5763 %{
 5764   constraint(ALLOC_IN_RC(ptr_reg));
 5765   match(AddP reg off);
 5766   op_cost(0);
 5767   format %{ "[$reg, $off]" %}
 5768   interface(MEMORY_INTER) %{
 5769     base($reg);
 5770     index(0xffffffff);
 5771     scale(0x0);
 5772     disp($off);
 5773   %}
 5774 %}
 5775 
 5776 operand indOffI1(iRegP reg, immIOffset1 off)
 5777 %{
 5778   constraint(ALLOC_IN_RC(ptr_reg));
 5779   match(AddP reg off);
 5780   op_cost(0);
 5781   format %{ "[$reg, $off]" %}
 5782   interface(MEMORY_INTER) %{
 5783     base($reg);
 5784     index(0xffffffff);
 5785     scale(0x0);
 5786     disp($off);
 5787   %}
 5788 %}
 5789 
 5790 operand indOffI2(iRegP reg, immIOffset2 off)
 5791 %{
 5792   constraint(ALLOC_IN_RC(ptr_reg));
 5793   match(AddP reg off);
 5794   op_cost(0);
 5795   format %{ "[$reg, $off]" %}
 5796   interface(MEMORY_INTER) %{
 5797     base($reg);
 5798     index(0xffffffff);
 5799     scale(0x0);
 5800     disp($off);
 5801   %}
 5802 %}
 5803 
 5804 operand indOffI4(iRegP reg, immIOffset4 off)
 5805 %{
 5806   constraint(ALLOC_IN_RC(ptr_reg));
 5807   match(AddP reg off);
 5808   op_cost(0);
 5809   format %{ "[$reg, $off]" %}
 5810   interface(MEMORY_INTER) %{
 5811     base($reg);
 5812     index(0xffffffff);
 5813     scale(0x0);
 5814     disp($off);
 5815   %}
 5816 %}
 5817 
 5818 operand indOffI8(iRegP reg, immIOffset8 off)
 5819 %{
 5820   constraint(ALLOC_IN_RC(ptr_reg));
 5821   match(AddP reg off);
 5822   op_cost(0);
 5823   format %{ "[$reg, $off]" %}
 5824   interface(MEMORY_INTER) %{
 5825     base($reg);
 5826     index(0xffffffff);
 5827     scale(0x0);
 5828     disp($off);
 5829   %}
 5830 %}
 5831 
 5832 operand indOffI16(iRegP reg, immIOffset16 off)
 5833 %{
 5834   constraint(ALLOC_IN_RC(ptr_reg));
 5835   match(AddP reg off);
 5836   op_cost(0);
 5837   format %{ "[$reg, $off]" %}
 5838   interface(MEMORY_INTER) %{
 5839     base($reg);
 5840     index(0xffffffff);
 5841     scale(0x0);
 5842     disp($off);
 5843   %}
 5844 %}
 5845 
 5846 operand indOffL(iRegP reg, immLoffset off)
 5847 %{
 5848   constraint(ALLOC_IN_RC(ptr_reg));
 5849   match(AddP reg off);
 5850   op_cost(0);
 5851   format %{ "[$reg, $off]" %}
 5852   interface(MEMORY_INTER) %{
 5853     base($reg);
 5854     index(0xffffffff);
 5855     scale(0x0);
 5856     disp($off);
 5857   %}
 5858 %}
 5859 
 5860 operand indOffL1(iRegP reg, immLoffset1 off)
 5861 %{
 5862   constraint(ALLOC_IN_RC(ptr_reg));
 5863   match(AddP reg off);
 5864   op_cost(0);
 5865   format %{ "[$reg, $off]" %}
 5866   interface(MEMORY_INTER) %{
 5867     base($reg);
 5868     index(0xffffffff);
 5869     scale(0x0);
 5870     disp($off);
 5871   %}
 5872 %}
 5873 
 5874 operand indOffL2(iRegP reg, immLoffset2 off)
 5875 %{
 5876   constraint(ALLOC_IN_RC(ptr_reg));
 5877   match(AddP reg off);
 5878   op_cost(0);
 5879   format %{ "[$reg, $off]" %}
 5880   interface(MEMORY_INTER) %{
 5881     base($reg);
 5882     index(0xffffffff);
 5883     scale(0x0);
 5884     disp($off);
 5885   %}
 5886 %}
 5887 
 5888 operand indOffL4(iRegP reg, immLoffset4 off)
 5889 %{
 5890   constraint(ALLOC_IN_RC(ptr_reg));
 5891   match(AddP reg off);
 5892   op_cost(0);
 5893   format %{ "[$reg, $off]" %}
 5894   interface(MEMORY_INTER) %{
 5895     base($reg);
 5896     index(0xffffffff);
 5897     scale(0x0);
 5898     disp($off);
 5899   %}
 5900 %}
 5901 
 5902 operand indOffL8(iRegP reg, immLoffset8 off)
 5903 %{
 5904   constraint(ALLOC_IN_RC(ptr_reg));
 5905   match(AddP reg off);
 5906   op_cost(0);
 5907   format %{ "[$reg, $off]" %}
 5908   interface(MEMORY_INTER) %{
 5909     base($reg);
 5910     index(0xffffffff);
 5911     scale(0x0);
 5912     disp($off);
 5913   %}
 5914 %}
 5915 
 5916 operand indOffL16(iRegP reg, immLoffset16 off)
 5917 %{
 5918   constraint(ALLOC_IN_RC(ptr_reg));
 5919   match(AddP reg off);
 5920   op_cost(0);
 5921   format %{ "[$reg, $off]" %}
 5922   interface(MEMORY_INTER) %{
 5923     base($reg);
 5924     index(0xffffffff);
 5925     scale(0x0);
 5926     disp($off);
 5927   %}
 5928 %}
 5929 
 5930 operand indirectN(iRegN reg)
 5931 %{
 5932   predicate(CompressedOops::shift() == 0);
 5933   constraint(ALLOC_IN_RC(ptr_reg));
 5934   match(DecodeN reg);
 5935   op_cost(0);
 5936   format %{ "[$reg]\t# narrow" %}
 5937   interface(MEMORY_INTER) %{
 5938     base($reg);
 5939     index(0xffffffff);
 5940     scale(0x0);
 5941     disp(0x0);
 5942   %}
 5943 %}
 5944 
 5945 operand indIndexScaledI2LN(iRegN reg, iRegI ireg, immIScale scale)
 5946 %{
 5947   predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 5948   constraint(ALLOC_IN_RC(ptr_reg));
 5949   match(AddP (DecodeN reg) (LShiftL (ConvI2L ireg) scale));
 5950   op_cost(0);
 5951   format %{ "$reg, $ireg sxtw($scale), 0, I2L\t# narrow" %}
 5952   interface(MEMORY_INTER) %{
 5953     base($reg);
 5954     index($ireg);
 5955     scale($scale);
 5956     disp(0x0);
 5957   %}
 5958 %}
 5959 
 5960 operand indIndexScaledN(iRegN reg, iRegL lreg, immIScale scale)
 5961 %{
 5962   predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 5963   constraint(ALLOC_IN_RC(ptr_reg));
 5964   match(AddP (DecodeN reg) (LShiftL lreg scale));
 5965   op_cost(0);
 5966   format %{ "$reg, $lreg lsl($scale)\t# narrow" %}
 5967   interface(MEMORY_INTER) %{
 5968     base($reg);
 5969     index($lreg);
 5970     scale($scale);
 5971     disp(0x0);
 5972   %}
 5973 %}
 5974 
 5975 operand indIndexI2LN(iRegN reg, iRegI ireg)
 5976 %{
 5977   predicate(CompressedOops::shift() == 0);
 5978   constraint(ALLOC_IN_RC(ptr_reg));
 5979   match(AddP (DecodeN reg) (ConvI2L ireg));
 5980   op_cost(0);
 5981   format %{ "$reg, $ireg, 0, I2L\t# narrow" %}
 5982   interface(MEMORY_INTER) %{
 5983     base($reg);
 5984     index($ireg);
 5985     scale(0x0);
 5986     disp(0x0);
 5987   %}
 5988 %}
 5989 
 5990 operand indIndexN(iRegN reg, iRegL lreg)
 5991 %{
 5992   predicate(CompressedOops::shift() == 0);
 5993   constraint(ALLOC_IN_RC(ptr_reg));
 5994   match(AddP (DecodeN reg) lreg);
 5995   op_cost(0);
 5996   format %{ "$reg, $lreg\t# narrow" %}
 5997   interface(MEMORY_INTER) %{
 5998     base($reg);
 5999     index($lreg);
 6000     scale(0x0);
 6001     disp(0x0);
 6002   %}
 6003 %}
 6004 
 6005 operand indOffIN(iRegN reg, immIOffset off)
 6006 %{
 6007   predicate(CompressedOops::shift() == 0);
 6008   constraint(ALLOC_IN_RC(ptr_reg));
 6009   match(AddP (DecodeN reg) off);
 6010   op_cost(0);
 6011   format %{ "[$reg, $off]\t# narrow" %}
 6012   interface(MEMORY_INTER) %{
 6013     base($reg);
 6014     index(0xffffffff);
 6015     scale(0x0);
 6016     disp($off);
 6017   %}
 6018 %}
 6019 
 6020 operand indOffLN(iRegN reg, immLoffset off)
 6021 %{
 6022   predicate(CompressedOops::shift() == 0);
 6023   constraint(ALLOC_IN_RC(ptr_reg));
 6024   match(AddP (DecodeN reg) off);
 6025   op_cost(0);
 6026   format %{ "[$reg, $off]\t# narrow" %}
 6027   interface(MEMORY_INTER) %{
 6028     base($reg);
 6029     index(0xffffffff);
 6030     scale(0x0);
 6031     disp($off);
 6032   %}
 6033 %}
 6034 
 6035 
 6036 
 6037 // AArch64 opto stubs need to write to the pc slot in the thread anchor
 6038 operand thread_anchor_pc(thread_RegP reg, immL_pc_off off)
 6039 %{
 6040   constraint(ALLOC_IN_RC(ptr_reg));
 6041   match(AddP reg off);
 6042   op_cost(0);
 6043   format %{ "[$reg, $off]" %}
 6044   interface(MEMORY_INTER) %{
 6045     base($reg);
 6046     index(0xffffffff);
 6047     scale(0x0);
 6048     disp($off);
 6049   %}
 6050 %}
 6051 
 6052 //----------Special Memory Operands--------------------------------------------
 6053 // Stack Slot Operand - This operand is used for loading and storing temporary
 6054 //                      values on the stack where a match requires a value to
 6055 //                      flow through memory.
 6056 operand stackSlotP(sRegP reg)
 6057 %{
 6058   constraint(ALLOC_IN_RC(stack_slots));
 6059   op_cost(100);
 6060   // No match rule because this operand is only generated in matching
 6061   // match(RegP);
 6062   format %{ "[$reg]" %}
 6063   interface(MEMORY_INTER) %{
 6064     base(0x1e);  // RSP
 6065     index(0x0);  // No Index
 6066     scale(0x0);  // No Scale
 6067     disp($reg);  // Stack Offset
 6068   %}
 6069 %}
 6070 
 6071 operand stackSlotI(sRegI reg)
 6072 %{
 6073   constraint(ALLOC_IN_RC(stack_slots));
 6074   // No match rule because this operand is only generated in matching
 6075   // match(RegI);
 6076   format %{ "[$reg]" %}
 6077   interface(MEMORY_INTER) %{
 6078     base(0x1e);  // RSP
 6079     index(0x0);  // No Index
 6080     scale(0x0);  // No Scale
 6081     disp($reg);  // Stack Offset
 6082   %}
 6083 %}
 6084 
 6085 operand stackSlotF(sRegF reg)
 6086 %{
 6087   constraint(ALLOC_IN_RC(stack_slots));
 6088   // No match rule because this operand is only generated in matching
 6089   // match(RegF);
 6090   format %{ "[$reg]" %}
 6091   interface(MEMORY_INTER) %{
 6092     base(0x1e);  // RSP
 6093     index(0x0);  // No Index
 6094     scale(0x0);  // No Scale
 6095     disp($reg);  // Stack Offset
 6096   %}
 6097 %}
 6098 
 6099 operand stackSlotD(sRegD reg)
 6100 %{
 6101   constraint(ALLOC_IN_RC(stack_slots));
 6102   // No match rule because this operand is only generated in matching
 6103   // match(RegD);
 6104   format %{ "[$reg]" %}
 6105   interface(MEMORY_INTER) %{
 6106     base(0x1e);  // RSP
 6107     index(0x0);  // No Index
 6108     scale(0x0);  // No Scale
 6109     disp($reg);  // Stack Offset
 6110   %}
 6111 %}
 6112 
 6113 operand stackSlotL(sRegL reg)
 6114 %{
 6115   constraint(ALLOC_IN_RC(stack_slots));
 6116   // No match rule because this operand is only generated in matching
 6117   // match(RegL);
 6118   format %{ "[$reg]" %}
 6119   interface(MEMORY_INTER) %{
 6120     base(0x1e);  // RSP
 6121     index(0x0);  // No Index
 6122     scale(0x0);  // No Scale
 6123     disp($reg);  // Stack Offset
 6124   %}
 6125 %}
 6126 
 6127 // Operands for expressing Control Flow
 6128 // NOTE: Label is a predefined operand which should not be redefined in
 6129 //       the AD file. It is generically handled within the ADLC.
 6130 
 6131 //----------Conditional Branch Operands----------------------------------------
 6132 // Comparison Op  - This is the operation of the comparison, and is limited to
 6133 //                  the following set of codes:
 6134 //                  L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
 6135 //
 6136 // Other attributes of the comparison, such as unsignedness, are specified
 6137 // by the comparison instruction that sets a condition code flags register.
 6138 // That result is represented by a flags operand whose subtype is appropriate
 6139 // to the unsignedness (etc.) of the comparison.
 6140 //
 6141 // Later, the instruction which matches both the Comparison Op (a Bool) and
 6142 // the flags (produced by the Cmp) specifies the coding of the comparison op
 6143 // by matching a specific subtype of Bool operand below, such as cmpOpU.
 6144 
 6145 // used for signed integral comparisons and fp comparisons
 6146 
 6147 operand cmpOp()
 6148 %{
 6149   match(Bool);
 6150 
 6151   format %{ "" %}
 6152   interface(COND_INTER) %{
 6153     equal(0x0, "eq");
 6154     not_equal(0x1, "ne");
 6155     less(0xb, "lt");
 6156     greater_equal(0xa, "ge");
 6157     less_equal(0xd, "le");
 6158     greater(0xc, "gt");
 6159     overflow(0x6, "vs");
 6160     no_overflow(0x7, "vc");
 6161   %}
 6162 %}
 6163 
 6164 // used for unsigned integral comparisons
 6165 
 6166 operand cmpOpU()
 6167 %{
 6168   match(Bool);
 6169 
 6170   format %{ "" %}
 6171   interface(COND_INTER) %{
 6172     equal(0x0, "eq");
 6173     not_equal(0x1, "ne");
 6174     less(0x3, "lo");
 6175     greater_equal(0x2, "hs");
 6176     less_equal(0x9, "ls");
 6177     greater(0x8, "hi");
 6178     overflow(0x6, "vs");
 6179     no_overflow(0x7, "vc");
 6180   %}
 6181 %}
 6182 
 6183 // used for certain integral comparisons which can be
 6184 // converted to cbxx or tbxx instructions
 6185 
 6186 operand cmpOpEqNe()
 6187 %{
 6188   match(Bool);
 6189   op_cost(0);
 6190   predicate(n->as_Bool()->_test._test == BoolTest::ne
 6191             || n->as_Bool()->_test._test == BoolTest::eq);
 6192 
 6193   format %{ "" %}
 6194   interface(COND_INTER) %{
 6195     equal(0x0, "eq");
 6196     not_equal(0x1, "ne");
 6197     less(0xb, "lt");
 6198     greater_equal(0xa, "ge");
 6199     less_equal(0xd, "le");
 6200     greater(0xc, "gt");
 6201     overflow(0x6, "vs");
 6202     no_overflow(0x7, "vc");
 6203   %}
 6204 %}
 6205 
 6206 // used for certain integral comparisons which can be
 6207 // converted to cbxx or tbxx instructions
 6208 
 6209 operand cmpOpLtGe()
 6210 %{
 6211   match(Bool);
 6212   op_cost(0);
 6213 
 6214   predicate(n->as_Bool()->_test._test == BoolTest::lt
 6215             || n->as_Bool()->_test._test == BoolTest::ge);
 6216 
 6217   format %{ "" %}
 6218   interface(COND_INTER) %{
 6219     equal(0x0, "eq");
 6220     not_equal(0x1, "ne");
 6221     less(0xb, "lt");
 6222     greater_equal(0xa, "ge");
 6223     less_equal(0xd, "le");
 6224     greater(0xc, "gt");
 6225     overflow(0x6, "vs");
 6226     no_overflow(0x7, "vc");
 6227   %}
 6228 %}
 6229 
 6230 // used for certain unsigned integral comparisons which can be
 6231 // converted to cbxx or tbxx instructions
 6232 
 6233 operand cmpOpUEqNeLtGe()
 6234 %{
 6235   match(Bool);
 6236   op_cost(0);
 6237 
 6238   predicate(n->as_Bool()->_test._test == BoolTest::eq
 6239             || n->as_Bool()->_test._test == BoolTest::ne
 6240             || n->as_Bool()->_test._test == BoolTest::lt
 6241             || n->as_Bool()->_test._test == BoolTest::ge);
 6242 
 6243   format %{ "" %}
 6244   interface(COND_INTER) %{
 6245     equal(0x0, "eq");
 6246     not_equal(0x1, "ne");
 6247     less(0xb, "lt");
 6248     greater_equal(0xa, "ge");
 6249     less_equal(0xd, "le");
 6250     greater(0xc, "gt");
 6251     overflow(0x6, "vs");
 6252     no_overflow(0x7, "vc");
 6253   %}
 6254 %}
 6255 
 6256 // Special operand allowing long args to int ops to be truncated for free
 6257 
 6258 operand iRegL2I(iRegL reg) %{
 6259 
 6260   op_cost(0);
 6261 
 6262   match(ConvL2I reg);
 6263 
 6264   format %{ "l2i($reg)" %}
 6265 
 6266   interface(REG_INTER)
 6267 %}
 6268 
 6269 opclass vmem2(indirect, indIndex, indOffI2, indOffL2);
 6270 opclass vmem4(indirect, indIndex, indOffI4, indOffL4);
 6271 opclass vmem8(indirect, indIndex, indOffI8, indOffL8);
 6272 opclass vmem16(indirect, indIndex, indOffI16, indOffL16);
 6273 
 6274 //----------OPERAND CLASSES----------------------------------------------------
 6275 // Operand Classes are groups of operands that are used as to simplify
 6276 // instruction definitions by not requiring the AD writer to specify
 6277 // separate instructions for every form of operand when the
 6278 // instruction accepts multiple operand types with the same basic
 6279 // encoding and format. The classic case of this is memory operands.
 6280 
 6281 // memory is used to define read/write location for load/store
 6282 // instruction defs. we can turn a memory op into an Address
 6283 
 6284 opclass memory1(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI1, indOffL1,
 6285                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN);
 6286 
 6287 opclass memory2(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI2, indOffL2,
 6288                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN);
 6289 
 6290 opclass memory4(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI4, indOffL4,
 6291                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
 6292 
 6293 opclass memory8(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI8, indOffL8,
 6294                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
 6295 
 6296 // All of the memory operands. For the pipeline description.
 6297 opclass memory(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex,
 6298                indOffI1, indOffL1, indOffI2, indOffL2, indOffI4, indOffL4, indOffI8, indOffL8,
 6299                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
 6300 
 6301 
 6302 // iRegIorL2I is used for src inputs in rules for 32 bit int (I)
 6303 // operations. it allows the src to be either an iRegI or a (ConvL2I
 6304 // iRegL). in the latter case the l2i normally planted for a ConvL2I
 6305 // can be elided because the 32-bit instruction will just employ the
 6306 // lower 32 bits anyway.
 6307 //
 6308 // n.b. this does not elide all L2I conversions. if the truncated
 6309 // value is consumed by more than one operation then the ConvL2I
 6310 // cannot be bundled into the consuming nodes so an l2i gets planted
 6311 // (actually a movw $dst $src) and the downstream instructions consume
 6312 // the result of the l2i as an iRegI input. That's a shame since the
 6313 // movw is actually redundant but its not too costly.
 6314 
 6315 opclass iRegIorL2I(iRegI, iRegL2I);
 6316 
 6317 //----------PIPELINE-----------------------------------------------------------
 6318 // Rules which define the behavior of the target architectures pipeline.
 6319 
 6320 // For specific pipelines, eg A53, define the stages of that pipeline
 6321 //pipe_desc(ISS, EX1, EX2, WR);
 6322 #define ISS S0
 6323 #define EX1 S1
 6324 #define EX2 S2
 6325 #define WR  S3
 6326 
 6327 // Integer ALU reg operation
 6328 pipeline %{
 6329 
 6330 attributes %{
 6331   // ARM instructions are of fixed length
 6332   fixed_size_instructions;        // Fixed size instructions TODO does
 6333   max_instructions_per_bundle = 4;   // A53 = 2, A57 = 4
 6334   // ARM instructions come in 32-bit word units
 6335   instruction_unit_size = 4;         // An instruction is 4 bytes long
 6336   instruction_fetch_unit_size = 64;  // The processor fetches one line
 6337   instruction_fetch_units = 1;       // of 64 bytes
 6338 
 6339   // List of nop instructions
 6340   nops( MachNop );
 6341 %}
 6342 
 6343 // We don't use an actual pipeline model so don't care about resources
 6344 // or description. we do use pipeline classes to introduce fixed
 6345 // latencies
 6346 
 6347 //----------RESOURCES----------------------------------------------------------
 6348 // Resources are the functional units available to the machine
 6349 
 6350 resources( INS0, INS1, INS01 = INS0 | INS1,
 6351            ALU0, ALU1, ALU = ALU0 | ALU1,
 6352            MAC,
 6353            DIV,
 6354            BRANCH,
 6355            LDST,
 6356            NEON_FP);
 6357 
 6358 //----------PIPELINE DESCRIPTION-----------------------------------------------
 6359 // Pipeline Description specifies the stages in the machine's pipeline
 6360 
 6361 // Define the pipeline as a generic 6 stage pipeline
 6362 pipe_desc(S0, S1, S2, S3, S4, S5);
 6363 
 6364 //----------PIPELINE CLASSES---------------------------------------------------
 6365 // Pipeline Classes describe the stages in which input and output are
 6366 // referenced by the hardware pipeline.
 6367 
 6368 pipe_class fp_dop_reg_reg_s(vRegF dst, vRegF src1, vRegF src2)
 6369 %{
 6370   single_instruction;
 6371   src1   : S1(read);
 6372   src2   : S2(read);
 6373   dst    : S5(write);
 6374   INS01  : ISS;
 6375   NEON_FP : S5;
 6376 %}
 6377 
 6378 pipe_class fp_dop_reg_reg_d(vRegD dst, vRegD src1, vRegD src2)
 6379 %{
 6380   single_instruction;
 6381   src1   : S1(read);
 6382   src2   : S2(read);
 6383   dst    : S5(write);
 6384   INS01  : ISS;
 6385   NEON_FP : S5;
 6386 %}
 6387 
 6388 pipe_class fp_uop_s(vRegF dst, vRegF src)
 6389 %{
 6390   single_instruction;
 6391   src    : S1(read);
 6392   dst    : S5(write);
 6393   INS01  : ISS;
 6394   NEON_FP : S5;
 6395 %}
 6396 
 6397 pipe_class fp_uop_d(vRegD dst, vRegD src)
 6398 %{
 6399   single_instruction;
 6400   src    : S1(read);
 6401   dst    : S5(write);
 6402   INS01  : ISS;
 6403   NEON_FP : S5;
 6404 %}
 6405 
 6406 pipe_class fp_d2f(vRegF dst, vRegD src)
 6407 %{
 6408   single_instruction;
 6409   src    : S1(read);
 6410   dst    : S5(write);
 6411   INS01  : ISS;
 6412   NEON_FP : S5;
 6413 %}
 6414 
 6415 pipe_class fp_f2d(vRegD dst, vRegF src)
 6416 %{
 6417   single_instruction;
 6418   src    : S1(read);
 6419   dst    : S5(write);
 6420   INS01  : ISS;
 6421   NEON_FP : S5;
 6422 %}
 6423 
 6424 pipe_class fp_f2i(iRegINoSp dst, vRegF src)
 6425 %{
 6426   single_instruction;
 6427   src    : S1(read);
 6428   dst    : S5(write);
 6429   INS01  : ISS;
 6430   NEON_FP : S5;
 6431 %}
 6432 
 6433 pipe_class fp_f2l(iRegLNoSp dst, vRegF src)
 6434 %{
 6435   single_instruction;
 6436   src    : S1(read);
 6437   dst    : S5(write);
 6438   INS01  : ISS;
 6439   NEON_FP : S5;
 6440 %}
 6441 
 6442 pipe_class fp_i2f(vRegF dst, iRegIorL2I src)
 6443 %{
 6444   single_instruction;
 6445   src    : S1(read);
 6446   dst    : S5(write);
 6447   INS01  : ISS;
 6448   NEON_FP : S5;
 6449 %}
 6450 
 6451 pipe_class fp_l2f(vRegF dst, iRegL src)
 6452 %{
 6453   single_instruction;
 6454   src    : S1(read);
 6455   dst    : S5(write);
 6456   INS01  : ISS;
 6457   NEON_FP : S5;
 6458 %}
 6459 
 6460 pipe_class fp_d2i(iRegINoSp dst, vRegD src)
 6461 %{
 6462   single_instruction;
 6463   src    : S1(read);
 6464   dst    : S5(write);
 6465   INS01  : ISS;
 6466   NEON_FP : S5;
 6467 %}
 6468 
 6469 pipe_class fp_d2l(iRegLNoSp dst, vRegD src)
 6470 %{
 6471   single_instruction;
 6472   src    : S1(read);
 6473   dst    : S5(write);
 6474   INS01  : ISS;
 6475   NEON_FP : S5;
 6476 %}
 6477 
 6478 pipe_class fp_i2d(vRegD dst, iRegIorL2I src)
 6479 %{
 6480   single_instruction;
 6481   src    : S1(read);
 6482   dst    : S5(write);
 6483   INS01  : ISS;
 6484   NEON_FP : S5;
 6485 %}
 6486 
 6487 pipe_class fp_l2d(vRegD dst, iRegIorL2I src)
 6488 %{
 6489   single_instruction;
 6490   src    : S1(read);
 6491   dst    : S5(write);
 6492   INS01  : ISS;
 6493   NEON_FP : S5;
 6494 %}
 6495 
 6496 pipe_class fp_div_s(vRegF dst, vRegF src1, vRegF src2)
 6497 %{
 6498   single_instruction;
 6499   src1   : S1(read);
 6500   src2   : S2(read);
 6501   dst    : S5(write);
 6502   INS0   : ISS;
 6503   NEON_FP : S5;
 6504 %}
 6505 
 6506 pipe_class fp_div_d(vRegD dst, vRegD src1, vRegD src2)
 6507 %{
 6508   single_instruction;
 6509   src1   : S1(read);
 6510   src2   : S2(read);
 6511   dst    : S5(write);
 6512   INS0   : ISS;
 6513   NEON_FP : S5;
 6514 %}
 6515 
 6516 pipe_class fp_cond_reg_reg_s(vRegF dst, vRegF src1, vRegF src2, rFlagsReg cr)
 6517 %{
 6518   single_instruction;
 6519   cr     : S1(read);
 6520   src1   : S1(read);
 6521   src2   : S1(read);
 6522   dst    : S3(write);
 6523   INS01  : ISS;
 6524   NEON_FP : S3;
 6525 %}
 6526 
 6527 pipe_class fp_cond_reg_reg_d(vRegD dst, vRegD src1, vRegD src2, rFlagsReg cr)
 6528 %{
 6529   single_instruction;
 6530   cr     : S1(read);
 6531   src1   : S1(read);
 6532   src2   : S1(read);
 6533   dst    : S3(write);
 6534   INS01  : ISS;
 6535   NEON_FP : S3;
 6536 %}
 6537 
 6538 pipe_class fp_imm_s(vRegF dst)
 6539 %{
 6540   single_instruction;
 6541   dst    : S3(write);
 6542   INS01  : ISS;
 6543   NEON_FP : S3;
 6544 %}
 6545 
 6546 pipe_class fp_imm_d(vRegD dst)
 6547 %{
 6548   single_instruction;
 6549   dst    : S3(write);
 6550   INS01  : ISS;
 6551   NEON_FP : S3;
 6552 %}
 6553 
 6554 pipe_class fp_load_constant_s(vRegF dst)
 6555 %{
 6556   single_instruction;
 6557   dst    : S4(write);
 6558   INS01  : ISS;
 6559   NEON_FP : S4;
 6560 %}
 6561 
 6562 pipe_class fp_load_constant_d(vRegD dst)
 6563 %{
 6564   single_instruction;
 6565   dst    : S4(write);
 6566   INS01  : ISS;
 6567   NEON_FP : S4;
 6568 %}
 6569 
 6570 //------- Integer ALU operations --------------------------
 6571 
 6572 // Integer ALU reg-reg operation
 6573 // Operands needed in EX1, result generated in EX2
 6574 // Eg.  ADD     x0, x1, x2
 6575 pipe_class ialu_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 6576 %{
 6577   single_instruction;
 6578   dst    : EX2(write);
 6579   src1   : EX1(read);
 6580   src2   : EX1(read);
 6581   INS01  : ISS; // Dual issue as instruction 0 or 1
 6582   ALU    : EX2;
 6583 %}
 6584 
 6585 // Integer ALU reg-reg operation with constant shift
 6586 // Shifted register must be available in LATE_ISS instead of EX1
 6587 // Eg.  ADD     x0, x1, x2, LSL #2
 6588 pipe_class ialu_reg_reg_shift(iRegI dst, iRegI src1, iRegI src2, immI shift)
 6589 %{
 6590   single_instruction;
 6591   dst    : EX2(write);
 6592   src1   : EX1(read);
 6593   src2   : ISS(read);
 6594   INS01  : ISS;
 6595   ALU    : EX2;
 6596 %}
 6597 
 6598 // Integer ALU reg operation with constant shift
 6599 // Eg.  LSL     x0, x1, #shift
 6600 pipe_class ialu_reg_shift(iRegI dst, iRegI src1)
 6601 %{
 6602   single_instruction;
 6603   dst    : EX2(write);
 6604   src1   : ISS(read);
 6605   INS01  : ISS;
 6606   ALU    : EX2;
 6607 %}
 6608 
 6609 // Integer ALU reg-reg operation with variable shift
 6610 // Both operands must be available in LATE_ISS instead of EX1
 6611 // Result is available in EX1 instead of EX2
 6612 // Eg.  LSLV    x0, x1, x2
 6613 pipe_class ialu_reg_reg_vshift(iRegI dst, iRegI src1, iRegI src2)
 6614 %{
 6615   single_instruction;
 6616   dst    : EX1(write);
 6617   src1   : ISS(read);
 6618   src2   : ISS(read);
 6619   INS01  : ISS;
 6620   ALU    : EX1;
 6621 %}
 6622 
 6623 // Integer ALU reg-reg operation with extract
 6624 // As for _vshift above, but result generated in EX2
 6625 // Eg.  EXTR    x0, x1, x2, #N
 6626 pipe_class ialu_reg_reg_extr(iRegI dst, iRegI src1, iRegI src2)
 6627 %{
 6628   single_instruction;
 6629   dst    : EX2(write);
 6630   src1   : ISS(read);
 6631   src2   : ISS(read);
 6632   INS1   : ISS; // Can only dual issue as Instruction 1
 6633   ALU    : EX1;
 6634 %}
 6635 
 6636 // Integer ALU reg operation
 6637 // Eg.  NEG     x0, x1
 6638 pipe_class ialu_reg(iRegI dst, iRegI src)
 6639 %{
 6640   single_instruction;
 6641   dst    : EX2(write);
 6642   src    : EX1(read);
 6643   INS01  : ISS;
 6644   ALU    : EX2;
 6645 %}
 6646 
 6647 // Integer ALU reg mmediate operation
 6648 // Eg.  ADD     x0, x1, #N
 6649 pipe_class ialu_reg_imm(iRegI dst, iRegI src1)
 6650 %{
 6651   single_instruction;
 6652   dst    : EX2(write);
 6653   src1   : EX1(read);
 6654   INS01  : ISS;
 6655   ALU    : EX2;
 6656 %}
 6657 
 6658 // Integer ALU immediate operation (no source operands)
 6659 // Eg.  MOV     x0, #N
 6660 pipe_class ialu_imm(iRegI dst)
 6661 %{
 6662   single_instruction;
 6663   dst    : EX1(write);
 6664   INS01  : ISS;
 6665   ALU    : EX1;
 6666 %}
 6667 
 6668 //------- Compare operation -------------------------------
 6669 
 6670 // Compare reg-reg
 6671 // Eg.  CMP     x0, x1
 6672 pipe_class icmp_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
 6673 %{
 6674   single_instruction;
 6675 //  fixed_latency(16);
 6676   cr     : EX2(write);
 6677   op1    : EX1(read);
 6678   op2    : EX1(read);
 6679   INS01  : ISS;
 6680   ALU    : EX2;
 6681 %}
 6682 
 6683 // Compare reg-reg
 6684 // Eg.  CMP     x0, #N
 6685 pipe_class icmp_reg_imm(rFlagsReg cr, iRegI op1)
 6686 %{
 6687   single_instruction;
 6688 //  fixed_latency(16);
 6689   cr     : EX2(write);
 6690   op1    : EX1(read);
 6691   INS01  : ISS;
 6692   ALU    : EX2;
 6693 %}
 6694 
 6695 //------- Conditional instructions ------------------------
 6696 
 6697 // Conditional no operands
 6698 // Eg.  CSINC   x0, zr, zr, <cond>
 6699 pipe_class icond_none(iRegI dst, rFlagsReg cr)
 6700 %{
 6701   single_instruction;
 6702   cr     : EX1(read);
 6703   dst    : EX2(write);
 6704   INS01  : ISS;
 6705   ALU    : EX2;
 6706 %}
 6707 
 6708 // Conditional 2 operand
 6709 // EG.  CSEL    X0, X1, X2, <cond>
 6710 pipe_class icond_reg_reg(iRegI dst, iRegI src1, iRegI src2, rFlagsReg cr)
 6711 %{
 6712   single_instruction;
 6713   cr     : EX1(read);
 6714   src1   : EX1(read);
 6715   src2   : EX1(read);
 6716   dst    : EX2(write);
 6717   INS01  : ISS;
 6718   ALU    : EX2;
 6719 %}
 6720 
 6721 // Conditional 2 operand
 6722 // EG.  CSEL    X0, X1, X2, <cond>
 6723 pipe_class icond_reg(iRegI dst, iRegI src, rFlagsReg cr)
 6724 %{
 6725   single_instruction;
 6726   cr     : EX1(read);
 6727   src    : EX1(read);
 6728   dst    : EX2(write);
 6729   INS01  : ISS;
 6730   ALU    : EX2;
 6731 %}
 6732 
 6733 //------- Multiply pipeline operations --------------------
 6734 
 6735 // Multiply reg-reg
 6736 // Eg.  MUL     w0, w1, w2
 6737 pipe_class imul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 6738 %{
 6739   single_instruction;
 6740   dst    : WR(write);
 6741   src1   : ISS(read);
 6742   src2   : ISS(read);
 6743   INS01  : ISS;
 6744   MAC    : WR;
 6745 %}
 6746 
 6747 // Multiply accumulate
 6748 // Eg.  MADD    w0, w1, w2, w3
 6749 pipe_class imac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
 6750 %{
 6751   single_instruction;
 6752   dst    : WR(write);
 6753   src1   : ISS(read);
 6754   src2   : ISS(read);
 6755   src3   : ISS(read);
 6756   INS01  : ISS;
 6757   MAC    : WR;
 6758 %}
 6759 
 6760 // Eg.  MUL     w0, w1, w2
 6761 pipe_class lmul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 6762 %{
 6763   single_instruction;
 6764   fixed_latency(3); // Maximum latency for 64 bit mul
 6765   dst    : WR(write);
 6766   src1   : ISS(read);
 6767   src2   : ISS(read);
 6768   INS01  : ISS;
 6769   MAC    : WR;
 6770 %}
 6771 
 6772 // Multiply accumulate
 6773 // Eg.  MADD    w0, w1, w2, w3
 6774 pipe_class lmac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
 6775 %{
 6776   single_instruction;
 6777   fixed_latency(3); // Maximum latency for 64 bit mul
 6778   dst    : WR(write);
 6779   src1   : ISS(read);
 6780   src2   : ISS(read);
 6781   src3   : ISS(read);
 6782   INS01  : ISS;
 6783   MAC    : WR;
 6784 %}
 6785 
 6786 //------- Divide pipeline operations --------------------
 6787 
 6788 // Eg.  SDIV    w0, w1, w2
 6789 pipe_class idiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 6790 %{
 6791   single_instruction;
 6792   fixed_latency(8); // Maximum latency for 32 bit divide
 6793   dst    : WR(write);
 6794   src1   : ISS(read);
 6795   src2   : ISS(read);
 6796   INS0   : ISS; // Can only dual issue as instruction 0
 6797   DIV    : WR;
 6798 %}
 6799 
 6800 // Eg.  SDIV    x0, x1, x2
 6801 pipe_class ldiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 6802 %{
 6803   single_instruction;
 6804   fixed_latency(16); // Maximum latency for 64 bit divide
 6805   dst    : WR(write);
 6806   src1   : ISS(read);
 6807   src2   : ISS(read);
 6808   INS0   : ISS; // Can only dual issue as instruction 0
 6809   DIV    : WR;
 6810 %}
 6811 
 6812 //------- Load pipeline operations ------------------------
 6813 
 6814 // Load - prefetch
 6815 // Eg.  PFRM    <mem>
 6816 pipe_class iload_prefetch(memory mem)
 6817 %{
 6818   single_instruction;
 6819   mem    : ISS(read);
 6820   INS01  : ISS;
 6821   LDST   : WR;
 6822 %}
 6823 
 6824 // Load - reg, mem
 6825 // Eg.  LDR     x0, <mem>
 6826 pipe_class iload_reg_mem(iRegI dst, memory mem)
 6827 %{
 6828   single_instruction;
 6829   dst    : WR(write);
 6830   mem    : ISS(read);
 6831   INS01  : ISS;
 6832   LDST   : WR;
 6833 %}
 6834 
 6835 // Load - reg, reg
 6836 // Eg.  LDR     x0, [sp, x1]
 6837 pipe_class iload_reg_reg(iRegI dst, iRegI src)
 6838 %{
 6839   single_instruction;
 6840   dst    : WR(write);
 6841   src    : ISS(read);
 6842   INS01  : ISS;
 6843   LDST   : WR;
 6844 %}
 6845 
 6846 //------- Store pipeline operations -----------------------
 6847 
 6848 // Store - zr, mem
 6849 // Eg.  STR     zr, <mem>
 6850 pipe_class istore_mem(memory mem)
 6851 %{
 6852   single_instruction;
 6853   mem    : ISS(read);
 6854   INS01  : ISS;
 6855   LDST   : WR;
 6856 %}
 6857 
 6858 // Store - reg, mem
 6859 // Eg.  STR     x0, <mem>
 6860 pipe_class istore_reg_mem(iRegI src, memory mem)
 6861 %{
 6862   single_instruction;
 6863   mem    : ISS(read);
 6864   src    : EX2(read);
 6865   INS01  : ISS;
 6866   LDST   : WR;
 6867 %}
 6868 
 6869 // Store - reg, reg
 6870 // Eg. STR      x0, [sp, x1]
 6871 pipe_class istore_reg_reg(iRegI dst, iRegI src)
 6872 %{
 6873   single_instruction;
 6874   dst    : ISS(read);
 6875   src    : EX2(read);
 6876   INS01  : ISS;
 6877   LDST   : WR;
 6878 %}
 6879 
 6880 //------- Store pipeline operations -----------------------
 6881 
 6882 // Branch
 6883 pipe_class pipe_branch()
 6884 %{
 6885   single_instruction;
 6886   INS01  : ISS;
 6887   BRANCH : EX1;
 6888 %}
 6889 
 6890 // Conditional branch
 6891 pipe_class pipe_branch_cond(rFlagsReg cr)
 6892 %{
 6893   single_instruction;
 6894   cr     : EX1(read);
 6895   INS01  : ISS;
 6896   BRANCH : EX1;
 6897 %}
 6898 
 6899 // Compare & Branch
 6900 // EG.  CBZ/CBNZ
 6901 pipe_class pipe_cmp_branch(iRegI op1)
 6902 %{
 6903   single_instruction;
 6904   op1    : EX1(read);
 6905   INS01  : ISS;
 6906   BRANCH : EX1;
 6907 %}
 6908 
 6909 //------- Synchronisation operations ----------------------
 6910 
 6911 // Any operation requiring serialization.
 6912 // EG.  DMB/Atomic Ops/Load Acquire/Str Release
 6913 pipe_class pipe_serial()
 6914 %{
 6915   single_instruction;
 6916   force_serialization;
 6917   fixed_latency(16);
 6918   INS01  : ISS(2); // Cannot dual issue with any other instruction
 6919   LDST   : WR;
 6920 %}
 6921 
 6922 // Generic big/slow expanded idiom - also serialized
 6923 pipe_class pipe_slow()
 6924 %{
 6925   instruction_count(10);
 6926   multiple_bundles;
 6927   force_serialization;
 6928   fixed_latency(16);
 6929   INS01  : ISS(2); // Cannot dual issue with any other instruction
 6930   LDST   : WR;
 6931 %}
 6932 
 6933 // Empty pipeline class
 6934 pipe_class pipe_class_empty()
 6935 %{
 6936   single_instruction;
 6937   fixed_latency(0);
 6938 %}
 6939 
 6940 // Default pipeline class.
 6941 pipe_class pipe_class_default()
 6942 %{
 6943   single_instruction;
 6944   fixed_latency(2);
 6945 %}
 6946 
 6947 // Pipeline class for compares.
 6948 pipe_class pipe_class_compare()
 6949 %{
 6950   single_instruction;
 6951   fixed_latency(16);
 6952 %}
 6953 
 6954 // Pipeline class for memory operations.
 6955 pipe_class pipe_class_memory()
 6956 %{
 6957   single_instruction;
 6958   fixed_latency(16);
 6959 %}
 6960 
 6961 // Pipeline class for call.
 6962 pipe_class pipe_class_call()
 6963 %{
 6964   single_instruction;
 6965   fixed_latency(100);
 6966 %}
 6967 
 6968 // Define the class for the Nop node.
 6969 define %{
 6970    MachNop = pipe_class_empty;
 6971 %}
 6972 
 6973 %}
 6974 //----------INSTRUCTIONS-------------------------------------------------------
 6975 //
 6976 // match      -- States which machine-independent subtree may be replaced
 6977 //               by this instruction.
 6978 // ins_cost   -- The estimated cost of this instruction is used by instruction
 6979 //               selection to identify a minimum cost tree of machine
 6980 //               instructions that matches a tree of machine-independent
 6981 //               instructions.
 6982 // format     -- A string providing the disassembly for this instruction.
 6983 //               The value of an instruction's operand may be inserted
 6984 //               by referring to it with a '$' prefix.
 6985 // opcode     -- Three instruction opcodes may be provided.  These are referred
 6986 //               to within an encode class as $primary, $secondary, and $tertiary
 6987 //               rrspectively.  The primary opcode is commonly used to
 6988 //               indicate the type of machine instruction, while secondary
 6989 //               and tertiary are often used for prefix options or addressing
 6990 //               modes.
 6991 // ins_encode -- A list of encode classes with parameters. The encode class
 6992 //               name must have been defined in an 'enc_class' specification
 6993 //               in the encode section of the architecture description.
 6994 
 6995 // ============================================================================
 6996 // Memory (Load/Store) Instructions
 6997 
 6998 // Load Instructions
 6999 
 7000 // Load Byte (8 bit signed)
 7001 instruct loadB(iRegINoSp dst, memory1 mem)
 7002 %{
 7003   match(Set dst (LoadB mem));
 7004   predicate(!needs_acquiring_load(n));
 7005 
 7006   ins_cost(4 * INSN_COST);
 7007   format %{ "ldrsbw  $dst, $mem\t# byte" %}
 7008 
 7009   ins_encode(aarch64_enc_ldrsbw(dst, mem));
 7010 
 7011   ins_pipe(iload_reg_mem);
 7012 %}
 7013 
 7014 // Load Byte (8 bit signed) into long
 7015 instruct loadB2L(iRegLNoSp dst, memory1 mem)
 7016 %{
 7017   match(Set dst (ConvI2L (LoadB mem)));
 7018   predicate(!needs_acquiring_load(n->in(1)));
 7019 
 7020   ins_cost(4 * INSN_COST);
 7021   format %{ "ldrsb  $dst, $mem\t# byte" %}
 7022 
 7023   ins_encode(aarch64_enc_ldrsb(dst, mem));
 7024 
 7025   ins_pipe(iload_reg_mem);
 7026 %}
 7027 
 7028 // Load Byte (8 bit unsigned)
 7029 instruct loadUB(iRegINoSp dst, memory1 mem)
 7030 %{
 7031   match(Set dst (LoadUB mem));
 7032   predicate(!needs_acquiring_load(n));
 7033 
 7034   ins_cost(4 * INSN_COST);
 7035   format %{ "ldrbw  $dst, $mem\t# byte" %}
 7036 
 7037   ins_encode(aarch64_enc_ldrb(dst, mem));
 7038 
 7039   ins_pipe(iload_reg_mem);
 7040 %}
 7041 
 7042 // Load Byte (8 bit unsigned) into long
 7043 instruct loadUB2L(iRegLNoSp dst, memory1 mem)
 7044 %{
 7045   match(Set dst (ConvI2L (LoadUB mem)));
 7046   predicate(!needs_acquiring_load(n->in(1)));
 7047 
 7048   ins_cost(4 * INSN_COST);
 7049   format %{ "ldrb  $dst, $mem\t# byte" %}
 7050 
 7051   ins_encode(aarch64_enc_ldrb(dst, mem));
 7052 
 7053   ins_pipe(iload_reg_mem);
 7054 %}
 7055 
 7056 // Load Short (16 bit signed)
 7057 instruct loadS(iRegINoSp dst, memory2 mem)
 7058 %{
 7059   match(Set dst (LoadS mem));
 7060   predicate(!needs_acquiring_load(n));
 7061 
 7062   ins_cost(4 * INSN_COST);
 7063   format %{ "ldrshw  $dst, $mem\t# short" %}
 7064 
 7065   ins_encode(aarch64_enc_ldrshw(dst, mem));
 7066 
 7067   ins_pipe(iload_reg_mem);
 7068 %}
 7069 
 7070 // Load Short (16 bit signed) into long
 7071 instruct loadS2L(iRegLNoSp dst, memory2 mem)
 7072 %{
 7073   match(Set dst (ConvI2L (LoadS mem)));
 7074   predicate(!needs_acquiring_load(n->in(1)));
 7075 
 7076   ins_cost(4 * INSN_COST);
 7077   format %{ "ldrsh  $dst, $mem\t# short" %}
 7078 
 7079   ins_encode(aarch64_enc_ldrsh(dst, mem));
 7080 
 7081   ins_pipe(iload_reg_mem);
 7082 %}
 7083 
 7084 // Load Char (16 bit unsigned)
 7085 instruct loadUS(iRegINoSp dst, memory2 mem)
 7086 %{
 7087   match(Set dst (LoadUS mem));
 7088   predicate(!needs_acquiring_load(n));
 7089 
 7090   ins_cost(4 * INSN_COST);
 7091   format %{ "ldrh  $dst, $mem\t# short" %}
 7092 
 7093   ins_encode(aarch64_enc_ldrh(dst, mem));
 7094 
 7095   ins_pipe(iload_reg_mem);
 7096 %}
 7097 
 7098 // Load Short/Char (16 bit unsigned) into long
 7099 instruct loadUS2L(iRegLNoSp dst, memory2 mem)
 7100 %{
 7101   match(Set dst (ConvI2L (LoadUS mem)));
 7102   predicate(!needs_acquiring_load(n->in(1)));
 7103 
 7104   ins_cost(4 * INSN_COST);
 7105   format %{ "ldrh  $dst, $mem\t# short" %}
 7106 
 7107   ins_encode(aarch64_enc_ldrh(dst, mem));
 7108 
 7109   ins_pipe(iload_reg_mem);
 7110 %}
 7111 
 7112 // Load Integer (32 bit signed)
 7113 instruct loadI(iRegINoSp dst, memory4 mem)
 7114 %{
 7115   match(Set dst (LoadI mem));
 7116   predicate(!needs_acquiring_load(n));
 7117 
 7118   ins_cost(4 * INSN_COST);
 7119   format %{ "ldrw  $dst, $mem\t# int" %}
 7120 
 7121   ins_encode(aarch64_enc_ldrw(dst, mem));
 7122 
 7123   ins_pipe(iload_reg_mem);
 7124 %}
 7125 
 7126 // Load Integer (32 bit signed) into long
 7127 instruct loadI2L(iRegLNoSp dst, memory4 mem)
 7128 %{
 7129   match(Set dst (ConvI2L (LoadI mem)));
 7130   predicate(!needs_acquiring_load(n->in(1)));
 7131 
 7132   ins_cost(4 * INSN_COST);
 7133   format %{ "ldrsw  $dst, $mem\t# int" %}
 7134 
 7135   ins_encode(aarch64_enc_ldrsw(dst, mem));
 7136 
 7137   ins_pipe(iload_reg_mem);
 7138 %}
 7139 
 7140 // Load Integer (32 bit unsigned) into long
 7141 instruct loadUI2L(iRegLNoSp dst, memory4 mem, immL_32bits mask)
 7142 %{
 7143   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
 7144   predicate(!needs_acquiring_load(n->in(1)->in(1)->as_Load()));
 7145 
 7146   ins_cost(4 * INSN_COST);
 7147   format %{ "ldrw  $dst, $mem\t# int" %}
 7148 
 7149   ins_encode(aarch64_enc_ldrw(dst, mem));
 7150 
 7151   ins_pipe(iload_reg_mem);
 7152 %}
 7153 
 7154 // Load Long (64 bit signed)
 7155 instruct loadL(iRegLNoSp dst, memory8 mem)
 7156 %{
 7157   match(Set dst (LoadL mem));
 7158   predicate(!needs_acquiring_load(n));
 7159 
 7160   ins_cost(4 * INSN_COST);
 7161   format %{ "ldr  $dst, $mem\t# int" %}
 7162 
 7163   ins_encode(aarch64_enc_ldr(dst, mem));
 7164 
 7165   ins_pipe(iload_reg_mem);
 7166 %}
 7167 
 7168 // Load Range
 7169 instruct loadRange(iRegINoSp dst, memory4 mem)
 7170 %{
 7171   match(Set dst (LoadRange mem));
 7172 
 7173   ins_cost(4 * INSN_COST);
 7174   format %{ "ldrw  $dst, $mem\t# range" %}
 7175 
 7176   ins_encode(aarch64_enc_ldrw(dst, mem));
 7177 
 7178   ins_pipe(iload_reg_mem);
 7179 %}
 7180 
 7181 // Load Pointer
 7182 instruct loadP(iRegPNoSp dst, memory8 mem)
 7183 %{
 7184   match(Set dst (LoadP mem));
 7185   predicate(!needs_acquiring_load(n) && (n->as_Load()->barrier_data() == 0));
 7186 
 7187   ins_cost(4 * INSN_COST);
 7188   format %{ "ldr  $dst, $mem\t# ptr" %}
 7189 
 7190   ins_encode(aarch64_enc_ldr(dst, mem));
 7191 
 7192   ins_pipe(iload_reg_mem);
 7193 %}
 7194 
 7195 // Load Compressed Pointer
 7196 instruct loadN(iRegNNoSp dst, memory4 mem)
 7197 %{
 7198   match(Set dst (LoadN mem));
 7199   predicate(!needs_acquiring_load(n));
 7200 
 7201   ins_cost(4 * INSN_COST);
 7202   format %{ "ldrw  $dst, $mem\t# compressed ptr" %}
 7203 
 7204   ins_encode(aarch64_enc_ldrw(dst, mem));
 7205 
 7206   ins_pipe(iload_reg_mem);
 7207 %}
 7208 
 7209 // Load Klass Pointer
 7210 instruct loadKlass(iRegPNoSp dst, memory8 mem)
 7211 %{
 7212   match(Set dst (LoadKlass mem));
 7213   predicate(!needs_acquiring_load(n));
 7214 
 7215   ins_cost(4 * INSN_COST);
 7216   format %{ "ldr  $dst, $mem\t# class" %}
 7217 
 7218   ins_encode(aarch64_enc_ldr(dst, mem));
 7219 
 7220   ins_pipe(iload_reg_mem);
 7221 %}
 7222 
 7223 // Load Narrow Klass Pointer
 7224 instruct loadNKlass(iRegNNoSp dst, memory4 mem)
 7225 %{
 7226   match(Set dst (LoadNKlass mem));
 7227   predicate(!needs_acquiring_load(n));
 7228 
 7229   ins_cost(4 * INSN_COST);
 7230   format %{ "ldrw  $dst, $mem\t# compressed class ptr" %}
 7231 
 7232   ins_encode(aarch64_enc_ldrw(dst, mem));
 7233 
 7234   ins_pipe(iload_reg_mem);
 7235 %}
 7236 
 7237 // Load Float
 7238 instruct loadF(vRegF dst, memory4 mem)
 7239 %{
 7240   match(Set dst (LoadF mem));
 7241   predicate(!needs_acquiring_load(n));
 7242 
 7243   ins_cost(4 * INSN_COST);
 7244   format %{ "ldrs  $dst, $mem\t# float" %}
 7245 
 7246   ins_encode( aarch64_enc_ldrs(dst, mem) );
 7247 
 7248   ins_pipe(pipe_class_memory);
 7249 %}
 7250 
 7251 // Load Double
 7252 instruct loadD(vRegD dst, memory8 mem)
 7253 %{
 7254   match(Set dst (LoadD mem));
 7255   predicate(!needs_acquiring_load(n));
 7256 
 7257   ins_cost(4 * INSN_COST);
 7258   format %{ "ldrd  $dst, $mem\t# double" %}
 7259 
 7260   ins_encode( aarch64_enc_ldrd(dst, mem) );
 7261 
 7262   ins_pipe(pipe_class_memory);
 7263 %}
 7264 
 7265 
 7266 // Load Int Constant
 7267 instruct loadConI(iRegINoSp dst, immI src)
 7268 %{
 7269   match(Set dst src);
 7270 
 7271   ins_cost(INSN_COST);
 7272   format %{ "mov $dst, $src\t# int" %}
 7273 
 7274   ins_encode( aarch64_enc_movw_imm(dst, src) );
 7275 
 7276   ins_pipe(ialu_imm);
 7277 %}
 7278 
 7279 // Load Long Constant
 7280 instruct loadConL(iRegLNoSp dst, immL src)
 7281 %{
 7282   match(Set dst src);
 7283 
 7284   ins_cost(INSN_COST);
 7285   format %{ "mov $dst, $src\t# long" %}
 7286 
 7287   ins_encode( aarch64_enc_mov_imm(dst, src) );
 7288 
 7289   ins_pipe(ialu_imm);
 7290 %}
 7291 
 7292 // Load Pointer Constant
 7293 
 7294 instruct loadConP(iRegPNoSp dst, immP con)
 7295 %{
 7296   match(Set dst con);
 7297 
 7298   ins_cost(INSN_COST * 4);
 7299   format %{
 7300     "mov  $dst, $con\t# ptr"
 7301   %}
 7302 
 7303   ins_encode(aarch64_enc_mov_p(dst, con));
 7304 
 7305   ins_pipe(ialu_imm);
 7306 %}
 7307 
 7308 // Load Null Pointer Constant
 7309 
 7310 instruct loadConP0(iRegPNoSp dst, immP0 con)
 7311 %{
 7312   match(Set dst con);
 7313 
 7314   ins_cost(INSN_COST);
 7315   format %{ "mov  $dst, $con\t# NULL ptr" %}
 7316 
 7317   ins_encode(aarch64_enc_mov_p0(dst, con));
 7318 
 7319   ins_pipe(ialu_imm);
 7320 %}
 7321 
 7322 // Load Pointer Constant One
 7323 
 7324 instruct loadConP1(iRegPNoSp dst, immP_1 con)
 7325 %{
 7326   match(Set dst con);
 7327 
 7328   ins_cost(INSN_COST);
 7329   format %{ "mov  $dst, $con\t# NULL ptr" %}
 7330 
 7331   ins_encode(aarch64_enc_mov_p1(dst, con));
 7332 
 7333   ins_pipe(ialu_imm);
 7334 %}
 7335 
 7336 // Load Byte Map Base Constant
 7337 
 7338 instruct loadByteMapBase(iRegPNoSp dst, immByteMapBase con)
 7339 %{
 7340   match(Set dst con);
 7341 
 7342   ins_cost(INSN_COST);
 7343   format %{ "adr  $dst, $con\t# Byte Map Base" %}
 7344 
 7345   ins_encode(aarch64_enc_mov_byte_map_base(dst, con));
 7346 
 7347   ins_pipe(ialu_imm);
 7348 %}
 7349 
 7350 // Load Narrow Pointer Constant
 7351 
 7352 instruct loadConN(iRegNNoSp dst, immN con)
 7353 %{
 7354   match(Set dst con);
 7355 
 7356   ins_cost(INSN_COST * 4);
 7357   format %{ "mov  $dst, $con\t# compressed ptr" %}
 7358 
 7359   ins_encode(aarch64_enc_mov_n(dst, con));
 7360 
 7361   ins_pipe(ialu_imm);
 7362 %}
 7363 
 7364 // Load Narrow Null Pointer Constant
 7365 
 7366 instruct loadConN0(iRegNNoSp dst, immN0 con)
 7367 %{
 7368   match(Set dst con);
 7369 
 7370   ins_cost(INSN_COST);
 7371   format %{ "mov  $dst, $con\t# compressed NULL ptr" %}
 7372 
 7373   ins_encode(aarch64_enc_mov_n0(dst, con));
 7374 
 7375   ins_pipe(ialu_imm);
 7376 %}
 7377 
 7378 // Load Narrow Klass Constant
 7379 
 7380 instruct loadConNKlass(iRegNNoSp dst, immNKlass con)
 7381 %{
 7382   match(Set dst con);
 7383 
 7384   ins_cost(INSN_COST);
 7385   format %{ "mov  $dst, $con\t# compressed klass ptr" %}
 7386 
 7387   ins_encode(aarch64_enc_mov_nk(dst, con));
 7388 
 7389   ins_pipe(ialu_imm);
 7390 %}
 7391 
 7392 // Load Packed Float Constant
 7393 
 7394 instruct loadConF_packed(vRegF dst, immFPacked con) %{
 7395   match(Set dst con);
 7396   ins_cost(INSN_COST * 4);
 7397   format %{ "fmovs  $dst, $con"%}
 7398   ins_encode %{
 7399     __ fmovs(as_FloatRegister($dst$$reg), (double)$con$$constant);
 7400   %}
 7401 
 7402   ins_pipe(fp_imm_s);
 7403 %}
 7404 
 7405 // Load Float Constant
 7406 
 7407 instruct loadConF(vRegF dst, immF con) %{
 7408   match(Set dst con);
 7409 
 7410   ins_cost(INSN_COST * 4);
 7411 
 7412   format %{
 7413     "ldrs $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
 7414   %}
 7415 
 7416   ins_encode %{
 7417     __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
 7418   %}
 7419 
 7420   ins_pipe(fp_load_constant_s);
 7421 %}
 7422 
 7423 // Load Packed Double Constant
 7424 
 7425 instruct loadConD_packed(vRegD dst, immDPacked con) %{
 7426   match(Set dst con);
 7427   ins_cost(INSN_COST);
 7428   format %{ "fmovd  $dst, $con"%}
 7429   ins_encode %{
 7430     __ fmovd(as_FloatRegister($dst$$reg), $con$$constant);
 7431   %}
 7432 
 7433   ins_pipe(fp_imm_d);
 7434 %}
 7435 
 7436 // Load Double Constant
 7437 
 7438 instruct loadConD(vRegD dst, immD con) %{
 7439   match(Set dst con);
 7440 
 7441   ins_cost(INSN_COST * 5);
 7442   format %{
 7443     "ldrd $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
 7444   %}
 7445 
 7446   ins_encode %{
 7447     __ ldrd(as_FloatRegister($dst$$reg), $constantaddress($con));
 7448   %}
 7449 
 7450   ins_pipe(fp_load_constant_d);
 7451 %}
 7452 
 7453 // Store Instructions
 7454 
 7455 // Store CMS card-mark Immediate
 7456 instruct storeimmCM0(immI0 zero, memory1 mem)
 7457 %{
 7458   match(Set mem (StoreCM mem zero));
 7459 
 7460   ins_cost(INSN_COST);
 7461   format %{ "storestore (elided)\n\t"
 7462             "strb zr, $mem\t# byte" %}
 7463 
 7464   ins_encode(aarch64_enc_strb0(mem));
 7465 
 7466   ins_pipe(istore_mem);
 7467 %}
 7468 
 7469 // Store CMS card-mark Immediate with intervening StoreStore
 7470 // needed when using CMS with no conditional card marking
 7471 instruct storeimmCM0_ordered(immI0 zero, memory1 mem)
 7472 %{
 7473   match(Set mem (StoreCM mem zero));
 7474 
 7475   ins_cost(INSN_COST * 2);
 7476   format %{ "storestore\n\t"
 7477             "dmb ishst"
 7478             "\n\tstrb zr, $mem\t# byte" %}
 7479 
 7480   ins_encode(aarch64_enc_strb0_ordered(mem));
 7481 
 7482   ins_pipe(istore_mem);
 7483 %}
 7484 
 7485 // Store Byte
 7486 instruct storeB(iRegIorL2I src, memory1 mem)
 7487 %{
 7488   match(Set mem (StoreB mem src));
 7489   predicate(!needs_releasing_store(n));
 7490 
 7491   ins_cost(INSN_COST);
 7492   format %{ "strb  $src, $mem\t# byte" %}
 7493 
 7494   ins_encode(aarch64_enc_strb(src, mem));
 7495 
 7496   ins_pipe(istore_reg_mem);
 7497 %}
 7498 
 7499 
 7500 instruct storeimmB0(immI0 zero, memory1 mem)
 7501 %{
 7502   match(Set mem (StoreB mem zero));
 7503   predicate(!needs_releasing_store(n));
 7504 
 7505   ins_cost(INSN_COST);
 7506   format %{ "strb rscractch2, $mem\t# byte" %}
 7507 
 7508   ins_encode(aarch64_enc_strb0(mem));
 7509 
 7510   ins_pipe(istore_mem);
 7511 %}
 7512 
 7513 // Store Char/Short
 7514 instruct storeC(iRegIorL2I src, memory2 mem)
 7515 %{
 7516   match(Set mem (StoreC mem src));
 7517   predicate(!needs_releasing_store(n));
 7518 
 7519   ins_cost(INSN_COST);
 7520   format %{ "strh  $src, $mem\t# short" %}
 7521 
 7522   ins_encode(aarch64_enc_strh(src, mem));
 7523 
 7524   ins_pipe(istore_reg_mem);
 7525 %}
 7526 
 7527 instruct storeimmC0(immI0 zero, memory2 mem)
 7528 %{
 7529   match(Set mem (StoreC mem zero));
 7530   predicate(!needs_releasing_store(n));
 7531 
 7532   ins_cost(INSN_COST);
 7533   format %{ "strh  zr, $mem\t# short" %}
 7534 
 7535   ins_encode(aarch64_enc_strh0(mem));
 7536 
 7537   ins_pipe(istore_mem);
 7538 %}
 7539 
 7540 // Store Integer
 7541 
 7542 instruct storeI(iRegIorL2I src, memory4 mem)
 7543 %{
 7544   match(Set mem(StoreI mem src));
 7545   predicate(!needs_releasing_store(n));
 7546 
 7547   ins_cost(INSN_COST);
 7548   format %{ "strw  $src, $mem\t# int" %}
 7549 
 7550   ins_encode(aarch64_enc_strw(src, mem));
 7551 
 7552   ins_pipe(istore_reg_mem);
 7553 %}
 7554 
 7555 instruct storeimmI0(immI0 zero, memory4 mem)
 7556 %{
 7557   match(Set mem(StoreI mem zero));
 7558   predicate(!needs_releasing_store(n));
 7559 
 7560   ins_cost(INSN_COST);
 7561   format %{ "strw  zr, $mem\t# int" %}
 7562 
 7563   ins_encode(aarch64_enc_strw0(mem));
 7564 
 7565   ins_pipe(istore_mem);
 7566 %}
 7567 
 7568 // Store Long (64 bit signed)
 7569 instruct storeL(iRegL src, memory8 mem)
 7570 %{
 7571   match(Set mem (StoreL mem src));
 7572   predicate(!needs_releasing_store(n));
 7573 
 7574   ins_cost(INSN_COST);
 7575   format %{ "str  $src, $mem\t# int" %}
 7576 
 7577   ins_encode(aarch64_enc_str(src, mem));
 7578 
 7579   ins_pipe(istore_reg_mem);
 7580 %}
 7581 
 7582 // Store Long (64 bit signed)
 7583 instruct storeimmL0(immL0 zero, memory8 mem)
 7584 %{
 7585   match(Set mem (StoreL mem zero));
 7586   predicate(!needs_releasing_store(n));
 7587 
 7588   ins_cost(INSN_COST);
 7589   format %{ "str  zr, $mem\t# int" %}
 7590 
 7591   ins_encode(aarch64_enc_str0(mem));
 7592 
 7593   ins_pipe(istore_mem);
 7594 %}
 7595 
 7596 // Store Pointer
 7597 instruct storeP(iRegP src, memory8 mem)
 7598 %{
 7599   match(Set mem (StoreP mem src));
 7600   predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
 7601 
 7602   ins_cost(INSN_COST);
 7603   format %{ "str  $src, $mem\t# ptr" %}
 7604 
 7605   ins_encode(aarch64_enc_str(src, mem));
 7606 
 7607   ins_pipe(istore_reg_mem);
 7608 %}
 7609 
 7610 // Store Pointer
 7611 instruct storeimmP0(immP0 zero, memory8 mem)
 7612 %{
 7613   match(Set mem (StoreP mem zero));
 7614   predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
 7615 
 7616   ins_cost(INSN_COST);
 7617   format %{ "str zr, $mem\t# ptr" %}
 7618 
 7619   ins_encode(aarch64_enc_str0(mem));
 7620 
 7621   ins_pipe(istore_mem);
 7622 %}
 7623 
 7624 // Store Compressed Pointer
 7625 instruct storeN(iRegN src, memory4 mem)
 7626 %{
 7627   match(Set mem (StoreN mem src));
 7628   predicate(!needs_releasing_store(n));
 7629 
 7630   ins_cost(INSN_COST);
 7631   format %{ "strw  $src, $mem\t# compressed ptr" %}
 7632 
 7633   ins_encode(aarch64_enc_strw(src, mem));
 7634 
 7635   ins_pipe(istore_reg_mem);
 7636 %}
 7637 
 7638 instruct storeImmN0(immN0 zero, memory4 mem)
 7639 %{
 7640   match(Set mem (StoreN mem zero));
 7641   predicate(!needs_releasing_store(n));
 7642 
 7643   ins_cost(INSN_COST);
 7644   format %{ "strw  zr, $mem\t# compressed ptr" %}
 7645 
 7646   ins_encode(aarch64_enc_strw0(mem));
 7647 
 7648   ins_pipe(istore_mem);
 7649 %}
 7650 
 7651 // Store Float
 7652 instruct storeF(vRegF src, memory4 mem)
 7653 %{
 7654   match(Set mem (StoreF mem src));
 7655   predicate(!needs_releasing_store(n));
 7656 
 7657   ins_cost(INSN_COST);
 7658   format %{ "strs  $src, $mem\t# float" %}
 7659 
 7660   ins_encode( aarch64_enc_strs(src, mem) );
 7661 
 7662   ins_pipe(pipe_class_memory);
 7663 %}
 7664 
 7665 // TODO
 7666 // implement storeImmF0 and storeFImmPacked
 7667 
 7668 // Store Double
 7669 instruct storeD(vRegD src, memory8 mem)
 7670 %{
 7671   match(Set mem (StoreD mem src));
 7672   predicate(!needs_releasing_store(n));
 7673 
 7674   ins_cost(INSN_COST);
 7675   format %{ "strd  $src, $mem\t# double" %}
 7676 
 7677   ins_encode( aarch64_enc_strd(src, mem) );
 7678 
 7679   ins_pipe(pipe_class_memory);
 7680 %}
 7681 
 7682 // Store Compressed Klass Pointer
 7683 instruct storeNKlass(iRegN src, memory4 mem)
 7684 %{
 7685   predicate(!needs_releasing_store(n));
 7686   match(Set mem (StoreNKlass mem src));
 7687 
 7688   ins_cost(INSN_COST);
 7689   format %{ "strw  $src, $mem\t# compressed klass ptr" %}
 7690 
 7691   ins_encode(aarch64_enc_strw(src, mem));
 7692 
 7693   ins_pipe(istore_reg_mem);
 7694 %}
 7695 
 7696 // TODO
 7697 // implement storeImmD0 and storeDImmPacked
 7698 
 7699 // prefetch instructions
 7700 // Must be safe to execute with invalid address (cannot fault).
 7701 
 7702 instruct prefetchalloc( memory8 mem ) %{
 7703   match(PrefetchAllocation mem);
 7704 
 7705   ins_cost(INSN_COST);
 7706   format %{ "prfm $mem, PSTL1KEEP\t# Prefetch into level 1 cache write keep" %}
 7707 
 7708   ins_encode( aarch64_enc_prefetchw(mem) );
 7709 
 7710   ins_pipe(iload_prefetch);
 7711 %}
 7712 
 7713 //  ---------------- volatile loads and stores ----------------
 7714 
 7715 // Load Byte (8 bit signed)
 7716 instruct loadB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 7717 %{
 7718   match(Set dst (LoadB mem));
 7719 
 7720   ins_cost(VOLATILE_REF_COST);
 7721   format %{ "ldarsb  $dst, $mem\t# byte" %}
 7722 
 7723   ins_encode(aarch64_enc_ldarsb(dst, mem));
 7724 
 7725   ins_pipe(pipe_serial);
 7726 %}
 7727 
 7728 // Load Byte (8 bit signed) into long
 7729 instruct loadB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 7730 %{
 7731   match(Set dst (ConvI2L (LoadB mem)));
 7732 
 7733   ins_cost(VOLATILE_REF_COST);
 7734   format %{ "ldarsb  $dst, $mem\t# byte" %}
 7735 
 7736   ins_encode(aarch64_enc_ldarsb(dst, mem));
 7737 
 7738   ins_pipe(pipe_serial);
 7739 %}
 7740 
 7741 // Load Byte (8 bit unsigned)
 7742 instruct loadUB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 7743 %{
 7744   match(Set dst (LoadUB mem));
 7745 
 7746   ins_cost(VOLATILE_REF_COST);
 7747   format %{ "ldarb  $dst, $mem\t# byte" %}
 7748 
 7749   ins_encode(aarch64_enc_ldarb(dst, mem));
 7750 
 7751   ins_pipe(pipe_serial);
 7752 %}
 7753 
 7754 // Load Byte (8 bit unsigned) into long
 7755 instruct loadUB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 7756 %{
 7757   match(Set dst (ConvI2L (LoadUB mem)));
 7758 
 7759   ins_cost(VOLATILE_REF_COST);
 7760   format %{ "ldarb  $dst, $mem\t# byte" %}
 7761 
 7762   ins_encode(aarch64_enc_ldarb(dst, mem));
 7763 
 7764   ins_pipe(pipe_serial);
 7765 %}
 7766 
 7767 // Load Short (16 bit signed)
 7768 instruct loadS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 7769 %{
 7770   match(Set dst (LoadS mem));
 7771 
 7772   ins_cost(VOLATILE_REF_COST);
 7773   format %{ "ldarshw  $dst, $mem\t# short" %}
 7774 
 7775   ins_encode(aarch64_enc_ldarshw(dst, mem));
 7776 
 7777   ins_pipe(pipe_serial);
 7778 %}
 7779 
 7780 instruct loadUS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 7781 %{
 7782   match(Set dst (LoadUS mem));
 7783 
 7784   ins_cost(VOLATILE_REF_COST);
 7785   format %{ "ldarhw  $dst, $mem\t# short" %}
 7786 
 7787   ins_encode(aarch64_enc_ldarhw(dst, mem));
 7788 
 7789   ins_pipe(pipe_serial);
 7790 %}
 7791 
 7792 // Load Short/Char (16 bit unsigned) into long
 7793 instruct loadUS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 7794 %{
 7795   match(Set dst (ConvI2L (LoadUS mem)));
 7796 
 7797   ins_cost(VOLATILE_REF_COST);
 7798   format %{ "ldarh  $dst, $mem\t# short" %}
 7799 
 7800   ins_encode(aarch64_enc_ldarh(dst, mem));
 7801 
 7802   ins_pipe(pipe_serial);
 7803 %}
 7804 
 7805 // Load Short/Char (16 bit signed) into long
 7806 instruct loadS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 7807 %{
 7808   match(Set dst (ConvI2L (LoadS mem)));
 7809 
 7810   ins_cost(VOLATILE_REF_COST);
 7811   format %{ "ldarh  $dst, $mem\t# short" %}
 7812 
 7813   ins_encode(aarch64_enc_ldarsh(dst, mem));
 7814 
 7815   ins_pipe(pipe_serial);
 7816 %}
 7817 
 7818 // Load Integer (32 bit signed)
 7819 instruct loadI_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 7820 %{
 7821   match(Set dst (LoadI mem));
 7822 
 7823   ins_cost(VOLATILE_REF_COST);
 7824   format %{ "ldarw  $dst, $mem\t# int" %}
 7825 
 7826   ins_encode(aarch64_enc_ldarw(dst, mem));
 7827 
 7828   ins_pipe(pipe_serial);
 7829 %}
 7830 
 7831 // Load Integer (32 bit unsigned) into long
 7832 instruct loadUI2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem, immL_32bits mask)
 7833 %{
 7834   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
 7835 
 7836   ins_cost(VOLATILE_REF_COST);
 7837   format %{ "ldarw  $dst, $mem\t# int" %}
 7838 
 7839   ins_encode(aarch64_enc_ldarw(dst, mem));
 7840 
 7841   ins_pipe(pipe_serial);
 7842 %}
 7843 
 7844 // Load Long (64 bit signed)
 7845 instruct loadL_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 7846 %{
 7847   match(Set dst (LoadL mem));
 7848 
 7849   ins_cost(VOLATILE_REF_COST);
 7850   format %{ "ldar  $dst, $mem\t# int" %}
 7851 
 7852   ins_encode(aarch64_enc_ldar(dst, mem));
 7853 
 7854   ins_pipe(pipe_serial);
 7855 %}
 7856 
 7857 // Load Pointer
 7858 instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem)
 7859 %{
 7860   match(Set dst (LoadP mem));
 7861   predicate(n->as_Load()->barrier_data() == 0);
 7862 
 7863   ins_cost(VOLATILE_REF_COST);
 7864   format %{ "ldar  $dst, $mem\t# ptr" %}
 7865 
 7866   ins_encode(aarch64_enc_ldar(dst, mem));
 7867 
 7868   ins_pipe(pipe_serial);
 7869 %}
 7870 
 7871 // Load Compressed Pointer
 7872 instruct loadN_volatile(iRegNNoSp dst, /* sync_memory*/indirect mem)
 7873 %{
 7874   match(Set dst (LoadN mem));
 7875 
 7876   ins_cost(VOLATILE_REF_COST);
 7877   format %{ "ldarw  $dst, $mem\t# compressed ptr" %}
 7878 
 7879   ins_encode(aarch64_enc_ldarw(dst, mem));
 7880 
 7881   ins_pipe(pipe_serial);
 7882 %}
 7883 
 7884 // Load Float
 7885 instruct loadF_volatile(vRegF dst, /* sync_memory*/indirect mem)
 7886 %{
 7887   match(Set dst (LoadF mem));
 7888 
 7889   ins_cost(VOLATILE_REF_COST);
 7890   format %{ "ldars  $dst, $mem\t# float" %}
 7891 
 7892   ins_encode( aarch64_enc_fldars(dst, mem) );
 7893 
 7894   ins_pipe(pipe_serial);
 7895 %}
 7896 
 7897 // Load Double
 7898 instruct loadD_volatile(vRegD dst, /* sync_memory*/indirect mem)
 7899 %{
 7900   match(Set dst (LoadD mem));
 7901 
 7902   ins_cost(VOLATILE_REF_COST);
 7903   format %{ "ldard  $dst, $mem\t# double" %}
 7904 
 7905   ins_encode( aarch64_enc_fldard(dst, mem) );
 7906 
 7907   ins_pipe(pipe_serial);
 7908 %}
 7909 
 7910 // Store Byte
 7911 instruct storeB_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
 7912 %{
 7913   match(Set mem (StoreB mem src));
 7914 
 7915   ins_cost(VOLATILE_REF_COST);
 7916   format %{ "stlrb  $src, $mem\t# byte" %}
 7917 
 7918   ins_encode(aarch64_enc_stlrb(src, mem));
 7919 
 7920   ins_pipe(pipe_class_memory);
 7921 %}
 7922 
 7923 instruct storeimmB0_volatile(immI0 zero, /* sync_memory*/indirect mem)
 7924 %{
 7925   match(Set mem (StoreB mem zero));
 7926 
 7927   ins_cost(VOLATILE_REF_COST);
 7928   format %{ "stlrb  zr, $mem\t# byte" %}
 7929 
 7930   ins_encode(aarch64_enc_stlrb0(mem));
 7931 
 7932   ins_pipe(pipe_class_memory);
 7933 %}
 7934 
 7935 // Store Char/Short
 7936 instruct storeC_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
 7937 %{
 7938   match(Set mem (StoreC mem src));
 7939 
 7940   ins_cost(VOLATILE_REF_COST);
 7941   format %{ "stlrh  $src, $mem\t# short" %}
 7942 
 7943   ins_encode(aarch64_enc_stlrh(src, mem));
 7944 
 7945   ins_pipe(pipe_class_memory);
 7946 %}
 7947 
 7948 instruct storeimmC0_volatile(immI0 zero, /* sync_memory*/indirect mem)
 7949 %{
 7950   match(Set mem (StoreC mem zero));
 7951 
 7952   ins_cost(VOLATILE_REF_COST);
 7953   format %{ "stlrh  zr, $mem\t# short" %}
 7954 
 7955   ins_encode(aarch64_enc_stlrh0(mem));
 7956 
 7957   ins_pipe(pipe_class_memory);
 7958 %}
 7959 
 7960 // Store Integer
 7961 
 7962 instruct storeI_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
 7963 %{
 7964   match(Set mem(StoreI mem src));
 7965 
 7966   ins_cost(VOLATILE_REF_COST);
 7967   format %{ "stlrw  $src, $mem\t# int" %}
 7968 
 7969   ins_encode(aarch64_enc_stlrw(src, mem));
 7970 
 7971   ins_pipe(pipe_class_memory);
 7972 %}
 7973 
 7974 instruct storeimmI0_volatile(immI0 zero, /* sync_memory*/indirect mem)
 7975 %{
 7976   match(Set mem(StoreI mem zero));
 7977 
 7978   ins_cost(VOLATILE_REF_COST);
 7979   format %{ "stlrw  zr, $mem\t# int" %}
 7980 
 7981   ins_encode(aarch64_enc_stlrw0(mem));
 7982 
 7983   ins_pipe(pipe_class_memory);
 7984 %}
 7985 
 7986 // Store Long (64 bit signed)
 7987 instruct storeL_volatile(iRegL src, /* sync_memory*/indirect mem)
 7988 %{
 7989   match(Set mem (StoreL mem src));
 7990 
 7991   ins_cost(VOLATILE_REF_COST);
 7992   format %{ "stlr  $src, $mem\t# int" %}
 7993 
 7994   ins_encode(aarch64_enc_stlr(src, mem));
 7995 
 7996   ins_pipe(pipe_class_memory);
 7997 %}
 7998 
 7999 instruct storeimmL0_volatile(immL0 zero, /* sync_memory*/indirect mem)
 8000 %{
 8001   match(Set mem (StoreL mem zero));
 8002 
 8003   ins_cost(VOLATILE_REF_COST);
 8004   format %{ "stlr  zr, $mem\t# int" %}
 8005 
 8006   ins_encode(aarch64_enc_stlr0(mem));
 8007 
 8008   ins_pipe(pipe_class_memory);
 8009 %}
 8010 
 8011 // Store Pointer
 8012 instruct storeP_volatile(iRegP src, /* sync_memory*/indirect mem)
 8013 %{
 8014   match(Set mem (StoreP mem src));
 8015   predicate(n->as_Store()->barrier_data() == 0);
 8016 
 8017   ins_cost(VOLATILE_REF_COST);
 8018   format %{ "stlr  $src, $mem\t# ptr" %}
 8019 
 8020   ins_encode(aarch64_enc_stlr(src, mem));
 8021 
 8022   ins_pipe(pipe_class_memory);
 8023 %}
 8024 
 8025 instruct storeimmP0_volatile(immP0 zero, /* sync_memory*/indirect mem)
 8026 %{
 8027   match(Set mem (StoreP mem zero));
 8028   predicate(n->as_Store()->barrier_data() == 0);
 8029 
 8030   ins_cost(VOLATILE_REF_COST);
 8031   format %{ "stlr  zr, $mem\t# ptr" %}
 8032 
 8033   ins_encode(aarch64_enc_stlr0(mem));
 8034 
 8035   ins_pipe(pipe_class_memory);
 8036 %}
 8037 
 8038 // Store Compressed Pointer
 8039 instruct storeN_volatile(iRegN src, /* sync_memory*/indirect mem)
 8040 %{
 8041   match(Set mem (StoreN mem src));
 8042 
 8043   ins_cost(VOLATILE_REF_COST);
 8044   format %{ "stlrw  $src, $mem\t# compressed ptr" %}
 8045 
 8046   ins_encode(aarch64_enc_stlrw(src, mem));
 8047 
 8048   ins_pipe(pipe_class_memory);
 8049 %}
 8050 
 8051 instruct storeimmN0_volatile(immN0 zero, /* sync_memory*/indirect mem)
 8052 %{
 8053   match(Set mem (StoreN mem zero));
 8054 
 8055   ins_cost(VOLATILE_REF_COST);
 8056   format %{ "stlrw  zr, $mem\t# compressed ptr" %}
 8057 
 8058   ins_encode(aarch64_enc_stlrw0(mem));
 8059 
 8060   ins_pipe(pipe_class_memory);
 8061 %}
 8062 
 8063 // Store Float
 8064 instruct storeF_volatile(vRegF src, /* sync_memory*/indirect mem)
 8065 %{
 8066   match(Set mem (StoreF mem src));
 8067 
 8068   ins_cost(VOLATILE_REF_COST);
 8069   format %{ "stlrs  $src, $mem\t# float" %}
 8070 
 8071   ins_encode( aarch64_enc_fstlrs(src, mem) );
 8072 
 8073   ins_pipe(pipe_class_memory);
 8074 %}
 8075 
 8076 // TODO
 8077 // implement storeImmF0 and storeFImmPacked
 8078 
 8079 // Store Double
 8080 instruct storeD_volatile(vRegD src, /* sync_memory*/indirect mem)
 8081 %{
 8082   match(Set mem (StoreD mem src));
 8083 
 8084   ins_cost(VOLATILE_REF_COST);
 8085   format %{ "stlrd  $src, $mem\t# double" %}
 8086 
 8087   ins_encode( aarch64_enc_fstlrd(src, mem) );
 8088 
 8089   ins_pipe(pipe_class_memory);
 8090 %}
 8091 
 8092 //  ---------------- end of volatile loads and stores ----------------
 8093 
 8094 instruct cacheWB(indirect addr)
 8095 %{
 8096   predicate(VM_Version::supports_data_cache_line_flush());
 8097   match(CacheWB addr);
 8098 
 8099   ins_cost(100);
 8100   format %{"cache wb $addr" %}
 8101   ins_encode %{
 8102     assert($addr->index_position() < 0, "should be");
 8103     assert($addr$$disp == 0, "should be");
 8104     __ cache_wb(Address($addr$$base$$Register, 0));
 8105   %}
 8106   ins_pipe(pipe_slow); // XXX
 8107 %}
 8108 
 8109 instruct cacheWBPreSync()
 8110 %{
 8111   predicate(VM_Version::supports_data_cache_line_flush());
 8112   match(CacheWBPreSync);
 8113 
 8114   ins_cost(100);
 8115   format %{"cache wb presync" %}
 8116   ins_encode %{
 8117     __ cache_wbsync(true);
 8118   %}
 8119   ins_pipe(pipe_slow); // XXX
 8120 %}
 8121 
 8122 instruct cacheWBPostSync()
 8123 %{
 8124   predicate(VM_Version::supports_data_cache_line_flush());
 8125   match(CacheWBPostSync);
 8126 
 8127   ins_cost(100);
 8128   format %{"cache wb postsync" %}
 8129   ins_encode %{
 8130     __ cache_wbsync(false);
 8131   %}
 8132   ins_pipe(pipe_slow); // XXX
 8133 %}
 8134 
 8135 // ============================================================================
 8136 // BSWAP Instructions
 8137 
 8138 instruct bytes_reverse_int(iRegINoSp dst, iRegIorL2I src) %{
 8139   match(Set dst (ReverseBytesI src));
 8140 
 8141   ins_cost(INSN_COST);
 8142   format %{ "revw  $dst, $src" %}
 8143 
 8144   ins_encode %{
 8145     __ revw(as_Register($dst$$reg), as_Register($src$$reg));
 8146   %}
 8147 
 8148   ins_pipe(ialu_reg);
 8149 %}
 8150 
 8151 instruct bytes_reverse_long(iRegLNoSp dst, iRegL src) %{
 8152   match(Set dst (ReverseBytesL src));
 8153 
 8154   ins_cost(INSN_COST);
 8155   format %{ "rev  $dst, $src" %}
 8156 
 8157   ins_encode %{
 8158     __ rev(as_Register($dst$$reg), as_Register($src$$reg));
 8159   %}
 8160 
 8161   ins_pipe(ialu_reg);
 8162 %}
 8163 
 8164 instruct bytes_reverse_unsigned_short(iRegINoSp dst, iRegIorL2I src) %{
 8165   match(Set dst (ReverseBytesUS src));
 8166 
 8167   ins_cost(INSN_COST);
 8168   format %{ "rev16w  $dst, $src" %}
 8169 
 8170   ins_encode %{
 8171     __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
 8172   %}
 8173 
 8174   ins_pipe(ialu_reg);
 8175 %}
 8176 
 8177 instruct bytes_reverse_short(iRegINoSp dst, iRegIorL2I src) %{
 8178   match(Set dst (ReverseBytesS src));
 8179 
 8180   ins_cost(INSN_COST);
 8181   format %{ "rev16w  $dst, $src\n\t"
 8182             "sbfmw $dst, $dst, #0, #15" %}
 8183 
 8184   ins_encode %{
 8185     __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
 8186     __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 15U);
 8187   %}
 8188 
 8189   ins_pipe(ialu_reg);
 8190 %}
 8191 
 8192 // ============================================================================
 8193 // Zero Count Instructions
 8194 
 8195 instruct countLeadingZerosI(iRegINoSp dst, iRegIorL2I src) %{
 8196   match(Set dst (CountLeadingZerosI src));
 8197 
 8198   ins_cost(INSN_COST);
 8199   format %{ "clzw  $dst, $src" %}
 8200   ins_encode %{
 8201     __ clzw(as_Register($dst$$reg), as_Register($src$$reg));
 8202   %}
 8203 
 8204   ins_pipe(ialu_reg);
 8205 %}
 8206 
 8207 instruct countLeadingZerosL(iRegINoSp dst, iRegL src) %{
 8208   match(Set dst (CountLeadingZerosL src));
 8209 
 8210   ins_cost(INSN_COST);
 8211   format %{ "clz   $dst, $src" %}
 8212   ins_encode %{
 8213     __ clz(as_Register($dst$$reg), as_Register($src$$reg));
 8214   %}
 8215 
 8216   ins_pipe(ialu_reg);
 8217 %}
 8218 
 8219 instruct countTrailingZerosI(iRegINoSp dst, iRegIorL2I src) %{
 8220   match(Set dst (CountTrailingZerosI src));
 8221 
 8222   ins_cost(INSN_COST * 2);
 8223   format %{ "rbitw  $dst, $src\n\t"
 8224             "clzw   $dst, $dst" %}
 8225   ins_encode %{
 8226     __ rbitw(as_Register($dst$$reg), as_Register($src$$reg));
 8227     __ clzw(as_Register($dst$$reg), as_Register($dst$$reg));
 8228   %}
 8229 
 8230   ins_pipe(ialu_reg);
 8231 %}
 8232 
 8233 instruct countTrailingZerosL(iRegINoSp dst, iRegL src) %{
 8234   match(Set dst (CountTrailingZerosL src));
 8235 
 8236   ins_cost(INSN_COST * 2);
 8237   format %{ "rbit   $dst, $src\n\t"
 8238             "clz    $dst, $dst" %}
 8239   ins_encode %{
 8240     __ rbit(as_Register($dst$$reg), as_Register($src$$reg));
 8241     __ clz(as_Register($dst$$reg), as_Register($dst$$reg));
 8242   %}
 8243 
 8244   ins_pipe(ialu_reg);
 8245 %}
 8246 
 8247 //---------- Population Count Instructions -------------------------------------
 8248 //
 8249 
 8250 instruct popCountI(iRegINoSp dst, iRegIorL2I src, vRegF tmp) %{
 8251   match(Set dst (PopCountI src));
 8252   effect(TEMP tmp);
 8253   ins_cost(INSN_COST * 13);
 8254 
 8255   format %{ "movw   $src, $src\n\t"
 8256             "mov    $tmp, $src\t# vector (1D)\n\t"
 8257             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 8258             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 8259             "mov    $dst, $tmp\t# vector (1D)" %}
 8260   ins_encode %{
 8261     __ movw($src$$Register, $src$$Register); // ensure top 32 bits 0
 8262     __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
 8263     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8264     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8265     __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
 8266   %}
 8267 
 8268   ins_pipe(pipe_class_default);
 8269 %}
 8270 
 8271 instruct popCountI_mem(iRegINoSp dst, memory4 mem, vRegF tmp) %{
 8272   match(Set dst (PopCountI (LoadI mem)));
 8273   effect(TEMP tmp);
 8274   ins_cost(INSN_COST * 13);
 8275 
 8276   format %{ "ldrs   $tmp, $mem\n\t"
 8277             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 8278             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 8279             "mov    $dst, $tmp\t# vector (1D)" %}
 8280   ins_encode %{
 8281     FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
 8282     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrs, tmp_reg, $mem->opcode(),
 8283               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 8284     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8285     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8286     __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
 8287   %}
 8288 
 8289   ins_pipe(pipe_class_default);
 8290 %}
 8291 
 8292 // Note: Long.bitCount(long) returns an int.
 8293 instruct popCountL(iRegINoSp dst, iRegL src, vRegD tmp) %{
 8294   match(Set dst (PopCountL src));
 8295   effect(TEMP tmp);
 8296   ins_cost(INSN_COST * 13);
 8297 
 8298   format %{ "mov    $tmp, $src\t# vector (1D)\n\t"
 8299             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 8300             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 8301             "mov    $dst, $tmp\t# vector (1D)" %}
 8302   ins_encode %{
 8303     __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
 8304     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8305     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8306     __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
 8307   %}
 8308 
 8309   ins_pipe(pipe_class_default);
 8310 %}
 8311 
 8312 instruct popCountL_mem(iRegINoSp dst, memory8 mem, vRegD tmp) %{
 8313   match(Set dst (PopCountL (LoadL mem)));
 8314   effect(TEMP tmp);
 8315   ins_cost(INSN_COST * 13);
 8316 
 8317   format %{ "ldrd   $tmp, $mem\n\t"
 8318             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 8319             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 8320             "mov    $dst, $tmp\t# vector (1D)" %}
 8321   ins_encode %{
 8322     FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
 8323     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrd, tmp_reg, $mem->opcode(),
 8324               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 8325     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8326     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8327     __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
 8328   %}
 8329 
 8330   ins_pipe(pipe_class_default);
 8331 %}
 8332 
 8333 // ============================================================================
 8334 // MemBar Instruction
 8335 
 8336 instruct load_fence() %{
 8337   match(LoadFence);
 8338   ins_cost(VOLATILE_REF_COST);
 8339 
 8340   format %{ "load_fence" %}
 8341 
 8342   ins_encode %{
 8343     __ membar(Assembler::LoadLoad|Assembler::LoadStore);
 8344   %}
 8345   ins_pipe(pipe_serial);
 8346 %}
 8347 
 8348 instruct unnecessary_membar_acquire() %{
 8349   predicate(unnecessary_acquire(n));
 8350   match(MemBarAcquire);
 8351   ins_cost(0);
 8352 
 8353   format %{ "membar_acquire (elided)" %}
 8354 
 8355   ins_encode %{
 8356     __ block_comment("membar_acquire (elided)");
 8357   %}
 8358 
 8359   ins_pipe(pipe_class_empty);
 8360 %}
 8361 
 8362 instruct membar_acquire() %{
 8363   match(MemBarAcquire);
 8364   ins_cost(VOLATILE_REF_COST);
 8365 
 8366   format %{ "membar_acquire\n\t"
 8367             "dmb ish" %}
 8368 
 8369   ins_encode %{
 8370     __ block_comment("membar_acquire");
 8371     __ membar(Assembler::LoadLoad|Assembler::LoadStore);
 8372   %}
 8373 
 8374   ins_pipe(pipe_serial);
 8375 %}
 8376 
 8377 
 8378 instruct membar_acquire_lock() %{
 8379   match(MemBarAcquireLock);
 8380   ins_cost(VOLATILE_REF_COST);
 8381 
 8382   format %{ "membar_acquire_lock (elided)" %}
 8383 
 8384   ins_encode %{
 8385     __ block_comment("membar_acquire_lock (elided)");
 8386   %}
 8387 
 8388   ins_pipe(pipe_serial);
 8389 %}
 8390 
 8391 instruct store_fence() %{
 8392   match(StoreFence);
 8393   ins_cost(VOLATILE_REF_COST);
 8394 
 8395   format %{ "store_fence" %}
 8396 
 8397   ins_encode %{
 8398     __ membar(Assembler::LoadStore|Assembler::StoreStore);
 8399   %}
 8400   ins_pipe(pipe_serial);
 8401 %}
 8402 
 8403 instruct unnecessary_membar_release() %{
 8404   predicate(unnecessary_release(n));
 8405   match(MemBarRelease);
 8406   ins_cost(0);
 8407 
 8408   format %{ "membar_release (elided)" %}
 8409 
 8410   ins_encode %{
 8411     __ block_comment("membar_release (elided)");
 8412   %}
 8413   ins_pipe(pipe_serial);
 8414 %}
 8415 
 8416 instruct membar_release() %{
 8417   match(MemBarRelease);
 8418   ins_cost(VOLATILE_REF_COST);
 8419 
 8420   format %{ "membar_release\n\t"
 8421             "dmb ish" %}
 8422 
 8423   ins_encode %{
 8424     __ block_comment("membar_release");
 8425     __ membar(Assembler::LoadStore|Assembler::StoreStore);
 8426   %}
 8427   ins_pipe(pipe_serial);
 8428 %}
 8429 
 8430 instruct membar_storestore() %{
 8431   match(MemBarStoreStore);
 8432   match(StoreStoreFence);
 8433   ins_cost(VOLATILE_REF_COST);
 8434 
 8435   format %{ "MEMBAR-store-store" %}
 8436 
 8437   ins_encode %{
 8438     __ membar(Assembler::StoreStore);
 8439   %}
 8440   ins_pipe(pipe_serial);
 8441 %}
 8442 
 8443 instruct membar_release_lock() %{
 8444   match(MemBarReleaseLock);
 8445   ins_cost(VOLATILE_REF_COST);
 8446 
 8447   format %{ "membar_release_lock (elided)" %}
 8448 
 8449   ins_encode %{
 8450     __ block_comment("membar_release_lock (elided)");
 8451   %}
 8452 
 8453   ins_pipe(pipe_serial);
 8454 %}
 8455 
 8456 instruct unnecessary_membar_volatile() %{
 8457   predicate(unnecessary_volatile(n));
 8458   match(MemBarVolatile);
 8459   ins_cost(0);
 8460 
 8461   format %{ "membar_volatile (elided)" %}
 8462 
 8463   ins_encode %{
 8464     __ block_comment("membar_volatile (elided)");
 8465   %}
 8466 
 8467   ins_pipe(pipe_serial);
 8468 %}
 8469 
 8470 instruct membar_volatile() %{
 8471   match(MemBarVolatile);
 8472   ins_cost(VOLATILE_REF_COST*100);
 8473 
 8474   format %{ "membar_volatile\n\t"
 8475              "dmb ish"%}
 8476 
 8477   ins_encode %{
 8478     __ block_comment("membar_volatile");
 8479     __ membar(Assembler::StoreLoad);
 8480   %}
 8481 
 8482   ins_pipe(pipe_serial);
 8483 %}
 8484 
 8485 // ============================================================================
 8486 // Cast/Convert Instructions
 8487 
 8488 instruct castX2P(iRegPNoSp dst, iRegL src) %{
 8489   match(Set dst (CastX2P src));
 8490 
 8491   ins_cost(INSN_COST);
 8492   format %{ "mov $dst, $src\t# long -> ptr" %}
 8493 
 8494   ins_encode %{
 8495     if ($dst$$reg != $src$$reg) {
 8496       __ mov(as_Register($dst$$reg), as_Register($src$$reg));
 8497     }
 8498   %}
 8499 
 8500   ins_pipe(ialu_reg);
 8501 %}
 8502 
 8503 instruct castN2X(iRegLNoSp dst, iRegN src) %{
 8504   match(Set dst (CastP2X src));
 8505 
 8506   ins_cost(INSN_COST);
 8507   format %{ "mov $dst, $src\t# ptr -> long" %}
 8508 
 8509   ins_encode %{
 8510     if ($dst$$reg != $src$$reg) {
 8511       __ mov(as_Register($dst$$reg), as_Register($src$$reg));
 8512     }
 8513   %}
 8514 
 8515   ins_pipe(ialu_reg);
 8516 %}
 8517 
 8518 instruct castP2X(iRegLNoSp dst, iRegP src) %{
 8519   match(Set dst (CastP2X src));
 8520 
 8521   ins_cost(INSN_COST);
 8522   format %{ "mov $dst, $src\t# ptr -> long" %}
 8523 
 8524   ins_encode %{
 8525     if ($dst$$reg != $src$$reg) {
 8526       __ mov(as_Register($dst$$reg), as_Register($src$$reg));
 8527     }
 8528   %}
 8529 
 8530   ins_pipe(ialu_reg);
 8531 %}
 8532 
 8533 // Convert oop into int for vectors alignment masking
 8534 instruct convP2I(iRegINoSp dst, iRegP src) %{
 8535   match(Set dst (ConvL2I (CastP2X src)));
 8536 
 8537   ins_cost(INSN_COST);
 8538   format %{ "movw $dst, $src\t# ptr -> int" %}
 8539   ins_encode %{
 8540     __ movw($dst$$Register, $src$$Register);
 8541   %}
 8542 
 8543   ins_pipe(ialu_reg);
 8544 %}
 8545 
 8546 // Convert compressed oop into int for vectors alignment masking
 8547 // in case of 32bit oops (heap < 4Gb).
 8548 instruct convN2I(iRegINoSp dst, iRegN src)
 8549 %{
 8550   predicate(CompressedOops::shift() == 0);
 8551   match(Set dst (ConvL2I (CastP2X (DecodeN src))));
 8552 
 8553   ins_cost(INSN_COST);
 8554   format %{ "mov dst, $src\t# compressed ptr -> int" %}
 8555   ins_encode %{
 8556     __ movw($dst$$Register, $src$$Register);
 8557   %}
 8558 
 8559   ins_pipe(ialu_reg);
 8560 %}
 8561 
 8562 
 8563 // Convert oop pointer into compressed form
 8564 instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
 8565   predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
 8566   match(Set dst (EncodeP src));
 8567   effect(KILL cr);
 8568   ins_cost(INSN_COST * 3);
 8569   format %{ "encode_heap_oop $dst, $src" %}
 8570   ins_encode %{
 8571     Register s = $src$$Register;
 8572     Register d = $dst$$Register;
 8573     __ encode_heap_oop(d, s);
 8574   %}
 8575   ins_pipe(ialu_reg);
 8576 %}
 8577 
 8578 instruct encodeHeapOop_not_null(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
 8579   predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
 8580   match(Set dst (EncodeP src));
 8581   ins_cost(INSN_COST * 3);
 8582   format %{ "encode_heap_oop_not_null $dst, $src" %}
 8583   ins_encode %{
 8584     __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
 8585   %}
 8586   ins_pipe(ialu_reg);
 8587 %}
 8588 
 8589 instruct decodeHeapOop(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
 8590   predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
 8591             n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
 8592   match(Set dst (DecodeN src));
 8593   ins_cost(INSN_COST * 3);
 8594   format %{ "decode_heap_oop $dst, $src" %}
 8595   ins_encode %{
 8596     Register s = $src$$Register;
 8597     Register d = $dst$$Register;
 8598     __ decode_heap_oop(d, s);
 8599   %}
 8600   ins_pipe(ialu_reg);
 8601 %}
 8602 
 8603 instruct decodeHeapOop_not_null(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
 8604   predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
 8605             n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
 8606   match(Set dst (DecodeN src));
 8607   ins_cost(INSN_COST * 3);
 8608   format %{ "decode_heap_oop_not_null $dst, $src" %}
 8609   ins_encode %{
 8610     Register s = $src$$Register;
 8611     Register d = $dst$$Register;
 8612     __ decode_heap_oop_not_null(d, s);
 8613   %}
 8614   ins_pipe(ialu_reg);
 8615 %}
 8616 
 8617 // n.b. AArch64 implementations of encode_klass_not_null and
 8618 // decode_klass_not_null do not modify the flags register so, unlike
 8619 // Intel, we don't kill CR as a side effect here
 8620 
 8621 instruct encodeKlass_not_null(iRegNNoSp dst, iRegP src) %{
 8622   match(Set dst (EncodePKlass src));
 8623 
 8624   ins_cost(INSN_COST * 3);
 8625   format %{ "encode_klass_not_null $dst,$src" %}
 8626 
 8627   ins_encode %{
 8628     Register src_reg = as_Register($src$$reg);
 8629     Register dst_reg = as_Register($dst$$reg);
 8630     __ encode_klass_not_null(dst_reg, src_reg);
 8631   %}
 8632 
 8633    ins_pipe(ialu_reg);
 8634 %}
 8635 
 8636 instruct decodeKlass_not_null(iRegPNoSp dst, iRegN src) %{
 8637   match(Set dst (DecodeNKlass src));
 8638 
 8639   ins_cost(INSN_COST * 3);
 8640   format %{ "decode_klass_not_null $dst,$src" %}
 8641 
 8642   ins_encode %{
 8643     Register src_reg = as_Register($src$$reg);
 8644     Register dst_reg = as_Register($dst$$reg);
 8645     if (dst_reg != src_reg) {
 8646       __ decode_klass_not_null(dst_reg, src_reg);
 8647     } else {
 8648       __ decode_klass_not_null(dst_reg);
 8649     }
 8650   %}
 8651 
 8652    ins_pipe(ialu_reg);
 8653 %}
 8654 
 8655 instruct checkCastPP(iRegPNoSp dst)
 8656 %{
 8657   match(Set dst (CheckCastPP dst));
 8658 
 8659   size(0);
 8660   format %{ "# checkcastPP of $dst" %}
 8661   ins_encode(/* empty encoding */);
 8662   ins_pipe(pipe_class_empty);
 8663 %}
 8664 
 8665 instruct castPP(iRegPNoSp dst)
 8666 %{
 8667   match(Set dst (CastPP dst));
 8668 
 8669   size(0);
 8670   format %{ "# castPP of $dst" %}
 8671   ins_encode(/* empty encoding */);
 8672   ins_pipe(pipe_class_empty);
 8673 %}
 8674 
 8675 instruct castII(iRegI dst)
 8676 %{
 8677   match(Set dst (CastII dst));
 8678 
 8679   size(0);
 8680   format %{ "# castII of $dst" %}
 8681   ins_encode(/* empty encoding */);
 8682   ins_cost(0);
 8683   ins_pipe(pipe_class_empty);
 8684 %}
 8685 
 8686 instruct castLL(iRegL dst)
 8687 %{
 8688   match(Set dst (CastLL dst));
 8689 
 8690   size(0);
 8691   format %{ "# castLL of $dst" %}
 8692   ins_encode(/* empty encoding */);
 8693   ins_cost(0);
 8694   ins_pipe(pipe_class_empty);
 8695 %}
 8696 
 8697 instruct castFF(vRegF dst)
 8698 %{
 8699   match(Set dst (CastFF dst));
 8700 
 8701   size(0);
 8702   format %{ "# castFF of $dst" %}
 8703   ins_encode(/* empty encoding */);
 8704   ins_cost(0);
 8705   ins_pipe(pipe_class_empty);
 8706 %}
 8707 
 8708 instruct castDD(vRegD dst)
 8709 %{
 8710   match(Set dst (CastDD dst));
 8711 
 8712   size(0);
 8713   format %{ "# castDD of $dst" %}
 8714   ins_encode(/* empty encoding */);
 8715   ins_cost(0);
 8716   ins_pipe(pipe_class_empty);
 8717 %}
 8718 
 8719 instruct castVV(vReg dst)
 8720 %{
 8721   match(Set dst (CastVV dst));
 8722 
 8723   size(0);
 8724   format %{ "# castVV of $dst" %}
 8725   ins_encode(/* empty encoding */);
 8726   ins_cost(0);
 8727   ins_pipe(pipe_class_empty);
 8728 %}
 8729 
 8730 instruct castVVMask(pRegGov dst)
 8731 %{
 8732   match(Set dst (CastVV dst));
 8733 
 8734   size(0);
 8735   format %{ "# castVV of $dst" %}
 8736   ins_encode(/* empty encoding */);
 8737   ins_cost(0);
 8738   ins_pipe(pipe_class_empty);
 8739 %}
 8740 
 8741 // ============================================================================
 8742 // Atomic operation instructions
 8743 //
 8744 
 8745 // standard CompareAndSwapX when we are using barriers
 8746 // these have higher priority than the rules selected by a predicate
 8747 
 8748 // XXX No flag versions for CompareAndSwap{I,L,P,N} because matcher
 8749 // can't match them
 8750 
 8751 instruct compareAndSwapB(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 8752 
 8753   match(Set res (CompareAndSwapB mem (Binary oldval newval)));
 8754   ins_cost(2 * VOLATILE_REF_COST);
 8755 
 8756   effect(KILL cr);
 8757 
 8758   format %{
 8759     "cmpxchgb $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 8760     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8761   %}
 8762 
 8763   ins_encode(aarch64_enc_cmpxchgb(mem, oldval, newval),
 8764             aarch64_enc_cset_eq(res));
 8765 
 8766   ins_pipe(pipe_slow);
 8767 %}
 8768 
 8769 instruct compareAndSwapS(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 8770 
 8771   match(Set res (CompareAndSwapS mem (Binary oldval newval)));
 8772   ins_cost(2 * VOLATILE_REF_COST);
 8773 
 8774   effect(KILL cr);
 8775 
 8776   format %{
 8777     "cmpxchgs $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 8778     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8779   %}
 8780 
 8781   ins_encode(aarch64_enc_cmpxchgs(mem, oldval, newval),
 8782             aarch64_enc_cset_eq(res));
 8783 
 8784   ins_pipe(pipe_slow);
 8785 %}
 8786 
 8787 instruct compareAndSwapI(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 8788 
 8789   match(Set res (CompareAndSwapI mem (Binary oldval newval)));
 8790   ins_cost(2 * VOLATILE_REF_COST);
 8791 
 8792   effect(KILL cr);
 8793 
 8794  format %{
 8795     "cmpxchgw $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 8796     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8797  %}
 8798 
 8799  ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
 8800             aarch64_enc_cset_eq(res));
 8801 
 8802   ins_pipe(pipe_slow);
 8803 %}
 8804 
 8805 instruct compareAndSwapL(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
 8806 
 8807   match(Set res (CompareAndSwapL mem (Binary oldval newval)));
 8808   ins_cost(2 * VOLATILE_REF_COST);
 8809 
 8810   effect(KILL cr);
 8811 
 8812  format %{
 8813     "cmpxchg $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
 8814     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8815  %}
 8816 
 8817  ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
 8818             aarch64_enc_cset_eq(res));
 8819 
 8820   ins_pipe(pipe_slow);
 8821 %}
 8822 
 8823 instruct compareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 8824 
 8825   match(Set res (CompareAndSwapP mem (Binary oldval newval)));
 8826   predicate(n->as_LoadStore()->barrier_data() == 0);
 8827   ins_cost(2 * VOLATILE_REF_COST);
 8828 
 8829   effect(KILL cr);
 8830 
 8831  format %{
 8832     "cmpxchg $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
 8833     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8834  %}
 8835 
 8836  ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
 8837             aarch64_enc_cset_eq(res));
 8838 
 8839   ins_pipe(pipe_slow);
 8840 %}
 8841 
 8842 instruct compareAndSwapN(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
 8843 
 8844   match(Set res (CompareAndSwapN mem (Binary oldval newval)));
 8845   ins_cost(2 * VOLATILE_REF_COST);
 8846 
 8847   effect(KILL cr);
 8848 
 8849  format %{
 8850     "cmpxchgw $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
 8851     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8852  %}
 8853 
 8854  ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
 8855             aarch64_enc_cset_eq(res));
 8856 
 8857   ins_pipe(pipe_slow);
 8858 %}
 8859 
 8860 // alternative CompareAndSwapX when we are eliding barriers
 8861 
 8862 instruct compareAndSwapBAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 8863 
 8864   predicate(needs_acquiring_load_exclusive(n));
 8865   match(Set res (CompareAndSwapB mem (Binary oldval newval)));
 8866   ins_cost(VOLATILE_REF_COST);
 8867 
 8868   effect(KILL cr);
 8869 
 8870   format %{
 8871     "cmpxchgb_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 8872     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8873   %}
 8874 
 8875   ins_encode(aarch64_enc_cmpxchgb_acq(mem, oldval, newval),
 8876             aarch64_enc_cset_eq(res));
 8877 
 8878   ins_pipe(pipe_slow);
 8879 %}
 8880 
 8881 instruct compareAndSwapSAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 8882 
 8883   predicate(needs_acquiring_load_exclusive(n));
 8884   match(Set res (CompareAndSwapS mem (Binary oldval newval)));
 8885   ins_cost(VOLATILE_REF_COST);
 8886 
 8887   effect(KILL cr);
 8888 
 8889   format %{
 8890     "cmpxchgs_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 8891     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8892   %}
 8893 
 8894   ins_encode(aarch64_enc_cmpxchgs_acq(mem, oldval, newval),
 8895             aarch64_enc_cset_eq(res));
 8896 
 8897   ins_pipe(pipe_slow);
 8898 %}
 8899 
 8900 instruct compareAndSwapIAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 8901 
 8902   predicate(needs_acquiring_load_exclusive(n));
 8903   match(Set res (CompareAndSwapI mem (Binary oldval newval)));
 8904   ins_cost(VOLATILE_REF_COST);
 8905 
 8906   effect(KILL cr);
 8907 
 8908  format %{
 8909     "cmpxchgw_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 8910     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8911  %}
 8912 
 8913  ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
 8914             aarch64_enc_cset_eq(res));
 8915 
 8916   ins_pipe(pipe_slow);
 8917 %}
 8918 
 8919 instruct compareAndSwapLAcq(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
 8920 
 8921   predicate(needs_acquiring_load_exclusive(n));
 8922   match(Set res (CompareAndSwapL mem (Binary oldval newval)));
 8923   ins_cost(VOLATILE_REF_COST);
 8924 
 8925   effect(KILL cr);
 8926 
 8927  format %{
 8928     "cmpxchg_acq $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
 8929     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8930  %}
 8931 
 8932  ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
 8933             aarch64_enc_cset_eq(res));
 8934 
 8935   ins_pipe(pipe_slow);
 8936 %}
 8937 
 8938 instruct compareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 8939 
 8940   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 8941   match(Set res (CompareAndSwapP mem (Binary oldval newval)));
 8942   ins_cost(VOLATILE_REF_COST);
 8943 
 8944   effect(KILL cr);
 8945 
 8946  format %{
 8947     "cmpxchg_acq $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
 8948     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8949  %}
 8950 
 8951  ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
 8952             aarch64_enc_cset_eq(res));
 8953 
 8954   ins_pipe(pipe_slow);
 8955 %}
 8956 
 8957 instruct compareAndSwapNAcq(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
 8958 
 8959   predicate(needs_acquiring_load_exclusive(n));
 8960   match(Set res (CompareAndSwapN mem (Binary oldval newval)));
 8961   ins_cost(VOLATILE_REF_COST);
 8962 
 8963   effect(KILL cr);
 8964 
 8965  format %{
 8966     "cmpxchgw_acq $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
 8967     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8968  %}
 8969 
 8970  ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
 8971             aarch64_enc_cset_eq(res));
 8972 
 8973   ins_pipe(pipe_slow);
 8974 %}
 8975 
 8976 
 8977 // ---------------------------------------------------------------------
 8978 
 8979 // BEGIN This section of the file is automatically generated. Do not edit --------------
 8980 
 8981 // Sundry CAS operations.  Note that release is always true,
 8982 // regardless of the memory ordering of the CAS.  This is because we
 8983 // need the volatile case to be sequentially consistent but there is
 8984 // no trailing StoreLoad barrier emitted by C2.  Unfortunately we
 8985 // can't check the type of memory ordering here, so we always emit a
 8986 // STLXR.
 8987 
 8988 // This section is generated from aarch64_ad_cas.m4
 8989 
 8990 
 8991 
 8992 // This pattern is generated automatically from cas.m4.
 8993 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8994 instruct compareAndExchangeB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 8995 
 8996   match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
 8997   ins_cost(2 * VOLATILE_REF_COST);
 8998   effect(TEMP_DEF res, KILL cr);
 8999   format %{
 9000     "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 9001   %}
 9002   ins_encode %{
 9003     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9004                Assembler::byte, /*acquire*/ false, /*release*/ true,
 9005                /*weak*/ false, $res$$Register);
 9006     __ sxtbw($res$$Register, $res$$Register);
 9007   %}
 9008   ins_pipe(pipe_slow);
 9009 %}
 9010 
 9011 // This pattern is generated automatically from cas.m4.
 9012 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9013 instruct compareAndExchangeS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9014 
 9015   match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
 9016   ins_cost(2 * VOLATILE_REF_COST);
 9017   effect(TEMP_DEF res, KILL cr);
 9018   format %{
 9019     "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 9020   %}
 9021   ins_encode %{
 9022     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9023                Assembler::halfword, /*acquire*/ false, /*release*/ true,
 9024                /*weak*/ false, $res$$Register);
 9025     __ sxthw($res$$Register, $res$$Register);
 9026   %}
 9027   ins_pipe(pipe_slow);
 9028 %}
 9029 
 9030 // This pattern is generated automatically from cas.m4.
 9031 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9032 instruct compareAndExchangeI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9033 
 9034   match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
 9035   ins_cost(2 * VOLATILE_REF_COST);
 9036   effect(TEMP_DEF res, KILL cr);
 9037   format %{
 9038     "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 9039   %}
 9040   ins_encode %{
 9041     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9042                Assembler::word, /*acquire*/ false, /*release*/ true,
 9043                /*weak*/ false, $res$$Register);
 9044   %}
 9045   ins_pipe(pipe_slow);
 9046 %}
 9047 
 9048 // This pattern is generated automatically from cas.m4.
 9049 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9050 instruct compareAndExchangeL(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 9051 
 9052   match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
 9053   ins_cost(2 * VOLATILE_REF_COST);
 9054   effect(TEMP_DEF res, KILL cr);
 9055   format %{
 9056     "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 9057   %}
 9058   ins_encode %{
 9059     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9060                Assembler::xword, /*acquire*/ false, /*release*/ true,
 9061                /*weak*/ false, $res$$Register);
 9062   %}
 9063   ins_pipe(pipe_slow);
 9064 %}
 9065 
 9066 // This pattern is generated automatically from cas.m4.
 9067 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9068 instruct compareAndExchangeN(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 9069 
 9070   match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
 9071   ins_cost(2 * VOLATILE_REF_COST);
 9072   effect(TEMP_DEF res, KILL cr);
 9073   format %{
 9074     "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 9075   %}
 9076   ins_encode %{
 9077     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9078                Assembler::word, /*acquire*/ false, /*release*/ true,
 9079                /*weak*/ false, $res$$Register);
 9080   %}
 9081   ins_pipe(pipe_slow);
 9082 %}
 9083 
 9084 // This pattern is generated automatically from cas.m4.
 9085 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9086 instruct compareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9087   predicate(n->as_LoadStore()->barrier_data() == 0);
 9088   match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
 9089   ins_cost(2 * VOLATILE_REF_COST);
 9090   effect(TEMP_DEF res, KILL cr);
 9091   format %{
 9092     "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 9093   %}
 9094   ins_encode %{
 9095     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9096                Assembler::xword, /*acquire*/ false, /*release*/ true,
 9097                /*weak*/ false, $res$$Register);
 9098   %}
 9099   ins_pipe(pipe_slow);
 9100 %}
 9101 
 9102 // This pattern is generated automatically from cas.m4.
 9103 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9104 instruct compareAndExchangeBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9105   predicate(needs_acquiring_load_exclusive(n));
 9106   match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
 9107   ins_cost(VOLATILE_REF_COST);
 9108   effect(TEMP_DEF res, KILL cr);
 9109   format %{
 9110     "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 9111   %}
 9112   ins_encode %{
 9113     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9114                Assembler::byte, /*acquire*/ true, /*release*/ true,
 9115                /*weak*/ false, $res$$Register);
 9116     __ sxtbw($res$$Register, $res$$Register);
 9117   %}
 9118   ins_pipe(pipe_slow);
 9119 %}
 9120 
 9121 // This pattern is generated automatically from cas.m4.
 9122 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9123 instruct compareAndExchangeSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9124   predicate(needs_acquiring_load_exclusive(n));
 9125   match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
 9126   ins_cost(VOLATILE_REF_COST);
 9127   effect(TEMP_DEF res, KILL cr);
 9128   format %{
 9129     "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 9130   %}
 9131   ins_encode %{
 9132     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9133                Assembler::halfword, /*acquire*/ true, /*release*/ true,
 9134                /*weak*/ false, $res$$Register);
 9135     __ sxthw($res$$Register, $res$$Register);
 9136   %}
 9137   ins_pipe(pipe_slow);
 9138 %}
 9139 
 9140 // This pattern is generated automatically from cas.m4.
 9141 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9142 instruct compareAndExchangeIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9143   predicate(needs_acquiring_load_exclusive(n));
 9144   match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
 9145   ins_cost(VOLATILE_REF_COST);
 9146   effect(TEMP_DEF res, KILL cr);
 9147   format %{
 9148     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 9149   %}
 9150   ins_encode %{
 9151     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9152                Assembler::word, /*acquire*/ true, /*release*/ true,
 9153                /*weak*/ false, $res$$Register);
 9154   %}
 9155   ins_pipe(pipe_slow);
 9156 %}
 9157 
 9158 // This pattern is generated automatically from cas.m4.
 9159 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9160 instruct compareAndExchangeLAcq(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 9161   predicate(needs_acquiring_load_exclusive(n));
 9162   match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
 9163   ins_cost(VOLATILE_REF_COST);
 9164   effect(TEMP_DEF res, KILL cr);
 9165   format %{
 9166     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 9167   %}
 9168   ins_encode %{
 9169     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9170                Assembler::xword, /*acquire*/ true, /*release*/ true,
 9171                /*weak*/ false, $res$$Register);
 9172   %}
 9173   ins_pipe(pipe_slow);
 9174 %}
 9175 
 9176 // This pattern is generated automatically from cas.m4.
 9177 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9178 instruct compareAndExchangeNAcq(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 9179   predicate(needs_acquiring_load_exclusive(n));
 9180   match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
 9181   ins_cost(VOLATILE_REF_COST);
 9182   effect(TEMP_DEF res, KILL cr);
 9183   format %{
 9184     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 9185   %}
 9186   ins_encode %{
 9187     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9188                Assembler::word, /*acquire*/ true, /*release*/ true,
 9189                /*weak*/ false, $res$$Register);
 9190   %}
 9191   ins_pipe(pipe_slow);
 9192 %}
 9193 
 9194 // This pattern is generated automatically from cas.m4.
 9195 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9196 instruct compareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9197   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 9198   match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
 9199   ins_cost(VOLATILE_REF_COST);
 9200   effect(TEMP_DEF res, KILL cr);
 9201   format %{
 9202     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 9203   %}
 9204   ins_encode %{
 9205     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9206                Assembler::xword, /*acquire*/ true, /*release*/ true,
 9207                /*weak*/ false, $res$$Register);
 9208   %}
 9209   ins_pipe(pipe_slow);
 9210 %}
 9211 
 9212 // This pattern is generated automatically from cas.m4.
 9213 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9214 instruct weakCompareAndSwapB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9215 
 9216   match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
 9217   ins_cost(2 * VOLATILE_REF_COST);
 9218   effect(KILL cr);
 9219   format %{
 9220     "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 9221     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9222   %}
 9223   ins_encode %{
 9224     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9225                Assembler::byte, /*acquire*/ false, /*release*/ true,
 9226                /*weak*/ true, noreg);
 9227     __ csetw($res$$Register, Assembler::EQ);
 9228   %}
 9229   ins_pipe(pipe_slow);
 9230 %}
 9231 
 9232 // This pattern is generated automatically from cas.m4.
 9233 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9234 instruct weakCompareAndSwapS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9235 
 9236   match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
 9237   ins_cost(2 * VOLATILE_REF_COST);
 9238   effect(KILL cr);
 9239   format %{
 9240     "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 9241     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9242   %}
 9243   ins_encode %{
 9244     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9245                Assembler::halfword, /*acquire*/ false, /*release*/ true,
 9246                /*weak*/ true, noreg);
 9247     __ csetw($res$$Register, Assembler::EQ);
 9248   %}
 9249   ins_pipe(pipe_slow);
 9250 %}
 9251 
 9252 // This pattern is generated automatically from cas.m4.
 9253 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9254 instruct weakCompareAndSwapI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9255 
 9256   match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
 9257   ins_cost(2 * VOLATILE_REF_COST);
 9258   effect(KILL cr);
 9259   format %{
 9260     "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 9261     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9262   %}
 9263   ins_encode %{
 9264     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9265                Assembler::word, /*acquire*/ false, /*release*/ true,
 9266                /*weak*/ true, noreg);
 9267     __ csetw($res$$Register, Assembler::EQ);
 9268   %}
 9269   ins_pipe(pipe_slow);
 9270 %}
 9271 
 9272 // This pattern is generated automatically from cas.m4.
 9273 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9274 instruct weakCompareAndSwapL(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 9275 
 9276   match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
 9277   ins_cost(2 * VOLATILE_REF_COST);
 9278   effect(KILL cr);
 9279   format %{
 9280     "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 9281     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9282   %}
 9283   ins_encode %{
 9284     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9285                Assembler::xword, /*acquire*/ false, /*release*/ true,
 9286                /*weak*/ true, noreg);
 9287     __ csetw($res$$Register, Assembler::EQ);
 9288   %}
 9289   ins_pipe(pipe_slow);
 9290 %}
 9291 
 9292 // This pattern is generated automatically from cas.m4.
 9293 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9294 instruct weakCompareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 9295 
 9296   match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
 9297   ins_cost(2 * VOLATILE_REF_COST);
 9298   effect(KILL cr);
 9299   format %{
 9300     "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 9301     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9302   %}
 9303   ins_encode %{
 9304     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9305                Assembler::word, /*acquire*/ false, /*release*/ true,
 9306                /*weak*/ true, noreg);
 9307     __ csetw($res$$Register, Assembler::EQ);
 9308   %}
 9309   ins_pipe(pipe_slow);
 9310 %}
 9311 
 9312 // This pattern is generated automatically from cas.m4.
 9313 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9314 instruct weakCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9315   predicate(n->as_LoadStore()->barrier_data() == 0);
 9316   match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
 9317   ins_cost(2 * VOLATILE_REF_COST);
 9318   effect(KILL cr);
 9319   format %{
 9320     "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 9321     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9322   %}
 9323   ins_encode %{
 9324     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9325                Assembler::xword, /*acquire*/ false, /*release*/ true,
 9326                /*weak*/ true, noreg);
 9327     __ csetw($res$$Register, Assembler::EQ);
 9328   %}
 9329   ins_pipe(pipe_slow);
 9330 %}
 9331 
 9332 // This pattern is generated automatically from cas.m4.
 9333 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9334 instruct weakCompareAndSwapBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9335   predicate(needs_acquiring_load_exclusive(n));
 9336   match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
 9337   ins_cost(VOLATILE_REF_COST);
 9338   effect(KILL cr);
 9339   format %{
 9340     "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 9341     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9342   %}
 9343   ins_encode %{
 9344     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9345                Assembler::byte, /*acquire*/ true, /*release*/ true,
 9346                /*weak*/ true, noreg);
 9347     __ csetw($res$$Register, Assembler::EQ);
 9348   %}
 9349   ins_pipe(pipe_slow);
 9350 %}
 9351 
 9352 // This pattern is generated automatically from cas.m4.
 9353 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9354 instruct weakCompareAndSwapSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9355   predicate(needs_acquiring_load_exclusive(n));
 9356   match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
 9357   ins_cost(VOLATILE_REF_COST);
 9358   effect(KILL cr);
 9359   format %{
 9360     "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 9361     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9362   %}
 9363   ins_encode %{
 9364     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9365                Assembler::halfword, /*acquire*/ true, /*release*/ true,
 9366                /*weak*/ true, noreg);
 9367     __ csetw($res$$Register, Assembler::EQ);
 9368   %}
 9369   ins_pipe(pipe_slow);
 9370 %}
 9371 
 9372 // This pattern is generated automatically from cas.m4.
 9373 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9374 instruct weakCompareAndSwapIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9375   predicate(needs_acquiring_load_exclusive(n));
 9376   match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
 9377   ins_cost(VOLATILE_REF_COST);
 9378   effect(KILL cr);
 9379   format %{
 9380     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 9381     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9382   %}
 9383   ins_encode %{
 9384     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9385                Assembler::word, /*acquire*/ true, /*release*/ true,
 9386                /*weak*/ true, noreg);
 9387     __ csetw($res$$Register, Assembler::EQ);
 9388   %}
 9389   ins_pipe(pipe_slow);
 9390 %}
 9391 
 9392 // This pattern is generated automatically from cas.m4.
 9393 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9394 instruct weakCompareAndSwapLAcq(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 9395   predicate(needs_acquiring_load_exclusive(n));
 9396   match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
 9397   ins_cost(VOLATILE_REF_COST);
 9398   effect(KILL cr);
 9399   format %{
 9400     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 9401     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9402   %}
 9403   ins_encode %{
 9404     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9405                Assembler::xword, /*acquire*/ true, /*release*/ true,
 9406                /*weak*/ true, noreg);
 9407     __ csetw($res$$Register, Assembler::EQ);
 9408   %}
 9409   ins_pipe(pipe_slow);
 9410 %}
 9411 
 9412 // This pattern is generated automatically from cas.m4.
 9413 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9414 instruct weakCompareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 9415   predicate(needs_acquiring_load_exclusive(n));
 9416   match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
 9417   ins_cost(VOLATILE_REF_COST);
 9418   effect(KILL cr);
 9419   format %{
 9420     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 9421     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9422   %}
 9423   ins_encode %{
 9424     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9425                Assembler::word, /*acquire*/ true, /*release*/ true,
 9426                /*weak*/ true, noreg);
 9427     __ csetw($res$$Register, Assembler::EQ);
 9428   %}
 9429   ins_pipe(pipe_slow);
 9430 %}
 9431 
 9432 // This pattern is generated automatically from cas.m4.
 9433 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9434 instruct weakCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9435   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 9436   match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
 9437   ins_cost(VOLATILE_REF_COST);
 9438   effect(KILL cr);
 9439   format %{
 9440     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 9441     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9442   %}
 9443   ins_encode %{
 9444     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9445                Assembler::xword, /*acquire*/ true, /*release*/ true,
 9446                /*weak*/ true, noreg);
 9447     __ csetw($res$$Register, Assembler::EQ);
 9448   %}
 9449   ins_pipe(pipe_slow);
 9450 %}
 9451 
 9452 // END This section of the file is automatically generated. Do not edit --------------
 9453 // ---------------------------------------------------------------------
 9454 
 9455 instruct get_and_setI(indirect mem, iRegI newv, iRegINoSp prev) %{
 9456   match(Set prev (GetAndSetI mem newv));
 9457   ins_cost(2 * VOLATILE_REF_COST);
 9458   format %{ "atomic_xchgw  $prev, $newv, [$mem]" %}
 9459   ins_encode %{
 9460     __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9461   %}
 9462   ins_pipe(pipe_serial);
 9463 %}
 9464 
 9465 instruct get_and_setL(indirect mem, iRegL newv, iRegLNoSp prev) %{
 9466   match(Set prev (GetAndSetL mem newv));
 9467   ins_cost(2 * VOLATILE_REF_COST);
 9468   format %{ "atomic_xchg  $prev, $newv, [$mem]" %}
 9469   ins_encode %{
 9470     __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9471   %}
 9472   ins_pipe(pipe_serial);
 9473 %}
 9474 
 9475 instruct get_and_setN(indirect mem, iRegN newv, iRegINoSp prev) %{
 9476   match(Set prev (GetAndSetN mem newv));
 9477   ins_cost(2 * VOLATILE_REF_COST);
 9478   format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
 9479   ins_encode %{
 9480     __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9481   %}
 9482   ins_pipe(pipe_serial);
 9483 %}
 9484 
 9485 instruct get_and_setP(indirect mem, iRegP newv, iRegPNoSp prev) %{
 9486   predicate(n->as_LoadStore()->barrier_data() == 0);
 9487   match(Set prev (GetAndSetP mem newv));
 9488   ins_cost(2 * VOLATILE_REF_COST);
 9489   format %{ "atomic_xchg  $prev, $newv, [$mem]" %}
 9490   ins_encode %{
 9491     __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9492   %}
 9493   ins_pipe(pipe_serial);
 9494 %}
 9495 
 9496 instruct get_and_setIAcq(indirect mem, iRegI newv, iRegINoSp prev) %{
 9497   predicate(needs_acquiring_load_exclusive(n));
 9498   match(Set prev (GetAndSetI mem newv));
 9499   ins_cost(VOLATILE_REF_COST);
 9500   format %{ "atomic_xchgw_acq  $prev, $newv, [$mem]" %}
 9501   ins_encode %{
 9502     __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9503   %}
 9504   ins_pipe(pipe_serial);
 9505 %}
 9506 
 9507 instruct get_and_setLAcq(indirect mem, iRegL newv, iRegLNoSp prev) %{
 9508   predicate(needs_acquiring_load_exclusive(n));
 9509   match(Set prev (GetAndSetL mem newv));
 9510   ins_cost(VOLATILE_REF_COST);
 9511   format %{ "atomic_xchg_acq  $prev, $newv, [$mem]" %}
 9512   ins_encode %{
 9513     __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9514   %}
 9515   ins_pipe(pipe_serial);
 9516 %}
 9517 
 9518 instruct get_and_setNAcq(indirect mem, iRegN newv, iRegINoSp prev) %{
 9519   predicate(needs_acquiring_load_exclusive(n));
 9520   match(Set prev (GetAndSetN mem newv));
 9521   ins_cost(VOLATILE_REF_COST);
 9522   format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
 9523   ins_encode %{
 9524     __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9525   %}
 9526   ins_pipe(pipe_serial);
 9527 %}
 9528 
 9529 instruct get_and_setPAcq(indirect mem, iRegP newv, iRegPNoSp prev) %{
 9530   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 9531   match(Set prev (GetAndSetP mem newv));
 9532   ins_cost(VOLATILE_REF_COST);
 9533   format %{ "atomic_xchg_acq  $prev, $newv, [$mem]" %}
 9534   ins_encode %{
 9535     __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9536   %}
 9537   ins_pipe(pipe_serial);
 9538 %}
 9539 
 9540 
 9541 instruct get_and_addL(indirect mem, iRegLNoSp newval, iRegL incr) %{
 9542   match(Set newval (GetAndAddL mem incr));
 9543   ins_cost(2 * VOLATILE_REF_COST + 1);
 9544   format %{ "get_and_addL $newval, [$mem], $incr" %}
 9545   ins_encode %{
 9546     __ atomic_add($newval$$Register, $incr$$Register, as_Register($mem$$base));
 9547   %}
 9548   ins_pipe(pipe_serial);
 9549 %}
 9550 
 9551 instruct get_and_addL_no_res(indirect mem, Universe dummy, iRegL incr) %{
 9552   predicate(n->as_LoadStore()->result_not_used());
 9553   match(Set dummy (GetAndAddL mem incr));
 9554   ins_cost(2 * VOLATILE_REF_COST);
 9555   format %{ "get_and_addL [$mem], $incr" %}
 9556   ins_encode %{
 9557     __ atomic_add(noreg, $incr$$Register, as_Register($mem$$base));
 9558   %}
 9559   ins_pipe(pipe_serial);
 9560 %}
 9561 
 9562 instruct get_and_addLi(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
 9563   match(Set newval (GetAndAddL mem incr));
 9564   ins_cost(2 * VOLATILE_REF_COST + 1);
 9565   format %{ "get_and_addL $newval, [$mem], $incr" %}
 9566   ins_encode %{
 9567     __ atomic_add($newval$$Register, $incr$$constant, as_Register($mem$$base));
 9568   %}
 9569   ins_pipe(pipe_serial);
 9570 %}
 9571 
 9572 instruct get_and_addLi_no_res(indirect mem, Universe dummy, immLAddSub incr) %{
 9573   predicate(n->as_LoadStore()->result_not_used());
 9574   match(Set dummy (GetAndAddL mem incr));
 9575   ins_cost(2 * VOLATILE_REF_COST);
 9576   format %{ "get_and_addL [$mem], $incr" %}
 9577   ins_encode %{
 9578     __ atomic_add(noreg, $incr$$constant, as_Register($mem$$base));
 9579   %}
 9580   ins_pipe(pipe_serial);
 9581 %}
 9582 
 9583 instruct get_and_addI(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
 9584   match(Set newval (GetAndAddI mem incr));
 9585   ins_cost(2 * VOLATILE_REF_COST + 1);
 9586   format %{ "get_and_addI $newval, [$mem], $incr" %}
 9587   ins_encode %{
 9588     __ atomic_addw($newval$$Register, $incr$$Register, as_Register($mem$$base));
 9589   %}
 9590   ins_pipe(pipe_serial);
 9591 %}
 9592 
 9593 instruct get_and_addI_no_res(indirect mem, Universe dummy, iRegIorL2I incr) %{
 9594   predicate(n->as_LoadStore()->result_not_used());
 9595   match(Set dummy (GetAndAddI mem incr));
 9596   ins_cost(2 * VOLATILE_REF_COST);
 9597   format %{ "get_and_addI [$mem], $incr" %}
 9598   ins_encode %{
 9599     __ atomic_addw(noreg, $incr$$Register, as_Register($mem$$base));
 9600   %}
 9601   ins_pipe(pipe_serial);
 9602 %}
 9603 
 9604 instruct get_and_addIi(indirect mem, iRegINoSp newval, immIAddSub incr) %{
 9605   match(Set newval (GetAndAddI mem incr));
 9606   ins_cost(2 * VOLATILE_REF_COST + 1);
 9607   format %{ "get_and_addI $newval, [$mem], $incr" %}
 9608   ins_encode %{
 9609     __ atomic_addw($newval$$Register, $incr$$constant, as_Register($mem$$base));
 9610   %}
 9611   ins_pipe(pipe_serial);
 9612 %}
 9613 
 9614 instruct get_and_addIi_no_res(indirect mem, Universe dummy, immIAddSub incr) %{
 9615   predicate(n->as_LoadStore()->result_not_used());
 9616   match(Set dummy (GetAndAddI mem incr));
 9617   ins_cost(2 * VOLATILE_REF_COST);
 9618   format %{ "get_and_addI [$mem], $incr" %}
 9619   ins_encode %{
 9620     __ atomic_addw(noreg, $incr$$constant, as_Register($mem$$base));
 9621   %}
 9622   ins_pipe(pipe_serial);
 9623 %}
 9624 
 9625 instruct get_and_addLAcq(indirect mem, iRegLNoSp newval, iRegL incr) %{
 9626   predicate(needs_acquiring_load_exclusive(n));
 9627   match(Set newval (GetAndAddL mem incr));
 9628   ins_cost(VOLATILE_REF_COST + 1);
 9629   format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
 9630   ins_encode %{
 9631     __ atomic_addal($newval$$Register, $incr$$Register, as_Register($mem$$base));
 9632   %}
 9633   ins_pipe(pipe_serial);
 9634 %}
 9635 
 9636 instruct get_and_addL_no_resAcq(indirect mem, Universe dummy, iRegL incr) %{
 9637   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
 9638   match(Set dummy (GetAndAddL mem incr));
 9639   ins_cost(VOLATILE_REF_COST);
 9640   format %{ "get_and_addL_acq [$mem], $incr" %}
 9641   ins_encode %{
 9642     __ atomic_addal(noreg, $incr$$Register, as_Register($mem$$base));
 9643   %}
 9644   ins_pipe(pipe_serial);
 9645 %}
 9646 
 9647 instruct get_and_addLiAcq(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
 9648   predicate(needs_acquiring_load_exclusive(n));
 9649   match(Set newval (GetAndAddL mem incr));
 9650   ins_cost(VOLATILE_REF_COST + 1);
 9651   format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
 9652   ins_encode %{
 9653     __ atomic_addal($newval$$Register, $incr$$constant, as_Register($mem$$base));
 9654   %}
 9655   ins_pipe(pipe_serial);
 9656 %}
 9657 
 9658 instruct get_and_addLi_no_resAcq(indirect mem, Universe dummy, immLAddSub incr) %{
 9659   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
 9660   match(Set dummy (GetAndAddL mem incr));
 9661   ins_cost(VOLATILE_REF_COST);
 9662   format %{ "get_and_addL_acq [$mem], $incr" %}
 9663   ins_encode %{
 9664     __ atomic_addal(noreg, $incr$$constant, as_Register($mem$$base));
 9665   %}
 9666   ins_pipe(pipe_serial);
 9667 %}
 9668 
 9669 instruct get_and_addIAcq(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
 9670   predicate(needs_acquiring_load_exclusive(n));
 9671   match(Set newval (GetAndAddI mem incr));
 9672   ins_cost(VOLATILE_REF_COST + 1);
 9673   format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
 9674   ins_encode %{
 9675     __ atomic_addalw($newval$$Register, $incr$$Register, as_Register($mem$$base));
 9676   %}
 9677   ins_pipe(pipe_serial);
 9678 %}
 9679 
 9680 instruct get_and_addI_no_resAcq(indirect mem, Universe dummy, iRegIorL2I incr) %{
 9681   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
 9682   match(Set dummy (GetAndAddI mem incr));
 9683   ins_cost(VOLATILE_REF_COST);
 9684   format %{ "get_and_addI_acq [$mem], $incr" %}
 9685   ins_encode %{
 9686     __ atomic_addalw(noreg, $incr$$Register, as_Register($mem$$base));
 9687   %}
 9688   ins_pipe(pipe_serial);
 9689 %}
 9690 
 9691 instruct get_and_addIiAcq(indirect mem, iRegINoSp newval, immIAddSub incr) %{
 9692   predicate(needs_acquiring_load_exclusive(n));
 9693   match(Set newval (GetAndAddI mem incr));
 9694   ins_cost(VOLATILE_REF_COST + 1);
 9695   format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
 9696   ins_encode %{
 9697     __ atomic_addalw($newval$$Register, $incr$$constant, as_Register($mem$$base));
 9698   %}
 9699   ins_pipe(pipe_serial);
 9700 %}
 9701 
 9702 instruct get_and_addIi_no_resAcq(indirect mem, Universe dummy, immIAddSub incr) %{
 9703   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
 9704   match(Set dummy (GetAndAddI mem incr));
 9705   ins_cost(VOLATILE_REF_COST);
 9706   format %{ "get_and_addI_acq [$mem], $incr" %}
 9707   ins_encode %{
 9708     __ atomic_addalw(noreg, $incr$$constant, as_Register($mem$$base));
 9709   %}
 9710   ins_pipe(pipe_serial);
 9711 %}
 9712 
 9713 // Manifest a CmpU result in an integer register.
 9714 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
 9715 instruct cmpU3_reg_reg(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg flags)
 9716 %{
 9717   match(Set dst (CmpU3 src1 src2));
 9718   effect(KILL flags);
 9719 
 9720   ins_cost(INSN_COST * 3);
 9721   format %{
 9722       "cmpw $src1, $src2\n\t"
 9723       "csetw $dst, ne\n\t"
 9724       "cnegw $dst, lo\t# CmpU3(reg)"
 9725   %}
 9726   ins_encode %{
 9727     __ cmpw($src1$$Register, $src2$$Register);
 9728     __ csetw($dst$$Register, Assembler::NE);
 9729     __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
 9730   %}
 9731 
 9732   ins_pipe(pipe_class_default);
 9733 %}
 9734 
 9735 instruct cmpU3_reg_imm(iRegINoSp dst, iRegI src1, immIAddSub src2, rFlagsReg flags)
 9736 %{
 9737   match(Set dst (CmpU3 src1 src2));
 9738   effect(KILL flags);
 9739 
 9740   ins_cost(INSN_COST * 3);
 9741   format %{
 9742       "subsw zr, $src1, $src2\n\t"
 9743       "csetw $dst, ne\n\t"
 9744       "cnegw $dst, lo\t# CmpU3(imm)"
 9745   %}
 9746   ins_encode %{
 9747     __ subsw(zr, $src1$$Register, (int32_t)$src2$$constant);
 9748     __ csetw($dst$$Register, Assembler::NE);
 9749     __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
 9750   %}
 9751 
 9752   ins_pipe(pipe_class_default);
 9753 %}
 9754 
 9755 // Manifest a CmpUL result in an integer register.
 9756 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
 9757 instruct cmpUL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
 9758 %{
 9759   match(Set dst (CmpUL3 src1 src2));
 9760   effect(KILL flags);
 9761 
 9762   ins_cost(INSN_COST * 3);
 9763   format %{
 9764       "cmp $src1, $src2\n\t"
 9765       "csetw $dst, ne\n\t"
 9766       "cnegw $dst, lo\t# CmpUL3(reg)"
 9767   %}
 9768   ins_encode %{
 9769     __ cmp($src1$$Register, $src2$$Register);
 9770     __ csetw($dst$$Register, Assembler::NE);
 9771     __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
 9772   %}
 9773 
 9774   ins_pipe(pipe_class_default);
 9775 %}
 9776 
 9777 instruct cmpUL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
 9778 %{
 9779   match(Set dst (CmpUL3 src1 src2));
 9780   effect(KILL flags);
 9781 
 9782   ins_cost(INSN_COST * 3);
 9783   format %{
 9784       "subs zr, $src1, $src2\n\t"
 9785       "csetw $dst, ne\n\t"
 9786       "cnegw $dst, lo\t# CmpUL3(imm)"
 9787   %}
 9788   ins_encode %{
 9789     __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
 9790     __ csetw($dst$$Register, Assembler::NE);
 9791     __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
 9792   %}
 9793 
 9794   ins_pipe(pipe_class_default);
 9795 %}
 9796 
 9797 // Manifest a CmpL result in an integer register.
 9798 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
 9799 instruct cmpL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
 9800 %{
 9801   match(Set dst (CmpL3 src1 src2));
 9802   effect(KILL flags);
 9803 
 9804   ins_cost(INSN_COST * 3);
 9805   format %{
 9806       "cmp $src1, $src2\n\t"
 9807       "csetw $dst, ne\n\t"
 9808       "cnegw $dst, lt\t# CmpL3(reg)"
 9809   %}
 9810   ins_encode %{
 9811     __ cmp($src1$$Register, $src2$$Register);
 9812     __ csetw($dst$$Register, Assembler::NE);
 9813     __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
 9814   %}
 9815 
 9816   ins_pipe(pipe_class_default);
 9817 %}
 9818 
 9819 instruct cmpL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
 9820 %{
 9821   match(Set dst (CmpL3 src1 src2));
 9822   effect(KILL flags);
 9823 
 9824   ins_cost(INSN_COST * 3);
 9825   format %{
 9826       "subs zr, $src1, $src2\n\t"
 9827       "csetw $dst, ne\n\t"
 9828       "cnegw $dst, lt\t# CmpL3(imm)"
 9829   %}
 9830   ins_encode %{
 9831     __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
 9832     __ csetw($dst$$Register, Assembler::NE);
 9833     __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
 9834   %}
 9835 
 9836   ins_pipe(pipe_class_default);
 9837 %}
 9838 
 9839 // ============================================================================
 9840 // Conditional Move Instructions
 9841 
 9842 // n.b. we have identical rules for both a signed compare op (cmpOp)
 9843 // and an unsigned compare op (cmpOpU). it would be nice if we could
 9844 // define an op class which merged both inputs and use it to type the
 9845 // argument to a single rule. unfortunatelyt his fails because the
 9846 // opclass does not live up to the COND_INTER interface of its
 9847 // component operands. When the generic code tries to negate the
 9848 // operand it ends up running the generci Machoper::negate method
 9849 // which throws a ShouldNotHappen. So, we have to provide two flavours
 9850 // of each rule, one for a cmpOp and a second for a cmpOpU (sigh).
 9851 
 9852 instruct cmovI_reg_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
 9853   match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
 9854 
 9855   ins_cost(INSN_COST * 2);
 9856   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, int"  %}
 9857 
 9858   ins_encode %{
 9859     __ cselw(as_Register($dst$$reg),
 9860              as_Register($src2$$reg),
 9861              as_Register($src1$$reg),
 9862              (Assembler::Condition)$cmp$$cmpcode);
 9863   %}
 9864 
 9865   ins_pipe(icond_reg_reg);
 9866 %}
 9867 
 9868 instruct cmovUI_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
 9869   match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
 9870 
 9871   ins_cost(INSN_COST * 2);
 9872   format %{ "cselw $dst, $src2, $src1 $cmp\t# unsigned, int"  %}
 9873 
 9874   ins_encode %{
 9875     __ cselw(as_Register($dst$$reg),
 9876              as_Register($src2$$reg),
 9877              as_Register($src1$$reg),
 9878              (Assembler::Condition)$cmp$$cmpcode);
 9879   %}
 9880 
 9881   ins_pipe(icond_reg_reg);
 9882 %}
 9883 
 9884 // special cases where one arg is zero
 9885 
 9886 // n.b. this is selected in preference to the rule above because it
 9887 // avoids loading constant 0 into a source register
 9888 
 9889 // TODO
 9890 // we ought only to be able to cull one of these variants as the ideal
 9891 // transforms ought always to order the zero consistently (to left/right?)
 9892 
 9893 instruct cmovI_zero_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
 9894   match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
 9895 
 9896   ins_cost(INSN_COST * 2);
 9897   format %{ "cselw $dst, $src, zr $cmp\t# signed, int"  %}
 9898 
 9899   ins_encode %{
 9900     __ cselw(as_Register($dst$$reg),
 9901              as_Register($src$$reg),
 9902              zr,
 9903              (Assembler::Condition)$cmp$$cmpcode);
 9904   %}
 9905 
 9906   ins_pipe(icond_reg);
 9907 %}
 9908 
 9909 instruct cmovUI_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
 9910   match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
 9911 
 9912   ins_cost(INSN_COST * 2);
 9913   format %{ "cselw $dst, $src, zr $cmp\t# unsigned, int"  %}
 9914 
 9915   ins_encode %{
 9916     __ cselw(as_Register($dst$$reg),
 9917              as_Register($src$$reg),
 9918              zr,
 9919              (Assembler::Condition)$cmp$$cmpcode);
 9920   %}
 9921 
 9922   ins_pipe(icond_reg);
 9923 %}
 9924 
 9925 instruct cmovI_reg_zero(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
 9926   match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
 9927 
 9928   ins_cost(INSN_COST * 2);
 9929   format %{ "cselw $dst, zr, $src $cmp\t# signed, int"  %}
 9930 
 9931   ins_encode %{
 9932     __ cselw(as_Register($dst$$reg),
 9933              zr,
 9934              as_Register($src$$reg),
 9935              (Assembler::Condition)$cmp$$cmpcode);
 9936   %}
 9937 
 9938   ins_pipe(icond_reg);
 9939 %}
 9940 
 9941 instruct cmovUI_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
 9942   match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
 9943 
 9944   ins_cost(INSN_COST * 2);
 9945   format %{ "cselw $dst, zr, $src $cmp\t# unsigned, int"  %}
 9946 
 9947   ins_encode %{
 9948     __ cselw(as_Register($dst$$reg),
 9949              zr,
 9950              as_Register($src$$reg),
 9951              (Assembler::Condition)$cmp$$cmpcode);
 9952   %}
 9953 
 9954   ins_pipe(icond_reg);
 9955 %}
 9956 
 9957 // special case for creating a boolean 0 or 1
 9958 
 9959 // n.b. this is selected in preference to the rule above because it
 9960 // avoids loading constants 0 and 1 into a source register
 9961 
 9962 instruct cmovI_reg_zero_one(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
 9963   match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
 9964 
 9965   ins_cost(INSN_COST * 2);
 9966   format %{ "csincw $dst, zr, zr $cmp\t# signed, int"  %}
 9967 
 9968   ins_encode %{
 9969     // equivalently
 9970     // cset(as_Register($dst$$reg),
 9971     //      negate_condition((Assembler::Condition)$cmp$$cmpcode));
 9972     __ csincw(as_Register($dst$$reg),
 9973              zr,
 9974              zr,
 9975              (Assembler::Condition)$cmp$$cmpcode);
 9976   %}
 9977 
 9978   ins_pipe(icond_none);
 9979 %}
 9980 
 9981 instruct cmovUI_reg_zero_one(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
 9982   match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
 9983 
 9984   ins_cost(INSN_COST * 2);
 9985   format %{ "csincw $dst, zr, zr $cmp\t# unsigned, int"  %}
 9986 
 9987   ins_encode %{
 9988     // equivalently
 9989     // cset(as_Register($dst$$reg),
 9990     //      negate_condition((Assembler::Condition)$cmp$$cmpcode));
 9991     __ csincw(as_Register($dst$$reg),
 9992              zr,
 9993              zr,
 9994              (Assembler::Condition)$cmp$$cmpcode);
 9995   %}
 9996 
 9997   ins_pipe(icond_none);
 9998 %}
 9999 
10000 instruct cmovL_reg_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
10001   match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
10002 
10003   ins_cost(INSN_COST * 2);
10004   format %{ "csel $dst, $src2, $src1 $cmp\t# signed, long"  %}
10005 
10006   ins_encode %{
10007     __ csel(as_Register($dst$$reg),
10008             as_Register($src2$$reg),
10009             as_Register($src1$$reg),
10010             (Assembler::Condition)$cmp$$cmpcode);
10011   %}
10012 
10013   ins_pipe(icond_reg_reg);
10014 %}
10015 
10016 instruct cmovUL_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
10017   match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
10018 
10019   ins_cost(INSN_COST * 2);
10020   format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, long"  %}
10021 
10022   ins_encode %{
10023     __ csel(as_Register($dst$$reg),
10024             as_Register($src2$$reg),
10025             as_Register($src1$$reg),
10026             (Assembler::Condition)$cmp$$cmpcode);
10027   %}
10028 
10029   ins_pipe(icond_reg_reg);
10030 %}
10031 
10032 // special cases where one arg is zero
10033 
10034 instruct cmovL_reg_zero(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
10035   match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
10036 
10037   ins_cost(INSN_COST * 2);
10038   format %{ "csel $dst, zr, $src $cmp\t# signed, long"  %}
10039 
10040   ins_encode %{
10041     __ csel(as_Register($dst$$reg),
10042             zr,
10043             as_Register($src$$reg),
10044             (Assembler::Condition)$cmp$$cmpcode);
10045   %}
10046 
10047   ins_pipe(icond_reg);
10048 %}
10049 
10050 instruct cmovUL_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
10051   match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
10052 
10053   ins_cost(INSN_COST * 2);
10054   format %{ "csel $dst, zr, $src $cmp\t# unsigned, long"  %}
10055 
10056   ins_encode %{
10057     __ csel(as_Register($dst$$reg),
10058             zr,
10059             as_Register($src$$reg),
10060             (Assembler::Condition)$cmp$$cmpcode);
10061   %}
10062 
10063   ins_pipe(icond_reg);
10064 %}
10065 
10066 instruct cmovL_zero_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
10067   match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
10068 
10069   ins_cost(INSN_COST * 2);
10070   format %{ "csel $dst, $src, zr $cmp\t# signed, long"  %}
10071 
10072   ins_encode %{
10073     __ csel(as_Register($dst$$reg),
10074             as_Register($src$$reg),
10075             zr,
10076             (Assembler::Condition)$cmp$$cmpcode);
10077   %}
10078 
10079   ins_pipe(icond_reg);
10080 %}
10081 
10082 instruct cmovUL_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
10083   match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
10084 
10085   ins_cost(INSN_COST * 2);
10086   format %{ "csel $dst, $src, zr $cmp\t# unsigned, long"  %}
10087 
10088   ins_encode %{
10089     __ csel(as_Register($dst$$reg),
10090             as_Register($src$$reg),
10091             zr,
10092             (Assembler::Condition)$cmp$$cmpcode);
10093   %}
10094 
10095   ins_pipe(icond_reg);
10096 %}
10097 
10098 instruct cmovP_reg_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
10099   match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
10100 
10101   ins_cost(INSN_COST * 2);
10102   format %{ "csel $dst, $src2, $src1 $cmp\t# signed, ptr"  %}
10103 
10104   ins_encode %{
10105     __ csel(as_Register($dst$$reg),
10106             as_Register($src2$$reg),
10107             as_Register($src1$$reg),
10108             (Assembler::Condition)$cmp$$cmpcode);
10109   %}
10110 
10111   ins_pipe(icond_reg_reg);
10112 %}
10113 
10114 instruct cmovUP_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
10115   match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
10116 
10117   ins_cost(INSN_COST * 2);
10118   format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, ptr"  %}
10119 
10120   ins_encode %{
10121     __ csel(as_Register($dst$$reg),
10122             as_Register($src2$$reg),
10123             as_Register($src1$$reg),
10124             (Assembler::Condition)$cmp$$cmpcode);
10125   %}
10126 
10127   ins_pipe(icond_reg_reg);
10128 %}
10129 
10130 // special cases where one arg is zero
10131 
10132 instruct cmovP_reg_zero(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
10133   match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
10134 
10135   ins_cost(INSN_COST * 2);
10136   format %{ "csel $dst, zr, $src $cmp\t# signed, ptr"  %}
10137 
10138   ins_encode %{
10139     __ csel(as_Register($dst$$reg),
10140             zr,
10141             as_Register($src$$reg),
10142             (Assembler::Condition)$cmp$$cmpcode);
10143   %}
10144 
10145   ins_pipe(icond_reg);
10146 %}
10147 
10148 instruct cmovUP_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
10149   match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
10150 
10151   ins_cost(INSN_COST * 2);
10152   format %{ "csel $dst, zr, $src $cmp\t# unsigned, ptr"  %}
10153 
10154   ins_encode %{
10155     __ csel(as_Register($dst$$reg),
10156             zr,
10157             as_Register($src$$reg),
10158             (Assembler::Condition)$cmp$$cmpcode);
10159   %}
10160 
10161   ins_pipe(icond_reg);
10162 %}
10163 
10164 instruct cmovP_zero_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
10165   match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
10166 
10167   ins_cost(INSN_COST * 2);
10168   format %{ "csel $dst, $src, zr $cmp\t# signed, ptr"  %}
10169 
10170   ins_encode %{
10171     __ csel(as_Register($dst$$reg),
10172             as_Register($src$$reg),
10173             zr,
10174             (Assembler::Condition)$cmp$$cmpcode);
10175   %}
10176 
10177   ins_pipe(icond_reg);
10178 %}
10179 
10180 instruct cmovUP_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
10181   match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
10182 
10183   ins_cost(INSN_COST * 2);
10184   format %{ "csel $dst, $src, zr $cmp\t# unsigned, ptr"  %}
10185 
10186   ins_encode %{
10187     __ csel(as_Register($dst$$reg),
10188             as_Register($src$$reg),
10189             zr,
10190             (Assembler::Condition)$cmp$$cmpcode);
10191   %}
10192 
10193   ins_pipe(icond_reg);
10194 %}
10195 
10196 instruct cmovN_reg_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
10197   match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
10198 
10199   ins_cost(INSN_COST * 2);
10200   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr"  %}
10201 
10202   ins_encode %{
10203     __ cselw(as_Register($dst$$reg),
10204              as_Register($src2$$reg),
10205              as_Register($src1$$reg),
10206              (Assembler::Condition)$cmp$$cmpcode);
10207   %}
10208 
10209   ins_pipe(icond_reg_reg);
10210 %}
10211 
10212 instruct cmovUN_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
10213   match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
10214 
10215   ins_cost(INSN_COST * 2);
10216   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr"  %}
10217 
10218   ins_encode %{
10219     __ cselw(as_Register($dst$$reg),
10220              as_Register($src2$$reg),
10221              as_Register($src1$$reg),
10222              (Assembler::Condition)$cmp$$cmpcode);
10223   %}
10224 
10225   ins_pipe(icond_reg_reg);
10226 %}
10227 
10228 // special cases where one arg is zero
10229 
10230 instruct cmovN_reg_zero(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
10231   match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
10232 
10233   ins_cost(INSN_COST * 2);
10234   format %{ "cselw $dst, zr, $src $cmp\t# signed, compressed ptr"  %}
10235 
10236   ins_encode %{
10237     __ cselw(as_Register($dst$$reg),
10238              zr,
10239              as_Register($src$$reg),
10240              (Assembler::Condition)$cmp$$cmpcode);
10241   %}
10242 
10243   ins_pipe(icond_reg);
10244 %}
10245 
10246 instruct cmovUN_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
10247   match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
10248 
10249   ins_cost(INSN_COST * 2);
10250   format %{ "cselw $dst, zr, $src $cmp\t# unsigned, compressed ptr"  %}
10251 
10252   ins_encode %{
10253     __ cselw(as_Register($dst$$reg),
10254              zr,
10255              as_Register($src$$reg),
10256              (Assembler::Condition)$cmp$$cmpcode);
10257   %}
10258 
10259   ins_pipe(icond_reg);
10260 %}
10261 
10262 instruct cmovN_zero_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
10263   match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
10264 
10265   ins_cost(INSN_COST * 2);
10266   format %{ "cselw $dst, $src, zr $cmp\t# signed, compressed ptr"  %}
10267 
10268   ins_encode %{
10269     __ cselw(as_Register($dst$$reg),
10270              as_Register($src$$reg),
10271              zr,
10272              (Assembler::Condition)$cmp$$cmpcode);
10273   %}
10274 
10275   ins_pipe(icond_reg);
10276 %}
10277 
10278 instruct cmovUN_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
10279   match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
10280 
10281   ins_cost(INSN_COST * 2);
10282   format %{ "cselw $dst, $src, zr $cmp\t# unsigned, compressed ptr"  %}
10283 
10284   ins_encode %{
10285     __ cselw(as_Register($dst$$reg),
10286              as_Register($src$$reg),
10287              zr,
10288              (Assembler::Condition)$cmp$$cmpcode);
10289   %}
10290 
10291   ins_pipe(icond_reg);
10292 %}
10293 
10294 instruct cmovF_reg(cmpOp cmp, rFlagsReg cr, vRegF dst, vRegF src1,  vRegF src2)
10295 %{
10296   match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
10297 
10298   ins_cost(INSN_COST * 3);
10299 
10300   format %{ "fcsels $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
10301   ins_encode %{
10302     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10303     __ fcsels(as_FloatRegister($dst$$reg),
10304               as_FloatRegister($src2$$reg),
10305               as_FloatRegister($src1$$reg),
10306               cond);
10307   %}
10308 
10309   ins_pipe(fp_cond_reg_reg_s);
10310 %}
10311 
10312 instruct cmovUF_reg(cmpOpU cmp, rFlagsRegU cr, vRegF dst, vRegF src1,  vRegF src2)
10313 %{
10314   match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
10315 
10316   ins_cost(INSN_COST * 3);
10317 
10318   format %{ "fcsels $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
10319   ins_encode %{
10320     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10321     __ fcsels(as_FloatRegister($dst$$reg),
10322               as_FloatRegister($src2$$reg),
10323               as_FloatRegister($src1$$reg),
10324               cond);
10325   %}
10326 
10327   ins_pipe(fp_cond_reg_reg_s);
10328 %}
10329 
10330 instruct cmovD_reg(cmpOp cmp, rFlagsReg cr, vRegD dst, vRegD src1,  vRegD src2)
10331 %{
10332   match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
10333 
10334   ins_cost(INSN_COST * 3);
10335 
10336   format %{ "fcseld $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
10337   ins_encode %{
10338     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10339     __ fcseld(as_FloatRegister($dst$$reg),
10340               as_FloatRegister($src2$$reg),
10341               as_FloatRegister($src1$$reg),
10342               cond);
10343   %}
10344 
10345   ins_pipe(fp_cond_reg_reg_d);
10346 %}
10347 
10348 instruct cmovUD_reg(cmpOpU cmp, rFlagsRegU cr, vRegD dst, vRegD src1,  vRegD src2)
10349 %{
10350   match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
10351 
10352   ins_cost(INSN_COST * 3);
10353 
10354   format %{ "fcseld $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
10355   ins_encode %{
10356     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10357     __ fcseld(as_FloatRegister($dst$$reg),
10358               as_FloatRegister($src2$$reg),
10359               as_FloatRegister($src1$$reg),
10360               cond);
10361   %}
10362 
10363   ins_pipe(fp_cond_reg_reg_d);
10364 %}
10365 
10366 // ============================================================================
10367 // Arithmetic Instructions
10368 //
10369 
10370 // Integer Addition
10371 
10372 // TODO
10373 // these currently employ operations which do not set CR and hence are
10374 // not flagged as killing CR but we would like to isolate the cases
10375 // where we want to set flags from those where we don't. need to work
10376 // out how to do that.
10377 
10378 instruct addI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10379   match(Set dst (AddI src1 src2));
10380 
10381   ins_cost(INSN_COST);
10382   format %{ "addw  $dst, $src1, $src2" %}
10383 
10384   ins_encode %{
10385     __ addw(as_Register($dst$$reg),
10386             as_Register($src1$$reg),
10387             as_Register($src2$$reg));
10388   %}
10389 
10390   ins_pipe(ialu_reg_reg);
10391 %}
10392 
10393 instruct addI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10394   match(Set dst (AddI src1 src2));
10395 
10396   ins_cost(INSN_COST);
10397   format %{ "addw $dst, $src1, $src2" %}
10398 
10399   // use opcode to indicate that this is an add not a sub
10400   opcode(0x0);
10401 
10402   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10403 
10404   ins_pipe(ialu_reg_imm);
10405 %}
10406 
10407 instruct addI_reg_imm_i2l(iRegINoSp dst, iRegL src1, immIAddSub src2) %{
10408   match(Set dst (AddI (ConvL2I src1) src2));
10409 
10410   ins_cost(INSN_COST);
10411   format %{ "addw $dst, $src1, $src2" %}
10412 
10413   // use opcode to indicate that this is an add not a sub
10414   opcode(0x0);
10415 
10416   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10417 
10418   ins_pipe(ialu_reg_imm);
10419 %}
10420 
10421 // Pointer Addition
10422 instruct addP_reg_reg(iRegPNoSp dst, iRegP src1, iRegL src2) %{
10423   match(Set dst (AddP src1 src2));
10424 
10425   ins_cost(INSN_COST);
10426   format %{ "add $dst, $src1, $src2\t# ptr" %}
10427 
10428   ins_encode %{
10429     __ add(as_Register($dst$$reg),
10430            as_Register($src1$$reg),
10431            as_Register($src2$$reg));
10432   %}
10433 
10434   ins_pipe(ialu_reg_reg);
10435 %}
10436 
10437 instruct addP_reg_reg_ext(iRegPNoSp dst, iRegP src1, iRegIorL2I src2) %{
10438   match(Set dst (AddP src1 (ConvI2L src2)));
10439 
10440   ins_cost(1.9 * INSN_COST);
10441   format %{ "add $dst, $src1, $src2, sxtw\t# ptr" %}
10442 
10443   ins_encode %{
10444     __ add(as_Register($dst$$reg),
10445            as_Register($src1$$reg),
10446            as_Register($src2$$reg), ext::sxtw);
10447   %}
10448 
10449   ins_pipe(ialu_reg_reg);
10450 %}
10451 
10452 instruct addP_reg_reg_lsl(iRegPNoSp dst, iRegP src1, iRegL src2, immIScale scale) %{
10453   match(Set dst (AddP src1 (LShiftL src2 scale)));
10454 
10455   ins_cost(1.9 * INSN_COST);
10456   format %{ "add $dst, $src1, $src2, LShiftL $scale\t# ptr" %}
10457 
10458   ins_encode %{
10459     __ lea(as_Register($dst$$reg),
10460            Address(as_Register($src1$$reg), as_Register($src2$$reg),
10461                    Address::lsl($scale$$constant)));
10462   %}
10463 
10464   ins_pipe(ialu_reg_reg_shift);
10465 %}
10466 
10467 instruct addP_reg_reg_ext_shift(iRegPNoSp dst, iRegP src1, iRegIorL2I src2, immIScale scale) %{
10468   match(Set dst (AddP src1 (LShiftL (ConvI2L src2) scale)));
10469 
10470   ins_cost(1.9 * INSN_COST);
10471   format %{ "add $dst, $src1, $src2, I2L $scale\t# ptr" %}
10472 
10473   ins_encode %{
10474     __ lea(as_Register($dst$$reg),
10475            Address(as_Register($src1$$reg), as_Register($src2$$reg),
10476                    Address::sxtw($scale$$constant)));
10477   %}
10478 
10479   ins_pipe(ialu_reg_reg_shift);
10480 %}
10481 
10482 instruct lshift_ext(iRegLNoSp dst, iRegIorL2I src, immI scale, rFlagsReg cr) %{
10483   match(Set dst (LShiftL (ConvI2L src) scale));
10484 
10485   ins_cost(INSN_COST);
10486   format %{ "sbfiz $dst, $src, $scale & 63, -$scale & 63\t" %}
10487 
10488   ins_encode %{
10489     __ sbfiz(as_Register($dst$$reg),
10490           as_Register($src$$reg),
10491           $scale$$constant & 63, MIN2(32, (int)((-$scale$$constant) & 63)));
10492   %}
10493 
10494   ins_pipe(ialu_reg_shift);
10495 %}
10496 
10497 // Pointer Immediate Addition
10498 // n.b. this needs to be more expensive than using an indirect memory
10499 // operand
10500 instruct addP_reg_imm(iRegPNoSp dst, iRegP src1, immLAddSub src2) %{
10501   match(Set dst (AddP src1 src2));
10502 
10503   ins_cost(INSN_COST);
10504   format %{ "add $dst, $src1, $src2\t# ptr" %}
10505 
10506   // use opcode to indicate that this is an add not a sub
10507   opcode(0x0);
10508 
10509   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10510 
10511   ins_pipe(ialu_reg_imm);
10512 %}
10513 
10514 // Long Addition
10515 instruct addL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10516 
10517   match(Set dst (AddL src1 src2));
10518 
10519   ins_cost(INSN_COST);
10520   format %{ "add  $dst, $src1, $src2" %}
10521 
10522   ins_encode %{
10523     __ add(as_Register($dst$$reg),
10524            as_Register($src1$$reg),
10525            as_Register($src2$$reg));
10526   %}
10527 
10528   ins_pipe(ialu_reg_reg);
10529 %}
10530 
10531 // No constant pool entries requiredLong Immediate Addition.
10532 instruct addL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10533   match(Set dst (AddL src1 src2));
10534 
10535   ins_cost(INSN_COST);
10536   format %{ "add $dst, $src1, $src2" %}
10537 
10538   // use opcode to indicate that this is an add not a sub
10539   opcode(0x0);
10540 
10541   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10542 
10543   ins_pipe(ialu_reg_imm);
10544 %}
10545 
10546 // Integer Subtraction
10547 instruct subI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10548   match(Set dst (SubI src1 src2));
10549 
10550   ins_cost(INSN_COST);
10551   format %{ "subw  $dst, $src1, $src2" %}
10552 
10553   ins_encode %{
10554     __ subw(as_Register($dst$$reg),
10555             as_Register($src1$$reg),
10556             as_Register($src2$$reg));
10557   %}
10558 
10559   ins_pipe(ialu_reg_reg);
10560 %}
10561 
10562 // Immediate Subtraction
10563 instruct subI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10564   match(Set dst (SubI src1 src2));
10565 
10566   ins_cost(INSN_COST);
10567   format %{ "subw $dst, $src1, $src2" %}
10568 
10569   // use opcode to indicate that this is a sub not an add
10570   opcode(0x1);
10571 
10572   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10573 
10574   ins_pipe(ialu_reg_imm);
10575 %}
10576 
10577 // Long Subtraction
10578 instruct subL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10579 
10580   match(Set dst (SubL src1 src2));
10581 
10582   ins_cost(INSN_COST);
10583   format %{ "sub  $dst, $src1, $src2" %}
10584 
10585   ins_encode %{
10586     __ sub(as_Register($dst$$reg),
10587            as_Register($src1$$reg),
10588            as_Register($src2$$reg));
10589   %}
10590 
10591   ins_pipe(ialu_reg_reg);
10592 %}
10593 
10594 // No constant pool entries requiredLong Immediate Subtraction.
10595 instruct subL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10596   match(Set dst (SubL src1 src2));
10597 
10598   ins_cost(INSN_COST);
10599   format %{ "sub$dst, $src1, $src2" %}
10600 
10601   // use opcode to indicate that this is a sub not an add
10602   opcode(0x1);
10603 
10604   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10605 
10606   ins_pipe(ialu_reg_imm);
10607 %}
10608 
10609 // Integer Negation (special case for sub)
10610 
10611 instruct negI_reg(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr) %{
10612   match(Set dst (SubI zero src));
10613 
10614   ins_cost(INSN_COST);
10615   format %{ "negw $dst, $src\t# int" %}
10616 
10617   ins_encode %{
10618     __ negw(as_Register($dst$$reg),
10619             as_Register($src$$reg));
10620   %}
10621 
10622   ins_pipe(ialu_reg);
10623 %}
10624 
10625 // Long Negation
10626 
10627 instruct negL_reg(iRegLNoSp dst, iRegL src, immL0 zero, rFlagsReg cr) %{
10628   match(Set dst (SubL zero src));
10629 
10630   ins_cost(INSN_COST);
10631   format %{ "neg $dst, $src\t# long" %}
10632 
10633   ins_encode %{
10634     __ neg(as_Register($dst$$reg),
10635            as_Register($src$$reg));
10636   %}
10637 
10638   ins_pipe(ialu_reg);
10639 %}
10640 
10641 // Integer Multiply
10642 
10643 instruct mulI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10644   match(Set dst (MulI src1 src2));
10645 
10646   ins_cost(INSN_COST * 3);
10647   format %{ "mulw  $dst, $src1, $src2" %}
10648 
10649   ins_encode %{
10650     __ mulw(as_Register($dst$$reg),
10651             as_Register($src1$$reg),
10652             as_Register($src2$$reg));
10653   %}
10654 
10655   ins_pipe(imul_reg_reg);
10656 %}
10657 
10658 instruct smulI(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10659   match(Set dst (MulL (ConvI2L src1) (ConvI2L src2)));
10660 
10661   ins_cost(INSN_COST * 3);
10662   format %{ "smull  $dst, $src1, $src2" %}
10663 
10664   ins_encode %{
10665     __ smull(as_Register($dst$$reg),
10666              as_Register($src1$$reg),
10667              as_Register($src2$$reg));
10668   %}
10669 
10670   ins_pipe(imul_reg_reg);
10671 %}
10672 
10673 // Long Multiply
10674 
10675 instruct mulL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10676   match(Set dst (MulL src1 src2));
10677 
10678   ins_cost(INSN_COST * 5);
10679   format %{ "mul  $dst, $src1, $src2" %}
10680 
10681   ins_encode %{
10682     __ mul(as_Register($dst$$reg),
10683            as_Register($src1$$reg),
10684            as_Register($src2$$reg));
10685   %}
10686 
10687   ins_pipe(lmul_reg_reg);
10688 %}
10689 
10690 instruct mulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10691 %{
10692   match(Set dst (MulHiL src1 src2));
10693 
10694   ins_cost(INSN_COST * 7);
10695   format %{ "smulh   $dst, $src1, $src2\t# mulhi" %}
10696 
10697   ins_encode %{
10698     __ smulh(as_Register($dst$$reg),
10699              as_Register($src1$$reg),
10700              as_Register($src2$$reg));
10701   %}
10702 
10703   ins_pipe(lmul_reg_reg);
10704 %}
10705 
10706 instruct umulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10707 %{
10708   match(Set dst (UMulHiL src1 src2));
10709 
10710   ins_cost(INSN_COST * 7);
10711   format %{ "umulh   $dst, $src1, $src2\t# umulhi" %}
10712 
10713   ins_encode %{
10714     __ umulh(as_Register($dst$$reg),
10715              as_Register($src1$$reg),
10716              as_Register($src2$$reg));
10717   %}
10718 
10719   ins_pipe(lmul_reg_reg);
10720 %}
10721 
10722 // Combined Integer Multiply & Add/Sub
10723 
10724 instruct maddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10725   match(Set dst (AddI src3 (MulI src1 src2)));
10726 
10727   ins_cost(INSN_COST * 3);
10728   format %{ "madd  $dst, $src1, $src2, $src3" %}
10729 
10730   ins_encode %{
10731     __ maddw(as_Register($dst$$reg),
10732              as_Register($src1$$reg),
10733              as_Register($src2$$reg),
10734              as_Register($src3$$reg));
10735   %}
10736 
10737   ins_pipe(imac_reg_reg);
10738 %}
10739 
10740 instruct msubI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10741   match(Set dst (SubI src3 (MulI src1 src2)));
10742 
10743   ins_cost(INSN_COST * 3);
10744   format %{ "msub  $dst, $src1, $src2, $src3" %}
10745 
10746   ins_encode %{
10747     __ msubw(as_Register($dst$$reg),
10748              as_Register($src1$$reg),
10749              as_Register($src2$$reg),
10750              as_Register($src3$$reg));
10751   %}
10752 
10753   ins_pipe(imac_reg_reg);
10754 %}
10755 
10756 // Combined Integer Multiply & Neg
10757 
10758 instruct mnegI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI0 zero) %{
10759   match(Set dst (MulI (SubI zero src1) src2));
10760 
10761   ins_cost(INSN_COST * 3);
10762   format %{ "mneg  $dst, $src1, $src2" %}
10763 
10764   ins_encode %{
10765     __ mnegw(as_Register($dst$$reg),
10766              as_Register($src1$$reg),
10767              as_Register($src2$$reg));
10768   %}
10769 
10770   ins_pipe(imac_reg_reg);
10771 %}
10772 
10773 // Combined Long Multiply & Add/Sub
10774 
10775 instruct maddL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10776   match(Set dst (AddL src3 (MulL src1 src2)));
10777 
10778   ins_cost(INSN_COST * 5);
10779   format %{ "madd  $dst, $src1, $src2, $src3" %}
10780 
10781   ins_encode %{
10782     __ madd(as_Register($dst$$reg),
10783             as_Register($src1$$reg),
10784             as_Register($src2$$reg),
10785             as_Register($src3$$reg));
10786   %}
10787 
10788   ins_pipe(lmac_reg_reg);
10789 %}
10790 
10791 instruct msubL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10792   match(Set dst (SubL src3 (MulL src1 src2)));
10793 
10794   ins_cost(INSN_COST * 5);
10795   format %{ "msub  $dst, $src1, $src2, $src3" %}
10796 
10797   ins_encode %{
10798     __ msub(as_Register($dst$$reg),
10799             as_Register($src1$$reg),
10800             as_Register($src2$$reg),
10801             as_Register($src3$$reg));
10802   %}
10803 
10804   ins_pipe(lmac_reg_reg);
10805 %}
10806 
10807 // Combined Long Multiply & Neg
10808 
10809 instruct mnegL(iRegLNoSp dst, iRegL src1, iRegL src2, immL0 zero) %{
10810   match(Set dst (MulL (SubL zero src1) src2));
10811 
10812   ins_cost(INSN_COST * 5);
10813   format %{ "mneg  $dst, $src1, $src2" %}
10814 
10815   ins_encode %{
10816     __ mneg(as_Register($dst$$reg),
10817             as_Register($src1$$reg),
10818             as_Register($src2$$reg));
10819   %}
10820 
10821   ins_pipe(lmac_reg_reg);
10822 %}
10823 
10824 // Combine Integer Signed Multiply & Add/Sub/Neg Long
10825 
10826 instruct smaddL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10827   match(Set dst (AddL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10828 
10829   ins_cost(INSN_COST * 3);
10830   format %{ "smaddl  $dst, $src1, $src2, $src3" %}
10831 
10832   ins_encode %{
10833     __ smaddl(as_Register($dst$$reg),
10834               as_Register($src1$$reg),
10835               as_Register($src2$$reg),
10836               as_Register($src3$$reg));
10837   %}
10838 
10839   ins_pipe(imac_reg_reg);
10840 %}
10841 
10842 instruct smsubL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10843   match(Set dst (SubL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10844 
10845   ins_cost(INSN_COST * 3);
10846   format %{ "smsubl  $dst, $src1, $src2, $src3" %}
10847 
10848   ins_encode %{
10849     __ smsubl(as_Register($dst$$reg),
10850               as_Register($src1$$reg),
10851               as_Register($src2$$reg),
10852               as_Register($src3$$reg));
10853   %}
10854 
10855   ins_pipe(imac_reg_reg);
10856 %}
10857 
10858 instruct smnegL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, immL0 zero) %{
10859   match(Set dst (MulL (SubL zero (ConvI2L src1)) (ConvI2L src2)));
10860 
10861   ins_cost(INSN_COST * 3);
10862   format %{ "smnegl  $dst, $src1, $src2" %}
10863 
10864   ins_encode %{
10865     __ smnegl(as_Register($dst$$reg),
10866               as_Register($src1$$reg),
10867               as_Register($src2$$reg));
10868   %}
10869 
10870   ins_pipe(imac_reg_reg);
10871 %}
10872 
10873 // Combined Multiply-Add Shorts into Integer (dst = src1 * src2 + src3 * src4)
10874 
10875 instruct muladdS2I(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3, iRegIorL2I src4) %{
10876   match(Set dst (MulAddS2I (Binary src1 src2) (Binary src3 src4)));
10877 
10878   ins_cost(INSN_COST * 5);
10879   format %{ "mulw  rscratch1, $src1, $src2\n\t"
10880             "maddw $dst, $src3, $src4, rscratch1" %}
10881 
10882   ins_encode %{
10883     __ mulw(rscratch1, as_Register($src1$$reg), as_Register($src2$$reg));
10884     __ maddw(as_Register($dst$$reg), as_Register($src3$$reg), as_Register($src4$$reg), rscratch1); %}
10885 
10886   ins_pipe(imac_reg_reg);
10887 %}
10888 
10889 // Integer Divide
10890 
10891 instruct divI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10892   match(Set dst (DivI src1 src2));
10893 
10894   ins_cost(INSN_COST * 19);
10895   format %{ "sdivw  $dst, $src1, $src2" %}
10896 
10897   ins_encode(aarch64_enc_divw(dst, src1, src2));
10898   ins_pipe(idiv_reg_reg);
10899 %}
10900 
10901 // Long Divide
10902 
10903 instruct divL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10904   match(Set dst (DivL src1 src2));
10905 
10906   ins_cost(INSN_COST * 35);
10907   format %{ "sdiv   $dst, $src1, $src2" %}
10908 
10909   ins_encode(aarch64_enc_div(dst, src1, src2));
10910   ins_pipe(ldiv_reg_reg);
10911 %}
10912 
10913 // Integer Remainder
10914 
10915 instruct modI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10916   match(Set dst (ModI src1 src2));
10917 
10918   ins_cost(INSN_COST * 22);
10919   format %{ "sdivw  rscratch1, $src1, $src2\n\t"
10920             "msubw  $dst, rscratch1, $src2, $src1" %}
10921 
10922   ins_encode(aarch64_enc_modw(dst, src1, src2));
10923   ins_pipe(idiv_reg_reg);
10924 %}
10925 
10926 // Long Remainder
10927 
10928 instruct modL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10929   match(Set dst (ModL src1 src2));
10930 
10931   ins_cost(INSN_COST * 38);
10932   format %{ "sdiv   rscratch1, $src1, $src2\n"
10933             "msub   $dst, rscratch1, $src2, $src1" %}
10934 
10935   ins_encode(aarch64_enc_mod(dst, src1, src2));
10936   ins_pipe(ldiv_reg_reg);
10937 %}
10938 
10939 // Unsigned Integer Divide
10940 
10941 instruct UdivI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10942   match(Set dst (UDivI src1 src2));
10943 
10944   ins_cost(INSN_COST * 19);
10945   format %{ "udivw  $dst, $src1, $src2" %}
10946 
10947   ins_encode %{
10948     __ udivw($dst$$Register, $src1$$Register, $src2$$Register);
10949   %}
10950 
10951   ins_pipe(idiv_reg_reg);
10952 %}
10953 
10954 //  Unsigned Long Divide
10955 
10956 instruct UdivL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10957   match(Set dst (UDivL src1 src2));
10958 
10959   ins_cost(INSN_COST * 35);
10960   format %{ "udiv   $dst, $src1, $src2" %}
10961 
10962   ins_encode %{
10963     __ udiv($dst$$Register, $src1$$Register, $src2$$Register);
10964   %}
10965 
10966   ins_pipe(ldiv_reg_reg);
10967 %}
10968 
10969 // Unsigned Integer Remainder
10970 
10971 instruct UmodI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10972   match(Set dst (UModI src1 src2));
10973 
10974   ins_cost(INSN_COST * 22);
10975   format %{ "udivw  rscratch1, $src1, $src2\n\t"
10976             "msubw  $dst, rscratch1, $src2, $src1" %}
10977 
10978   ins_encode %{
10979     __ udivw(rscratch1, $src1$$Register, $src2$$Register);
10980     __ msubw($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
10981   %}
10982 
10983   ins_pipe(idiv_reg_reg);
10984 %}
10985 
10986 // Unsigned Long Remainder
10987 
10988 instruct UModL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10989   match(Set dst (UModL src1 src2));
10990 
10991   ins_cost(INSN_COST * 38);
10992   format %{ "udiv   rscratch1, $src1, $src2\n"
10993             "msub   $dst, rscratch1, $src2, $src1" %}
10994 
10995   ins_encode %{
10996     __ udiv(rscratch1, $src1$$Register, $src2$$Register);
10997     __ msub($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
10998   %}
10999 
11000   ins_pipe(ldiv_reg_reg);
11001 %}
11002 
11003 // Integer Shifts
11004 
11005 // Shift Left Register
11006 instruct lShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11007   match(Set dst (LShiftI src1 src2));
11008 
11009   ins_cost(INSN_COST * 2);
11010   format %{ "lslvw  $dst, $src1, $src2" %}
11011 
11012   ins_encode %{
11013     __ lslvw(as_Register($dst$$reg),
11014              as_Register($src1$$reg),
11015              as_Register($src2$$reg));
11016   %}
11017 
11018   ins_pipe(ialu_reg_reg_vshift);
11019 %}
11020 
11021 // Shift Left Immediate
11022 instruct lShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
11023   match(Set dst (LShiftI src1 src2));
11024 
11025   ins_cost(INSN_COST);
11026   format %{ "lslw $dst, $src1, ($src2 & 0x1f)" %}
11027 
11028   ins_encode %{
11029     __ lslw(as_Register($dst$$reg),
11030             as_Register($src1$$reg),
11031             $src2$$constant & 0x1f);
11032   %}
11033 
11034   ins_pipe(ialu_reg_shift);
11035 %}
11036 
11037 // Shift Right Logical Register
11038 instruct urShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11039   match(Set dst (URShiftI src1 src2));
11040 
11041   ins_cost(INSN_COST * 2);
11042   format %{ "lsrvw  $dst, $src1, $src2" %}
11043 
11044   ins_encode %{
11045     __ lsrvw(as_Register($dst$$reg),
11046              as_Register($src1$$reg),
11047              as_Register($src2$$reg));
11048   %}
11049 
11050   ins_pipe(ialu_reg_reg_vshift);
11051 %}
11052 
11053 // Shift Right Logical Immediate
11054 instruct urShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
11055   match(Set dst (URShiftI src1 src2));
11056 
11057   ins_cost(INSN_COST);
11058   format %{ "lsrw $dst, $src1, ($src2 & 0x1f)" %}
11059 
11060   ins_encode %{
11061     __ lsrw(as_Register($dst$$reg),
11062             as_Register($src1$$reg),
11063             $src2$$constant & 0x1f);
11064   %}
11065 
11066   ins_pipe(ialu_reg_shift);
11067 %}
11068 
11069 // Shift Right Arithmetic Register
11070 instruct rShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11071   match(Set dst (RShiftI src1 src2));
11072 
11073   ins_cost(INSN_COST * 2);
11074   format %{ "asrvw  $dst, $src1, $src2" %}
11075 
11076   ins_encode %{
11077     __ asrvw(as_Register($dst$$reg),
11078              as_Register($src1$$reg),
11079              as_Register($src2$$reg));
11080   %}
11081 
11082   ins_pipe(ialu_reg_reg_vshift);
11083 %}
11084 
11085 // Shift Right Arithmetic Immediate
11086 instruct rShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
11087   match(Set dst (RShiftI src1 src2));
11088 
11089   ins_cost(INSN_COST);
11090   format %{ "asrw $dst, $src1, ($src2 & 0x1f)" %}
11091 
11092   ins_encode %{
11093     __ asrw(as_Register($dst$$reg),
11094             as_Register($src1$$reg),
11095             $src2$$constant & 0x1f);
11096   %}
11097 
11098   ins_pipe(ialu_reg_shift);
11099 %}
11100 
11101 // Combined Int Mask and Right Shift (using UBFM)
11102 // TODO
11103 
11104 // Long Shifts
11105 
11106 // Shift Left Register
11107 instruct lShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
11108   match(Set dst (LShiftL src1 src2));
11109 
11110   ins_cost(INSN_COST * 2);
11111   format %{ "lslv  $dst, $src1, $src2" %}
11112 
11113   ins_encode %{
11114     __ lslv(as_Register($dst$$reg),
11115             as_Register($src1$$reg),
11116             as_Register($src2$$reg));
11117   %}
11118 
11119   ins_pipe(ialu_reg_reg_vshift);
11120 %}
11121 
11122 // Shift Left Immediate
11123 instruct lShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
11124   match(Set dst (LShiftL src1 src2));
11125 
11126   ins_cost(INSN_COST);
11127   format %{ "lsl $dst, $src1, ($src2 & 0x3f)" %}
11128 
11129   ins_encode %{
11130     __ lsl(as_Register($dst$$reg),
11131             as_Register($src1$$reg),
11132             $src2$$constant & 0x3f);
11133   %}
11134 
11135   ins_pipe(ialu_reg_shift);
11136 %}
11137 
11138 // Shift Right Logical Register
11139 instruct urShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
11140   match(Set dst (URShiftL src1 src2));
11141 
11142   ins_cost(INSN_COST * 2);
11143   format %{ "lsrv  $dst, $src1, $src2" %}
11144 
11145   ins_encode %{
11146     __ lsrv(as_Register($dst$$reg),
11147             as_Register($src1$$reg),
11148             as_Register($src2$$reg));
11149   %}
11150 
11151   ins_pipe(ialu_reg_reg_vshift);
11152 %}
11153 
11154 // Shift Right Logical Immediate
11155 instruct urShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
11156   match(Set dst (URShiftL src1 src2));
11157 
11158   ins_cost(INSN_COST);
11159   format %{ "lsr $dst, $src1, ($src2 & 0x3f)" %}
11160 
11161   ins_encode %{
11162     __ lsr(as_Register($dst$$reg),
11163            as_Register($src1$$reg),
11164            $src2$$constant & 0x3f);
11165   %}
11166 
11167   ins_pipe(ialu_reg_shift);
11168 %}
11169 
11170 // A special-case pattern for card table stores.
11171 instruct urShiftP_reg_imm(iRegLNoSp dst, iRegP src1, immI src2) %{
11172   match(Set dst (URShiftL (CastP2X src1) src2));
11173 
11174   ins_cost(INSN_COST);
11175   format %{ "lsr $dst, p2x($src1), ($src2 & 0x3f)" %}
11176 
11177   ins_encode %{
11178     __ lsr(as_Register($dst$$reg),
11179            as_Register($src1$$reg),
11180            $src2$$constant & 0x3f);
11181   %}
11182 
11183   ins_pipe(ialu_reg_shift);
11184 %}
11185 
11186 // Shift Right Arithmetic Register
11187 instruct rShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
11188   match(Set dst (RShiftL src1 src2));
11189 
11190   ins_cost(INSN_COST * 2);
11191   format %{ "asrv  $dst, $src1, $src2" %}
11192 
11193   ins_encode %{
11194     __ asrv(as_Register($dst$$reg),
11195             as_Register($src1$$reg),
11196             as_Register($src2$$reg));
11197   %}
11198 
11199   ins_pipe(ialu_reg_reg_vshift);
11200 %}
11201 
11202 // Shift Right Arithmetic Immediate
11203 instruct rShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
11204   match(Set dst (RShiftL src1 src2));
11205 
11206   ins_cost(INSN_COST);
11207   format %{ "asr $dst, $src1, ($src2 & 0x3f)" %}
11208 
11209   ins_encode %{
11210     __ asr(as_Register($dst$$reg),
11211            as_Register($src1$$reg),
11212            $src2$$constant & 0x3f);
11213   %}
11214 
11215   ins_pipe(ialu_reg_shift);
11216 %}
11217 
11218 // BEGIN This section of the file is automatically generated. Do not edit --------------
11219 // This section is generated from aarch64_ad.m4
11220 
11221 
11222 // This pattern is automatically generated from aarch64_ad.m4
11223 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11224 instruct regL_not_reg(iRegLNoSp dst,
11225                          iRegL src1, immL_M1 m1,
11226                          rFlagsReg cr) %{
11227   match(Set dst (XorL src1 m1));
11228   ins_cost(INSN_COST);
11229   format %{ "eon  $dst, $src1, zr" %}
11230 
11231   ins_encode %{
11232     __ eon(as_Register($dst$$reg),
11233               as_Register($src1$$reg),
11234               zr,
11235               Assembler::LSL, 0);
11236   %}
11237 
11238   ins_pipe(ialu_reg);
11239 %}
11240 
11241 // This pattern is automatically generated from aarch64_ad.m4
11242 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11243 instruct regI_not_reg(iRegINoSp dst,
11244                          iRegIorL2I src1, immI_M1 m1,
11245                          rFlagsReg cr) %{
11246   match(Set dst (XorI src1 m1));
11247   ins_cost(INSN_COST);
11248   format %{ "eonw  $dst, $src1, zr" %}
11249 
11250   ins_encode %{
11251     __ eonw(as_Register($dst$$reg),
11252               as_Register($src1$$reg),
11253               zr,
11254               Assembler::LSL, 0);
11255   %}
11256 
11257   ins_pipe(ialu_reg);
11258 %}
11259 
11260 // This pattern is automatically generated from aarch64_ad.m4
11261 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11262 instruct NegI_reg_URShift_reg(iRegINoSp dst,
11263                               immI0 zero, iRegIorL2I src1, immI src2) %{
11264   match(Set dst (SubI zero (URShiftI src1 src2)));
11265 
11266   ins_cost(1.9 * INSN_COST);
11267   format %{ "negw  $dst, $src1, LSR $src2" %}
11268 
11269   ins_encode %{
11270     __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
11271             Assembler::LSR, $src2$$constant & 0x1f);
11272   %}
11273 
11274   ins_pipe(ialu_reg_shift);
11275 %}
11276 
11277 // This pattern is automatically generated from aarch64_ad.m4
11278 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11279 instruct NegI_reg_RShift_reg(iRegINoSp dst,
11280                               immI0 zero, iRegIorL2I src1, immI src2) %{
11281   match(Set dst (SubI zero (RShiftI src1 src2)));
11282 
11283   ins_cost(1.9 * INSN_COST);
11284   format %{ "negw  $dst, $src1, ASR $src2" %}
11285 
11286   ins_encode %{
11287     __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
11288             Assembler::ASR, $src2$$constant & 0x1f);
11289   %}
11290 
11291   ins_pipe(ialu_reg_shift);
11292 %}
11293 
11294 // This pattern is automatically generated from aarch64_ad.m4
11295 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11296 instruct NegI_reg_LShift_reg(iRegINoSp dst,
11297                               immI0 zero, iRegIorL2I src1, immI src2) %{
11298   match(Set dst (SubI zero (LShiftI src1 src2)));
11299 
11300   ins_cost(1.9 * INSN_COST);
11301   format %{ "negw  $dst, $src1, LSL $src2" %}
11302 
11303   ins_encode %{
11304     __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
11305             Assembler::LSL, $src2$$constant & 0x1f);
11306   %}
11307 
11308   ins_pipe(ialu_reg_shift);
11309 %}
11310 
11311 // This pattern is automatically generated from aarch64_ad.m4
11312 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11313 instruct NegL_reg_URShift_reg(iRegLNoSp dst,
11314                               immL0 zero, iRegL src1, immI src2) %{
11315   match(Set dst (SubL zero (URShiftL src1 src2)));
11316 
11317   ins_cost(1.9 * INSN_COST);
11318   format %{ "neg  $dst, $src1, LSR $src2" %}
11319 
11320   ins_encode %{
11321     __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
11322             Assembler::LSR, $src2$$constant & 0x3f);
11323   %}
11324 
11325   ins_pipe(ialu_reg_shift);
11326 %}
11327 
11328 // This pattern is automatically generated from aarch64_ad.m4
11329 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11330 instruct NegL_reg_RShift_reg(iRegLNoSp dst,
11331                               immL0 zero, iRegL src1, immI src2) %{
11332   match(Set dst (SubL zero (RShiftL src1 src2)));
11333 
11334   ins_cost(1.9 * INSN_COST);
11335   format %{ "neg  $dst, $src1, ASR $src2" %}
11336 
11337   ins_encode %{
11338     __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
11339             Assembler::ASR, $src2$$constant & 0x3f);
11340   %}
11341 
11342   ins_pipe(ialu_reg_shift);
11343 %}
11344 
11345 // This pattern is automatically generated from aarch64_ad.m4
11346 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11347 instruct NegL_reg_LShift_reg(iRegLNoSp dst,
11348                               immL0 zero, iRegL src1, immI src2) %{
11349   match(Set dst (SubL zero (LShiftL src1 src2)));
11350 
11351   ins_cost(1.9 * INSN_COST);
11352   format %{ "neg  $dst, $src1, LSL $src2" %}
11353 
11354   ins_encode %{
11355     __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
11356             Assembler::LSL, $src2$$constant & 0x3f);
11357   %}
11358 
11359   ins_pipe(ialu_reg_shift);
11360 %}
11361 
11362 // This pattern is automatically generated from aarch64_ad.m4
11363 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11364 instruct AndI_reg_not_reg(iRegINoSp dst,
11365                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11366   match(Set dst (AndI src1 (XorI src2 m1)));
11367   ins_cost(INSN_COST);
11368   format %{ "bicw  $dst, $src1, $src2" %}
11369 
11370   ins_encode %{
11371     __ bicw(as_Register($dst$$reg),
11372               as_Register($src1$$reg),
11373               as_Register($src2$$reg),
11374               Assembler::LSL, 0);
11375   %}
11376 
11377   ins_pipe(ialu_reg_reg);
11378 %}
11379 
11380 // This pattern is automatically generated from aarch64_ad.m4
11381 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11382 instruct AndL_reg_not_reg(iRegLNoSp dst,
11383                          iRegL src1, iRegL src2, immL_M1 m1) %{
11384   match(Set dst (AndL src1 (XorL src2 m1)));
11385   ins_cost(INSN_COST);
11386   format %{ "bic  $dst, $src1, $src2" %}
11387 
11388   ins_encode %{
11389     __ bic(as_Register($dst$$reg),
11390               as_Register($src1$$reg),
11391               as_Register($src2$$reg),
11392               Assembler::LSL, 0);
11393   %}
11394 
11395   ins_pipe(ialu_reg_reg);
11396 %}
11397 
11398 // This pattern is automatically generated from aarch64_ad.m4
11399 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11400 instruct OrI_reg_not_reg(iRegINoSp dst,
11401                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11402   match(Set dst (OrI src1 (XorI src2 m1)));
11403   ins_cost(INSN_COST);
11404   format %{ "ornw  $dst, $src1, $src2" %}
11405 
11406   ins_encode %{
11407     __ ornw(as_Register($dst$$reg),
11408               as_Register($src1$$reg),
11409               as_Register($src2$$reg),
11410               Assembler::LSL, 0);
11411   %}
11412 
11413   ins_pipe(ialu_reg_reg);
11414 %}
11415 
11416 // This pattern is automatically generated from aarch64_ad.m4
11417 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11418 instruct OrL_reg_not_reg(iRegLNoSp dst,
11419                          iRegL src1, iRegL src2, immL_M1 m1) %{
11420   match(Set dst (OrL src1 (XorL src2 m1)));
11421   ins_cost(INSN_COST);
11422   format %{ "orn  $dst, $src1, $src2" %}
11423 
11424   ins_encode %{
11425     __ orn(as_Register($dst$$reg),
11426               as_Register($src1$$reg),
11427               as_Register($src2$$reg),
11428               Assembler::LSL, 0);
11429   %}
11430 
11431   ins_pipe(ialu_reg_reg);
11432 %}
11433 
11434 // This pattern is automatically generated from aarch64_ad.m4
11435 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11436 instruct XorI_reg_not_reg(iRegINoSp dst,
11437                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11438   match(Set dst (XorI m1 (XorI src2 src1)));
11439   ins_cost(INSN_COST);
11440   format %{ "eonw  $dst, $src1, $src2" %}
11441 
11442   ins_encode %{
11443     __ eonw(as_Register($dst$$reg),
11444               as_Register($src1$$reg),
11445               as_Register($src2$$reg),
11446               Assembler::LSL, 0);
11447   %}
11448 
11449   ins_pipe(ialu_reg_reg);
11450 %}
11451 
11452 // This pattern is automatically generated from aarch64_ad.m4
11453 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11454 instruct XorL_reg_not_reg(iRegLNoSp dst,
11455                          iRegL src1, iRegL src2, immL_M1 m1) %{
11456   match(Set dst (XorL m1 (XorL src2 src1)));
11457   ins_cost(INSN_COST);
11458   format %{ "eon  $dst, $src1, $src2" %}
11459 
11460   ins_encode %{
11461     __ eon(as_Register($dst$$reg),
11462               as_Register($src1$$reg),
11463               as_Register($src2$$reg),
11464               Assembler::LSL, 0);
11465   %}
11466 
11467   ins_pipe(ialu_reg_reg);
11468 %}
11469 
11470 // This pattern is automatically generated from aarch64_ad.m4
11471 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11472 // val & (-1 ^ (val >>> shift)) ==> bicw
11473 instruct AndI_reg_URShift_not_reg(iRegINoSp dst,
11474                          iRegIorL2I src1, iRegIorL2I src2,
11475                          immI src3, immI_M1 src4) %{
11476   match(Set dst (AndI src1 (XorI(URShiftI src2 src3) src4)));
11477   ins_cost(1.9 * INSN_COST);
11478   format %{ "bicw  $dst, $src1, $src2, LSR $src3" %}
11479 
11480   ins_encode %{
11481     __ bicw(as_Register($dst$$reg),
11482               as_Register($src1$$reg),
11483               as_Register($src2$$reg),
11484               Assembler::LSR,
11485               $src3$$constant & 0x1f);
11486   %}
11487 
11488   ins_pipe(ialu_reg_reg_shift);
11489 %}
11490 
11491 // This pattern is automatically generated from aarch64_ad.m4
11492 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11493 // val & (-1 ^ (val >>> shift)) ==> bic
11494 instruct AndL_reg_URShift_not_reg(iRegLNoSp dst,
11495                          iRegL src1, iRegL src2,
11496                          immI src3, immL_M1 src4) %{
11497   match(Set dst (AndL src1 (XorL(URShiftL src2 src3) src4)));
11498   ins_cost(1.9 * INSN_COST);
11499   format %{ "bic  $dst, $src1, $src2, LSR $src3" %}
11500 
11501   ins_encode %{
11502     __ bic(as_Register($dst$$reg),
11503               as_Register($src1$$reg),
11504               as_Register($src2$$reg),
11505               Assembler::LSR,
11506               $src3$$constant & 0x3f);
11507   %}
11508 
11509   ins_pipe(ialu_reg_reg_shift);
11510 %}
11511 
11512 // This pattern is automatically generated from aarch64_ad.m4
11513 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11514 // val & (-1 ^ (val >> shift)) ==> bicw
11515 instruct AndI_reg_RShift_not_reg(iRegINoSp dst,
11516                          iRegIorL2I src1, iRegIorL2I src2,
11517                          immI src3, immI_M1 src4) %{
11518   match(Set dst (AndI src1 (XorI(RShiftI src2 src3) src4)));
11519   ins_cost(1.9 * INSN_COST);
11520   format %{ "bicw  $dst, $src1, $src2, ASR $src3" %}
11521 
11522   ins_encode %{
11523     __ bicw(as_Register($dst$$reg),
11524               as_Register($src1$$reg),
11525               as_Register($src2$$reg),
11526               Assembler::ASR,
11527               $src3$$constant & 0x1f);
11528   %}
11529 
11530   ins_pipe(ialu_reg_reg_shift);
11531 %}
11532 
11533 // This pattern is automatically generated from aarch64_ad.m4
11534 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11535 // val & (-1 ^ (val >> shift)) ==> bic
11536 instruct AndL_reg_RShift_not_reg(iRegLNoSp dst,
11537                          iRegL src1, iRegL src2,
11538                          immI src3, immL_M1 src4) %{
11539   match(Set dst (AndL src1 (XorL(RShiftL src2 src3) src4)));
11540   ins_cost(1.9 * INSN_COST);
11541   format %{ "bic  $dst, $src1, $src2, ASR $src3" %}
11542 
11543   ins_encode %{
11544     __ bic(as_Register($dst$$reg),
11545               as_Register($src1$$reg),
11546               as_Register($src2$$reg),
11547               Assembler::ASR,
11548               $src3$$constant & 0x3f);
11549   %}
11550 
11551   ins_pipe(ialu_reg_reg_shift);
11552 %}
11553 
11554 // This pattern is automatically generated from aarch64_ad.m4
11555 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11556 // val & (-1 ^ (val ror shift)) ==> bicw
11557 instruct AndI_reg_RotateRight_not_reg(iRegINoSp dst,
11558                          iRegIorL2I src1, iRegIorL2I src2,
11559                          immI src3, immI_M1 src4) %{
11560   match(Set dst (AndI src1 (XorI(RotateRight src2 src3) src4)));
11561   ins_cost(1.9 * INSN_COST);
11562   format %{ "bicw  $dst, $src1, $src2, ROR $src3" %}
11563 
11564   ins_encode %{
11565     __ bicw(as_Register($dst$$reg),
11566               as_Register($src1$$reg),
11567               as_Register($src2$$reg),
11568               Assembler::ROR,
11569               $src3$$constant & 0x1f);
11570   %}
11571 
11572   ins_pipe(ialu_reg_reg_shift);
11573 %}
11574 
11575 // This pattern is automatically generated from aarch64_ad.m4
11576 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11577 // val & (-1 ^ (val ror shift)) ==> bic
11578 instruct AndL_reg_RotateRight_not_reg(iRegLNoSp dst,
11579                          iRegL src1, iRegL src2,
11580                          immI src3, immL_M1 src4) %{
11581   match(Set dst (AndL src1 (XorL(RotateRight src2 src3) src4)));
11582   ins_cost(1.9 * INSN_COST);
11583   format %{ "bic  $dst, $src1, $src2, ROR $src3" %}
11584 
11585   ins_encode %{
11586     __ bic(as_Register($dst$$reg),
11587               as_Register($src1$$reg),
11588               as_Register($src2$$reg),
11589               Assembler::ROR,
11590               $src3$$constant & 0x3f);
11591   %}
11592 
11593   ins_pipe(ialu_reg_reg_shift);
11594 %}
11595 
11596 // This pattern is automatically generated from aarch64_ad.m4
11597 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11598 // val & (-1 ^ (val << shift)) ==> bicw
11599 instruct AndI_reg_LShift_not_reg(iRegINoSp dst,
11600                          iRegIorL2I src1, iRegIorL2I src2,
11601                          immI src3, immI_M1 src4) %{
11602   match(Set dst (AndI src1 (XorI(LShiftI src2 src3) src4)));
11603   ins_cost(1.9 * INSN_COST);
11604   format %{ "bicw  $dst, $src1, $src2, LSL $src3" %}
11605 
11606   ins_encode %{
11607     __ bicw(as_Register($dst$$reg),
11608               as_Register($src1$$reg),
11609               as_Register($src2$$reg),
11610               Assembler::LSL,
11611               $src3$$constant & 0x1f);
11612   %}
11613 
11614   ins_pipe(ialu_reg_reg_shift);
11615 %}
11616 
11617 // This pattern is automatically generated from aarch64_ad.m4
11618 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11619 // val & (-1 ^ (val << shift)) ==> bic
11620 instruct AndL_reg_LShift_not_reg(iRegLNoSp dst,
11621                          iRegL src1, iRegL src2,
11622                          immI src3, immL_M1 src4) %{
11623   match(Set dst (AndL src1 (XorL(LShiftL src2 src3) src4)));
11624   ins_cost(1.9 * INSN_COST);
11625   format %{ "bic  $dst, $src1, $src2, LSL $src3" %}
11626 
11627   ins_encode %{
11628     __ bic(as_Register($dst$$reg),
11629               as_Register($src1$$reg),
11630               as_Register($src2$$reg),
11631               Assembler::LSL,
11632               $src3$$constant & 0x3f);
11633   %}
11634 
11635   ins_pipe(ialu_reg_reg_shift);
11636 %}
11637 
11638 // This pattern is automatically generated from aarch64_ad.m4
11639 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11640 // val ^ (-1 ^ (val >>> shift)) ==> eonw
11641 instruct XorI_reg_URShift_not_reg(iRegINoSp dst,
11642                          iRegIorL2I src1, iRegIorL2I src2,
11643                          immI src3, immI_M1 src4) %{
11644   match(Set dst (XorI src4 (XorI(URShiftI src2 src3) src1)));
11645   ins_cost(1.9 * INSN_COST);
11646   format %{ "eonw  $dst, $src1, $src2, LSR $src3" %}
11647 
11648   ins_encode %{
11649     __ eonw(as_Register($dst$$reg),
11650               as_Register($src1$$reg),
11651               as_Register($src2$$reg),
11652               Assembler::LSR,
11653               $src3$$constant & 0x1f);
11654   %}
11655 
11656   ins_pipe(ialu_reg_reg_shift);
11657 %}
11658 
11659 // This pattern is automatically generated from aarch64_ad.m4
11660 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11661 // val ^ (-1 ^ (val >>> shift)) ==> eon
11662 instruct XorL_reg_URShift_not_reg(iRegLNoSp dst,
11663                          iRegL src1, iRegL src2,
11664                          immI src3, immL_M1 src4) %{
11665   match(Set dst (XorL src4 (XorL(URShiftL src2 src3) src1)));
11666   ins_cost(1.9 * INSN_COST);
11667   format %{ "eon  $dst, $src1, $src2, LSR $src3" %}
11668 
11669   ins_encode %{
11670     __ eon(as_Register($dst$$reg),
11671               as_Register($src1$$reg),
11672               as_Register($src2$$reg),
11673               Assembler::LSR,
11674               $src3$$constant & 0x3f);
11675   %}
11676 
11677   ins_pipe(ialu_reg_reg_shift);
11678 %}
11679 
11680 // This pattern is automatically generated from aarch64_ad.m4
11681 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11682 // val ^ (-1 ^ (val >> shift)) ==> eonw
11683 instruct XorI_reg_RShift_not_reg(iRegINoSp dst,
11684                          iRegIorL2I src1, iRegIorL2I src2,
11685                          immI src3, immI_M1 src4) %{
11686   match(Set dst (XorI src4 (XorI(RShiftI src2 src3) src1)));
11687   ins_cost(1.9 * INSN_COST);
11688   format %{ "eonw  $dst, $src1, $src2, ASR $src3" %}
11689 
11690   ins_encode %{
11691     __ eonw(as_Register($dst$$reg),
11692               as_Register($src1$$reg),
11693               as_Register($src2$$reg),
11694               Assembler::ASR,
11695               $src3$$constant & 0x1f);
11696   %}
11697 
11698   ins_pipe(ialu_reg_reg_shift);
11699 %}
11700 
11701 // This pattern is automatically generated from aarch64_ad.m4
11702 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11703 // val ^ (-1 ^ (val >> shift)) ==> eon
11704 instruct XorL_reg_RShift_not_reg(iRegLNoSp dst,
11705                          iRegL src1, iRegL src2,
11706                          immI src3, immL_M1 src4) %{
11707   match(Set dst (XorL src4 (XorL(RShiftL src2 src3) src1)));
11708   ins_cost(1.9 * INSN_COST);
11709   format %{ "eon  $dst, $src1, $src2, ASR $src3" %}
11710 
11711   ins_encode %{
11712     __ eon(as_Register($dst$$reg),
11713               as_Register($src1$$reg),
11714               as_Register($src2$$reg),
11715               Assembler::ASR,
11716               $src3$$constant & 0x3f);
11717   %}
11718 
11719   ins_pipe(ialu_reg_reg_shift);
11720 %}
11721 
11722 // This pattern is automatically generated from aarch64_ad.m4
11723 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11724 // val ^ (-1 ^ (val ror shift)) ==> eonw
11725 instruct XorI_reg_RotateRight_not_reg(iRegINoSp dst,
11726                          iRegIorL2I src1, iRegIorL2I src2,
11727                          immI src3, immI_M1 src4) %{
11728   match(Set dst (XorI src4 (XorI(RotateRight src2 src3) src1)));
11729   ins_cost(1.9 * INSN_COST);
11730   format %{ "eonw  $dst, $src1, $src2, ROR $src3" %}
11731 
11732   ins_encode %{
11733     __ eonw(as_Register($dst$$reg),
11734               as_Register($src1$$reg),
11735               as_Register($src2$$reg),
11736               Assembler::ROR,
11737               $src3$$constant & 0x1f);
11738   %}
11739 
11740   ins_pipe(ialu_reg_reg_shift);
11741 %}
11742 
11743 // This pattern is automatically generated from aarch64_ad.m4
11744 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11745 // val ^ (-1 ^ (val ror shift)) ==> eon
11746 instruct XorL_reg_RotateRight_not_reg(iRegLNoSp dst,
11747                          iRegL src1, iRegL src2,
11748                          immI src3, immL_M1 src4) %{
11749   match(Set dst (XorL src4 (XorL(RotateRight src2 src3) src1)));
11750   ins_cost(1.9 * INSN_COST);
11751   format %{ "eon  $dst, $src1, $src2, ROR $src3" %}
11752 
11753   ins_encode %{
11754     __ eon(as_Register($dst$$reg),
11755               as_Register($src1$$reg),
11756               as_Register($src2$$reg),
11757               Assembler::ROR,
11758               $src3$$constant & 0x3f);
11759   %}
11760 
11761   ins_pipe(ialu_reg_reg_shift);
11762 %}
11763 
11764 // This pattern is automatically generated from aarch64_ad.m4
11765 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11766 // val ^ (-1 ^ (val << shift)) ==> eonw
11767 instruct XorI_reg_LShift_not_reg(iRegINoSp dst,
11768                          iRegIorL2I src1, iRegIorL2I src2,
11769                          immI src3, immI_M1 src4) %{
11770   match(Set dst (XorI src4 (XorI(LShiftI src2 src3) src1)));
11771   ins_cost(1.9 * INSN_COST);
11772   format %{ "eonw  $dst, $src1, $src2, LSL $src3" %}
11773 
11774   ins_encode %{
11775     __ eonw(as_Register($dst$$reg),
11776               as_Register($src1$$reg),
11777               as_Register($src2$$reg),
11778               Assembler::LSL,
11779               $src3$$constant & 0x1f);
11780   %}
11781 
11782   ins_pipe(ialu_reg_reg_shift);
11783 %}
11784 
11785 // This pattern is automatically generated from aarch64_ad.m4
11786 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11787 // val ^ (-1 ^ (val << shift)) ==> eon
11788 instruct XorL_reg_LShift_not_reg(iRegLNoSp dst,
11789                          iRegL src1, iRegL src2,
11790                          immI src3, immL_M1 src4) %{
11791   match(Set dst (XorL src4 (XorL(LShiftL src2 src3) src1)));
11792   ins_cost(1.9 * INSN_COST);
11793   format %{ "eon  $dst, $src1, $src2, LSL $src3" %}
11794 
11795   ins_encode %{
11796     __ eon(as_Register($dst$$reg),
11797               as_Register($src1$$reg),
11798               as_Register($src2$$reg),
11799               Assembler::LSL,
11800               $src3$$constant & 0x3f);
11801   %}
11802 
11803   ins_pipe(ialu_reg_reg_shift);
11804 %}
11805 
11806 // This pattern is automatically generated from aarch64_ad.m4
11807 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11808 // val | (-1 ^ (val >>> shift)) ==> ornw
11809 instruct OrI_reg_URShift_not_reg(iRegINoSp dst,
11810                          iRegIorL2I src1, iRegIorL2I src2,
11811                          immI src3, immI_M1 src4) %{
11812   match(Set dst (OrI src1 (XorI(URShiftI src2 src3) src4)));
11813   ins_cost(1.9 * INSN_COST);
11814   format %{ "ornw  $dst, $src1, $src2, LSR $src3" %}
11815 
11816   ins_encode %{
11817     __ ornw(as_Register($dst$$reg),
11818               as_Register($src1$$reg),
11819               as_Register($src2$$reg),
11820               Assembler::LSR,
11821               $src3$$constant & 0x1f);
11822   %}
11823 
11824   ins_pipe(ialu_reg_reg_shift);
11825 %}
11826 
11827 // This pattern is automatically generated from aarch64_ad.m4
11828 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11829 // val | (-1 ^ (val >>> shift)) ==> orn
11830 instruct OrL_reg_URShift_not_reg(iRegLNoSp dst,
11831                          iRegL src1, iRegL src2,
11832                          immI src3, immL_M1 src4) %{
11833   match(Set dst (OrL src1 (XorL(URShiftL src2 src3) src4)));
11834   ins_cost(1.9 * INSN_COST);
11835   format %{ "orn  $dst, $src1, $src2, LSR $src3" %}
11836 
11837   ins_encode %{
11838     __ orn(as_Register($dst$$reg),
11839               as_Register($src1$$reg),
11840               as_Register($src2$$reg),
11841               Assembler::LSR,
11842               $src3$$constant & 0x3f);
11843   %}
11844 
11845   ins_pipe(ialu_reg_reg_shift);
11846 %}
11847 
11848 // This pattern is automatically generated from aarch64_ad.m4
11849 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11850 // val | (-1 ^ (val >> shift)) ==> ornw
11851 instruct OrI_reg_RShift_not_reg(iRegINoSp dst,
11852                          iRegIorL2I src1, iRegIorL2I src2,
11853                          immI src3, immI_M1 src4) %{
11854   match(Set dst (OrI src1 (XorI(RShiftI src2 src3) src4)));
11855   ins_cost(1.9 * INSN_COST);
11856   format %{ "ornw  $dst, $src1, $src2, ASR $src3" %}
11857 
11858   ins_encode %{
11859     __ ornw(as_Register($dst$$reg),
11860               as_Register($src1$$reg),
11861               as_Register($src2$$reg),
11862               Assembler::ASR,
11863               $src3$$constant & 0x1f);
11864   %}
11865 
11866   ins_pipe(ialu_reg_reg_shift);
11867 %}
11868 
11869 // This pattern is automatically generated from aarch64_ad.m4
11870 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11871 // val | (-1 ^ (val >> shift)) ==> orn
11872 instruct OrL_reg_RShift_not_reg(iRegLNoSp dst,
11873                          iRegL src1, iRegL src2,
11874                          immI src3, immL_M1 src4) %{
11875   match(Set dst (OrL src1 (XorL(RShiftL src2 src3) src4)));
11876   ins_cost(1.9 * INSN_COST);
11877   format %{ "orn  $dst, $src1, $src2, ASR $src3" %}
11878 
11879   ins_encode %{
11880     __ orn(as_Register($dst$$reg),
11881               as_Register($src1$$reg),
11882               as_Register($src2$$reg),
11883               Assembler::ASR,
11884               $src3$$constant & 0x3f);
11885   %}
11886 
11887   ins_pipe(ialu_reg_reg_shift);
11888 %}
11889 
11890 // This pattern is automatically generated from aarch64_ad.m4
11891 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11892 // val | (-1 ^ (val ror shift)) ==> ornw
11893 instruct OrI_reg_RotateRight_not_reg(iRegINoSp dst,
11894                          iRegIorL2I src1, iRegIorL2I src2,
11895                          immI src3, immI_M1 src4) %{
11896   match(Set dst (OrI src1 (XorI(RotateRight src2 src3) src4)));
11897   ins_cost(1.9 * INSN_COST);
11898   format %{ "ornw  $dst, $src1, $src2, ROR $src3" %}
11899 
11900   ins_encode %{
11901     __ ornw(as_Register($dst$$reg),
11902               as_Register($src1$$reg),
11903               as_Register($src2$$reg),
11904               Assembler::ROR,
11905               $src3$$constant & 0x1f);
11906   %}
11907 
11908   ins_pipe(ialu_reg_reg_shift);
11909 %}
11910 
11911 // This pattern is automatically generated from aarch64_ad.m4
11912 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11913 // val | (-1 ^ (val ror shift)) ==> orn
11914 instruct OrL_reg_RotateRight_not_reg(iRegLNoSp dst,
11915                          iRegL src1, iRegL src2,
11916                          immI src3, immL_M1 src4) %{
11917   match(Set dst (OrL src1 (XorL(RotateRight src2 src3) src4)));
11918   ins_cost(1.9 * INSN_COST);
11919   format %{ "orn  $dst, $src1, $src2, ROR $src3" %}
11920 
11921   ins_encode %{
11922     __ orn(as_Register($dst$$reg),
11923               as_Register($src1$$reg),
11924               as_Register($src2$$reg),
11925               Assembler::ROR,
11926               $src3$$constant & 0x3f);
11927   %}
11928 
11929   ins_pipe(ialu_reg_reg_shift);
11930 %}
11931 
11932 // This pattern is automatically generated from aarch64_ad.m4
11933 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11934 // val | (-1 ^ (val << shift)) ==> ornw
11935 instruct OrI_reg_LShift_not_reg(iRegINoSp dst,
11936                          iRegIorL2I src1, iRegIorL2I src2,
11937                          immI src3, immI_M1 src4) %{
11938   match(Set dst (OrI src1 (XorI(LShiftI src2 src3) src4)));
11939   ins_cost(1.9 * INSN_COST);
11940   format %{ "ornw  $dst, $src1, $src2, LSL $src3" %}
11941 
11942   ins_encode %{
11943     __ ornw(as_Register($dst$$reg),
11944               as_Register($src1$$reg),
11945               as_Register($src2$$reg),
11946               Assembler::LSL,
11947               $src3$$constant & 0x1f);
11948   %}
11949 
11950   ins_pipe(ialu_reg_reg_shift);
11951 %}
11952 
11953 // This pattern is automatically generated from aarch64_ad.m4
11954 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11955 // val | (-1 ^ (val << shift)) ==> orn
11956 instruct OrL_reg_LShift_not_reg(iRegLNoSp dst,
11957                          iRegL src1, iRegL src2,
11958                          immI src3, immL_M1 src4) %{
11959   match(Set dst (OrL src1 (XorL(LShiftL src2 src3) src4)));
11960   ins_cost(1.9 * INSN_COST);
11961   format %{ "orn  $dst, $src1, $src2, LSL $src3" %}
11962 
11963   ins_encode %{
11964     __ orn(as_Register($dst$$reg),
11965               as_Register($src1$$reg),
11966               as_Register($src2$$reg),
11967               Assembler::LSL,
11968               $src3$$constant & 0x3f);
11969   %}
11970 
11971   ins_pipe(ialu_reg_reg_shift);
11972 %}
11973 
11974 // This pattern is automatically generated from aarch64_ad.m4
11975 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11976 instruct AndI_reg_URShift_reg(iRegINoSp dst,
11977                          iRegIorL2I src1, iRegIorL2I src2,
11978                          immI src3) %{
11979   match(Set dst (AndI src1 (URShiftI src2 src3)));
11980 
11981   ins_cost(1.9 * INSN_COST);
11982   format %{ "andw  $dst, $src1, $src2, LSR $src3" %}
11983 
11984   ins_encode %{
11985     __ andw(as_Register($dst$$reg),
11986               as_Register($src1$$reg),
11987               as_Register($src2$$reg),
11988               Assembler::LSR,
11989               $src3$$constant & 0x1f);
11990   %}
11991 
11992   ins_pipe(ialu_reg_reg_shift);
11993 %}
11994 
11995 // This pattern is automatically generated from aarch64_ad.m4
11996 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11997 instruct AndL_reg_URShift_reg(iRegLNoSp dst,
11998                          iRegL src1, iRegL src2,
11999                          immI src3) %{
12000   match(Set dst (AndL src1 (URShiftL src2 src3)));
12001 
12002   ins_cost(1.9 * INSN_COST);
12003   format %{ "andr  $dst, $src1, $src2, LSR $src3" %}
12004 
12005   ins_encode %{
12006     __ andr(as_Register($dst$$reg),
12007               as_Register($src1$$reg),
12008               as_Register($src2$$reg),
12009               Assembler::LSR,
12010               $src3$$constant & 0x3f);
12011   %}
12012 
12013   ins_pipe(ialu_reg_reg_shift);
12014 %}
12015 
12016 // This pattern is automatically generated from aarch64_ad.m4
12017 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12018 instruct AndI_reg_RShift_reg(iRegINoSp dst,
12019                          iRegIorL2I src1, iRegIorL2I src2,
12020                          immI src3) %{
12021   match(Set dst (AndI src1 (RShiftI src2 src3)));
12022 
12023   ins_cost(1.9 * INSN_COST);
12024   format %{ "andw  $dst, $src1, $src2, ASR $src3" %}
12025 
12026   ins_encode %{
12027     __ andw(as_Register($dst$$reg),
12028               as_Register($src1$$reg),
12029               as_Register($src2$$reg),
12030               Assembler::ASR,
12031               $src3$$constant & 0x1f);
12032   %}
12033 
12034   ins_pipe(ialu_reg_reg_shift);
12035 %}
12036 
12037 // This pattern is automatically generated from aarch64_ad.m4
12038 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12039 instruct AndL_reg_RShift_reg(iRegLNoSp dst,
12040                          iRegL src1, iRegL src2,
12041                          immI src3) %{
12042   match(Set dst (AndL src1 (RShiftL src2 src3)));
12043 
12044   ins_cost(1.9 * INSN_COST);
12045   format %{ "andr  $dst, $src1, $src2, ASR $src3" %}
12046 
12047   ins_encode %{
12048     __ andr(as_Register($dst$$reg),
12049               as_Register($src1$$reg),
12050               as_Register($src2$$reg),
12051               Assembler::ASR,
12052               $src3$$constant & 0x3f);
12053   %}
12054 
12055   ins_pipe(ialu_reg_reg_shift);
12056 %}
12057 
12058 // This pattern is automatically generated from aarch64_ad.m4
12059 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12060 instruct AndI_reg_LShift_reg(iRegINoSp dst,
12061                          iRegIorL2I src1, iRegIorL2I src2,
12062                          immI src3) %{
12063   match(Set dst (AndI src1 (LShiftI src2 src3)));
12064 
12065   ins_cost(1.9 * INSN_COST);
12066   format %{ "andw  $dst, $src1, $src2, LSL $src3" %}
12067 
12068   ins_encode %{
12069     __ andw(as_Register($dst$$reg),
12070               as_Register($src1$$reg),
12071               as_Register($src2$$reg),
12072               Assembler::LSL,
12073               $src3$$constant & 0x1f);
12074   %}
12075 
12076   ins_pipe(ialu_reg_reg_shift);
12077 %}
12078 
12079 // This pattern is automatically generated from aarch64_ad.m4
12080 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12081 instruct AndL_reg_LShift_reg(iRegLNoSp dst,
12082                          iRegL src1, iRegL src2,
12083                          immI src3) %{
12084   match(Set dst (AndL src1 (LShiftL src2 src3)));
12085 
12086   ins_cost(1.9 * INSN_COST);
12087   format %{ "andr  $dst, $src1, $src2, LSL $src3" %}
12088 
12089   ins_encode %{
12090     __ andr(as_Register($dst$$reg),
12091               as_Register($src1$$reg),
12092               as_Register($src2$$reg),
12093               Assembler::LSL,
12094               $src3$$constant & 0x3f);
12095   %}
12096 
12097   ins_pipe(ialu_reg_reg_shift);
12098 %}
12099 
12100 // This pattern is automatically generated from aarch64_ad.m4
12101 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12102 instruct AndI_reg_RotateRight_reg(iRegINoSp dst,
12103                          iRegIorL2I src1, iRegIorL2I src2,
12104                          immI src3) %{
12105   match(Set dst (AndI src1 (RotateRight src2 src3)));
12106 
12107   ins_cost(1.9 * INSN_COST);
12108   format %{ "andw  $dst, $src1, $src2, ROR $src3" %}
12109 
12110   ins_encode %{
12111     __ andw(as_Register($dst$$reg),
12112               as_Register($src1$$reg),
12113               as_Register($src2$$reg),
12114               Assembler::ROR,
12115               $src3$$constant & 0x1f);
12116   %}
12117 
12118   ins_pipe(ialu_reg_reg_shift);
12119 %}
12120 
12121 // This pattern is automatically generated from aarch64_ad.m4
12122 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12123 instruct AndL_reg_RotateRight_reg(iRegLNoSp dst,
12124                          iRegL src1, iRegL src2,
12125                          immI src3) %{
12126   match(Set dst (AndL src1 (RotateRight src2 src3)));
12127 
12128   ins_cost(1.9 * INSN_COST);
12129   format %{ "andr  $dst, $src1, $src2, ROR $src3" %}
12130 
12131   ins_encode %{
12132     __ andr(as_Register($dst$$reg),
12133               as_Register($src1$$reg),
12134               as_Register($src2$$reg),
12135               Assembler::ROR,
12136               $src3$$constant & 0x3f);
12137   %}
12138 
12139   ins_pipe(ialu_reg_reg_shift);
12140 %}
12141 
12142 // This pattern is automatically generated from aarch64_ad.m4
12143 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12144 instruct XorI_reg_URShift_reg(iRegINoSp dst,
12145                          iRegIorL2I src1, iRegIorL2I src2,
12146                          immI src3) %{
12147   match(Set dst (XorI src1 (URShiftI src2 src3)));
12148 
12149   ins_cost(1.9 * INSN_COST);
12150   format %{ "eorw  $dst, $src1, $src2, LSR $src3" %}
12151 
12152   ins_encode %{
12153     __ eorw(as_Register($dst$$reg),
12154               as_Register($src1$$reg),
12155               as_Register($src2$$reg),
12156               Assembler::LSR,
12157               $src3$$constant & 0x1f);
12158   %}
12159 
12160   ins_pipe(ialu_reg_reg_shift);
12161 %}
12162 
12163 // This pattern is automatically generated from aarch64_ad.m4
12164 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12165 instruct XorL_reg_URShift_reg(iRegLNoSp dst,
12166                          iRegL src1, iRegL src2,
12167                          immI src3) %{
12168   match(Set dst (XorL src1 (URShiftL src2 src3)));
12169 
12170   ins_cost(1.9 * INSN_COST);
12171   format %{ "eor  $dst, $src1, $src2, LSR $src3" %}
12172 
12173   ins_encode %{
12174     __ eor(as_Register($dst$$reg),
12175               as_Register($src1$$reg),
12176               as_Register($src2$$reg),
12177               Assembler::LSR,
12178               $src3$$constant & 0x3f);
12179   %}
12180 
12181   ins_pipe(ialu_reg_reg_shift);
12182 %}
12183 
12184 // This pattern is automatically generated from aarch64_ad.m4
12185 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12186 instruct XorI_reg_RShift_reg(iRegINoSp dst,
12187                          iRegIorL2I src1, iRegIorL2I src2,
12188                          immI src3) %{
12189   match(Set dst (XorI src1 (RShiftI src2 src3)));
12190 
12191   ins_cost(1.9 * INSN_COST);
12192   format %{ "eorw  $dst, $src1, $src2, ASR $src3" %}
12193 
12194   ins_encode %{
12195     __ eorw(as_Register($dst$$reg),
12196               as_Register($src1$$reg),
12197               as_Register($src2$$reg),
12198               Assembler::ASR,
12199               $src3$$constant & 0x1f);
12200   %}
12201 
12202   ins_pipe(ialu_reg_reg_shift);
12203 %}
12204 
12205 // This pattern is automatically generated from aarch64_ad.m4
12206 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12207 instruct XorL_reg_RShift_reg(iRegLNoSp dst,
12208                          iRegL src1, iRegL src2,
12209                          immI src3) %{
12210   match(Set dst (XorL src1 (RShiftL src2 src3)));
12211 
12212   ins_cost(1.9 * INSN_COST);
12213   format %{ "eor  $dst, $src1, $src2, ASR $src3" %}
12214 
12215   ins_encode %{
12216     __ eor(as_Register($dst$$reg),
12217               as_Register($src1$$reg),
12218               as_Register($src2$$reg),
12219               Assembler::ASR,
12220               $src3$$constant & 0x3f);
12221   %}
12222 
12223   ins_pipe(ialu_reg_reg_shift);
12224 %}
12225 
12226 // This pattern is automatically generated from aarch64_ad.m4
12227 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12228 instruct XorI_reg_LShift_reg(iRegINoSp dst,
12229                          iRegIorL2I src1, iRegIorL2I src2,
12230                          immI src3) %{
12231   match(Set dst (XorI src1 (LShiftI src2 src3)));
12232 
12233   ins_cost(1.9 * INSN_COST);
12234   format %{ "eorw  $dst, $src1, $src2, LSL $src3" %}
12235 
12236   ins_encode %{
12237     __ eorw(as_Register($dst$$reg),
12238               as_Register($src1$$reg),
12239               as_Register($src2$$reg),
12240               Assembler::LSL,
12241               $src3$$constant & 0x1f);
12242   %}
12243 
12244   ins_pipe(ialu_reg_reg_shift);
12245 %}
12246 
12247 // This pattern is automatically generated from aarch64_ad.m4
12248 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12249 instruct XorL_reg_LShift_reg(iRegLNoSp dst,
12250                          iRegL src1, iRegL src2,
12251                          immI src3) %{
12252   match(Set dst (XorL src1 (LShiftL src2 src3)));
12253 
12254   ins_cost(1.9 * INSN_COST);
12255   format %{ "eor  $dst, $src1, $src2, LSL $src3" %}
12256 
12257   ins_encode %{
12258     __ eor(as_Register($dst$$reg),
12259               as_Register($src1$$reg),
12260               as_Register($src2$$reg),
12261               Assembler::LSL,
12262               $src3$$constant & 0x3f);
12263   %}
12264 
12265   ins_pipe(ialu_reg_reg_shift);
12266 %}
12267 
12268 // This pattern is automatically generated from aarch64_ad.m4
12269 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12270 instruct XorI_reg_RotateRight_reg(iRegINoSp dst,
12271                          iRegIorL2I src1, iRegIorL2I src2,
12272                          immI src3) %{
12273   match(Set dst (XorI src1 (RotateRight src2 src3)));
12274 
12275   ins_cost(1.9 * INSN_COST);
12276   format %{ "eorw  $dst, $src1, $src2, ROR $src3" %}
12277 
12278   ins_encode %{
12279     __ eorw(as_Register($dst$$reg),
12280               as_Register($src1$$reg),
12281               as_Register($src2$$reg),
12282               Assembler::ROR,
12283               $src3$$constant & 0x1f);
12284   %}
12285 
12286   ins_pipe(ialu_reg_reg_shift);
12287 %}
12288 
12289 // This pattern is automatically generated from aarch64_ad.m4
12290 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12291 instruct XorL_reg_RotateRight_reg(iRegLNoSp dst,
12292                          iRegL src1, iRegL src2,
12293                          immI src3) %{
12294   match(Set dst (XorL src1 (RotateRight src2 src3)));
12295 
12296   ins_cost(1.9 * INSN_COST);
12297   format %{ "eor  $dst, $src1, $src2, ROR $src3" %}
12298 
12299   ins_encode %{
12300     __ eor(as_Register($dst$$reg),
12301               as_Register($src1$$reg),
12302               as_Register($src2$$reg),
12303               Assembler::ROR,
12304               $src3$$constant & 0x3f);
12305   %}
12306 
12307   ins_pipe(ialu_reg_reg_shift);
12308 %}
12309 
12310 // This pattern is automatically generated from aarch64_ad.m4
12311 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12312 instruct OrI_reg_URShift_reg(iRegINoSp dst,
12313                          iRegIorL2I src1, iRegIorL2I src2,
12314                          immI src3) %{
12315   match(Set dst (OrI src1 (URShiftI src2 src3)));
12316 
12317   ins_cost(1.9 * INSN_COST);
12318   format %{ "orrw  $dst, $src1, $src2, LSR $src3" %}
12319 
12320   ins_encode %{
12321     __ orrw(as_Register($dst$$reg),
12322               as_Register($src1$$reg),
12323               as_Register($src2$$reg),
12324               Assembler::LSR,
12325               $src3$$constant & 0x1f);
12326   %}
12327 
12328   ins_pipe(ialu_reg_reg_shift);
12329 %}
12330 
12331 // This pattern is automatically generated from aarch64_ad.m4
12332 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12333 instruct OrL_reg_URShift_reg(iRegLNoSp dst,
12334                          iRegL src1, iRegL src2,
12335                          immI src3) %{
12336   match(Set dst (OrL src1 (URShiftL src2 src3)));
12337 
12338   ins_cost(1.9 * INSN_COST);
12339   format %{ "orr  $dst, $src1, $src2, LSR $src3" %}
12340 
12341   ins_encode %{
12342     __ orr(as_Register($dst$$reg),
12343               as_Register($src1$$reg),
12344               as_Register($src2$$reg),
12345               Assembler::LSR,
12346               $src3$$constant & 0x3f);
12347   %}
12348 
12349   ins_pipe(ialu_reg_reg_shift);
12350 %}
12351 
12352 // This pattern is automatically generated from aarch64_ad.m4
12353 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12354 instruct OrI_reg_RShift_reg(iRegINoSp dst,
12355                          iRegIorL2I src1, iRegIorL2I src2,
12356                          immI src3) %{
12357   match(Set dst (OrI src1 (RShiftI src2 src3)));
12358 
12359   ins_cost(1.9 * INSN_COST);
12360   format %{ "orrw  $dst, $src1, $src2, ASR $src3" %}
12361 
12362   ins_encode %{
12363     __ orrw(as_Register($dst$$reg),
12364               as_Register($src1$$reg),
12365               as_Register($src2$$reg),
12366               Assembler::ASR,
12367               $src3$$constant & 0x1f);
12368   %}
12369 
12370   ins_pipe(ialu_reg_reg_shift);
12371 %}
12372 
12373 // This pattern is automatically generated from aarch64_ad.m4
12374 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12375 instruct OrL_reg_RShift_reg(iRegLNoSp dst,
12376                          iRegL src1, iRegL src2,
12377                          immI src3) %{
12378   match(Set dst (OrL src1 (RShiftL src2 src3)));
12379 
12380   ins_cost(1.9 * INSN_COST);
12381   format %{ "orr  $dst, $src1, $src2, ASR $src3" %}
12382 
12383   ins_encode %{
12384     __ orr(as_Register($dst$$reg),
12385               as_Register($src1$$reg),
12386               as_Register($src2$$reg),
12387               Assembler::ASR,
12388               $src3$$constant & 0x3f);
12389   %}
12390 
12391   ins_pipe(ialu_reg_reg_shift);
12392 %}
12393 
12394 // This pattern is automatically generated from aarch64_ad.m4
12395 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12396 instruct OrI_reg_LShift_reg(iRegINoSp dst,
12397                          iRegIorL2I src1, iRegIorL2I src2,
12398                          immI src3) %{
12399   match(Set dst (OrI src1 (LShiftI src2 src3)));
12400 
12401   ins_cost(1.9 * INSN_COST);
12402   format %{ "orrw  $dst, $src1, $src2, LSL $src3" %}
12403 
12404   ins_encode %{
12405     __ orrw(as_Register($dst$$reg),
12406               as_Register($src1$$reg),
12407               as_Register($src2$$reg),
12408               Assembler::LSL,
12409               $src3$$constant & 0x1f);
12410   %}
12411 
12412   ins_pipe(ialu_reg_reg_shift);
12413 %}
12414 
12415 // This pattern is automatically generated from aarch64_ad.m4
12416 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12417 instruct OrL_reg_LShift_reg(iRegLNoSp dst,
12418                          iRegL src1, iRegL src2,
12419                          immI src3) %{
12420   match(Set dst (OrL src1 (LShiftL src2 src3)));
12421 
12422   ins_cost(1.9 * INSN_COST);
12423   format %{ "orr  $dst, $src1, $src2, LSL $src3" %}
12424 
12425   ins_encode %{
12426     __ orr(as_Register($dst$$reg),
12427               as_Register($src1$$reg),
12428               as_Register($src2$$reg),
12429               Assembler::LSL,
12430               $src3$$constant & 0x3f);
12431   %}
12432 
12433   ins_pipe(ialu_reg_reg_shift);
12434 %}
12435 
12436 // This pattern is automatically generated from aarch64_ad.m4
12437 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12438 instruct OrI_reg_RotateRight_reg(iRegINoSp dst,
12439                          iRegIorL2I src1, iRegIorL2I src2,
12440                          immI src3) %{
12441   match(Set dst (OrI src1 (RotateRight src2 src3)));
12442 
12443   ins_cost(1.9 * INSN_COST);
12444   format %{ "orrw  $dst, $src1, $src2, ROR $src3" %}
12445 
12446   ins_encode %{
12447     __ orrw(as_Register($dst$$reg),
12448               as_Register($src1$$reg),
12449               as_Register($src2$$reg),
12450               Assembler::ROR,
12451               $src3$$constant & 0x1f);
12452   %}
12453 
12454   ins_pipe(ialu_reg_reg_shift);
12455 %}
12456 
12457 // This pattern is automatically generated from aarch64_ad.m4
12458 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12459 instruct OrL_reg_RotateRight_reg(iRegLNoSp dst,
12460                          iRegL src1, iRegL src2,
12461                          immI src3) %{
12462   match(Set dst (OrL src1 (RotateRight src2 src3)));
12463 
12464   ins_cost(1.9 * INSN_COST);
12465   format %{ "orr  $dst, $src1, $src2, ROR $src3" %}
12466 
12467   ins_encode %{
12468     __ orr(as_Register($dst$$reg),
12469               as_Register($src1$$reg),
12470               as_Register($src2$$reg),
12471               Assembler::ROR,
12472               $src3$$constant & 0x3f);
12473   %}
12474 
12475   ins_pipe(ialu_reg_reg_shift);
12476 %}
12477 
12478 // This pattern is automatically generated from aarch64_ad.m4
12479 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12480 instruct AddI_reg_URShift_reg(iRegINoSp dst,
12481                          iRegIorL2I src1, iRegIorL2I src2,
12482                          immI src3) %{
12483   match(Set dst (AddI src1 (URShiftI src2 src3)));
12484 
12485   ins_cost(1.9 * INSN_COST);
12486   format %{ "addw  $dst, $src1, $src2, LSR $src3" %}
12487 
12488   ins_encode %{
12489     __ addw(as_Register($dst$$reg),
12490               as_Register($src1$$reg),
12491               as_Register($src2$$reg),
12492               Assembler::LSR,
12493               $src3$$constant & 0x1f);
12494   %}
12495 
12496   ins_pipe(ialu_reg_reg_shift);
12497 %}
12498 
12499 // This pattern is automatically generated from aarch64_ad.m4
12500 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12501 instruct AddL_reg_URShift_reg(iRegLNoSp dst,
12502                          iRegL src1, iRegL src2,
12503                          immI src3) %{
12504   match(Set dst (AddL src1 (URShiftL src2 src3)));
12505 
12506   ins_cost(1.9 * INSN_COST);
12507   format %{ "add  $dst, $src1, $src2, LSR $src3" %}
12508 
12509   ins_encode %{
12510     __ add(as_Register($dst$$reg),
12511               as_Register($src1$$reg),
12512               as_Register($src2$$reg),
12513               Assembler::LSR,
12514               $src3$$constant & 0x3f);
12515   %}
12516 
12517   ins_pipe(ialu_reg_reg_shift);
12518 %}
12519 
12520 // This pattern is automatically generated from aarch64_ad.m4
12521 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12522 instruct AddI_reg_RShift_reg(iRegINoSp dst,
12523                          iRegIorL2I src1, iRegIorL2I src2,
12524                          immI src3) %{
12525   match(Set dst (AddI src1 (RShiftI src2 src3)));
12526 
12527   ins_cost(1.9 * INSN_COST);
12528   format %{ "addw  $dst, $src1, $src2, ASR $src3" %}
12529 
12530   ins_encode %{
12531     __ addw(as_Register($dst$$reg),
12532               as_Register($src1$$reg),
12533               as_Register($src2$$reg),
12534               Assembler::ASR,
12535               $src3$$constant & 0x1f);
12536   %}
12537 
12538   ins_pipe(ialu_reg_reg_shift);
12539 %}
12540 
12541 // This pattern is automatically generated from aarch64_ad.m4
12542 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12543 instruct AddL_reg_RShift_reg(iRegLNoSp dst,
12544                          iRegL src1, iRegL src2,
12545                          immI src3) %{
12546   match(Set dst (AddL src1 (RShiftL src2 src3)));
12547 
12548   ins_cost(1.9 * INSN_COST);
12549   format %{ "add  $dst, $src1, $src2, ASR $src3" %}
12550 
12551   ins_encode %{
12552     __ add(as_Register($dst$$reg),
12553               as_Register($src1$$reg),
12554               as_Register($src2$$reg),
12555               Assembler::ASR,
12556               $src3$$constant & 0x3f);
12557   %}
12558 
12559   ins_pipe(ialu_reg_reg_shift);
12560 %}
12561 
12562 // This pattern is automatically generated from aarch64_ad.m4
12563 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12564 instruct AddI_reg_LShift_reg(iRegINoSp dst,
12565                          iRegIorL2I src1, iRegIorL2I src2,
12566                          immI src3) %{
12567   match(Set dst (AddI src1 (LShiftI src2 src3)));
12568 
12569   ins_cost(1.9 * INSN_COST);
12570   format %{ "addw  $dst, $src1, $src2, LSL $src3" %}
12571 
12572   ins_encode %{
12573     __ addw(as_Register($dst$$reg),
12574               as_Register($src1$$reg),
12575               as_Register($src2$$reg),
12576               Assembler::LSL,
12577               $src3$$constant & 0x1f);
12578   %}
12579 
12580   ins_pipe(ialu_reg_reg_shift);
12581 %}
12582 
12583 // This pattern is automatically generated from aarch64_ad.m4
12584 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12585 instruct AddL_reg_LShift_reg(iRegLNoSp dst,
12586                          iRegL src1, iRegL src2,
12587                          immI src3) %{
12588   match(Set dst (AddL src1 (LShiftL src2 src3)));
12589 
12590   ins_cost(1.9 * INSN_COST);
12591   format %{ "add  $dst, $src1, $src2, LSL $src3" %}
12592 
12593   ins_encode %{
12594     __ add(as_Register($dst$$reg),
12595               as_Register($src1$$reg),
12596               as_Register($src2$$reg),
12597               Assembler::LSL,
12598               $src3$$constant & 0x3f);
12599   %}
12600 
12601   ins_pipe(ialu_reg_reg_shift);
12602 %}
12603 
12604 // This pattern is automatically generated from aarch64_ad.m4
12605 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12606 instruct SubI_reg_URShift_reg(iRegINoSp dst,
12607                          iRegIorL2I src1, iRegIorL2I src2,
12608                          immI src3) %{
12609   match(Set dst (SubI src1 (URShiftI src2 src3)));
12610 
12611   ins_cost(1.9 * INSN_COST);
12612   format %{ "subw  $dst, $src1, $src2, LSR $src3" %}
12613 
12614   ins_encode %{
12615     __ subw(as_Register($dst$$reg),
12616               as_Register($src1$$reg),
12617               as_Register($src2$$reg),
12618               Assembler::LSR,
12619               $src3$$constant & 0x1f);
12620   %}
12621 
12622   ins_pipe(ialu_reg_reg_shift);
12623 %}
12624 
12625 // This pattern is automatically generated from aarch64_ad.m4
12626 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12627 instruct SubL_reg_URShift_reg(iRegLNoSp dst,
12628                          iRegL src1, iRegL src2,
12629                          immI src3) %{
12630   match(Set dst (SubL src1 (URShiftL src2 src3)));
12631 
12632   ins_cost(1.9 * INSN_COST);
12633   format %{ "sub  $dst, $src1, $src2, LSR $src3" %}
12634 
12635   ins_encode %{
12636     __ sub(as_Register($dst$$reg),
12637               as_Register($src1$$reg),
12638               as_Register($src2$$reg),
12639               Assembler::LSR,
12640               $src3$$constant & 0x3f);
12641   %}
12642 
12643   ins_pipe(ialu_reg_reg_shift);
12644 %}
12645 
12646 // This pattern is automatically generated from aarch64_ad.m4
12647 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12648 instruct SubI_reg_RShift_reg(iRegINoSp dst,
12649                          iRegIorL2I src1, iRegIorL2I src2,
12650                          immI src3) %{
12651   match(Set dst (SubI src1 (RShiftI src2 src3)));
12652 
12653   ins_cost(1.9 * INSN_COST);
12654   format %{ "subw  $dst, $src1, $src2, ASR $src3" %}
12655 
12656   ins_encode %{
12657     __ subw(as_Register($dst$$reg),
12658               as_Register($src1$$reg),
12659               as_Register($src2$$reg),
12660               Assembler::ASR,
12661               $src3$$constant & 0x1f);
12662   %}
12663 
12664   ins_pipe(ialu_reg_reg_shift);
12665 %}
12666 
12667 // This pattern is automatically generated from aarch64_ad.m4
12668 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12669 instruct SubL_reg_RShift_reg(iRegLNoSp dst,
12670                          iRegL src1, iRegL src2,
12671                          immI src3) %{
12672   match(Set dst (SubL src1 (RShiftL src2 src3)));
12673 
12674   ins_cost(1.9 * INSN_COST);
12675   format %{ "sub  $dst, $src1, $src2, ASR $src3" %}
12676 
12677   ins_encode %{
12678     __ sub(as_Register($dst$$reg),
12679               as_Register($src1$$reg),
12680               as_Register($src2$$reg),
12681               Assembler::ASR,
12682               $src3$$constant & 0x3f);
12683   %}
12684 
12685   ins_pipe(ialu_reg_reg_shift);
12686 %}
12687 
12688 // This pattern is automatically generated from aarch64_ad.m4
12689 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12690 instruct SubI_reg_LShift_reg(iRegINoSp dst,
12691                          iRegIorL2I src1, iRegIorL2I src2,
12692                          immI src3) %{
12693   match(Set dst (SubI src1 (LShiftI src2 src3)));
12694 
12695   ins_cost(1.9 * INSN_COST);
12696   format %{ "subw  $dst, $src1, $src2, LSL $src3" %}
12697 
12698   ins_encode %{
12699     __ subw(as_Register($dst$$reg),
12700               as_Register($src1$$reg),
12701               as_Register($src2$$reg),
12702               Assembler::LSL,
12703               $src3$$constant & 0x1f);
12704   %}
12705 
12706   ins_pipe(ialu_reg_reg_shift);
12707 %}
12708 
12709 // This pattern is automatically generated from aarch64_ad.m4
12710 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12711 instruct SubL_reg_LShift_reg(iRegLNoSp dst,
12712                          iRegL src1, iRegL src2,
12713                          immI src3) %{
12714   match(Set dst (SubL src1 (LShiftL src2 src3)));
12715 
12716   ins_cost(1.9 * INSN_COST);
12717   format %{ "sub  $dst, $src1, $src2, LSL $src3" %}
12718 
12719   ins_encode %{
12720     __ sub(as_Register($dst$$reg),
12721               as_Register($src1$$reg),
12722               as_Register($src2$$reg),
12723               Assembler::LSL,
12724               $src3$$constant & 0x3f);
12725   %}
12726 
12727   ins_pipe(ialu_reg_reg_shift);
12728 %}
12729 
12730 // This pattern is automatically generated from aarch64_ad.m4
12731 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12732 
12733 // Shift Left followed by Shift Right.
12734 // This idiom is used by the compiler for the i2b bytecode etc.
12735 instruct sbfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12736 %{
12737   match(Set dst (RShiftL (LShiftL src lshift_count) rshift_count));
12738   ins_cost(INSN_COST * 2);
12739   format %{ "sbfm  $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12740   ins_encode %{
12741     int lshift = $lshift_count$$constant & 63;
12742     int rshift = $rshift_count$$constant & 63;
12743     int s = 63 - lshift;
12744     int r = (rshift - lshift) & 63;
12745     __ sbfm(as_Register($dst$$reg),
12746             as_Register($src$$reg),
12747             r, s);
12748   %}
12749 
12750   ins_pipe(ialu_reg_shift);
12751 %}
12752 
12753 // This pattern is automatically generated from aarch64_ad.m4
12754 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12755 
12756 // Shift Left followed by Shift Right.
12757 // This idiom is used by the compiler for the i2b bytecode etc.
12758 instruct sbfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12759 %{
12760   match(Set dst (RShiftI (LShiftI src lshift_count) rshift_count));
12761   ins_cost(INSN_COST * 2);
12762   format %{ "sbfmw  $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12763   ins_encode %{
12764     int lshift = $lshift_count$$constant & 31;
12765     int rshift = $rshift_count$$constant & 31;
12766     int s = 31 - lshift;
12767     int r = (rshift - lshift) & 31;
12768     __ sbfmw(as_Register($dst$$reg),
12769             as_Register($src$$reg),
12770             r, s);
12771   %}
12772 
12773   ins_pipe(ialu_reg_shift);
12774 %}
12775 
12776 // This pattern is automatically generated from aarch64_ad.m4
12777 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12778 
12779 // Shift Left followed by Shift Right.
12780 // This idiom is used by the compiler for the i2b bytecode etc.
12781 instruct ubfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12782 %{
12783   match(Set dst (URShiftL (LShiftL src lshift_count) rshift_count));
12784   ins_cost(INSN_COST * 2);
12785   format %{ "ubfm  $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12786   ins_encode %{
12787     int lshift = $lshift_count$$constant & 63;
12788     int rshift = $rshift_count$$constant & 63;
12789     int s = 63 - lshift;
12790     int r = (rshift - lshift) & 63;
12791     __ ubfm(as_Register($dst$$reg),
12792             as_Register($src$$reg),
12793             r, s);
12794   %}
12795 
12796   ins_pipe(ialu_reg_shift);
12797 %}
12798 
12799 // This pattern is automatically generated from aarch64_ad.m4
12800 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12801 
12802 // Shift Left followed by Shift Right.
12803 // This idiom is used by the compiler for the i2b bytecode etc.
12804 instruct ubfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12805 %{
12806   match(Set dst (URShiftI (LShiftI src lshift_count) rshift_count));
12807   ins_cost(INSN_COST * 2);
12808   format %{ "ubfmw  $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12809   ins_encode %{
12810     int lshift = $lshift_count$$constant & 31;
12811     int rshift = $rshift_count$$constant & 31;
12812     int s = 31 - lshift;
12813     int r = (rshift - lshift) & 31;
12814     __ ubfmw(as_Register($dst$$reg),
12815             as_Register($src$$reg),
12816             r, s);
12817   %}
12818 
12819   ins_pipe(ialu_reg_shift);
12820 %}
12821 
12822 // Bitfield extract with shift & mask
12823 
12824 // This pattern is automatically generated from aarch64_ad.m4
12825 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12826 instruct ubfxwI(iRegINoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12827 %{
12828   match(Set dst (AndI (URShiftI src rshift) mask));
12829   // Make sure we are not going to exceed what ubfxw can do.
12830   predicate((exact_log2(n->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12831 
12832   ins_cost(INSN_COST);
12833   format %{ "ubfxw $dst, $src, $rshift, $mask" %}
12834   ins_encode %{
12835     int rshift = $rshift$$constant & 31;
12836     intptr_t mask = $mask$$constant;
12837     int width = exact_log2(mask+1);
12838     __ ubfxw(as_Register($dst$$reg),
12839             as_Register($src$$reg), rshift, width);
12840   %}
12841   ins_pipe(ialu_reg_shift);
12842 %}
12843 
12844 // This pattern is automatically generated from aarch64_ad.m4
12845 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12846 instruct ubfxL(iRegLNoSp dst, iRegL src, immI rshift, immL_bitmask mask)
12847 %{
12848   match(Set dst (AndL (URShiftL src rshift) mask));
12849   // Make sure we are not going to exceed what ubfx can do.
12850   predicate((exact_log2_long(n->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= (63 + 1));
12851 
12852   ins_cost(INSN_COST);
12853   format %{ "ubfx $dst, $src, $rshift, $mask" %}
12854   ins_encode %{
12855     int rshift = $rshift$$constant & 63;
12856     intptr_t mask = $mask$$constant;
12857     int width = exact_log2_long(mask+1);
12858     __ ubfx(as_Register($dst$$reg),
12859             as_Register($src$$reg), rshift, width);
12860   %}
12861   ins_pipe(ialu_reg_shift);
12862 %}
12863 
12864 
12865 // This pattern is automatically generated from aarch64_ad.m4
12866 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12867 
12868 // We can use ubfx when extending an And with a mask when we know mask
12869 // is positive.  We know that because immI_bitmask guarantees it.
12870 instruct ubfxIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12871 %{
12872   match(Set dst (ConvI2L (AndI (URShiftI src rshift) mask)));
12873   // Make sure we are not going to exceed what ubfxw can do.
12874   predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12875 
12876   ins_cost(INSN_COST * 2);
12877   format %{ "ubfx $dst, $src, $rshift, $mask" %}
12878   ins_encode %{
12879     int rshift = $rshift$$constant & 31;
12880     intptr_t mask = $mask$$constant;
12881     int width = exact_log2(mask+1);
12882     __ ubfx(as_Register($dst$$reg),
12883             as_Register($src$$reg), rshift, width);
12884   %}
12885   ins_pipe(ialu_reg_shift);
12886 %}
12887 
12888 
12889 // This pattern is automatically generated from aarch64_ad.m4
12890 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12891 
12892 // We can use ubfiz when masking by a positive number and then left shifting the result.
12893 // We know that the mask is positive because immI_bitmask guarantees it.
12894 instruct ubfizwI(iRegINoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12895 %{
12896   match(Set dst (LShiftI (AndI src mask) lshift));
12897   predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 31)) <= (31 + 1));
12898 
12899   ins_cost(INSN_COST);
12900   format %{ "ubfizw $dst, $src, $lshift, $mask" %}
12901   ins_encode %{
12902     int lshift = $lshift$$constant & 31;
12903     intptr_t mask = $mask$$constant;
12904     int width = exact_log2(mask+1);
12905     __ ubfizw(as_Register($dst$$reg),
12906           as_Register($src$$reg), lshift, width);
12907   %}
12908   ins_pipe(ialu_reg_shift);
12909 %}
12910 
12911 // This pattern is automatically generated from aarch64_ad.m4
12912 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12913 
12914 // We can use ubfiz when masking by a positive number and then left shifting the result.
12915 // We know that the mask is positive because immL_bitmask guarantees it.
12916 instruct ubfizL(iRegLNoSp dst, iRegL src, immI lshift, immL_bitmask mask)
12917 %{
12918   match(Set dst (LShiftL (AndL src mask) lshift));
12919   predicate((exact_log2_long(n->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
12920 
12921   ins_cost(INSN_COST);
12922   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12923   ins_encode %{
12924     int lshift = $lshift$$constant & 63;
12925     intptr_t mask = $mask$$constant;
12926     int width = exact_log2_long(mask+1);
12927     __ ubfiz(as_Register($dst$$reg),
12928           as_Register($src$$reg), lshift, width);
12929   %}
12930   ins_pipe(ialu_reg_shift);
12931 %}
12932 
12933 // This pattern is automatically generated from aarch64_ad.m4
12934 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12935 
12936 // We can use ubfiz when masking by a positive number and then left shifting the result.
12937 // We know that the mask is positive because immI_bitmask guarantees it.
12938 instruct ubfizwIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12939 %{
12940   match(Set dst (ConvI2L (LShiftI (AndI src mask) lshift)));
12941   predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= 31);
12942 
12943   ins_cost(INSN_COST);
12944   format %{ "ubfizw $dst, $src, $lshift, $mask" %}
12945   ins_encode %{
12946     int lshift = $lshift$$constant & 31;
12947     intptr_t mask = $mask$$constant;
12948     int width = exact_log2(mask+1);
12949     __ ubfizw(as_Register($dst$$reg),
12950           as_Register($src$$reg), lshift, width);
12951   %}
12952   ins_pipe(ialu_reg_shift);
12953 %}
12954 
12955 // This pattern is automatically generated from aarch64_ad.m4
12956 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12957 
12958 // We can use ubfiz when masking by a positive number and then left shifting the result.
12959 // We know that the mask is positive because immL_bitmask guarantees it.
12960 instruct ubfizLConvL2I(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
12961 %{
12962   match(Set dst (ConvL2I (LShiftL (AndL src mask) lshift)));
12963   predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= 31);
12964 
12965   ins_cost(INSN_COST);
12966   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12967   ins_encode %{
12968     int lshift = $lshift$$constant & 63;
12969     intptr_t mask = $mask$$constant;
12970     int width = exact_log2_long(mask+1);
12971     __ ubfiz(as_Register($dst$$reg),
12972           as_Register($src$$reg), lshift, width);
12973   %}
12974   ins_pipe(ialu_reg_shift);
12975 %}
12976 
12977 
12978 // This pattern is automatically generated from aarch64_ad.m4
12979 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12980 
12981 // If there is a convert I to L block between and AndI and a LShiftL, we can also match ubfiz
12982 instruct ubfizIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12983 %{
12984   match(Set dst (LShiftL (ConvI2L (AndI src mask)) lshift));
12985   predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
12986 
12987   ins_cost(INSN_COST);
12988   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12989   ins_encode %{
12990     int lshift = $lshift$$constant & 63;
12991     intptr_t mask = $mask$$constant;
12992     int width = exact_log2(mask+1);
12993     __ ubfiz(as_Register($dst$$reg),
12994              as_Register($src$$reg), lshift, width);
12995   %}
12996   ins_pipe(ialu_reg_shift);
12997 %}
12998 
12999 // This pattern is automatically generated from aarch64_ad.m4
13000 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13001 
13002 // If there is a convert L to I block between and AndL and a LShiftI, we can also match ubfiz
13003 instruct ubfizLConvL2Ix(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
13004 %{
13005   match(Set dst (LShiftI (ConvL2I (AndL src mask)) lshift));
13006   predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 31)) <= 31);
13007 
13008   ins_cost(INSN_COST);
13009   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
13010   ins_encode %{
13011     int lshift = $lshift$$constant & 31;
13012     intptr_t mask = $mask$$constant;
13013     int width = exact_log2(mask+1);
13014     __ ubfiz(as_Register($dst$$reg),
13015              as_Register($src$$reg), lshift, width);
13016   %}
13017   ins_pipe(ialu_reg_shift);
13018 %}
13019 
13020 // This pattern is automatically generated from aarch64_ad.m4
13021 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13022 
13023 // Can skip int2long conversions after AND with small bitmask
13024 instruct ubfizIConvI2LAndI(iRegLNoSp dst, iRegI src, immI_bitmask msk)
13025 %{
13026   match(Set dst (ConvI2L (AndI src msk)));
13027   ins_cost(INSN_COST);
13028   format %{ "ubfiz $dst, $src, 0, exact_log2($msk + 1) " %}
13029   ins_encode %{
13030     __ ubfiz(as_Register($dst$$reg), as_Register($src$$reg), 0, exact_log2($msk$$constant + 1));
13031   %}
13032   ins_pipe(ialu_reg_shift);
13033 %}
13034 
13035 
13036 // Rotations
13037 // This pattern is automatically generated from aarch64_ad.m4
13038 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13039 instruct extrOrL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
13040 %{
13041   match(Set dst (OrL (LShiftL src1 lshift) (URShiftL src2 rshift)));
13042   predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
13043 
13044   ins_cost(INSN_COST);
13045   format %{ "extr $dst, $src1, $src2, #$rshift" %}
13046 
13047   ins_encode %{
13048     __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13049             $rshift$$constant & 63);
13050   %}
13051   ins_pipe(ialu_reg_reg_extr);
13052 %}
13053 
13054 
13055 // This pattern is automatically generated from aarch64_ad.m4
13056 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13057 instruct extrOrI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
13058 %{
13059   match(Set dst (OrI (LShiftI src1 lshift) (URShiftI src2 rshift)));
13060   predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
13061 
13062   ins_cost(INSN_COST);
13063   format %{ "extr $dst, $src1, $src2, #$rshift" %}
13064 
13065   ins_encode %{
13066     __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13067             $rshift$$constant & 31);
13068   %}
13069   ins_pipe(ialu_reg_reg_extr);
13070 %}
13071 
13072 
13073 // This pattern is automatically generated from aarch64_ad.m4
13074 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13075 instruct extrAddL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
13076 %{
13077   match(Set dst (AddL (LShiftL src1 lshift) (URShiftL src2 rshift)));
13078   predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
13079 
13080   ins_cost(INSN_COST);
13081   format %{ "extr $dst, $src1, $src2, #$rshift" %}
13082 
13083   ins_encode %{
13084     __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13085             $rshift$$constant & 63);
13086   %}
13087   ins_pipe(ialu_reg_reg_extr);
13088 %}
13089 
13090 
13091 // This pattern is automatically generated from aarch64_ad.m4
13092 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13093 instruct extrAddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
13094 %{
13095   match(Set dst (AddI (LShiftI src1 lshift) (URShiftI src2 rshift)));
13096   predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
13097 
13098   ins_cost(INSN_COST);
13099   format %{ "extr $dst, $src1, $src2, #$rshift" %}
13100 
13101   ins_encode %{
13102     __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13103             $rshift$$constant & 31);
13104   %}
13105   ins_pipe(ialu_reg_reg_extr);
13106 %}
13107 
13108 
13109 // This pattern is automatically generated from aarch64_ad.m4
13110 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13111 instruct rorI_imm(iRegINoSp dst, iRegI src, immI shift)
13112 %{
13113   match(Set dst (RotateRight src shift));
13114 
13115   ins_cost(INSN_COST);
13116   format %{ "ror    $dst, $src, $shift" %}
13117 
13118   ins_encode %{
13119      __ extrw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
13120                $shift$$constant & 0x1f);
13121   %}
13122   ins_pipe(ialu_reg_reg_vshift);
13123 %}
13124 
13125 // This pattern is automatically generated from aarch64_ad.m4
13126 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13127 instruct rorL_imm(iRegLNoSp dst, iRegL src, immI shift)
13128 %{
13129   match(Set dst (RotateRight src shift));
13130 
13131   ins_cost(INSN_COST);
13132   format %{ "ror    $dst, $src, $shift" %}
13133 
13134   ins_encode %{
13135      __ extr(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
13136                $shift$$constant & 0x3f);
13137   %}
13138   ins_pipe(ialu_reg_reg_vshift);
13139 %}
13140 
13141 // This pattern is automatically generated from aarch64_ad.m4
13142 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13143 instruct rorI_reg(iRegINoSp dst, iRegI src, iRegI shift)
13144 %{
13145   match(Set dst (RotateRight src shift));
13146 
13147   ins_cost(INSN_COST);
13148   format %{ "ror    $dst, $src, $shift" %}
13149 
13150   ins_encode %{
13151      __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
13152   %}
13153   ins_pipe(ialu_reg_reg_vshift);
13154 %}
13155 
13156 // This pattern is automatically generated from aarch64_ad.m4
13157 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13158 instruct rorL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
13159 %{
13160   match(Set dst (RotateRight src shift));
13161 
13162   ins_cost(INSN_COST);
13163   format %{ "ror    $dst, $src, $shift" %}
13164 
13165   ins_encode %{
13166      __ rorv(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
13167   %}
13168   ins_pipe(ialu_reg_reg_vshift);
13169 %}
13170 
13171 // This pattern is automatically generated from aarch64_ad.m4
13172 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13173 instruct rolI_reg(iRegINoSp dst, iRegI src, iRegI shift)
13174 %{
13175   match(Set dst (RotateLeft src shift));
13176 
13177   ins_cost(INSN_COST);
13178   format %{ "rol    $dst, $src, $shift" %}
13179 
13180   ins_encode %{
13181      __ subw(rscratch1, zr, as_Register($shift$$reg));
13182      __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
13183   %}
13184   ins_pipe(ialu_reg_reg_vshift);
13185 %}
13186 
13187 // This pattern is automatically generated from aarch64_ad.m4
13188 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13189 instruct rolL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
13190 %{
13191   match(Set dst (RotateLeft src shift));
13192 
13193   ins_cost(INSN_COST);
13194   format %{ "rol    $dst, $src, $shift" %}
13195 
13196   ins_encode %{
13197      __ subw(rscratch1, zr, as_Register($shift$$reg));
13198      __ rorv(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
13199   %}
13200   ins_pipe(ialu_reg_reg_vshift);
13201 %}
13202 
13203 
13204 // Add/subtract (extended)
13205 
13206 // This pattern is automatically generated from aarch64_ad.m4
13207 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13208 instruct AddExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
13209 %{
13210   match(Set dst (AddL src1 (ConvI2L src2)));
13211   ins_cost(INSN_COST);
13212   format %{ "add  $dst, $src1, $src2, sxtw" %}
13213 
13214    ins_encode %{
13215      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13216             as_Register($src2$$reg), ext::sxtw);
13217    %}
13218   ins_pipe(ialu_reg_reg);
13219 %}
13220 
13221 // This pattern is automatically generated from aarch64_ad.m4
13222 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13223 instruct SubExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
13224 %{
13225   match(Set dst (SubL src1 (ConvI2L src2)));
13226   ins_cost(INSN_COST);
13227   format %{ "sub  $dst, $src1, $src2, sxtw" %}
13228 
13229    ins_encode %{
13230      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13231             as_Register($src2$$reg), ext::sxtw);
13232    %}
13233   ins_pipe(ialu_reg_reg);
13234 %}
13235 
13236 // This pattern is automatically generated from aarch64_ad.m4
13237 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13238 instruct AddExtI_sxth(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_16 lshift, immI_16 rshift, rFlagsReg cr)
13239 %{
13240   match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
13241   ins_cost(INSN_COST);
13242   format %{ "add  $dst, $src1, $src2, sxth" %}
13243 
13244    ins_encode %{
13245      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13246             as_Register($src2$$reg), ext::sxth);
13247    %}
13248   ins_pipe(ialu_reg_reg);
13249 %}
13250 
13251 // This pattern is automatically generated from aarch64_ad.m4
13252 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13253 instruct AddExtI_sxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
13254 %{
13255   match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
13256   ins_cost(INSN_COST);
13257   format %{ "add  $dst, $src1, $src2, sxtb" %}
13258 
13259    ins_encode %{
13260      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13261             as_Register($src2$$reg), ext::sxtb);
13262    %}
13263   ins_pipe(ialu_reg_reg);
13264 %}
13265 
13266 // This pattern is automatically generated from aarch64_ad.m4
13267 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13268 instruct AddExtI_uxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
13269 %{
13270   match(Set dst (AddI src1 (URShiftI (LShiftI src2 lshift) rshift)));
13271   ins_cost(INSN_COST);
13272   format %{ "add  $dst, $src1, $src2, uxtb" %}
13273 
13274    ins_encode %{
13275      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13276             as_Register($src2$$reg), ext::uxtb);
13277    %}
13278   ins_pipe(ialu_reg_reg);
13279 %}
13280 
13281 // This pattern is automatically generated from aarch64_ad.m4
13282 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13283 instruct AddExtL_sxth(iRegLNoSp dst, iRegL src1, iRegL src2, immI_48 lshift, immI_48 rshift, rFlagsReg cr)
13284 %{
13285   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13286   ins_cost(INSN_COST);
13287   format %{ "add  $dst, $src1, $src2, sxth" %}
13288 
13289    ins_encode %{
13290      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13291             as_Register($src2$$reg), ext::sxth);
13292    %}
13293   ins_pipe(ialu_reg_reg);
13294 %}
13295 
13296 // This pattern is automatically generated from aarch64_ad.m4
13297 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13298 instruct AddExtL_sxtw(iRegLNoSp dst, iRegL src1, iRegL src2, immI_32 lshift, immI_32 rshift, rFlagsReg cr)
13299 %{
13300   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13301   ins_cost(INSN_COST);
13302   format %{ "add  $dst, $src1, $src2, sxtw" %}
13303 
13304    ins_encode %{
13305      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13306             as_Register($src2$$reg), ext::sxtw);
13307    %}
13308   ins_pipe(ialu_reg_reg);
13309 %}
13310 
13311 // This pattern is automatically generated from aarch64_ad.m4
13312 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13313 instruct AddExtL_sxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
13314 %{
13315   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13316   ins_cost(INSN_COST);
13317   format %{ "add  $dst, $src1, $src2, sxtb" %}
13318 
13319    ins_encode %{
13320      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13321             as_Register($src2$$reg), ext::sxtb);
13322    %}
13323   ins_pipe(ialu_reg_reg);
13324 %}
13325 
13326 // This pattern is automatically generated from aarch64_ad.m4
13327 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13328 instruct AddExtL_uxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
13329 %{
13330   match(Set dst (AddL src1 (URShiftL (LShiftL src2 lshift) rshift)));
13331   ins_cost(INSN_COST);
13332   format %{ "add  $dst, $src1, $src2, uxtb" %}
13333 
13334    ins_encode %{
13335      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13336             as_Register($src2$$reg), ext::uxtb);
13337    %}
13338   ins_pipe(ialu_reg_reg);
13339 %}
13340 
13341 // This pattern is automatically generated from aarch64_ad.m4
13342 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13343 instruct AddExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
13344 %{
13345   match(Set dst (AddI src1 (AndI src2 mask)));
13346   ins_cost(INSN_COST);
13347   format %{ "addw  $dst, $src1, $src2, uxtb" %}
13348 
13349    ins_encode %{
13350      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13351             as_Register($src2$$reg), ext::uxtb);
13352    %}
13353   ins_pipe(ialu_reg_reg);
13354 %}
13355 
13356 // This pattern is automatically generated from aarch64_ad.m4
13357 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13358 instruct AddExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
13359 %{
13360   match(Set dst (AddI src1 (AndI src2 mask)));
13361   ins_cost(INSN_COST);
13362   format %{ "addw  $dst, $src1, $src2, uxth" %}
13363 
13364    ins_encode %{
13365      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13366             as_Register($src2$$reg), ext::uxth);
13367    %}
13368   ins_pipe(ialu_reg_reg);
13369 %}
13370 
13371 // This pattern is automatically generated from aarch64_ad.m4
13372 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13373 instruct AddExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
13374 %{
13375   match(Set dst (AddL src1 (AndL src2 mask)));
13376   ins_cost(INSN_COST);
13377   format %{ "add  $dst, $src1, $src2, uxtb" %}
13378 
13379    ins_encode %{
13380      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13381             as_Register($src2$$reg), ext::uxtb);
13382    %}
13383   ins_pipe(ialu_reg_reg);
13384 %}
13385 
13386 // This pattern is automatically generated from aarch64_ad.m4
13387 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13388 instruct AddExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
13389 %{
13390   match(Set dst (AddL src1 (AndL src2 mask)));
13391   ins_cost(INSN_COST);
13392   format %{ "add  $dst, $src1, $src2, uxth" %}
13393 
13394    ins_encode %{
13395      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13396             as_Register($src2$$reg), ext::uxth);
13397    %}
13398   ins_pipe(ialu_reg_reg);
13399 %}
13400 
13401 // This pattern is automatically generated from aarch64_ad.m4
13402 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13403 instruct AddExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
13404 %{
13405   match(Set dst (AddL src1 (AndL src2 mask)));
13406   ins_cost(INSN_COST);
13407   format %{ "add  $dst, $src1, $src2, uxtw" %}
13408 
13409    ins_encode %{
13410      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13411             as_Register($src2$$reg), ext::uxtw);
13412    %}
13413   ins_pipe(ialu_reg_reg);
13414 %}
13415 
13416 // This pattern is automatically generated from aarch64_ad.m4
13417 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13418 instruct SubExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
13419 %{
13420   match(Set dst (SubI src1 (AndI src2 mask)));
13421   ins_cost(INSN_COST);
13422   format %{ "subw  $dst, $src1, $src2, uxtb" %}
13423 
13424    ins_encode %{
13425      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13426             as_Register($src2$$reg), ext::uxtb);
13427    %}
13428   ins_pipe(ialu_reg_reg);
13429 %}
13430 
13431 // This pattern is automatically generated from aarch64_ad.m4
13432 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13433 instruct SubExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
13434 %{
13435   match(Set dst (SubI src1 (AndI src2 mask)));
13436   ins_cost(INSN_COST);
13437   format %{ "subw  $dst, $src1, $src2, uxth" %}
13438 
13439    ins_encode %{
13440      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13441             as_Register($src2$$reg), ext::uxth);
13442    %}
13443   ins_pipe(ialu_reg_reg);
13444 %}
13445 
13446 // This pattern is automatically generated from aarch64_ad.m4
13447 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13448 instruct SubExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
13449 %{
13450   match(Set dst (SubL src1 (AndL src2 mask)));
13451   ins_cost(INSN_COST);
13452   format %{ "sub  $dst, $src1, $src2, uxtb" %}
13453 
13454    ins_encode %{
13455      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13456             as_Register($src2$$reg), ext::uxtb);
13457    %}
13458   ins_pipe(ialu_reg_reg);
13459 %}
13460 
13461 // This pattern is automatically generated from aarch64_ad.m4
13462 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13463 instruct SubExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
13464 %{
13465   match(Set dst (SubL src1 (AndL src2 mask)));
13466   ins_cost(INSN_COST);
13467   format %{ "sub  $dst, $src1, $src2, uxth" %}
13468 
13469    ins_encode %{
13470      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13471             as_Register($src2$$reg), ext::uxth);
13472    %}
13473   ins_pipe(ialu_reg_reg);
13474 %}
13475 
13476 // This pattern is automatically generated from aarch64_ad.m4
13477 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13478 instruct SubExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
13479 %{
13480   match(Set dst (SubL src1 (AndL src2 mask)));
13481   ins_cost(INSN_COST);
13482   format %{ "sub  $dst, $src1, $src2, uxtw" %}
13483 
13484    ins_encode %{
13485      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13486             as_Register($src2$$reg), ext::uxtw);
13487    %}
13488   ins_pipe(ialu_reg_reg);
13489 %}
13490 
13491 
13492 // This pattern is automatically generated from aarch64_ad.m4
13493 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13494 instruct AddExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13495 %{
13496   match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13497   ins_cost(1.9 * INSN_COST);
13498   format %{ "add  $dst, $src1, $src2, sxtb #lshift2" %}
13499 
13500    ins_encode %{
13501      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13502             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13503    %}
13504   ins_pipe(ialu_reg_reg_shift);
13505 %}
13506 
13507 // This pattern is automatically generated from aarch64_ad.m4
13508 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13509 instruct AddExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13510 %{
13511   match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13512   ins_cost(1.9 * INSN_COST);
13513   format %{ "add  $dst, $src1, $src2, sxth #lshift2" %}
13514 
13515    ins_encode %{
13516      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13517             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13518    %}
13519   ins_pipe(ialu_reg_reg_shift);
13520 %}
13521 
13522 // This pattern is automatically generated from aarch64_ad.m4
13523 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13524 instruct AddExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13525 %{
13526   match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13527   ins_cost(1.9 * INSN_COST);
13528   format %{ "add  $dst, $src1, $src2, sxtw #lshift2" %}
13529 
13530    ins_encode %{
13531      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13532             as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13533    %}
13534   ins_pipe(ialu_reg_reg_shift);
13535 %}
13536 
13537 // This pattern is automatically generated from aarch64_ad.m4
13538 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13539 instruct SubExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13540 %{
13541   match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13542   ins_cost(1.9 * INSN_COST);
13543   format %{ "sub  $dst, $src1, $src2, sxtb #lshift2" %}
13544 
13545    ins_encode %{
13546      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13547             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13548    %}
13549   ins_pipe(ialu_reg_reg_shift);
13550 %}
13551 
13552 // This pattern is automatically generated from aarch64_ad.m4
13553 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13554 instruct SubExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13555 %{
13556   match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13557   ins_cost(1.9 * INSN_COST);
13558   format %{ "sub  $dst, $src1, $src2, sxth #lshift2" %}
13559 
13560    ins_encode %{
13561      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13562             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13563    %}
13564   ins_pipe(ialu_reg_reg_shift);
13565 %}
13566 
13567 // This pattern is automatically generated from aarch64_ad.m4
13568 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13569 instruct SubExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13570 %{
13571   match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13572   ins_cost(1.9 * INSN_COST);
13573   format %{ "sub  $dst, $src1, $src2, sxtw #lshift2" %}
13574 
13575    ins_encode %{
13576      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13577             as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13578    %}
13579   ins_pipe(ialu_reg_reg_shift);
13580 %}
13581 
13582 // This pattern is automatically generated from aarch64_ad.m4
13583 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13584 instruct AddExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13585 %{
13586   match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13587   ins_cost(1.9 * INSN_COST);
13588   format %{ "addw  $dst, $src1, $src2, sxtb #lshift2" %}
13589 
13590    ins_encode %{
13591      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13592             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13593    %}
13594   ins_pipe(ialu_reg_reg_shift);
13595 %}
13596 
13597 // This pattern is automatically generated from aarch64_ad.m4
13598 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13599 instruct AddExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13600 %{
13601   match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13602   ins_cost(1.9 * INSN_COST);
13603   format %{ "addw  $dst, $src1, $src2, sxth #lshift2" %}
13604 
13605    ins_encode %{
13606      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13607             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13608    %}
13609   ins_pipe(ialu_reg_reg_shift);
13610 %}
13611 
13612 // This pattern is automatically generated from aarch64_ad.m4
13613 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13614 instruct SubExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13615 %{
13616   match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13617   ins_cost(1.9 * INSN_COST);
13618   format %{ "subw  $dst, $src1, $src2, sxtb #lshift2" %}
13619 
13620    ins_encode %{
13621      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13622             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13623    %}
13624   ins_pipe(ialu_reg_reg_shift);
13625 %}
13626 
13627 // This pattern is automatically generated from aarch64_ad.m4
13628 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13629 instruct SubExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13630 %{
13631   match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13632   ins_cost(1.9 * INSN_COST);
13633   format %{ "subw  $dst, $src1, $src2, sxth #lshift2" %}
13634 
13635    ins_encode %{
13636      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13637             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13638    %}
13639   ins_pipe(ialu_reg_reg_shift);
13640 %}
13641 
13642 // This pattern is automatically generated from aarch64_ad.m4
13643 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13644 instruct AddExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13645 %{
13646   match(Set dst (AddL src1 (LShiftL (ConvI2L src2) lshift)));
13647   ins_cost(1.9 * INSN_COST);
13648   format %{ "add  $dst, $src1, $src2, sxtw #lshift" %}
13649 
13650    ins_encode %{
13651      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13652             as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13653    %}
13654   ins_pipe(ialu_reg_reg_shift);
13655 %}
13656 
13657 // This pattern is automatically generated from aarch64_ad.m4
13658 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13659 instruct SubExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13660 %{
13661   match(Set dst (SubL src1 (LShiftL (ConvI2L src2) lshift)));
13662   ins_cost(1.9 * INSN_COST);
13663   format %{ "sub  $dst, $src1, $src2, sxtw #lshift" %}
13664 
13665    ins_encode %{
13666      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13667             as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13668    %}
13669   ins_pipe(ialu_reg_reg_shift);
13670 %}
13671 
13672 // This pattern is automatically generated from aarch64_ad.m4
13673 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13674 instruct AddExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13675 %{
13676   match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13677   ins_cost(1.9 * INSN_COST);
13678   format %{ "add  $dst, $src1, $src2, uxtb #lshift" %}
13679 
13680    ins_encode %{
13681      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13682             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13683    %}
13684   ins_pipe(ialu_reg_reg_shift);
13685 %}
13686 
13687 // This pattern is automatically generated from aarch64_ad.m4
13688 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13689 instruct AddExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13690 %{
13691   match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13692   ins_cost(1.9 * INSN_COST);
13693   format %{ "add  $dst, $src1, $src2, uxth #lshift" %}
13694 
13695    ins_encode %{
13696      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13697             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13698    %}
13699   ins_pipe(ialu_reg_reg_shift);
13700 %}
13701 
13702 // This pattern is automatically generated from aarch64_ad.m4
13703 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13704 instruct AddExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13705 %{
13706   match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13707   ins_cost(1.9 * INSN_COST);
13708   format %{ "add  $dst, $src1, $src2, uxtw #lshift" %}
13709 
13710    ins_encode %{
13711      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13712             as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13713    %}
13714   ins_pipe(ialu_reg_reg_shift);
13715 %}
13716 
13717 // This pattern is automatically generated from aarch64_ad.m4
13718 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13719 instruct SubExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13720 %{
13721   match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13722   ins_cost(1.9 * INSN_COST);
13723   format %{ "sub  $dst, $src1, $src2, uxtb #lshift" %}
13724 
13725    ins_encode %{
13726      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13727             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13728    %}
13729   ins_pipe(ialu_reg_reg_shift);
13730 %}
13731 
13732 // This pattern is automatically generated from aarch64_ad.m4
13733 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13734 instruct SubExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13735 %{
13736   match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13737   ins_cost(1.9 * INSN_COST);
13738   format %{ "sub  $dst, $src1, $src2, uxth #lshift" %}
13739 
13740    ins_encode %{
13741      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13742             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13743    %}
13744   ins_pipe(ialu_reg_reg_shift);
13745 %}
13746 
13747 // This pattern is automatically generated from aarch64_ad.m4
13748 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13749 instruct SubExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13750 %{
13751   match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13752   ins_cost(1.9 * INSN_COST);
13753   format %{ "sub  $dst, $src1, $src2, uxtw #lshift" %}
13754 
13755    ins_encode %{
13756      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13757             as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13758    %}
13759   ins_pipe(ialu_reg_reg_shift);
13760 %}
13761 
13762 // This pattern is automatically generated from aarch64_ad.m4
13763 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13764 instruct AddExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13765 %{
13766   match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13767   ins_cost(1.9 * INSN_COST);
13768   format %{ "addw  $dst, $src1, $src2, uxtb #lshift" %}
13769 
13770    ins_encode %{
13771      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13772             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13773    %}
13774   ins_pipe(ialu_reg_reg_shift);
13775 %}
13776 
13777 // This pattern is automatically generated from aarch64_ad.m4
13778 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13779 instruct AddExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13780 %{
13781   match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13782   ins_cost(1.9 * INSN_COST);
13783   format %{ "addw  $dst, $src1, $src2, uxth #lshift" %}
13784 
13785    ins_encode %{
13786      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13787             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13788    %}
13789   ins_pipe(ialu_reg_reg_shift);
13790 %}
13791 
13792 // This pattern is automatically generated from aarch64_ad.m4
13793 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13794 instruct SubExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13795 %{
13796   match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13797   ins_cost(1.9 * INSN_COST);
13798   format %{ "subw  $dst, $src1, $src2, uxtb #lshift" %}
13799 
13800    ins_encode %{
13801      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13802             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13803    %}
13804   ins_pipe(ialu_reg_reg_shift);
13805 %}
13806 
13807 // This pattern is automatically generated from aarch64_ad.m4
13808 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13809 instruct SubExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13810 %{
13811   match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13812   ins_cost(1.9 * INSN_COST);
13813   format %{ "subw  $dst, $src1, $src2, uxth #lshift" %}
13814 
13815    ins_encode %{
13816      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13817             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13818    %}
13819   ins_pipe(ialu_reg_reg_shift);
13820 %}
13821 
13822 
13823 
13824 // END This section of the file is automatically generated. Do not edit --------------
13825 
13826 
13827 // ============================================================================
13828 // Floating Point Arithmetic Instructions
13829 
13830 instruct addF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13831   match(Set dst (AddF src1 src2));
13832 
13833   ins_cost(INSN_COST * 5);
13834   format %{ "fadds   $dst, $src1, $src2" %}
13835 
13836   ins_encode %{
13837     __ fadds(as_FloatRegister($dst$$reg),
13838              as_FloatRegister($src1$$reg),
13839              as_FloatRegister($src2$$reg));
13840   %}
13841 
13842   ins_pipe(fp_dop_reg_reg_s);
13843 %}
13844 
13845 instruct addD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13846   match(Set dst (AddD src1 src2));
13847 
13848   ins_cost(INSN_COST * 5);
13849   format %{ "faddd   $dst, $src1, $src2" %}
13850 
13851   ins_encode %{
13852     __ faddd(as_FloatRegister($dst$$reg),
13853              as_FloatRegister($src1$$reg),
13854              as_FloatRegister($src2$$reg));
13855   %}
13856 
13857   ins_pipe(fp_dop_reg_reg_d);
13858 %}
13859 
13860 instruct subF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13861   match(Set dst (SubF src1 src2));
13862 
13863   ins_cost(INSN_COST * 5);
13864   format %{ "fsubs   $dst, $src1, $src2" %}
13865 
13866   ins_encode %{
13867     __ fsubs(as_FloatRegister($dst$$reg),
13868              as_FloatRegister($src1$$reg),
13869              as_FloatRegister($src2$$reg));
13870   %}
13871 
13872   ins_pipe(fp_dop_reg_reg_s);
13873 %}
13874 
13875 instruct subD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13876   match(Set dst (SubD src1 src2));
13877 
13878   ins_cost(INSN_COST * 5);
13879   format %{ "fsubd   $dst, $src1, $src2" %}
13880 
13881   ins_encode %{
13882     __ fsubd(as_FloatRegister($dst$$reg),
13883              as_FloatRegister($src1$$reg),
13884              as_FloatRegister($src2$$reg));
13885   %}
13886 
13887   ins_pipe(fp_dop_reg_reg_d);
13888 %}
13889 
13890 instruct mulF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13891   match(Set dst (MulF src1 src2));
13892 
13893   ins_cost(INSN_COST * 6);
13894   format %{ "fmuls   $dst, $src1, $src2" %}
13895 
13896   ins_encode %{
13897     __ fmuls(as_FloatRegister($dst$$reg),
13898              as_FloatRegister($src1$$reg),
13899              as_FloatRegister($src2$$reg));
13900   %}
13901 
13902   ins_pipe(fp_dop_reg_reg_s);
13903 %}
13904 
13905 instruct mulD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13906   match(Set dst (MulD src1 src2));
13907 
13908   ins_cost(INSN_COST * 6);
13909   format %{ "fmuld   $dst, $src1, $src2" %}
13910 
13911   ins_encode %{
13912     __ fmuld(as_FloatRegister($dst$$reg),
13913              as_FloatRegister($src1$$reg),
13914              as_FloatRegister($src2$$reg));
13915   %}
13916 
13917   ins_pipe(fp_dop_reg_reg_d);
13918 %}
13919 
13920 // src1 * src2 + src3
13921 instruct maddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13922   predicate(UseFMA);
13923   match(Set dst (FmaF src3 (Binary src1 src2)));
13924 
13925   format %{ "fmadds   $dst, $src1, $src2, $src3" %}
13926 
13927   ins_encode %{
13928     __ fmadds(as_FloatRegister($dst$$reg),
13929              as_FloatRegister($src1$$reg),
13930              as_FloatRegister($src2$$reg),
13931              as_FloatRegister($src3$$reg));
13932   %}
13933 
13934   ins_pipe(pipe_class_default);
13935 %}
13936 
13937 // src1 * src2 + src3
13938 instruct maddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13939   predicate(UseFMA);
13940   match(Set dst (FmaD src3 (Binary src1 src2)));
13941 
13942   format %{ "fmaddd   $dst, $src1, $src2, $src3" %}
13943 
13944   ins_encode %{
13945     __ fmaddd(as_FloatRegister($dst$$reg),
13946              as_FloatRegister($src1$$reg),
13947              as_FloatRegister($src2$$reg),
13948              as_FloatRegister($src3$$reg));
13949   %}
13950 
13951   ins_pipe(pipe_class_default);
13952 %}
13953 
13954 // -src1 * src2 + src3
13955 instruct msubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13956   predicate(UseFMA);
13957   match(Set dst (FmaF src3 (Binary (NegF src1) src2)));
13958   match(Set dst (FmaF src3 (Binary src1 (NegF src2))));
13959 
13960   format %{ "fmsubs   $dst, $src1, $src2, $src3" %}
13961 
13962   ins_encode %{
13963     __ fmsubs(as_FloatRegister($dst$$reg),
13964               as_FloatRegister($src1$$reg),
13965               as_FloatRegister($src2$$reg),
13966               as_FloatRegister($src3$$reg));
13967   %}
13968 
13969   ins_pipe(pipe_class_default);
13970 %}
13971 
13972 // -src1 * src2 + src3
13973 instruct msubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13974   predicate(UseFMA);
13975   match(Set dst (FmaD src3 (Binary (NegD src1) src2)));
13976   match(Set dst (FmaD src3 (Binary src1 (NegD src2))));
13977 
13978   format %{ "fmsubd   $dst, $src1, $src2, $src3" %}
13979 
13980   ins_encode %{
13981     __ fmsubd(as_FloatRegister($dst$$reg),
13982               as_FloatRegister($src1$$reg),
13983               as_FloatRegister($src2$$reg),
13984               as_FloatRegister($src3$$reg));
13985   %}
13986 
13987   ins_pipe(pipe_class_default);
13988 %}
13989 
13990 // -src1 * src2 - src3
13991 instruct mnaddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13992   predicate(UseFMA);
13993   match(Set dst (FmaF (NegF src3) (Binary (NegF src1) src2)));
13994   match(Set dst (FmaF (NegF src3) (Binary src1 (NegF src2))));
13995 
13996   format %{ "fnmadds  $dst, $src1, $src2, $src3" %}
13997 
13998   ins_encode %{
13999     __ fnmadds(as_FloatRegister($dst$$reg),
14000                as_FloatRegister($src1$$reg),
14001                as_FloatRegister($src2$$reg),
14002                as_FloatRegister($src3$$reg));
14003   %}
14004 
14005   ins_pipe(pipe_class_default);
14006 %}
14007 
14008 // -src1 * src2 - src3
14009 instruct mnaddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
14010   predicate(UseFMA);
14011   match(Set dst (FmaD (NegD src3) (Binary (NegD src1) src2)));
14012   match(Set dst (FmaD (NegD src3) (Binary src1 (NegD src2))));
14013 
14014   format %{ "fnmaddd   $dst, $src1, $src2, $src3" %}
14015 
14016   ins_encode %{
14017     __ fnmaddd(as_FloatRegister($dst$$reg),
14018                as_FloatRegister($src1$$reg),
14019                as_FloatRegister($src2$$reg),
14020                as_FloatRegister($src3$$reg));
14021   %}
14022 
14023   ins_pipe(pipe_class_default);
14024 %}
14025 
14026 // src1 * src2 - src3
14027 instruct mnsubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3, immF0 zero) %{
14028   predicate(UseFMA);
14029   match(Set dst (FmaF (NegF src3) (Binary src1 src2)));
14030 
14031   format %{ "fnmsubs  $dst, $src1, $src2, $src3" %}
14032 
14033   ins_encode %{
14034     __ fnmsubs(as_FloatRegister($dst$$reg),
14035                as_FloatRegister($src1$$reg),
14036                as_FloatRegister($src2$$reg),
14037                as_FloatRegister($src3$$reg));
14038   %}
14039 
14040   ins_pipe(pipe_class_default);
14041 %}
14042 
14043 // src1 * src2 - src3
14044 instruct mnsubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3, immD0 zero) %{
14045   predicate(UseFMA);
14046   match(Set dst (FmaD (NegD src3) (Binary src1 src2)));
14047 
14048   format %{ "fnmsubd   $dst, $src1, $src2, $src3" %}
14049 
14050   ins_encode %{
14051   // n.b. insn name should be fnmsubd
14052     __ fnmsub(as_FloatRegister($dst$$reg),
14053               as_FloatRegister($src1$$reg),
14054               as_FloatRegister($src2$$reg),
14055               as_FloatRegister($src3$$reg));
14056   %}
14057 
14058   ins_pipe(pipe_class_default);
14059 %}
14060 
14061 
14062 // Math.max(FF)F
14063 instruct maxF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14064   match(Set dst (MaxF src1 src2));
14065 
14066   format %{ "fmaxs   $dst, $src1, $src2" %}
14067   ins_encode %{
14068     __ fmaxs(as_FloatRegister($dst$$reg),
14069              as_FloatRegister($src1$$reg),
14070              as_FloatRegister($src2$$reg));
14071   %}
14072 
14073   ins_pipe(fp_dop_reg_reg_s);
14074 %}
14075 
14076 // Math.min(FF)F
14077 instruct minF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14078   match(Set dst (MinF src1 src2));
14079 
14080   format %{ "fmins   $dst, $src1, $src2" %}
14081   ins_encode %{
14082     __ fmins(as_FloatRegister($dst$$reg),
14083              as_FloatRegister($src1$$reg),
14084              as_FloatRegister($src2$$reg));
14085   %}
14086 
14087   ins_pipe(fp_dop_reg_reg_s);
14088 %}
14089 
14090 // Math.max(DD)D
14091 instruct maxD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14092   match(Set dst (MaxD src1 src2));
14093 
14094   format %{ "fmaxd   $dst, $src1, $src2" %}
14095   ins_encode %{
14096     __ fmaxd(as_FloatRegister($dst$$reg),
14097              as_FloatRegister($src1$$reg),
14098              as_FloatRegister($src2$$reg));
14099   %}
14100 
14101   ins_pipe(fp_dop_reg_reg_d);
14102 %}
14103 
14104 // Math.min(DD)D
14105 instruct minD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14106   match(Set dst (MinD src1 src2));
14107 
14108   format %{ "fmind   $dst, $src1, $src2" %}
14109   ins_encode %{
14110     __ fmind(as_FloatRegister($dst$$reg),
14111              as_FloatRegister($src1$$reg),
14112              as_FloatRegister($src2$$reg));
14113   %}
14114 
14115   ins_pipe(fp_dop_reg_reg_d);
14116 %}
14117 
14118 
14119 instruct divF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14120   match(Set dst (DivF src1  src2));
14121 
14122   ins_cost(INSN_COST * 18);
14123   format %{ "fdivs   $dst, $src1, $src2" %}
14124 
14125   ins_encode %{
14126     __ fdivs(as_FloatRegister($dst$$reg),
14127              as_FloatRegister($src1$$reg),
14128              as_FloatRegister($src2$$reg));
14129   %}
14130 
14131   ins_pipe(fp_div_s);
14132 %}
14133 
14134 instruct divD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14135   match(Set dst (DivD src1  src2));
14136 
14137   ins_cost(INSN_COST * 32);
14138   format %{ "fdivd   $dst, $src1, $src2" %}
14139 
14140   ins_encode %{
14141     __ fdivd(as_FloatRegister($dst$$reg),
14142              as_FloatRegister($src1$$reg),
14143              as_FloatRegister($src2$$reg));
14144   %}
14145 
14146   ins_pipe(fp_div_d);
14147 %}
14148 
14149 instruct negF_reg_reg(vRegF dst, vRegF src) %{
14150   match(Set dst (NegF src));
14151 
14152   ins_cost(INSN_COST * 3);
14153   format %{ "fneg   $dst, $src" %}
14154 
14155   ins_encode %{
14156     __ fnegs(as_FloatRegister($dst$$reg),
14157              as_FloatRegister($src$$reg));
14158   %}
14159 
14160   ins_pipe(fp_uop_s);
14161 %}
14162 
14163 instruct negD_reg_reg(vRegD dst, vRegD src) %{
14164   match(Set dst (NegD src));
14165 
14166   ins_cost(INSN_COST * 3);
14167   format %{ "fnegd   $dst, $src" %}
14168 
14169   ins_encode %{
14170     __ fnegd(as_FloatRegister($dst$$reg),
14171              as_FloatRegister($src$$reg));
14172   %}
14173 
14174   ins_pipe(fp_uop_d);
14175 %}
14176 
14177 instruct absI_reg(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
14178 %{
14179   match(Set dst (AbsI src));
14180 
14181   effect(KILL cr);
14182   ins_cost(INSN_COST * 2);
14183   format %{ "cmpw  $src, zr\n\t"
14184             "cnegw $dst, $src, Assembler::LT\t# int abs"
14185   %}
14186 
14187   ins_encode %{
14188     __ cmpw(as_Register($src$$reg), zr);
14189     __ cnegw(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14190   %}
14191   ins_pipe(pipe_class_default);
14192 %}
14193 
14194 instruct absL_reg(iRegLNoSp dst, iRegL src, rFlagsReg cr)
14195 %{
14196   match(Set dst (AbsL src));
14197 
14198   effect(KILL cr);
14199   ins_cost(INSN_COST * 2);
14200   format %{ "cmp  $src, zr\n\t"
14201             "cneg $dst, $src, Assembler::LT\t# long abs"
14202   %}
14203 
14204   ins_encode %{
14205     __ cmp(as_Register($src$$reg), zr);
14206     __ cneg(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14207   %}
14208   ins_pipe(pipe_class_default);
14209 %}
14210 
14211 instruct absF_reg(vRegF dst, vRegF src) %{
14212   match(Set dst (AbsF src));
14213 
14214   ins_cost(INSN_COST * 3);
14215   format %{ "fabss   $dst, $src" %}
14216   ins_encode %{
14217     __ fabss(as_FloatRegister($dst$$reg),
14218              as_FloatRegister($src$$reg));
14219   %}
14220 
14221   ins_pipe(fp_uop_s);
14222 %}
14223 
14224 instruct absD_reg(vRegD dst, vRegD src) %{
14225   match(Set dst (AbsD src));
14226 
14227   ins_cost(INSN_COST * 3);
14228   format %{ "fabsd   $dst, $src" %}
14229   ins_encode %{
14230     __ fabsd(as_FloatRegister($dst$$reg),
14231              as_FloatRegister($src$$reg));
14232   %}
14233 
14234   ins_pipe(fp_uop_d);
14235 %}
14236 
14237 instruct absdF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14238   match(Set dst (AbsF (SubF src1 src2)));
14239 
14240   ins_cost(INSN_COST * 3);
14241   format %{ "fabds   $dst, $src1, $src2" %}
14242   ins_encode %{
14243     __ fabds(as_FloatRegister($dst$$reg),
14244              as_FloatRegister($src1$$reg),
14245              as_FloatRegister($src2$$reg));
14246   %}
14247 
14248   ins_pipe(fp_uop_s);
14249 %}
14250 
14251 instruct absdD_reg(vRegD dst, vRegD src1, vRegD src2) %{
14252   match(Set dst (AbsD (SubD src1 src2)));
14253 
14254   ins_cost(INSN_COST * 3);
14255   format %{ "fabdd   $dst, $src1, $src2" %}
14256   ins_encode %{
14257     __ fabdd(as_FloatRegister($dst$$reg),
14258              as_FloatRegister($src1$$reg),
14259              as_FloatRegister($src2$$reg));
14260   %}
14261 
14262   ins_pipe(fp_uop_d);
14263 %}
14264 
14265 instruct sqrtD_reg(vRegD dst, vRegD src) %{
14266   match(Set dst (SqrtD src));
14267 
14268   ins_cost(INSN_COST * 50);
14269   format %{ "fsqrtd  $dst, $src" %}
14270   ins_encode %{
14271     __ fsqrtd(as_FloatRegister($dst$$reg),
14272              as_FloatRegister($src$$reg));
14273   %}
14274 
14275   ins_pipe(fp_div_s);
14276 %}
14277 
14278 instruct sqrtF_reg(vRegF dst, vRegF src) %{
14279   match(Set dst (SqrtF src));
14280 
14281   ins_cost(INSN_COST * 50);
14282   format %{ "fsqrts  $dst, $src" %}
14283   ins_encode %{
14284     __ fsqrts(as_FloatRegister($dst$$reg),
14285              as_FloatRegister($src$$reg));
14286   %}
14287 
14288   ins_pipe(fp_div_d);
14289 %}
14290 
14291 // Math.rint, floor, ceil
14292 instruct roundD_reg(vRegD dst, vRegD src, immI rmode) %{
14293   match(Set dst (RoundDoubleMode src rmode));
14294   format %{ "frint  $dst, $src, $rmode" %}
14295   ins_encode %{
14296     switch ($rmode$$constant) {
14297       case RoundDoubleModeNode::rmode_rint:
14298         __ frintnd(as_FloatRegister($dst$$reg),
14299                    as_FloatRegister($src$$reg));
14300         break;
14301       case RoundDoubleModeNode::rmode_floor:
14302         __ frintmd(as_FloatRegister($dst$$reg),
14303                    as_FloatRegister($src$$reg));
14304         break;
14305       case RoundDoubleModeNode::rmode_ceil:
14306         __ frintpd(as_FloatRegister($dst$$reg),
14307                    as_FloatRegister($src$$reg));
14308         break;
14309     }
14310   %}
14311   ins_pipe(fp_uop_d);
14312 %}
14313 
14314 instruct copySignD_reg(vRegD dst, vRegD src1, vRegD src2, vRegD zero) %{
14315   match(Set dst (CopySignD src1 (Binary src2 zero)));
14316   effect(TEMP_DEF dst, USE src1, USE src2, USE zero);
14317   format %{ "CopySignD  $dst $src1 $src2" %}
14318   ins_encode %{
14319     FloatRegister dst = as_FloatRegister($dst$$reg),
14320                   src1 = as_FloatRegister($src1$$reg),
14321                   src2 = as_FloatRegister($src2$$reg),
14322                   zero = as_FloatRegister($zero$$reg);
14323     __ fnegd(dst, zero);
14324     __ bsl(dst, __ T8B, src2, src1);
14325   %}
14326   ins_pipe(fp_uop_d);
14327 %}
14328 
14329 instruct copySignF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14330   match(Set dst (CopySignF src1 src2));
14331   effect(TEMP_DEF dst, USE src1, USE src2);
14332   format %{ "CopySignF  $dst $src1 $src2" %}
14333   ins_encode %{
14334     FloatRegister dst = as_FloatRegister($dst$$reg),
14335                   src1 = as_FloatRegister($src1$$reg),
14336                   src2 = as_FloatRegister($src2$$reg);
14337     __ movi(dst, __ T2S, 0x80, 24);
14338     __ bsl(dst, __ T8B, src2, src1);
14339   %}
14340   ins_pipe(fp_uop_d);
14341 %}
14342 
14343 instruct signumD_reg(vRegD dst, vRegD src, vRegD zero, vRegD one) %{
14344   match(Set dst (SignumD src (Binary zero one)));
14345   effect(TEMP_DEF dst, USE src, USE zero, USE one);
14346   format %{ "signumD  $dst, $src" %}
14347   ins_encode %{
14348     FloatRegister src = as_FloatRegister($src$$reg),
14349                   dst = as_FloatRegister($dst$$reg),
14350                   zero = as_FloatRegister($zero$$reg),
14351                   one = as_FloatRegister($one$$reg);
14352     __ facgtd(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14353     __ ushrd(dst, dst, 1);     // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14354     // Bit selection instruction gets bit from "one" for each enabled bit in
14355     // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14356     // NaN the whole "src" will be copied because "dst" is zero. For all other
14357     // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14358     // from "src", and all other bits are copied from 1.0.
14359     __ bsl(dst, __ T8B, one, src);
14360   %}
14361   ins_pipe(fp_uop_d);
14362 %}
14363 
14364 instruct signumF_reg(vRegF dst, vRegF src, vRegF zero, vRegF one) %{
14365   match(Set dst (SignumF src (Binary zero one)));
14366   effect(TEMP_DEF dst, USE src, USE zero, USE one);
14367   format %{ "signumF  $dst, $src" %}
14368   ins_encode %{
14369     FloatRegister src = as_FloatRegister($src$$reg),
14370                   dst = as_FloatRegister($dst$$reg),
14371                   zero = as_FloatRegister($zero$$reg),
14372                   one = as_FloatRegister($one$$reg);
14373     __ facgts(dst, src, zero);    // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14374     __ ushr(dst, __ T2S, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14375     // Bit selection instruction gets bit from "one" for each enabled bit in
14376     // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14377     // NaN the whole "src" will be copied because "dst" is zero. For all other
14378     // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14379     // from "src", and all other bits are copied from 1.0.
14380     __ bsl(dst, __ T8B, one, src);
14381   %}
14382   ins_pipe(fp_uop_d);
14383 %}
14384 
14385 instruct onspinwait() %{
14386   match(OnSpinWait);
14387   ins_cost(INSN_COST);
14388 
14389   format %{ "onspinwait" %}
14390 
14391   ins_encode %{
14392     __ spin_wait();
14393   %}
14394   ins_pipe(pipe_class_empty);
14395 %}
14396 
14397 // ============================================================================
14398 // Logical Instructions
14399 
14400 // Integer Logical Instructions
14401 
14402 // And Instructions
14403 
14404 
14405 instruct andI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, rFlagsReg cr) %{
14406   match(Set dst (AndI src1 src2));
14407 
14408   format %{ "andw  $dst, $src1, $src2\t# int" %}
14409 
14410   ins_cost(INSN_COST);
14411   ins_encode %{
14412     __ andw(as_Register($dst$$reg),
14413             as_Register($src1$$reg),
14414             as_Register($src2$$reg));
14415   %}
14416 
14417   ins_pipe(ialu_reg_reg);
14418 %}
14419 
14420 instruct andI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2, rFlagsReg cr) %{
14421   match(Set dst (AndI src1 src2));
14422 
14423   format %{ "andsw  $dst, $src1, $src2\t# int" %}
14424 
14425   ins_cost(INSN_COST);
14426   ins_encode %{
14427     __ andw(as_Register($dst$$reg),
14428             as_Register($src1$$reg),
14429             (uint64_t)($src2$$constant));
14430   %}
14431 
14432   ins_pipe(ialu_reg_imm);
14433 %}
14434 
14435 // Or Instructions
14436 
14437 instruct orI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14438   match(Set dst (OrI src1 src2));
14439 
14440   format %{ "orrw  $dst, $src1, $src2\t# int" %}
14441 
14442   ins_cost(INSN_COST);
14443   ins_encode %{
14444     __ orrw(as_Register($dst$$reg),
14445             as_Register($src1$$reg),
14446             as_Register($src2$$reg));
14447   %}
14448 
14449   ins_pipe(ialu_reg_reg);
14450 %}
14451 
14452 instruct orI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14453   match(Set dst (OrI src1 src2));
14454 
14455   format %{ "orrw  $dst, $src1, $src2\t# int" %}
14456 
14457   ins_cost(INSN_COST);
14458   ins_encode %{
14459     __ orrw(as_Register($dst$$reg),
14460             as_Register($src1$$reg),
14461             (uint64_t)($src2$$constant));
14462   %}
14463 
14464   ins_pipe(ialu_reg_imm);
14465 %}
14466 
14467 // Xor Instructions
14468 
14469 instruct xorI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14470   match(Set dst (XorI src1 src2));
14471 
14472   format %{ "eorw  $dst, $src1, $src2\t# int" %}
14473 
14474   ins_cost(INSN_COST);
14475   ins_encode %{
14476     __ eorw(as_Register($dst$$reg),
14477             as_Register($src1$$reg),
14478             as_Register($src2$$reg));
14479   %}
14480 
14481   ins_pipe(ialu_reg_reg);
14482 %}
14483 
14484 instruct xorI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14485   match(Set dst (XorI src1 src2));
14486 
14487   format %{ "eorw  $dst, $src1, $src2\t# int" %}
14488 
14489   ins_cost(INSN_COST);
14490   ins_encode %{
14491     __ eorw(as_Register($dst$$reg),
14492             as_Register($src1$$reg),
14493             (uint64_t)($src2$$constant));
14494   %}
14495 
14496   ins_pipe(ialu_reg_imm);
14497 %}
14498 
14499 // Long Logical Instructions
14500 // TODO
14501 
14502 instruct andL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr) %{
14503   match(Set dst (AndL src1 src2));
14504 
14505   format %{ "and  $dst, $src1, $src2\t# int" %}
14506 
14507   ins_cost(INSN_COST);
14508   ins_encode %{
14509     __ andr(as_Register($dst$$reg),
14510             as_Register($src1$$reg),
14511             as_Register($src2$$reg));
14512   %}
14513 
14514   ins_pipe(ialu_reg_reg);
14515 %}
14516 
14517 instruct andL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2, rFlagsReg cr) %{
14518   match(Set dst (AndL src1 src2));
14519 
14520   format %{ "and  $dst, $src1, $src2\t# int" %}
14521 
14522   ins_cost(INSN_COST);
14523   ins_encode %{
14524     __ andr(as_Register($dst$$reg),
14525             as_Register($src1$$reg),
14526             (uint64_t)($src2$$constant));
14527   %}
14528 
14529   ins_pipe(ialu_reg_imm);
14530 %}
14531 
14532 // Or Instructions
14533 
14534 instruct orL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14535   match(Set dst (OrL src1 src2));
14536 
14537   format %{ "orr  $dst, $src1, $src2\t# int" %}
14538 
14539   ins_cost(INSN_COST);
14540   ins_encode %{
14541     __ orr(as_Register($dst$$reg),
14542            as_Register($src1$$reg),
14543            as_Register($src2$$reg));
14544   %}
14545 
14546   ins_pipe(ialu_reg_reg);
14547 %}
14548 
14549 instruct orL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14550   match(Set dst (OrL src1 src2));
14551 
14552   format %{ "orr  $dst, $src1, $src2\t# int" %}
14553 
14554   ins_cost(INSN_COST);
14555   ins_encode %{
14556     __ orr(as_Register($dst$$reg),
14557            as_Register($src1$$reg),
14558            (uint64_t)($src2$$constant));
14559   %}
14560 
14561   ins_pipe(ialu_reg_imm);
14562 %}
14563 
14564 // Xor Instructions
14565 
14566 instruct xorL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14567   match(Set dst (XorL src1 src2));
14568 
14569   format %{ "eor  $dst, $src1, $src2\t# int" %}
14570 
14571   ins_cost(INSN_COST);
14572   ins_encode %{
14573     __ eor(as_Register($dst$$reg),
14574            as_Register($src1$$reg),
14575            as_Register($src2$$reg));
14576   %}
14577 
14578   ins_pipe(ialu_reg_reg);
14579 %}
14580 
14581 instruct xorL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14582   match(Set dst (XorL src1 src2));
14583 
14584   ins_cost(INSN_COST);
14585   format %{ "eor  $dst, $src1, $src2\t# int" %}
14586 
14587   ins_encode %{
14588     __ eor(as_Register($dst$$reg),
14589            as_Register($src1$$reg),
14590            (uint64_t)($src2$$constant));
14591   %}
14592 
14593   ins_pipe(ialu_reg_imm);
14594 %}
14595 
14596 instruct convI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src)
14597 %{
14598   match(Set dst (ConvI2L src));
14599 
14600   ins_cost(INSN_COST);
14601   format %{ "sxtw  $dst, $src\t# i2l" %}
14602   ins_encode %{
14603     __ sbfm($dst$$Register, $src$$Register, 0, 31);
14604   %}
14605   ins_pipe(ialu_reg_shift);
14606 %}
14607 
14608 // this pattern occurs in bigmath arithmetic
14609 instruct convUI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src, immL_32bits mask)
14610 %{
14611   match(Set dst (AndL (ConvI2L src) mask));
14612 
14613   ins_cost(INSN_COST);
14614   format %{ "ubfm  $dst, $src, 0, 31\t# ui2l" %}
14615   ins_encode %{
14616     __ ubfm($dst$$Register, $src$$Register, 0, 31);
14617   %}
14618 
14619   ins_pipe(ialu_reg_shift);
14620 %}
14621 
14622 instruct convL2I_reg(iRegINoSp dst, iRegL src) %{
14623   match(Set dst (ConvL2I src));
14624 
14625   ins_cost(INSN_COST);
14626   format %{ "movw  $dst, $src \t// l2i" %}
14627 
14628   ins_encode %{
14629     __ movw(as_Register($dst$$reg), as_Register($src$$reg));
14630   %}
14631 
14632   ins_pipe(ialu_reg);
14633 %}
14634 
14635 instruct convI2B(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
14636 %{
14637   match(Set dst (Conv2B src));
14638   effect(KILL cr);
14639 
14640   format %{
14641     "cmpw $src, zr\n\t"
14642     "cset $dst, ne"
14643   %}
14644 
14645   ins_encode %{
14646     __ cmpw(as_Register($src$$reg), zr);
14647     __ cset(as_Register($dst$$reg), Assembler::NE);
14648   %}
14649 
14650   ins_pipe(ialu_reg);
14651 %}
14652 
14653 instruct convP2B(iRegINoSp dst, iRegP src, rFlagsReg cr)
14654 %{
14655   match(Set dst (Conv2B src));
14656   effect(KILL cr);
14657 
14658   format %{
14659     "cmp  $src, zr\n\t"
14660     "cset $dst, ne"
14661   %}
14662 
14663   ins_encode %{
14664     __ cmp(as_Register($src$$reg), zr);
14665     __ cset(as_Register($dst$$reg), Assembler::NE);
14666   %}
14667 
14668   ins_pipe(ialu_reg);
14669 %}
14670 
14671 instruct convD2F_reg(vRegF dst, vRegD src) %{
14672   match(Set dst (ConvD2F src));
14673 
14674   ins_cost(INSN_COST * 5);
14675   format %{ "fcvtd  $dst, $src \t// d2f" %}
14676 
14677   ins_encode %{
14678     __ fcvtd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14679   %}
14680 
14681   ins_pipe(fp_d2f);
14682 %}
14683 
14684 instruct convF2D_reg(vRegD dst, vRegF src) %{
14685   match(Set dst (ConvF2D src));
14686 
14687   ins_cost(INSN_COST * 5);
14688   format %{ "fcvts  $dst, $src \t// f2d" %}
14689 
14690   ins_encode %{
14691     __ fcvts(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14692   %}
14693 
14694   ins_pipe(fp_f2d);
14695 %}
14696 
14697 instruct convF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14698   match(Set dst (ConvF2I src));
14699 
14700   ins_cost(INSN_COST * 5);
14701   format %{ "fcvtzsw  $dst, $src \t// f2i" %}
14702 
14703   ins_encode %{
14704     __ fcvtzsw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14705   %}
14706 
14707   ins_pipe(fp_f2i);
14708 %}
14709 
14710 instruct convF2L_reg_reg(iRegLNoSp dst, vRegF src) %{
14711   match(Set dst (ConvF2L src));
14712 
14713   ins_cost(INSN_COST * 5);
14714   format %{ "fcvtzs  $dst, $src \t// f2l" %}
14715 
14716   ins_encode %{
14717     __ fcvtzs(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14718   %}
14719 
14720   ins_pipe(fp_f2l);
14721 %}
14722 
14723 instruct convF2HF_reg_reg(iRegINoSp dst, vRegF src, vRegF tmp) %{
14724   match(Set dst (ConvF2HF src));
14725   format %{ "fcvt $tmp, $src\t# convert single to half precision\n\t"
14726             "smov $dst, $tmp\t# move result from $tmp to $dst"
14727   %}
14728   effect(TEMP tmp);
14729   ins_encode %{
14730       __ fcvtsh($tmp$$FloatRegister, $src$$FloatRegister);
14731       __ smov($dst$$Register, $tmp$$FloatRegister, __ H, 0);
14732   %}
14733   ins_pipe(pipe_slow);
14734 %}
14735 
14736 instruct convHF2F_reg_reg(vRegF dst, iRegINoSp src, vRegF tmp) %{
14737   match(Set dst (ConvHF2F src));
14738   format %{ "mov $tmp, $src\t# move source from $src to $tmp\n\t"
14739             "fcvt $dst, $tmp\t# convert half to single precision"
14740   %}
14741   effect(TEMP tmp);
14742   ins_encode %{
14743       __ mov($tmp$$FloatRegister, __ H, 0, $src$$Register);
14744       __ fcvths($dst$$FloatRegister, $tmp$$FloatRegister);
14745   %}
14746   ins_pipe(pipe_slow);
14747 %}
14748 
14749 instruct convI2F_reg_reg(vRegF dst, iRegIorL2I src) %{
14750   match(Set dst (ConvI2F src));
14751 
14752   ins_cost(INSN_COST * 5);
14753   format %{ "scvtfws  $dst, $src \t// i2f" %}
14754 
14755   ins_encode %{
14756     __ scvtfws(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14757   %}
14758 
14759   ins_pipe(fp_i2f);
14760 %}
14761 
14762 instruct convL2F_reg_reg(vRegF dst, iRegL src) %{
14763   match(Set dst (ConvL2F src));
14764 
14765   ins_cost(INSN_COST * 5);
14766   format %{ "scvtfs  $dst, $src \t// l2f" %}
14767 
14768   ins_encode %{
14769     __ scvtfs(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14770   %}
14771 
14772   ins_pipe(fp_l2f);
14773 %}
14774 
14775 instruct convD2I_reg_reg(iRegINoSp dst, vRegD src) %{
14776   match(Set dst (ConvD2I src));
14777 
14778   ins_cost(INSN_COST * 5);
14779   format %{ "fcvtzdw  $dst, $src \t// d2i" %}
14780 
14781   ins_encode %{
14782     __ fcvtzdw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14783   %}
14784 
14785   ins_pipe(fp_d2i);
14786 %}
14787 
14788 instruct convD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14789   match(Set dst (ConvD2L src));
14790 
14791   ins_cost(INSN_COST * 5);
14792   format %{ "fcvtzd  $dst, $src \t// d2l" %}
14793 
14794   ins_encode %{
14795     __ fcvtzd(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14796   %}
14797 
14798   ins_pipe(fp_d2l);
14799 %}
14800 
14801 instruct convI2D_reg_reg(vRegD dst, iRegIorL2I src) %{
14802   match(Set dst (ConvI2D src));
14803 
14804   ins_cost(INSN_COST * 5);
14805   format %{ "scvtfwd  $dst, $src \t// i2d" %}
14806 
14807   ins_encode %{
14808     __ scvtfwd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14809   %}
14810 
14811   ins_pipe(fp_i2d);
14812 %}
14813 
14814 instruct convL2D_reg_reg(vRegD dst, iRegL src) %{
14815   match(Set dst (ConvL2D src));
14816 
14817   ins_cost(INSN_COST * 5);
14818   format %{ "scvtfd  $dst, $src \t// l2d" %}
14819 
14820   ins_encode %{
14821     __ scvtfd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14822   %}
14823 
14824   ins_pipe(fp_l2d);
14825 %}
14826 
14827 instruct round_double_reg(iRegLNoSp dst, vRegD src, vRegD ftmp, rFlagsReg cr)
14828 %{
14829   match(Set dst (RoundD src));
14830   effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
14831   format %{ "java_round_double $dst,$src"%}
14832   ins_encode %{
14833     __ java_round_double($dst$$Register, as_FloatRegister($src$$reg),
14834                          as_FloatRegister($ftmp$$reg));
14835   %}
14836   ins_pipe(pipe_slow);
14837 %}
14838 
14839 instruct round_float_reg(iRegINoSp dst, vRegF src, vRegF ftmp, rFlagsReg cr)
14840 %{
14841   match(Set dst (RoundF src));
14842   effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
14843   format %{ "java_round_float $dst,$src"%}
14844   ins_encode %{
14845     __ java_round_float($dst$$Register, as_FloatRegister($src$$reg),
14846                         as_FloatRegister($ftmp$$reg));
14847   %}
14848   ins_pipe(pipe_slow);
14849 %}
14850 
14851 // stack <-> reg and reg <-> reg shuffles with no conversion
14852 
14853 instruct MoveF2I_stack_reg(iRegINoSp dst, stackSlotF src) %{
14854 
14855   match(Set dst (MoveF2I src));
14856 
14857   effect(DEF dst, USE src);
14858 
14859   ins_cost(4 * INSN_COST);
14860 
14861   format %{ "ldrw $dst, $src\t# MoveF2I_stack_reg" %}
14862 
14863   ins_encode %{
14864     __ ldrw($dst$$Register, Address(sp, $src$$disp));
14865   %}
14866 
14867   ins_pipe(iload_reg_reg);
14868 
14869 %}
14870 
14871 instruct MoveI2F_stack_reg(vRegF dst, stackSlotI src) %{
14872 
14873   match(Set dst (MoveI2F src));
14874 
14875   effect(DEF dst, USE src);
14876 
14877   ins_cost(4 * INSN_COST);
14878 
14879   format %{ "ldrs $dst, $src\t# MoveI2F_stack_reg" %}
14880 
14881   ins_encode %{
14882     __ ldrs(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
14883   %}
14884 
14885   ins_pipe(pipe_class_memory);
14886 
14887 %}
14888 
14889 instruct MoveD2L_stack_reg(iRegLNoSp dst, stackSlotD src) %{
14890 
14891   match(Set dst (MoveD2L src));
14892 
14893   effect(DEF dst, USE src);
14894 
14895   ins_cost(4 * INSN_COST);
14896 
14897   format %{ "ldr $dst, $src\t# MoveD2L_stack_reg" %}
14898 
14899   ins_encode %{
14900     __ ldr($dst$$Register, Address(sp, $src$$disp));
14901   %}
14902 
14903   ins_pipe(iload_reg_reg);
14904 
14905 %}
14906 
14907 instruct MoveL2D_stack_reg(vRegD dst, stackSlotL src) %{
14908 
14909   match(Set dst (MoveL2D src));
14910 
14911   effect(DEF dst, USE src);
14912 
14913   ins_cost(4 * INSN_COST);
14914 
14915   format %{ "ldrd $dst, $src\t# MoveL2D_stack_reg" %}
14916 
14917   ins_encode %{
14918     __ ldrd(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
14919   %}
14920 
14921   ins_pipe(pipe_class_memory);
14922 
14923 %}
14924 
14925 instruct MoveF2I_reg_stack(stackSlotI dst, vRegF src) %{
14926 
14927   match(Set dst (MoveF2I src));
14928 
14929   effect(DEF dst, USE src);
14930 
14931   ins_cost(INSN_COST);
14932 
14933   format %{ "strs $src, $dst\t# MoveF2I_reg_stack" %}
14934 
14935   ins_encode %{
14936     __ strs(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
14937   %}
14938 
14939   ins_pipe(pipe_class_memory);
14940 
14941 %}
14942 
14943 instruct MoveI2F_reg_stack(stackSlotF dst, iRegI src) %{
14944 
14945   match(Set dst (MoveI2F src));
14946 
14947   effect(DEF dst, USE src);
14948 
14949   ins_cost(INSN_COST);
14950 
14951   format %{ "strw $src, $dst\t# MoveI2F_reg_stack" %}
14952 
14953   ins_encode %{
14954     __ strw($src$$Register, Address(sp, $dst$$disp));
14955   %}
14956 
14957   ins_pipe(istore_reg_reg);
14958 
14959 %}
14960 
14961 instruct MoveD2L_reg_stack(stackSlotL dst, vRegD src) %{
14962 
14963   match(Set dst (MoveD2L src));
14964 
14965   effect(DEF dst, USE src);
14966 
14967   ins_cost(INSN_COST);
14968 
14969   format %{ "strd $dst, $src\t# MoveD2L_reg_stack" %}
14970 
14971   ins_encode %{
14972     __ strd(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
14973   %}
14974 
14975   ins_pipe(pipe_class_memory);
14976 
14977 %}
14978 
14979 instruct MoveL2D_reg_stack(stackSlotD dst, iRegL src) %{
14980 
14981   match(Set dst (MoveL2D src));
14982 
14983   effect(DEF dst, USE src);
14984 
14985   ins_cost(INSN_COST);
14986 
14987   format %{ "str $src, $dst\t# MoveL2D_reg_stack" %}
14988 
14989   ins_encode %{
14990     __ str($src$$Register, Address(sp, $dst$$disp));
14991   %}
14992 
14993   ins_pipe(istore_reg_reg);
14994 
14995 %}
14996 
14997 instruct MoveF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14998 
14999   match(Set dst (MoveF2I src));
15000 
15001   effect(DEF dst, USE src);
15002 
15003   ins_cost(INSN_COST);
15004 
15005   format %{ "fmovs $dst, $src\t# MoveF2I_reg_reg" %}
15006 
15007   ins_encode %{
15008     __ fmovs($dst$$Register, as_FloatRegister($src$$reg));
15009   %}
15010 
15011   ins_pipe(fp_f2i);
15012 
15013 %}
15014 
15015 instruct MoveI2F_reg_reg(vRegF dst, iRegI src) %{
15016 
15017   match(Set dst (MoveI2F src));
15018 
15019   effect(DEF dst, USE src);
15020 
15021   ins_cost(INSN_COST);
15022 
15023   format %{ "fmovs $dst, $src\t# MoveI2F_reg_reg" %}
15024 
15025   ins_encode %{
15026     __ fmovs(as_FloatRegister($dst$$reg), $src$$Register);
15027   %}
15028 
15029   ins_pipe(fp_i2f);
15030 
15031 %}
15032 
15033 instruct MoveD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
15034 
15035   match(Set dst (MoveD2L src));
15036 
15037   effect(DEF dst, USE src);
15038 
15039   ins_cost(INSN_COST);
15040 
15041   format %{ "fmovd $dst, $src\t# MoveD2L_reg_reg" %}
15042 
15043   ins_encode %{
15044     __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
15045   %}
15046 
15047   ins_pipe(fp_d2l);
15048 
15049 %}
15050 
15051 instruct MoveL2D_reg_reg(vRegD dst, iRegL src) %{
15052 
15053   match(Set dst (MoveL2D src));
15054 
15055   effect(DEF dst, USE src);
15056 
15057   ins_cost(INSN_COST);
15058 
15059   format %{ "fmovd $dst, $src\t# MoveL2D_reg_reg" %}
15060 
15061   ins_encode %{
15062     __ fmovd(as_FloatRegister($dst$$reg), $src$$Register);
15063   %}
15064 
15065   ins_pipe(fp_l2d);
15066 
15067 %}
15068 
15069 // ============================================================================
15070 // clearing of an array
15071 
15072 instruct clearArray_reg_reg_immL0(iRegL_R11 cnt, iRegP_R10 base, immL0 zero, Universe dummy, rFlagsReg cr)
15073 %{
15074   match(Set dummy (ClearArray (Binary cnt base) zero));
15075   effect(USE_KILL cnt, USE_KILL base, KILL cr);
15076 
15077   ins_cost(4 * INSN_COST);
15078   format %{ "ClearArray $cnt, $base" %}
15079 
15080   ins_encode %{
15081     address tpc = __ zero_words($base$$Register, $cnt$$Register);
15082     if (tpc == NULL) {
15083       ciEnv::current()->record_failure("CodeCache is full");
15084       return;
15085     }
15086   %}
15087 
15088   ins_pipe(pipe_class_memory);
15089 %}
15090 
15091 instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, iRegL val, Universe dummy, rFlagsReg cr)
15092 %{
15093   predicate(((ClearArrayNode*)n)->word_copy_only());
15094   match(Set dummy (ClearArray (Binary cnt base) val));
15095   effect(USE_KILL cnt, USE_KILL base, KILL cr);
15096 
15097   ins_cost(4 * INSN_COST);
15098   format %{ "ClearArray $cnt, $base, $val" %}
15099 
15100   ins_encode %{
15101     __ fill_words($base$$Register, $cnt$$Register, $val$$Register);
15102   %}
15103 
15104   ins_pipe(pipe_class_memory);
15105 %}
15106 
15107 instruct clearArray_imm_reg(immL cnt, iRegP_R10 base, iRegL_R11 temp, Universe dummy, rFlagsReg cr)
15108 %{
15109   predicate((uint64_t)n->in(2)->get_long()
15110             < (uint64_t)(BlockZeroingLowLimit >> LogBytesPerWord)
15111             && !((ClearArrayNode*)n)->word_copy_only());
15112   match(Set dummy (ClearArray cnt base));
15113   effect(TEMP temp, USE_KILL base, KILL cr);
15114 
15115   ins_cost(4 * INSN_COST);
15116   format %{ "ClearArray $cnt, $base" %}
15117 
15118   ins_encode %{
15119     address tpc = __ zero_words($base$$Register, (uint64_t)$cnt$$constant);
15120     if (tpc == NULL) {
15121       ciEnv::current()->record_failure("CodeCache is full");
15122       return;
15123     }
15124   %}
15125 
15126   ins_pipe(pipe_class_memory);
15127 %}
15128 
15129 // ============================================================================
15130 // Overflow Math Instructions
15131 
15132 instruct overflowAddI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15133 %{
15134   match(Set cr (OverflowAddI op1 op2));
15135 
15136   format %{ "cmnw  $op1, $op2\t# overflow check int" %}
15137   ins_cost(INSN_COST);
15138   ins_encode %{
15139     __ cmnw($op1$$Register, $op2$$Register);
15140   %}
15141 
15142   ins_pipe(icmp_reg_reg);
15143 %}
15144 
15145 instruct overflowAddI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
15146 %{
15147   match(Set cr (OverflowAddI op1 op2));
15148 
15149   format %{ "cmnw  $op1, $op2\t# overflow check int" %}
15150   ins_cost(INSN_COST);
15151   ins_encode %{
15152     __ cmnw($op1$$Register, $op2$$constant);
15153   %}
15154 
15155   ins_pipe(icmp_reg_imm);
15156 %}
15157 
15158 instruct overflowAddL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15159 %{
15160   match(Set cr (OverflowAddL op1 op2));
15161 
15162   format %{ "cmn   $op1, $op2\t# overflow check long" %}
15163   ins_cost(INSN_COST);
15164   ins_encode %{
15165     __ cmn($op1$$Register, $op2$$Register);
15166   %}
15167 
15168   ins_pipe(icmp_reg_reg);
15169 %}
15170 
15171 instruct overflowAddL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
15172 %{
15173   match(Set cr (OverflowAddL op1 op2));
15174 
15175   format %{ "adds  zr, $op1, $op2\t# overflow check long" %}
15176   ins_cost(INSN_COST);
15177   ins_encode %{
15178     __ adds(zr, $op1$$Register, $op2$$constant);
15179   %}
15180 
15181   ins_pipe(icmp_reg_imm);
15182 %}
15183 
15184 instruct overflowSubI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15185 %{
15186   match(Set cr (OverflowSubI op1 op2));
15187 
15188   format %{ "cmpw  $op1, $op2\t# overflow check int" %}
15189   ins_cost(INSN_COST);
15190   ins_encode %{
15191     __ cmpw($op1$$Register, $op2$$Register);
15192   %}
15193 
15194   ins_pipe(icmp_reg_reg);
15195 %}
15196 
15197 instruct overflowSubI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
15198 %{
15199   match(Set cr (OverflowSubI op1 op2));
15200 
15201   format %{ "cmpw  $op1, $op2\t# overflow check int" %}
15202   ins_cost(INSN_COST);
15203   ins_encode %{
15204     __ cmpw($op1$$Register, $op2$$constant);
15205   %}
15206 
15207   ins_pipe(icmp_reg_imm);
15208 %}
15209 
15210 instruct overflowSubL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15211 %{
15212   match(Set cr (OverflowSubL op1 op2));
15213 
15214   format %{ "cmp   $op1, $op2\t# overflow check long" %}
15215   ins_cost(INSN_COST);
15216   ins_encode %{
15217     __ cmp($op1$$Register, $op2$$Register);
15218   %}
15219 
15220   ins_pipe(icmp_reg_reg);
15221 %}
15222 
15223 instruct overflowSubL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
15224 %{
15225   match(Set cr (OverflowSubL op1 op2));
15226 
15227   format %{ "cmp   $op1, $op2\t# overflow check long" %}
15228   ins_cost(INSN_COST);
15229   ins_encode %{
15230     __ subs(zr, $op1$$Register, $op2$$constant);
15231   %}
15232 
15233   ins_pipe(icmp_reg_imm);
15234 %}
15235 
15236 instruct overflowNegI_reg(rFlagsReg cr, immI0 zero, iRegIorL2I op1)
15237 %{
15238   match(Set cr (OverflowSubI zero op1));
15239 
15240   format %{ "cmpw  zr, $op1\t# overflow check int" %}
15241   ins_cost(INSN_COST);
15242   ins_encode %{
15243     __ cmpw(zr, $op1$$Register);
15244   %}
15245 
15246   ins_pipe(icmp_reg_imm);
15247 %}
15248 
15249 instruct overflowNegL_reg(rFlagsReg cr, immI0 zero, iRegL op1)
15250 %{
15251   match(Set cr (OverflowSubL zero op1));
15252 
15253   format %{ "cmp   zr, $op1\t# overflow check long" %}
15254   ins_cost(INSN_COST);
15255   ins_encode %{
15256     __ cmp(zr, $op1$$Register);
15257   %}
15258 
15259   ins_pipe(icmp_reg_imm);
15260 %}
15261 
15262 instruct overflowMulI_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15263 %{
15264   match(Set cr (OverflowMulI op1 op2));
15265 
15266   format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15267             "cmp   rscratch1, rscratch1, sxtw\n\t"
15268             "movw  rscratch1, #0x80000000\n\t"
15269             "cselw rscratch1, rscratch1, zr, NE\n\t"
15270             "cmpw  rscratch1, #1" %}
15271   ins_cost(5 * INSN_COST);
15272   ins_encode %{
15273     __ smull(rscratch1, $op1$$Register, $op2$$Register);
15274     __ subs(zr, rscratch1, rscratch1, ext::sxtw);      // NE => overflow
15275     __ movw(rscratch1, 0x80000000);                    // Develop 0 (EQ),
15276     __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15277     __ cmpw(rscratch1, 1);                             // 0x80000000 - 1 => VS
15278   %}
15279 
15280   ins_pipe(pipe_slow);
15281 %}
15282 
15283 instruct overflowMulI_reg_branch(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, label labl, rFlagsReg cr)
15284 %{
15285   match(If cmp (OverflowMulI op1 op2));
15286   predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15287             || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15288   effect(USE labl, KILL cr);
15289 
15290   format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15291             "cmp   rscratch1, rscratch1, sxtw\n\t"
15292             "b$cmp   $labl" %}
15293   ins_cost(3 * INSN_COST); // Branch is rare so treat as INSN_COST
15294   ins_encode %{
15295     Label* L = $labl$$label;
15296     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15297     __ smull(rscratch1, $op1$$Register, $op2$$Register);
15298     __ subs(zr, rscratch1, rscratch1, ext::sxtw);      // NE => overflow
15299     __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15300   %}
15301 
15302   ins_pipe(pipe_serial);
15303 %}
15304 
15305 instruct overflowMulL_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15306 %{
15307   match(Set cr (OverflowMulL op1 op2));
15308 
15309   format %{ "mul   rscratch1, $op1, $op2\t#overflow check long\n\t"
15310             "smulh rscratch2, $op1, $op2\n\t"
15311             "cmp   rscratch2, rscratch1, ASR #63\n\t"
15312             "movw  rscratch1, #0x80000000\n\t"
15313             "cselw rscratch1, rscratch1, zr, NE\n\t"
15314             "cmpw  rscratch1, #1" %}
15315   ins_cost(6 * INSN_COST);
15316   ins_encode %{
15317     __ mul(rscratch1, $op1$$Register, $op2$$Register);   // Result bits 0..63
15318     __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15319     __ cmp(rscratch2, rscratch1, Assembler::ASR, 63);    // Top is pure sign ext
15320     __ movw(rscratch1, 0x80000000);                    // Develop 0 (EQ),
15321     __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15322     __ cmpw(rscratch1, 1);                             // 0x80000000 - 1 => VS
15323   %}
15324 
15325   ins_pipe(pipe_slow);
15326 %}
15327 
15328 instruct overflowMulL_reg_branch(cmpOp cmp, iRegL op1, iRegL op2, label labl, rFlagsReg cr)
15329 %{
15330   match(If cmp (OverflowMulL op1 op2));
15331   predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15332             || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15333   effect(USE labl, KILL cr);
15334 
15335   format %{ "mul   rscratch1, $op1, $op2\t#overflow check long\n\t"
15336             "smulh rscratch2, $op1, $op2\n\t"
15337             "cmp   rscratch2, rscratch1, ASR #63\n\t"
15338             "b$cmp $labl" %}
15339   ins_cost(4 * INSN_COST); // Branch is rare so treat as INSN_COST
15340   ins_encode %{
15341     Label* L = $labl$$label;
15342     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15343     __ mul(rscratch1, $op1$$Register, $op2$$Register);   // Result bits 0..63
15344     __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15345     __ cmp(rscratch2, rscratch1, Assembler::ASR, 63);    // Top is pure sign ext
15346     __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15347   %}
15348 
15349   ins_pipe(pipe_serial);
15350 %}
15351 
15352 // ============================================================================
15353 // Compare Instructions
15354 
15355 instruct compI_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
15356 %{
15357   match(Set cr (CmpI op1 op2));
15358 
15359   effect(DEF cr, USE op1, USE op2);
15360 
15361   ins_cost(INSN_COST);
15362   format %{ "cmpw  $op1, $op2" %}
15363 
15364   ins_encode(aarch64_enc_cmpw(op1, op2));
15365 
15366   ins_pipe(icmp_reg_reg);
15367 %}
15368 
15369 instruct compI_reg_immI0(rFlagsReg cr, iRegI op1, immI0 zero)
15370 %{
15371   match(Set cr (CmpI op1 zero));
15372 
15373   effect(DEF cr, USE op1);
15374 
15375   ins_cost(INSN_COST);
15376   format %{ "cmpw $op1, 0" %}
15377 
15378   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15379 
15380   ins_pipe(icmp_reg_imm);
15381 %}
15382 
15383 instruct compI_reg_immIAddSub(rFlagsReg cr, iRegI op1, immIAddSub op2)
15384 %{
15385   match(Set cr (CmpI op1 op2));
15386 
15387   effect(DEF cr, USE op1);
15388 
15389   ins_cost(INSN_COST);
15390   format %{ "cmpw  $op1, $op2" %}
15391 
15392   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15393 
15394   ins_pipe(icmp_reg_imm);
15395 %}
15396 
15397 instruct compI_reg_immI(rFlagsReg cr, iRegI op1, immI op2)
15398 %{
15399   match(Set cr (CmpI op1 op2));
15400 
15401   effect(DEF cr, USE op1);
15402 
15403   ins_cost(INSN_COST * 2);
15404   format %{ "cmpw  $op1, $op2" %}
15405 
15406   ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15407 
15408   ins_pipe(icmp_reg_imm);
15409 %}
15410 
15411 // Unsigned compare Instructions; really, same as signed compare
15412 // except it should only be used to feed an If or a CMovI which takes a
15413 // cmpOpU.
15414 
15415 instruct compU_reg_reg(rFlagsRegU cr, iRegI op1, iRegI op2)
15416 %{
15417   match(Set cr (CmpU op1 op2));
15418 
15419   effect(DEF cr, USE op1, USE op2);
15420 
15421   ins_cost(INSN_COST);
15422   format %{ "cmpw  $op1, $op2\t# unsigned" %}
15423 
15424   ins_encode(aarch64_enc_cmpw(op1, op2));
15425 
15426   ins_pipe(icmp_reg_reg);
15427 %}
15428 
15429 instruct compU_reg_immI0(rFlagsRegU cr, iRegI op1, immI0 zero)
15430 %{
15431   match(Set cr (CmpU op1 zero));
15432 
15433   effect(DEF cr, USE op1);
15434 
15435   ins_cost(INSN_COST);
15436   format %{ "cmpw $op1, #0\t# unsigned" %}
15437 
15438   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15439 
15440   ins_pipe(icmp_reg_imm);
15441 %}
15442 
15443 instruct compU_reg_immIAddSub(rFlagsRegU cr, iRegI op1, immIAddSub op2)
15444 %{
15445   match(Set cr (CmpU op1 op2));
15446 
15447   effect(DEF cr, USE op1);
15448 
15449   ins_cost(INSN_COST);
15450   format %{ "cmpw  $op1, $op2\t# unsigned" %}
15451 
15452   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15453 
15454   ins_pipe(icmp_reg_imm);
15455 %}
15456 
15457 instruct compU_reg_immI(rFlagsRegU cr, iRegI op1, immI op2)
15458 %{
15459   match(Set cr (CmpU op1 op2));
15460 
15461   effect(DEF cr, USE op1);
15462 
15463   ins_cost(INSN_COST * 2);
15464   format %{ "cmpw  $op1, $op2\t# unsigned" %}
15465 
15466   ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15467 
15468   ins_pipe(icmp_reg_imm);
15469 %}
15470 
15471 instruct compL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15472 %{
15473   match(Set cr (CmpL op1 op2));
15474 
15475   effect(DEF cr, USE op1, USE op2);
15476 
15477   ins_cost(INSN_COST);
15478   format %{ "cmp  $op1, $op2" %}
15479 
15480   ins_encode(aarch64_enc_cmp(op1, op2));
15481 
15482   ins_pipe(icmp_reg_reg);
15483 %}
15484 
15485 instruct compL_reg_immL0(rFlagsReg cr, iRegL op1, immL0 zero)
15486 %{
15487   match(Set cr (CmpL op1 zero));
15488 
15489   effect(DEF cr, USE op1);
15490 
15491   ins_cost(INSN_COST);
15492   format %{ "tst  $op1" %}
15493 
15494   ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15495 
15496   ins_pipe(icmp_reg_imm);
15497 %}
15498 
15499 instruct compL_reg_immLAddSub(rFlagsReg cr, iRegL op1, immLAddSub op2)
15500 %{
15501   match(Set cr (CmpL op1 op2));
15502 
15503   effect(DEF cr, USE op1);
15504 
15505   ins_cost(INSN_COST);
15506   format %{ "cmp  $op1, $op2" %}
15507 
15508   ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15509 
15510   ins_pipe(icmp_reg_imm);
15511 %}
15512 
15513 instruct compL_reg_immL(rFlagsReg cr, iRegL op1, immL op2)
15514 %{
15515   match(Set cr (CmpL op1 op2));
15516 
15517   effect(DEF cr, USE op1);
15518 
15519   ins_cost(INSN_COST * 2);
15520   format %{ "cmp  $op1, $op2" %}
15521 
15522   ins_encode(aarch64_enc_cmp_imm(op1, op2));
15523 
15524   ins_pipe(icmp_reg_imm);
15525 %}
15526 
15527 instruct compUL_reg_reg(rFlagsRegU cr, iRegL op1, iRegL op2)
15528 %{
15529   match(Set cr (CmpUL op1 op2));
15530 
15531   effect(DEF cr, USE op1, USE op2);
15532 
15533   ins_cost(INSN_COST);
15534   format %{ "cmp  $op1, $op2" %}
15535 
15536   ins_encode(aarch64_enc_cmp(op1, op2));
15537 
15538   ins_pipe(icmp_reg_reg);
15539 %}
15540 
15541 instruct compUL_reg_immL0(rFlagsRegU cr, iRegL op1, immL0 zero)
15542 %{
15543   match(Set cr (CmpUL op1 zero));
15544 
15545   effect(DEF cr, USE op1);
15546 
15547   ins_cost(INSN_COST);
15548   format %{ "tst  $op1" %}
15549 
15550   ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15551 
15552   ins_pipe(icmp_reg_imm);
15553 %}
15554 
15555 instruct compUL_reg_immLAddSub(rFlagsRegU cr, iRegL op1, immLAddSub op2)
15556 %{
15557   match(Set cr (CmpUL op1 op2));
15558 
15559   effect(DEF cr, USE op1);
15560 
15561   ins_cost(INSN_COST);
15562   format %{ "cmp  $op1, $op2" %}
15563 
15564   ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15565 
15566   ins_pipe(icmp_reg_imm);
15567 %}
15568 
15569 instruct compUL_reg_immL(rFlagsRegU cr, iRegL op1, immL op2)
15570 %{
15571   match(Set cr (CmpUL op1 op2));
15572 
15573   effect(DEF cr, USE op1);
15574 
15575   ins_cost(INSN_COST * 2);
15576   format %{ "cmp  $op1, $op2" %}
15577 
15578   ins_encode(aarch64_enc_cmp_imm(op1, op2));
15579 
15580   ins_pipe(icmp_reg_imm);
15581 %}
15582 
15583 instruct compP_reg_reg(rFlagsRegU cr, iRegP op1, iRegP op2)
15584 %{
15585   match(Set cr (CmpP op1 op2));
15586 
15587   effect(DEF cr, USE op1, USE op2);
15588 
15589   ins_cost(INSN_COST);
15590   format %{ "cmp  $op1, $op2\t // ptr" %}
15591 
15592   ins_encode(aarch64_enc_cmpp(op1, op2));
15593 
15594   ins_pipe(icmp_reg_reg);
15595 %}
15596 
15597 instruct compN_reg_reg(rFlagsRegU cr, iRegN op1, iRegN op2)
15598 %{
15599   match(Set cr (CmpN op1 op2));
15600 
15601   effect(DEF cr, USE op1, USE op2);
15602 
15603   ins_cost(INSN_COST);
15604   format %{ "cmp  $op1, $op2\t // compressed ptr" %}
15605 
15606   ins_encode(aarch64_enc_cmpn(op1, op2));
15607 
15608   ins_pipe(icmp_reg_reg);
15609 %}
15610 
15611 instruct testP_reg(rFlagsRegU cr, iRegP op1, immP0 zero)
15612 %{
15613   match(Set cr (CmpP op1 zero));
15614 
15615   effect(DEF cr, USE op1, USE zero);
15616 
15617   ins_cost(INSN_COST);
15618   format %{ "cmp  $op1, 0\t // ptr" %}
15619 
15620   ins_encode(aarch64_enc_testp(op1));
15621 
15622   ins_pipe(icmp_reg_imm);
15623 %}
15624 
15625 instruct testN_reg(rFlagsRegU cr, iRegN op1, immN0 zero)
15626 %{
15627   match(Set cr (CmpN op1 zero));
15628 
15629   effect(DEF cr, USE op1, USE zero);
15630 
15631   ins_cost(INSN_COST);
15632   format %{ "cmp  $op1, 0\t // compressed ptr" %}
15633 
15634   ins_encode(aarch64_enc_testn(op1));
15635 
15636   ins_pipe(icmp_reg_imm);
15637 %}
15638 
15639 // FP comparisons
15640 //
15641 // n.b. CmpF/CmpD set a normal flags reg which then gets compared
15642 // using normal cmpOp. See declaration of rFlagsReg for details.
15643 
15644 instruct compF_reg_reg(rFlagsReg cr, vRegF src1, vRegF src2)
15645 %{
15646   match(Set cr (CmpF src1 src2));
15647 
15648   ins_cost(3 * INSN_COST);
15649   format %{ "fcmps $src1, $src2" %}
15650 
15651   ins_encode %{
15652     __ fcmps(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15653   %}
15654 
15655   ins_pipe(pipe_class_compare);
15656 %}
15657 
15658 instruct compF_reg_zero(rFlagsReg cr, vRegF src1, immF0 src2)
15659 %{
15660   match(Set cr (CmpF src1 src2));
15661 
15662   ins_cost(3 * INSN_COST);
15663   format %{ "fcmps $src1, 0.0" %}
15664 
15665   ins_encode %{
15666     __ fcmps(as_FloatRegister($src1$$reg), 0.0);
15667   %}
15668 
15669   ins_pipe(pipe_class_compare);
15670 %}
15671 // FROM HERE
15672 
15673 instruct compD_reg_reg(rFlagsReg cr, vRegD src1, vRegD src2)
15674 %{
15675   match(Set cr (CmpD src1 src2));
15676 
15677   ins_cost(3 * INSN_COST);
15678   format %{ "fcmpd $src1, $src2" %}
15679 
15680   ins_encode %{
15681     __ fcmpd(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15682   %}
15683 
15684   ins_pipe(pipe_class_compare);
15685 %}
15686 
15687 instruct compD_reg_zero(rFlagsReg cr, vRegD src1, immD0 src2)
15688 %{
15689   match(Set cr (CmpD src1 src2));
15690 
15691   ins_cost(3 * INSN_COST);
15692   format %{ "fcmpd $src1, 0.0" %}
15693 
15694   ins_encode %{
15695     __ fcmpd(as_FloatRegister($src1$$reg), 0.0);
15696   %}
15697 
15698   ins_pipe(pipe_class_compare);
15699 %}
15700 
15701 instruct compF3_reg_reg(iRegINoSp dst, vRegF src1, vRegF src2, rFlagsReg cr)
15702 %{
15703   match(Set dst (CmpF3 src1 src2));
15704   effect(KILL cr);
15705 
15706   ins_cost(5 * INSN_COST);
15707   format %{ "fcmps $src1, $src2\n\t"
15708             "csinvw($dst, zr, zr, eq\n\t"
15709             "csnegw($dst, $dst, $dst, lt)"
15710   %}
15711 
15712   ins_encode %{
15713     Label done;
15714     FloatRegister s1 = as_FloatRegister($src1$$reg);
15715     FloatRegister s2 = as_FloatRegister($src2$$reg);
15716     Register d = as_Register($dst$$reg);
15717     __ fcmps(s1, s2);
15718     // installs 0 if EQ else -1
15719     __ csinvw(d, zr, zr, Assembler::EQ);
15720     // keeps -1 if less or unordered else installs 1
15721     __ csnegw(d, d, d, Assembler::LT);
15722     __ bind(done);
15723   %}
15724 
15725   ins_pipe(pipe_class_default);
15726 
15727 %}
15728 
15729 instruct compD3_reg_reg(iRegINoSp dst, vRegD src1, vRegD src2, rFlagsReg cr)
15730 %{
15731   match(Set dst (CmpD3 src1 src2));
15732   effect(KILL cr);
15733 
15734   ins_cost(5 * INSN_COST);
15735   format %{ "fcmpd $src1, $src2\n\t"
15736             "csinvw($dst, zr, zr, eq\n\t"
15737             "csnegw($dst, $dst, $dst, lt)"
15738   %}
15739 
15740   ins_encode %{
15741     Label done;
15742     FloatRegister s1 = as_FloatRegister($src1$$reg);
15743     FloatRegister s2 = as_FloatRegister($src2$$reg);
15744     Register d = as_Register($dst$$reg);
15745     __ fcmpd(s1, s2);
15746     // installs 0 if EQ else -1
15747     __ csinvw(d, zr, zr, Assembler::EQ);
15748     // keeps -1 if less or unordered else installs 1
15749     __ csnegw(d, d, d, Assembler::LT);
15750     __ bind(done);
15751   %}
15752   ins_pipe(pipe_class_default);
15753 
15754 %}
15755 
15756 instruct compF3_reg_immF0(iRegINoSp dst, vRegF src1, immF0 zero, rFlagsReg cr)
15757 %{
15758   match(Set dst (CmpF3 src1 zero));
15759   effect(KILL cr);
15760 
15761   ins_cost(5 * INSN_COST);
15762   format %{ "fcmps $src1, 0.0\n\t"
15763             "csinvw($dst, zr, zr, eq\n\t"
15764             "csnegw($dst, $dst, $dst, lt)"
15765   %}
15766 
15767   ins_encode %{
15768     Label done;
15769     FloatRegister s1 = as_FloatRegister($src1$$reg);
15770     Register d = as_Register($dst$$reg);
15771     __ fcmps(s1, 0.0);
15772     // installs 0 if EQ else -1
15773     __ csinvw(d, zr, zr, Assembler::EQ);
15774     // keeps -1 if less or unordered else installs 1
15775     __ csnegw(d, d, d, Assembler::LT);
15776     __ bind(done);
15777   %}
15778 
15779   ins_pipe(pipe_class_default);
15780 
15781 %}
15782 
15783 instruct compD3_reg_immD0(iRegINoSp dst, vRegD src1, immD0 zero, rFlagsReg cr)
15784 %{
15785   match(Set dst (CmpD3 src1 zero));
15786   effect(KILL cr);
15787 
15788   ins_cost(5 * INSN_COST);
15789   format %{ "fcmpd $src1, 0.0\n\t"
15790             "csinvw($dst, zr, zr, eq\n\t"
15791             "csnegw($dst, $dst, $dst, lt)"
15792   %}
15793 
15794   ins_encode %{
15795     Label done;
15796     FloatRegister s1 = as_FloatRegister($src1$$reg);
15797     Register d = as_Register($dst$$reg);
15798     __ fcmpd(s1, 0.0);
15799     // installs 0 if EQ else -1
15800     __ csinvw(d, zr, zr, Assembler::EQ);
15801     // keeps -1 if less or unordered else installs 1
15802     __ csnegw(d, d, d, Assembler::LT);
15803     __ bind(done);
15804   %}
15805   ins_pipe(pipe_class_default);
15806 
15807 %}
15808 
15809 instruct cmpLTMask_reg_reg(iRegINoSp dst, iRegIorL2I p, iRegIorL2I q, rFlagsReg cr)
15810 %{
15811   match(Set dst (CmpLTMask p q));
15812   effect(KILL cr);
15813 
15814   ins_cost(3 * INSN_COST);
15815 
15816   format %{ "cmpw $p, $q\t# cmpLTMask\n\t"
15817             "csetw $dst, lt\n\t"
15818             "subw $dst, zr, $dst"
15819   %}
15820 
15821   ins_encode %{
15822     __ cmpw(as_Register($p$$reg), as_Register($q$$reg));
15823     __ csetw(as_Register($dst$$reg), Assembler::LT);
15824     __ subw(as_Register($dst$$reg), zr, as_Register($dst$$reg));
15825   %}
15826 
15827   ins_pipe(ialu_reg_reg);
15828 %}
15829 
15830 instruct cmpLTMask_reg_zero(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr)
15831 %{
15832   match(Set dst (CmpLTMask src zero));
15833   effect(KILL cr);
15834 
15835   ins_cost(INSN_COST);
15836 
15837   format %{ "asrw $dst, $src, #31\t# cmpLTMask0" %}
15838 
15839   ins_encode %{
15840     __ asrw(as_Register($dst$$reg), as_Register($src$$reg), 31);
15841   %}
15842 
15843   ins_pipe(ialu_reg_shift);
15844 %}
15845 
15846 // ============================================================================
15847 // Max and Min
15848 
15849 instruct cmovI_reg_reg_lt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
15850 %{
15851   effect( DEF dst, USE src1, USE src2, USE cr );
15852 
15853   ins_cost(INSN_COST * 2);
15854   format %{ "cselw $dst, $src1, $src2 lt\t"  %}
15855 
15856   ins_encode %{
15857     __ cselw(as_Register($dst$$reg),
15858              as_Register($src1$$reg),
15859              as_Register($src2$$reg),
15860              Assembler::LT);
15861   %}
15862 
15863   ins_pipe(icond_reg_reg);
15864 %}
15865 
15866 instruct minI_rReg(iRegINoSp dst, iRegI src1, iRegI src2)
15867 %{
15868   match(Set dst (MinI src1 src2));
15869   ins_cost(INSN_COST * 3);
15870 
15871   expand %{
15872     rFlagsReg cr;
15873     compI_reg_reg(cr, src1, src2);
15874     cmovI_reg_reg_lt(dst, src1, src2, cr);
15875   %}
15876 
15877 %}
15878 // FROM HERE
15879 
15880 instruct cmovI_reg_reg_gt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
15881 %{
15882   effect( DEF dst, USE src1, USE src2, USE cr );
15883 
15884   ins_cost(INSN_COST * 2);
15885   format %{ "cselw $dst, $src1, $src2 gt\t"  %}
15886 
15887   ins_encode %{
15888     __ cselw(as_Register($dst$$reg),
15889              as_Register($src1$$reg),
15890              as_Register($src2$$reg),
15891              Assembler::GT);
15892   %}
15893 
15894   ins_pipe(icond_reg_reg);
15895 %}
15896 
15897 instruct maxI_rReg(iRegINoSp dst, iRegI src1, iRegI src2)
15898 %{
15899   match(Set dst (MaxI src1 src2));
15900   ins_cost(INSN_COST * 3);
15901   expand %{
15902     rFlagsReg cr;
15903     compI_reg_reg(cr, src1, src2);
15904     cmovI_reg_reg_gt(dst, src1, src2, cr);
15905   %}
15906 %}
15907 
15908 // ============================================================================
15909 // Branch Instructions
15910 
15911 // Direct Branch.
15912 instruct branch(label lbl)
15913 %{
15914   match(Goto);
15915 
15916   effect(USE lbl);
15917 
15918   ins_cost(BRANCH_COST);
15919   format %{ "b  $lbl" %}
15920 
15921   ins_encode(aarch64_enc_b(lbl));
15922 
15923   ins_pipe(pipe_branch);
15924 %}
15925 
15926 // Conditional Near Branch
15927 instruct branchCon(cmpOp cmp, rFlagsReg cr, label lbl)
15928 %{
15929   // Same match rule as `branchConFar'.
15930   match(If cmp cr);
15931 
15932   effect(USE lbl);
15933 
15934   ins_cost(BRANCH_COST);
15935   // If set to 1 this indicates that the current instruction is a
15936   // short variant of a long branch. This avoids using this
15937   // instruction in first-pass matching. It will then only be used in
15938   // the `Shorten_branches' pass.
15939   // ins_short_branch(1);
15940   format %{ "b$cmp  $lbl" %}
15941 
15942   ins_encode(aarch64_enc_br_con(cmp, lbl));
15943 
15944   ins_pipe(pipe_branch_cond);
15945 %}
15946 
15947 // Conditional Near Branch Unsigned
15948 instruct branchConU(cmpOpU cmp, rFlagsRegU cr, label lbl)
15949 %{
15950   // Same match rule as `branchConFar'.
15951   match(If cmp cr);
15952 
15953   effect(USE lbl);
15954 
15955   ins_cost(BRANCH_COST);
15956   // If set to 1 this indicates that the current instruction is a
15957   // short variant of a long branch. This avoids using this
15958   // instruction in first-pass matching. It will then only be used in
15959   // the `Shorten_branches' pass.
15960   // ins_short_branch(1);
15961   format %{ "b$cmp  $lbl\t# unsigned" %}
15962 
15963   ins_encode(aarch64_enc_br_conU(cmp, lbl));
15964 
15965   ins_pipe(pipe_branch_cond);
15966 %}
15967 
15968 // Make use of CBZ and CBNZ.  These instructions, as well as being
15969 // shorter than (cmp; branch), have the additional benefit of not
15970 // killing the flags.
15971 
15972 instruct cmpI_imm0_branch(cmpOpEqNe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsReg cr) %{
15973   match(If cmp (CmpI op1 op2));
15974   effect(USE labl);
15975 
15976   ins_cost(BRANCH_COST);
15977   format %{ "cbw$cmp   $op1, $labl" %}
15978   ins_encode %{
15979     Label* L = $labl$$label;
15980     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15981     if (cond == Assembler::EQ)
15982       __ cbzw($op1$$Register, *L);
15983     else
15984       __ cbnzw($op1$$Register, *L);
15985   %}
15986   ins_pipe(pipe_cmp_branch);
15987 %}
15988 
15989 instruct cmpL_imm0_branch(cmpOpEqNe cmp, iRegL op1, immL0 op2, label labl, rFlagsReg cr) %{
15990   match(If cmp (CmpL op1 op2));
15991   effect(USE labl);
15992 
15993   ins_cost(BRANCH_COST);
15994   format %{ "cb$cmp   $op1, $labl" %}
15995   ins_encode %{
15996     Label* L = $labl$$label;
15997     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15998     if (cond == Assembler::EQ)
15999       __ cbz($op1$$Register, *L);
16000     else
16001       __ cbnz($op1$$Register, *L);
16002   %}
16003   ins_pipe(pipe_cmp_branch);
16004 %}
16005 
16006 instruct cmpP_imm0_branch(cmpOpEqNe cmp, iRegP op1, immP0 op2, label labl, rFlagsReg cr) %{
16007   match(If cmp (CmpP op1 op2));
16008   effect(USE labl);
16009 
16010   ins_cost(BRANCH_COST);
16011   format %{ "cb$cmp   $op1, $labl" %}
16012   ins_encode %{
16013     Label* L = $labl$$label;
16014     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16015     if (cond == Assembler::EQ)
16016       __ cbz($op1$$Register, *L);
16017     else
16018       __ cbnz($op1$$Register, *L);
16019   %}
16020   ins_pipe(pipe_cmp_branch);
16021 %}
16022 
16023 instruct cmpN_imm0_branch(cmpOpEqNe cmp, iRegN op1, immN0 op2, label labl, rFlagsReg cr) %{
16024   match(If cmp (CmpN op1 op2));
16025   effect(USE labl);
16026 
16027   ins_cost(BRANCH_COST);
16028   format %{ "cbw$cmp   $op1, $labl" %}
16029   ins_encode %{
16030     Label* L = $labl$$label;
16031     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16032     if (cond == Assembler::EQ)
16033       __ cbzw($op1$$Register, *L);
16034     else
16035       __ cbnzw($op1$$Register, *L);
16036   %}
16037   ins_pipe(pipe_cmp_branch);
16038 %}
16039 
16040 instruct cmpP_narrowOop_imm0_branch(cmpOpEqNe cmp, iRegN oop, immP0 zero, label labl, rFlagsReg cr) %{
16041   match(If cmp (CmpP (DecodeN oop) zero));
16042   effect(USE labl);
16043 
16044   ins_cost(BRANCH_COST);
16045   format %{ "cb$cmp   $oop, $labl" %}
16046   ins_encode %{
16047     Label* L = $labl$$label;
16048     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16049     if (cond == Assembler::EQ)
16050       __ cbzw($oop$$Register, *L);
16051     else
16052       __ cbnzw($oop$$Register, *L);
16053   %}
16054   ins_pipe(pipe_cmp_branch);
16055 %}
16056 
16057 instruct cmpUI_imm0_branch(cmpOpUEqNeLtGe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsRegU cr) %{
16058   match(If cmp (CmpU op1 op2));
16059   effect(USE labl);
16060 
16061   ins_cost(BRANCH_COST);
16062   format %{ "cbw$cmp   $op1, $labl" %}
16063   ins_encode %{
16064     Label* L = $labl$$label;
16065     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16066     if (cond == Assembler::EQ || cond == Assembler::LS)
16067       __ cbzw($op1$$Register, *L);
16068     else
16069       __ cbnzw($op1$$Register, *L);
16070   %}
16071   ins_pipe(pipe_cmp_branch);
16072 %}
16073 
16074 instruct cmpUL_imm0_branch(cmpOpUEqNeLtGe cmp, iRegL op1, immL0 op2, label labl, rFlagsRegU cr) %{
16075   match(If cmp (CmpUL op1 op2));
16076   effect(USE labl);
16077 
16078   ins_cost(BRANCH_COST);
16079   format %{ "cb$cmp   $op1, $labl" %}
16080   ins_encode %{
16081     Label* L = $labl$$label;
16082     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16083     if (cond == Assembler::EQ || cond == Assembler::LS)
16084       __ cbz($op1$$Register, *L);
16085     else
16086       __ cbnz($op1$$Register, *L);
16087   %}
16088   ins_pipe(pipe_cmp_branch);
16089 %}
16090 
16091 // Test bit and Branch
16092 
16093 // Patterns for short (< 32KiB) variants
16094 instruct cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
16095   match(If cmp (CmpL op1 op2));
16096   effect(USE labl);
16097 
16098   ins_cost(BRANCH_COST);
16099   format %{ "cb$cmp   $op1, $labl # long" %}
16100   ins_encode %{
16101     Label* L = $labl$$label;
16102     Assembler::Condition cond =
16103       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16104     __ tbr(cond, $op1$$Register, 63, *L);
16105   %}
16106   ins_pipe(pipe_cmp_branch);
16107   ins_short_branch(1);
16108 %}
16109 
16110 instruct cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16111   match(If cmp (CmpI op1 op2));
16112   effect(USE labl);
16113 
16114   ins_cost(BRANCH_COST);
16115   format %{ "cb$cmp   $op1, $labl # int" %}
16116   ins_encode %{
16117     Label* L = $labl$$label;
16118     Assembler::Condition cond =
16119       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16120     __ tbr(cond, $op1$$Register, 31, *L);
16121   %}
16122   ins_pipe(pipe_cmp_branch);
16123   ins_short_branch(1);
16124 %}
16125 
16126 instruct cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
16127   match(If cmp (CmpL (AndL op1 op2) op3));
16128   predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
16129   effect(USE labl);
16130 
16131   ins_cost(BRANCH_COST);
16132   format %{ "tb$cmp   $op1, $op2, $labl" %}
16133   ins_encode %{
16134     Label* L = $labl$$label;
16135     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16136     int bit = exact_log2_long($op2$$constant);
16137     __ tbr(cond, $op1$$Register, bit, *L);
16138   %}
16139   ins_pipe(pipe_cmp_branch);
16140   ins_short_branch(1);
16141 %}
16142 
16143 instruct cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
16144   match(If cmp (CmpI (AndI op1 op2) op3));
16145   predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
16146   effect(USE labl);
16147 
16148   ins_cost(BRANCH_COST);
16149   format %{ "tb$cmp   $op1, $op2, $labl" %}
16150   ins_encode %{
16151     Label* L = $labl$$label;
16152     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16153     int bit = exact_log2((juint)$op2$$constant);
16154     __ tbr(cond, $op1$$Register, bit, *L);
16155   %}
16156   ins_pipe(pipe_cmp_branch);
16157   ins_short_branch(1);
16158 %}
16159 
16160 // And far variants
16161 instruct far_cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
16162   match(If cmp (CmpL op1 op2));
16163   effect(USE labl);
16164 
16165   ins_cost(BRANCH_COST);
16166   format %{ "cb$cmp   $op1, $labl # long" %}
16167   ins_encode %{
16168     Label* L = $labl$$label;
16169     Assembler::Condition cond =
16170       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16171     __ tbr(cond, $op1$$Register, 63, *L, /*far*/true);
16172   %}
16173   ins_pipe(pipe_cmp_branch);
16174 %}
16175 
16176 instruct far_cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16177   match(If cmp (CmpI op1 op2));
16178   effect(USE labl);
16179 
16180   ins_cost(BRANCH_COST);
16181   format %{ "cb$cmp   $op1, $labl # int" %}
16182   ins_encode %{
16183     Label* L = $labl$$label;
16184     Assembler::Condition cond =
16185       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16186     __ tbr(cond, $op1$$Register, 31, *L, /*far*/true);
16187   %}
16188   ins_pipe(pipe_cmp_branch);
16189 %}
16190 
16191 instruct far_cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
16192   match(If cmp (CmpL (AndL op1 op2) op3));
16193   predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
16194   effect(USE labl);
16195 
16196   ins_cost(BRANCH_COST);
16197   format %{ "tb$cmp   $op1, $op2, $labl" %}
16198   ins_encode %{
16199     Label* L = $labl$$label;
16200     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16201     int bit = exact_log2_long($op2$$constant);
16202     __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
16203   %}
16204   ins_pipe(pipe_cmp_branch);
16205 %}
16206 
16207 instruct far_cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
16208   match(If cmp (CmpI (AndI op1 op2) op3));
16209   predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
16210   effect(USE labl);
16211 
16212   ins_cost(BRANCH_COST);
16213   format %{ "tb$cmp   $op1, $op2, $labl" %}
16214   ins_encode %{
16215     Label* L = $labl$$label;
16216     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16217     int bit = exact_log2((juint)$op2$$constant);
16218     __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
16219   %}
16220   ins_pipe(pipe_cmp_branch);
16221 %}
16222 
16223 // Test bits
16224 
16225 instruct cmpL_and(cmpOp cmp, iRegL op1, immL op2, immL0 op3, rFlagsReg cr) %{
16226   match(Set cr (CmpL (AndL op1 op2) op3));
16227   predicate(Assembler::operand_valid_for_logical_immediate
16228             (/*is_32*/false, n->in(1)->in(2)->get_long()));
16229 
16230   ins_cost(INSN_COST);
16231   format %{ "tst $op1, $op2 # long" %}
16232   ins_encode %{
16233     __ tst($op1$$Register, $op2$$constant);
16234   %}
16235   ins_pipe(ialu_reg_reg);
16236 %}
16237 
16238 instruct cmpI_and(cmpOp cmp, iRegIorL2I op1, immI op2, immI0 op3, rFlagsReg cr) %{
16239   match(Set cr (CmpI (AndI op1 op2) op3));
16240   predicate(Assembler::operand_valid_for_logical_immediate
16241             (/*is_32*/true, n->in(1)->in(2)->get_int()));
16242 
16243   ins_cost(INSN_COST);
16244   format %{ "tst $op1, $op2 # int" %}
16245   ins_encode %{
16246     __ tstw($op1$$Register, $op2$$constant);
16247   %}
16248   ins_pipe(ialu_reg_reg);
16249 %}
16250 
16251 instruct cmpL_and_reg(cmpOp cmp, iRegL op1, iRegL op2, immL0 op3, rFlagsReg cr) %{
16252   match(Set cr (CmpL (AndL op1 op2) op3));
16253 
16254   ins_cost(INSN_COST);
16255   format %{ "tst $op1, $op2 # long" %}
16256   ins_encode %{
16257     __ tst($op1$$Register, $op2$$Register);
16258   %}
16259   ins_pipe(ialu_reg_reg);
16260 %}
16261 
16262 instruct cmpI_and_reg(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, immI0 op3, rFlagsReg cr) %{
16263   match(Set cr (CmpI (AndI op1 op2) op3));
16264 
16265   ins_cost(INSN_COST);
16266   format %{ "tstw $op1, $op2 # int" %}
16267   ins_encode %{
16268     __ tstw($op1$$Register, $op2$$Register);
16269   %}
16270   ins_pipe(ialu_reg_reg);
16271 %}
16272 
16273 
16274 // Conditional Far Branch
16275 // Conditional Far Branch Unsigned
16276 // TODO: fixme
16277 
16278 // counted loop end branch near
16279 instruct branchLoopEnd(cmpOp cmp, rFlagsReg cr, label lbl)
16280 %{
16281   match(CountedLoopEnd cmp cr);
16282 
16283   effect(USE lbl);
16284 
16285   ins_cost(BRANCH_COST);
16286   // short variant.
16287   // ins_short_branch(1);
16288   format %{ "b$cmp $lbl \t// counted loop end" %}
16289 
16290   ins_encode(aarch64_enc_br_con(cmp, lbl));
16291 
16292   ins_pipe(pipe_branch);
16293 %}
16294 
16295 // counted loop end branch near Unsigned
16296 instruct branchLoopEndU(cmpOpU cmp, rFlagsRegU cr, label lbl)
16297 %{
16298   match(CountedLoopEnd cmp cr);
16299 
16300   effect(USE lbl);
16301 
16302   ins_cost(BRANCH_COST);
16303   // short variant.
16304   // ins_short_branch(1);
16305   format %{ "b$cmp $lbl \t// counted loop end unsigned" %}
16306 
16307   ins_encode(aarch64_enc_br_conU(cmp, lbl));
16308 
16309   ins_pipe(pipe_branch);
16310 %}
16311 
16312 // counted loop end branch far
16313 // counted loop end branch far unsigned
16314 // TODO: fixme
16315 
16316 // ============================================================================
16317 // inlined locking and unlocking
16318 
16319 instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
16320 %{
16321   match(Set cr (FastLock object box));
16322   effect(TEMP tmp, TEMP tmp2);
16323 
16324   // TODO
16325   // identify correct cost
16326   ins_cost(5 * INSN_COST);
16327   format %{ "fastlock $object,$box\t! kills $tmp,$tmp2" %}
16328 
16329   ins_encode(aarch64_enc_fast_lock(object, box, tmp, tmp2));
16330 
16331   ins_pipe(pipe_serial);
16332 %}
16333 
16334 instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
16335 %{
16336   match(Set cr (FastUnlock object box));
16337   effect(TEMP tmp, TEMP tmp2);
16338 
16339   ins_cost(5 * INSN_COST);
16340   format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2" %}
16341 
16342   ins_encode(aarch64_enc_fast_unlock(object, box, tmp, tmp2));
16343 
16344   ins_pipe(pipe_serial);
16345 %}
16346 
16347 
16348 // ============================================================================
16349 // Safepoint Instructions
16350 
16351 // TODO
16352 // provide a near and far version of this code
16353 
16354 instruct safePoint(rFlagsReg cr, iRegP poll)
16355 %{
16356   match(SafePoint poll);
16357   effect(KILL cr);
16358 
16359   format %{
16360     "ldrw zr, [$poll]\t# Safepoint: poll for GC"
16361   %}
16362   ins_encode %{
16363     __ read_polling_page(as_Register($poll$$reg), relocInfo::poll_type);
16364   %}
16365   ins_pipe(pipe_serial); // ins_pipe(iload_reg_mem);
16366 %}
16367 
16368 
16369 // ============================================================================
16370 // Procedure Call/Return Instructions
16371 
16372 // Call Java Static Instruction
16373 
16374 instruct CallStaticJavaDirect(method meth)
16375 %{
16376   match(CallStaticJava);
16377 
16378   effect(USE meth);
16379 
16380   ins_cost(CALL_COST);
16381 
16382   format %{ "call,static $meth \t// ==> " %}
16383 
16384   ins_encode(aarch64_enc_java_static_call(meth),
16385              aarch64_enc_call_epilog);
16386 
16387   ins_pipe(pipe_class_call);
16388 %}
16389 
16390 // TO HERE
16391 
16392 // Call Java Dynamic Instruction
16393 instruct CallDynamicJavaDirect(method meth)
16394 %{
16395   match(CallDynamicJava);
16396 
16397   effect(USE meth);
16398 
16399   ins_cost(CALL_COST);
16400 
16401   format %{ "CALL,dynamic $meth \t// ==> " %}
16402 
16403   ins_encode(aarch64_enc_java_dynamic_call(meth),
16404              aarch64_enc_call_epilog);
16405 
16406   ins_pipe(pipe_class_call);
16407 %}
16408 
16409 // Call Runtime Instruction
16410 
16411 instruct CallRuntimeDirect(method meth)
16412 %{
16413   match(CallRuntime);
16414 
16415   effect(USE meth);
16416 
16417   ins_cost(CALL_COST);
16418 
16419   format %{ "CALL, runtime $meth" %}
16420 
16421   ins_encode( aarch64_enc_java_to_runtime(meth) );
16422 
16423   ins_pipe(pipe_class_call);
16424 %}
16425 
16426 // Call Runtime Instruction
16427 
16428 instruct CallLeafDirect(method meth)
16429 %{
16430   match(CallLeaf);
16431 
16432   effect(USE meth);
16433 
16434   ins_cost(CALL_COST);
16435 
16436   format %{ "CALL, runtime leaf $meth" %}
16437 
16438   ins_encode( aarch64_enc_java_to_runtime(meth) );
16439 
16440   ins_pipe(pipe_class_call);
16441 %}
16442 
16443 // Call Runtime Instruction
16444 
16445 // entry point is null, target holds the address to call
16446 instruct CallLeafNoFPIndirect(iRegP target)
16447 %{
16448   predicate(n->as_Call()->entry_point() == NULL);
16449 
16450   match(CallLeafNoFP target);
16451 
16452   ins_cost(CALL_COST);
16453 
16454   format %{ "CALL, runtime leaf nofp indirect $target" %}
16455 
16456   ins_encode %{
16457     __ blr($target$$Register);
16458   %}
16459 
16460   ins_pipe(pipe_class_call);
16461 %}
16462 
16463 instruct CallLeafNoFPDirect(method meth)
16464 %{
16465   predicate(n->as_Call()->entry_point() != NULL);
16466 
16467   match(CallLeafNoFP);
16468 
16469   effect(USE meth);
16470 
16471   ins_cost(CALL_COST);
16472 
16473   format %{ "CALL, runtime leaf nofp $meth" %}
16474 
16475   ins_encode( aarch64_enc_java_to_runtime(meth) );
16476 
16477   ins_pipe(pipe_class_call);
16478 %}
16479 
16480 // Tail Call; Jump from runtime stub to Java code.
16481 // Also known as an 'interprocedural jump'.
16482 // Target of jump will eventually return to caller.
16483 // TailJump below removes the return address.
16484 instruct TailCalljmpInd(iRegPNoSp jump_target, inline_cache_RegP method_ptr)
16485 %{
16486   match(TailCall jump_target method_ptr);
16487 
16488   ins_cost(CALL_COST);
16489 
16490   format %{ "br $jump_target\t# $method_ptr holds method" %}
16491 
16492   ins_encode(aarch64_enc_tail_call(jump_target));
16493 
16494   ins_pipe(pipe_class_call);
16495 %}
16496 
16497 instruct TailjmpInd(iRegPNoSp jump_target, iRegP_R0 ex_oop)
16498 %{
16499   match(TailJump jump_target ex_oop);
16500 
16501   ins_cost(CALL_COST);
16502 
16503   format %{ "br $jump_target\t# $ex_oop holds exception oop" %}
16504 
16505   ins_encode(aarch64_enc_tail_jmp(jump_target));
16506 
16507   ins_pipe(pipe_class_call);
16508 %}
16509 
16510 // Create exception oop: created by stack-crawling runtime code.
16511 // Created exception is now available to this handler, and is setup
16512 // just prior to jumping to this handler. No code emitted.
16513 // TODO check
16514 // should ex_oop be in r0? intel uses rax, ppc cannot use r0 so uses rarg1
16515 instruct CreateException(iRegP_R0 ex_oop)
16516 %{
16517   match(Set ex_oop (CreateEx));
16518 
16519   format %{ " -- \t// exception oop; no code emitted" %}
16520 
16521   size(0);
16522 
16523   ins_encode( /*empty*/ );
16524 
16525   ins_pipe(pipe_class_empty);
16526 %}
16527 
16528 // Rethrow exception: The exception oop will come in the first
16529 // argument position. Then JUMP (not call) to the rethrow stub code.
16530 instruct RethrowException() %{
16531   match(Rethrow);
16532   ins_cost(CALL_COST);
16533 
16534   format %{ "b rethrow_stub" %}
16535 
16536   ins_encode( aarch64_enc_rethrow() );
16537 
16538   ins_pipe(pipe_class_call);
16539 %}
16540 
16541 
16542 // Return Instruction
16543 // epilog node loads ret address into lr as part of frame pop
16544 instruct Ret()
16545 %{
16546   match(Return);
16547 
16548   format %{ "ret\t// return register" %}
16549 
16550   ins_encode( aarch64_enc_ret() );
16551 
16552   ins_pipe(pipe_branch);
16553 %}
16554 
16555 // Die now.
16556 instruct ShouldNotReachHere() %{
16557   match(Halt);
16558 
16559   ins_cost(CALL_COST);
16560   format %{ "ShouldNotReachHere" %}
16561 
16562   ins_encode %{
16563     if (is_reachable()) {
16564       __ stop(_halt_reason);
16565     }
16566   %}
16567 
16568   ins_pipe(pipe_class_default);
16569 %}
16570 
16571 // ============================================================================
16572 // Partial Subtype Check
16573 //
16574 // superklass array for an instance of the superklass.  Set a hidden
16575 // internal cache on a hit (cache is checked with exposed code in
16576 // gen_subtype_check()).  Return NZ for a miss or zero for a hit.  The
16577 // encoding ALSO sets flags.
16578 
16579 instruct partialSubtypeCheck(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, rFlagsReg cr)
16580 %{
16581   match(Set result (PartialSubtypeCheck sub super));
16582   effect(KILL cr, KILL temp);
16583 
16584   ins_cost(1100);  // slightly larger than the next version
16585   format %{ "partialSubtypeCheck $result, $sub, $super" %}
16586 
16587   ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
16588 
16589   opcode(0x1); // Force zero of result reg on hit
16590 
16591   ins_pipe(pipe_class_memory);
16592 %}
16593 
16594 instruct partialSubtypeCheckVsZero(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, immP0 zero, rFlagsReg cr)
16595 %{
16596   match(Set cr (CmpP (PartialSubtypeCheck sub super) zero));
16597   effect(KILL temp, KILL result);
16598 
16599   ins_cost(1100);  // slightly larger than the next version
16600   format %{ "partialSubtypeCheck $result, $sub, $super == 0" %}
16601 
16602   ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
16603 
16604   opcode(0x0); // Don't zero result reg on hit
16605 
16606   ins_pipe(pipe_class_memory);
16607 %}
16608 
16609 // Intrisics for String.compareTo()
16610 
16611 instruct string_compareU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16612                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16613 %{
16614   predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
16615   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16616   effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16617 
16618   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1" %}
16619   ins_encode %{
16620     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16621     __ string_compare($str1$$Register, $str2$$Register,
16622                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16623                       $tmp1$$Register, $tmp2$$Register,
16624                       fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::UU);
16625   %}
16626   ins_pipe(pipe_class_memory);
16627 %}
16628 
16629 instruct string_compareL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16630                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16631 %{
16632   predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
16633   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16634   effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16635 
16636   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1" %}
16637   ins_encode %{
16638     __ string_compare($str1$$Register, $str2$$Register,
16639                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16640                       $tmp1$$Register, $tmp2$$Register,
16641                       fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::LL);
16642   %}
16643   ins_pipe(pipe_class_memory);
16644 %}
16645 
16646 instruct string_compareUL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16647                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16648                         vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16649 %{
16650   predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
16651   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16652   effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16653          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16654 
16655   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16656   ins_encode %{
16657     __ string_compare($str1$$Register, $str2$$Register,
16658                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16659                       $tmp1$$Register, $tmp2$$Register,
16660                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16661                       $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::UL);
16662   %}
16663   ins_pipe(pipe_class_memory);
16664 %}
16665 
16666 instruct string_compareLU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16667                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16668                         vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16669 %{
16670   predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
16671   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16672   effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16673          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16674 
16675   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16676   ins_encode %{
16677     __ string_compare($str1$$Register, $str2$$Register,
16678                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16679                       $tmp1$$Register, $tmp2$$Register,
16680                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16681                       $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::LU);
16682   %}
16683   ins_pipe(pipe_class_memory);
16684 %}
16685 
16686 // Note that Z registers alias the corresponding NEON registers, we declare the vector operands of
16687 // these string_compare variants as NEON register type for convenience so that the prototype of
16688 // string_compare can be shared with all variants.
16689 
16690 instruct string_compareLL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16691                               iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16692                               vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16693                               pRegGov_P1 pgtmp2, rFlagsReg cr)
16694 %{
16695   predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
16696   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16697   effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16698          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16699 
16700   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # USE sve" %}
16701   ins_encode %{
16702     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16703     __ string_compare($str1$$Register, $str2$$Register,
16704                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16705                       $tmp1$$Register, $tmp2$$Register,
16706                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16707                       as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16708                       StrIntrinsicNode::LL);
16709   %}
16710   ins_pipe(pipe_class_memory);
16711 %}
16712 
16713 instruct string_compareLU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16714                               iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16715                               vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16716                               pRegGov_P1 pgtmp2, rFlagsReg cr)
16717 %{
16718   predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
16719   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16720   effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16721          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16722 
16723   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # USE sve" %}
16724   ins_encode %{
16725     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16726     __ string_compare($str1$$Register, $str2$$Register,
16727                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16728                       $tmp1$$Register, $tmp2$$Register,
16729                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16730                       as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16731                       StrIntrinsicNode::LU);
16732   %}
16733   ins_pipe(pipe_class_memory);
16734 %}
16735 
16736 instruct string_compareUL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16737                               iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16738                               vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16739                               pRegGov_P1 pgtmp2, rFlagsReg cr)
16740 %{
16741   predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
16742   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16743   effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16744          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16745 
16746   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # USE sve" %}
16747   ins_encode %{
16748     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16749     __ string_compare($str1$$Register, $str2$$Register,
16750                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16751                       $tmp1$$Register, $tmp2$$Register,
16752                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16753                       as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16754                       StrIntrinsicNode::UL);
16755   %}
16756   ins_pipe(pipe_class_memory);
16757 %}
16758 
16759 instruct string_compareUU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16760                               iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16761                               vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16762                               pRegGov_P1 pgtmp2, rFlagsReg cr)
16763 %{
16764   predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
16765   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16766   effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16767          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16768 
16769   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # USE sve" %}
16770   ins_encode %{
16771     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16772     __ string_compare($str1$$Register, $str2$$Register,
16773                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16774                       $tmp1$$Register, $tmp2$$Register,
16775                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16776                       as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16777                       StrIntrinsicNode::UU);
16778   %}
16779   ins_pipe(pipe_class_memory);
16780 %}
16781 
16782 instruct string_indexofUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16783        iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
16784        iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6, rFlagsReg cr)
16785 %{
16786   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16787   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16788   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16789          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6, KILL cr);
16790   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UU)" %}
16791 
16792   ins_encode %{
16793     __ string_indexof($str1$$Register, $str2$$Register,
16794                       $cnt1$$Register, $cnt2$$Register,
16795                       $tmp1$$Register, $tmp2$$Register,
16796                       $tmp3$$Register, $tmp4$$Register,
16797                       $tmp5$$Register, $tmp6$$Register,
16798                       -1, $result$$Register, StrIntrinsicNode::UU);
16799   %}
16800   ins_pipe(pipe_class_memory);
16801 %}
16802 
16803 instruct string_indexofLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16804        iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
16805        iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6, rFlagsReg cr)
16806 %{
16807   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16808   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16809   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16810          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6, KILL cr);
16811   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (LL)" %}
16812 
16813   ins_encode %{
16814     __ string_indexof($str1$$Register, $str2$$Register,
16815                       $cnt1$$Register, $cnt2$$Register,
16816                       $tmp1$$Register, $tmp2$$Register,
16817                       $tmp3$$Register, $tmp4$$Register,
16818                       $tmp5$$Register, $tmp6$$Register,
16819                       -1, $result$$Register, StrIntrinsicNode::LL);
16820   %}
16821   ins_pipe(pipe_class_memory);
16822 %}
16823 
16824 instruct string_indexofUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16825        iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
16826        iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6, rFlagsReg cr)
16827 %{
16828   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16829   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16830   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16831          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6, KILL cr);
16832   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UL)" %}
16833 
16834   ins_encode %{
16835     __ string_indexof($str1$$Register, $str2$$Register,
16836                       $cnt1$$Register, $cnt2$$Register,
16837                       $tmp1$$Register, $tmp2$$Register,
16838                       $tmp3$$Register, $tmp4$$Register,
16839                       $tmp5$$Register, $tmp6$$Register,
16840                       -1, $result$$Register, StrIntrinsicNode::UL);
16841   %}
16842   ins_pipe(pipe_class_memory);
16843 %}
16844 
16845 instruct string_indexof_conUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16846                  immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16847                  iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16848 %{
16849   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16850   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16851   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16852          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16853   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UU)" %}
16854 
16855   ins_encode %{
16856     int icnt2 = (int)$int_cnt2$$constant;
16857     __ string_indexof($str1$$Register, $str2$$Register,
16858                       $cnt1$$Register, zr,
16859                       $tmp1$$Register, $tmp2$$Register,
16860                       $tmp3$$Register, $tmp4$$Register, zr, zr,
16861                       icnt2, $result$$Register, StrIntrinsicNode::UU);
16862   %}
16863   ins_pipe(pipe_class_memory);
16864 %}
16865 
16866 instruct string_indexof_conLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16867                  immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16868                  iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16869 %{
16870   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16871   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16872   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16873          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16874   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (LL)" %}
16875 
16876   ins_encode %{
16877     int icnt2 = (int)$int_cnt2$$constant;
16878     __ string_indexof($str1$$Register, $str2$$Register,
16879                       $cnt1$$Register, zr,
16880                       $tmp1$$Register, $tmp2$$Register,
16881                       $tmp3$$Register, $tmp4$$Register, zr, zr,
16882                       icnt2, $result$$Register, StrIntrinsicNode::LL);
16883   %}
16884   ins_pipe(pipe_class_memory);
16885 %}
16886 
16887 instruct string_indexof_conUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16888                  immI_1 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16889                  iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16890 %{
16891   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16892   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16893   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16894          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16895   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UL)" %}
16896 
16897   ins_encode %{
16898     int icnt2 = (int)$int_cnt2$$constant;
16899     __ string_indexof($str1$$Register, $str2$$Register,
16900                       $cnt1$$Register, zr,
16901                       $tmp1$$Register, $tmp2$$Register,
16902                       $tmp3$$Register, $tmp4$$Register, zr, zr,
16903                       icnt2, $result$$Register, StrIntrinsicNode::UL);
16904   %}
16905   ins_pipe(pipe_class_memory);
16906 %}
16907 
16908 instruct string_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16909                              iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16910                              iRegINoSp tmp3, rFlagsReg cr)
16911 %{
16912   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16913   predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
16914   effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16915          TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16916 
16917   format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16918 
16919   ins_encode %{
16920     __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16921                            $result$$Register, $tmp1$$Register, $tmp2$$Register,
16922                            $tmp3$$Register);
16923   %}
16924   ins_pipe(pipe_class_memory);
16925 %}
16926 
16927 instruct stringL_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16928                               iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16929                               iRegINoSp tmp3, rFlagsReg cr)
16930 %{
16931   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16932   predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
16933   effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16934          TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16935 
16936   format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16937 
16938   ins_encode %{
16939     __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16940                             $result$$Register, $tmp1$$Register, $tmp2$$Register,
16941                             $tmp3$$Register);
16942   %}
16943   ins_pipe(pipe_class_memory);
16944 %}
16945 
16946 instruct stringL_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16947                                   iRegI_R0 result, vecA ztmp1, vecA ztmp2,
16948                                   pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
16949   predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L);
16950   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16951   effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
16952   format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
16953   ins_encode %{
16954     __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
16955                                $result$$Register, $ztmp1$$FloatRegister,
16956                                $ztmp2$$FloatRegister, $pgtmp$$PRegister,
16957                                $ptmp$$PRegister, true /* isL */);
16958   %}
16959   ins_pipe(pipe_class_memory);
16960 %}
16961 
16962 instruct stringU_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16963                                   iRegI_R0 result, vecA ztmp1, vecA ztmp2,
16964                                   pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
16965   predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U);
16966   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16967   effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
16968   format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
16969   ins_encode %{
16970     __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
16971                                $result$$Register, $ztmp1$$FloatRegister,
16972                                $ztmp2$$FloatRegister, $pgtmp$$PRegister,
16973                                $ptmp$$PRegister, false /* isL */);
16974   %}
16975   ins_pipe(pipe_class_memory);
16976 %}
16977 
16978 instruct string_equalsL(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
16979                         iRegI_R0 result, rFlagsReg cr)
16980 %{
16981   predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::LL);
16982   match(Set result (StrEquals (Binary str1 str2) cnt));
16983   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
16984 
16985   format %{ "String Equals $str1,$str2,$cnt -> $result" %}
16986   ins_encode %{
16987     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16988     __ string_equals($str1$$Register, $str2$$Register,
16989                      $result$$Register, $cnt$$Register, 1);
16990   %}
16991   ins_pipe(pipe_class_memory);
16992 %}
16993 
16994 instruct string_equalsU(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
16995                         iRegI_R0 result, rFlagsReg cr)
16996 %{
16997   predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::UU);
16998   match(Set result (StrEquals (Binary str1 str2) cnt));
16999   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
17000 
17001   format %{ "String Equals $str1,$str2,$cnt -> $result" %}
17002   ins_encode %{
17003     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
17004     __ string_equals($str1$$Register, $str2$$Register,
17005                      $result$$Register, $cnt$$Register, 2);
17006   %}
17007   ins_pipe(pipe_class_memory);
17008 %}
17009 
17010 instruct array_equalsB(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
17011                        iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
17012                        iRegP_R10 tmp, rFlagsReg cr)
17013 %{
17014   predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
17015   match(Set result (AryEq ary1 ary2));
17016   effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
17017 
17018   format %{ "Array Equals $ary1,ary2 -> $result    // KILL $tmp" %}
17019   ins_encode %{
17020     address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
17021                                    $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
17022                                    $result$$Register, $tmp$$Register, 1);
17023     if (tpc == NULL) {
17024       ciEnv::current()->record_failure("CodeCache is full");
17025       return;
17026     }
17027   %}
17028   ins_pipe(pipe_class_memory);
17029 %}
17030 
17031 instruct array_equalsC(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
17032                        iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
17033                        iRegP_R10 tmp, rFlagsReg cr)
17034 %{
17035   predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
17036   match(Set result (AryEq ary1 ary2));
17037   effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
17038 
17039   format %{ "Array Equals $ary1,ary2 -> $result    // KILL $tmp" %}
17040   ins_encode %{
17041     address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
17042                                    $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
17043                                    $result$$Register, $tmp$$Register, 2);
17044     if (tpc == NULL) {
17045       ciEnv::current()->record_failure("CodeCache is full");
17046       return;
17047     }
17048   %}
17049   ins_pipe(pipe_class_memory);
17050 %}
17051 
17052 instruct count_positives(iRegP_R1 ary1, iRegI_R2 len, iRegI_R0 result, rFlagsReg cr)
17053 %{
17054   match(Set result (CountPositives ary1 len));
17055   effect(USE_KILL ary1, USE_KILL len, KILL cr);
17056   format %{ "count positives byte[] $ary1,$len -> $result" %}
17057   ins_encode %{
17058     address tpc = __ count_positives($ary1$$Register, $len$$Register, $result$$Register);
17059     if (tpc == NULL) {
17060       ciEnv::current()->record_failure("CodeCache is full");
17061       return;
17062     }
17063   %}
17064   ins_pipe( pipe_slow );
17065 %}
17066 
17067 // fast char[] to byte[] compression
17068 instruct string_compress(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17069                          vRegD_V0 tmp1, vRegD_V1 tmp2,
17070                          vRegD_V2 tmp3, vRegD_V3 tmp4,
17071                          iRegI_R0 result, rFlagsReg cr)
17072 %{
17073   match(Set result (StrCompressedCopy src (Binary dst len)));
17074   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4,
17075          USE_KILL src, USE_KILL dst, USE len, KILL cr);
17076 
17077   format %{ "String Compress $src,$dst,$len -> $result  // KILL $src,$dst" %}
17078   ins_encode %{
17079     __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
17080                            $result$$Register,
17081                            $tmp1$$FloatRegister, $tmp2$$FloatRegister,
17082                            $tmp3$$FloatRegister, $tmp4$$FloatRegister);
17083   %}
17084   ins_pipe(pipe_slow);
17085 %}
17086 
17087 // fast byte[] to char[] inflation
17088 instruct string_inflate(Universe dummy, iRegP_R0 src, iRegP_R1 dst, iRegI_R2 len,
17089                         vRegD_V0 tmp1, vRegD_V1 tmp2, vRegD_V2 tmp3, iRegP_R3 tmp4, rFlagsReg cr)
17090 %{
17091   match(Set dummy (StrInflatedCopy src (Binary dst len)));
17092   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
17093 
17094   format %{ "String Inflate $src,$dst    // KILL $tmp1, $tmp2" %}
17095   ins_encode %{
17096     address tpc = __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
17097                                         $tmp1$$FloatRegister, $tmp2$$FloatRegister,
17098                                         $tmp3$$FloatRegister, $tmp4$$Register);
17099     if (tpc == NULL) {
17100       ciEnv::current()->record_failure("CodeCache is full");
17101       return;
17102     }
17103   %}
17104   ins_pipe(pipe_class_memory);
17105 %}
17106 
17107 // encode char[] to byte[] in ISO_8859_1
17108 instruct encode_iso_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17109                           vRegD_V0 vtmp0, vRegD_V1 vtmp1,
17110                           vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17111                           iRegI_R0 result, rFlagsReg cr)
17112 %{
17113   predicate(!((EncodeISOArrayNode*)n)->is_ascii());
17114   match(Set result (EncodeISOArray src (Binary dst len)));
17115   effect(USE_KILL src, USE_KILL dst, USE len,
17116          KILL vtmp0, KILL vtmp1, KILL vtmp2, KILL vtmp3, KILL cr);
17117 
17118   format %{ "Encode ISO array $src,$dst,$len -> $result" %}
17119   ins_encode %{
17120     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
17121                         $result$$Register, false,
17122                         $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17123                         $vtmp2$$FloatRegister, $vtmp3$$FloatRegister);
17124   %}
17125   ins_pipe(pipe_class_memory);
17126 %}
17127 
17128 instruct encode_ascii_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17129                             vRegD_V0 vtmp0, vRegD_V1 vtmp1,
17130                             vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17131                             iRegI_R0 result, rFlagsReg cr)
17132 %{
17133   predicate(((EncodeISOArrayNode*)n)->is_ascii());
17134   match(Set result (EncodeISOArray src (Binary dst len)));
17135   effect(USE_KILL src, USE_KILL dst, USE len,
17136          KILL vtmp0, KILL vtmp1, KILL vtmp2, KILL vtmp3, KILL cr);
17137 
17138   format %{ "Encode ASCII array $src,$dst,$len -> $result" %}
17139   ins_encode %{
17140     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
17141                         $result$$Register, true,
17142                         $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17143                         $vtmp2$$FloatRegister, $vtmp3$$FloatRegister);
17144   %}
17145   ins_pipe(pipe_class_memory);
17146 %}
17147 
17148 // ============================================================================
17149 // This name is KNOWN by the ADLC and cannot be changed.
17150 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
17151 // for this guy.
17152 instruct tlsLoadP(thread_RegP dst)
17153 %{
17154   match(Set dst (ThreadLocal));
17155 
17156   ins_cost(0);
17157 
17158   format %{ " -- \t// $dst=Thread::current(), empty" %}
17159 
17160   size(0);
17161 
17162   ins_encode( /*empty*/ );
17163 
17164   ins_pipe(pipe_class_empty);
17165 %}
17166 
17167 //----------PEEPHOLE RULES-----------------------------------------------------
17168 // These must follow all instruction definitions as they use the names
17169 // defined in the instructions definitions.
17170 //
17171 // peepmatch ( root_instr_name [preceding_instruction]* );
17172 //
17173 // peepconstraint %{
17174 // (instruction_number.operand_name relational_op instruction_number.operand_name
17175 //  [, ...] );
17176 // // instruction numbers are zero-based using left to right order in peepmatch
17177 //
17178 // peepreplace ( instr_name  ( [instruction_number.operand_name]* ) );
17179 // // provide an instruction_number.operand_name for each operand that appears
17180 // // in the replacement instruction's match rule
17181 //
17182 // ---------VM FLAGS---------------------------------------------------------
17183 //
17184 // All peephole optimizations can be turned off using -XX:-OptoPeephole
17185 //
17186 // Each peephole rule is given an identifying number starting with zero and
17187 // increasing by one in the order seen by the parser.  An individual peephole
17188 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
17189 // on the command-line.
17190 //
17191 // ---------CURRENT LIMITATIONS----------------------------------------------
17192 //
17193 // Only match adjacent instructions in same basic block
17194 // Only equality constraints
17195 // Only constraints between operands, not (0.dest_reg == RAX_enc)
17196 // Only one replacement instruction
17197 //
17198 // ---------EXAMPLE----------------------------------------------------------
17199 //
17200 // // pertinent parts of existing instructions in architecture description
17201 // instruct movI(iRegINoSp dst, iRegI src)
17202 // %{
17203 //   match(Set dst (CopyI src));
17204 // %}
17205 //
17206 // instruct incI_iReg(iRegINoSp dst, immI1 src, rFlagsReg cr)
17207 // %{
17208 //   match(Set dst (AddI dst src));
17209 //   effect(KILL cr);
17210 // %}
17211 //
17212 // // Change (inc mov) to lea
17213 // peephole %{
17214 //   // increment preceded by register-register move
17215 //   peepmatch ( incI_iReg movI );
17216 //   // require that the destination register of the increment
17217 //   // match the destination register of the move
17218 //   peepconstraint ( 0.dst == 1.dst );
17219 //   // construct a replacement instruction that sets
17220 //   // the destination to ( move's source register + one )
17221 //   peepreplace ( leaI_iReg_immI( 0.dst 1.src 0.src ) );
17222 // %}
17223 //
17224 
17225 // Implementation no longer uses movX instructions since
17226 // machine-independent system no longer uses CopyX nodes.
17227 //
17228 // peephole
17229 // %{
17230 //   peepmatch (incI_iReg movI);
17231 //   peepconstraint (0.dst == 1.dst);
17232 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17233 // %}
17234 
17235 // peephole
17236 // %{
17237 //   peepmatch (decI_iReg movI);
17238 //   peepconstraint (0.dst == 1.dst);
17239 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17240 // %}
17241 
17242 // peephole
17243 // %{
17244 //   peepmatch (addI_iReg_imm movI);
17245 //   peepconstraint (0.dst == 1.dst);
17246 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17247 // %}
17248 
17249 // peephole
17250 // %{
17251 //   peepmatch (incL_iReg movL);
17252 //   peepconstraint (0.dst == 1.dst);
17253 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17254 // %}
17255 
17256 // peephole
17257 // %{
17258 //   peepmatch (decL_iReg movL);
17259 //   peepconstraint (0.dst == 1.dst);
17260 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17261 // %}
17262 
17263 // peephole
17264 // %{
17265 //   peepmatch (addL_iReg_imm movL);
17266 //   peepconstraint (0.dst == 1.dst);
17267 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17268 // %}
17269 
17270 // peephole
17271 // %{
17272 //   peepmatch (addP_iReg_imm movP);
17273 //   peepconstraint (0.dst == 1.dst);
17274 //   peepreplace (leaP_iReg_imm(0.dst 1.src 0.src));
17275 // %}
17276 
17277 // // Change load of spilled value to only a spill
17278 // instruct storeI(memory mem, iRegI src)
17279 // %{
17280 //   match(Set mem (StoreI mem src));
17281 // %}
17282 //
17283 // instruct loadI(iRegINoSp dst, memory mem)
17284 // %{
17285 //   match(Set dst (LoadI mem));
17286 // %}
17287 //
17288 
17289 //----------SMARTSPILL RULES---------------------------------------------------
17290 // These must follow all instruction definitions as they use the names
17291 // defined in the instructions definitions.
17292 
17293 // Local Variables:
17294 // mode: c++
17295 // End: