1 //
    2 // Copyright (c) 2003, 2025, Oracle and/or its affiliates. All rights reserved.
    3 // Copyright (c) 2014, 2024, Red Hat, Inc. All rights reserved.
    4 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
    5 //
    6 // This code is free software; you can redistribute it and/or modify it
    7 // under the terms of the GNU General Public License version 2 only, as
    8 // published by the Free Software Foundation.
    9 //
   10 // This code is distributed in the hope that it will be useful, but WITHOUT
   11 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
   12 // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
   13 // version 2 for more details (a copy is included in the LICENSE file that
   14 // accompanied this code).
   15 //
   16 // You should have received a copy of the GNU General Public License version
   17 // 2 along with this work; if not, write to the Free Software Foundation,
   18 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
   19 //
   20 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
   21 // or visit www.oracle.com if you need additional information or have any
   22 // questions.
   23 //
   24 //
   25 
   26 // AArch64 Architecture Description File
   27 
   28 //----------REGISTER DEFINITION BLOCK------------------------------------------
   29 // This information is used by the matcher and the register allocator to
   30 // describe individual registers and classes of registers within the target
   31 // architecture.
   32 
   33 register %{
   34 //----------Architecture Description Register Definitions----------------------
   35 // General Registers
   36 // "reg_def"  name ( register save type, C convention save type,
   37 //                   ideal register type, encoding );
   38 // Register Save Types:
   39 //
   40 // NS  = No-Save:       The register allocator assumes that these registers
   41 //                      can be used without saving upon entry to the method, &
   42 //                      that they do not need to be saved at call sites.
   43 //
   44 // SOC = Save-On-Call:  The register allocator assumes that these registers
   45 //                      can be used without saving upon entry to the method,
   46 //                      but that they must be saved at call sites.
   47 //
   48 // SOE = Save-On-Entry: The register allocator assumes that these registers
   49 //                      must be saved before using them upon entry to the
   50 //                      method, but they do not need to be saved at call
   51 //                      sites.
   52 //
   53 // AS  = Always-Save:   The register allocator assumes that these registers
   54 //                      must be saved before using them upon entry to the
   55 //                      method, & that they must be saved at call sites.
   56 //
   57 // Ideal Register Type is used to determine how to save & restore a
   58 // register.  Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
   59 // spilled with LoadP/StoreP.  If the register supports both, use Op_RegI.
   60 //
   61 // The encoding number is the actual bit-pattern placed into the opcodes.
   62 
   63 // We must define the 64 bit int registers in two 32 bit halves, the
   64 // real lower register and a virtual upper half register. upper halves
   65 // are used by the register allocator but are not actually supplied as
   66 // operands to memory ops.
   67 //
   68 // follow the C1 compiler in making registers
   69 //
   70 //   r0-r7,r10-r26 volatile (caller save)
   71 //   r27-r32 system (no save, no allocate)
   72 //   r8-r9 non-allocatable (so we can use them as scratch regs)
   73 //
   74 // as regards Java usage. we don't use any callee save registers
   75 // because this makes it difficult to de-optimise a frame (see comment
   76 // in x86 implementation of Deoptimization::unwind_callee_save_values)
   77 //
   78 
   79 // General Registers
   80 
   81 reg_def R0      ( SOC, SOC, Op_RegI,  0, r0->as_VMReg()         );
   82 reg_def R0_H    ( SOC, SOC, Op_RegI,  0, r0->as_VMReg()->next() );
   83 reg_def R1      ( SOC, SOC, Op_RegI,  1, r1->as_VMReg()         );
   84 reg_def R1_H    ( SOC, SOC, Op_RegI,  1, r1->as_VMReg()->next() );
   85 reg_def R2      ( SOC, SOC, Op_RegI,  2, r2->as_VMReg()         );
   86 reg_def R2_H    ( SOC, SOC, Op_RegI,  2, r2->as_VMReg()->next() );
   87 reg_def R3      ( SOC, SOC, Op_RegI,  3, r3->as_VMReg()         );
   88 reg_def R3_H    ( SOC, SOC, Op_RegI,  3, r3->as_VMReg()->next() );
   89 reg_def R4      ( SOC, SOC, Op_RegI,  4, r4->as_VMReg()         );
   90 reg_def R4_H    ( SOC, SOC, Op_RegI,  4, r4->as_VMReg()->next() );
   91 reg_def R5      ( SOC, SOC, Op_RegI,  5, r5->as_VMReg()         );
   92 reg_def R5_H    ( SOC, SOC, Op_RegI,  5, r5->as_VMReg()->next() );
   93 reg_def R6      ( SOC, SOC, Op_RegI,  6, r6->as_VMReg()         );
   94 reg_def R6_H    ( SOC, SOC, Op_RegI,  6, r6->as_VMReg()->next() );
   95 reg_def R7      ( SOC, SOC, Op_RegI,  7, r7->as_VMReg()         );
   96 reg_def R7_H    ( SOC, SOC, Op_RegI,  7, r7->as_VMReg()->next() );
   97 reg_def R8      ( NS,  SOC, Op_RegI,  8, r8->as_VMReg()         ); // rscratch1, non-allocatable
   98 reg_def R8_H    ( NS,  SOC, Op_RegI,  8, r8->as_VMReg()->next() );
   99 reg_def R9      ( NS,  SOC, Op_RegI,  9, r9->as_VMReg()         ); // rscratch2, non-allocatable
  100 reg_def R9_H    ( NS,  SOC, Op_RegI,  9, r9->as_VMReg()->next() );
  101 reg_def R10     ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()        );
  102 reg_def R10_H   ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
  103 reg_def R11     ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()        );
  104 reg_def R11_H   ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
  105 reg_def R12     ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()        );
  106 reg_def R12_H   ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()->next());
  107 reg_def R13     ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()        );
  108 reg_def R13_H   ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()->next());
  109 reg_def R14     ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()        );
  110 reg_def R14_H   ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()->next());
  111 reg_def R15     ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()        );
  112 reg_def R15_H   ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()->next());
  113 reg_def R16     ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()        );
  114 reg_def R16_H   ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
  115 reg_def R17     ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()        );
  116 reg_def R17_H   ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
  117 reg_def R18     ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()        );
  118 reg_def R18_H   ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()->next());
  119 reg_def R19     ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()        );
  120 reg_def R19_H   ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()->next());
  121 reg_def R20     ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()        ); // caller esp
  122 reg_def R20_H   ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()->next());
  123 reg_def R21     ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()        );
  124 reg_def R21_H   ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()->next());
  125 reg_def R22     ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()        );
  126 reg_def R22_H   ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()->next());
  127 reg_def R23     ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()        );
  128 reg_def R23_H   ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()->next());
  129 reg_def R24     ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()        );
  130 reg_def R24_H   ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()->next());
  131 reg_def R25     ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()        );
  132 reg_def R25_H   ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()->next());
  133 reg_def R26     ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()        );
  134 reg_def R26_H   ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()->next());
  135 reg_def R27     ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()        ); // heapbase
  136 reg_def R27_H   ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()->next());
  137 reg_def R28     (  NS, SOE, Op_RegI, 28, r28->as_VMReg()        ); // thread
  138 reg_def R28_H   (  NS, SOE, Op_RegI, 28, r28->as_VMReg()->next());
  139 reg_def R29     (  NS,  NS, Op_RegI, 29, r29->as_VMReg()        ); // fp
  140 reg_def R29_H   (  NS,  NS, Op_RegI, 29, r29->as_VMReg()->next());
  141 reg_def R30     (  NS,  NS, Op_RegI, 30, r30->as_VMReg()        ); // lr
  142 reg_def R30_H   (  NS,  NS, Op_RegI, 30, r30->as_VMReg()->next());
  143 reg_def R31     (  NS,  NS, Op_RegI, 31, r31_sp->as_VMReg()     ); // sp
  144 reg_def R31_H   (  NS,  NS, Op_RegI, 31, r31_sp->as_VMReg()->next());
  145 
  146 // ----------------------------
  147 // Float/Double/Vector Registers
  148 // ----------------------------
  149 
  150 // Double Registers
  151 
  152 // The rules of ADL require that double registers be defined in pairs.
  153 // Each pair must be two 32-bit values, but not necessarily a pair of
  154 // single float registers. In each pair, ADLC-assigned register numbers
  155 // must be adjacent, with the lower number even. Finally, when the
  156 // CPU stores such a register pair to memory, the word associated with
  157 // the lower ADLC-assigned number must be stored to the lower address.
  158 
  159 // AArch64 has 32 floating-point registers. Each can store a vector of
  160 // single or double precision floating-point values up to 8 * 32
  161 // floats, 4 * 64 bit floats or 2 * 128 bit floats.  We currently only
  162 // use the first float or double element of the vector.
  163 
  164 // for Java use float registers v0-v15 are always save on call whereas
  165 // the platform ABI treats v8-v15 as callee save). float registers
  166 // v16-v31 are SOC as per the platform spec
  167 
  168 // For SVE vector registers, we simply extend vector register size to 8
  169 // 'logical' slots. This is nominally 256 bits but it actually covers
  170 // all possible 'physical' SVE vector register lengths from 128 ~ 2048
  171 // bits. The 'physical' SVE vector register length is detected during
  172 // startup, so the register allocator is able to identify the correct
  173 // number of bytes needed for an SVE spill/unspill.
  174 // Note that a vector register with 4 slots denotes a 128-bit NEON
  175 // register allowing it to be distinguished from the corresponding SVE
  176 // vector register when the SVE vector length is 128 bits.
  177 
  178   reg_def V0   ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()          );
  179   reg_def V0_H ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next()  );
  180   reg_def V0_J ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(2) );
  181   reg_def V0_K ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(3) );
  182 
  183   reg_def V1   ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()          );
  184   reg_def V1_H ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next()  );
  185   reg_def V1_J ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(2) );
  186   reg_def V1_K ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(3) );
  187 
  188   reg_def V2   ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()          );
  189   reg_def V2_H ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next()  );
  190   reg_def V2_J ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(2) );
  191   reg_def V2_K ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(3) );
  192 
  193   reg_def V3   ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()          );
  194   reg_def V3_H ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next()  );
  195   reg_def V3_J ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(2) );
  196   reg_def V3_K ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(3) );
  197 
  198   reg_def V4   ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()          );
  199   reg_def V4_H ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next()  );
  200   reg_def V4_J ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(2) );
  201   reg_def V4_K ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(3) );
  202 
  203   reg_def V5   ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()          );
  204   reg_def V5_H ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next()  );
  205   reg_def V5_J ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(2) );
  206   reg_def V5_K ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(3) );
  207 
  208   reg_def V6   ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()          );
  209   reg_def V6_H ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next()  );
  210   reg_def V6_J ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(2) );
  211   reg_def V6_K ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(3) );
  212 
  213   reg_def V7   ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()          );
  214   reg_def V7_H ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next()  );
  215   reg_def V7_J ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(2) );
  216   reg_def V7_K ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(3) );
  217 
  218   reg_def V8   ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()          );
  219   reg_def V8_H ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()->next()  );
  220   reg_def V8_J ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(2) );
  221   reg_def V8_K ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(3) );
  222 
  223   reg_def V9   ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()          );
  224   reg_def V9_H ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()->next()  );
  225   reg_def V9_J ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(2) );
  226   reg_def V9_K ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(3) );
  227 
  228   reg_def V10   ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()          );
  229   reg_def V10_H ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()->next()  );
  230   reg_def V10_J ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(2) );
  231   reg_def V10_K ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(3) );
  232 
  233   reg_def V11   ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()          );
  234   reg_def V11_H ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()->next()  );
  235   reg_def V11_J ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(2) );
  236   reg_def V11_K ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(3) );
  237 
  238   reg_def V12   ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()          );
  239   reg_def V12_H ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()->next()  );
  240   reg_def V12_J ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(2) );
  241   reg_def V12_K ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(3) );
  242 
  243   reg_def V13   ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()          );
  244   reg_def V13_H ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()->next()  );
  245   reg_def V13_J ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(2) );
  246   reg_def V13_K ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(3) );
  247 
  248   reg_def V14   ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()          );
  249   reg_def V14_H ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()->next()  );
  250   reg_def V14_J ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(2) );
  251   reg_def V14_K ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(3) );
  252 
  253   reg_def V15   ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()          );
  254   reg_def V15_H ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()->next()  );
  255   reg_def V15_J ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(2) );
  256   reg_def V15_K ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(3) );
  257 
  258   reg_def V16   ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()          );
  259   reg_def V16_H ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next()  );
  260   reg_def V16_J ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(2) );
  261   reg_def V16_K ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(3) );
  262 
  263   reg_def V17   ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()          );
  264   reg_def V17_H ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next()  );
  265   reg_def V17_J ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(2) );
  266   reg_def V17_K ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(3) );
  267 
  268   reg_def V18   ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()          );
  269   reg_def V18_H ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next()  );
  270   reg_def V18_J ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(2) );
  271   reg_def V18_K ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(3) );
  272 
  273   reg_def V19   ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()          );
  274   reg_def V19_H ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next()  );
  275   reg_def V19_J ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(2) );
  276   reg_def V19_K ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(3) );
  277 
  278   reg_def V20   ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()          );
  279   reg_def V20_H ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next()  );
  280   reg_def V20_J ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(2) );
  281   reg_def V20_K ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(3) );
  282 
  283   reg_def V21   ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()          );
  284   reg_def V21_H ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next()  );
  285   reg_def V21_J ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(2) );
  286   reg_def V21_K ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(3) );
  287 
  288   reg_def V22   ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()          );
  289   reg_def V22_H ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next()  );
  290   reg_def V22_J ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(2) );
  291   reg_def V22_K ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(3) );
  292 
  293   reg_def V23   ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()          );
  294   reg_def V23_H ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next()  );
  295   reg_def V23_J ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(2) );
  296   reg_def V23_K ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(3) );
  297 
  298   reg_def V24   ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()          );
  299   reg_def V24_H ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next()  );
  300   reg_def V24_J ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(2) );
  301   reg_def V24_K ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(3) );
  302 
  303   reg_def V25   ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()          );
  304   reg_def V25_H ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next()  );
  305   reg_def V25_J ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(2) );
  306   reg_def V25_K ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(3) );
  307 
  308   reg_def V26   ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()          );
  309   reg_def V26_H ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next()  );
  310   reg_def V26_J ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(2) );
  311   reg_def V26_K ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(3) );
  312 
  313   reg_def V27   ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()          );
  314   reg_def V27_H ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next()  );
  315   reg_def V27_J ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(2) );
  316   reg_def V27_K ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(3) );
  317 
  318   reg_def V28   ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()          );
  319   reg_def V28_H ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next()  );
  320   reg_def V28_J ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(2) );
  321   reg_def V28_K ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(3) );
  322 
  323   reg_def V29   ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()          );
  324   reg_def V29_H ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next()  );
  325   reg_def V29_J ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(2) );
  326   reg_def V29_K ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(3) );
  327 
  328   reg_def V30   ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()          );
  329   reg_def V30_H ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next()  );
  330   reg_def V30_J ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(2) );
  331   reg_def V30_K ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(3) );
  332 
  333   reg_def V31   ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()          );
  334   reg_def V31_H ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next()  );
  335   reg_def V31_J ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(2) );
  336   reg_def V31_K ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(3) );
  337 
  338 // ----------------------------
  339 // SVE Predicate Registers
  340 // ----------------------------
  341   reg_def P0 (SOC, SOC, Op_RegVectMask, 0, p0->as_VMReg());
  342   reg_def P1 (SOC, SOC, Op_RegVectMask, 1, p1->as_VMReg());
  343   reg_def P2 (SOC, SOC, Op_RegVectMask, 2, p2->as_VMReg());
  344   reg_def P3 (SOC, SOC, Op_RegVectMask, 3, p3->as_VMReg());
  345   reg_def P4 (SOC, SOC, Op_RegVectMask, 4, p4->as_VMReg());
  346   reg_def P5 (SOC, SOC, Op_RegVectMask, 5, p5->as_VMReg());
  347   reg_def P6 (SOC, SOC, Op_RegVectMask, 6, p6->as_VMReg());
  348   reg_def P7 (SOC, SOC, Op_RegVectMask, 7, p7->as_VMReg());
  349   reg_def P8 (SOC, SOC, Op_RegVectMask, 8, p8->as_VMReg());
  350   reg_def P9 (SOC, SOC, Op_RegVectMask, 9, p9->as_VMReg());
  351   reg_def P10 (SOC, SOC, Op_RegVectMask, 10, p10->as_VMReg());
  352   reg_def P11 (SOC, SOC, Op_RegVectMask, 11, p11->as_VMReg());
  353   reg_def P12 (SOC, SOC, Op_RegVectMask, 12, p12->as_VMReg());
  354   reg_def P13 (SOC, SOC, Op_RegVectMask, 13, p13->as_VMReg());
  355   reg_def P14 (SOC, SOC, Op_RegVectMask, 14, p14->as_VMReg());
  356   reg_def P15 (SOC, SOC, Op_RegVectMask, 15, p15->as_VMReg());
  357 
  358 // ----------------------------
  359 // Special Registers
  360 // ----------------------------
  361 
  362 // the AArch64 CSPR status flag register is not directly accessible as
  363 // instruction operand. the FPSR status flag register is a system
  364 // register which can be written/read using MSR/MRS but again does not
  365 // appear as an operand (a code identifying the FSPR occurs as an
  366 // immediate value in the instruction).
  367 
  368 reg_def RFLAGS(SOC, SOC, 0, 32, VMRegImpl::Bad());
  369 
  370 // Specify priority of register selection within phases of register
  371 // allocation.  Highest priority is first.  A useful heuristic is to
  372 // give registers a low priority when they are required by machine
  373 // instructions, like EAX and EDX on I486, and choose no-save registers
  374 // before save-on-call, & save-on-call before save-on-entry.  Registers
  375 // which participate in fixed calling sequences should come last.
  376 // Registers which are used as pairs must fall on an even boundary.
  377 
  378 alloc_class chunk0(
  379     // volatiles
  380     R10, R10_H,
  381     R11, R11_H,
  382     R12, R12_H,
  383     R13, R13_H,
  384     R14, R14_H,
  385     R15, R15_H,
  386     R16, R16_H,
  387     R17, R17_H,
  388     R18, R18_H,
  389 
  390     // arg registers
  391     R0, R0_H,
  392     R1, R1_H,
  393     R2, R2_H,
  394     R3, R3_H,
  395     R4, R4_H,
  396     R5, R5_H,
  397     R6, R6_H,
  398     R7, R7_H,
  399 
  400     // non-volatiles
  401     R19, R19_H,
  402     R20, R20_H,
  403     R21, R21_H,
  404     R22, R22_H,
  405     R23, R23_H,
  406     R24, R24_H,
  407     R25, R25_H,
  408     R26, R26_H,
  409 
  410     // non-allocatable registers
  411 
  412     R27, R27_H, // heapbase
  413     R28, R28_H, // thread
  414     R29, R29_H, // fp
  415     R30, R30_H, // lr
  416     R31, R31_H, // sp
  417     R8, R8_H,   // rscratch1
  418     R9, R9_H,   // rscratch2
  419 );
  420 
  421 alloc_class chunk1(
  422 
  423     // no save
  424     V16, V16_H, V16_J, V16_K,
  425     V17, V17_H, V17_J, V17_K,
  426     V18, V18_H, V18_J, V18_K,
  427     V19, V19_H, V19_J, V19_K,
  428     V20, V20_H, V20_J, V20_K,
  429     V21, V21_H, V21_J, V21_K,
  430     V22, V22_H, V22_J, V22_K,
  431     V23, V23_H, V23_J, V23_K,
  432     V24, V24_H, V24_J, V24_K,
  433     V25, V25_H, V25_J, V25_K,
  434     V26, V26_H, V26_J, V26_K,
  435     V27, V27_H, V27_J, V27_K,
  436     V28, V28_H, V28_J, V28_K,
  437     V29, V29_H, V29_J, V29_K,
  438     V30, V30_H, V30_J, V30_K,
  439     V31, V31_H, V31_J, V31_K,
  440 
  441     // arg registers
  442     V0, V0_H, V0_J, V0_K,
  443     V1, V1_H, V1_J, V1_K,
  444     V2, V2_H, V2_J, V2_K,
  445     V3, V3_H, V3_J, V3_K,
  446     V4, V4_H, V4_J, V4_K,
  447     V5, V5_H, V5_J, V5_K,
  448     V6, V6_H, V6_J, V6_K,
  449     V7, V7_H, V7_J, V7_K,
  450 
  451     // non-volatiles
  452     V8, V8_H, V8_J, V8_K,
  453     V9, V9_H, V9_J, V9_K,
  454     V10, V10_H, V10_J, V10_K,
  455     V11, V11_H, V11_J, V11_K,
  456     V12, V12_H, V12_J, V12_K,
  457     V13, V13_H, V13_J, V13_K,
  458     V14, V14_H, V14_J, V14_K,
  459     V15, V15_H, V15_J, V15_K,
  460 );
  461 
  462 alloc_class chunk2 (
  463     // Governing predicates for load/store and arithmetic
  464     P0,
  465     P1,
  466     P2,
  467     P3,
  468     P4,
  469     P5,
  470     P6,
  471 
  472     // Extra predicates
  473     P8,
  474     P9,
  475     P10,
  476     P11,
  477     P12,
  478     P13,
  479     P14,
  480     P15,
  481 
  482     // Preserved for all-true predicate
  483     P7,
  484 );
  485 
  486 alloc_class chunk3(RFLAGS);
  487 
  488 //----------Architecture Description Register Classes--------------------------
  489 // Several register classes are automatically defined based upon information in
  490 // this architecture description.
  491 // 1) reg_class inline_cache_reg           ( /* as def'd in frame section */ )
  492 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
  493 //
  494 
  495 // Class for all 32 bit general purpose registers
  496 reg_class all_reg32(
  497     R0,
  498     R1,
  499     R2,
  500     R3,
  501     R4,
  502     R5,
  503     R6,
  504     R7,
  505     R10,
  506     R11,
  507     R12,
  508     R13,
  509     R14,
  510     R15,
  511     R16,
  512     R17,
  513     R18,
  514     R19,
  515     R20,
  516     R21,
  517     R22,
  518     R23,
  519     R24,
  520     R25,
  521     R26,
  522     R27,
  523     R28,
  524     R29,
  525     R30,
  526     R31
  527 );
  528 
  529 
  530 // Class for all 32 bit integer registers (excluding SP which
  531 // will never be used as an integer register)
  532 reg_class any_reg32 %{
  533   return _ANY_REG32_mask;
  534 %}
  535 
  536 // Singleton class for R0 int register
  537 reg_class int_r0_reg(R0);
  538 
  539 // Singleton class for R2 int register
  540 reg_class int_r2_reg(R2);
  541 
  542 // Singleton class for R3 int register
  543 reg_class int_r3_reg(R3);
  544 
  545 // Singleton class for R4 int register
  546 reg_class int_r4_reg(R4);
  547 
  548 // Singleton class for R31 int register
  549 reg_class int_r31_reg(R31);
  550 
  551 // Class for all 64 bit general purpose registers
  552 reg_class all_reg(
  553     R0, R0_H,
  554     R1, R1_H,
  555     R2, R2_H,
  556     R3, R3_H,
  557     R4, R4_H,
  558     R5, R5_H,
  559     R6, R6_H,
  560     R7, R7_H,
  561     R10, R10_H,
  562     R11, R11_H,
  563     R12, R12_H,
  564     R13, R13_H,
  565     R14, R14_H,
  566     R15, R15_H,
  567     R16, R16_H,
  568     R17, R17_H,
  569     R18, R18_H,
  570     R19, R19_H,
  571     R20, R20_H,
  572     R21, R21_H,
  573     R22, R22_H,
  574     R23, R23_H,
  575     R24, R24_H,
  576     R25, R25_H,
  577     R26, R26_H,
  578     R27, R27_H,
  579     R28, R28_H,
  580     R29, R29_H,
  581     R30, R30_H,
  582     R31, R31_H
  583 );
  584 
  585 // Class for all long integer registers (including SP)
  586 reg_class any_reg %{
  587   return _ANY_REG_mask;
  588 %}
  589 
  590 // Class for non-allocatable 32 bit registers
  591 reg_class non_allocatable_reg32(
  592 #ifdef R18_RESERVED
  593     // See comment in register_aarch64.hpp
  594     R18,                        // tls on Windows
  595 #endif
  596     R28,                        // thread
  597     R30,                        // lr
  598     R31                         // sp
  599 );
  600 
  601 // Class for non-allocatable 64 bit registers
  602 reg_class non_allocatable_reg(
  603 #ifdef R18_RESERVED
  604     // See comment in register_aarch64.hpp
  605     R18, R18_H,                 // tls on Windows, platform register on macOS
  606 #endif
  607     R28, R28_H,                 // thread
  608     R30, R30_H,                 // lr
  609     R31, R31_H                  // sp
  610 );
  611 
  612 // Class for all non-special integer registers
  613 reg_class no_special_reg32 %{
  614   return _NO_SPECIAL_REG32_mask;
  615 %}
  616 
  617 // Class for all non-special long integer registers
  618 reg_class no_special_reg %{
  619   return _NO_SPECIAL_REG_mask;
  620 %}
  621 
  622 // Class for 64 bit register r0
  623 reg_class r0_reg(
  624     R0, R0_H
  625 );
  626 
  627 // Class for 64 bit register r1
  628 reg_class r1_reg(
  629     R1, R1_H
  630 );
  631 
  632 // Class for 64 bit register r2
  633 reg_class r2_reg(
  634     R2, R2_H
  635 );
  636 
  637 // Class for 64 bit register r3
  638 reg_class r3_reg(
  639     R3, R3_H
  640 );
  641 
  642 // Class for 64 bit register r4
  643 reg_class r4_reg(
  644     R4, R4_H
  645 );
  646 
  647 // Class for 64 bit register r5
  648 reg_class r5_reg(
  649     R5, R5_H
  650 );
  651 
  652 // Class for 64 bit register r10
  653 reg_class r10_reg(
  654     R10, R10_H
  655 );
  656 
  657 // Class for 64 bit register r11
  658 reg_class r11_reg(
  659     R11, R11_H
  660 );
  661 
  662 // Class for method register
  663 reg_class method_reg(
  664     R12, R12_H
  665 );
  666 
  667 // Class for thread register
  668 reg_class thread_reg(
  669     R28, R28_H
  670 );
  671 
  672 // Class for frame pointer register
  673 reg_class fp_reg(
  674     R29, R29_H
  675 );
  676 
  677 // Class for link register
  678 reg_class lr_reg(
  679     R30, R30_H
  680 );
  681 
  682 // Class for long sp register
  683 reg_class sp_reg(
  684   R31, R31_H
  685 );
  686 
  687 // Class for all pointer registers
  688 reg_class ptr_reg %{
  689   return _PTR_REG_mask;
  690 %}
  691 
  692 // Class for all non_special pointer registers
  693 reg_class no_special_ptr_reg %{
  694   return _NO_SPECIAL_PTR_REG_mask;
  695 %}
  696 
  697 // Class for all non_special pointer registers (excluding rfp)
  698 reg_class no_special_no_rfp_ptr_reg %{
  699   return _NO_SPECIAL_NO_RFP_PTR_REG_mask;
  700 %}
  701 
  702 // Class for all float registers
  703 reg_class float_reg(
  704     V0,
  705     V1,
  706     V2,
  707     V3,
  708     V4,
  709     V5,
  710     V6,
  711     V7,
  712     V8,
  713     V9,
  714     V10,
  715     V11,
  716     V12,
  717     V13,
  718     V14,
  719     V15,
  720     V16,
  721     V17,
  722     V18,
  723     V19,
  724     V20,
  725     V21,
  726     V22,
  727     V23,
  728     V24,
  729     V25,
  730     V26,
  731     V27,
  732     V28,
  733     V29,
  734     V30,
  735     V31
  736 );
  737 
  738 // Double precision float registers have virtual `high halves' that
  739 // are needed by the allocator.
  740 // Class for all double registers
  741 reg_class double_reg(
  742     V0, V0_H,
  743     V1, V1_H,
  744     V2, V2_H,
  745     V3, V3_H,
  746     V4, V4_H,
  747     V5, V5_H,
  748     V6, V6_H,
  749     V7, V7_H,
  750     V8, V8_H,
  751     V9, V9_H,
  752     V10, V10_H,
  753     V11, V11_H,
  754     V12, V12_H,
  755     V13, V13_H,
  756     V14, V14_H,
  757     V15, V15_H,
  758     V16, V16_H,
  759     V17, V17_H,
  760     V18, V18_H,
  761     V19, V19_H,
  762     V20, V20_H,
  763     V21, V21_H,
  764     V22, V22_H,
  765     V23, V23_H,
  766     V24, V24_H,
  767     V25, V25_H,
  768     V26, V26_H,
  769     V27, V27_H,
  770     V28, V28_H,
  771     V29, V29_H,
  772     V30, V30_H,
  773     V31, V31_H
  774 );
  775 
  776 // Class for all SVE vector registers.
  777 reg_class vectora_reg (
  778     V0, V0_H, V0_J, V0_K,
  779     V1, V1_H, V1_J, V1_K,
  780     V2, V2_H, V2_J, V2_K,
  781     V3, V3_H, V3_J, V3_K,
  782     V4, V4_H, V4_J, V4_K,
  783     V5, V5_H, V5_J, V5_K,
  784     V6, V6_H, V6_J, V6_K,
  785     V7, V7_H, V7_J, V7_K,
  786     V8, V8_H, V8_J, V8_K,
  787     V9, V9_H, V9_J, V9_K,
  788     V10, V10_H, V10_J, V10_K,
  789     V11, V11_H, V11_J, V11_K,
  790     V12, V12_H, V12_J, V12_K,
  791     V13, V13_H, V13_J, V13_K,
  792     V14, V14_H, V14_J, V14_K,
  793     V15, V15_H, V15_J, V15_K,
  794     V16, V16_H, V16_J, V16_K,
  795     V17, V17_H, V17_J, V17_K,
  796     V18, V18_H, V18_J, V18_K,
  797     V19, V19_H, V19_J, V19_K,
  798     V20, V20_H, V20_J, V20_K,
  799     V21, V21_H, V21_J, V21_K,
  800     V22, V22_H, V22_J, V22_K,
  801     V23, V23_H, V23_J, V23_K,
  802     V24, V24_H, V24_J, V24_K,
  803     V25, V25_H, V25_J, V25_K,
  804     V26, V26_H, V26_J, V26_K,
  805     V27, V27_H, V27_J, V27_K,
  806     V28, V28_H, V28_J, V28_K,
  807     V29, V29_H, V29_J, V29_K,
  808     V30, V30_H, V30_J, V30_K,
  809     V31, V31_H, V31_J, V31_K,
  810 );
  811 
  812 // Class for all 64bit vector registers
  813 reg_class vectord_reg(
  814     V0, V0_H,
  815     V1, V1_H,
  816     V2, V2_H,
  817     V3, V3_H,
  818     V4, V4_H,
  819     V5, V5_H,
  820     V6, V6_H,
  821     V7, V7_H,
  822     V8, V8_H,
  823     V9, V9_H,
  824     V10, V10_H,
  825     V11, V11_H,
  826     V12, V12_H,
  827     V13, V13_H,
  828     V14, V14_H,
  829     V15, V15_H,
  830     V16, V16_H,
  831     V17, V17_H,
  832     V18, V18_H,
  833     V19, V19_H,
  834     V20, V20_H,
  835     V21, V21_H,
  836     V22, V22_H,
  837     V23, V23_H,
  838     V24, V24_H,
  839     V25, V25_H,
  840     V26, V26_H,
  841     V27, V27_H,
  842     V28, V28_H,
  843     V29, V29_H,
  844     V30, V30_H,
  845     V31, V31_H
  846 );
  847 
  848 // Class for all 128bit vector registers
  849 reg_class vectorx_reg(
  850     V0, V0_H, V0_J, V0_K,
  851     V1, V1_H, V1_J, V1_K,
  852     V2, V2_H, V2_J, V2_K,
  853     V3, V3_H, V3_J, V3_K,
  854     V4, V4_H, V4_J, V4_K,
  855     V5, V5_H, V5_J, V5_K,
  856     V6, V6_H, V6_J, V6_K,
  857     V7, V7_H, V7_J, V7_K,
  858     V8, V8_H, V8_J, V8_K,
  859     V9, V9_H, V9_J, V9_K,
  860     V10, V10_H, V10_J, V10_K,
  861     V11, V11_H, V11_J, V11_K,
  862     V12, V12_H, V12_J, V12_K,
  863     V13, V13_H, V13_J, V13_K,
  864     V14, V14_H, V14_J, V14_K,
  865     V15, V15_H, V15_J, V15_K,
  866     V16, V16_H, V16_J, V16_K,
  867     V17, V17_H, V17_J, V17_K,
  868     V18, V18_H, V18_J, V18_K,
  869     V19, V19_H, V19_J, V19_K,
  870     V20, V20_H, V20_J, V20_K,
  871     V21, V21_H, V21_J, V21_K,
  872     V22, V22_H, V22_J, V22_K,
  873     V23, V23_H, V23_J, V23_K,
  874     V24, V24_H, V24_J, V24_K,
  875     V25, V25_H, V25_J, V25_K,
  876     V26, V26_H, V26_J, V26_K,
  877     V27, V27_H, V27_J, V27_K,
  878     V28, V28_H, V28_J, V28_K,
  879     V29, V29_H, V29_J, V29_K,
  880     V30, V30_H, V30_J, V30_K,
  881     V31, V31_H, V31_J, V31_K
  882 );
  883 
  884 // Class for 128 bit register v0
  885 reg_class v0_reg(
  886     V0, V0_H
  887 );
  888 
  889 // Class for 128 bit register v1
  890 reg_class v1_reg(
  891     V1, V1_H
  892 );
  893 
  894 // Class for 128 bit register v2
  895 reg_class v2_reg(
  896     V2, V2_H
  897 );
  898 
  899 // Class for 128 bit register v3
  900 reg_class v3_reg(
  901     V3, V3_H
  902 );
  903 
  904 // Class for 128 bit register v4
  905 reg_class v4_reg(
  906     V4, V4_H
  907 );
  908 
  909 // Class for 128 bit register v5
  910 reg_class v5_reg(
  911     V5, V5_H
  912 );
  913 
  914 // Class for 128 bit register v6
  915 reg_class v6_reg(
  916     V6, V6_H
  917 );
  918 
  919 // Class for 128 bit register v7
  920 reg_class v7_reg(
  921     V7, V7_H
  922 );
  923 
  924 // Class for 128 bit register v8
  925 reg_class v8_reg(
  926     V8, V8_H
  927 );
  928 
  929 // Class for 128 bit register v9
  930 reg_class v9_reg(
  931     V9, V9_H
  932 );
  933 
  934 // Class for 128 bit register v10
  935 reg_class v10_reg(
  936     V10, V10_H
  937 );
  938 
  939 // Class for 128 bit register v11
  940 reg_class v11_reg(
  941     V11, V11_H
  942 );
  943 
  944 // Class for 128 bit register v12
  945 reg_class v12_reg(
  946     V12, V12_H
  947 );
  948 
  949 // Class for 128 bit register v13
  950 reg_class v13_reg(
  951     V13, V13_H
  952 );
  953 
  954 // Class for 128 bit register v14
  955 reg_class v14_reg(
  956     V14, V14_H
  957 );
  958 
  959 // Class for 128 bit register v15
  960 reg_class v15_reg(
  961     V15, V15_H
  962 );
  963 
  964 // Class for 128 bit register v16
  965 reg_class v16_reg(
  966     V16, V16_H
  967 );
  968 
  969 // Class for 128 bit register v17
  970 reg_class v17_reg(
  971     V17, V17_H
  972 );
  973 
  974 // Class for 128 bit register v18
  975 reg_class v18_reg(
  976     V18, V18_H
  977 );
  978 
  979 // Class for 128 bit register v19
  980 reg_class v19_reg(
  981     V19, V19_H
  982 );
  983 
  984 // Class for 128 bit register v20
  985 reg_class v20_reg(
  986     V20, V20_H
  987 );
  988 
  989 // Class for 128 bit register v21
  990 reg_class v21_reg(
  991     V21, V21_H
  992 );
  993 
  994 // Class for 128 bit register v22
  995 reg_class v22_reg(
  996     V22, V22_H
  997 );
  998 
  999 // Class for 128 bit register v23
 1000 reg_class v23_reg(
 1001     V23, V23_H
 1002 );
 1003 
 1004 // Class for 128 bit register v24
 1005 reg_class v24_reg(
 1006     V24, V24_H
 1007 );
 1008 
 1009 // Class for 128 bit register v25
 1010 reg_class v25_reg(
 1011     V25, V25_H
 1012 );
 1013 
 1014 // Class for 128 bit register v26
 1015 reg_class v26_reg(
 1016     V26, V26_H
 1017 );
 1018 
 1019 // Class for 128 bit register v27
 1020 reg_class v27_reg(
 1021     V27, V27_H
 1022 );
 1023 
 1024 // Class for 128 bit register v28
 1025 reg_class v28_reg(
 1026     V28, V28_H
 1027 );
 1028 
 1029 // Class for 128 bit register v29
 1030 reg_class v29_reg(
 1031     V29, V29_H
 1032 );
 1033 
 1034 // Class for 128 bit register v30
 1035 reg_class v30_reg(
 1036     V30, V30_H
 1037 );
 1038 
 1039 // Class for 128 bit register v31
 1040 reg_class v31_reg(
 1041     V31, V31_H
 1042 );
 1043 
 1044 // Class for all SVE predicate registers.
 1045 reg_class pr_reg (
 1046     P0,
 1047     P1,
 1048     P2,
 1049     P3,
 1050     P4,
 1051     P5,
 1052     P6,
 1053     // P7, non-allocatable, preserved with all elements preset to TRUE.
 1054     P8,
 1055     P9,
 1056     P10,
 1057     P11,
 1058     P12,
 1059     P13,
 1060     P14,
 1061     P15
 1062 );
 1063 
 1064 // Class for SVE governing predicate registers, which are used
 1065 // to determine the active elements of a predicated instruction.
 1066 reg_class gov_pr (
 1067     P0,
 1068     P1,
 1069     P2,
 1070     P3,
 1071     P4,
 1072     P5,
 1073     P6,
 1074     // P7, non-allocatable, preserved with all elements preset to TRUE.
 1075 );
 1076 
 1077 reg_class p0_reg(P0);
 1078 reg_class p1_reg(P1);
 1079 
 1080 // Singleton class for condition codes
 1081 reg_class int_flags(RFLAGS);
 1082 
 1083 %}
 1084 
 1085 //----------DEFINITION BLOCK---------------------------------------------------
 1086 // Define name --> value mappings to inform the ADLC of an integer valued name
 1087 // Current support includes integer values in the range [0, 0x7FFFFFFF]
 1088 // Format:
 1089 //        int_def  <name>         ( <int_value>, <expression>);
 1090 // Generated Code in ad_<arch>.hpp
 1091 //        #define  <name>   (<expression>)
 1092 //        // value == <int_value>
 1093 // Generated code in ad_<arch>.cpp adlc_verification()
 1094 //        assert( <name> == <int_value>, "Expect (<expression>) to equal <int_value>");
 1095 //
 1096 
 1097 // we follow the ppc-aix port in using a simple cost model which ranks
 1098 // register operations as cheap, memory ops as more expensive and
 1099 // branches as most expensive. the first two have a low as well as a
 1100 // normal cost. huge cost appears to be a way of saying don't do
 1101 // something
 1102 
 1103 definitions %{
 1104   // The default cost (of a register move instruction).
 1105   int_def INSN_COST            (    100,     100);
 1106   int_def BRANCH_COST          (    200,     2 * INSN_COST);
 1107   int_def CALL_COST            (    200,     2 * INSN_COST);
 1108   int_def VOLATILE_REF_COST    (   1000,     10 * INSN_COST);
 1109 %}
 1110 
 1111 
 1112 //----------SOURCE BLOCK-------------------------------------------------------
 1113 // This is a block of C++ code which provides values, functions, and
 1114 // definitions necessary in the rest of the architecture description
 1115 
 1116 source_hpp %{
 1117 
 1118 #include "asm/macroAssembler.hpp"
 1119 #include "gc/shared/barrierSetAssembler.hpp"
 1120 #include "gc/shared/cardTable.hpp"
 1121 #include "gc/shared/cardTableBarrierSet.hpp"
 1122 #include "gc/shared/collectedHeap.hpp"
 1123 #include "opto/addnode.hpp"
 1124 #include "opto/convertnode.hpp"
 1125 #include "runtime/objectMonitor.hpp"
 1126 
 1127 extern RegMask _ANY_REG32_mask;
 1128 extern RegMask _ANY_REG_mask;
 1129 extern RegMask _PTR_REG_mask;
 1130 extern RegMask _NO_SPECIAL_REG32_mask;
 1131 extern RegMask _NO_SPECIAL_REG_mask;
 1132 extern RegMask _NO_SPECIAL_PTR_REG_mask;
 1133 extern RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
 1134 
 1135 class CallStubImpl {
 1136 
 1137   //--------------------------------------------------------------
 1138   //---<  Used for optimization in Compile::shorten_branches  >---
 1139   //--------------------------------------------------------------
 1140 
 1141  public:
 1142   // Size of call trampoline stub.
 1143   static uint size_call_trampoline() {
 1144     return 0; // no call trampolines on this platform
 1145   }
 1146 
 1147   // number of relocations needed by a call trampoline stub
 1148   static uint reloc_call_trampoline() {
 1149     return 0; // no call trampolines on this platform
 1150   }
 1151 };
 1152 
 1153 class HandlerImpl {
 1154 
 1155  public:
 1156 
 1157   static int emit_exception_handler(C2_MacroAssembler *masm);
 1158   static int emit_deopt_handler(C2_MacroAssembler* masm);
 1159 
 1160   static uint size_exception_handler() {
 1161     return MacroAssembler::far_codestub_branch_size();
 1162   }
 1163 
 1164   static uint size_deopt_handler() {
 1165     // count one adr and one far branch instruction
 1166     return NativeInstruction::instruction_size + MacroAssembler::far_codestub_branch_size();
 1167   }
 1168 };
 1169 
 1170 class Node::PD {
 1171 public:
 1172   enum NodeFlags {
 1173     _last_flag = Node::_last_flag
 1174   };
 1175 };
 1176 
 1177   bool is_CAS(int opcode, bool maybe_volatile);
 1178 
 1179   // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
 1180 
 1181   bool unnecessary_acquire(const Node *barrier);
 1182   bool needs_acquiring_load(const Node *load);
 1183 
 1184   // predicates controlling emit of str<x>/stlr<x> and associated dmbs
 1185 
 1186   bool unnecessary_release(const Node *barrier);
 1187   bool unnecessary_volatile(const Node *barrier);
 1188   bool needs_releasing_store(const Node *store);
 1189 
 1190   // predicate controlling translation of CompareAndSwapX
 1191   bool needs_acquiring_load_exclusive(const Node *load);
 1192 
 1193   // predicate controlling addressing modes
 1194   bool size_fits_all_mem_uses(AddPNode* addp, int shift);
 1195 
 1196   // Convert BootTest condition to Assembler condition.
 1197   // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
 1198   Assembler::Condition to_assembler_cond(BoolTest::mask cond);
 1199 %}
 1200 
 1201 source %{
 1202 
 1203   // Derived RegMask with conditionally allocatable registers
 1204 
 1205   void PhaseOutput::pd_perform_mach_node_analysis() {
 1206   }
 1207 
 1208   int MachNode::pd_alignment_required() const {
 1209     return 1;
 1210   }
 1211 
 1212   int MachNode::compute_padding(int current_offset) const {
 1213     return 0;
 1214   }
 1215 
 1216   RegMask _ANY_REG32_mask;
 1217   RegMask _ANY_REG_mask;
 1218   RegMask _PTR_REG_mask;
 1219   RegMask _NO_SPECIAL_REG32_mask;
 1220   RegMask _NO_SPECIAL_REG_mask;
 1221   RegMask _NO_SPECIAL_PTR_REG_mask;
 1222   RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
 1223 
 1224   void reg_mask_init() {
 1225     // We derive below RegMask(s) from the ones which are auto-generated from
 1226     // adlc register classes to make AArch64 rheapbase (r27) and rfp (r29)
 1227     // registers conditionally reserved.
 1228 
 1229     _ANY_REG32_mask = _ALL_REG32_mask;
 1230     _ANY_REG32_mask.Remove(OptoReg::as_OptoReg(r31_sp->as_VMReg()));
 1231 
 1232     _ANY_REG_mask = _ALL_REG_mask;
 1233 
 1234     _PTR_REG_mask = _ALL_REG_mask;
 1235 
 1236     _NO_SPECIAL_REG32_mask = _ALL_REG32_mask;
 1237     _NO_SPECIAL_REG32_mask.SUBTRACT(_NON_ALLOCATABLE_REG32_mask);
 1238 
 1239     _NO_SPECIAL_REG_mask = _ALL_REG_mask;
 1240     _NO_SPECIAL_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
 1241 
 1242     _NO_SPECIAL_PTR_REG_mask = _ALL_REG_mask;
 1243     _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
 1244 
 1245     // r27 is not allocatable when compressed oops is on and heapbase is not
 1246     // zero, compressed klass pointers doesn't use r27 after JDK-8234794
 1247     if (UseCompressedOops && (CompressedOops::base() != nullptr)) {
 1248       _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
 1249       _NO_SPECIAL_REG_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
 1250       _NO_SPECIAL_PTR_REG_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
 1251     }
 1252 
 1253     // r29 is not allocatable when PreserveFramePointer is on
 1254     if (PreserveFramePointer) {
 1255       _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
 1256       _NO_SPECIAL_REG_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
 1257       _NO_SPECIAL_PTR_REG_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
 1258     }
 1259 
 1260     _NO_SPECIAL_NO_RFP_PTR_REG_mask = _NO_SPECIAL_PTR_REG_mask;
 1261     _NO_SPECIAL_NO_RFP_PTR_REG_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
 1262   }
 1263 
 1264   // Optimizaton of volatile gets and puts
 1265   // -------------------------------------
 1266   //
 1267   // AArch64 has ldar<x> and stlr<x> instructions which we can safely
 1268   // use to implement volatile reads and writes. For a volatile read
 1269   // we simply need
 1270   //
 1271   //   ldar<x>
 1272   //
 1273   // and for a volatile write we need
 1274   //
 1275   //   stlr<x>
 1276   //
 1277   // Alternatively, we can implement them by pairing a normal
 1278   // load/store with a memory barrier. For a volatile read we need
 1279   //
 1280   //   ldr<x>
 1281   //   dmb ishld
 1282   //
 1283   // for a volatile write
 1284   //
 1285   //   dmb ish
 1286   //   str<x>
 1287   //   dmb ish
 1288   //
 1289   // We can also use ldaxr and stlxr to implement compare and swap CAS
 1290   // sequences. These are normally translated to an instruction
 1291   // sequence like the following
 1292   //
 1293   //   dmb      ish
 1294   // retry:
 1295   //   ldxr<x>   rval raddr
 1296   //   cmp       rval rold
 1297   //   b.ne done
 1298   //   stlxr<x>  rval, rnew, rold
 1299   //   cbnz      rval retry
 1300   // done:
 1301   //   cset      r0, eq
 1302   //   dmb ishld
 1303   //
 1304   // Note that the exclusive store is already using an stlxr
 1305   // instruction. That is required to ensure visibility to other
 1306   // threads of the exclusive write (assuming it succeeds) before that
 1307   // of any subsequent writes.
 1308   //
 1309   // The following instruction sequence is an improvement on the above
 1310   //
 1311   // retry:
 1312   //   ldaxr<x>  rval raddr
 1313   //   cmp       rval rold
 1314   //   b.ne done
 1315   //   stlxr<x>  rval, rnew, rold
 1316   //   cbnz      rval retry
 1317   // done:
 1318   //   cset      r0, eq
 1319   //
 1320   // We don't need the leading dmb ish since the stlxr guarantees
 1321   // visibility of prior writes in the case that the swap is
 1322   // successful. Crucially we don't have to worry about the case where
 1323   // the swap is not successful since no valid program should be
 1324   // relying on visibility of prior changes by the attempting thread
 1325   // in the case where the CAS fails.
 1326   //
 1327   // Similarly, we don't need the trailing dmb ishld if we substitute
 1328   // an ldaxr instruction since that will provide all the guarantees we
 1329   // require regarding observation of changes made by other threads
 1330   // before any change to the CAS address observed by the load.
 1331   //
 1332   // In order to generate the desired instruction sequence we need to
 1333   // be able to identify specific 'signature' ideal graph node
 1334   // sequences which i) occur as a translation of a volatile reads or
 1335   // writes or CAS operations and ii) do not occur through any other
 1336   // translation or graph transformation. We can then provide
 1337   // alternative aldc matching rules which translate these node
 1338   // sequences to the desired machine code sequences. Selection of the
 1339   // alternative rules can be implemented by predicates which identify
 1340   // the relevant node sequences.
 1341   //
 1342   // The ideal graph generator translates a volatile read to the node
 1343   // sequence
 1344   //
 1345   //   LoadX[mo_acquire]
 1346   //   MemBarAcquire
 1347   //
 1348   // As a special case when using the compressed oops optimization we
 1349   // may also see this variant
 1350   //
 1351   //   LoadN[mo_acquire]
 1352   //   DecodeN
 1353   //   MemBarAcquire
 1354   //
 1355   // A volatile write is translated to the node sequence
 1356   //
 1357   //   MemBarRelease
 1358   //   StoreX[mo_release] {CardMark}-optional
 1359   //   MemBarVolatile
 1360   //
 1361   // n.b. the above node patterns are generated with a strict
 1362   // 'signature' configuration of input and output dependencies (see
 1363   // the predicates below for exact details). The card mark may be as
 1364   // simple as a few extra nodes or, in a few GC configurations, may
 1365   // include more complex control flow between the leading and
 1366   // trailing memory barriers. However, whatever the card mark
 1367   // configuration these signatures are unique to translated volatile
 1368   // reads/stores -- they will not appear as a result of any other
 1369   // bytecode translation or inlining nor as a consequence of
 1370   // optimizing transforms.
 1371   //
 1372   // We also want to catch inlined unsafe volatile gets and puts and
 1373   // be able to implement them using either ldar<x>/stlr<x> or some
 1374   // combination of ldr<x>/stlr<x> and dmb instructions.
 1375   //
 1376   // Inlined unsafe volatiles puts manifest as a minor variant of the
 1377   // normal volatile put node sequence containing an extra cpuorder
 1378   // membar
 1379   //
 1380   //   MemBarRelease
 1381   //   MemBarCPUOrder
 1382   //   StoreX[mo_release] {CardMark}-optional
 1383   //   MemBarCPUOrder
 1384   //   MemBarVolatile
 1385   //
 1386   // n.b. as an aside, a cpuorder membar is not itself subject to
 1387   // matching and translation by adlc rules.  However, the rule
 1388   // predicates need to detect its presence in order to correctly
 1389   // select the desired adlc rules.
 1390   //
 1391   // Inlined unsafe volatile gets manifest as a slightly different
 1392   // node sequence to a normal volatile get because of the
 1393   // introduction of some CPUOrder memory barriers to bracket the
 1394   // Load. However, but the same basic skeleton of a LoadX feeding a
 1395   // MemBarAcquire, possibly through an optional DecodeN, is still
 1396   // present
 1397   //
 1398   //   MemBarCPUOrder
 1399   //        ||       \\
 1400   //   MemBarCPUOrder LoadX[mo_acquire]
 1401   //        ||            |
 1402   //        ||       {DecodeN} optional
 1403   //        ||       /
 1404   //     MemBarAcquire
 1405   //
 1406   // In this case the acquire membar does not directly depend on the
 1407   // load. However, we can be sure that the load is generated from an
 1408   // inlined unsafe volatile get if we see it dependent on this unique
 1409   // sequence of membar nodes. Similarly, given an acquire membar we
 1410   // can know that it was added because of an inlined unsafe volatile
 1411   // get if it is fed and feeds a cpuorder membar and if its feed
 1412   // membar also feeds an acquiring load.
 1413   //
 1414   // Finally an inlined (Unsafe) CAS operation is translated to the
 1415   // following ideal graph
 1416   //
 1417   //   MemBarRelease
 1418   //   MemBarCPUOrder
 1419   //   CompareAndSwapX {CardMark}-optional
 1420   //   MemBarCPUOrder
 1421   //   MemBarAcquire
 1422   //
 1423   // So, where we can identify these volatile read and write
 1424   // signatures we can choose to plant either of the above two code
 1425   // sequences. For a volatile read we can simply plant a normal
 1426   // ldr<x> and translate the MemBarAcquire to a dmb. However, we can
 1427   // also choose to inhibit translation of the MemBarAcquire and
 1428   // inhibit planting of the ldr<x>, instead planting an ldar<x>.
 1429   //
 1430   // When we recognise a volatile store signature we can choose to
 1431   // plant at a dmb ish as a translation for the MemBarRelease, a
 1432   // normal str<x> and then a dmb ish for the MemBarVolatile.
 1433   // Alternatively, we can inhibit translation of the MemBarRelease
 1434   // and MemBarVolatile and instead plant a simple stlr<x>
 1435   // instruction.
 1436   //
 1437   // when we recognise a CAS signature we can choose to plant a dmb
 1438   // ish as a translation for the MemBarRelease, the conventional
 1439   // macro-instruction sequence for the CompareAndSwap node (which
 1440   // uses ldxr<x>) and then a dmb ishld for the MemBarAcquire.
 1441   // Alternatively, we can elide generation of the dmb instructions
 1442   // and plant the alternative CompareAndSwap macro-instruction
 1443   // sequence (which uses ldaxr<x>).
 1444   //
 1445   // Of course, the above only applies when we see these signature
 1446   // configurations. We still want to plant dmb instructions in any
 1447   // other cases where we may see a MemBarAcquire, MemBarRelease or
 1448   // MemBarVolatile. For example, at the end of a constructor which
 1449   // writes final/volatile fields we will see a MemBarRelease
 1450   // instruction and this needs a 'dmb ish' lest we risk the
 1451   // constructed object being visible without making the
 1452   // final/volatile field writes visible.
 1453   //
 1454   // n.b. the translation rules below which rely on detection of the
 1455   // volatile signatures and insert ldar<x> or stlr<x> are failsafe.
 1456   // If we see anything other than the signature configurations we
 1457   // always just translate the loads and stores to ldr<x> and str<x>
 1458   // and translate acquire, release and volatile membars to the
 1459   // relevant dmb instructions.
 1460   //
 1461 
 1462   // is_CAS(int opcode, bool maybe_volatile)
 1463   //
 1464   // return true if opcode is one of the possible CompareAndSwapX
 1465   // values otherwise false.
 1466 
 1467   bool is_CAS(int opcode, bool maybe_volatile)
 1468   {
 1469     switch(opcode) {
 1470       // We handle these
 1471     case Op_CompareAndSwapI:
 1472     case Op_CompareAndSwapL:
 1473     case Op_CompareAndSwapP:
 1474     case Op_CompareAndSwapN:
 1475     case Op_ShenandoahCompareAndSwapP:
 1476     case Op_ShenandoahCompareAndSwapN:
 1477     case Op_CompareAndSwapB:
 1478     case Op_CompareAndSwapS:
 1479     case Op_GetAndSetI:
 1480     case Op_GetAndSetL:
 1481     case Op_GetAndSetP:
 1482     case Op_GetAndSetN:
 1483     case Op_GetAndAddI:
 1484     case Op_GetAndAddL:
 1485       return true;
 1486     case Op_CompareAndExchangeI:
 1487     case Op_CompareAndExchangeN:
 1488     case Op_CompareAndExchangeB:
 1489     case Op_CompareAndExchangeS:
 1490     case Op_CompareAndExchangeL:
 1491     case Op_CompareAndExchangeP:
 1492     case Op_WeakCompareAndSwapB:
 1493     case Op_WeakCompareAndSwapS:
 1494     case Op_WeakCompareAndSwapI:
 1495     case Op_WeakCompareAndSwapL:
 1496     case Op_WeakCompareAndSwapP:
 1497     case Op_WeakCompareAndSwapN:
 1498     case Op_ShenandoahWeakCompareAndSwapP:
 1499     case Op_ShenandoahWeakCompareAndSwapN:
 1500     case Op_ShenandoahCompareAndExchangeP:
 1501     case Op_ShenandoahCompareAndExchangeN:
 1502       return maybe_volatile;
 1503     default:
 1504       return false;
 1505     }
 1506   }
 1507 
 1508   // helper to determine the maximum number of Phi nodes we may need to
 1509   // traverse when searching from a card mark membar for the merge mem
 1510   // feeding a trailing membar or vice versa
 1511 
 1512 // predicates controlling emit of ldr<x>/ldar<x>
 1513 
 1514 bool unnecessary_acquire(const Node *barrier)
 1515 {
 1516   assert(barrier->is_MemBar(), "expecting a membar");
 1517 
 1518   MemBarNode* mb = barrier->as_MemBar();
 1519 
 1520   if (mb->trailing_load()) {
 1521     return true;
 1522   }
 1523 
 1524   if (mb->trailing_load_store()) {
 1525     Node* load_store = mb->in(MemBarNode::Precedent);
 1526     assert(load_store->is_LoadStore(), "unexpected graph shape");
 1527     return is_CAS(load_store->Opcode(), true);
 1528   }
 1529 
 1530   return false;
 1531 }
 1532 
 1533 bool needs_acquiring_load(const Node *n)
 1534 {
 1535   assert(n->is_Load(), "expecting a load");
 1536   LoadNode *ld = n->as_Load();
 1537   return ld->is_acquire();
 1538 }
 1539 
 1540 bool unnecessary_release(const Node *n)
 1541 {
 1542   assert((n->is_MemBar() &&
 1543           n->Opcode() == Op_MemBarRelease),
 1544          "expecting a release membar");
 1545 
 1546   MemBarNode *barrier = n->as_MemBar();
 1547   if (!barrier->leading()) {
 1548     return false;
 1549   } else {
 1550     Node* trailing = barrier->trailing_membar();
 1551     MemBarNode* trailing_mb = trailing->as_MemBar();
 1552     assert(trailing_mb->trailing(), "Not a trailing membar?");
 1553     assert(trailing_mb->leading_membar() == n, "inconsistent leading/trailing membars");
 1554 
 1555     Node* mem = trailing_mb->in(MemBarNode::Precedent);
 1556     if (mem->is_Store()) {
 1557       assert(mem->as_Store()->is_release(), "");
 1558       assert(trailing_mb->Opcode() == Op_MemBarVolatile, "");
 1559       return true;
 1560     } else {
 1561       assert(mem->is_LoadStore(), "");
 1562       assert(trailing_mb->Opcode() == Op_MemBarAcquire, "");
 1563       return is_CAS(mem->Opcode(), true);
 1564     }
 1565   }
 1566   return false;
 1567 }
 1568 
 1569 bool unnecessary_volatile(const Node *n)
 1570 {
 1571   // assert n->is_MemBar();
 1572   MemBarNode *mbvol = n->as_MemBar();
 1573 
 1574   bool release = mbvol->trailing_store();
 1575   assert(!release || (mbvol->in(MemBarNode::Precedent)->is_Store() && mbvol->in(MemBarNode::Precedent)->as_Store()->is_release()), "");
 1576 #ifdef ASSERT
 1577   if (release) {
 1578     Node* leading = mbvol->leading_membar();
 1579     assert(leading->Opcode() == Op_MemBarRelease, "");
 1580     assert(leading->as_MemBar()->leading_store(), "");
 1581     assert(leading->as_MemBar()->trailing_membar() == mbvol, "");
 1582   }
 1583 #endif
 1584 
 1585   return release;
 1586 }
 1587 
 1588 // predicates controlling emit of str<x>/stlr<x>
 1589 
 1590 bool needs_releasing_store(const Node *n)
 1591 {
 1592   // assert n->is_Store();
 1593   StoreNode *st = n->as_Store();
 1594   return st->trailing_membar() != nullptr;
 1595 }
 1596 
 1597 // predicate controlling translation of CAS
 1598 //
 1599 // returns true if CAS needs to use an acquiring load otherwise false
 1600 
 1601 bool needs_acquiring_load_exclusive(const Node *n)
 1602 {
 1603   assert(is_CAS(n->Opcode(), true), "expecting a compare and swap");
 1604   LoadStoreNode* ldst = n->as_LoadStore();
 1605   if (is_CAS(n->Opcode(), false)) {
 1606     assert(ldst->trailing_membar() != nullptr, "expected trailing membar");
 1607   } else {
 1608     return ldst->trailing_membar() != nullptr;
 1609   }
 1610 
 1611   // so we can just return true here
 1612   return true;
 1613 }
 1614 
 1615 #define __ masm->
 1616 
 1617 // advance declarations for helper functions to convert register
 1618 // indices to register objects
 1619 
 1620 // the ad file has to provide implementations of certain methods
 1621 // expected by the generic code
 1622 //
 1623 // REQUIRED FUNCTIONALITY
 1624 
 1625 //=============================================================================
 1626 
 1627 // !!!!! Special hack to get all types of calls to specify the byte offset
 1628 //       from the start of the call to the point where the return address
 1629 //       will point.
 1630 
 1631 int MachCallStaticJavaNode::ret_addr_offset()
 1632 {
 1633   // call should be a simple bl
 1634   int off = 4;
 1635   return off;
 1636 }
 1637 
 1638 int MachCallDynamicJavaNode::ret_addr_offset()
 1639 {
 1640   return 16; // movz, movk, movk, bl
 1641 }
 1642 
 1643 int MachCallRuntimeNode::ret_addr_offset() {
 1644   // for generated stubs the call will be
 1645   //   bl(addr)
 1646   // or with far branches
 1647   //   bl(trampoline_stub)
 1648   // for real runtime callouts it will be six instructions
 1649   // see aarch64_enc_java_to_runtime
 1650   //   adr(rscratch2, retaddr)
 1651   //   str(rscratch2, Address(rthread, JavaThread::last_Java_pc_offset()));
 1652   //   lea(rscratch1, RuntimeAddress(addr)
 1653   //   blr(rscratch1)
 1654   CodeBlob *cb = CodeCache::find_blob(_entry_point);
 1655   if (cb) {
 1656     return 1 * NativeInstruction::instruction_size;
 1657   } else if (_entry_point == nullptr) {
 1658     // See CallLeafNoFPIndirect
 1659     return 1 * NativeInstruction::instruction_size;
 1660   } else {
 1661     return 6 * NativeInstruction::instruction_size;
 1662   }
 1663 }
 1664 
 1665 //=============================================================================
 1666 
 1667 #ifndef PRODUCT
 1668 void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 1669   st->print("BREAKPOINT");
 1670 }
 1671 #endif
 1672 
 1673 void MachBreakpointNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
 1674   __ brk(0);
 1675 }
 1676 
 1677 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
 1678   return MachNode::size(ra_);
 1679 }
 1680 
 1681 //=============================================================================
 1682 
 1683 #ifndef PRODUCT
 1684   void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
 1685     st->print("nop \t# %d bytes pad for loops and calls", _count);
 1686   }
 1687 #endif
 1688 
 1689   void MachNopNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc*) const {
 1690     for (int i = 0; i < _count; i++) {
 1691       __ nop();
 1692     }
 1693   }
 1694 
 1695   uint MachNopNode::size(PhaseRegAlloc*) const {
 1696     return _count * NativeInstruction::instruction_size;
 1697   }
 1698 
 1699 //=============================================================================
 1700 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
 1701 
 1702 int ConstantTable::calculate_table_base_offset() const {
 1703   return 0;  // absolute addressing, no offset
 1704 }
 1705 
 1706 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
 1707 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
 1708   ShouldNotReachHere();
 1709 }
 1710 
 1711 void MachConstantBaseNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const {
 1712   // Empty encoding
 1713 }
 1714 
 1715 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
 1716   return 0;
 1717 }
 1718 
 1719 #ifndef PRODUCT
 1720 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
 1721   st->print("-- \t// MachConstantBaseNode (empty encoding)");
 1722 }
 1723 #endif
 1724 
 1725 #ifndef PRODUCT
 1726 void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 1727   Compile* C = ra_->C;
 1728 
 1729   int framesize = C->output()->frame_slots() << LogBytesPerInt;
 1730 
 1731   if (C->output()->need_stack_bang(framesize))
 1732     st->print("# stack bang size=%d\n\t", framesize);
 1733 
 1734   if (VM_Version::use_rop_protection()) {
 1735     st->print("ldr  zr, [lr]\n\t");
 1736     st->print("paciaz\n\t");
 1737   }
 1738   if (framesize < ((1 << 9) + 2 * wordSize)) {
 1739     st->print("sub  sp, sp, #%d\n\t", framesize);
 1740     st->print("stp  rfp, lr, [sp, #%d]", framesize - 2 * wordSize);
 1741     if (PreserveFramePointer) st->print("\n\tadd  rfp, sp, #%d", framesize - 2 * wordSize);
 1742   } else {
 1743     st->print("stp  lr, rfp, [sp, #%d]!\n\t", -(2 * wordSize));
 1744     if (PreserveFramePointer) st->print("mov  rfp, sp\n\t");
 1745     st->print("mov  rscratch1, #%d\n\t", framesize - 2 * wordSize);
 1746     st->print("sub  sp, sp, rscratch1");
 1747   }
 1748   if (C->stub_function() == nullptr) {
 1749     st->print("\n\t");
 1750     st->print("ldr  rscratch1, [guard]\n\t");
 1751     st->print("dmb ishld\n\t");
 1752     st->print("ldr  rscratch2, [rthread, #thread_disarmed_guard_value_offset]\n\t");
 1753     st->print("cmp  rscratch1, rscratch2\n\t");
 1754     st->print("b.eq skip");
 1755     st->print("\n\t");
 1756     st->print("blr #nmethod_entry_barrier_stub\n\t");
 1757     st->print("b skip\n\t");
 1758     st->print("guard: int\n\t");
 1759     st->print("\n\t");
 1760     st->print("skip:\n\t");
 1761   }
 1762 }
 1763 #endif
 1764 
 1765 void MachPrologNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
 1766   Compile* C = ra_->C;
 1767 
 1768   // insert a nop at the start of the prolog so we can patch in a
 1769   // branch if we need to invalidate the method later
 1770   __ nop();
 1771 
 1772   __ verified_entry(C, 0);
 1773 
 1774   if (C->stub_function() == nullptr) {
 1775     __ entry_barrier();
 1776   }
 1777 
 1778   if (!Compile::current()->output()->in_scratch_emit_size()) {
 1779     __ bind(*_verified_entry);
 1780   }
 1781 
 1782   if (VerifyStackAtCalls) {
 1783     Unimplemented();
 1784   }
 1785 
 1786   C->output()->set_frame_complete(__ offset());
 1787 
 1788   if (C->has_mach_constant_base_node()) {
 1789     // NOTE: We set the table base offset here because users might be
 1790     // emitted before MachConstantBaseNode.
 1791     ConstantTable& constant_table = C->output()->constant_table();
 1792     constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
 1793   }
 1794 }
 1795 
 1796 int MachPrologNode::reloc() const
 1797 {
 1798   return 0;
 1799 }
 1800 
 1801 //=============================================================================
 1802 
 1803 #ifndef PRODUCT
 1804 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 1805   Compile* C = ra_->C;
 1806   int framesize = C->output()->frame_slots() << LogBytesPerInt;
 1807 
 1808   st->print("# pop frame %d\n\t",framesize);
 1809 
 1810   if (framesize == 0) {
 1811     st->print("ldp  lr, rfp, [sp],#%d\n\t", (2 * wordSize));
 1812   } else if (framesize < ((1 << 9) + 2 * wordSize)) {
 1813     st->print("ldp  lr, rfp, [sp,#%d]\n\t", framesize - 2 * wordSize);
 1814     st->print("add  sp, sp, #%d\n\t", framesize);
 1815   } else {
 1816     st->print("mov  rscratch1, #%d\n\t", framesize - 2 * wordSize);
 1817     st->print("add  sp, sp, rscratch1\n\t");
 1818     st->print("ldp  lr, rfp, [sp],#%d\n\t", (2 * wordSize));
 1819   }
 1820   if (VM_Version::use_rop_protection()) {
 1821     st->print("autiaz\n\t");
 1822     st->print("ldr  zr, [lr]\n\t");
 1823   }
 1824 
 1825   if (do_polling() && C->is_method_compilation()) {
 1826     st->print("# test polling word\n\t");
 1827     st->print("ldr  rscratch1, [rthread],#%d\n\t", in_bytes(JavaThread::polling_word_offset()));
 1828     st->print("cmp  sp, rscratch1\n\t");
 1829     st->print("bhi #slow_path");
 1830   }
 1831 }
 1832 #endif
 1833 
 1834 void MachEpilogNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
 1835   Compile* C = ra_->C;
 1836   int framesize = C->output()->frame_slots() << LogBytesPerInt;
 1837 
 1838   __ remove_frame(framesize, C->needs_stack_repair());
 1839 
 1840   if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
 1841     __ reserved_stack_check();
 1842   }
 1843 
 1844   if (do_polling() && C->is_method_compilation()) {
 1845     Label dummy_label;
 1846     Label* code_stub = &dummy_label;
 1847     if (!C->output()->in_scratch_emit_size()) {
 1848       C2SafepointPollStub* stub = new (C->comp_arena()) C2SafepointPollStub(__ offset());
 1849       C->output()->add_stub(stub);
 1850       code_stub = &stub->entry();
 1851     }
 1852     __ relocate(relocInfo::poll_return_type);
 1853     __ safepoint_poll(*code_stub, true /* at_return */, false /* acquire */, true /* in_nmethod */);
 1854   }
 1855 }
 1856 
 1857 int MachEpilogNode::reloc() const {
 1858   // Return number of relocatable values contained in this instruction.
 1859   return 1; // 1 for polling page.
 1860 }
 1861 
 1862 const Pipeline * MachEpilogNode::pipeline() const {
 1863   return MachNode::pipeline_class();
 1864 }
 1865 
 1866 //=============================================================================
 1867 
 1868 static enum RC rc_class(OptoReg::Name reg) {
 1869 
 1870   if (reg == OptoReg::Bad) {
 1871     return rc_bad;
 1872   }
 1873 
 1874   // we have 32 int registers * 2 halves
 1875   int slots_of_int_registers = Register::number_of_registers * Register::max_slots_per_register;
 1876 
 1877   if (reg < slots_of_int_registers) {
 1878     return rc_int;
 1879   }
 1880 
 1881   // we have 32 float register * 8 halves
 1882   int slots_of_float_registers = FloatRegister::number_of_registers * FloatRegister::max_slots_per_register;
 1883   if (reg < slots_of_int_registers + slots_of_float_registers) {
 1884     return rc_float;
 1885   }
 1886 
 1887   int slots_of_predicate_registers = PRegister::number_of_registers * PRegister::max_slots_per_register;
 1888   if (reg < slots_of_int_registers + slots_of_float_registers + slots_of_predicate_registers) {
 1889     return rc_predicate;
 1890   }
 1891 
 1892   // Between predicate regs & stack is the flags.
 1893   assert(OptoReg::is_stack(reg), "blow up if spilling flags");
 1894 
 1895   return rc_stack;
 1896 }
 1897 
 1898 uint MachSpillCopyNode::implementation(C2_MacroAssembler *masm, PhaseRegAlloc *ra_, bool do_size, outputStream *st) const {
 1899   Compile* C = ra_->C;
 1900 
 1901   // Get registers to move.
 1902   OptoReg::Name src_hi = ra_->get_reg_second(in(1));
 1903   OptoReg::Name src_lo = ra_->get_reg_first(in(1));
 1904   OptoReg::Name dst_hi = ra_->get_reg_second(this);
 1905   OptoReg::Name dst_lo = ra_->get_reg_first(this);
 1906 
 1907   enum RC src_hi_rc = rc_class(src_hi);
 1908   enum RC src_lo_rc = rc_class(src_lo);
 1909   enum RC dst_hi_rc = rc_class(dst_hi);
 1910   enum RC dst_lo_rc = rc_class(dst_lo);
 1911 
 1912   assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
 1913 
 1914   if (src_hi != OptoReg::Bad && !bottom_type()->isa_vectmask()) {
 1915     assert((src_lo&1)==0 && src_lo+1==src_hi &&
 1916            (dst_lo&1)==0 && dst_lo+1==dst_hi,
 1917            "expected aligned-adjacent pairs");
 1918   }
 1919 
 1920   if (src_lo == dst_lo && src_hi == dst_hi) {
 1921     return 0;            // Self copy, no move.
 1922   }
 1923 
 1924   bool is64 = (src_lo & 1) == 0 && src_lo + 1 == src_hi &&
 1925               (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi;
 1926   int src_offset = ra_->reg2offset(src_lo);
 1927   int dst_offset = ra_->reg2offset(dst_lo);
 1928 
 1929   if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
 1930     uint ireg = ideal_reg();
 1931     if (ireg == Op_VecA && masm) {
 1932       int sve_vector_reg_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
 1933       if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
 1934         // stack->stack
 1935         __ spill_copy_sve_vector_stack_to_stack(src_offset, dst_offset,
 1936                                                 sve_vector_reg_size_in_bytes);
 1937       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
 1938         __ spill_sve_vector(as_FloatRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
 1939                             sve_vector_reg_size_in_bytes);
 1940       } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
 1941         __ unspill_sve_vector(as_FloatRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
 1942                               sve_vector_reg_size_in_bytes);
 1943       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
 1944         __ sve_orr(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 1945                    as_FloatRegister(Matcher::_regEncode[src_lo]),
 1946                    as_FloatRegister(Matcher::_regEncode[src_lo]));
 1947       } else {
 1948         ShouldNotReachHere();
 1949       }
 1950     } else if (masm) {
 1951       assert(ireg == Op_VecD || ireg == Op_VecX, "must be 64 bit or 128 bit vector");
 1952       assert((src_lo_rc != rc_int && dst_lo_rc != rc_int), "sanity");
 1953       if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
 1954         // stack->stack
 1955         assert((src_offset & 7) == 0 && (dst_offset & 7) == 0, "unaligned stack offset");
 1956         if (ireg == Op_VecD) {
 1957           __ unspill(rscratch1, true, src_offset);
 1958           __ spill(rscratch1, true, dst_offset);
 1959         } else {
 1960           __ spill_copy128(src_offset, dst_offset);
 1961         }
 1962       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
 1963         __ mov(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 1964                ireg == Op_VecD ? __ T8B : __ T16B,
 1965                as_FloatRegister(Matcher::_regEncode[src_lo]));
 1966       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
 1967         __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
 1968                  ireg == Op_VecD ? __ D : __ Q,
 1969                  ra_->reg2offset(dst_lo));
 1970       } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
 1971         __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 1972                    ireg == Op_VecD ? __ D : __ Q,
 1973                    ra_->reg2offset(src_lo));
 1974       } else {
 1975         ShouldNotReachHere();
 1976       }
 1977     }
 1978   } else if (masm) {
 1979     switch (src_lo_rc) {
 1980     case rc_int:
 1981       if (dst_lo_rc == rc_int) {  // gpr --> gpr copy
 1982         if (is64) {
 1983             __ mov(as_Register(Matcher::_regEncode[dst_lo]),
 1984                    as_Register(Matcher::_regEncode[src_lo]));
 1985         } else {
 1986             __ movw(as_Register(Matcher::_regEncode[dst_lo]),
 1987                     as_Register(Matcher::_regEncode[src_lo]));
 1988         }
 1989       } else if (dst_lo_rc == rc_float) { // gpr --> fpr copy
 1990         if (is64) {
 1991             __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 1992                      as_Register(Matcher::_regEncode[src_lo]));
 1993         } else {
 1994             __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 1995                      as_Register(Matcher::_regEncode[src_lo]));
 1996         }
 1997       } else {                    // gpr --> stack spill
 1998         assert(dst_lo_rc == rc_stack, "spill to bad register class");
 1999         __ spill(as_Register(Matcher::_regEncode[src_lo]), is64, dst_offset);
 2000       }
 2001       break;
 2002     case rc_float:
 2003       if (dst_lo_rc == rc_int) {  // fpr --> gpr copy
 2004         if (is64) {
 2005             __ fmovd(as_Register(Matcher::_regEncode[dst_lo]),
 2006                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2007         } else {
 2008             __ fmovs(as_Register(Matcher::_regEncode[dst_lo]),
 2009                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2010         }
 2011       } else if (dst_lo_rc == rc_float) { // fpr --> fpr copy
 2012         if (is64) {
 2013             __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2014                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2015         } else {
 2016             __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2017                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2018         }
 2019       } else {                    // fpr --> stack spill
 2020         assert(dst_lo_rc == rc_stack, "spill to bad register class");
 2021         __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
 2022                  is64 ? __ D : __ S, dst_offset);
 2023       }
 2024       break;
 2025     case rc_stack:
 2026       if (dst_lo_rc == rc_int) {  // stack --> gpr load
 2027         __ unspill(as_Register(Matcher::_regEncode[dst_lo]), is64, src_offset);
 2028       } else if (dst_lo_rc == rc_float) { // stack --> fpr load
 2029         __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2030                    is64 ? __ D : __ S, src_offset);
 2031       } else if (dst_lo_rc == rc_predicate) {
 2032         __ unspill_sve_predicate(as_PRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
 2033                                  Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
 2034       } else {                    // stack --> stack copy
 2035         assert(dst_lo_rc == rc_stack, "spill to bad register class");
 2036         if (ideal_reg() == Op_RegVectMask) {
 2037           __ spill_copy_sve_predicate_stack_to_stack(src_offset, dst_offset,
 2038                                                      Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
 2039         } else {
 2040           __ unspill(rscratch1, is64, src_offset);
 2041           __ spill(rscratch1, is64, dst_offset);
 2042         }
 2043       }
 2044       break;
 2045     case rc_predicate:
 2046       if (dst_lo_rc == rc_predicate) {
 2047         __ sve_mov(as_PRegister(Matcher::_regEncode[dst_lo]), as_PRegister(Matcher::_regEncode[src_lo]));
 2048       } else if (dst_lo_rc == rc_stack) {
 2049         __ spill_sve_predicate(as_PRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
 2050                                Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
 2051       } else {
 2052         assert(false, "bad src and dst rc_class combination.");
 2053         ShouldNotReachHere();
 2054       }
 2055       break;
 2056     default:
 2057       assert(false, "bad rc_class for spill");
 2058       ShouldNotReachHere();
 2059     }
 2060   }
 2061 
 2062   if (st) {
 2063     st->print("spill ");
 2064     if (src_lo_rc == rc_stack) {
 2065       st->print("[sp, #%d] -> ", ra_->reg2offset(src_lo));
 2066     } else {
 2067       st->print("%s -> ", Matcher::regName[src_lo]);
 2068     }
 2069     if (dst_lo_rc == rc_stack) {
 2070       st->print("[sp, #%d]", ra_->reg2offset(dst_lo));
 2071     } else {
 2072       st->print("%s", Matcher::regName[dst_lo]);
 2073     }
 2074     if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
 2075       int vsize = 0;
 2076       switch (ideal_reg()) {
 2077       case Op_VecD:
 2078         vsize = 64;
 2079         break;
 2080       case Op_VecX:
 2081         vsize = 128;
 2082         break;
 2083       case Op_VecA:
 2084         vsize = Matcher::scalable_vector_reg_size(T_BYTE) * 8;
 2085         break;
 2086       default:
 2087         assert(false, "bad register type for spill");
 2088         ShouldNotReachHere();
 2089       }
 2090       st->print("\t# vector spill size = %d", vsize);
 2091     } else if (ideal_reg() == Op_RegVectMask) {
 2092       assert(Matcher::supports_scalable_vector(), "bad register type for spill");
 2093       int vsize = Matcher::scalable_predicate_reg_slots() * 32;
 2094       st->print("\t# predicate spill size = %d", vsize);
 2095     } else {
 2096       st->print("\t# spill size = %d", is64 ? 64 : 32);
 2097     }
 2098   }
 2099 
 2100   return 0;
 2101 
 2102 }
 2103 
 2104 #ifndef PRODUCT
 2105 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 2106   if (!ra_)
 2107     st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
 2108   else
 2109     implementation(nullptr, ra_, false, st);
 2110 }
 2111 #endif
 2112 
 2113 void MachSpillCopyNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
 2114   implementation(masm, ra_, false, nullptr);
 2115 }
 2116 
 2117 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
 2118   return MachNode::size(ra_);
 2119 }
 2120 
 2121 //=============================================================================
 2122 
 2123 #ifndef PRODUCT
 2124 void BoxLockNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 2125   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 2126   int reg = ra_->get_reg_first(this);
 2127   st->print("add %s, rsp, #%d]\t# box lock",
 2128             Matcher::regName[reg], offset);
 2129 }
 2130 #endif
 2131 
 2132 void BoxLockNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
 2133   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 2134   int reg    = ra_->get_encode(this);
 2135 
 2136   // This add will handle any 24-bit signed offset. 24 bits allows an
 2137   // 8 megabyte stack frame.
 2138   __ add(as_Register(reg), sp, offset);
 2139 }
 2140 
 2141 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
 2142   // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_).
 2143   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 2144 
 2145   if (Assembler::operand_valid_for_add_sub_immediate(offset)) {
 2146     return NativeInstruction::instruction_size;
 2147   } else {
 2148     return 2 * NativeInstruction::instruction_size;
 2149   }
 2150 }
 2151 
 2152 ///=============================================================================
 2153 #ifndef PRODUCT
 2154 void MachVEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 2155 {
 2156   st->print_cr("# MachVEPNode");
 2157   if (!_verified) {
 2158     st->print_cr("\t load_class");
 2159   } else {
 2160     st->print_cr("\t unpack_inline_arg");
 2161   }
 2162 }
 2163 #endif
 2164 
 2165 void MachVEPNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc* ra_) const
 2166 {
 2167   if (!_verified) {
 2168     __ ic_check(1);
 2169   } else {
 2170     // insert a nop at the start of the prolog so we can patch in a
 2171     // branch if we need to invalidate the method later
 2172     __ nop();
 2173 
 2174     // TODO 8284443 Avoid creation of temporary frame
 2175     if (ra_->C->stub_function() == nullptr) {
 2176       __ verified_entry(ra_->C, 0);
 2177       __ entry_barrier();
 2178       int framesize = ra_->C->output()->frame_slots() << LogBytesPerInt;
 2179       __ remove_frame(framesize, false);
 2180     }
 2181     // Unpack inline type args passed as oop and then jump to
 2182     // the verified entry point (skipping the unverified entry).
 2183     int sp_inc = __ unpack_inline_args(ra_->C, _receiver_only);
 2184     // Emit code for verified entry and save increment for stack repair on return
 2185     __ verified_entry(ra_->C, sp_inc);
 2186     if (Compile::current()->output()->in_scratch_emit_size()) {
 2187       Label dummy_verified_entry;
 2188       __ b(dummy_verified_entry);
 2189     } else {
 2190       __ b(*_verified_entry);
 2191     }
 2192   }
 2193 }
 2194 
 2195 //=============================================================================
 2196 #ifndef PRODUCT
 2197 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 2198 {
 2199   st->print_cr("# MachUEPNode");
 2200   if (UseCompressedClassPointers) {
 2201     st->print_cr("\tldrw rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
 2202     st->print_cr("\tldrw r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
 2203     st->print_cr("\tcmpw rscratch1, r10");
 2204   } else {
 2205     st->print_cr("\tldr rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
 2206     st->print_cr("\tldr r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
 2207     st->print_cr("\tcmp rscratch1, r10");
 2208   }
 2209   st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
 2210 }
 2211 #endif
 2212 
 2213 void MachUEPNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
 2214 {
 2215   __ ic_check(InteriorEntryAlignment);
 2216 }
 2217 
 2218 // REQUIRED EMIT CODE
 2219 
 2220 //=============================================================================
 2221 
 2222 // Emit exception handler code.
 2223 int HandlerImpl::emit_exception_handler(C2_MacroAssembler* masm)
 2224 {
 2225   // mov rscratch1 #exception_blob_entry_point
 2226   // br rscratch1
 2227   // Note that the code buffer's insts_mark is always relative to insts.
 2228   // That's why we must use the macroassembler to generate a handler.
 2229   address base = __ start_a_stub(size_exception_handler());
 2230   if (base == nullptr) {
 2231     ciEnv::current()->record_failure("CodeCache is full");
 2232     return 0;  // CodeBuffer::expand failed
 2233   }
 2234   int offset = __ offset();
 2235   __ far_jump(RuntimeAddress(OptoRuntime::exception_blob()->entry_point()));
 2236   assert(__ offset() - offset <= (int) size_exception_handler(), "overflow");
 2237   __ end_a_stub();
 2238   return offset;
 2239 }
 2240 
 2241 // Emit deopt handler code.
 2242 int HandlerImpl::emit_deopt_handler(C2_MacroAssembler* masm)
 2243 {
 2244   // Note that the code buffer's insts_mark is always relative to insts.
 2245   // That's why we must use the macroassembler to generate a handler.
 2246   address base = __ start_a_stub(size_deopt_handler());
 2247   if (base == nullptr) {
 2248     ciEnv::current()->record_failure("CodeCache is full");
 2249     return 0;  // CodeBuffer::expand failed
 2250   }
 2251   int offset = __ offset();
 2252 
 2253   __ adr(lr, __ pc());
 2254   __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
 2255 
 2256   assert(__ offset() - offset == (int) size_deopt_handler(), "overflow");
 2257   __ end_a_stub();
 2258   return offset;
 2259 }
 2260 
 2261 // REQUIRED MATCHER CODE
 2262 
 2263 //=============================================================================
 2264 
 2265 bool Matcher::match_rule_supported(int opcode) {
 2266   if (!has_match_rule(opcode))
 2267     return false;
 2268 
 2269   switch (opcode) {
 2270     case Op_OnSpinWait:
 2271       return VM_Version::supports_on_spin_wait();
 2272     case Op_CacheWB:
 2273     case Op_CacheWBPreSync:
 2274     case Op_CacheWBPostSync:
 2275       if (!VM_Version::supports_data_cache_line_flush()) {
 2276         return false;
 2277       }
 2278       break;
 2279     case Op_ExpandBits:
 2280     case Op_CompressBits:
 2281       if (!VM_Version::supports_svebitperm()) {
 2282         return false;
 2283       }
 2284       break;
 2285     case Op_FmaF:
 2286     case Op_FmaD:
 2287     case Op_FmaVF:
 2288     case Op_FmaVD:
 2289       if (!UseFMA) {
 2290         return false;
 2291       }
 2292       break;
 2293     case Op_FmaHF:
 2294       // UseFMA flag also needs to be checked along with FEAT_FP16
 2295       if (!UseFMA || !is_feat_fp16_supported()) {
 2296         return false;
 2297       }
 2298       break;
 2299     case Op_AddHF:
 2300     case Op_SubHF:
 2301     case Op_MulHF:
 2302     case Op_DivHF:
 2303     case Op_MinHF:
 2304     case Op_MaxHF:
 2305     case Op_SqrtHF:
 2306       // Half-precision floating point scalar operations require FEAT_FP16
 2307       // to be available. FEAT_FP16 is enabled if both "fphp" and "asimdhp"
 2308       // features are supported.
 2309       if (!is_feat_fp16_supported()) {
 2310         return false;
 2311       }
 2312       break;
 2313   }
 2314 
 2315   return true; // Per default match rules are supported.
 2316 }
 2317 
 2318 const RegMask* Matcher::predicate_reg_mask(void) {
 2319   return &_PR_REG_mask;
 2320 }
 2321 
 2322 bool Matcher::supports_vector_calling_convention(void) {
 2323   return EnableVectorSupport;
 2324 }
 2325 
 2326 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
 2327   assert(EnableVectorSupport, "sanity");
 2328   int lo = V0_num;
 2329   int hi = V0_H_num;
 2330   if (ideal_reg == Op_VecX || ideal_reg == Op_VecA) {
 2331     hi = V0_K_num;
 2332   }
 2333   return OptoRegPair(hi, lo);
 2334 }
 2335 
 2336 // Is this branch offset short enough that a short branch can be used?
 2337 //
 2338 // NOTE: If the platform does not provide any short branch variants, then
 2339 //       this method should return false for offset 0.
 2340 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
 2341   // The passed offset is relative to address of the branch.
 2342 
 2343   return (-32768 <= offset && offset < 32768);
 2344 }
 2345 
 2346 // Vector width in bytes.
 2347 int Matcher::vector_width_in_bytes(BasicType bt) {
 2348   // The MaxVectorSize should have been set by detecting SVE max vector register size.
 2349   int size = MIN2((UseSVE > 0) ? (int)FloatRegister::sve_vl_max : (int)FloatRegister::neon_vl, (int)MaxVectorSize);
 2350   // Minimum 2 values in vector
 2351   if (size < 2*type2aelembytes(bt)) size = 0;
 2352   // But never < 4
 2353   if (size < 4) size = 0;
 2354   return size;
 2355 }
 2356 
 2357 // Limits on vector size (number of elements) loaded into vector.
 2358 int Matcher::max_vector_size(const BasicType bt) {
 2359   return vector_width_in_bytes(bt)/type2aelembytes(bt);
 2360 }
 2361 
 2362 int Matcher::min_vector_size(const BasicType bt) {
 2363   int max_size = max_vector_size(bt);
 2364   // Limit the min vector size to 8 bytes.
 2365   int size = 8 / type2aelembytes(bt);
 2366   if (bt == T_BYTE) {
 2367     // To support vector api shuffle/rearrange.
 2368     size = 4;
 2369   } else if (bt == T_BOOLEAN) {
 2370     // To support vector api load/store mask.
 2371     size = 2;
 2372   }
 2373   if (size < 2) size = 2;
 2374   return MIN2(size, max_size);
 2375 }
 2376 
 2377 int Matcher::max_vector_size_auto_vectorization(const BasicType bt) {
 2378   return Matcher::max_vector_size(bt);
 2379 }
 2380 
 2381 // Actual max scalable vector register length.
 2382 int Matcher::scalable_vector_reg_size(const BasicType bt) {
 2383   return Matcher::max_vector_size(bt);
 2384 }
 2385 
 2386 // Vector ideal reg.
 2387 uint Matcher::vector_ideal_reg(int len) {
 2388   if (UseSVE > 0 && FloatRegister::neon_vl < len && len <= FloatRegister::sve_vl_max) {
 2389     return Op_VecA;
 2390   }
 2391   switch(len) {
 2392     // For 16-bit/32-bit mask vector, reuse VecD.
 2393     case  2:
 2394     case  4:
 2395     case  8: return Op_VecD;
 2396     case 16: return Op_VecX;
 2397   }
 2398   ShouldNotReachHere();
 2399   return 0;
 2400 }
 2401 
 2402 MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* generic_opnd, uint ideal_reg, bool is_temp) {
 2403   assert(Matcher::is_generic_vector(generic_opnd), "not generic");
 2404   switch (ideal_reg) {
 2405     case Op_VecA: return new vecAOper();
 2406     case Op_VecD: return new vecDOper();
 2407     case Op_VecX: return new vecXOper();
 2408   }
 2409   ShouldNotReachHere();
 2410   return nullptr;
 2411 }
 2412 
 2413 bool Matcher::is_reg2reg_move(MachNode* m) {
 2414   return false;
 2415 }
 2416 
 2417 bool Matcher::is_generic_vector(MachOper* opnd)  {
 2418   return opnd->opcode() == VREG;
 2419 }
 2420 
 2421 // Return whether or not this register is ever used as an argument.
 2422 // This function is used on startup to build the trampoline stubs in
 2423 // generateOptoStub.  Registers not mentioned will be killed by the VM
 2424 // call in the trampoline, and arguments in those registers not be
 2425 // available to the callee.
 2426 bool Matcher::can_be_java_arg(int reg)
 2427 {
 2428   return
 2429     reg ==  R0_num || reg == R0_H_num ||
 2430     reg ==  R1_num || reg == R1_H_num ||
 2431     reg ==  R2_num || reg == R2_H_num ||
 2432     reg ==  R3_num || reg == R3_H_num ||
 2433     reg ==  R4_num || reg == R4_H_num ||
 2434     reg ==  R5_num || reg == R5_H_num ||
 2435     reg ==  R6_num || reg == R6_H_num ||
 2436     reg ==  R7_num || reg == R7_H_num ||
 2437     reg ==  V0_num || reg == V0_H_num ||
 2438     reg ==  V1_num || reg == V1_H_num ||
 2439     reg ==  V2_num || reg == V2_H_num ||
 2440     reg ==  V3_num || reg == V3_H_num ||
 2441     reg ==  V4_num || reg == V4_H_num ||
 2442     reg ==  V5_num || reg == V5_H_num ||
 2443     reg ==  V6_num || reg == V6_H_num ||
 2444     reg ==  V7_num || reg == V7_H_num;
 2445 }
 2446 
 2447 bool Matcher::is_spillable_arg(int reg)
 2448 {
 2449   return can_be_java_arg(reg);
 2450 }
 2451 
 2452 uint Matcher::int_pressure_limit()
 2453 {
 2454   // JDK-8183543: When taking the number of available registers as int
 2455   // register pressure threshold, the jtreg test:
 2456   // test/hotspot/jtreg/compiler/regalloc/TestC2IntPressure.java
 2457   // failed due to C2 compilation failure with
 2458   // "COMPILE SKIPPED: failed spill-split-recycle sanity check".
 2459   //
 2460   // A derived pointer is live at CallNode and then is flagged by RA
 2461   // as a spilled LRG. Spilling heuristics(Spill-USE) explicitly skip
 2462   // derived pointers and lastly fail to spill after reaching maximum
 2463   // number of iterations. Lowering the default pressure threshold to
 2464   // (_NO_SPECIAL_REG32_mask.Size() minus 1) forces CallNode to become
 2465   // a high register pressure area of the code so that split_DEF can
 2466   // generate DefinitionSpillCopy for the derived pointer.
 2467   uint default_int_pressure_threshold = _NO_SPECIAL_REG32_mask.Size() - 1;
 2468   if (!PreserveFramePointer) {
 2469     // When PreserveFramePointer is off, frame pointer is allocatable,
 2470     // but different from other SOC registers, it is excluded from
 2471     // fatproj's mask because its save type is No-Save. Decrease 1 to
 2472     // ensure high pressure at fatproj when PreserveFramePointer is off.
 2473     // See check_pressure_at_fatproj().
 2474     default_int_pressure_threshold--;
 2475   }
 2476   return (INTPRESSURE == -1) ? default_int_pressure_threshold : INTPRESSURE;
 2477 }
 2478 
 2479 uint Matcher::float_pressure_limit()
 2480 {
 2481   // _FLOAT_REG_mask is generated by adlc from the float_reg register class.
 2482   return (FLOATPRESSURE == -1) ? _FLOAT_REG_mask.Size() : FLOATPRESSURE;
 2483 }
 2484 
 2485 bool Matcher::use_asm_for_ldiv_by_con(jlong divisor) {
 2486   return false;
 2487 }
 2488 
 2489 RegMask Matcher::divI_proj_mask() {
 2490   ShouldNotReachHere();
 2491   return RegMask();
 2492 }
 2493 
 2494 // Register for MODI projection of divmodI.
 2495 RegMask Matcher::modI_proj_mask() {
 2496   ShouldNotReachHere();
 2497   return RegMask();
 2498 }
 2499 
 2500 // Register for DIVL projection of divmodL.
 2501 RegMask Matcher::divL_proj_mask() {
 2502   ShouldNotReachHere();
 2503   return RegMask();
 2504 }
 2505 
 2506 // Register for MODL projection of divmodL.
 2507 RegMask Matcher::modL_proj_mask() {
 2508   ShouldNotReachHere();
 2509   return RegMask();
 2510 }
 2511 
 2512 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
 2513   return FP_REG_mask();
 2514 }
 2515 
 2516 bool size_fits_all_mem_uses(AddPNode* addp, int shift) {
 2517   for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) {
 2518     Node* u = addp->fast_out(i);
 2519     if (u->is_LoadStore()) {
 2520       // On AArch64, LoadStoreNodes (i.e. compare and swap
 2521       // instructions) only take register indirect as an operand, so
 2522       // any attempt to use an AddPNode as an input to a LoadStoreNode
 2523       // must fail.
 2524       return false;
 2525     }
 2526     if (u->is_Mem()) {
 2527       int opsize = u->as_Mem()->memory_size();
 2528       assert(opsize > 0, "unexpected memory operand size");
 2529       if (u->as_Mem()->memory_size() != (1<<shift)) {
 2530         return false;
 2531       }
 2532     }
 2533   }
 2534   return true;
 2535 }
 2536 
 2537 // Convert BootTest condition to Assembler condition.
 2538 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
 2539 Assembler::Condition to_assembler_cond(BoolTest::mask cond) {
 2540   Assembler::Condition result;
 2541   switch(cond) {
 2542     case BoolTest::eq:
 2543       result = Assembler::EQ; break;
 2544     case BoolTest::ne:
 2545       result = Assembler::NE; break;
 2546     case BoolTest::le:
 2547       result = Assembler::LE; break;
 2548     case BoolTest::ge:
 2549       result = Assembler::GE; break;
 2550     case BoolTest::lt:
 2551       result = Assembler::LT; break;
 2552     case BoolTest::gt:
 2553       result = Assembler::GT; break;
 2554     case BoolTest::ule:
 2555       result = Assembler::LS; break;
 2556     case BoolTest::uge:
 2557       result = Assembler::HS; break;
 2558     case BoolTest::ult:
 2559       result = Assembler::LO; break;
 2560     case BoolTest::ugt:
 2561       result = Assembler::HI; break;
 2562     case BoolTest::overflow:
 2563       result = Assembler::VS; break;
 2564     case BoolTest::no_overflow:
 2565       result = Assembler::VC; break;
 2566     default:
 2567       ShouldNotReachHere();
 2568       return Assembler::Condition(-1);
 2569   }
 2570 
 2571   // Check conversion
 2572   if (cond & BoolTest::unsigned_compare) {
 2573     assert(cmpOpUOper((BoolTest::mask)((int)cond & ~(BoolTest::unsigned_compare))).ccode() == result, "Invalid conversion");
 2574   } else {
 2575     assert(cmpOpOper(cond).ccode() == result, "Invalid conversion");
 2576   }
 2577 
 2578   return result;
 2579 }
 2580 
 2581 // Binary src (Replicate con)
 2582 static bool is_valid_sve_arith_imm_pattern(Node* n, Node* m) {
 2583   if (n == nullptr || m == nullptr) {
 2584     return false;
 2585   }
 2586 
 2587   if (UseSVE == 0 || m->Opcode() != Op_Replicate) {
 2588     return false;
 2589   }
 2590 
 2591   Node* imm_node = m->in(1);
 2592   if (!imm_node->is_Con()) {
 2593     return false;
 2594   }
 2595 
 2596   const Type* t = imm_node->bottom_type();
 2597   if (!(t->isa_int() || t->isa_long())) {
 2598     return false;
 2599   }
 2600 
 2601   switch (n->Opcode()) {
 2602   case Op_AndV:
 2603   case Op_OrV:
 2604   case Op_XorV: {
 2605     Assembler::SIMD_RegVariant T = Assembler::elemType_to_regVariant(Matcher::vector_element_basic_type(n));
 2606     uint64_t value = t->isa_long() ? (uint64_t)imm_node->get_long() : (uint64_t)imm_node->get_int();
 2607     return Assembler::operand_valid_for_sve_logical_immediate(Assembler::regVariant_to_elemBits(T), value);
 2608   }
 2609   case Op_AddVB:
 2610     return (imm_node->get_int() <= 255 && imm_node->get_int() >= -255);
 2611   case Op_AddVS:
 2612   case Op_AddVI:
 2613     return Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)imm_node->get_int());
 2614   case Op_AddVL:
 2615     return Assembler::operand_valid_for_sve_add_sub_immediate(imm_node->get_long());
 2616   default:
 2617     return false;
 2618   }
 2619 }
 2620 
 2621 // (XorV src (Replicate m1))
 2622 // (XorVMask src (MaskAll m1))
 2623 static bool is_vector_bitwise_not_pattern(Node* n, Node* m) {
 2624   if (n != nullptr && m != nullptr) {
 2625     return (n->Opcode() == Op_XorV || n->Opcode() == Op_XorVMask) &&
 2626            VectorNode::is_all_ones_vector(m);
 2627   }
 2628   return false;
 2629 }
 2630 
 2631 // Should the matcher clone input 'm' of node 'n'?
 2632 bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
 2633   if (is_vshift_con_pattern(n, m) ||
 2634       is_vector_bitwise_not_pattern(n, m) ||
 2635       is_valid_sve_arith_imm_pattern(n, m) ||
 2636       is_encode_and_store_pattern(n, m)) {
 2637     mstack.push(m, Visit);
 2638     return true;
 2639   }
 2640   return false;
 2641 }
 2642 
 2643 // Should the Matcher clone shifts on addressing modes, expecting them
 2644 // to be subsumed into complex addressing expressions or compute them
 2645 // into registers?
 2646 bool Matcher::pd_clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
 2647 
 2648   // Loads and stores with indirect memory input (e.g., volatile loads and
 2649   // stores) do not subsume the input into complex addressing expressions. If
 2650   // the addressing expression is input to at least one such load or store, do
 2651   // not clone the addressing expression. Query needs_acquiring_load and
 2652   // needs_releasing_store as a proxy for indirect memory input, as it is not
 2653   // possible to directly query for indirect memory input at this stage.
 2654   for (DUIterator_Fast imax, i = m->fast_outs(imax); i < imax; i++) {
 2655     Node* n = m->fast_out(i);
 2656     if (n->is_Load() && needs_acquiring_load(n)) {
 2657       return false;
 2658     }
 2659     if (n->is_Store() && needs_releasing_store(n)) {
 2660       return false;
 2661     }
 2662   }
 2663 
 2664   if (clone_base_plus_offset_address(m, mstack, address_visited)) {
 2665     return true;
 2666   }
 2667 
 2668   Node *off = m->in(AddPNode::Offset);
 2669   if (off->Opcode() == Op_LShiftL && off->in(2)->is_Con() &&
 2670       size_fits_all_mem_uses(m, off->in(2)->get_int()) &&
 2671       // Are there other uses besides address expressions?
 2672       !is_visited(off)) {
 2673     address_visited.set(off->_idx); // Flag as address_visited
 2674     mstack.push(off->in(2), Visit);
 2675     Node *conv = off->in(1);
 2676     if (conv->Opcode() == Op_ConvI2L &&
 2677         // Are there other uses besides address expressions?
 2678         !is_visited(conv)) {
 2679       address_visited.set(conv->_idx); // Flag as address_visited
 2680       mstack.push(conv->in(1), Pre_Visit);
 2681     } else {
 2682       mstack.push(conv, Pre_Visit);
 2683     }
 2684     address_visited.test_set(m->_idx); // Flag as address_visited
 2685     mstack.push(m->in(AddPNode::Address), Pre_Visit);
 2686     mstack.push(m->in(AddPNode::Base), Pre_Visit);
 2687     return true;
 2688   } else if (off->Opcode() == Op_ConvI2L &&
 2689              // Are there other uses besides address expressions?
 2690              !is_visited(off)) {
 2691     address_visited.test_set(m->_idx); // Flag as address_visited
 2692     address_visited.set(off->_idx); // Flag as address_visited
 2693     mstack.push(off->in(1), Pre_Visit);
 2694     mstack.push(m->in(AddPNode::Address), Pre_Visit);
 2695     mstack.push(m->in(AddPNode::Base), Pre_Visit);
 2696     return true;
 2697   }
 2698   return false;
 2699 }
 2700 
 2701 #define MOV_VOLATILE(REG, BASE, INDEX, SCALE, DISP, SCRATCH, INSN)      \
 2702   {                                                                     \
 2703     guarantee(INDEX == -1, "mode not permitted for volatile");          \
 2704     guarantee(DISP == 0, "mode not permitted for volatile");            \
 2705     guarantee(SCALE == 0, "mode not permitted for volatile");           \
 2706     __ INSN(REG, as_Register(BASE));                                    \
 2707   }
 2708 
 2709 
 2710 static Address mem2address(int opcode, Register base, int index, int size, int disp)
 2711   {
 2712     Address::extend scale;
 2713 
 2714     // Hooboy, this is fugly.  We need a way to communicate to the
 2715     // encoder that the index needs to be sign extended, so we have to
 2716     // enumerate all the cases.
 2717     switch (opcode) {
 2718     case INDINDEXSCALEDI2L:
 2719     case INDINDEXSCALEDI2LN:
 2720     case INDINDEXI2L:
 2721     case INDINDEXI2LN:
 2722       scale = Address::sxtw(size);
 2723       break;
 2724     default:
 2725       scale = Address::lsl(size);
 2726     }
 2727 
 2728     if (index == -1) {
 2729       return Address(base, disp);
 2730     } else {
 2731       assert(disp == 0, "unsupported address mode: disp = %d", disp);
 2732       return Address(base, as_Register(index), scale);
 2733     }
 2734   }
 2735 
 2736 
 2737 typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr);
 2738 typedef void (MacroAssembler::* mem_insn2)(Register Rt, Register adr);
 2739 typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr);
 2740 typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt,
 2741                                   MacroAssembler::SIMD_RegVariant T, const Address &adr);
 2742 
 2743   // Used for all non-volatile memory accesses.  The use of
 2744   // $mem->opcode() to discover whether this pattern uses sign-extended
 2745   // offsets is something of a kludge.
 2746   static void loadStore(C2_MacroAssembler* masm, mem_insn insn,
 2747                         Register reg, int opcode,
 2748                         Register base, int index, int scale, int disp,
 2749                         int size_in_memory)
 2750   {
 2751     Address addr = mem2address(opcode, base, index, scale, disp);
 2752     if (addr.getMode() == Address::base_plus_offset) {
 2753       /* Fix up any out-of-range offsets. */
 2754       assert_different_registers(rscratch1, base);
 2755       assert_different_registers(rscratch1, reg);
 2756       addr = __ legitimize_address(addr, size_in_memory, rscratch1);
 2757     }
 2758     (masm->*insn)(reg, addr);
 2759   }
 2760 
 2761   static void loadStore(C2_MacroAssembler* masm, mem_float_insn insn,
 2762                         FloatRegister reg, int opcode,
 2763                         Register base, int index, int size, int disp,
 2764                         int size_in_memory)
 2765   {
 2766     Address::extend scale;
 2767 
 2768     switch (opcode) {
 2769     case INDINDEXSCALEDI2L:
 2770     case INDINDEXSCALEDI2LN:
 2771       scale = Address::sxtw(size);
 2772       break;
 2773     default:
 2774       scale = Address::lsl(size);
 2775     }
 2776 
 2777     if (index == -1) {
 2778       // Fix up any out-of-range offsets.
 2779       assert_different_registers(rscratch1, base);
 2780       Address addr = Address(base, disp);
 2781       addr = __ legitimize_address(addr, size_in_memory, rscratch1);
 2782       (masm->*insn)(reg, addr);
 2783     } else {
 2784       assert(disp == 0, "unsupported address mode: disp = %d", disp);
 2785       (masm->*insn)(reg, Address(base, as_Register(index), scale));
 2786     }
 2787   }
 2788 
 2789   static void loadStore(C2_MacroAssembler* masm, mem_vector_insn insn,
 2790                         FloatRegister reg, MacroAssembler::SIMD_RegVariant T,
 2791                         int opcode, Register base, int index, int size, int disp)
 2792   {
 2793     if (index == -1) {
 2794       (masm->*insn)(reg, T, Address(base, disp));
 2795     } else {
 2796       assert(disp == 0, "unsupported address mode");
 2797       (masm->*insn)(reg, T, Address(base, as_Register(index), Address::lsl(size)));
 2798     }
 2799   }
 2800 
 2801 %}
 2802 
 2803 
 2804 
 2805 //----------ENCODING BLOCK-----------------------------------------------------
 2806 // This block specifies the encoding classes used by the compiler to
 2807 // output byte streams.  Encoding classes are parameterized macros
 2808 // used by Machine Instruction Nodes in order to generate the bit
 2809 // encoding of the instruction.  Operands specify their base encoding
 2810 // interface with the interface keyword.  There are currently
 2811 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
 2812 // COND_INTER.  REG_INTER causes an operand to generate a function
 2813 // which returns its register number when queried.  CONST_INTER causes
 2814 // an operand to generate a function which returns the value of the
 2815 // constant when queried.  MEMORY_INTER causes an operand to generate
 2816 // four functions which return the Base Register, the Index Register,
 2817 // the Scale Value, and the Offset Value of the operand when queried.
 2818 // COND_INTER causes an operand to generate six functions which return
 2819 // the encoding code (ie - encoding bits for the instruction)
 2820 // associated with each basic boolean condition for a conditional
 2821 // instruction.
 2822 //
 2823 // Instructions specify two basic values for encoding.  Again, a
 2824 // function is available to check if the constant displacement is an
 2825 // oop. They use the ins_encode keyword to specify their encoding
 2826 // classes (which must be a sequence of enc_class names, and their
 2827 // parameters, specified in the encoding block), and they use the
 2828 // opcode keyword to specify, in order, their primary, secondary, and
 2829 // tertiary opcode.  Only the opcode sections which a particular
 2830 // instruction needs for encoding need to be specified.
 2831 encode %{
 2832   // Build emit functions for each basic byte or larger field in the
 2833   // intel encoding scheme (opcode, rm, sib, immediate), and call them
 2834   // from C++ code in the enc_class source block.  Emit functions will
 2835   // live in the main source block for now.  In future, we can
 2836   // generalize this by adding a syntax that specifies the sizes of
 2837   // fields in an order, so that the adlc can build the emit functions
 2838   // automagically
 2839 
 2840   // catch all for unimplemented encodings
 2841   enc_class enc_unimplemented %{
 2842     __ unimplemented("C2 catch all");
 2843   %}
 2844 
 2845   // BEGIN Non-volatile memory access
 2846 
 2847   // This encoding class is generated automatically from ad_encode.m4.
 2848   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2849   enc_class aarch64_enc_ldrsbw(iRegI dst, memory1 mem) %{
 2850     Register dst_reg = as_Register($dst$$reg);
 2851     loadStore(masm, &MacroAssembler::ldrsbw, dst_reg, $mem->opcode(),
 2852                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2853   %}
 2854 
 2855   // This encoding class is generated automatically from ad_encode.m4.
 2856   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2857   enc_class aarch64_enc_ldrsb(iRegI dst, memory1 mem) %{
 2858     Register dst_reg = as_Register($dst$$reg);
 2859     loadStore(masm, &MacroAssembler::ldrsb, dst_reg, $mem->opcode(),
 2860                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2861   %}
 2862 
 2863   // This encoding class is generated automatically from ad_encode.m4.
 2864   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2865   enc_class aarch64_enc_ldrb(iRegI dst, memory1 mem) %{
 2866     Register dst_reg = as_Register($dst$$reg);
 2867     loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
 2868                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2869   %}
 2870 
 2871   // This encoding class is generated automatically from ad_encode.m4.
 2872   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2873   enc_class aarch64_enc_ldrb(iRegL dst, memory1 mem) %{
 2874     Register dst_reg = as_Register($dst$$reg);
 2875     loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
 2876                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2877   %}
 2878 
 2879   // This encoding class is generated automatically from ad_encode.m4.
 2880   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2881   enc_class aarch64_enc_ldrshw(iRegI dst, memory2 mem) %{
 2882     Register dst_reg = as_Register($dst$$reg);
 2883     loadStore(masm, &MacroAssembler::ldrshw, dst_reg, $mem->opcode(),
 2884                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2885   %}
 2886 
 2887   // This encoding class is generated automatically from ad_encode.m4.
 2888   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2889   enc_class aarch64_enc_ldrsh(iRegI dst, memory2 mem) %{
 2890     Register dst_reg = as_Register($dst$$reg);
 2891     loadStore(masm, &MacroAssembler::ldrsh, dst_reg, $mem->opcode(),
 2892                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2893   %}
 2894 
 2895   // This encoding class is generated automatically from ad_encode.m4.
 2896   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2897   enc_class aarch64_enc_ldrh(iRegI dst, memory2 mem) %{
 2898     Register dst_reg = as_Register($dst$$reg);
 2899     loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
 2900                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2901   %}
 2902 
 2903   // This encoding class is generated automatically from ad_encode.m4.
 2904   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2905   enc_class aarch64_enc_ldrh(iRegL dst, memory2 mem) %{
 2906     Register dst_reg = as_Register($dst$$reg);
 2907     loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
 2908                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2909   %}
 2910 
 2911   // This encoding class is generated automatically from ad_encode.m4.
 2912   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2913   enc_class aarch64_enc_ldrw(iRegI dst, memory4 mem) %{
 2914     Register dst_reg = as_Register($dst$$reg);
 2915     loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
 2916                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2917   %}
 2918 
 2919   // This encoding class is generated automatically from ad_encode.m4.
 2920   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2921   enc_class aarch64_enc_ldrw(iRegL dst, memory4 mem) %{
 2922     Register dst_reg = as_Register($dst$$reg);
 2923     loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
 2924                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2925   %}
 2926 
 2927   // This encoding class is generated automatically from ad_encode.m4.
 2928   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2929   enc_class aarch64_enc_ldrsw(iRegL dst, memory4 mem) %{
 2930     Register dst_reg = as_Register($dst$$reg);
 2931     loadStore(masm, &MacroAssembler::ldrsw, dst_reg, $mem->opcode(),
 2932                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2933   %}
 2934 
 2935   // This encoding class is generated automatically from ad_encode.m4.
 2936   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2937   enc_class aarch64_enc_ldr(iRegL dst, memory8 mem) %{
 2938     Register dst_reg = as_Register($dst$$reg);
 2939     loadStore(masm, &MacroAssembler::ldr, dst_reg, $mem->opcode(),
 2940                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 2941   %}
 2942 
 2943   // This encoding class is generated automatically from ad_encode.m4.
 2944   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2945   enc_class aarch64_enc_ldrs(vRegF dst, memory4 mem) %{
 2946     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 2947     loadStore(masm, &MacroAssembler::ldrs, dst_reg, $mem->opcode(),
 2948                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2949   %}
 2950 
 2951   // This encoding class is generated automatically from ad_encode.m4.
 2952   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2953   enc_class aarch64_enc_ldrd(vRegD dst, memory8 mem) %{
 2954     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 2955     loadStore(masm, &MacroAssembler::ldrd, dst_reg, $mem->opcode(),
 2956                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 2957   %}
 2958 
 2959   // This encoding class is generated automatically from ad_encode.m4.
 2960   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2961   enc_class aarch64_enc_strb(iRegI src, memory1 mem) %{
 2962     Register src_reg = as_Register($src$$reg);
 2963     loadStore(masm, &MacroAssembler::strb, src_reg, $mem->opcode(),
 2964                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2965   %}
 2966 
 2967   // This encoding class is generated automatically from ad_encode.m4.
 2968   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2969   enc_class aarch64_enc_strb0(memory1 mem) %{
 2970     loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
 2971                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2972   %}
 2973 
 2974   // This encoding class is generated automatically from ad_encode.m4.
 2975   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2976   enc_class aarch64_enc_strh(iRegI src, memory2 mem) %{
 2977     Register src_reg = as_Register($src$$reg);
 2978     loadStore(masm, &MacroAssembler::strh, src_reg, $mem->opcode(),
 2979                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2980   %}
 2981 
 2982   // This encoding class is generated automatically from ad_encode.m4.
 2983   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2984   enc_class aarch64_enc_strh0(memory2 mem) %{
 2985     loadStore(masm, &MacroAssembler::strh, zr, $mem->opcode(),
 2986                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2987   %}
 2988 
 2989   // This encoding class is generated automatically from ad_encode.m4.
 2990   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2991   enc_class aarch64_enc_strw(iRegI src, memory4 mem) %{
 2992     Register src_reg = as_Register($src$$reg);
 2993     loadStore(masm, &MacroAssembler::strw, src_reg, $mem->opcode(),
 2994                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2995   %}
 2996 
 2997   // This encoding class is generated automatically from ad_encode.m4.
 2998   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2999   enc_class aarch64_enc_strw0(memory4 mem) %{
 3000     loadStore(masm, &MacroAssembler::strw, zr, $mem->opcode(),
 3001                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 3002   %}
 3003 
 3004   // This encoding class is generated automatically from ad_encode.m4.
 3005   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3006   enc_class aarch64_enc_str(iRegL src, memory8 mem) %{
 3007     Register src_reg = as_Register($src$$reg);
 3008     // we sometimes get asked to store the stack pointer into the
 3009     // current thread -- we cannot do that directly on AArch64
 3010     if (src_reg == r31_sp) {
 3011       assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
 3012       __ mov(rscratch2, sp);
 3013       src_reg = rscratch2;
 3014     }
 3015     loadStore(masm, &MacroAssembler::str, src_reg, $mem->opcode(),
 3016                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 3017   %}
 3018 
 3019   // This encoding class is generated automatically from ad_encode.m4.
 3020   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3021   enc_class aarch64_enc_str0(memory8 mem) %{
 3022     loadStore(masm, &MacroAssembler::str, zr, $mem->opcode(),
 3023                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 3024   %}
 3025 
 3026   // This encoding class is generated automatically from ad_encode.m4.
 3027   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3028   enc_class aarch64_enc_strs(vRegF src, memory4 mem) %{
 3029     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3030     loadStore(masm, &MacroAssembler::strs, src_reg, $mem->opcode(),
 3031                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 3032   %}
 3033 
 3034   // This encoding class is generated automatically from ad_encode.m4.
 3035   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3036   enc_class aarch64_enc_strd(vRegD src, memory8 mem) %{
 3037     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3038     loadStore(masm, &MacroAssembler::strd, src_reg, $mem->opcode(),
 3039                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 3040   %}
 3041 
 3042   // This encoding class is generated automatically from ad_encode.m4.
 3043   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3044   enc_class aarch64_enc_strb0_ordered(memory4 mem) %{
 3045       __ membar(Assembler::StoreStore);
 3046       loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
 3047                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 3048   %}
 3049 
 3050   // END Non-volatile memory access
 3051 
 3052   // Vector loads and stores
 3053   enc_class aarch64_enc_ldrvH(vReg dst, memory mem) %{
 3054     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3055     loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::H,
 3056        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3057   %}
 3058 
 3059   enc_class aarch64_enc_ldrvS(vReg dst, memory mem) %{
 3060     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3061     loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::S,
 3062        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3063   %}
 3064 
 3065   enc_class aarch64_enc_ldrvD(vReg dst, memory mem) %{
 3066     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3067     loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::D,
 3068        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3069   %}
 3070 
 3071   enc_class aarch64_enc_ldrvQ(vReg dst, memory mem) %{
 3072     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3073     loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::Q,
 3074        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3075   %}
 3076 
 3077   enc_class aarch64_enc_strvH(vReg src, memory mem) %{
 3078     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3079     loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::H,
 3080        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3081   %}
 3082 
 3083   enc_class aarch64_enc_strvS(vReg src, memory mem) %{
 3084     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3085     loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::S,
 3086        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3087   %}
 3088 
 3089   enc_class aarch64_enc_strvD(vReg src, memory mem) %{
 3090     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3091     loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::D,
 3092        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3093   %}
 3094 
 3095   enc_class aarch64_enc_strvQ(vReg src, memory mem) %{
 3096     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3097     loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::Q,
 3098        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3099   %}
 3100 
 3101   // volatile loads and stores
 3102 
 3103   enc_class aarch64_enc_stlrb(iRegI src, memory mem) %{
 3104     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3105                  rscratch1, stlrb);
 3106   %}
 3107 
 3108   enc_class aarch64_enc_stlrb0(memory mem) %{
 3109     MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3110                  rscratch1, stlrb);
 3111   %}
 3112 
 3113   enc_class aarch64_enc_stlrh(iRegI src, memory mem) %{
 3114     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3115                  rscratch1, stlrh);
 3116   %}
 3117 
 3118   enc_class aarch64_enc_stlrh0(memory mem) %{
 3119     MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3120                  rscratch1, stlrh);
 3121   %}
 3122 
 3123   enc_class aarch64_enc_stlrw(iRegI src, memory mem) %{
 3124     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3125                  rscratch1, stlrw);
 3126   %}
 3127 
 3128   enc_class aarch64_enc_stlrw0(memory mem) %{
 3129     MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3130                  rscratch1, stlrw);
 3131   %}
 3132 
 3133   enc_class aarch64_enc_ldarsbw(iRegI dst, memory mem) %{
 3134     Register dst_reg = as_Register($dst$$reg);
 3135     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3136              rscratch1, ldarb);
 3137     __ sxtbw(dst_reg, dst_reg);
 3138   %}
 3139 
 3140   enc_class aarch64_enc_ldarsb(iRegL dst, memory mem) %{
 3141     Register dst_reg = as_Register($dst$$reg);
 3142     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3143              rscratch1, ldarb);
 3144     __ sxtb(dst_reg, dst_reg);
 3145   %}
 3146 
 3147   enc_class aarch64_enc_ldarbw(iRegI dst, memory mem) %{
 3148     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3149              rscratch1, ldarb);
 3150   %}
 3151 
 3152   enc_class aarch64_enc_ldarb(iRegL dst, memory mem) %{
 3153     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3154              rscratch1, ldarb);
 3155   %}
 3156 
 3157   enc_class aarch64_enc_ldarshw(iRegI dst, memory mem) %{
 3158     Register dst_reg = as_Register($dst$$reg);
 3159     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3160              rscratch1, ldarh);
 3161     __ sxthw(dst_reg, dst_reg);
 3162   %}
 3163 
 3164   enc_class aarch64_enc_ldarsh(iRegL dst, memory mem) %{
 3165     Register dst_reg = as_Register($dst$$reg);
 3166     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3167              rscratch1, ldarh);
 3168     __ sxth(dst_reg, dst_reg);
 3169   %}
 3170 
 3171   enc_class aarch64_enc_ldarhw(iRegI dst, memory mem) %{
 3172     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3173              rscratch1, ldarh);
 3174   %}
 3175 
 3176   enc_class aarch64_enc_ldarh(iRegL dst, memory mem) %{
 3177     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3178              rscratch1, ldarh);
 3179   %}
 3180 
 3181   enc_class aarch64_enc_ldarw(iRegI dst, memory mem) %{
 3182     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3183              rscratch1, ldarw);
 3184   %}
 3185 
 3186   enc_class aarch64_enc_ldarw(iRegL dst, memory mem) %{
 3187     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3188              rscratch1, ldarw);
 3189   %}
 3190 
 3191   enc_class aarch64_enc_ldar(iRegL dst, memory mem) %{
 3192     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3193              rscratch1, ldar);
 3194   %}
 3195 
 3196   enc_class aarch64_enc_fldars(vRegF dst, memory mem) %{
 3197     MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3198              rscratch1, ldarw);
 3199     __ fmovs(as_FloatRegister($dst$$reg), rscratch1);
 3200   %}
 3201 
 3202   enc_class aarch64_enc_fldard(vRegD dst, memory mem) %{
 3203     MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3204              rscratch1, ldar);
 3205     __ fmovd(as_FloatRegister($dst$$reg), rscratch1);
 3206   %}
 3207 
 3208   enc_class aarch64_enc_stlr(iRegL src, memory mem) %{
 3209     Register src_reg = as_Register($src$$reg);
 3210     // we sometimes get asked to store the stack pointer into the
 3211     // current thread -- we cannot do that directly on AArch64
 3212     if (src_reg == r31_sp) {
 3213       assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
 3214       __ mov(rscratch2, sp);
 3215       src_reg = rscratch2;
 3216     }
 3217     MOV_VOLATILE(src_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3218                  rscratch1, stlr);
 3219   %}
 3220 
 3221   enc_class aarch64_enc_stlr0(memory mem) %{
 3222     MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3223                  rscratch1, stlr);
 3224   %}
 3225 
 3226   enc_class aarch64_enc_fstlrs(vRegF src, memory mem) %{
 3227     {
 3228       FloatRegister src_reg = as_FloatRegister($src$$reg);
 3229       __ fmovs(rscratch2, src_reg);
 3230     }
 3231     MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3232                  rscratch1, stlrw);
 3233   %}
 3234 
 3235   enc_class aarch64_enc_fstlrd(vRegD src, memory mem) %{
 3236     {
 3237       FloatRegister src_reg = as_FloatRegister($src$$reg);
 3238       __ fmovd(rscratch2, src_reg);
 3239     }
 3240     MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3241                  rscratch1, stlr);
 3242   %}
 3243 
 3244   // synchronized read/update encodings
 3245 
 3246   enc_class aarch64_enc_ldaxr(iRegL dst, memory8 mem) %{
 3247     Register dst_reg = as_Register($dst$$reg);
 3248     Register base = as_Register($mem$$base);
 3249     int index = $mem$$index;
 3250     int scale = $mem$$scale;
 3251     int disp = $mem$$disp;
 3252     if (index == -1) {
 3253        if (disp != 0) {
 3254         __ lea(rscratch1, Address(base, disp));
 3255         __ ldaxr(dst_reg, rscratch1);
 3256       } else {
 3257         // TODO
 3258         // should we ever get anything other than this case?
 3259         __ ldaxr(dst_reg, base);
 3260       }
 3261     } else {
 3262       Register index_reg = as_Register(index);
 3263       if (disp == 0) {
 3264         __ lea(rscratch1, Address(base, index_reg, Address::lsl(scale)));
 3265         __ ldaxr(dst_reg, rscratch1);
 3266       } else {
 3267         __ lea(rscratch1, Address(base, disp));
 3268         __ lea(rscratch1, Address(rscratch1, index_reg, Address::lsl(scale)));
 3269         __ ldaxr(dst_reg, rscratch1);
 3270       }
 3271     }
 3272   %}
 3273 
 3274   enc_class aarch64_enc_stlxr(iRegLNoSp src, memory8 mem) %{
 3275     Register src_reg = as_Register($src$$reg);
 3276     Register base = as_Register($mem$$base);
 3277     int index = $mem$$index;
 3278     int scale = $mem$$scale;
 3279     int disp = $mem$$disp;
 3280     if (index == -1) {
 3281        if (disp != 0) {
 3282         __ lea(rscratch2, Address(base, disp));
 3283         __ stlxr(rscratch1, src_reg, rscratch2);
 3284       } else {
 3285         // TODO
 3286         // should we ever get anything other than this case?
 3287         __ stlxr(rscratch1, src_reg, base);
 3288       }
 3289     } else {
 3290       Register index_reg = as_Register(index);
 3291       if (disp == 0) {
 3292         __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
 3293         __ stlxr(rscratch1, src_reg, rscratch2);
 3294       } else {
 3295         __ lea(rscratch2, Address(base, disp));
 3296         __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
 3297         __ stlxr(rscratch1, src_reg, rscratch2);
 3298       }
 3299     }
 3300     __ cmpw(rscratch1, zr);
 3301   %}
 3302 
 3303   enc_class aarch64_enc_cmpxchg(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
 3304     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3305     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3306                Assembler::xword, /*acquire*/ false, /*release*/ true,
 3307                /*weak*/ false, noreg);
 3308   %}
 3309 
 3310   enc_class aarch64_enc_cmpxchgw(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3311     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3312     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3313                Assembler::word, /*acquire*/ false, /*release*/ true,
 3314                /*weak*/ false, noreg);
 3315   %}
 3316 
 3317   enc_class aarch64_enc_cmpxchgs(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3318     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3319     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3320                Assembler::halfword, /*acquire*/ false, /*release*/ true,
 3321                /*weak*/ false, noreg);
 3322   %}
 3323 
 3324   enc_class aarch64_enc_cmpxchgb(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3325     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3326     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3327                Assembler::byte, /*acquire*/ false, /*release*/ true,
 3328                /*weak*/ false, noreg);
 3329   %}
 3330 
 3331 
 3332   // The only difference between aarch64_enc_cmpxchg and
 3333   // aarch64_enc_cmpxchg_acq is that we use load-acquire in the
 3334   // CompareAndSwap sequence to serve as a barrier on acquiring a
 3335   // lock.
 3336   enc_class aarch64_enc_cmpxchg_acq(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
 3337     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3338     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3339                Assembler::xword, /*acquire*/ true, /*release*/ true,
 3340                /*weak*/ false, noreg);
 3341   %}
 3342 
 3343   enc_class aarch64_enc_cmpxchgw_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3344     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3345     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3346                Assembler::word, /*acquire*/ true, /*release*/ true,
 3347                /*weak*/ false, noreg);
 3348   %}
 3349 
 3350   enc_class aarch64_enc_cmpxchgs_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3351     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3352     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3353                Assembler::halfword, /*acquire*/ true, /*release*/ true,
 3354                /*weak*/ false, noreg);
 3355   %}
 3356 
 3357   enc_class aarch64_enc_cmpxchgb_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3358     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3359     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3360                Assembler::byte, /*acquire*/ true, /*release*/ true,
 3361                /*weak*/ false, noreg);
 3362   %}
 3363 
 3364   // auxiliary used for CompareAndSwapX to set result register
 3365   enc_class aarch64_enc_cset_eq(iRegINoSp res) %{
 3366     Register res_reg = as_Register($res$$reg);
 3367     __ cset(res_reg, Assembler::EQ);
 3368   %}
 3369 
 3370   // prefetch encodings
 3371 
 3372   enc_class aarch64_enc_prefetchw(memory mem) %{
 3373     Register base = as_Register($mem$$base);
 3374     int index = $mem$$index;
 3375     int scale = $mem$$scale;
 3376     int disp = $mem$$disp;
 3377     if (index == -1) {
 3378       // Fix up any out-of-range offsets.
 3379       assert_different_registers(rscratch1, base);
 3380       Address addr = Address(base, disp);
 3381       addr = __ legitimize_address(addr, 8, rscratch1);
 3382       __ prfm(addr, PSTL1KEEP);
 3383     } else {
 3384       Register index_reg = as_Register(index);
 3385       if (disp == 0) {
 3386         __ prfm(Address(base, index_reg, Address::lsl(scale)), PSTL1KEEP);
 3387       } else {
 3388         __ lea(rscratch1, Address(base, disp));
 3389 	__ prfm(Address(rscratch1, index_reg, Address::lsl(scale)), PSTL1KEEP);
 3390       }
 3391     }
 3392   %}
 3393 
 3394   // mov encodings
 3395 
 3396   enc_class aarch64_enc_movw_imm(iRegI dst, immI src) %{
 3397     uint32_t con = (uint32_t)$src$$constant;
 3398     Register dst_reg = as_Register($dst$$reg);
 3399     if (con == 0) {
 3400       __ movw(dst_reg, zr);
 3401     } else {
 3402       __ movw(dst_reg, con);
 3403     }
 3404   %}
 3405 
 3406   enc_class aarch64_enc_mov_imm(iRegL dst, immL src) %{
 3407     Register dst_reg = as_Register($dst$$reg);
 3408     uint64_t con = (uint64_t)$src$$constant;
 3409     if (con == 0) {
 3410       __ mov(dst_reg, zr);
 3411     } else {
 3412       __ mov(dst_reg, con);
 3413     }
 3414   %}
 3415 
 3416   enc_class aarch64_enc_mov_p(iRegP dst, immP src) %{
 3417     Register dst_reg = as_Register($dst$$reg);
 3418     address con = (address)$src$$constant;
 3419     if (con == nullptr || con == (address)1) {
 3420       ShouldNotReachHere();
 3421     } else {
 3422       relocInfo::relocType rtype = $src->constant_reloc();
 3423       if (rtype == relocInfo::oop_type) {
 3424         __ movoop(dst_reg, (jobject)con);
 3425       } else if (rtype == relocInfo::metadata_type) {
 3426         __ mov_metadata(dst_reg, (Metadata*)con);
 3427       } else {
 3428         assert(rtype == relocInfo::none, "unexpected reloc type");
 3429         if (! __ is_valid_AArch64_address(con) ||
 3430             con < (address)(uintptr_t)os::vm_page_size()) {
 3431           __ mov(dst_reg, con);
 3432         } else {
 3433           uint64_t offset;
 3434           __ adrp(dst_reg, con, offset);
 3435           __ add(dst_reg, dst_reg, offset);
 3436         }
 3437       }
 3438     }
 3439   %}
 3440 
 3441   enc_class aarch64_enc_mov_p0(iRegP dst, immP0 src) %{
 3442     Register dst_reg = as_Register($dst$$reg);
 3443     __ mov(dst_reg, zr);
 3444   %}
 3445 
 3446   enc_class aarch64_enc_mov_p1(iRegP dst, immP_1 src) %{
 3447     Register dst_reg = as_Register($dst$$reg);
 3448     __ mov(dst_reg, (uint64_t)1);
 3449   %}
 3450 
 3451   enc_class aarch64_enc_mov_byte_map_base(iRegP dst, immByteMapBase src) %{
 3452     __ load_byte_map_base($dst$$Register);
 3453   %}
 3454 
 3455   enc_class aarch64_enc_mov_n(iRegN dst, immN src) %{
 3456     Register dst_reg = as_Register($dst$$reg);
 3457     address con = (address)$src$$constant;
 3458     if (con == nullptr) {
 3459       ShouldNotReachHere();
 3460     } else {
 3461       relocInfo::relocType rtype = $src->constant_reloc();
 3462       assert(rtype == relocInfo::oop_type, "unexpected reloc type");
 3463       __ set_narrow_oop(dst_reg, (jobject)con);
 3464     }
 3465   %}
 3466 
 3467   enc_class aarch64_enc_mov_n0(iRegN dst, immN0 src) %{
 3468     Register dst_reg = as_Register($dst$$reg);
 3469     __ mov(dst_reg, zr);
 3470   %}
 3471 
 3472   enc_class aarch64_enc_mov_nk(iRegN dst, immNKlass src) %{
 3473     Register dst_reg = as_Register($dst$$reg);
 3474     address con = (address)$src$$constant;
 3475     if (con == nullptr) {
 3476       ShouldNotReachHere();
 3477     } else {
 3478       relocInfo::relocType rtype = $src->constant_reloc();
 3479       assert(rtype == relocInfo::metadata_type, "unexpected reloc type");
 3480       __ set_narrow_klass(dst_reg, (Klass *)con);
 3481     }
 3482   %}
 3483 
 3484   // arithmetic encodings
 3485 
 3486   enc_class aarch64_enc_addsubw_imm(iRegI dst, iRegI src1, immIAddSub src2) %{
 3487     Register dst_reg = as_Register($dst$$reg);
 3488     Register src_reg = as_Register($src1$$reg);
 3489     int32_t con = (int32_t)$src2$$constant;
 3490     // add has primary == 0, subtract has primary == 1
 3491     if ($primary) { con = -con; }
 3492     if (con < 0) {
 3493       __ subw(dst_reg, src_reg, -con);
 3494     } else {
 3495       __ addw(dst_reg, src_reg, con);
 3496     }
 3497   %}
 3498 
 3499   enc_class aarch64_enc_addsub_imm(iRegL dst, iRegL src1, immLAddSub src2) %{
 3500     Register dst_reg = as_Register($dst$$reg);
 3501     Register src_reg = as_Register($src1$$reg);
 3502     int32_t con = (int32_t)$src2$$constant;
 3503     // add has primary == 0, subtract has primary == 1
 3504     if ($primary) { con = -con; }
 3505     if (con < 0) {
 3506       __ sub(dst_reg, src_reg, -con);
 3507     } else {
 3508       __ add(dst_reg, src_reg, con);
 3509     }
 3510   %}
 3511 
 3512   enc_class aarch64_enc_divw(iRegI dst, iRegI src1, iRegI src2) %{
 3513    Register dst_reg = as_Register($dst$$reg);
 3514    Register src1_reg = as_Register($src1$$reg);
 3515    Register src2_reg = as_Register($src2$$reg);
 3516     __ corrected_idivl(dst_reg, src1_reg, src2_reg, false, rscratch1);
 3517   %}
 3518 
 3519   enc_class aarch64_enc_div(iRegI dst, iRegI src1, iRegI src2) %{
 3520    Register dst_reg = as_Register($dst$$reg);
 3521    Register src1_reg = as_Register($src1$$reg);
 3522    Register src2_reg = as_Register($src2$$reg);
 3523     __ corrected_idivq(dst_reg, src1_reg, src2_reg, false, rscratch1);
 3524   %}
 3525 
 3526   enc_class aarch64_enc_modw(iRegI dst, iRegI src1, iRegI src2) %{
 3527    Register dst_reg = as_Register($dst$$reg);
 3528    Register src1_reg = as_Register($src1$$reg);
 3529    Register src2_reg = as_Register($src2$$reg);
 3530     __ corrected_idivl(dst_reg, src1_reg, src2_reg, true, rscratch1);
 3531   %}
 3532 
 3533   enc_class aarch64_enc_mod(iRegI dst, iRegI src1, iRegI src2) %{
 3534    Register dst_reg = as_Register($dst$$reg);
 3535    Register src1_reg = as_Register($src1$$reg);
 3536    Register src2_reg = as_Register($src2$$reg);
 3537     __ corrected_idivq(dst_reg, src1_reg, src2_reg, true, rscratch1);
 3538   %}
 3539 
 3540   // compare instruction encodings
 3541 
 3542   enc_class aarch64_enc_cmpw(iRegI src1, iRegI src2) %{
 3543     Register reg1 = as_Register($src1$$reg);
 3544     Register reg2 = as_Register($src2$$reg);
 3545     __ cmpw(reg1, reg2);
 3546   %}
 3547 
 3548   enc_class aarch64_enc_cmpw_imm_addsub(iRegI src1, immIAddSub src2) %{
 3549     Register reg = as_Register($src1$$reg);
 3550     int32_t val = $src2$$constant;
 3551     if (val >= 0) {
 3552       __ subsw(zr, reg, val);
 3553     } else {
 3554       __ addsw(zr, reg, -val);
 3555     }
 3556   %}
 3557 
 3558   enc_class aarch64_enc_cmpw_imm(iRegI src1, immI src2) %{
 3559     Register reg1 = as_Register($src1$$reg);
 3560     uint32_t val = (uint32_t)$src2$$constant;
 3561     __ movw(rscratch1, val);
 3562     __ cmpw(reg1, rscratch1);
 3563   %}
 3564 
 3565   enc_class aarch64_enc_cmp(iRegL src1, iRegL src2) %{
 3566     Register reg1 = as_Register($src1$$reg);
 3567     Register reg2 = as_Register($src2$$reg);
 3568     __ cmp(reg1, reg2);
 3569   %}
 3570 
 3571   enc_class aarch64_enc_cmp_imm_addsub(iRegL src1, immL12 src2) %{
 3572     Register reg = as_Register($src1$$reg);
 3573     int64_t val = $src2$$constant;
 3574     if (val >= 0) {
 3575       __ subs(zr, reg, val);
 3576     } else if (val != -val) {
 3577       __ adds(zr, reg, -val);
 3578     } else {
 3579     // aargh, Long.MIN_VALUE is a special case
 3580       __ orr(rscratch1, zr, (uint64_t)val);
 3581       __ subs(zr, reg, rscratch1);
 3582     }
 3583   %}
 3584 
 3585   enc_class aarch64_enc_cmp_imm(iRegL src1, immL src2) %{
 3586     Register reg1 = as_Register($src1$$reg);
 3587     uint64_t val = (uint64_t)$src2$$constant;
 3588     __ mov(rscratch1, val);
 3589     __ cmp(reg1, rscratch1);
 3590   %}
 3591 
 3592   enc_class aarch64_enc_cmpp(iRegP src1, iRegP src2) %{
 3593     Register reg1 = as_Register($src1$$reg);
 3594     Register reg2 = as_Register($src2$$reg);
 3595     __ cmp(reg1, reg2);
 3596   %}
 3597 
 3598   enc_class aarch64_enc_cmpn(iRegN src1, iRegN src2) %{
 3599     Register reg1 = as_Register($src1$$reg);
 3600     Register reg2 = as_Register($src2$$reg);
 3601     __ cmpw(reg1, reg2);
 3602   %}
 3603 
 3604   enc_class aarch64_enc_testp(iRegP src) %{
 3605     Register reg = as_Register($src$$reg);
 3606     __ cmp(reg, zr);
 3607   %}
 3608 
 3609   enc_class aarch64_enc_testn(iRegN src) %{
 3610     Register reg = as_Register($src$$reg);
 3611     __ cmpw(reg, zr);
 3612   %}
 3613 
 3614   enc_class aarch64_enc_b(label lbl) %{
 3615     Label *L = $lbl$$label;
 3616     __ b(*L);
 3617   %}
 3618 
 3619   enc_class aarch64_enc_br_con(cmpOp cmp, label lbl) %{
 3620     Label *L = $lbl$$label;
 3621     __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
 3622   %}
 3623 
 3624   enc_class aarch64_enc_br_conU(cmpOpU cmp, label lbl) %{
 3625     Label *L = $lbl$$label;
 3626     __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
 3627   %}
 3628 
 3629   enc_class aarch64_enc_partial_subtype_check(iRegP sub, iRegP super, iRegP temp, iRegP result)
 3630   %{
 3631      Register sub_reg = as_Register($sub$$reg);
 3632      Register super_reg = as_Register($super$$reg);
 3633      Register temp_reg = as_Register($temp$$reg);
 3634      Register result_reg = as_Register($result$$reg);
 3635 
 3636      Label miss;
 3637      __ check_klass_subtype_slow_path(sub_reg, super_reg, temp_reg, result_reg,
 3638                                      nullptr, &miss,
 3639                                      /*set_cond_codes:*/ true);
 3640      if ($primary) {
 3641        __ mov(result_reg, zr);
 3642      }
 3643      __ bind(miss);
 3644   %}
 3645 
 3646   enc_class aarch64_enc_java_static_call(method meth) %{
 3647     address addr = (address)$meth$$method;
 3648     address call;
 3649     if (!_method) {
 3650       // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
 3651       call = __ trampoline_call(Address(addr, relocInfo::runtime_call_type));
 3652       if (call == nullptr) {
 3653         ciEnv::current()->record_failure("CodeCache is full");
 3654         return;
 3655       }
 3656     } else if (_method->intrinsic_id() == vmIntrinsicID::_ensureMaterializedForStackWalk) {
 3657       // The NOP here is purely to ensure that eliding a call to
 3658       // JVM_EnsureMaterializedForStackWalk doesn't change the code size.
 3659       __ nop();
 3660       __ block_comment("call JVM_EnsureMaterializedForStackWalk (elided)");
 3661     } else {
 3662       int method_index = resolved_method_index(masm);
 3663       RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
 3664                                                   : static_call_Relocation::spec(method_index);
 3665       call = __ trampoline_call(Address(addr, rspec));
 3666       if (call == nullptr) {
 3667         ciEnv::current()->record_failure("CodeCache is full");
 3668         return;
 3669       }
 3670       if (CodeBuffer::supports_shared_stubs() && _method->can_be_statically_bound()) {
 3671         // Calls of the same statically bound method can share
 3672         // a stub to the interpreter.
 3673         __ code()->shared_stub_to_interp_for(_method, call - __ begin());
 3674       } else {
 3675         // Emit stub for static call
 3676         address stub = CompiledDirectCall::emit_to_interp_stub(masm, call);
 3677         if (stub == nullptr) {
 3678           ciEnv::current()->record_failure("CodeCache is full");
 3679           return;
 3680         }
 3681       }
 3682     }
 3683 
 3684     __ post_call_nop();
 3685 
 3686     // Only non uncommon_trap calls need to reinitialize ptrue.
 3687     if (Compile::current()->max_vector_size() > 0 && uncommon_trap_request() == 0) {
 3688       __ reinitialize_ptrue();
 3689     }
 3690   %}
 3691 
 3692   enc_class aarch64_enc_java_dynamic_call(method meth) %{
 3693     int method_index = resolved_method_index(masm);
 3694     address call = __ ic_call((address)$meth$$method, method_index);
 3695     if (call == nullptr) {
 3696       ciEnv::current()->record_failure("CodeCache is full");
 3697       return;
 3698     }
 3699     __ post_call_nop();
 3700     if (Compile::current()->max_vector_size() > 0) {
 3701       __ reinitialize_ptrue();
 3702     }
 3703   %}
 3704 
 3705   enc_class aarch64_enc_call_epilog() %{
 3706     if (VerifyStackAtCalls) {
 3707       // Check that stack depth is unchanged: find majik cookie on stack
 3708       __ call_Unimplemented();
 3709     }
 3710     if (tf()->returns_inline_type_as_fields() && !_method->is_method_handle_intrinsic()) {
 3711       // The last return value is not set by the callee but used to pass the null marker to compiled code.
 3712       // Search for the corresponding projection, get the register and emit code that initialized it.
 3713       uint con = (tf()->range_cc()->cnt() - 1);
 3714       for (DUIterator_Fast imax, i = fast_outs(imax); i < imax; i++) {
 3715         ProjNode* proj = fast_out(i)->as_Proj();
 3716         if (proj->_con == con) {
 3717           // Set null marker if r0 is non-null (a non-null value is returned buffered or scalarized)
 3718           OptoReg::Name optoReg = ra_->get_reg_first(proj);
 3719           VMReg reg = OptoReg::as_VMReg(optoReg, ra_->_framesize, OptoReg::reg2stack(ra_->_matcher._new_SP));
 3720           Register toReg = reg->is_reg() ? reg->as_Register() : rscratch1;
 3721           __ cmp(r0, zr);
 3722           __ cset(toReg, Assembler::NE);
 3723           if (reg->is_stack()) {
 3724             int st_off = reg->reg2stack() * VMRegImpl::stack_slot_size;
 3725             __ str(toReg, Address(sp, st_off));
 3726           }
 3727           break;
 3728         }
 3729       }
 3730       if (return_value_is_used()) {
 3731         // An inline type is returned as fields in multiple registers.
 3732         // R0 either contains an oop if the inline type is buffered or a pointer
 3733         // to the corresponding InlineKlass with the lowest bit set to 1. Zero r0
 3734         // if the lowest bit is set to allow C2 to use the oop after null checking.
 3735         // r0 &= (r0 & 1) - 1
 3736         __ andr(rscratch1, r0, 0x1);
 3737         __ sub(rscratch1, rscratch1, 0x1);
 3738         __ andr(r0, r0, rscratch1);
 3739       }
 3740     }
 3741   %}
 3742 
 3743   enc_class aarch64_enc_java_to_runtime(method meth) %{
 3744     // some calls to generated routines (arraycopy code) are scheduled
 3745     // by C2 as runtime calls. if so we can call them using a br (they
 3746     // will be in a reachable segment) otherwise we have to use a blr
 3747     // which loads the absolute address into a register.
 3748     address entry = (address)$meth$$method;
 3749     CodeBlob *cb = CodeCache::find_blob(entry);
 3750     if (cb) {
 3751       address call = __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
 3752       if (call == nullptr) {
 3753         ciEnv::current()->record_failure("CodeCache is full");
 3754         return;
 3755       }
 3756       __ post_call_nop();
 3757     } else {
 3758       Label retaddr;
 3759       // Make the anchor frame walkable
 3760       __ adr(rscratch2, retaddr);
 3761       __ str(rscratch2, Address(rthread, JavaThread::last_Java_pc_offset()));
 3762       __ lea(rscratch1, RuntimeAddress(entry));
 3763       __ blr(rscratch1);
 3764       __ bind(retaddr);
 3765       __ post_call_nop();
 3766     }
 3767     if (Compile::current()->max_vector_size() > 0) {
 3768       __ reinitialize_ptrue();
 3769     }
 3770   %}
 3771 
 3772   enc_class aarch64_enc_rethrow() %{
 3773     __ far_jump(RuntimeAddress(OptoRuntime::rethrow_stub()));
 3774   %}
 3775 
 3776   enc_class aarch64_enc_ret() %{
 3777 #ifdef ASSERT
 3778     if (Compile::current()->max_vector_size() > 0) {
 3779       __ verify_ptrue();
 3780     }
 3781 #endif
 3782     __ ret(lr);
 3783   %}
 3784 
 3785   enc_class aarch64_enc_tail_call(iRegP jump_target) %{
 3786     Register target_reg = as_Register($jump_target$$reg);
 3787     __ br(target_reg);
 3788   %}
 3789 
 3790   enc_class aarch64_enc_tail_jmp(iRegP jump_target) %{
 3791     Register target_reg = as_Register($jump_target$$reg);
 3792     // exception oop should be in r0
 3793     // ret addr has been popped into lr
 3794     // callee expects it in r3
 3795     __ mov(r3, lr);
 3796     __ br(target_reg);
 3797   %}
 3798 
 3799 %}
 3800 
 3801 //----------FRAME--------------------------------------------------------------
 3802 // Definition of frame structure and management information.
 3803 //
 3804 //  S T A C K   L A Y O U T    Allocators stack-slot number
 3805 //                             |   (to get allocators register number
 3806 //  G  Owned by    |        |  v    add OptoReg::stack0())
 3807 //  r   CALLER     |        |
 3808 //  o     |        +--------+      pad to even-align allocators stack-slot
 3809 //  w     V        |  pad0  |        numbers; owned by CALLER
 3810 //  t   -----------+--------+----> Matcher::_in_arg_limit, unaligned
 3811 //  h     ^        |   in   |  5
 3812 //        |        |  args  |  4   Holes in incoming args owned by SELF
 3813 //  |     |        |        |  3
 3814 //  |     |        +--------+
 3815 //  V     |        | old out|      Empty on Intel, window on Sparc
 3816 //        |    old |preserve|      Must be even aligned.
 3817 //        |     SP-+--------+----> Matcher::_old_SP, even aligned
 3818 //        |        |   in   |  3   area for Intel ret address
 3819 //     Owned by    |preserve|      Empty on Sparc.
 3820 //       SELF      +--------+
 3821 //        |        |  pad2  |  2   pad to align old SP
 3822 //        |        +--------+  1
 3823 //        |        | locks  |  0
 3824 //        |        +--------+----> OptoReg::stack0(), even aligned
 3825 //        |        |  pad1  | 11   pad to align new SP
 3826 //        |        +--------+
 3827 //        |        |        | 10
 3828 //        |        | spills |  9   spills
 3829 //        V        |        |  8   (pad0 slot for callee)
 3830 //      -----------+--------+----> Matcher::_out_arg_limit, unaligned
 3831 //        ^        |  out   |  7
 3832 //        |        |  args  |  6   Holes in outgoing args owned by CALLEE
 3833 //     Owned by    +--------+
 3834 //      CALLEE     | new out|  6   Empty on Intel, window on Sparc
 3835 //        |    new |preserve|      Must be even-aligned.
 3836 //        |     SP-+--------+----> Matcher::_new_SP, even aligned
 3837 //        |        |        |
 3838 //
 3839 // Note 1: Only region 8-11 is determined by the allocator.  Region 0-5 is
 3840 //         known from SELF's arguments and the Java calling convention.
 3841 //         Region 6-7 is determined per call site.
 3842 // Note 2: If the calling convention leaves holes in the incoming argument
 3843 //         area, those holes are owned by SELF.  Holes in the outgoing area
 3844 //         are owned by the CALLEE.  Holes should not be necessary in the
 3845 //         incoming area, as the Java calling convention is completely under
 3846 //         the control of the AD file.  Doubles can be sorted and packed to
 3847 //         avoid holes.  Holes in the outgoing arguments may be necessary for
 3848 //         varargs C calling conventions.
 3849 // Note 3: Region 0-3 is even aligned, with pad2 as needed.  Region 3-5 is
 3850 //         even aligned with pad0 as needed.
 3851 //         Region 6 is even aligned.  Region 6-7 is NOT even aligned;
 3852 //           (the latter is true on Intel but is it false on AArch64?)
 3853 //         region 6-11 is even aligned; it may be padded out more so that
 3854 //         the region from SP to FP meets the minimum stack alignment.
 3855 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
 3856 //         alignment.  Region 11, pad1, may be dynamically extended so that
 3857 //         SP meets the minimum alignment.
 3858 
 3859 frame %{
 3860   // These three registers define part of the calling convention
 3861   // between compiled code and the interpreter.
 3862 
 3863   // Inline Cache Register or Method for I2C.
 3864   inline_cache_reg(R12);
 3865 
 3866   // Number of stack slots consumed by locking an object
 3867   sync_stack_slots(2);
 3868 
 3869   // Compiled code's Frame Pointer
 3870   frame_pointer(R31);
 3871 
 3872   // Interpreter stores its frame pointer in a register which is
 3873   // stored to the stack by I2CAdaptors.
 3874   // I2CAdaptors convert from interpreted java to compiled java.
 3875   interpreter_frame_pointer(R29);
 3876 
 3877   // Stack alignment requirement
 3878   stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
 3879 
 3880   // Number of outgoing stack slots killed above the out_preserve_stack_slots
 3881   // for calls to C.  Supports the var-args backing area for register parms.
 3882   varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
 3883 
 3884   // The after-PROLOG location of the return address.  Location of
 3885   // return address specifies a type (REG or STACK) and a number
 3886   // representing the register number (i.e. - use a register name) or
 3887   // stack slot.
 3888   // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
 3889   // Otherwise, it is above the locks and verification slot and alignment word
 3890   // TODO this may well be correct but need to check why that - 2 is there
 3891   // ppc port uses 0 but we definitely need to allow for fixed_slots
 3892   // which folds in the space used for monitors
 3893   return_addr(STACK - 2 +
 3894               align_up((Compile::current()->in_preserve_stack_slots() +
 3895                         Compile::current()->fixed_slots()),
 3896                        stack_alignment_in_slots()));
 3897 
 3898   // Location of compiled Java return values.  Same as C for now.
 3899   return_value
 3900   %{
 3901     // TODO do we allow ideal_reg == Op_RegN???
 3902     assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
 3903            "only return normal values");
 3904 
 3905     static const int lo[Op_RegL + 1] = { // enum name
 3906       0,                                 // Op_Node
 3907       0,                                 // Op_Set
 3908       R0_num,                            // Op_RegN
 3909       R0_num,                            // Op_RegI
 3910       R0_num,                            // Op_RegP
 3911       V0_num,                            // Op_RegF
 3912       V0_num,                            // Op_RegD
 3913       R0_num                             // Op_RegL
 3914     };
 3915 
 3916     static const int hi[Op_RegL + 1] = { // enum name
 3917       0,                                 // Op_Node
 3918       0,                                 // Op_Set
 3919       OptoReg::Bad,                      // Op_RegN
 3920       OptoReg::Bad,                      // Op_RegI
 3921       R0_H_num,                          // Op_RegP
 3922       OptoReg::Bad,                      // Op_RegF
 3923       V0_H_num,                          // Op_RegD
 3924       R0_H_num                           // Op_RegL
 3925     };
 3926 
 3927     return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
 3928   %}
 3929 %}
 3930 
 3931 //----------ATTRIBUTES---------------------------------------------------------
 3932 //----------Operand Attributes-------------------------------------------------
 3933 op_attrib op_cost(1);        // Required cost attribute
 3934 
 3935 //----------Instruction Attributes---------------------------------------------
 3936 ins_attrib ins_cost(INSN_COST); // Required cost attribute
 3937 ins_attrib ins_size(32);        // Required size attribute (in bits)
 3938 ins_attrib ins_short_branch(0); // Required flag: is this instruction
 3939                                 // a non-matching short branch variant
 3940                                 // of some long branch?
 3941 ins_attrib ins_alignment(4);    // Required alignment attribute (must
 3942                                 // be a power of 2) specifies the
 3943                                 // alignment that some part of the
 3944                                 // instruction (not necessarily the
 3945                                 // start) requires.  If > 1, a
 3946                                 // compute_padding() function must be
 3947                                 // provided for the instruction
 3948 
 3949 //----------OPERANDS-----------------------------------------------------------
 3950 // Operand definitions must precede instruction definitions for correct parsing
 3951 // in the ADLC because operands constitute user defined types which are used in
 3952 // instruction definitions.
 3953 
 3954 //----------Simple Operands----------------------------------------------------
 3955 
 3956 // Integer operands 32 bit
 3957 // 32 bit immediate
 3958 operand immI()
 3959 %{
 3960   match(ConI);
 3961 
 3962   op_cost(0);
 3963   format %{ %}
 3964   interface(CONST_INTER);
 3965 %}
 3966 
 3967 // 32 bit zero
 3968 operand immI0()
 3969 %{
 3970   predicate(n->get_int() == 0);
 3971   match(ConI);
 3972 
 3973   op_cost(0);
 3974   format %{ %}
 3975   interface(CONST_INTER);
 3976 %}
 3977 
 3978 // 32 bit unit increment
 3979 operand immI_1()
 3980 %{
 3981   predicate(n->get_int() == 1);
 3982   match(ConI);
 3983 
 3984   op_cost(0);
 3985   format %{ %}
 3986   interface(CONST_INTER);
 3987 %}
 3988 
 3989 // 32 bit unit decrement
 3990 operand immI_M1()
 3991 %{
 3992   predicate(n->get_int() == -1);
 3993   match(ConI);
 3994 
 3995   op_cost(0);
 3996   format %{ %}
 3997   interface(CONST_INTER);
 3998 %}
 3999 
 4000 // Shift values for add/sub extension shift
 4001 operand immIExt()
 4002 %{
 4003   predicate(0 <= n->get_int() && (n->get_int() <= 4));
 4004   match(ConI);
 4005 
 4006   op_cost(0);
 4007   format %{ %}
 4008   interface(CONST_INTER);
 4009 %}
 4010 
 4011 operand immI_gt_1()
 4012 %{
 4013   predicate(n->get_int() > 1);
 4014   match(ConI);
 4015 
 4016   op_cost(0);
 4017   format %{ %}
 4018   interface(CONST_INTER);
 4019 %}
 4020 
 4021 operand immI_le_4()
 4022 %{
 4023   predicate(n->get_int() <= 4);
 4024   match(ConI);
 4025 
 4026   op_cost(0);
 4027   format %{ %}
 4028   interface(CONST_INTER);
 4029 %}
 4030 
 4031 operand immI_16()
 4032 %{
 4033   predicate(n->get_int() == 16);
 4034   match(ConI);
 4035 
 4036   op_cost(0);
 4037   format %{ %}
 4038   interface(CONST_INTER);
 4039 %}
 4040 
 4041 operand immI_24()
 4042 %{
 4043   predicate(n->get_int() == 24);
 4044   match(ConI);
 4045 
 4046   op_cost(0);
 4047   format %{ %}
 4048   interface(CONST_INTER);
 4049 %}
 4050 
 4051 operand immI_32()
 4052 %{
 4053   predicate(n->get_int() == 32);
 4054   match(ConI);
 4055 
 4056   op_cost(0);
 4057   format %{ %}
 4058   interface(CONST_INTER);
 4059 %}
 4060 
 4061 operand immI_48()
 4062 %{
 4063   predicate(n->get_int() == 48);
 4064   match(ConI);
 4065 
 4066   op_cost(0);
 4067   format %{ %}
 4068   interface(CONST_INTER);
 4069 %}
 4070 
 4071 operand immI_56()
 4072 %{
 4073   predicate(n->get_int() == 56);
 4074   match(ConI);
 4075 
 4076   op_cost(0);
 4077   format %{ %}
 4078   interface(CONST_INTER);
 4079 %}
 4080 
 4081 operand immI_255()
 4082 %{
 4083   predicate(n->get_int() == 255);
 4084   match(ConI);
 4085 
 4086   op_cost(0);
 4087   format %{ %}
 4088   interface(CONST_INTER);
 4089 %}
 4090 
 4091 operand immI_65535()
 4092 %{
 4093   predicate(n->get_int() == 65535);
 4094   match(ConI);
 4095 
 4096   op_cost(0);
 4097   format %{ %}
 4098   interface(CONST_INTER);
 4099 %}
 4100 
 4101 operand immI_positive()
 4102 %{
 4103   predicate(n->get_int() > 0);
 4104   match(ConI);
 4105 
 4106   op_cost(0);
 4107   format %{ %}
 4108   interface(CONST_INTER);
 4109 %}
 4110 
 4111 // BoolTest condition for signed compare
 4112 operand immI_cmp_cond()
 4113 %{
 4114   predicate(!Matcher::is_unsigned_booltest_pred(n->get_int()));
 4115   match(ConI);
 4116 
 4117   op_cost(0);
 4118   format %{ %}
 4119   interface(CONST_INTER);
 4120 %}
 4121 
 4122 // BoolTest condition for unsigned compare
 4123 operand immI_cmpU_cond()
 4124 %{
 4125   predicate(Matcher::is_unsigned_booltest_pred(n->get_int()));
 4126   match(ConI);
 4127 
 4128   op_cost(0);
 4129   format %{ %}
 4130   interface(CONST_INTER);
 4131 %}
 4132 
 4133 operand immL_255()
 4134 %{
 4135   predicate(n->get_long() == 255L);
 4136   match(ConL);
 4137 
 4138   op_cost(0);
 4139   format %{ %}
 4140   interface(CONST_INTER);
 4141 %}
 4142 
 4143 operand immL_65535()
 4144 %{
 4145   predicate(n->get_long() == 65535L);
 4146   match(ConL);
 4147 
 4148   op_cost(0);
 4149   format %{ %}
 4150   interface(CONST_INTER);
 4151 %}
 4152 
 4153 operand immL_4294967295()
 4154 %{
 4155   predicate(n->get_long() == 4294967295L);
 4156   match(ConL);
 4157 
 4158   op_cost(0);
 4159   format %{ %}
 4160   interface(CONST_INTER);
 4161 %}
 4162 
 4163 operand immL_bitmask()
 4164 %{
 4165   predicate((n->get_long() != 0)
 4166             && ((n->get_long() & 0xc000000000000000l) == 0)
 4167             && is_power_of_2(n->get_long() + 1));
 4168   match(ConL);
 4169 
 4170   op_cost(0);
 4171   format %{ %}
 4172   interface(CONST_INTER);
 4173 %}
 4174 
 4175 operand immI_bitmask()
 4176 %{
 4177   predicate((n->get_int() != 0)
 4178             && ((n->get_int() & 0xc0000000) == 0)
 4179             && is_power_of_2(n->get_int() + 1));
 4180   match(ConI);
 4181 
 4182   op_cost(0);
 4183   format %{ %}
 4184   interface(CONST_INTER);
 4185 %}
 4186 
 4187 operand immL_positive_bitmaskI()
 4188 %{
 4189   predicate((n->get_long() != 0)
 4190             && ((julong)n->get_long() < 0x80000000ULL)
 4191             && is_power_of_2(n->get_long() + 1));
 4192   match(ConL);
 4193 
 4194   op_cost(0);
 4195   format %{ %}
 4196   interface(CONST_INTER);
 4197 %}
 4198 
 4199 // Scale values for scaled offset addressing modes (up to long but not quad)
 4200 operand immIScale()
 4201 %{
 4202   predicate(0 <= n->get_int() && (n->get_int() <= 3));
 4203   match(ConI);
 4204 
 4205   op_cost(0);
 4206   format %{ %}
 4207   interface(CONST_INTER);
 4208 %}
 4209 
 4210 // 5 bit signed integer
 4211 operand immI5()
 4212 %{
 4213   predicate(Assembler::is_simm(n->get_int(), 5));
 4214   match(ConI);
 4215 
 4216   op_cost(0);
 4217   format %{ %}
 4218   interface(CONST_INTER);
 4219 %}
 4220 
 4221 // 7 bit unsigned integer
 4222 operand immIU7()
 4223 %{
 4224   predicate(Assembler::is_uimm(n->get_int(), 7));
 4225   match(ConI);
 4226 
 4227   op_cost(0);
 4228   format %{ %}
 4229   interface(CONST_INTER);
 4230 %}
 4231 
 4232 // Offset for scaled or unscaled immediate loads and stores
 4233 operand immIOffset()
 4234 %{
 4235   predicate(Address::offset_ok_for_immed(n->get_int(), 0));
 4236   match(ConI);
 4237 
 4238   op_cost(0);
 4239   format %{ %}
 4240   interface(CONST_INTER);
 4241 %}
 4242 
 4243 operand immIOffset1()
 4244 %{
 4245   predicate(Address::offset_ok_for_immed(n->get_int(), 0));
 4246   match(ConI);
 4247 
 4248   op_cost(0);
 4249   format %{ %}
 4250   interface(CONST_INTER);
 4251 %}
 4252 
 4253 operand immIOffset2()
 4254 %{
 4255   predicate(Address::offset_ok_for_immed(n->get_int(), 1));
 4256   match(ConI);
 4257 
 4258   op_cost(0);
 4259   format %{ %}
 4260   interface(CONST_INTER);
 4261 %}
 4262 
 4263 operand immIOffset4()
 4264 %{
 4265   predicate(Address::offset_ok_for_immed(n->get_int(), 2));
 4266   match(ConI);
 4267 
 4268   op_cost(0);
 4269   format %{ %}
 4270   interface(CONST_INTER);
 4271 %}
 4272 
 4273 operand immIOffset8()
 4274 %{
 4275   predicate(Address::offset_ok_for_immed(n->get_int(), 3));
 4276   match(ConI);
 4277 
 4278   op_cost(0);
 4279   format %{ %}
 4280   interface(CONST_INTER);
 4281 %}
 4282 
 4283 operand immIOffset16()
 4284 %{
 4285   predicate(Address::offset_ok_for_immed(n->get_int(), 4));
 4286   match(ConI);
 4287 
 4288   op_cost(0);
 4289   format %{ %}
 4290   interface(CONST_INTER);
 4291 %}
 4292 
 4293 operand immLOffset()
 4294 %{
 4295   predicate(n->get_long() >= -256 && n->get_long() <= 65520);
 4296   match(ConL);
 4297 
 4298   op_cost(0);
 4299   format %{ %}
 4300   interface(CONST_INTER);
 4301 %}
 4302 
 4303 operand immLoffset1()
 4304 %{
 4305   predicate(Address::offset_ok_for_immed(n->get_long(), 0));
 4306   match(ConL);
 4307 
 4308   op_cost(0);
 4309   format %{ %}
 4310   interface(CONST_INTER);
 4311 %}
 4312 
 4313 operand immLoffset2()
 4314 %{
 4315   predicate(Address::offset_ok_for_immed(n->get_long(), 1));
 4316   match(ConL);
 4317 
 4318   op_cost(0);
 4319   format %{ %}
 4320   interface(CONST_INTER);
 4321 %}
 4322 
 4323 operand immLoffset4()
 4324 %{
 4325   predicate(Address::offset_ok_for_immed(n->get_long(), 2));
 4326   match(ConL);
 4327 
 4328   op_cost(0);
 4329   format %{ %}
 4330   interface(CONST_INTER);
 4331 %}
 4332 
 4333 operand immLoffset8()
 4334 %{
 4335   predicate(Address::offset_ok_for_immed(n->get_long(), 3));
 4336   match(ConL);
 4337 
 4338   op_cost(0);
 4339   format %{ %}
 4340   interface(CONST_INTER);
 4341 %}
 4342 
 4343 operand immLoffset16()
 4344 %{
 4345   predicate(Address::offset_ok_for_immed(n->get_long(), 4));
 4346   match(ConL);
 4347 
 4348   op_cost(0);
 4349   format %{ %}
 4350   interface(CONST_INTER);
 4351 %}
 4352 
 4353 // 5 bit signed long integer
 4354 operand immL5()
 4355 %{
 4356   predicate(Assembler::is_simm(n->get_long(), 5));
 4357   match(ConL);
 4358 
 4359   op_cost(0);
 4360   format %{ %}
 4361   interface(CONST_INTER);
 4362 %}
 4363 
 4364 // 7 bit unsigned long integer
 4365 operand immLU7()
 4366 %{
 4367   predicate(Assembler::is_uimm(n->get_long(), 7));
 4368   match(ConL);
 4369 
 4370   op_cost(0);
 4371   format %{ %}
 4372   interface(CONST_INTER);
 4373 %}
 4374 
 4375 // 8 bit signed value.
 4376 operand immI8()
 4377 %{
 4378   predicate(n->get_int() <= 127 && n->get_int() >= -128);
 4379   match(ConI);
 4380 
 4381   op_cost(0);
 4382   format %{ %}
 4383   interface(CONST_INTER);
 4384 %}
 4385 
 4386 // 8 bit signed value (simm8), or #simm8 LSL 8.
 4387 operand immI8_shift8()
 4388 %{
 4389   predicate((n->get_int() <= 127 && n->get_int() >= -128) ||
 4390             (n->get_int() <= 32512 && n->get_int() >= -32768 && (n->get_int() & 0xff) == 0));
 4391   match(ConI);
 4392 
 4393   op_cost(0);
 4394   format %{ %}
 4395   interface(CONST_INTER);
 4396 %}
 4397 
 4398 // 8 bit signed value (simm8), or #simm8 LSL 8.
 4399 operand immL8_shift8()
 4400 %{
 4401   predicate((n->get_long() <= 127 && n->get_long() >= -128) ||
 4402             (n->get_long() <= 32512 && n->get_long() >= -32768 && (n->get_long() & 0xff) == 0));
 4403   match(ConL);
 4404 
 4405   op_cost(0);
 4406   format %{ %}
 4407   interface(CONST_INTER);
 4408 %}
 4409 
 4410 // 8 bit integer valid for vector add sub immediate
 4411 operand immBAddSubV()
 4412 %{
 4413   predicate(n->get_int() <= 255 && n->get_int() >= -255);
 4414   match(ConI);
 4415 
 4416   op_cost(0);
 4417   format %{ %}
 4418   interface(CONST_INTER);
 4419 %}
 4420 
 4421 // 32 bit integer valid for add sub immediate
 4422 operand immIAddSub()
 4423 %{
 4424   predicate(Assembler::operand_valid_for_add_sub_immediate((int64_t)n->get_int()));
 4425   match(ConI);
 4426   op_cost(0);
 4427   format %{ %}
 4428   interface(CONST_INTER);
 4429 %}
 4430 
 4431 // 32 bit integer valid for vector add sub immediate
 4432 operand immIAddSubV()
 4433 %{
 4434   predicate(Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)n->get_int()));
 4435   match(ConI);
 4436 
 4437   op_cost(0);
 4438   format %{ %}
 4439   interface(CONST_INTER);
 4440 %}
 4441 
 4442 // 32 bit unsigned integer valid for logical immediate
 4443 
 4444 operand immBLog()
 4445 %{
 4446   predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerByte, (uint64_t)n->get_int()));
 4447   match(ConI);
 4448 
 4449   op_cost(0);
 4450   format %{ %}
 4451   interface(CONST_INTER);
 4452 %}
 4453 
 4454 operand immSLog()
 4455 %{
 4456   predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerShort, (uint64_t)n->get_int()));
 4457   match(ConI);
 4458 
 4459   op_cost(0);
 4460   format %{ %}
 4461   interface(CONST_INTER);
 4462 %}
 4463 
 4464 operand immILog()
 4465 %{
 4466   predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/true, (uint64_t)n->get_int()));
 4467   match(ConI);
 4468 
 4469   op_cost(0);
 4470   format %{ %}
 4471   interface(CONST_INTER);
 4472 %}
 4473 
 4474 // Integer operands 64 bit
 4475 // 64 bit immediate
 4476 operand immL()
 4477 %{
 4478   match(ConL);
 4479 
 4480   op_cost(0);
 4481   format %{ %}
 4482   interface(CONST_INTER);
 4483 %}
 4484 
 4485 // 64 bit zero
 4486 operand immL0()
 4487 %{
 4488   predicate(n->get_long() == 0);
 4489   match(ConL);
 4490 
 4491   op_cost(0);
 4492   format %{ %}
 4493   interface(CONST_INTER);
 4494 %}
 4495 
 4496 // 64 bit unit decrement
 4497 operand immL_M1()
 4498 %{
 4499   predicate(n->get_long() == -1);
 4500   match(ConL);
 4501 
 4502   op_cost(0);
 4503   format %{ %}
 4504   interface(CONST_INTER);
 4505 %}
 4506 
 4507 // 64 bit integer valid for add sub immediate
 4508 operand immLAddSub()
 4509 %{
 4510   predicate(Assembler::operand_valid_for_add_sub_immediate(n->get_long()));
 4511   match(ConL);
 4512   op_cost(0);
 4513   format %{ %}
 4514   interface(CONST_INTER);
 4515 %}
 4516 
 4517 // 64 bit integer valid for addv subv immediate
 4518 operand immLAddSubV()
 4519 %{
 4520   predicate(Assembler::operand_valid_for_sve_add_sub_immediate(n->get_long()));
 4521   match(ConL);
 4522 
 4523   op_cost(0);
 4524   format %{ %}
 4525   interface(CONST_INTER);
 4526 %}
 4527 
 4528 // 64 bit integer valid for logical immediate
 4529 operand immLLog()
 4530 %{
 4531   predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/false, (uint64_t)n->get_long()));
 4532   match(ConL);
 4533   op_cost(0);
 4534   format %{ %}
 4535   interface(CONST_INTER);
 4536 %}
 4537 
 4538 // Long Immediate: low 32-bit mask
 4539 operand immL_32bits()
 4540 %{
 4541   predicate(n->get_long() == 0xFFFFFFFFL);
 4542   match(ConL);
 4543   op_cost(0);
 4544   format %{ %}
 4545   interface(CONST_INTER);
 4546 %}
 4547 
 4548 // Pointer operands
 4549 // Pointer Immediate
 4550 operand immP()
 4551 %{
 4552   match(ConP);
 4553 
 4554   op_cost(0);
 4555   format %{ %}
 4556   interface(CONST_INTER);
 4557 %}
 4558 
 4559 // nullptr Pointer Immediate
 4560 operand immP0()
 4561 %{
 4562   predicate(n->get_ptr() == 0);
 4563   match(ConP);
 4564 
 4565   op_cost(0);
 4566   format %{ %}
 4567   interface(CONST_INTER);
 4568 %}
 4569 
 4570 // Pointer Immediate One
 4571 // this is used in object initialization (initial object header)
 4572 operand immP_1()
 4573 %{
 4574   predicate(n->get_ptr() == 1);
 4575   match(ConP);
 4576 
 4577   op_cost(0);
 4578   format %{ %}
 4579   interface(CONST_INTER);
 4580 %}
 4581 
 4582 // Card Table Byte Map Base
 4583 operand immByteMapBase()
 4584 %{
 4585   // Get base of card map
 4586   predicate(BarrierSet::barrier_set()->is_a(BarrierSet::CardTableBarrierSet) &&
 4587             SHENANDOAHGC_ONLY(!BarrierSet::barrier_set()->is_a(BarrierSet::ShenandoahBarrierSet) &&)
 4588             (CardTable::CardValue*)n->get_ptr() == ((CardTableBarrierSet*)(BarrierSet::barrier_set()))->card_table()->byte_map_base());
 4589   match(ConP);
 4590 
 4591   op_cost(0);
 4592   format %{ %}
 4593   interface(CONST_INTER);
 4594 %}
 4595 
 4596 // Float and Double operands
 4597 // Double Immediate
 4598 operand immD()
 4599 %{
 4600   match(ConD);
 4601   op_cost(0);
 4602   format %{ %}
 4603   interface(CONST_INTER);
 4604 %}
 4605 
 4606 // Double Immediate: +0.0d
 4607 operand immD0()
 4608 %{
 4609   predicate(jlong_cast(n->getd()) == 0);
 4610   match(ConD);
 4611 
 4612   op_cost(0);
 4613   format %{ %}
 4614   interface(CONST_INTER);
 4615 %}
 4616 
 4617 // constant 'double +0.0'.
 4618 operand immDPacked()
 4619 %{
 4620   predicate(Assembler::operand_valid_for_float_immediate(n->getd()));
 4621   match(ConD);
 4622   op_cost(0);
 4623   format %{ %}
 4624   interface(CONST_INTER);
 4625 %}
 4626 
 4627 // Float Immediate
 4628 operand immF()
 4629 %{
 4630   match(ConF);
 4631   op_cost(0);
 4632   format %{ %}
 4633   interface(CONST_INTER);
 4634 %}
 4635 
 4636 // Float Immediate: +0.0f.
 4637 operand immF0()
 4638 %{
 4639   predicate(jint_cast(n->getf()) == 0);
 4640   match(ConF);
 4641 
 4642   op_cost(0);
 4643   format %{ %}
 4644   interface(CONST_INTER);
 4645 %}
 4646 
 4647 // Half Float (FP16) Immediate
 4648 operand immH()
 4649 %{
 4650   match(ConH);
 4651   op_cost(0);
 4652   format %{ %}
 4653   interface(CONST_INTER);
 4654 %}
 4655 
 4656 //
 4657 operand immFPacked()
 4658 %{
 4659   predicate(Assembler::operand_valid_for_float_immediate((double)n->getf()));
 4660   match(ConF);
 4661   op_cost(0);
 4662   format %{ %}
 4663   interface(CONST_INTER);
 4664 %}
 4665 
 4666 // Narrow pointer operands
 4667 // Narrow Pointer Immediate
 4668 operand immN()
 4669 %{
 4670   match(ConN);
 4671 
 4672   op_cost(0);
 4673   format %{ %}
 4674   interface(CONST_INTER);
 4675 %}
 4676 
 4677 // Narrow nullptr Pointer Immediate
 4678 operand immN0()
 4679 %{
 4680   predicate(n->get_narrowcon() == 0);
 4681   match(ConN);
 4682 
 4683   op_cost(0);
 4684   format %{ %}
 4685   interface(CONST_INTER);
 4686 %}
 4687 
 4688 operand immNKlass()
 4689 %{
 4690   match(ConNKlass);
 4691 
 4692   op_cost(0);
 4693   format %{ %}
 4694   interface(CONST_INTER);
 4695 %}
 4696 
 4697 // Integer 32 bit Register Operands
 4698 // Integer 32 bitRegister (excludes SP)
 4699 operand iRegI()
 4700 %{
 4701   constraint(ALLOC_IN_RC(any_reg32));
 4702   match(RegI);
 4703   match(iRegINoSp);
 4704   op_cost(0);
 4705   format %{ %}
 4706   interface(REG_INTER);
 4707 %}
 4708 
 4709 // Integer 32 bit Register not Special
 4710 operand iRegINoSp()
 4711 %{
 4712   constraint(ALLOC_IN_RC(no_special_reg32));
 4713   match(RegI);
 4714   op_cost(0);
 4715   format %{ %}
 4716   interface(REG_INTER);
 4717 %}
 4718 
 4719 // Integer 64 bit Register Operands
 4720 // Integer 64 bit Register (includes SP)
 4721 operand iRegL()
 4722 %{
 4723   constraint(ALLOC_IN_RC(any_reg));
 4724   match(RegL);
 4725   match(iRegLNoSp);
 4726   op_cost(0);
 4727   format %{ %}
 4728   interface(REG_INTER);
 4729 %}
 4730 
 4731 // Integer 64 bit Register not Special
 4732 operand iRegLNoSp()
 4733 %{
 4734   constraint(ALLOC_IN_RC(no_special_reg));
 4735   match(RegL);
 4736   match(iRegL_R0);
 4737   format %{ %}
 4738   interface(REG_INTER);
 4739 %}
 4740 
 4741 // Pointer Register Operands
 4742 // Pointer Register
 4743 operand iRegP()
 4744 %{
 4745   constraint(ALLOC_IN_RC(ptr_reg));
 4746   match(RegP);
 4747   match(iRegPNoSp);
 4748   match(iRegP_R0);
 4749   //match(iRegP_R2);
 4750   //match(iRegP_R4);
 4751   match(iRegP_R5);
 4752   match(thread_RegP);
 4753   op_cost(0);
 4754   format %{ %}
 4755   interface(REG_INTER);
 4756 %}
 4757 
 4758 // Pointer 64 bit Register not Special
 4759 operand iRegPNoSp()
 4760 %{
 4761   constraint(ALLOC_IN_RC(no_special_ptr_reg));
 4762   match(RegP);
 4763   // match(iRegP);
 4764   // match(iRegP_R0);
 4765   // match(iRegP_R2);
 4766   // match(iRegP_R4);
 4767   // match(iRegP_R5);
 4768   // match(thread_RegP);
 4769   op_cost(0);
 4770   format %{ %}
 4771   interface(REG_INTER);
 4772 %}
 4773 
 4774 // This operand is not allowed to use rfp even if
 4775 // rfp is not used to hold the frame pointer.
 4776 operand iRegPNoSpNoRfp()
 4777 %{
 4778   constraint(ALLOC_IN_RC(no_special_no_rfp_ptr_reg));
 4779   match(RegP);
 4780   match(iRegPNoSp);
 4781   op_cost(0);
 4782   format %{ %}
 4783   interface(REG_INTER);
 4784 %}
 4785 
 4786 // Pointer 64 bit Register R0 only
 4787 operand iRegP_R0()
 4788 %{
 4789   constraint(ALLOC_IN_RC(r0_reg));
 4790   match(RegP);
 4791   // match(iRegP);
 4792   match(iRegPNoSp);
 4793   op_cost(0);
 4794   format %{ %}
 4795   interface(REG_INTER);
 4796 %}
 4797 
 4798 // Pointer 64 bit Register R1 only
 4799 operand iRegP_R1()
 4800 %{
 4801   constraint(ALLOC_IN_RC(r1_reg));
 4802   match(RegP);
 4803   // match(iRegP);
 4804   match(iRegPNoSp);
 4805   op_cost(0);
 4806   format %{ %}
 4807   interface(REG_INTER);
 4808 %}
 4809 
 4810 // Pointer 64 bit Register R2 only
 4811 operand iRegP_R2()
 4812 %{
 4813   constraint(ALLOC_IN_RC(r2_reg));
 4814   match(RegP);
 4815   // match(iRegP);
 4816   match(iRegPNoSp);
 4817   op_cost(0);
 4818   format %{ %}
 4819   interface(REG_INTER);
 4820 %}
 4821 
 4822 // Pointer 64 bit Register R3 only
 4823 operand iRegP_R3()
 4824 %{
 4825   constraint(ALLOC_IN_RC(r3_reg));
 4826   match(RegP);
 4827   // match(iRegP);
 4828   match(iRegPNoSp);
 4829   op_cost(0);
 4830   format %{ %}
 4831   interface(REG_INTER);
 4832 %}
 4833 
 4834 // Pointer 64 bit Register R4 only
 4835 operand iRegP_R4()
 4836 %{
 4837   constraint(ALLOC_IN_RC(r4_reg));
 4838   match(RegP);
 4839   // match(iRegP);
 4840   match(iRegPNoSp);
 4841   op_cost(0);
 4842   format %{ %}
 4843   interface(REG_INTER);
 4844 %}
 4845 
 4846 // Pointer 64 bit Register R5 only
 4847 operand iRegP_R5()
 4848 %{
 4849   constraint(ALLOC_IN_RC(r5_reg));
 4850   match(RegP);
 4851   // match(iRegP);
 4852   match(iRegPNoSp);
 4853   op_cost(0);
 4854   format %{ %}
 4855   interface(REG_INTER);
 4856 %}
 4857 
 4858 // Pointer 64 bit Register R10 only
 4859 operand iRegP_R10()
 4860 %{
 4861   constraint(ALLOC_IN_RC(r10_reg));
 4862   match(RegP);
 4863   // match(iRegP);
 4864   match(iRegPNoSp);
 4865   op_cost(0);
 4866   format %{ %}
 4867   interface(REG_INTER);
 4868 %}
 4869 
 4870 // Long 64 bit Register R0 only
 4871 operand iRegL_R0()
 4872 %{
 4873   constraint(ALLOC_IN_RC(r0_reg));
 4874   match(RegL);
 4875   match(iRegLNoSp);
 4876   op_cost(0);
 4877   format %{ %}
 4878   interface(REG_INTER);
 4879 %}
 4880 
 4881 // Long 64 bit Register R11 only
 4882 operand iRegL_R11()
 4883 %{
 4884   constraint(ALLOC_IN_RC(r11_reg));
 4885   match(RegL);
 4886   match(iRegLNoSp);
 4887   op_cost(0);
 4888   format %{ %}
 4889   interface(REG_INTER);
 4890 %}
 4891 
 4892 // Register R0 only
 4893 operand iRegI_R0()
 4894 %{
 4895   constraint(ALLOC_IN_RC(int_r0_reg));
 4896   match(RegI);
 4897   match(iRegINoSp);
 4898   op_cost(0);
 4899   format %{ %}
 4900   interface(REG_INTER);
 4901 %}
 4902 
 4903 // Register R2 only
 4904 operand iRegI_R2()
 4905 %{
 4906   constraint(ALLOC_IN_RC(int_r2_reg));
 4907   match(RegI);
 4908   match(iRegINoSp);
 4909   op_cost(0);
 4910   format %{ %}
 4911   interface(REG_INTER);
 4912 %}
 4913 
 4914 // Register R3 only
 4915 operand iRegI_R3()
 4916 %{
 4917   constraint(ALLOC_IN_RC(int_r3_reg));
 4918   match(RegI);
 4919   match(iRegINoSp);
 4920   op_cost(0);
 4921   format %{ %}
 4922   interface(REG_INTER);
 4923 %}
 4924 
 4925 
 4926 // Register R4 only
 4927 operand iRegI_R4()
 4928 %{
 4929   constraint(ALLOC_IN_RC(int_r4_reg));
 4930   match(RegI);
 4931   match(iRegINoSp);
 4932   op_cost(0);
 4933   format %{ %}
 4934   interface(REG_INTER);
 4935 %}
 4936 
 4937 
 4938 // Pointer Register Operands
 4939 // Narrow Pointer Register
 4940 operand iRegN()
 4941 %{
 4942   constraint(ALLOC_IN_RC(any_reg32));
 4943   match(RegN);
 4944   match(iRegNNoSp);
 4945   op_cost(0);
 4946   format %{ %}
 4947   interface(REG_INTER);
 4948 %}
 4949 
 4950 // Integer 64 bit Register not Special
 4951 operand iRegNNoSp()
 4952 %{
 4953   constraint(ALLOC_IN_RC(no_special_reg32));
 4954   match(RegN);
 4955   op_cost(0);
 4956   format %{ %}
 4957   interface(REG_INTER);
 4958 %}
 4959 
 4960 // Float Register
 4961 // Float register operands
 4962 operand vRegF()
 4963 %{
 4964   constraint(ALLOC_IN_RC(float_reg));
 4965   match(RegF);
 4966 
 4967   op_cost(0);
 4968   format %{ %}
 4969   interface(REG_INTER);
 4970 %}
 4971 
 4972 // Double Register
 4973 // Double register operands
 4974 operand vRegD()
 4975 %{
 4976   constraint(ALLOC_IN_RC(double_reg));
 4977   match(RegD);
 4978 
 4979   op_cost(0);
 4980   format %{ %}
 4981   interface(REG_INTER);
 4982 %}
 4983 
 4984 // Generic vector class. This will be used for
 4985 // all vector operands, including NEON and SVE.
 4986 operand vReg()
 4987 %{
 4988   constraint(ALLOC_IN_RC(dynamic));
 4989   match(VecA);
 4990   match(VecD);
 4991   match(VecX);
 4992 
 4993   op_cost(0);
 4994   format %{ %}
 4995   interface(REG_INTER);
 4996 %}
 4997 
 4998 operand vecA()
 4999 %{
 5000   constraint(ALLOC_IN_RC(vectora_reg));
 5001   match(VecA);
 5002 
 5003   op_cost(0);
 5004   format %{ %}
 5005   interface(REG_INTER);
 5006 %}
 5007 
 5008 operand vecD()
 5009 %{
 5010   constraint(ALLOC_IN_RC(vectord_reg));
 5011   match(VecD);
 5012 
 5013   op_cost(0);
 5014   format %{ %}
 5015   interface(REG_INTER);
 5016 %}
 5017 
 5018 operand vecX()
 5019 %{
 5020   constraint(ALLOC_IN_RC(vectorx_reg));
 5021   match(VecX);
 5022 
 5023   op_cost(0);
 5024   format %{ %}
 5025   interface(REG_INTER);
 5026 %}
 5027 
 5028 operand vRegD_V0()
 5029 %{
 5030   constraint(ALLOC_IN_RC(v0_reg));
 5031   match(RegD);
 5032   op_cost(0);
 5033   format %{ %}
 5034   interface(REG_INTER);
 5035 %}
 5036 
 5037 operand vRegD_V1()
 5038 %{
 5039   constraint(ALLOC_IN_RC(v1_reg));
 5040   match(RegD);
 5041   op_cost(0);
 5042   format %{ %}
 5043   interface(REG_INTER);
 5044 %}
 5045 
 5046 operand vRegD_V2()
 5047 %{
 5048   constraint(ALLOC_IN_RC(v2_reg));
 5049   match(RegD);
 5050   op_cost(0);
 5051   format %{ %}
 5052   interface(REG_INTER);
 5053 %}
 5054 
 5055 operand vRegD_V3()
 5056 %{
 5057   constraint(ALLOC_IN_RC(v3_reg));
 5058   match(RegD);
 5059   op_cost(0);
 5060   format %{ %}
 5061   interface(REG_INTER);
 5062 %}
 5063 
 5064 operand vRegD_V4()
 5065 %{
 5066   constraint(ALLOC_IN_RC(v4_reg));
 5067   match(RegD);
 5068   op_cost(0);
 5069   format %{ %}
 5070   interface(REG_INTER);
 5071 %}
 5072 
 5073 operand vRegD_V5()
 5074 %{
 5075   constraint(ALLOC_IN_RC(v5_reg));
 5076   match(RegD);
 5077   op_cost(0);
 5078   format %{ %}
 5079   interface(REG_INTER);
 5080 %}
 5081 
 5082 operand vRegD_V6()
 5083 %{
 5084   constraint(ALLOC_IN_RC(v6_reg));
 5085   match(RegD);
 5086   op_cost(0);
 5087   format %{ %}
 5088   interface(REG_INTER);
 5089 %}
 5090 
 5091 operand vRegD_V7()
 5092 %{
 5093   constraint(ALLOC_IN_RC(v7_reg));
 5094   match(RegD);
 5095   op_cost(0);
 5096   format %{ %}
 5097   interface(REG_INTER);
 5098 %}
 5099 
 5100 operand vRegD_V12()
 5101 %{
 5102   constraint(ALLOC_IN_RC(v12_reg));
 5103   match(RegD);
 5104   op_cost(0);
 5105   format %{ %}
 5106   interface(REG_INTER);
 5107 %}
 5108 
 5109 operand vRegD_V13()
 5110 %{
 5111   constraint(ALLOC_IN_RC(v13_reg));
 5112   match(RegD);
 5113   op_cost(0);
 5114   format %{ %}
 5115   interface(REG_INTER);
 5116 %}
 5117 
 5118 operand pReg()
 5119 %{
 5120   constraint(ALLOC_IN_RC(pr_reg));
 5121   match(RegVectMask);
 5122   match(pRegGov);
 5123   op_cost(0);
 5124   format %{ %}
 5125   interface(REG_INTER);
 5126 %}
 5127 
 5128 operand pRegGov()
 5129 %{
 5130   constraint(ALLOC_IN_RC(gov_pr));
 5131   match(RegVectMask);
 5132   match(pReg);
 5133   op_cost(0);
 5134   format %{ %}
 5135   interface(REG_INTER);
 5136 %}
 5137 
 5138 operand pRegGov_P0()
 5139 %{
 5140   constraint(ALLOC_IN_RC(p0_reg));
 5141   match(RegVectMask);
 5142   op_cost(0);
 5143   format %{ %}
 5144   interface(REG_INTER);
 5145 %}
 5146 
 5147 operand pRegGov_P1()
 5148 %{
 5149   constraint(ALLOC_IN_RC(p1_reg));
 5150   match(RegVectMask);
 5151   op_cost(0);
 5152   format %{ %}
 5153   interface(REG_INTER);
 5154 %}
 5155 
 5156 // Flags register, used as output of signed compare instructions
 5157 
 5158 // note that on AArch64 we also use this register as the output for
 5159 // for floating point compare instructions (CmpF CmpD). this ensures
 5160 // that ordered inequality tests use GT, GE, LT or LE none of which
 5161 // pass through cases where the result is unordered i.e. one or both
 5162 // inputs to the compare is a NaN. this means that the ideal code can
 5163 // replace e.g. a GT with an LE and not end up capturing the NaN case
 5164 // (where the comparison should always fail). EQ and NE tests are
 5165 // always generated in ideal code so that unordered folds into the NE
 5166 // case, matching the behaviour of AArch64 NE.
 5167 //
 5168 // This differs from x86 where the outputs of FP compares use a
 5169 // special FP flags registers and where compares based on this
 5170 // register are distinguished into ordered inequalities (cmpOpUCF) and
 5171 // EQ/NEQ tests (cmpOpUCF2). x86 has to special case the latter tests
 5172 // to explicitly handle the unordered case in branches. x86 also has
 5173 // to include extra CMoveX rules to accept a cmpOpUCF input.
 5174 
 5175 operand rFlagsReg()
 5176 %{
 5177   constraint(ALLOC_IN_RC(int_flags));
 5178   match(RegFlags);
 5179 
 5180   op_cost(0);
 5181   format %{ "RFLAGS" %}
 5182   interface(REG_INTER);
 5183 %}
 5184 
 5185 // Flags register, used as output of unsigned compare instructions
 5186 operand rFlagsRegU()
 5187 %{
 5188   constraint(ALLOC_IN_RC(int_flags));
 5189   match(RegFlags);
 5190 
 5191   op_cost(0);
 5192   format %{ "RFLAGSU" %}
 5193   interface(REG_INTER);
 5194 %}
 5195 
 5196 // Special Registers
 5197 
 5198 // Method Register
 5199 operand inline_cache_RegP(iRegP reg)
 5200 %{
 5201   constraint(ALLOC_IN_RC(method_reg)); // inline_cache_reg
 5202   match(reg);
 5203   match(iRegPNoSp);
 5204   op_cost(0);
 5205   format %{ %}
 5206   interface(REG_INTER);
 5207 %}
 5208 
 5209 // Thread Register
 5210 operand thread_RegP(iRegP reg)
 5211 %{
 5212   constraint(ALLOC_IN_RC(thread_reg)); // link_reg
 5213   match(reg);
 5214   op_cost(0);
 5215   format %{ %}
 5216   interface(REG_INTER);
 5217 %}
 5218 
 5219 //----------Memory Operands----------------------------------------------------
 5220 
 5221 operand indirect(iRegP reg)
 5222 %{
 5223   constraint(ALLOC_IN_RC(ptr_reg));
 5224   match(reg);
 5225   op_cost(0);
 5226   format %{ "[$reg]" %}
 5227   interface(MEMORY_INTER) %{
 5228     base($reg);
 5229     index(0xffffffff);
 5230     scale(0x0);
 5231     disp(0x0);
 5232   %}
 5233 %}
 5234 
 5235 operand indIndexScaledI2L(iRegP reg, iRegI ireg, immIScale scale)
 5236 %{
 5237   constraint(ALLOC_IN_RC(ptr_reg));
 5238   predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 5239   match(AddP reg (LShiftL (ConvI2L ireg) scale));
 5240   op_cost(0);
 5241   format %{ "$reg, $ireg sxtw($scale), 0, I2L" %}
 5242   interface(MEMORY_INTER) %{
 5243     base($reg);
 5244     index($ireg);
 5245     scale($scale);
 5246     disp(0x0);
 5247   %}
 5248 %}
 5249 
 5250 operand indIndexScaled(iRegP reg, iRegL lreg, immIScale scale)
 5251 %{
 5252   constraint(ALLOC_IN_RC(ptr_reg));
 5253   predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 5254   match(AddP reg (LShiftL lreg scale));
 5255   op_cost(0);
 5256   format %{ "$reg, $lreg lsl($scale)" %}
 5257   interface(MEMORY_INTER) %{
 5258     base($reg);
 5259     index($lreg);
 5260     scale($scale);
 5261     disp(0x0);
 5262   %}
 5263 %}
 5264 
 5265 operand indIndexI2L(iRegP reg, iRegI ireg)
 5266 %{
 5267   constraint(ALLOC_IN_RC(ptr_reg));
 5268   match(AddP reg (ConvI2L ireg));
 5269   op_cost(0);
 5270   format %{ "$reg, $ireg, 0, I2L" %}
 5271   interface(MEMORY_INTER) %{
 5272     base($reg);
 5273     index($ireg);
 5274     scale(0x0);
 5275     disp(0x0);
 5276   %}
 5277 %}
 5278 
 5279 operand indIndex(iRegP reg, iRegL lreg)
 5280 %{
 5281   constraint(ALLOC_IN_RC(ptr_reg));
 5282   match(AddP reg lreg);
 5283   op_cost(0);
 5284   format %{ "$reg, $lreg" %}
 5285   interface(MEMORY_INTER) %{
 5286     base($reg);
 5287     index($lreg);
 5288     scale(0x0);
 5289     disp(0x0);
 5290   %}
 5291 %}
 5292 
 5293 operand indOffI1(iRegP reg, immIOffset1 off)
 5294 %{
 5295   constraint(ALLOC_IN_RC(ptr_reg));
 5296   match(AddP reg off);
 5297   op_cost(0);
 5298   format %{ "[$reg, $off]" %}
 5299   interface(MEMORY_INTER) %{
 5300     base($reg);
 5301     index(0xffffffff);
 5302     scale(0x0);
 5303     disp($off);
 5304   %}
 5305 %}
 5306 
 5307 operand indOffI2(iRegP reg, immIOffset2 off)
 5308 %{
 5309   constraint(ALLOC_IN_RC(ptr_reg));
 5310   match(AddP reg off);
 5311   op_cost(0);
 5312   format %{ "[$reg, $off]" %}
 5313   interface(MEMORY_INTER) %{
 5314     base($reg);
 5315     index(0xffffffff);
 5316     scale(0x0);
 5317     disp($off);
 5318   %}
 5319 %}
 5320 
 5321 operand indOffI4(iRegP reg, immIOffset4 off)
 5322 %{
 5323   constraint(ALLOC_IN_RC(ptr_reg));
 5324   match(AddP reg off);
 5325   op_cost(0);
 5326   format %{ "[$reg, $off]" %}
 5327   interface(MEMORY_INTER) %{
 5328     base($reg);
 5329     index(0xffffffff);
 5330     scale(0x0);
 5331     disp($off);
 5332   %}
 5333 %}
 5334 
 5335 operand indOffI8(iRegP reg, immIOffset8 off)
 5336 %{
 5337   constraint(ALLOC_IN_RC(ptr_reg));
 5338   match(AddP reg off);
 5339   op_cost(0);
 5340   format %{ "[$reg, $off]" %}
 5341   interface(MEMORY_INTER) %{
 5342     base($reg);
 5343     index(0xffffffff);
 5344     scale(0x0);
 5345     disp($off);
 5346   %}
 5347 %}
 5348 
 5349 operand indOffI16(iRegP reg, immIOffset16 off)
 5350 %{
 5351   constraint(ALLOC_IN_RC(ptr_reg));
 5352   match(AddP reg off);
 5353   op_cost(0);
 5354   format %{ "[$reg, $off]" %}
 5355   interface(MEMORY_INTER) %{
 5356     base($reg);
 5357     index(0xffffffff);
 5358     scale(0x0);
 5359     disp($off);
 5360   %}
 5361 %}
 5362 
 5363 operand indOffL1(iRegP reg, immLoffset1 off)
 5364 %{
 5365   constraint(ALLOC_IN_RC(ptr_reg));
 5366   match(AddP reg off);
 5367   op_cost(0);
 5368   format %{ "[$reg, $off]" %}
 5369   interface(MEMORY_INTER) %{
 5370     base($reg);
 5371     index(0xffffffff);
 5372     scale(0x0);
 5373     disp($off);
 5374   %}
 5375 %}
 5376 
 5377 operand indOffL2(iRegP reg, immLoffset2 off)
 5378 %{
 5379   constraint(ALLOC_IN_RC(ptr_reg));
 5380   match(AddP reg off);
 5381   op_cost(0);
 5382   format %{ "[$reg, $off]" %}
 5383   interface(MEMORY_INTER) %{
 5384     base($reg);
 5385     index(0xffffffff);
 5386     scale(0x0);
 5387     disp($off);
 5388   %}
 5389 %}
 5390 
 5391 operand indOffL4(iRegP reg, immLoffset4 off)
 5392 %{
 5393   constraint(ALLOC_IN_RC(ptr_reg));
 5394   match(AddP reg off);
 5395   op_cost(0);
 5396   format %{ "[$reg, $off]" %}
 5397   interface(MEMORY_INTER) %{
 5398     base($reg);
 5399     index(0xffffffff);
 5400     scale(0x0);
 5401     disp($off);
 5402   %}
 5403 %}
 5404 
 5405 operand indOffL8(iRegP reg, immLoffset8 off)
 5406 %{
 5407   constraint(ALLOC_IN_RC(ptr_reg));
 5408   match(AddP reg off);
 5409   op_cost(0);
 5410   format %{ "[$reg, $off]" %}
 5411   interface(MEMORY_INTER) %{
 5412     base($reg);
 5413     index(0xffffffff);
 5414     scale(0x0);
 5415     disp($off);
 5416   %}
 5417 %}
 5418 
 5419 operand indOffL16(iRegP reg, immLoffset16 off)
 5420 %{
 5421   constraint(ALLOC_IN_RC(ptr_reg));
 5422   match(AddP reg off);
 5423   op_cost(0);
 5424   format %{ "[$reg, $off]" %}
 5425   interface(MEMORY_INTER) %{
 5426     base($reg);
 5427     index(0xffffffff);
 5428     scale(0x0);
 5429     disp($off);
 5430   %}
 5431 %}
 5432 
 5433 operand indirectX2P(iRegL reg)
 5434 %{
 5435   constraint(ALLOC_IN_RC(ptr_reg));
 5436   match(CastX2P reg);
 5437   op_cost(0);
 5438   format %{ "[$reg]\t# long -> ptr" %}
 5439   interface(MEMORY_INTER) %{
 5440     base($reg);
 5441     index(0xffffffff);
 5442     scale(0x0);
 5443     disp(0x0);
 5444   %}
 5445 %}
 5446 
 5447 operand indOffX2P(iRegL reg, immLOffset off)
 5448 %{
 5449   constraint(ALLOC_IN_RC(ptr_reg));
 5450   match(AddP (CastX2P reg) off);
 5451   op_cost(0);
 5452   format %{ "[$reg, $off]\t# long -> ptr" %}
 5453   interface(MEMORY_INTER) %{
 5454     base($reg);
 5455     index(0xffffffff);
 5456     scale(0x0);
 5457     disp($off);
 5458   %}
 5459 %}
 5460 
 5461 operand indirectN(iRegN reg)
 5462 %{
 5463   predicate(CompressedOops::shift() == 0);
 5464   constraint(ALLOC_IN_RC(ptr_reg));
 5465   match(DecodeN reg);
 5466   op_cost(0);
 5467   format %{ "[$reg]\t# narrow" %}
 5468   interface(MEMORY_INTER) %{
 5469     base($reg);
 5470     index(0xffffffff);
 5471     scale(0x0);
 5472     disp(0x0);
 5473   %}
 5474 %}
 5475 
 5476 operand indIndexScaledI2LN(iRegN reg, iRegI ireg, immIScale scale)
 5477 %{
 5478   predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 5479   constraint(ALLOC_IN_RC(ptr_reg));
 5480   match(AddP (DecodeN reg) (LShiftL (ConvI2L ireg) scale));
 5481   op_cost(0);
 5482   format %{ "$reg, $ireg sxtw($scale), 0, I2L\t# narrow" %}
 5483   interface(MEMORY_INTER) %{
 5484     base($reg);
 5485     index($ireg);
 5486     scale($scale);
 5487     disp(0x0);
 5488   %}
 5489 %}
 5490 
 5491 operand indIndexScaledN(iRegN reg, iRegL lreg, immIScale scale)
 5492 %{
 5493   predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 5494   constraint(ALLOC_IN_RC(ptr_reg));
 5495   match(AddP (DecodeN reg) (LShiftL lreg scale));
 5496   op_cost(0);
 5497   format %{ "$reg, $lreg lsl($scale)\t# narrow" %}
 5498   interface(MEMORY_INTER) %{
 5499     base($reg);
 5500     index($lreg);
 5501     scale($scale);
 5502     disp(0x0);
 5503   %}
 5504 %}
 5505 
 5506 operand indIndexI2LN(iRegN reg, iRegI ireg)
 5507 %{
 5508   predicate(CompressedOops::shift() == 0);
 5509   constraint(ALLOC_IN_RC(ptr_reg));
 5510   match(AddP (DecodeN reg) (ConvI2L ireg));
 5511   op_cost(0);
 5512   format %{ "$reg, $ireg, 0, I2L\t# narrow" %}
 5513   interface(MEMORY_INTER) %{
 5514     base($reg);
 5515     index($ireg);
 5516     scale(0x0);
 5517     disp(0x0);
 5518   %}
 5519 %}
 5520 
 5521 operand indIndexN(iRegN reg, iRegL lreg)
 5522 %{
 5523   predicate(CompressedOops::shift() == 0);
 5524   constraint(ALLOC_IN_RC(ptr_reg));
 5525   match(AddP (DecodeN reg) lreg);
 5526   op_cost(0);
 5527   format %{ "$reg, $lreg\t# narrow" %}
 5528   interface(MEMORY_INTER) %{
 5529     base($reg);
 5530     index($lreg);
 5531     scale(0x0);
 5532     disp(0x0);
 5533   %}
 5534 %}
 5535 
 5536 operand indOffIN(iRegN reg, immIOffset off)
 5537 %{
 5538   predicate(CompressedOops::shift() == 0);
 5539   constraint(ALLOC_IN_RC(ptr_reg));
 5540   match(AddP (DecodeN reg) off);
 5541   op_cost(0);
 5542   format %{ "[$reg, $off]\t# narrow" %}
 5543   interface(MEMORY_INTER) %{
 5544     base($reg);
 5545     index(0xffffffff);
 5546     scale(0x0);
 5547     disp($off);
 5548   %}
 5549 %}
 5550 
 5551 operand indOffLN(iRegN reg, immLOffset off)
 5552 %{
 5553   predicate(CompressedOops::shift() == 0);
 5554   constraint(ALLOC_IN_RC(ptr_reg));
 5555   match(AddP (DecodeN reg) off);
 5556   op_cost(0);
 5557   format %{ "[$reg, $off]\t# narrow" %}
 5558   interface(MEMORY_INTER) %{
 5559     base($reg);
 5560     index(0xffffffff);
 5561     scale(0x0);
 5562     disp($off);
 5563   %}
 5564 %}
 5565 
 5566 
 5567 //----------Special Memory Operands--------------------------------------------
 5568 // Stack Slot Operand - This operand is used for loading and storing temporary
 5569 //                      values on the stack where a match requires a value to
 5570 //                      flow through memory.
 5571 operand stackSlotP(sRegP reg)
 5572 %{
 5573   constraint(ALLOC_IN_RC(stack_slots));
 5574   op_cost(100);
 5575   // No match rule because this operand is only generated in matching
 5576   // match(RegP);
 5577   format %{ "[$reg]" %}
 5578   interface(MEMORY_INTER) %{
 5579     base(0x1e);  // RSP
 5580     index(0x0);  // No Index
 5581     scale(0x0);  // No Scale
 5582     disp($reg);  // Stack Offset
 5583   %}
 5584 %}
 5585 
 5586 operand stackSlotI(sRegI reg)
 5587 %{
 5588   constraint(ALLOC_IN_RC(stack_slots));
 5589   // No match rule because this operand is only generated in matching
 5590   // match(RegI);
 5591   format %{ "[$reg]" %}
 5592   interface(MEMORY_INTER) %{
 5593     base(0x1e);  // RSP
 5594     index(0x0);  // No Index
 5595     scale(0x0);  // No Scale
 5596     disp($reg);  // Stack Offset
 5597   %}
 5598 %}
 5599 
 5600 operand stackSlotF(sRegF reg)
 5601 %{
 5602   constraint(ALLOC_IN_RC(stack_slots));
 5603   // No match rule because this operand is only generated in matching
 5604   // match(RegF);
 5605   format %{ "[$reg]" %}
 5606   interface(MEMORY_INTER) %{
 5607     base(0x1e);  // RSP
 5608     index(0x0);  // No Index
 5609     scale(0x0);  // No Scale
 5610     disp($reg);  // Stack Offset
 5611   %}
 5612 %}
 5613 
 5614 operand stackSlotD(sRegD reg)
 5615 %{
 5616   constraint(ALLOC_IN_RC(stack_slots));
 5617   // No match rule because this operand is only generated in matching
 5618   // match(RegD);
 5619   format %{ "[$reg]" %}
 5620   interface(MEMORY_INTER) %{
 5621     base(0x1e);  // RSP
 5622     index(0x0);  // No Index
 5623     scale(0x0);  // No Scale
 5624     disp($reg);  // Stack Offset
 5625   %}
 5626 %}
 5627 
 5628 operand stackSlotL(sRegL reg)
 5629 %{
 5630   constraint(ALLOC_IN_RC(stack_slots));
 5631   // No match rule because this operand is only generated in matching
 5632   // match(RegL);
 5633   format %{ "[$reg]" %}
 5634   interface(MEMORY_INTER) %{
 5635     base(0x1e);  // RSP
 5636     index(0x0);  // No Index
 5637     scale(0x0);  // No Scale
 5638     disp($reg);  // Stack Offset
 5639   %}
 5640 %}
 5641 
 5642 // Operands for expressing Control Flow
 5643 // NOTE: Label is a predefined operand which should not be redefined in
 5644 //       the AD file. It is generically handled within the ADLC.
 5645 
 5646 //----------Conditional Branch Operands----------------------------------------
 5647 // Comparison Op  - This is the operation of the comparison, and is limited to
 5648 //                  the following set of codes:
 5649 //                  L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
 5650 //
 5651 // Other attributes of the comparison, such as unsignedness, are specified
 5652 // by the comparison instruction that sets a condition code flags register.
 5653 // That result is represented by a flags operand whose subtype is appropriate
 5654 // to the unsignedness (etc.) of the comparison.
 5655 //
 5656 // Later, the instruction which matches both the Comparison Op (a Bool) and
 5657 // the flags (produced by the Cmp) specifies the coding of the comparison op
 5658 // by matching a specific subtype of Bool operand below, such as cmpOpU.
 5659 
 5660 // used for signed integral comparisons and fp comparisons
 5661 
 5662 operand cmpOp()
 5663 %{
 5664   match(Bool);
 5665 
 5666   format %{ "" %}
 5667   interface(COND_INTER) %{
 5668     equal(0x0, "eq");
 5669     not_equal(0x1, "ne");
 5670     less(0xb, "lt");
 5671     greater_equal(0xa, "ge");
 5672     less_equal(0xd, "le");
 5673     greater(0xc, "gt");
 5674     overflow(0x6, "vs");
 5675     no_overflow(0x7, "vc");
 5676   %}
 5677 %}
 5678 
 5679 // used for unsigned integral comparisons
 5680 
 5681 operand cmpOpU()
 5682 %{
 5683   match(Bool);
 5684 
 5685   format %{ "" %}
 5686   interface(COND_INTER) %{
 5687     equal(0x0, "eq");
 5688     not_equal(0x1, "ne");
 5689     less(0x3, "lo");
 5690     greater_equal(0x2, "hs");
 5691     less_equal(0x9, "ls");
 5692     greater(0x8, "hi");
 5693     overflow(0x6, "vs");
 5694     no_overflow(0x7, "vc");
 5695   %}
 5696 %}
 5697 
 5698 // used for certain integral comparisons which can be
 5699 // converted to cbxx or tbxx instructions
 5700 
 5701 operand cmpOpEqNe()
 5702 %{
 5703   match(Bool);
 5704   op_cost(0);
 5705   predicate(n->as_Bool()->_test._test == BoolTest::ne
 5706             || n->as_Bool()->_test._test == BoolTest::eq);
 5707 
 5708   format %{ "" %}
 5709   interface(COND_INTER) %{
 5710     equal(0x0, "eq");
 5711     not_equal(0x1, "ne");
 5712     less(0xb, "lt");
 5713     greater_equal(0xa, "ge");
 5714     less_equal(0xd, "le");
 5715     greater(0xc, "gt");
 5716     overflow(0x6, "vs");
 5717     no_overflow(0x7, "vc");
 5718   %}
 5719 %}
 5720 
 5721 // used for certain integral comparisons which can be
 5722 // converted to cbxx or tbxx instructions
 5723 
 5724 operand cmpOpLtGe()
 5725 %{
 5726   match(Bool);
 5727   op_cost(0);
 5728 
 5729   predicate(n->as_Bool()->_test._test == BoolTest::lt
 5730             || n->as_Bool()->_test._test == BoolTest::ge);
 5731 
 5732   format %{ "" %}
 5733   interface(COND_INTER) %{
 5734     equal(0x0, "eq");
 5735     not_equal(0x1, "ne");
 5736     less(0xb, "lt");
 5737     greater_equal(0xa, "ge");
 5738     less_equal(0xd, "le");
 5739     greater(0xc, "gt");
 5740     overflow(0x6, "vs");
 5741     no_overflow(0x7, "vc");
 5742   %}
 5743 %}
 5744 
 5745 // used for certain unsigned integral comparisons which can be
 5746 // converted to cbxx or tbxx instructions
 5747 
 5748 operand cmpOpUEqNeLeGt()
 5749 %{
 5750   match(Bool);
 5751   op_cost(0);
 5752 
 5753   predicate(n->as_Bool()->_test._test == BoolTest::eq ||
 5754             n->as_Bool()->_test._test == BoolTest::ne ||
 5755             n->as_Bool()->_test._test == BoolTest::le ||
 5756             n->as_Bool()->_test._test == BoolTest::gt);
 5757 
 5758   format %{ "" %}
 5759   interface(COND_INTER) %{
 5760     equal(0x0, "eq");
 5761     not_equal(0x1, "ne");
 5762     less(0x3, "lo");
 5763     greater_equal(0x2, "hs");
 5764     less_equal(0x9, "ls");
 5765     greater(0x8, "hi");
 5766     overflow(0x6, "vs");
 5767     no_overflow(0x7, "vc");
 5768   %}
 5769 %}
 5770 
 5771 // Special operand allowing long args to int ops to be truncated for free
 5772 
 5773 operand iRegL2I(iRegL reg) %{
 5774 
 5775   op_cost(0);
 5776 
 5777   match(ConvL2I reg);
 5778 
 5779   format %{ "l2i($reg)" %}
 5780 
 5781   interface(REG_INTER)
 5782 %}
 5783 
 5784 operand iRegL2P(iRegL reg) %{
 5785 
 5786   op_cost(0);
 5787 
 5788   match(CastX2P reg);
 5789 
 5790   format %{ "l2p($reg)" %}
 5791 
 5792   interface(REG_INTER)
 5793 %}
 5794 
 5795 opclass vmem2(indirect, indIndex, indOffI2, indOffL2);
 5796 opclass vmem4(indirect, indIndex, indOffI4, indOffL4);
 5797 opclass vmem8(indirect, indIndex, indOffI8, indOffL8);
 5798 opclass vmem16(indirect, indIndex, indOffI16, indOffL16);
 5799 
 5800 //----------OPERAND CLASSES----------------------------------------------------
 5801 // Operand Classes are groups of operands that are used as to simplify
 5802 // instruction definitions by not requiring the AD writer to specify
 5803 // separate instructions for every form of operand when the
 5804 // instruction accepts multiple operand types with the same basic
 5805 // encoding and format. The classic case of this is memory operands.
 5806 
 5807 // memory is used to define read/write location for load/store
 5808 // instruction defs. we can turn a memory op into an Address
 5809 
 5810 opclass memory1(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI1, indOffL1,
 5811                 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
 5812 
 5813 opclass memory2(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI2, indOffL2,
 5814                 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
 5815 
 5816 opclass memory4(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI4, indOffL4,
 5817                 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
 5818 
 5819 opclass memory8(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI8, indOffL8,
 5820                 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
 5821 
 5822 // All of the memory operands. For the pipeline description.
 5823 opclass memory(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex,
 5824                indOffI1, indOffL1, indOffI2, indOffL2, indOffI4, indOffL4, indOffI8, indOffL8,
 5825                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
 5826 
 5827 
 5828 // iRegIorL2I is used for src inputs in rules for 32 bit int (I)
 5829 // operations. it allows the src to be either an iRegI or a (ConvL2I
 5830 // iRegL). in the latter case the l2i normally planted for a ConvL2I
 5831 // can be elided because the 32-bit instruction will just employ the
 5832 // lower 32 bits anyway.
 5833 //
 5834 // n.b. this does not elide all L2I conversions. if the truncated
 5835 // value is consumed by more than one operation then the ConvL2I
 5836 // cannot be bundled into the consuming nodes so an l2i gets planted
 5837 // (actually a movw $dst $src) and the downstream instructions consume
 5838 // the result of the l2i as an iRegI input. That's a shame since the
 5839 // movw is actually redundant but its not too costly.
 5840 
 5841 opclass iRegIorL2I(iRegI, iRegL2I);
 5842 opclass iRegPorL2P(iRegP, iRegL2P);
 5843 
 5844 //----------PIPELINE-----------------------------------------------------------
 5845 // Rules which define the behavior of the target architectures pipeline.
 5846 
 5847 // For specific pipelines, eg A53, define the stages of that pipeline
 5848 //pipe_desc(ISS, EX1, EX2, WR);
 5849 #define ISS S0
 5850 #define EX1 S1
 5851 #define EX2 S2
 5852 #define WR  S3
 5853 
 5854 // Integer ALU reg operation
 5855 pipeline %{
 5856 
 5857 attributes %{
 5858   // ARM instructions are of fixed length
 5859   fixed_size_instructions;        // Fixed size instructions TODO does
 5860   max_instructions_per_bundle = 4;   // A53 = 2, A57 = 4
 5861   // ARM instructions come in 32-bit word units
 5862   instruction_unit_size = 4;         // An instruction is 4 bytes long
 5863   instruction_fetch_unit_size = 64;  // The processor fetches one line
 5864   instruction_fetch_units = 1;       // of 64 bytes
 5865 
 5866   // List of nop instructions
 5867   nops( MachNop );
 5868 %}
 5869 
 5870 // We don't use an actual pipeline model so don't care about resources
 5871 // or description. we do use pipeline classes to introduce fixed
 5872 // latencies
 5873 
 5874 //----------RESOURCES----------------------------------------------------------
 5875 // Resources are the functional units available to the machine
 5876 
 5877 resources( INS0, INS1, INS01 = INS0 | INS1,
 5878            ALU0, ALU1, ALU = ALU0 | ALU1,
 5879            MAC,
 5880            DIV,
 5881            BRANCH,
 5882            LDST,
 5883            NEON_FP);
 5884 
 5885 //----------PIPELINE DESCRIPTION-----------------------------------------------
 5886 // Pipeline Description specifies the stages in the machine's pipeline
 5887 
 5888 // Define the pipeline as a generic 6 stage pipeline
 5889 pipe_desc(S0, S1, S2, S3, S4, S5);
 5890 
 5891 //----------PIPELINE CLASSES---------------------------------------------------
 5892 // Pipeline Classes describe the stages in which input and output are
 5893 // referenced by the hardware pipeline.
 5894 
 5895 pipe_class fp_dop_reg_reg_s(vRegF dst, vRegF src1, vRegF src2)
 5896 %{
 5897   single_instruction;
 5898   src1   : S1(read);
 5899   src2   : S2(read);
 5900   dst    : S5(write);
 5901   INS01  : ISS;
 5902   NEON_FP : S5;
 5903 %}
 5904 
 5905 pipe_class fp_dop_reg_reg_d(vRegD dst, vRegD src1, vRegD src2)
 5906 %{
 5907   single_instruction;
 5908   src1   : S1(read);
 5909   src2   : S2(read);
 5910   dst    : S5(write);
 5911   INS01  : ISS;
 5912   NEON_FP : S5;
 5913 %}
 5914 
 5915 pipe_class fp_uop_s(vRegF dst, vRegF src)
 5916 %{
 5917   single_instruction;
 5918   src    : S1(read);
 5919   dst    : S5(write);
 5920   INS01  : ISS;
 5921   NEON_FP : S5;
 5922 %}
 5923 
 5924 pipe_class fp_uop_d(vRegD dst, vRegD src)
 5925 %{
 5926   single_instruction;
 5927   src    : S1(read);
 5928   dst    : S5(write);
 5929   INS01  : ISS;
 5930   NEON_FP : S5;
 5931 %}
 5932 
 5933 pipe_class fp_d2f(vRegF dst, vRegD src)
 5934 %{
 5935   single_instruction;
 5936   src    : S1(read);
 5937   dst    : S5(write);
 5938   INS01  : ISS;
 5939   NEON_FP : S5;
 5940 %}
 5941 
 5942 pipe_class fp_f2d(vRegD dst, vRegF src)
 5943 %{
 5944   single_instruction;
 5945   src    : S1(read);
 5946   dst    : S5(write);
 5947   INS01  : ISS;
 5948   NEON_FP : S5;
 5949 %}
 5950 
 5951 pipe_class fp_f2i(iRegINoSp dst, vRegF src)
 5952 %{
 5953   single_instruction;
 5954   src    : S1(read);
 5955   dst    : S5(write);
 5956   INS01  : ISS;
 5957   NEON_FP : S5;
 5958 %}
 5959 
 5960 pipe_class fp_f2l(iRegLNoSp dst, vRegF src)
 5961 %{
 5962   single_instruction;
 5963   src    : S1(read);
 5964   dst    : S5(write);
 5965   INS01  : ISS;
 5966   NEON_FP : S5;
 5967 %}
 5968 
 5969 pipe_class fp_i2f(vRegF dst, iRegIorL2I src)
 5970 %{
 5971   single_instruction;
 5972   src    : S1(read);
 5973   dst    : S5(write);
 5974   INS01  : ISS;
 5975   NEON_FP : S5;
 5976 %}
 5977 
 5978 pipe_class fp_l2f(vRegF dst, iRegL src)
 5979 %{
 5980   single_instruction;
 5981   src    : S1(read);
 5982   dst    : S5(write);
 5983   INS01  : ISS;
 5984   NEON_FP : S5;
 5985 %}
 5986 
 5987 pipe_class fp_d2i(iRegINoSp dst, vRegD src)
 5988 %{
 5989   single_instruction;
 5990   src    : S1(read);
 5991   dst    : S5(write);
 5992   INS01  : ISS;
 5993   NEON_FP : S5;
 5994 %}
 5995 
 5996 pipe_class fp_d2l(iRegLNoSp dst, vRegD src)
 5997 %{
 5998   single_instruction;
 5999   src    : S1(read);
 6000   dst    : S5(write);
 6001   INS01  : ISS;
 6002   NEON_FP : S5;
 6003 %}
 6004 
 6005 pipe_class fp_i2d(vRegD dst, iRegIorL2I src)
 6006 %{
 6007   single_instruction;
 6008   src    : S1(read);
 6009   dst    : S5(write);
 6010   INS01  : ISS;
 6011   NEON_FP : S5;
 6012 %}
 6013 
 6014 pipe_class fp_l2d(vRegD dst, iRegIorL2I src)
 6015 %{
 6016   single_instruction;
 6017   src    : S1(read);
 6018   dst    : S5(write);
 6019   INS01  : ISS;
 6020   NEON_FP : S5;
 6021 %}
 6022 
 6023 pipe_class fp_div_s(vRegF dst, vRegF src1, vRegF src2)
 6024 %{
 6025   single_instruction;
 6026   src1   : S1(read);
 6027   src2   : S2(read);
 6028   dst    : S5(write);
 6029   INS0   : ISS;
 6030   NEON_FP : S5;
 6031 %}
 6032 
 6033 pipe_class fp_div_d(vRegD dst, vRegD src1, vRegD src2)
 6034 %{
 6035   single_instruction;
 6036   src1   : S1(read);
 6037   src2   : S2(read);
 6038   dst    : S5(write);
 6039   INS0   : ISS;
 6040   NEON_FP : S5;
 6041 %}
 6042 
 6043 pipe_class fp_cond_reg_reg_s(vRegF dst, vRegF src1, vRegF src2, rFlagsReg cr)
 6044 %{
 6045   single_instruction;
 6046   cr     : S1(read);
 6047   src1   : S1(read);
 6048   src2   : S1(read);
 6049   dst    : S3(write);
 6050   INS01  : ISS;
 6051   NEON_FP : S3;
 6052 %}
 6053 
 6054 pipe_class fp_cond_reg_reg_d(vRegD dst, vRegD src1, vRegD src2, rFlagsReg cr)
 6055 %{
 6056   single_instruction;
 6057   cr     : S1(read);
 6058   src1   : S1(read);
 6059   src2   : S1(read);
 6060   dst    : S3(write);
 6061   INS01  : ISS;
 6062   NEON_FP : S3;
 6063 %}
 6064 
 6065 pipe_class fp_imm_s(vRegF dst)
 6066 %{
 6067   single_instruction;
 6068   dst    : S3(write);
 6069   INS01  : ISS;
 6070   NEON_FP : S3;
 6071 %}
 6072 
 6073 pipe_class fp_imm_d(vRegD dst)
 6074 %{
 6075   single_instruction;
 6076   dst    : S3(write);
 6077   INS01  : ISS;
 6078   NEON_FP : S3;
 6079 %}
 6080 
 6081 pipe_class fp_load_constant_s(vRegF dst)
 6082 %{
 6083   single_instruction;
 6084   dst    : S4(write);
 6085   INS01  : ISS;
 6086   NEON_FP : S4;
 6087 %}
 6088 
 6089 pipe_class fp_load_constant_d(vRegD dst)
 6090 %{
 6091   single_instruction;
 6092   dst    : S4(write);
 6093   INS01  : ISS;
 6094   NEON_FP : S4;
 6095 %}
 6096 
 6097 //------- Integer ALU operations --------------------------
 6098 
 6099 // Integer ALU reg-reg operation
 6100 // Operands needed in EX1, result generated in EX2
 6101 // Eg.  ADD     x0, x1, x2
 6102 pipe_class ialu_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 6103 %{
 6104   single_instruction;
 6105   dst    : EX2(write);
 6106   src1   : EX1(read);
 6107   src2   : EX1(read);
 6108   INS01  : ISS; // Dual issue as instruction 0 or 1
 6109   ALU    : EX2;
 6110 %}
 6111 
 6112 // Integer ALU reg-reg operation with constant shift
 6113 // Shifted register must be available in LATE_ISS instead of EX1
 6114 // Eg.  ADD     x0, x1, x2, LSL #2
 6115 pipe_class ialu_reg_reg_shift(iRegI dst, iRegI src1, iRegI src2, immI shift)
 6116 %{
 6117   single_instruction;
 6118   dst    : EX2(write);
 6119   src1   : EX1(read);
 6120   src2   : ISS(read);
 6121   INS01  : ISS;
 6122   ALU    : EX2;
 6123 %}
 6124 
 6125 // Integer ALU reg operation with constant shift
 6126 // Eg.  LSL     x0, x1, #shift
 6127 pipe_class ialu_reg_shift(iRegI dst, iRegI src1)
 6128 %{
 6129   single_instruction;
 6130   dst    : EX2(write);
 6131   src1   : ISS(read);
 6132   INS01  : ISS;
 6133   ALU    : EX2;
 6134 %}
 6135 
 6136 // Integer ALU reg-reg operation with variable shift
 6137 // Both operands must be available in LATE_ISS instead of EX1
 6138 // Result is available in EX1 instead of EX2
 6139 // Eg.  LSLV    x0, x1, x2
 6140 pipe_class ialu_reg_reg_vshift(iRegI dst, iRegI src1, iRegI src2)
 6141 %{
 6142   single_instruction;
 6143   dst    : EX1(write);
 6144   src1   : ISS(read);
 6145   src2   : ISS(read);
 6146   INS01  : ISS;
 6147   ALU    : EX1;
 6148 %}
 6149 
 6150 // Integer ALU reg-reg operation with extract
 6151 // As for _vshift above, but result generated in EX2
 6152 // Eg.  EXTR    x0, x1, x2, #N
 6153 pipe_class ialu_reg_reg_extr(iRegI dst, iRegI src1, iRegI src2)
 6154 %{
 6155   single_instruction;
 6156   dst    : EX2(write);
 6157   src1   : ISS(read);
 6158   src2   : ISS(read);
 6159   INS1   : ISS; // Can only dual issue as Instruction 1
 6160   ALU    : EX1;
 6161 %}
 6162 
 6163 // Integer ALU reg operation
 6164 // Eg.  NEG     x0, x1
 6165 pipe_class ialu_reg(iRegI dst, iRegI src)
 6166 %{
 6167   single_instruction;
 6168   dst    : EX2(write);
 6169   src    : EX1(read);
 6170   INS01  : ISS;
 6171   ALU    : EX2;
 6172 %}
 6173 
 6174 // Integer ALU reg mmediate operation
 6175 // Eg.  ADD     x0, x1, #N
 6176 pipe_class ialu_reg_imm(iRegI dst, iRegI src1)
 6177 %{
 6178   single_instruction;
 6179   dst    : EX2(write);
 6180   src1   : EX1(read);
 6181   INS01  : ISS;
 6182   ALU    : EX2;
 6183 %}
 6184 
 6185 // Integer ALU immediate operation (no source operands)
 6186 // Eg.  MOV     x0, #N
 6187 pipe_class ialu_imm(iRegI dst)
 6188 %{
 6189   single_instruction;
 6190   dst    : EX1(write);
 6191   INS01  : ISS;
 6192   ALU    : EX1;
 6193 %}
 6194 
 6195 //------- Compare operation -------------------------------
 6196 
 6197 // Compare reg-reg
 6198 // Eg.  CMP     x0, x1
 6199 pipe_class icmp_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
 6200 %{
 6201   single_instruction;
 6202 //  fixed_latency(16);
 6203   cr     : EX2(write);
 6204   op1    : EX1(read);
 6205   op2    : EX1(read);
 6206   INS01  : ISS;
 6207   ALU    : EX2;
 6208 %}
 6209 
 6210 // Compare reg-reg
 6211 // Eg.  CMP     x0, #N
 6212 pipe_class icmp_reg_imm(rFlagsReg cr, iRegI op1)
 6213 %{
 6214   single_instruction;
 6215 //  fixed_latency(16);
 6216   cr     : EX2(write);
 6217   op1    : EX1(read);
 6218   INS01  : ISS;
 6219   ALU    : EX2;
 6220 %}
 6221 
 6222 //------- Conditional instructions ------------------------
 6223 
 6224 // Conditional no operands
 6225 // Eg.  CSINC   x0, zr, zr, <cond>
 6226 pipe_class icond_none(iRegI dst, rFlagsReg cr)
 6227 %{
 6228   single_instruction;
 6229   cr     : EX1(read);
 6230   dst    : EX2(write);
 6231   INS01  : ISS;
 6232   ALU    : EX2;
 6233 %}
 6234 
 6235 // Conditional 2 operand
 6236 // EG.  CSEL    X0, X1, X2, <cond>
 6237 pipe_class icond_reg_reg(iRegI dst, iRegI src1, iRegI src2, rFlagsReg cr)
 6238 %{
 6239   single_instruction;
 6240   cr     : EX1(read);
 6241   src1   : EX1(read);
 6242   src2   : EX1(read);
 6243   dst    : EX2(write);
 6244   INS01  : ISS;
 6245   ALU    : EX2;
 6246 %}
 6247 
 6248 // Conditional 2 operand
 6249 // EG.  CSEL    X0, X1, X2, <cond>
 6250 pipe_class icond_reg(iRegI dst, iRegI src, rFlagsReg cr)
 6251 %{
 6252   single_instruction;
 6253   cr     : EX1(read);
 6254   src    : EX1(read);
 6255   dst    : EX2(write);
 6256   INS01  : ISS;
 6257   ALU    : EX2;
 6258 %}
 6259 
 6260 //------- Multiply pipeline operations --------------------
 6261 
 6262 // Multiply reg-reg
 6263 // Eg.  MUL     w0, w1, w2
 6264 pipe_class imul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 6265 %{
 6266   single_instruction;
 6267   dst    : WR(write);
 6268   src1   : ISS(read);
 6269   src2   : ISS(read);
 6270   INS01  : ISS;
 6271   MAC    : WR;
 6272 %}
 6273 
 6274 // Multiply accumulate
 6275 // Eg.  MADD    w0, w1, w2, w3
 6276 pipe_class imac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
 6277 %{
 6278   single_instruction;
 6279   dst    : WR(write);
 6280   src1   : ISS(read);
 6281   src2   : ISS(read);
 6282   src3   : ISS(read);
 6283   INS01  : ISS;
 6284   MAC    : WR;
 6285 %}
 6286 
 6287 // Eg.  MUL     w0, w1, w2
 6288 pipe_class lmul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 6289 %{
 6290   single_instruction;
 6291   fixed_latency(3); // Maximum latency for 64 bit mul
 6292   dst    : WR(write);
 6293   src1   : ISS(read);
 6294   src2   : ISS(read);
 6295   INS01  : ISS;
 6296   MAC    : WR;
 6297 %}
 6298 
 6299 // Multiply accumulate
 6300 // Eg.  MADD    w0, w1, w2, w3
 6301 pipe_class lmac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
 6302 %{
 6303   single_instruction;
 6304   fixed_latency(3); // Maximum latency for 64 bit mul
 6305   dst    : WR(write);
 6306   src1   : ISS(read);
 6307   src2   : ISS(read);
 6308   src3   : ISS(read);
 6309   INS01  : ISS;
 6310   MAC    : WR;
 6311 %}
 6312 
 6313 //------- Divide pipeline operations --------------------
 6314 
 6315 // Eg.  SDIV    w0, w1, w2
 6316 pipe_class idiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 6317 %{
 6318   single_instruction;
 6319   fixed_latency(8); // Maximum latency for 32 bit divide
 6320   dst    : WR(write);
 6321   src1   : ISS(read);
 6322   src2   : ISS(read);
 6323   INS0   : ISS; // Can only dual issue as instruction 0
 6324   DIV    : WR;
 6325 %}
 6326 
 6327 // Eg.  SDIV    x0, x1, x2
 6328 pipe_class ldiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 6329 %{
 6330   single_instruction;
 6331   fixed_latency(16); // Maximum latency for 64 bit divide
 6332   dst    : WR(write);
 6333   src1   : ISS(read);
 6334   src2   : ISS(read);
 6335   INS0   : ISS; // Can only dual issue as instruction 0
 6336   DIV    : WR;
 6337 %}
 6338 
 6339 //------- Load pipeline operations ------------------------
 6340 
 6341 // Load - prefetch
 6342 // Eg.  PFRM    <mem>
 6343 pipe_class iload_prefetch(memory mem)
 6344 %{
 6345   single_instruction;
 6346   mem    : ISS(read);
 6347   INS01  : ISS;
 6348   LDST   : WR;
 6349 %}
 6350 
 6351 // Load - reg, mem
 6352 // Eg.  LDR     x0, <mem>
 6353 pipe_class iload_reg_mem(iRegI dst, memory mem)
 6354 %{
 6355   single_instruction;
 6356   dst    : WR(write);
 6357   mem    : ISS(read);
 6358   INS01  : ISS;
 6359   LDST   : WR;
 6360 %}
 6361 
 6362 // Load - reg, reg
 6363 // Eg.  LDR     x0, [sp, x1]
 6364 pipe_class iload_reg_reg(iRegI dst, iRegI src)
 6365 %{
 6366   single_instruction;
 6367   dst    : WR(write);
 6368   src    : ISS(read);
 6369   INS01  : ISS;
 6370   LDST   : WR;
 6371 %}
 6372 
 6373 //------- Store pipeline operations -----------------------
 6374 
 6375 // Store - zr, mem
 6376 // Eg.  STR     zr, <mem>
 6377 pipe_class istore_mem(memory mem)
 6378 %{
 6379   single_instruction;
 6380   mem    : ISS(read);
 6381   INS01  : ISS;
 6382   LDST   : WR;
 6383 %}
 6384 
 6385 // Store - reg, mem
 6386 // Eg.  STR     x0, <mem>
 6387 pipe_class istore_reg_mem(iRegI src, memory mem)
 6388 %{
 6389   single_instruction;
 6390   mem    : ISS(read);
 6391   src    : EX2(read);
 6392   INS01  : ISS;
 6393   LDST   : WR;
 6394 %}
 6395 
 6396 // Store - reg, reg
 6397 // Eg. STR      x0, [sp, x1]
 6398 pipe_class istore_reg_reg(iRegI dst, iRegI src)
 6399 %{
 6400   single_instruction;
 6401   dst    : ISS(read);
 6402   src    : EX2(read);
 6403   INS01  : ISS;
 6404   LDST   : WR;
 6405 %}
 6406 
 6407 //------- Store pipeline operations -----------------------
 6408 
 6409 // Branch
 6410 pipe_class pipe_branch()
 6411 %{
 6412   single_instruction;
 6413   INS01  : ISS;
 6414   BRANCH : EX1;
 6415 %}
 6416 
 6417 // Conditional branch
 6418 pipe_class pipe_branch_cond(rFlagsReg cr)
 6419 %{
 6420   single_instruction;
 6421   cr     : EX1(read);
 6422   INS01  : ISS;
 6423   BRANCH : EX1;
 6424 %}
 6425 
 6426 // Compare & Branch
 6427 // EG.  CBZ/CBNZ
 6428 pipe_class pipe_cmp_branch(iRegI op1)
 6429 %{
 6430   single_instruction;
 6431   op1    : EX1(read);
 6432   INS01  : ISS;
 6433   BRANCH : EX1;
 6434 %}
 6435 
 6436 //------- Synchronisation operations ----------------------
 6437 
 6438 // Any operation requiring serialization.
 6439 // EG.  DMB/Atomic Ops/Load Acquire/Str Release
 6440 pipe_class pipe_serial()
 6441 %{
 6442   single_instruction;
 6443   force_serialization;
 6444   fixed_latency(16);
 6445   INS01  : ISS(2); // Cannot dual issue with any other instruction
 6446   LDST   : WR;
 6447 %}
 6448 
 6449 // Generic big/slow expanded idiom - also serialized
 6450 pipe_class pipe_slow()
 6451 %{
 6452   instruction_count(10);
 6453   multiple_bundles;
 6454   force_serialization;
 6455   fixed_latency(16);
 6456   INS01  : ISS(2); // Cannot dual issue with any other instruction
 6457   LDST   : WR;
 6458 %}
 6459 
 6460 // Empty pipeline class
 6461 pipe_class pipe_class_empty()
 6462 %{
 6463   single_instruction;
 6464   fixed_latency(0);
 6465 %}
 6466 
 6467 // Default pipeline class.
 6468 pipe_class pipe_class_default()
 6469 %{
 6470   single_instruction;
 6471   fixed_latency(2);
 6472 %}
 6473 
 6474 // Pipeline class for compares.
 6475 pipe_class pipe_class_compare()
 6476 %{
 6477   single_instruction;
 6478   fixed_latency(16);
 6479 %}
 6480 
 6481 // Pipeline class for memory operations.
 6482 pipe_class pipe_class_memory()
 6483 %{
 6484   single_instruction;
 6485   fixed_latency(16);
 6486 %}
 6487 
 6488 // Pipeline class for call.
 6489 pipe_class pipe_class_call()
 6490 %{
 6491   single_instruction;
 6492   fixed_latency(100);
 6493 %}
 6494 
 6495 // Define the class for the Nop node.
 6496 define %{
 6497    MachNop = pipe_class_empty;
 6498 %}
 6499 
 6500 %}
 6501 //----------INSTRUCTIONS-------------------------------------------------------
 6502 //
 6503 // match      -- States which machine-independent subtree may be replaced
 6504 //               by this instruction.
 6505 // ins_cost   -- The estimated cost of this instruction is used by instruction
 6506 //               selection to identify a minimum cost tree of machine
 6507 //               instructions that matches a tree of machine-independent
 6508 //               instructions.
 6509 // format     -- A string providing the disassembly for this instruction.
 6510 //               The value of an instruction's operand may be inserted
 6511 //               by referring to it with a '$' prefix.
 6512 // opcode     -- Three instruction opcodes may be provided.  These are referred
 6513 //               to within an encode class as $primary, $secondary, and $tertiary
 6514 //               rrspectively.  The primary opcode is commonly used to
 6515 //               indicate the type of machine instruction, while secondary
 6516 //               and tertiary are often used for prefix options or addressing
 6517 //               modes.
 6518 // ins_encode -- A list of encode classes with parameters. The encode class
 6519 //               name must have been defined in an 'enc_class' specification
 6520 //               in the encode section of the architecture description.
 6521 
 6522 // ============================================================================
 6523 // Memory (Load/Store) Instructions
 6524 
 6525 // Load Instructions
 6526 
 6527 // Load Byte (8 bit signed)
 6528 instruct loadB(iRegINoSp dst, memory1 mem)
 6529 %{
 6530   match(Set dst (LoadB mem));
 6531   predicate(!needs_acquiring_load(n));
 6532 
 6533   ins_cost(4 * INSN_COST);
 6534   format %{ "ldrsbw  $dst, $mem\t# byte" %}
 6535 
 6536   ins_encode(aarch64_enc_ldrsbw(dst, mem));
 6537 
 6538   ins_pipe(iload_reg_mem);
 6539 %}
 6540 
 6541 // Load Byte (8 bit signed) into long
 6542 instruct loadB2L(iRegLNoSp dst, memory1 mem)
 6543 %{
 6544   match(Set dst (ConvI2L (LoadB mem)));
 6545   predicate(!needs_acquiring_load(n->in(1)));
 6546 
 6547   ins_cost(4 * INSN_COST);
 6548   format %{ "ldrsb  $dst, $mem\t# byte" %}
 6549 
 6550   ins_encode(aarch64_enc_ldrsb(dst, mem));
 6551 
 6552   ins_pipe(iload_reg_mem);
 6553 %}
 6554 
 6555 // Load Byte (8 bit unsigned)
 6556 instruct loadUB(iRegINoSp dst, memory1 mem)
 6557 %{
 6558   match(Set dst (LoadUB mem));
 6559   predicate(!needs_acquiring_load(n));
 6560 
 6561   ins_cost(4 * INSN_COST);
 6562   format %{ "ldrbw  $dst, $mem\t# byte" %}
 6563 
 6564   ins_encode(aarch64_enc_ldrb(dst, mem));
 6565 
 6566   ins_pipe(iload_reg_mem);
 6567 %}
 6568 
 6569 // Load Byte (8 bit unsigned) into long
 6570 instruct loadUB2L(iRegLNoSp dst, memory1 mem)
 6571 %{
 6572   match(Set dst (ConvI2L (LoadUB mem)));
 6573   predicate(!needs_acquiring_load(n->in(1)));
 6574 
 6575   ins_cost(4 * INSN_COST);
 6576   format %{ "ldrb  $dst, $mem\t# byte" %}
 6577 
 6578   ins_encode(aarch64_enc_ldrb(dst, mem));
 6579 
 6580   ins_pipe(iload_reg_mem);
 6581 %}
 6582 
 6583 // Load Short (16 bit signed)
 6584 instruct loadS(iRegINoSp dst, memory2 mem)
 6585 %{
 6586   match(Set dst (LoadS mem));
 6587   predicate(!needs_acquiring_load(n));
 6588 
 6589   ins_cost(4 * INSN_COST);
 6590   format %{ "ldrshw  $dst, $mem\t# short" %}
 6591 
 6592   ins_encode(aarch64_enc_ldrshw(dst, mem));
 6593 
 6594   ins_pipe(iload_reg_mem);
 6595 %}
 6596 
 6597 // Load Short (16 bit signed) into long
 6598 instruct loadS2L(iRegLNoSp dst, memory2 mem)
 6599 %{
 6600   match(Set dst (ConvI2L (LoadS mem)));
 6601   predicate(!needs_acquiring_load(n->in(1)));
 6602 
 6603   ins_cost(4 * INSN_COST);
 6604   format %{ "ldrsh  $dst, $mem\t# short" %}
 6605 
 6606   ins_encode(aarch64_enc_ldrsh(dst, mem));
 6607 
 6608   ins_pipe(iload_reg_mem);
 6609 %}
 6610 
 6611 // Load Char (16 bit unsigned)
 6612 instruct loadUS(iRegINoSp dst, memory2 mem)
 6613 %{
 6614   match(Set dst (LoadUS mem));
 6615   predicate(!needs_acquiring_load(n));
 6616 
 6617   ins_cost(4 * INSN_COST);
 6618   format %{ "ldrh  $dst, $mem\t# short" %}
 6619 
 6620   ins_encode(aarch64_enc_ldrh(dst, mem));
 6621 
 6622   ins_pipe(iload_reg_mem);
 6623 %}
 6624 
 6625 // Load Short/Char (16 bit unsigned) into long
 6626 instruct loadUS2L(iRegLNoSp dst, memory2 mem)
 6627 %{
 6628   match(Set dst (ConvI2L (LoadUS mem)));
 6629   predicate(!needs_acquiring_load(n->in(1)));
 6630 
 6631   ins_cost(4 * INSN_COST);
 6632   format %{ "ldrh  $dst, $mem\t# short" %}
 6633 
 6634   ins_encode(aarch64_enc_ldrh(dst, mem));
 6635 
 6636   ins_pipe(iload_reg_mem);
 6637 %}
 6638 
 6639 // Load Integer (32 bit signed)
 6640 instruct loadI(iRegINoSp dst, memory4 mem)
 6641 %{
 6642   match(Set dst (LoadI mem));
 6643   predicate(!needs_acquiring_load(n));
 6644 
 6645   ins_cost(4 * INSN_COST);
 6646   format %{ "ldrw  $dst, $mem\t# int" %}
 6647 
 6648   ins_encode(aarch64_enc_ldrw(dst, mem));
 6649 
 6650   ins_pipe(iload_reg_mem);
 6651 %}
 6652 
 6653 // Load Integer (32 bit signed) into long
 6654 instruct loadI2L(iRegLNoSp dst, memory4 mem)
 6655 %{
 6656   match(Set dst (ConvI2L (LoadI mem)));
 6657   predicate(!needs_acquiring_load(n->in(1)));
 6658 
 6659   ins_cost(4 * INSN_COST);
 6660   format %{ "ldrsw  $dst, $mem\t# int" %}
 6661 
 6662   ins_encode(aarch64_enc_ldrsw(dst, mem));
 6663 
 6664   ins_pipe(iload_reg_mem);
 6665 %}
 6666 
 6667 // Load Integer (32 bit unsigned) into long
 6668 instruct loadUI2L(iRegLNoSp dst, memory4 mem, immL_32bits mask)
 6669 %{
 6670   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
 6671   predicate(!needs_acquiring_load(n->in(1)->in(1)->as_Load()));
 6672 
 6673   ins_cost(4 * INSN_COST);
 6674   format %{ "ldrw  $dst, $mem\t# int" %}
 6675 
 6676   ins_encode(aarch64_enc_ldrw(dst, mem));
 6677 
 6678   ins_pipe(iload_reg_mem);
 6679 %}
 6680 
 6681 // Load Long (64 bit signed)
 6682 instruct loadL(iRegLNoSp dst, memory8 mem)
 6683 %{
 6684   match(Set dst (LoadL mem));
 6685   predicate(!needs_acquiring_load(n));
 6686 
 6687   ins_cost(4 * INSN_COST);
 6688   format %{ "ldr  $dst, $mem\t# int" %}
 6689 
 6690   ins_encode(aarch64_enc_ldr(dst, mem));
 6691 
 6692   ins_pipe(iload_reg_mem);
 6693 %}
 6694 
 6695 // Load Range
 6696 instruct loadRange(iRegINoSp dst, memory4 mem)
 6697 %{
 6698   match(Set dst (LoadRange mem));
 6699 
 6700   ins_cost(4 * INSN_COST);
 6701   format %{ "ldrw  $dst, $mem\t# range" %}
 6702 
 6703   ins_encode(aarch64_enc_ldrw(dst, mem));
 6704 
 6705   ins_pipe(iload_reg_mem);
 6706 %}
 6707 
 6708 // Load Pointer
 6709 instruct loadP(iRegPNoSp dst, memory8 mem)
 6710 %{
 6711   match(Set dst (LoadP mem));
 6712   predicate(!needs_acquiring_load(n) && (n->as_Load()->barrier_data() == 0));
 6713 
 6714   ins_cost(4 * INSN_COST);
 6715   format %{ "ldr  $dst, $mem\t# ptr" %}
 6716 
 6717   ins_encode(aarch64_enc_ldr(dst, mem));
 6718 
 6719   ins_pipe(iload_reg_mem);
 6720 %}
 6721 
 6722 // Load Compressed Pointer
 6723 instruct loadN(iRegNNoSp dst, memory4 mem)
 6724 %{
 6725   match(Set dst (LoadN mem));
 6726   predicate(!needs_acquiring_load(n) && n->as_Load()->barrier_data() == 0);
 6727 
 6728   ins_cost(4 * INSN_COST);
 6729   format %{ "ldrw  $dst, $mem\t# compressed ptr" %}
 6730 
 6731   ins_encode(aarch64_enc_ldrw(dst, mem));
 6732 
 6733   ins_pipe(iload_reg_mem);
 6734 %}
 6735 
 6736 // Load Klass Pointer
 6737 instruct loadKlass(iRegPNoSp dst, memory8 mem)
 6738 %{
 6739   match(Set dst (LoadKlass mem));
 6740   predicate(!needs_acquiring_load(n));
 6741 
 6742   ins_cost(4 * INSN_COST);
 6743   format %{ "ldr  $dst, $mem\t# class" %}
 6744 
 6745   ins_encode(aarch64_enc_ldr(dst, mem));
 6746 
 6747   ins_pipe(iload_reg_mem);
 6748 %}
 6749 
 6750 // Load Narrow Klass Pointer
 6751 instruct loadNKlass(iRegNNoSp dst, memory4 mem)
 6752 %{
 6753   match(Set dst (LoadNKlass mem));
 6754   predicate(!needs_acquiring_load(n) && !UseCompactObjectHeaders);
 6755 
 6756   ins_cost(4 * INSN_COST);
 6757   format %{ "ldrw  $dst, $mem\t# compressed class ptr" %}
 6758 
 6759   ins_encode(aarch64_enc_ldrw(dst, mem));
 6760 
 6761   ins_pipe(iload_reg_mem);
 6762 %}
 6763 
 6764 instruct loadNKlassCompactHeaders(iRegNNoSp dst, memory4 mem)
 6765 %{
 6766   match(Set dst (LoadNKlass mem));
 6767   predicate(!needs_acquiring_load(n) && UseCompactObjectHeaders);
 6768 
 6769   ins_cost(4 * INSN_COST);
 6770   format %{
 6771     "ldrw  $dst, $mem\t# compressed class ptr, shifted\n\t"
 6772     "lsrw  $dst, $dst, markWord::klass_shift_at_offset"
 6773   %}
 6774   ins_encode %{
 6775     // inlined aarch64_enc_ldrw
 6776     loadStore(masm, &MacroAssembler::ldrw, $dst$$Register, $mem->opcode(),
 6777               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 6778     __ lsrw($dst$$Register, $dst$$Register, markWord::klass_shift_at_offset);
 6779   %}
 6780   ins_pipe(iload_reg_mem);
 6781 %}
 6782 
 6783 // Load Float
 6784 instruct loadF(vRegF dst, memory4 mem)
 6785 %{
 6786   match(Set dst (LoadF mem));
 6787   predicate(!needs_acquiring_load(n));
 6788 
 6789   ins_cost(4 * INSN_COST);
 6790   format %{ "ldrs  $dst, $mem\t# float" %}
 6791 
 6792   ins_encode( aarch64_enc_ldrs(dst, mem) );
 6793 
 6794   ins_pipe(pipe_class_memory);
 6795 %}
 6796 
 6797 // Load Double
 6798 instruct loadD(vRegD dst, memory8 mem)
 6799 %{
 6800   match(Set dst (LoadD mem));
 6801   predicate(!needs_acquiring_load(n));
 6802 
 6803   ins_cost(4 * INSN_COST);
 6804   format %{ "ldrd  $dst, $mem\t# double" %}
 6805 
 6806   ins_encode( aarch64_enc_ldrd(dst, mem) );
 6807 
 6808   ins_pipe(pipe_class_memory);
 6809 %}
 6810 
 6811 
 6812 // Load Int Constant
 6813 instruct loadConI(iRegINoSp dst, immI src)
 6814 %{
 6815   match(Set dst src);
 6816 
 6817   ins_cost(INSN_COST);
 6818   format %{ "mov $dst, $src\t# int" %}
 6819 
 6820   ins_encode( aarch64_enc_movw_imm(dst, src) );
 6821 
 6822   ins_pipe(ialu_imm);
 6823 %}
 6824 
 6825 // Load Long Constant
 6826 instruct loadConL(iRegLNoSp dst, immL src)
 6827 %{
 6828   match(Set dst src);
 6829 
 6830   ins_cost(INSN_COST);
 6831   format %{ "mov $dst, $src\t# long" %}
 6832 
 6833   ins_encode( aarch64_enc_mov_imm(dst, src) );
 6834 
 6835   ins_pipe(ialu_imm);
 6836 %}
 6837 
 6838 // Load Pointer Constant
 6839 
 6840 instruct loadConP(iRegPNoSp dst, immP con)
 6841 %{
 6842   match(Set dst con);
 6843 
 6844   ins_cost(INSN_COST * 4);
 6845   format %{
 6846     "mov  $dst, $con\t# ptr"
 6847   %}
 6848 
 6849   ins_encode(aarch64_enc_mov_p(dst, con));
 6850 
 6851   ins_pipe(ialu_imm);
 6852 %}
 6853 
 6854 // Load Null Pointer Constant
 6855 
 6856 instruct loadConP0(iRegPNoSp dst, immP0 con)
 6857 %{
 6858   match(Set dst con);
 6859 
 6860   ins_cost(INSN_COST);
 6861   format %{ "mov  $dst, $con\t# nullptr ptr" %}
 6862 
 6863   ins_encode(aarch64_enc_mov_p0(dst, con));
 6864 
 6865   ins_pipe(ialu_imm);
 6866 %}
 6867 
 6868 // Load Pointer Constant One
 6869 
 6870 instruct loadConP1(iRegPNoSp dst, immP_1 con)
 6871 %{
 6872   match(Set dst con);
 6873 
 6874   ins_cost(INSN_COST);
 6875   format %{ "mov  $dst, $con\t# nullptr ptr" %}
 6876 
 6877   ins_encode(aarch64_enc_mov_p1(dst, con));
 6878 
 6879   ins_pipe(ialu_imm);
 6880 %}
 6881 
 6882 // Load Byte Map Base Constant
 6883 
 6884 instruct loadByteMapBase(iRegPNoSp dst, immByteMapBase con)
 6885 %{
 6886   match(Set dst con);
 6887 
 6888   ins_cost(INSN_COST);
 6889   format %{ "adr  $dst, $con\t# Byte Map Base" %}
 6890 
 6891   ins_encode(aarch64_enc_mov_byte_map_base(dst, con));
 6892 
 6893   ins_pipe(ialu_imm);
 6894 %}
 6895 
 6896 // Load Narrow Pointer Constant
 6897 
 6898 instruct loadConN(iRegNNoSp dst, immN con)
 6899 %{
 6900   match(Set dst con);
 6901 
 6902   ins_cost(INSN_COST * 4);
 6903   format %{ "mov  $dst, $con\t# compressed ptr" %}
 6904 
 6905   ins_encode(aarch64_enc_mov_n(dst, con));
 6906 
 6907   ins_pipe(ialu_imm);
 6908 %}
 6909 
 6910 // Load Narrow Null Pointer Constant
 6911 
 6912 instruct loadConN0(iRegNNoSp dst, immN0 con)
 6913 %{
 6914   match(Set dst con);
 6915 
 6916   ins_cost(INSN_COST);
 6917   format %{ "mov  $dst, $con\t# compressed nullptr ptr" %}
 6918 
 6919   ins_encode(aarch64_enc_mov_n0(dst, con));
 6920 
 6921   ins_pipe(ialu_imm);
 6922 %}
 6923 
 6924 // Load Narrow Klass Constant
 6925 
 6926 instruct loadConNKlass(iRegNNoSp dst, immNKlass con)
 6927 %{
 6928   match(Set dst con);
 6929 
 6930   ins_cost(INSN_COST);
 6931   format %{ "mov  $dst, $con\t# compressed klass ptr" %}
 6932 
 6933   ins_encode(aarch64_enc_mov_nk(dst, con));
 6934 
 6935   ins_pipe(ialu_imm);
 6936 %}
 6937 
 6938 // Load Packed Float Constant
 6939 
 6940 instruct loadConF_packed(vRegF dst, immFPacked con) %{
 6941   match(Set dst con);
 6942   ins_cost(INSN_COST * 4);
 6943   format %{ "fmovs  $dst, $con"%}
 6944   ins_encode %{
 6945     __ fmovs(as_FloatRegister($dst$$reg), (double)$con$$constant);
 6946   %}
 6947 
 6948   ins_pipe(fp_imm_s);
 6949 %}
 6950 
 6951 // Load Float Constant
 6952 
 6953 instruct loadConF(vRegF dst, immF con) %{
 6954   match(Set dst con);
 6955 
 6956   ins_cost(INSN_COST * 4);
 6957 
 6958   format %{
 6959     "ldrs $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
 6960   %}
 6961 
 6962   ins_encode %{
 6963     __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
 6964   %}
 6965 
 6966   ins_pipe(fp_load_constant_s);
 6967 %}
 6968 
 6969 // Load Packed Double Constant
 6970 
 6971 instruct loadConD_packed(vRegD dst, immDPacked con) %{
 6972   match(Set dst con);
 6973   ins_cost(INSN_COST);
 6974   format %{ "fmovd  $dst, $con"%}
 6975   ins_encode %{
 6976     __ fmovd(as_FloatRegister($dst$$reg), $con$$constant);
 6977   %}
 6978 
 6979   ins_pipe(fp_imm_d);
 6980 %}
 6981 
 6982 // Load Double Constant
 6983 
 6984 instruct loadConD(vRegD dst, immD con) %{
 6985   match(Set dst con);
 6986 
 6987   ins_cost(INSN_COST * 5);
 6988   format %{
 6989     "ldrd $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
 6990   %}
 6991 
 6992   ins_encode %{
 6993     __ ldrd(as_FloatRegister($dst$$reg), $constantaddress($con));
 6994   %}
 6995 
 6996   ins_pipe(fp_load_constant_d);
 6997 %}
 6998 
 6999 // Load Half Float Constant
 7000 // The "ldr" instruction loads a 32-bit word from the constant pool into a
 7001 // 32-bit register but only the bottom half will be populated and the top
 7002 // 16 bits are zero.
 7003 instruct loadConH(vRegF dst, immH con) %{
 7004   match(Set dst con);
 7005   format %{
 7006     "ldrs $dst, [$constantaddress]\t# load from constant table: half float=$con\n\t"
 7007   %}
 7008   ins_encode %{
 7009     __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
 7010   %}
 7011   ins_pipe(fp_load_constant_s);
 7012 %}
 7013 
 7014 // Store Instructions
 7015 
 7016 // Store Byte
 7017 instruct storeB(iRegIorL2I src, memory1 mem)
 7018 %{
 7019   match(Set mem (StoreB mem src));
 7020   predicate(!needs_releasing_store(n));
 7021 
 7022   ins_cost(INSN_COST);
 7023   format %{ "strb  $src, $mem\t# byte" %}
 7024 
 7025   ins_encode(aarch64_enc_strb(src, mem));
 7026 
 7027   ins_pipe(istore_reg_mem);
 7028 %}
 7029 
 7030 
 7031 instruct storeimmB0(immI0 zero, memory1 mem)
 7032 %{
 7033   match(Set mem (StoreB mem zero));
 7034   predicate(!needs_releasing_store(n));
 7035 
 7036   ins_cost(INSN_COST);
 7037   format %{ "strb rscractch2, $mem\t# byte" %}
 7038 
 7039   ins_encode(aarch64_enc_strb0(mem));
 7040 
 7041   ins_pipe(istore_mem);
 7042 %}
 7043 
 7044 // Store Char/Short
 7045 instruct storeC(iRegIorL2I src, memory2 mem)
 7046 %{
 7047   match(Set mem (StoreC mem src));
 7048   predicate(!needs_releasing_store(n));
 7049 
 7050   ins_cost(INSN_COST);
 7051   format %{ "strh  $src, $mem\t# short" %}
 7052 
 7053   ins_encode(aarch64_enc_strh(src, mem));
 7054 
 7055   ins_pipe(istore_reg_mem);
 7056 %}
 7057 
 7058 instruct storeimmC0(immI0 zero, memory2 mem)
 7059 %{
 7060   match(Set mem (StoreC mem zero));
 7061   predicate(!needs_releasing_store(n));
 7062 
 7063   ins_cost(INSN_COST);
 7064   format %{ "strh  zr, $mem\t# short" %}
 7065 
 7066   ins_encode(aarch64_enc_strh0(mem));
 7067 
 7068   ins_pipe(istore_mem);
 7069 %}
 7070 
 7071 // Store Integer
 7072 
 7073 instruct storeI(iRegIorL2I src, memory4 mem)
 7074 %{
 7075   match(Set mem(StoreI mem src));
 7076   predicate(!needs_releasing_store(n));
 7077 
 7078   ins_cost(INSN_COST);
 7079   format %{ "strw  $src, $mem\t# int" %}
 7080 
 7081   ins_encode(aarch64_enc_strw(src, mem));
 7082 
 7083   ins_pipe(istore_reg_mem);
 7084 %}
 7085 
 7086 instruct storeimmI0(immI0 zero, memory4 mem)
 7087 %{
 7088   match(Set mem(StoreI mem zero));
 7089   predicate(!needs_releasing_store(n));
 7090 
 7091   ins_cost(INSN_COST);
 7092   format %{ "strw  zr, $mem\t# int" %}
 7093 
 7094   ins_encode(aarch64_enc_strw0(mem));
 7095 
 7096   ins_pipe(istore_mem);
 7097 %}
 7098 
 7099 // Store Long (64 bit signed)
 7100 instruct storeL(iRegL src, memory8 mem)
 7101 %{
 7102   match(Set mem (StoreL mem src));
 7103   predicate(!needs_releasing_store(n));
 7104 
 7105   ins_cost(INSN_COST);
 7106   format %{ "str  $src, $mem\t# int" %}
 7107 
 7108   ins_encode(aarch64_enc_str(src, mem));
 7109 
 7110   ins_pipe(istore_reg_mem);
 7111 %}
 7112 
 7113 // Store Long (64 bit signed)
 7114 instruct storeimmL0(immL0 zero, memory8 mem)
 7115 %{
 7116   match(Set mem (StoreL mem zero));
 7117   predicate(!needs_releasing_store(n));
 7118 
 7119   ins_cost(INSN_COST);
 7120   format %{ "str  zr, $mem\t# int" %}
 7121 
 7122   ins_encode(aarch64_enc_str0(mem));
 7123 
 7124   ins_pipe(istore_mem);
 7125 %}
 7126 
 7127 // Store Pointer
 7128 instruct storeP(iRegP src, memory8 mem)
 7129 %{
 7130   match(Set mem (StoreP mem src));
 7131   predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
 7132 
 7133   ins_cost(INSN_COST);
 7134   format %{ "str  $src, $mem\t# ptr" %}
 7135 
 7136   ins_encode(aarch64_enc_str(src, mem));
 7137 
 7138   ins_pipe(istore_reg_mem);
 7139 %}
 7140 
 7141 // Store Pointer
 7142 instruct storeimmP0(immP0 zero, memory8 mem)
 7143 %{
 7144   match(Set mem (StoreP mem zero));
 7145   predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
 7146 
 7147   ins_cost(INSN_COST);
 7148   format %{ "str zr, $mem\t# ptr" %}
 7149 
 7150   ins_encode(aarch64_enc_str0(mem));
 7151 
 7152   ins_pipe(istore_mem);
 7153 %}
 7154 
 7155 // Store Compressed Pointer
 7156 instruct storeN(iRegN src, memory4 mem)
 7157 %{
 7158   match(Set mem (StoreN mem src));
 7159   predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
 7160 
 7161   ins_cost(INSN_COST);
 7162   format %{ "strw  $src, $mem\t# compressed ptr" %}
 7163 
 7164   ins_encode(aarch64_enc_strw(src, mem));
 7165 
 7166   ins_pipe(istore_reg_mem);
 7167 %}
 7168 
 7169 instruct storeImmN0(immN0 zero, memory4 mem)
 7170 %{
 7171   match(Set mem (StoreN mem zero));
 7172   predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
 7173 
 7174   ins_cost(INSN_COST);
 7175   format %{ "strw  zr, $mem\t# compressed ptr" %}
 7176 
 7177   ins_encode(aarch64_enc_strw0(mem));
 7178 
 7179   ins_pipe(istore_mem);
 7180 %}
 7181 
 7182 // Store Float
 7183 instruct storeF(vRegF src, memory4 mem)
 7184 %{
 7185   match(Set mem (StoreF mem src));
 7186   predicate(!needs_releasing_store(n));
 7187 
 7188   ins_cost(INSN_COST);
 7189   format %{ "strs  $src, $mem\t# float" %}
 7190 
 7191   ins_encode( aarch64_enc_strs(src, mem) );
 7192 
 7193   ins_pipe(pipe_class_memory);
 7194 %}
 7195 
 7196 // TODO
 7197 // implement storeImmF0 and storeFImmPacked
 7198 
 7199 // Store Double
 7200 instruct storeD(vRegD src, memory8 mem)
 7201 %{
 7202   match(Set mem (StoreD mem src));
 7203   predicate(!needs_releasing_store(n));
 7204 
 7205   ins_cost(INSN_COST);
 7206   format %{ "strd  $src, $mem\t# double" %}
 7207 
 7208   ins_encode( aarch64_enc_strd(src, mem) );
 7209 
 7210   ins_pipe(pipe_class_memory);
 7211 %}
 7212 
 7213 // Store Compressed Klass Pointer
 7214 instruct storeNKlass(iRegN src, memory4 mem)
 7215 %{
 7216   predicate(!needs_releasing_store(n));
 7217   match(Set mem (StoreNKlass mem src));
 7218 
 7219   ins_cost(INSN_COST);
 7220   format %{ "strw  $src, $mem\t# compressed klass ptr" %}
 7221 
 7222   ins_encode(aarch64_enc_strw(src, mem));
 7223 
 7224   ins_pipe(istore_reg_mem);
 7225 %}
 7226 
 7227 // TODO
 7228 // implement storeImmD0 and storeDImmPacked
 7229 
 7230 // prefetch instructions
 7231 // Must be safe to execute with invalid address (cannot fault).
 7232 
 7233 instruct prefetchalloc( memory8 mem ) %{
 7234   match(PrefetchAllocation mem);
 7235 
 7236   ins_cost(INSN_COST);
 7237   format %{ "prfm $mem, PSTL1KEEP\t# Prefetch into level 1 cache write keep" %}
 7238 
 7239   ins_encode( aarch64_enc_prefetchw(mem) );
 7240 
 7241   ins_pipe(iload_prefetch);
 7242 %}
 7243 
 7244 //  ---------------- volatile loads and stores ----------------
 7245 
 7246 // Load Byte (8 bit signed)
 7247 instruct loadB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 7248 %{
 7249   match(Set dst (LoadB mem));
 7250 
 7251   ins_cost(VOLATILE_REF_COST);
 7252   format %{ "ldarsb  $dst, $mem\t# byte" %}
 7253 
 7254   ins_encode(aarch64_enc_ldarsb(dst, mem));
 7255 
 7256   ins_pipe(pipe_serial);
 7257 %}
 7258 
 7259 // Load Byte (8 bit signed) into long
 7260 instruct loadB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 7261 %{
 7262   match(Set dst (ConvI2L (LoadB mem)));
 7263 
 7264   ins_cost(VOLATILE_REF_COST);
 7265   format %{ "ldarsb  $dst, $mem\t# byte" %}
 7266 
 7267   ins_encode(aarch64_enc_ldarsb(dst, mem));
 7268 
 7269   ins_pipe(pipe_serial);
 7270 %}
 7271 
 7272 // Load Byte (8 bit unsigned)
 7273 instruct loadUB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 7274 %{
 7275   match(Set dst (LoadUB mem));
 7276 
 7277   ins_cost(VOLATILE_REF_COST);
 7278   format %{ "ldarb  $dst, $mem\t# byte" %}
 7279 
 7280   ins_encode(aarch64_enc_ldarb(dst, mem));
 7281 
 7282   ins_pipe(pipe_serial);
 7283 %}
 7284 
 7285 // Load Byte (8 bit unsigned) into long
 7286 instruct loadUB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 7287 %{
 7288   match(Set dst (ConvI2L (LoadUB mem)));
 7289 
 7290   ins_cost(VOLATILE_REF_COST);
 7291   format %{ "ldarb  $dst, $mem\t# byte" %}
 7292 
 7293   ins_encode(aarch64_enc_ldarb(dst, mem));
 7294 
 7295   ins_pipe(pipe_serial);
 7296 %}
 7297 
 7298 // Load Short (16 bit signed)
 7299 instruct loadS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 7300 %{
 7301   match(Set dst (LoadS mem));
 7302 
 7303   ins_cost(VOLATILE_REF_COST);
 7304   format %{ "ldarshw  $dst, $mem\t# short" %}
 7305 
 7306   ins_encode(aarch64_enc_ldarshw(dst, mem));
 7307 
 7308   ins_pipe(pipe_serial);
 7309 %}
 7310 
 7311 instruct loadUS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 7312 %{
 7313   match(Set dst (LoadUS mem));
 7314 
 7315   ins_cost(VOLATILE_REF_COST);
 7316   format %{ "ldarhw  $dst, $mem\t# short" %}
 7317 
 7318   ins_encode(aarch64_enc_ldarhw(dst, mem));
 7319 
 7320   ins_pipe(pipe_serial);
 7321 %}
 7322 
 7323 // Load Short/Char (16 bit unsigned) into long
 7324 instruct loadUS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 7325 %{
 7326   match(Set dst (ConvI2L (LoadUS mem)));
 7327 
 7328   ins_cost(VOLATILE_REF_COST);
 7329   format %{ "ldarh  $dst, $mem\t# short" %}
 7330 
 7331   ins_encode(aarch64_enc_ldarh(dst, mem));
 7332 
 7333   ins_pipe(pipe_serial);
 7334 %}
 7335 
 7336 // Load Short/Char (16 bit signed) into long
 7337 instruct loadS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 7338 %{
 7339   match(Set dst (ConvI2L (LoadS mem)));
 7340 
 7341   ins_cost(VOLATILE_REF_COST);
 7342   format %{ "ldarh  $dst, $mem\t# short" %}
 7343 
 7344   ins_encode(aarch64_enc_ldarsh(dst, mem));
 7345 
 7346   ins_pipe(pipe_serial);
 7347 %}
 7348 
 7349 // Load Integer (32 bit signed)
 7350 instruct loadI_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 7351 %{
 7352   match(Set dst (LoadI mem));
 7353 
 7354   ins_cost(VOLATILE_REF_COST);
 7355   format %{ "ldarw  $dst, $mem\t# int" %}
 7356 
 7357   ins_encode(aarch64_enc_ldarw(dst, mem));
 7358 
 7359   ins_pipe(pipe_serial);
 7360 %}
 7361 
 7362 // Load Integer (32 bit unsigned) into long
 7363 instruct loadUI2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem, immL_32bits mask)
 7364 %{
 7365   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
 7366 
 7367   ins_cost(VOLATILE_REF_COST);
 7368   format %{ "ldarw  $dst, $mem\t# int" %}
 7369 
 7370   ins_encode(aarch64_enc_ldarw(dst, mem));
 7371 
 7372   ins_pipe(pipe_serial);
 7373 %}
 7374 
 7375 // Load Long (64 bit signed)
 7376 instruct loadL_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 7377 %{
 7378   match(Set dst (LoadL mem));
 7379 
 7380   ins_cost(VOLATILE_REF_COST);
 7381   format %{ "ldar  $dst, $mem\t# int" %}
 7382 
 7383   ins_encode(aarch64_enc_ldar(dst, mem));
 7384 
 7385   ins_pipe(pipe_serial);
 7386 %}
 7387 
 7388 // Load Pointer
 7389 instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem)
 7390 %{
 7391   match(Set dst (LoadP mem));
 7392   predicate(n->as_Load()->barrier_data() == 0);
 7393 
 7394   ins_cost(VOLATILE_REF_COST);
 7395   format %{ "ldar  $dst, $mem\t# ptr" %}
 7396 
 7397   ins_encode(aarch64_enc_ldar(dst, mem));
 7398 
 7399   ins_pipe(pipe_serial);
 7400 %}
 7401 
 7402 // Load Compressed Pointer
 7403 instruct loadN_volatile(iRegNNoSp dst, /* sync_memory*/indirect mem)
 7404 %{
 7405   match(Set dst (LoadN mem));
 7406   predicate(n->as_Load()->barrier_data() == 0);
 7407 
 7408   ins_cost(VOLATILE_REF_COST);
 7409   format %{ "ldarw  $dst, $mem\t# compressed ptr" %}
 7410 
 7411   ins_encode(aarch64_enc_ldarw(dst, mem));
 7412 
 7413   ins_pipe(pipe_serial);
 7414 %}
 7415 
 7416 // Load Float
 7417 instruct loadF_volatile(vRegF dst, /* sync_memory*/indirect mem)
 7418 %{
 7419   match(Set dst (LoadF mem));
 7420 
 7421   ins_cost(VOLATILE_REF_COST);
 7422   format %{ "ldars  $dst, $mem\t# float" %}
 7423 
 7424   ins_encode( aarch64_enc_fldars(dst, mem) );
 7425 
 7426   ins_pipe(pipe_serial);
 7427 %}
 7428 
 7429 // Load Double
 7430 instruct loadD_volatile(vRegD dst, /* sync_memory*/indirect mem)
 7431 %{
 7432   match(Set dst (LoadD mem));
 7433 
 7434   ins_cost(VOLATILE_REF_COST);
 7435   format %{ "ldard  $dst, $mem\t# double" %}
 7436 
 7437   ins_encode( aarch64_enc_fldard(dst, mem) );
 7438 
 7439   ins_pipe(pipe_serial);
 7440 %}
 7441 
 7442 // Store Byte
 7443 instruct storeB_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
 7444 %{
 7445   match(Set mem (StoreB mem src));
 7446 
 7447   ins_cost(VOLATILE_REF_COST);
 7448   format %{ "stlrb  $src, $mem\t# byte" %}
 7449 
 7450   ins_encode(aarch64_enc_stlrb(src, mem));
 7451 
 7452   ins_pipe(pipe_class_memory);
 7453 %}
 7454 
 7455 instruct storeimmB0_volatile(immI0 zero, /* sync_memory*/indirect mem)
 7456 %{
 7457   match(Set mem (StoreB mem zero));
 7458 
 7459   ins_cost(VOLATILE_REF_COST);
 7460   format %{ "stlrb  zr, $mem\t# byte" %}
 7461 
 7462   ins_encode(aarch64_enc_stlrb0(mem));
 7463 
 7464   ins_pipe(pipe_class_memory);
 7465 %}
 7466 
 7467 // Store Char/Short
 7468 instruct storeC_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
 7469 %{
 7470   match(Set mem (StoreC mem src));
 7471 
 7472   ins_cost(VOLATILE_REF_COST);
 7473   format %{ "stlrh  $src, $mem\t# short" %}
 7474 
 7475   ins_encode(aarch64_enc_stlrh(src, mem));
 7476 
 7477   ins_pipe(pipe_class_memory);
 7478 %}
 7479 
 7480 instruct storeimmC0_volatile(immI0 zero, /* sync_memory*/indirect mem)
 7481 %{
 7482   match(Set mem (StoreC mem zero));
 7483 
 7484   ins_cost(VOLATILE_REF_COST);
 7485   format %{ "stlrh  zr, $mem\t# short" %}
 7486 
 7487   ins_encode(aarch64_enc_stlrh0(mem));
 7488 
 7489   ins_pipe(pipe_class_memory);
 7490 %}
 7491 
 7492 // Store Integer
 7493 
 7494 instruct storeI_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
 7495 %{
 7496   match(Set mem(StoreI mem src));
 7497 
 7498   ins_cost(VOLATILE_REF_COST);
 7499   format %{ "stlrw  $src, $mem\t# int" %}
 7500 
 7501   ins_encode(aarch64_enc_stlrw(src, mem));
 7502 
 7503   ins_pipe(pipe_class_memory);
 7504 %}
 7505 
 7506 instruct storeimmI0_volatile(immI0 zero, /* sync_memory*/indirect mem)
 7507 %{
 7508   match(Set mem(StoreI mem zero));
 7509 
 7510   ins_cost(VOLATILE_REF_COST);
 7511   format %{ "stlrw  zr, $mem\t# int" %}
 7512 
 7513   ins_encode(aarch64_enc_stlrw0(mem));
 7514 
 7515   ins_pipe(pipe_class_memory);
 7516 %}
 7517 
 7518 // Store Long (64 bit signed)
 7519 instruct storeL_volatile(iRegL src, /* sync_memory*/indirect mem)
 7520 %{
 7521   match(Set mem (StoreL mem src));
 7522 
 7523   ins_cost(VOLATILE_REF_COST);
 7524   format %{ "stlr  $src, $mem\t# int" %}
 7525 
 7526   ins_encode(aarch64_enc_stlr(src, mem));
 7527 
 7528   ins_pipe(pipe_class_memory);
 7529 %}
 7530 
 7531 instruct storeimmL0_volatile(immL0 zero, /* sync_memory*/indirect mem)
 7532 %{
 7533   match(Set mem (StoreL mem zero));
 7534 
 7535   ins_cost(VOLATILE_REF_COST);
 7536   format %{ "stlr  zr, $mem\t# int" %}
 7537 
 7538   ins_encode(aarch64_enc_stlr0(mem));
 7539 
 7540   ins_pipe(pipe_class_memory);
 7541 %}
 7542 
 7543 // Store Pointer
 7544 instruct storeP_volatile(iRegP src, /* sync_memory*/indirect mem)
 7545 %{
 7546   match(Set mem (StoreP mem src));
 7547   predicate(n->as_Store()->barrier_data() == 0);
 7548 
 7549   ins_cost(VOLATILE_REF_COST);
 7550   format %{ "stlr  $src, $mem\t# ptr" %}
 7551 
 7552   ins_encode(aarch64_enc_stlr(src, mem));
 7553 
 7554   ins_pipe(pipe_class_memory);
 7555 %}
 7556 
 7557 instruct storeimmP0_volatile(immP0 zero, /* sync_memory*/indirect mem)
 7558 %{
 7559   match(Set mem (StoreP mem zero));
 7560   predicate(n->as_Store()->barrier_data() == 0);
 7561 
 7562   ins_cost(VOLATILE_REF_COST);
 7563   format %{ "stlr  zr, $mem\t# ptr" %}
 7564 
 7565   ins_encode(aarch64_enc_stlr0(mem));
 7566 
 7567   ins_pipe(pipe_class_memory);
 7568 %}
 7569 
 7570 // Store Compressed Pointer
 7571 instruct storeN_volatile(iRegN src, /* sync_memory*/indirect mem)
 7572 %{
 7573   match(Set mem (StoreN mem src));
 7574   predicate(n->as_Store()->barrier_data() == 0);
 7575 
 7576   ins_cost(VOLATILE_REF_COST);
 7577   format %{ "stlrw  $src, $mem\t# compressed ptr" %}
 7578 
 7579   ins_encode(aarch64_enc_stlrw(src, mem));
 7580 
 7581   ins_pipe(pipe_class_memory);
 7582 %}
 7583 
 7584 instruct storeimmN0_volatile(immN0 zero, /* sync_memory*/indirect mem)
 7585 %{
 7586   match(Set mem (StoreN mem zero));
 7587   predicate(n->as_Store()->barrier_data() == 0);
 7588 
 7589   ins_cost(VOLATILE_REF_COST);
 7590   format %{ "stlrw  zr, $mem\t# compressed ptr" %}
 7591 
 7592   ins_encode(aarch64_enc_stlrw0(mem));
 7593 
 7594   ins_pipe(pipe_class_memory);
 7595 %}
 7596 
 7597 // Store Float
 7598 instruct storeF_volatile(vRegF src, /* sync_memory*/indirect mem)
 7599 %{
 7600   match(Set mem (StoreF mem src));
 7601 
 7602   ins_cost(VOLATILE_REF_COST);
 7603   format %{ "stlrs  $src, $mem\t# float" %}
 7604 
 7605   ins_encode( aarch64_enc_fstlrs(src, mem) );
 7606 
 7607   ins_pipe(pipe_class_memory);
 7608 %}
 7609 
 7610 // TODO
 7611 // implement storeImmF0 and storeFImmPacked
 7612 
 7613 // Store Double
 7614 instruct storeD_volatile(vRegD src, /* sync_memory*/indirect mem)
 7615 %{
 7616   match(Set mem (StoreD mem src));
 7617 
 7618   ins_cost(VOLATILE_REF_COST);
 7619   format %{ "stlrd  $src, $mem\t# double" %}
 7620 
 7621   ins_encode( aarch64_enc_fstlrd(src, mem) );
 7622 
 7623   ins_pipe(pipe_class_memory);
 7624 %}
 7625 
 7626 //  ---------------- end of volatile loads and stores ----------------
 7627 
 7628 instruct cacheWB(indirect addr)
 7629 %{
 7630   predicate(VM_Version::supports_data_cache_line_flush());
 7631   match(CacheWB addr);
 7632 
 7633   ins_cost(100);
 7634   format %{"cache wb $addr" %}
 7635   ins_encode %{
 7636     assert($addr->index_position() < 0, "should be");
 7637     assert($addr$$disp == 0, "should be");
 7638     __ cache_wb(Address($addr$$base$$Register, 0));
 7639   %}
 7640   ins_pipe(pipe_slow); // XXX
 7641 %}
 7642 
 7643 instruct cacheWBPreSync()
 7644 %{
 7645   predicate(VM_Version::supports_data_cache_line_flush());
 7646   match(CacheWBPreSync);
 7647 
 7648   ins_cost(100);
 7649   format %{"cache wb presync" %}
 7650   ins_encode %{
 7651     __ cache_wbsync(true);
 7652   %}
 7653   ins_pipe(pipe_slow); // XXX
 7654 %}
 7655 
 7656 instruct cacheWBPostSync()
 7657 %{
 7658   predicate(VM_Version::supports_data_cache_line_flush());
 7659   match(CacheWBPostSync);
 7660 
 7661   ins_cost(100);
 7662   format %{"cache wb postsync" %}
 7663   ins_encode %{
 7664     __ cache_wbsync(false);
 7665   %}
 7666   ins_pipe(pipe_slow); // XXX
 7667 %}
 7668 
 7669 // ============================================================================
 7670 // BSWAP Instructions
 7671 
 7672 instruct bytes_reverse_int(iRegINoSp dst, iRegIorL2I src) %{
 7673   match(Set dst (ReverseBytesI src));
 7674 
 7675   ins_cost(INSN_COST);
 7676   format %{ "revw  $dst, $src" %}
 7677 
 7678   ins_encode %{
 7679     __ revw(as_Register($dst$$reg), as_Register($src$$reg));
 7680   %}
 7681 
 7682   ins_pipe(ialu_reg);
 7683 %}
 7684 
 7685 instruct bytes_reverse_long(iRegLNoSp dst, iRegL src) %{
 7686   match(Set dst (ReverseBytesL src));
 7687 
 7688   ins_cost(INSN_COST);
 7689   format %{ "rev  $dst, $src" %}
 7690 
 7691   ins_encode %{
 7692     __ rev(as_Register($dst$$reg), as_Register($src$$reg));
 7693   %}
 7694 
 7695   ins_pipe(ialu_reg);
 7696 %}
 7697 
 7698 instruct bytes_reverse_unsigned_short(iRegINoSp dst, iRegIorL2I src) %{
 7699   match(Set dst (ReverseBytesUS src));
 7700 
 7701   ins_cost(INSN_COST);
 7702   format %{ "rev16w  $dst, $src" %}
 7703 
 7704   ins_encode %{
 7705     __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
 7706   %}
 7707 
 7708   ins_pipe(ialu_reg);
 7709 %}
 7710 
 7711 instruct bytes_reverse_short(iRegINoSp dst, iRegIorL2I src) %{
 7712   match(Set dst (ReverseBytesS src));
 7713 
 7714   ins_cost(INSN_COST);
 7715   format %{ "rev16w  $dst, $src\n\t"
 7716             "sbfmw $dst, $dst, #0, #15" %}
 7717 
 7718   ins_encode %{
 7719     __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
 7720     __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 15U);
 7721   %}
 7722 
 7723   ins_pipe(ialu_reg);
 7724 %}
 7725 
 7726 // ============================================================================
 7727 // Zero Count Instructions
 7728 
 7729 instruct countLeadingZerosI(iRegINoSp dst, iRegIorL2I src) %{
 7730   match(Set dst (CountLeadingZerosI src));
 7731 
 7732   ins_cost(INSN_COST);
 7733   format %{ "clzw  $dst, $src" %}
 7734   ins_encode %{
 7735     __ clzw(as_Register($dst$$reg), as_Register($src$$reg));
 7736   %}
 7737 
 7738   ins_pipe(ialu_reg);
 7739 %}
 7740 
 7741 instruct countLeadingZerosL(iRegINoSp dst, iRegL src) %{
 7742   match(Set dst (CountLeadingZerosL src));
 7743 
 7744   ins_cost(INSN_COST);
 7745   format %{ "clz   $dst, $src" %}
 7746   ins_encode %{
 7747     __ clz(as_Register($dst$$reg), as_Register($src$$reg));
 7748   %}
 7749 
 7750   ins_pipe(ialu_reg);
 7751 %}
 7752 
 7753 instruct countTrailingZerosI(iRegINoSp dst, iRegIorL2I src) %{
 7754   match(Set dst (CountTrailingZerosI src));
 7755 
 7756   ins_cost(INSN_COST * 2);
 7757   format %{ "rbitw  $dst, $src\n\t"
 7758             "clzw   $dst, $dst" %}
 7759   ins_encode %{
 7760     __ rbitw(as_Register($dst$$reg), as_Register($src$$reg));
 7761     __ clzw(as_Register($dst$$reg), as_Register($dst$$reg));
 7762   %}
 7763 
 7764   ins_pipe(ialu_reg);
 7765 %}
 7766 
 7767 instruct countTrailingZerosL(iRegINoSp dst, iRegL src) %{
 7768   match(Set dst (CountTrailingZerosL src));
 7769 
 7770   ins_cost(INSN_COST * 2);
 7771   format %{ "rbit   $dst, $src\n\t"
 7772             "clz    $dst, $dst" %}
 7773   ins_encode %{
 7774     __ rbit(as_Register($dst$$reg), as_Register($src$$reg));
 7775     __ clz(as_Register($dst$$reg), as_Register($dst$$reg));
 7776   %}
 7777 
 7778   ins_pipe(ialu_reg);
 7779 %}
 7780 
 7781 //---------- Population Count Instructions -------------------------------------
 7782 //
 7783 
 7784 instruct popCountI(iRegINoSp dst, iRegIorL2I src, vRegF tmp) %{
 7785   match(Set dst (PopCountI src));
 7786   effect(TEMP tmp);
 7787   ins_cost(INSN_COST * 13);
 7788 
 7789   format %{ "movw   $src, $src\n\t"
 7790             "mov    $tmp, $src\t# vector (1D)\n\t"
 7791             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 7792             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 7793             "mov    $dst, $tmp\t# vector (1D)" %}
 7794   ins_encode %{
 7795     __ movw($src$$Register, $src$$Register); // ensure top 32 bits 0
 7796     __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
 7797     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 7798     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 7799     __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
 7800   %}
 7801 
 7802   ins_pipe(pipe_class_default);
 7803 %}
 7804 
 7805 instruct popCountI_mem(iRegINoSp dst, memory4 mem, vRegF tmp) %{
 7806   match(Set dst (PopCountI (LoadI mem)));
 7807   effect(TEMP tmp);
 7808   ins_cost(INSN_COST * 13);
 7809 
 7810   format %{ "ldrs   $tmp, $mem\n\t"
 7811             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 7812             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 7813             "mov    $dst, $tmp\t# vector (1D)" %}
 7814   ins_encode %{
 7815     FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
 7816     loadStore(masm, &MacroAssembler::ldrs, tmp_reg, $mem->opcode(),
 7817               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 7818     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 7819     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 7820     __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
 7821   %}
 7822 
 7823   ins_pipe(pipe_class_default);
 7824 %}
 7825 
 7826 // Note: Long.bitCount(long) returns an int.
 7827 instruct popCountL(iRegINoSp dst, iRegL src, vRegD tmp) %{
 7828   match(Set dst (PopCountL src));
 7829   effect(TEMP tmp);
 7830   ins_cost(INSN_COST * 13);
 7831 
 7832   format %{ "mov    $tmp, $src\t# vector (1D)\n\t"
 7833             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 7834             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 7835             "mov    $dst, $tmp\t# vector (1D)" %}
 7836   ins_encode %{
 7837     __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
 7838     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 7839     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 7840     __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
 7841   %}
 7842 
 7843   ins_pipe(pipe_class_default);
 7844 %}
 7845 
 7846 instruct popCountL_mem(iRegINoSp dst, memory8 mem, vRegD tmp) %{
 7847   match(Set dst (PopCountL (LoadL mem)));
 7848   effect(TEMP tmp);
 7849   ins_cost(INSN_COST * 13);
 7850 
 7851   format %{ "ldrd   $tmp, $mem\n\t"
 7852             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 7853             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 7854             "mov    $dst, $tmp\t# vector (1D)" %}
 7855   ins_encode %{
 7856     FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
 7857     loadStore(masm, &MacroAssembler::ldrd, tmp_reg, $mem->opcode(),
 7858               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 7859     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 7860     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 7861     __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
 7862   %}
 7863 
 7864   ins_pipe(pipe_class_default);
 7865 %}
 7866 
 7867 // ============================================================================
 7868 // VerifyVectorAlignment Instruction
 7869 
 7870 instruct verify_vector_alignment(iRegP addr, immL_positive_bitmaskI mask, rFlagsReg cr) %{
 7871   match(Set addr (VerifyVectorAlignment addr mask));
 7872   effect(KILL cr);
 7873   format %{ "verify_vector_alignment $addr $mask \t! verify alignment" %}
 7874   ins_encode %{
 7875     Label Lskip;
 7876     // check if masked bits of addr are zero
 7877     __ tst($addr$$Register, $mask$$constant);
 7878     __ br(Assembler::EQ, Lskip);
 7879     __ stop("verify_vector_alignment found a misaligned vector memory access");
 7880     __ bind(Lskip);
 7881   %}
 7882   ins_pipe(pipe_slow);
 7883 %}
 7884 
 7885 // ============================================================================
 7886 // MemBar Instruction
 7887 
 7888 instruct load_fence() %{
 7889   match(LoadFence);
 7890   ins_cost(VOLATILE_REF_COST);
 7891 
 7892   format %{ "load_fence" %}
 7893 
 7894   ins_encode %{
 7895     __ membar(Assembler::LoadLoad|Assembler::LoadStore);
 7896   %}
 7897   ins_pipe(pipe_serial);
 7898 %}
 7899 
 7900 instruct unnecessary_membar_acquire() %{
 7901   predicate(unnecessary_acquire(n));
 7902   match(MemBarAcquire);
 7903   ins_cost(0);
 7904 
 7905   format %{ "membar_acquire (elided)" %}
 7906 
 7907   ins_encode %{
 7908     __ block_comment("membar_acquire (elided)");
 7909   %}
 7910 
 7911   ins_pipe(pipe_class_empty);
 7912 %}
 7913 
 7914 instruct membar_acquire() %{
 7915   match(MemBarAcquire);
 7916   ins_cost(VOLATILE_REF_COST);
 7917 
 7918   format %{ "membar_acquire\n\t"
 7919             "dmb ishld" %}
 7920 
 7921   ins_encode %{
 7922     __ block_comment("membar_acquire");
 7923     __ membar(Assembler::LoadLoad|Assembler::LoadStore);
 7924   %}
 7925 
 7926   ins_pipe(pipe_serial);
 7927 %}
 7928 
 7929 
 7930 instruct membar_acquire_lock() %{
 7931   match(MemBarAcquireLock);
 7932   ins_cost(VOLATILE_REF_COST);
 7933 
 7934   format %{ "membar_acquire_lock (elided)" %}
 7935 
 7936   ins_encode %{
 7937     __ block_comment("membar_acquire_lock (elided)");
 7938   %}
 7939 
 7940   ins_pipe(pipe_serial);
 7941 %}
 7942 
 7943 instruct store_fence() %{
 7944   match(StoreFence);
 7945   ins_cost(VOLATILE_REF_COST);
 7946 
 7947   format %{ "store_fence" %}
 7948 
 7949   ins_encode %{
 7950     __ membar(Assembler::LoadStore|Assembler::StoreStore);
 7951   %}
 7952   ins_pipe(pipe_serial);
 7953 %}
 7954 
 7955 instruct unnecessary_membar_release() %{
 7956   predicate(unnecessary_release(n));
 7957   match(MemBarRelease);
 7958   ins_cost(0);
 7959 
 7960   format %{ "membar_release (elided)" %}
 7961 
 7962   ins_encode %{
 7963     __ block_comment("membar_release (elided)");
 7964   %}
 7965   ins_pipe(pipe_serial);
 7966 %}
 7967 
 7968 instruct membar_release() %{
 7969   match(MemBarRelease);
 7970   ins_cost(VOLATILE_REF_COST);
 7971 
 7972   format %{ "membar_release\n\t"
 7973             "dmb ishst\n\tdmb ishld" %}
 7974 
 7975   ins_encode %{
 7976     __ block_comment("membar_release");
 7977     // These will be merged if AlwaysMergeDMB is enabled.
 7978     __ membar(Assembler::StoreStore);
 7979     __ membar(Assembler::LoadStore);
 7980   %}
 7981   ins_pipe(pipe_serial);
 7982 %}
 7983 
 7984 instruct membar_storestore() %{
 7985   match(MemBarStoreStore);
 7986   match(StoreStoreFence);
 7987   ins_cost(VOLATILE_REF_COST);
 7988 
 7989   format %{ "MEMBAR-store-store" %}
 7990 
 7991   ins_encode %{
 7992     __ membar(Assembler::StoreStore);
 7993   %}
 7994   ins_pipe(pipe_serial);
 7995 %}
 7996 
 7997 instruct membar_release_lock() %{
 7998   match(MemBarReleaseLock);
 7999   ins_cost(VOLATILE_REF_COST);
 8000 
 8001   format %{ "membar_release_lock (elided)" %}
 8002 
 8003   ins_encode %{
 8004     __ block_comment("membar_release_lock (elided)");
 8005   %}
 8006 
 8007   ins_pipe(pipe_serial);
 8008 %}
 8009 
 8010 instruct unnecessary_membar_volatile() %{
 8011   predicate(unnecessary_volatile(n));
 8012   match(MemBarVolatile);
 8013   ins_cost(0);
 8014 
 8015   format %{ "membar_volatile (elided)" %}
 8016 
 8017   ins_encode %{
 8018     __ block_comment("membar_volatile (elided)");
 8019   %}
 8020 
 8021   ins_pipe(pipe_serial);
 8022 %}
 8023 
 8024 instruct membar_volatile() %{
 8025   match(MemBarVolatile);
 8026   ins_cost(VOLATILE_REF_COST*100);
 8027 
 8028   format %{ "membar_volatile\n\t"
 8029              "dmb ish"%}
 8030 
 8031   ins_encode %{
 8032     __ block_comment("membar_volatile");
 8033     __ membar(Assembler::StoreLoad);
 8034   %}
 8035 
 8036   ins_pipe(pipe_serial);
 8037 %}
 8038 
 8039 // ============================================================================
 8040 // Cast/Convert Instructions
 8041 
 8042 instruct castX2P(iRegPNoSp dst, iRegL src) %{
 8043   match(Set dst (CastX2P src));
 8044 
 8045   ins_cost(INSN_COST);
 8046   format %{ "mov $dst, $src\t# long -> ptr" %}
 8047 
 8048   ins_encode %{
 8049     if ($dst$$reg != $src$$reg) {
 8050       __ mov(as_Register($dst$$reg), as_Register($src$$reg));
 8051     }
 8052   %}
 8053 
 8054   ins_pipe(ialu_reg);
 8055 %}
 8056 
 8057 instruct castI2N(iRegNNoSp dst, iRegI src) %{
 8058   match(Set dst (CastI2N src));
 8059 
 8060   ins_cost(INSN_COST);
 8061   format %{ "mov $dst, $src\t# int -> narrow ptr" %}
 8062 
 8063   ins_encode %{
 8064     if ($dst$$reg != $src$$reg) {
 8065       __ mov(as_Register($dst$$reg), as_Register($src$$reg));
 8066     }
 8067   %}
 8068 
 8069   ins_pipe(ialu_reg);
 8070 %}
 8071 
 8072 instruct castN2X(iRegLNoSp dst, iRegN src) %{
 8073   match(Set dst (CastP2X src));
 8074 
 8075   ins_cost(INSN_COST);
 8076   format %{ "mov $dst, $src\t# ptr -> long" %}
 8077 
 8078   ins_encode %{
 8079     if ($dst$$reg != $src$$reg) {
 8080       __ mov(as_Register($dst$$reg), as_Register($src$$reg));
 8081     }
 8082   %}
 8083 
 8084   ins_pipe(ialu_reg);
 8085 %}
 8086 
 8087 instruct castP2X(iRegLNoSp dst, iRegP src) %{
 8088   match(Set dst (CastP2X src));
 8089 
 8090   ins_cost(INSN_COST);
 8091   format %{ "mov $dst, $src\t# ptr -> long" %}
 8092 
 8093   ins_encode %{
 8094     if ($dst$$reg != $src$$reg) {
 8095       __ mov(as_Register($dst$$reg), as_Register($src$$reg));
 8096     }
 8097   %}
 8098 
 8099   ins_pipe(ialu_reg);
 8100 %}
 8101 
 8102 // Convert oop into int for vectors alignment masking
 8103 instruct convP2I(iRegINoSp dst, iRegP src) %{
 8104   match(Set dst (ConvL2I (CastP2X src)));
 8105 
 8106   ins_cost(INSN_COST);
 8107   format %{ "movw $dst, $src\t# ptr -> int" %}
 8108   ins_encode %{
 8109     __ movw($dst$$Register, $src$$Register);
 8110   %}
 8111 
 8112   ins_pipe(ialu_reg);
 8113 %}
 8114 
 8115 // Convert compressed oop into int for vectors alignment masking
 8116 // in case of 32bit oops (heap < 4Gb).
 8117 instruct convN2I(iRegINoSp dst, iRegN src)
 8118 %{
 8119   predicate(CompressedOops::shift() == 0);
 8120   match(Set dst (ConvL2I (CastP2X (DecodeN src))));
 8121 
 8122   ins_cost(INSN_COST);
 8123   format %{ "mov dst, $src\t# compressed ptr -> int" %}
 8124   ins_encode %{
 8125     __ movw($dst$$Register, $src$$Register);
 8126   %}
 8127 
 8128   ins_pipe(ialu_reg);
 8129 %}
 8130 
 8131 
 8132 // Convert oop pointer into compressed form
 8133 instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
 8134   predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
 8135   match(Set dst (EncodeP src));
 8136   effect(KILL cr);
 8137   ins_cost(INSN_COST * 3);
 8138   format %{ "encode_heap_oop $dst, $src" %}
 8139   ins_encode %{
 8140     Register s = $src$$Register;
 8141     Register d = $dst$$Register;
 8142     __ encode_heap_oop(d, s);
 8143   %}
 8144   ins_pipe(ialu_reg);
 8145 %}
 8146 
 8147 instruct encodeHeapOop_not_null(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
 8148   predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
 8149   match(Set dst (EncodeP src));
 8150   ins_cost(INSN_COST * 3);
 8151   format %{ "encode_heap_oop_not_null $dst, $src" %}
 8152   ins_encode %{
 8153     __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
 8154   %}
 8155   ins_pipe(ialu_reg);
 8156 %}
 8157 
 8158 instruct decodeHeapOop(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
 8159   predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
 8160             n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
 8161   match(Set dst (DecodeN src));
 8162   ins_cost(INSN_COST * 3);
 8163   format %{ "decode_heap_oop $dst, $src" %}
 8164   ins_encode %{
 8165     Register s = $src$$Register;
 8166     Register d = $dst$$Register;
 8167     __ decode_heap_oop(d, s);
 8168   %}
 8169   ins_pipe(ialu_reg);
 8170 %}
 8171 
 8172 instruct decodeHeapOop_not_null(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
 8173   predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
 8174             n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
 8175   match(Set dst (DecodeN src));
 8176   ins_cost(INSN_COST * 3);
 8177   format %{ "decode_heap_oop_not_null $dst, $src" %}
 8178   ins_encode %{
 8179     Register s = $src$$Register;
 8180     Register d = $dst$$Register;
 8181     __ decode_heap_oop_not_null(d, s);
 8182   %}
 8183   ins_pipe(ialu_reg);
 8184 %}
 8185 
 8186 // n.b. AArch64 implementations of encode_klass_not_null and
 8187 // decode_klass_not_null do not modify the flags register so, unlike
 8188 // Intel, we don't kill CR as a side effect here
 8189 
 8190 instruct encodeKlass_not_null(iRegNNoSp dst, iRegP src) %{
 8191   match(Set dst (EncodePKlass src));
 8192 
 8193   ins_cost(INSN_COST * 3);
 8194   format %{ "encode_klass_not_null $dst,$src" %}
 8195 
 8196   ins_encode %{
 8197     Register src_reg = as_Register($src$$reg);
 8198     Register dst_reg = as_Register($dst$$reg);
 8199     __ encode_klass_not_null(dst_reg, src_reg);
 8200   %}
 8201 
 8202    ins_pipe(ialu_reg);
 8203 %}
 8204 
 8205 instruct decodeKlass_not_null(iRegPNoSp dst, iRegN src) %{
 8206   match(Set dst (DecodeNKlass src));
 8207 
 8208   ins_cost(INSN_COST * 3);
 8209   format %{ "decode_klass_not_null $dst,$src" %}
 8210 
 8211   ins_encode %{
 8212     Register src_reg = as_Register($src$$reg);
 8213     Register dst_reg = as_Register($dst$$reg);
 8214     if (dst_reg != src_reg) {
 8215       __ decode_klass_not_null(dst_reg, src_reg);
 8216     } else {
 8217       __ decode_klass_not_null(dst_reg);
 8218     }
 8219   %}
 8220 
 8221    ins_pipe(ialu_reg);
 8222 %}
 8223 
 8224 instruct checkCastPP(iRegPNoSp dst)
 8225 %{
 8226   match(Set dst (CheckCastPP dst));
 8227 
 8228   size(0);
 8229   format %{ "# checkcastPP of $dst" %}
 8230   ins_encode(/* empty encoding */);
 8231   ins_pipe(pipe_class_empty);
 8232 %}
 8233 
 8234 instruct castPP(iRegPNoSp dst)
 8235 %{
 8236   match(Set dst (CastPP dst));
 8237 
 8238   size(0);
 8239   format %{ "# castPP of $dst" %}
 8240   ins_encode(/* empty encoding */);
 8241   ins_pipe(pipe_class_empty);
 8242 %}
 8243 
 8244 instruct castII(iRegI dst)
 8245 %{
 8246   predicate(VerifyConstraintCasts == 0);
 8247   match(Set dst (CastII dst));
 8248 
 8249   size(0);
 8250   format %{ "# castII of $dst" %}
 8251   ins_encode(/* empty encoding */);
 8252   ins_cost(0);
 8253   ins_pipe(pipe_class_empty);
 8254 %}
 8255 
 8256 instruct castII_checked(iRegI dst, rFlagsReg cr)
 8257 %{
 8258   predicate(VerifyConstraintCasts > 0);
 8259   match(Set dst (CastII dst));
 8260   effect(KILL cr);
 8261 
 8262   format %{ "# castII_checked of $dst" %}
 8263   ins_encode %{
 8264     __ verify_int_in_range(_idx, bottom_type()->is_int(), $dst$$Register, rscratch1);
 8265   %}
 8266   ins_pipe(pipe_slow);
 8267 %}
 8268 
 8269 instruct castLL(iRegL dst)
 8270 %{
 8271   predicate(VerifyConstraintCasts == 0);
 8272   match(Set dst (CastLL dst));
 8273 
 8274   size(0);
 8275   format %{ "# castLL of $dst" %}
 8276   ins_encode(/* empty encoding */);
 8277   ins_cost(0);
 8278   ins_pipe(pipe_class_empty);
 8279 %}
 8280 
 8281 instruct castLL_checked(iRegL dst, rFlagsReg cr)
 8282 %{
 8283   predicate(VerifyConstraintCasts > 0);
 8284   match(Set dst (CastLL dst));
 8285   effect(KILL cr);
 8286 
 8287   format %{ "# castLL_checked of $dst" %}
 8288   ins_encode %{
 8289     __ verify_long_in_range(_idx, bottom_type()->is_long(), $dst$$Register, rscratch1);
 8290   %}
 8291   ins_pipe(pipe_slow);
 8292 %}
 8293 
 8294 instruct castFF(vRegF dst)
 8295 %{
 8296   match(Set dst (CastFF dst));
 8297 
 8298   size(0);
 8299   format %{ "# castFF of $dst" %}
 8300   ins_encode(/* empty encoding */);
 8301   ins_cost(0);
 8302   ins_pipe(pipe_class_empty);
 8303 %}
 8304 
 8305 instruct castDD(vRegD dst)
 8306 %{
 8307   match(Set dst (CastDD dst));
 8308 
 8309   size(0);
 8310   format %{ "# castDD of $dst" %}
 8311   ins_encode(/* empty encoding */);
 8312   ins_cost(0);
 8313   ins_pipe(pipe_class_empty);
 8314 %}
 8315 
 8316 instruct castVV(vReg dst)
 8317 %{
 8318   match(Set dst (CastVV dst));
 8319 
 8320   size(0);
 8321   format %{ "# castVV of $dst" %}
 8322   ins_encode(/* empty encoding */);
 8323   ins_cost(0);
 8324   ins_pipe(pipe_class_empty);
 8325 %}
 8326 
 8327 instruct castVVMask(pRegGov dst)
 8328 %{
 8329   match(Set dst (CastVV dst));
 8330 
 8331   size(0);
 8332   format %{ "# castVV of $dst" %}
 8333   ins_encode(/* empty encoding */);
 8334   ins_cost(0);
 8335   ins_pipe(pipe_class_empty);
 8336 %}
 8337 
 8338 // ============================================================================
 8339 // Atomic operation instructions
 8340 //
 8341 
 8342 // standard CompareAndSwapX when we are using barriers
 8343 // these have higher priority than the rules selected by a predicate
 8344 
 8345 // XXX No flag versions for CompareAndSwap{I,L,P,N} because matcher
 8346 // can't match them
 8347 
 8348 instruct compareAndSwapB(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 8349 
 8350   match(Set res (CompareAndSwapB mem (Binary oldval newval)));
 8351   ins_cost(2 * VOLATILE_REF_COST);
 8352 
 8353   effect(KILL cr);
 8354 
 8355   format %{
 8356     "cmpxchgb $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 8357     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8358   %}
 8359 
 8360   ins_encode(aarch64_enc_cmpxchgb(mem, oldval, newval),
 8361             aarch64_enc_cset_eq(res));
 8362 
 8363   ins_pipe(pipe_slow);
 8364 %}
 8365 
 8366 instruct compareAndSwapS(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 8367 
 8368   match(Set res (CompareAndSwapS mem (Binary oldval newval)));
 8369   ins_cost(2 * VOLATILE_REF_COST);
 8370 
 8371   effect(KILL cr);
 8372 
 8373   format %{
 8374     "cmpxchgs $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 8375     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8376   %}
 8377 
 8378   ins_encode(aarch64_enc_cmpxchgs(mem, oldval, newval),
 8379             aarch64_enc_cset_eq(res));
 8380 
 8381   ins_pipe(pipe_slow);
 8382 %}
 8383 
 8384 instruct compareAndSwapI(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 8385 
 8386   match(Set res (CompareAndSwapI mem (Binary oldval newval)));
 8387   ins_cost(2 * VOLATILE_REF_COST);
 8388 
 8389   effect(KILL cr);
 8390 
 8391  format %{
 8392     "cmpxchgw $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 8393     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8394  %}
 8395 
 8396  ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
 8397             aarch64_enc_cset_eq(res));
 8398 
 8399   ins_pipe(pipe_slow);
 8400 %}
 8401 
 8402 instruct compareAndSwapL(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
 8403 
 8404   match(Set res (CompareAndSwapL mem (Binary oldval newval)));
 8405   ins_cost(2 * VOLATILE_REF_COST);
 8406 
 8407   effect(KILL cr);
 8408 
 8409  format %{
 8410     "cmpxchg $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
 8411     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8412  %}
 8413 
 8414  ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
 8415             aarch64_enc_cset_eq(res));
 8416 
 8417   ins_pipe(pipe_slow);
 8418 %}
 8419 
 8420 instruct compareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 8421 
 8422   match(Set res (CompareAndSwapP mem (Binary oldval newval)));
 8423   predicate(n->as_LoadStore()->barrier_data() == 0);
 8424   ins_cost(2 * VOLATILE_REF_COST);
 8425 
 8426   effect(KILL cr);
 8427 
 8428  format %{
 8429     "cmpxchg $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
 8430     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8431  %}
 8432 
 8433  ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
 8434             aarch64_enc_cset_eq(res));
 8435 
 8436   ins_pipe(pipe_slow);
 8437 %}
 8438 
 8439 instruct compareAndSwapN(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
 8440 
 8441   match(Set res (CompareAndSwapN mem (Binary oldval newval)));
 8442   predicate(n->as_LoadStore()->barrier_data() == 0);
 8443   ins_cost(2 * VOLATILE_REF_COST);
 8444 
 8445   effect(KILL cr);
 8446 
 8447  format %{
 8448     "cmpxchgw $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
 8449     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8450  %}
 8451 
 8452  ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
 8453             aarch64_enc_cset_eq(res));
 8454 
 8455   ins_pipe(pipe_slow);
 8456 %}
 8457 
 8458 // alternative CompareAndSwapX when we are eliding barriers
 8459 
 8460 instruct compareAndSwapBAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 8461 
 8462   predicate(needs_acquiring_load_exclusive(n));
 8463   match(Set res (CompareAndSwapB mem (Binary oldval newval)));
 8464   ins_cost(VOLATILE_REF_COST);
 8465 
 8466   effect(KILL cr);
 8467 
 8468   format %{
 8469     "cmpxchgb_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 8470     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8471   %}
 8472 
 8473   ins_encode(aarch64_enc_cmpxchgb_acq(mem, oldval, newval),
 8474             aarch64_enc_cset_eq(res));
 8475 
 8476   ins_pipe(pipe_slow);
 8477 %}
 8478 
 8479 instruct compareAndSwapSAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 8480 
 8481   predicate(needs_acquiring_load_exclusive(n));
 8482   match(Set res (CompareAndSwapS mem (Binary oldval newval)));
 8483   ins_cost(VOLATILE_REF_COST);
 8484 
 8485   effect(KILL cr);
 8486 
 8487   format %{
 8488     "cmpxchgs_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 8489     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8490   %}
 8491 
 8492   ins_encode(aarch64_enc_cmpxchgs_acq(mem, oldval, newval),
 8493             aarch64_enc_cset_eq(res));
 8494 
 8495   ins_pipe(pipe_slow);
 8496 %}
 8497 
 8498 instruct compareAndSwapIAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 8499 
 8500   predicate(needs_acquiring_load_exclusive(n));
 8501   match(Set res (CompareAndSwapI mem (Binary oldval newval)));
 8502   ins_cost(VOLATILE_REF_COST);
 8503 
 8504   effect(KILL cr);
 8505 
 8506  format %{
 8507     "cmpxchgw_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 8508     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8509  %}
 8510 
 8511  ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
 8512             aarch64_enc_cset_eq(res));
 8513 
 8514   ins_pipe(pipe_slow);
 8515 %}
 8516 
 8517 instruct compareAndSwapLAcq(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
 8518 
 8519   predicate(needs_acquiring_load_exclusive(n));
 8520   match(Set res (CompareAndSwapL mem (Binary oldval newval)));
 8521   ins_cost(VOLATILE_REF_COST);
 8522 
 8523   effect(KILL cr);
 8524 
 8525  format %{
 8526     "cmpxchg_acq $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
 8527     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8528  %}
 8529 
 8530  ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
 8531             aarch64_enc_cset_eq(res));
 8532 
 8533   ins_pipe(pipe_slow);
 8534 %}
 8535 
 8536 instruct compareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 8537 
 8538   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 8539   match(Set res (CompareAndSwapP mem (Binary oldval newval)));
 8540   ins_cost(VOLATILE_REF_COST);
 8541 
 8542   effect(KILL cr);
 8543 
 8544  format %{
 8545     "cmpxchg_acq $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
 8546     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8547  %}
 8548 
 8549  ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
 8550             aarch64_enc_cset_eq(res));
 8551 
 8552   ins_pipe(pipe_slow);
 8553 %}
 8554 
 8555 instruct compareAndSwapNAcq(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
 8556 
 8557   predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
 8558   match(Set res (CompareAndSwapN mem (Binary oldval newval)));
 8559   ins_cost(VOLATILE_REF_COST);
 8560 
 8561   effect(KILL cr);
 8562 
 8563  format %{
 8564     "cmpxchgw_acq $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
 8565     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8566  %}
 8567 
 8568  ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
 8569             aarch64_enc_cset_eq(res));
 8570 
 8571   ins_pipe(pipe_slow);
 8572 %}
 8573 
 8574 
 8575 // ---------------------------------------------------------------------
 8576 
 8577 // BEGIN This section of the file is automatically generated. Do not edit --------------
 8578 
 8579 // Sundry CAS operations.  Note that release is always true,
 8580 // regardless of the memory ordering of the CAS.  This is because we
 8581 // need the volatile case to be sequentially consistent but there is
 8582 // no trailing StoreLoad barrier emitted by C2.  Unfortunately we
 8583 // can't check the type of memory ordering here, so we always emit a
 8584 // STLXR.
 8585 
 8586 // This section is generated from cas.m4
 8587 
 8588 
 8589 // This pattern is generated automatically from cas.m4.
 8590 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8591 instruct compareAndExchangeB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 8592   match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
 8593   ins_cost(2 * VOLATILE_REF_COST);
 8594   effect(TEMP_DEF res, KILL cr);
 8595   format %{
 8596     "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 8597   %}
 8598   ins_encode %{
 8599     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8600                Assembler::byte, /*acquire*/ false, /*release*/ true,
 8601                /*weak*/ false, $res$$Register);
 8602     __ sxtbw($res$$Register, $res$$Register);
 8603   %}
 8604   ins_pipe(pipe_slow);
 8605 %}
 8606 
 8607 // This pattern is generated automatically from cas.m4.
 8608 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8609 instruct compareAndExchangeS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 8610   match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
 8611   ins_cost(2 * VOLATILE_REF_COST);
 8612   effect(TEMP_DEF res, KILL cr);
 8613   format %{
 8614     "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 8615   %}
 8616   ins_encode %{
 8617     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8618                Assembler::halfword, /*acquire*/ false, /*release*/ true,
 8619                /*weak*/ false, $res$$Register);
 8620     __ sxthw($res$$Register, $res$$Register);
 8621   %}
 8622   ins_pipe(pipe_slow);
 8623 %}
 8624 
 8625 // This pattern is generated automatically from cas.m4.
 8626 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8627 instruct compareAndExchangeI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 8628   match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
 8629   ins_cost(2 * VOLATILE_REF_COST);
 8630   effect(TEMP_DEF res, KILL cr);
 8631   format %{
 8632     "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 8633   %}
 8634   ins_encode %{
 8635     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8636                Assembler::word, /*acquire*/ false, /*release*/ true,
 8637                /*weak*/ false, $res$$Register);
 8638   %}
 8639   ins_pipe(pipe_slow);
 8640 %}
 8641 
 8642 // This pattern is generated automatically from cas.m4.
 8643 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8644 instruct compareAndExchangeL(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 8645   match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
 8646   ins_cost(2 * VOLATILE_REF_COST);
 8647   effect(TEMP_DEF res, KILL cr);
 8648   format %{
 8649     "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 8650   %}
 8651   ins_encode %{
 8652     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8653                Assembler::xword, /*acquire*/ false, /*release*/ true,
 8654                /*weak*/ false, $res$$Register);
 8655   %}
 8656   ins_pipe(pipe_slow);
 8657 %}
 8658 
 8659 // This pattern is generated automatically from cas.m4.
 8660 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8661 instruct compareAndExchangeN(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 8662   predicate(n->as_LoadStore()->barrier_data() == 0);
 8663   match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
 8664   ins_cost(2 * VOLATILE_REF_COST);
 8665   effect(TEMP_DEF res, KILL cr);
 8666   format %{
 8667     "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 8668   %}
 8669   ins_encode %{
 8670     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8671                Assembler::word, /*acquire*/ false, /*release*/ true,
 8672                /*weak*/ false, $res$$Register);
 8673   %}
 8674   ins_pipe(pipe_slow);
 8675 %}
 8676 
 8677 // This pattern is generated automatically from cas.m4.
 8678 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8679 instruct compareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 8680   predicate(n->as_LoadStore()->barrier_data() == 0);
 8681   match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
 8682   ins_cost(2 * VOLATILE_REF_COST);
 8683   effect(TEMP_DEF res, KILL cr);
 8684   format %{
 8685     "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 8686   %}
 8687   ins_encode %{
 8688     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8689                Assembler::xword, /*acquire*/ false, /*release*/ true,
 8690                /*weak*/ false, $res$$Register);
 8691   %}
 8692   ins_pipe(pipe_slow);
 8693 %}
 8694 
 8695 // This pattern is generated automatically from cas.m4.
 8696 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8697 instruct compareAndExchangeBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 8698   predicate(needs_acquiring_load_exclusive(n));
 8699   match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
 8700   ins_cost(VOLATILE_REF_COST);
 8701   effect(TEMP_DEF res, KILL cr);
 8702   format %{
 8703     "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 8704   %}
 8705   ins_encode %{
 8706     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8707                Assembler::byte, /*acquire*/ true, /*release*/ true,
 8708                /*weak*/ false, $res$$Register);
 8709     __ sxtbw($res$$Register, $res$$Register);
 8710   %}
 8711   ins_pipe(pipe_slow);
 8712 %}
 8713 
 8714 // This pattern is generated automatically from cas.m4.
 8715 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8716 instruct compareAndExchangeSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 8717   predicate(needs_acquiring_load_exclusive(n));
 8718   match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
 8719   ins_cost(VOLATILE_REF_COST);
 8720   effect(TEMP_DEF res, KILL cr);
 8721   format %{
 8722     "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 8723   %}
 8724   ins_encode %{
 8725     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8726                Assembler::halfword, /*acquire*/ true, /*release*/ true,
 8727                /*weak*/ false, $res$$Register);
 8728     __ sxthw($res$$Register, $res$$Register);
 8729   %}
 8730   ins_pipe(pipe_slow);
 8731 %}
 8732 
 8733 // This pattern is generated automatically from cas.m4.
 8734 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8735 instruct compareAndExchangeIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 8736   predicate(needs_acquiring_load_exclusive(n));
 8737   match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
 8738   ins_cost(VOLATILE_REF_COST);
 8739   effect(TEMP_DEF res, KILL cr);
 8740   format %{
 8741     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 8742   %}
 8743   ins_encode %{
 8744     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8745                Assembler::word, /*acquire*/ true, /*release*/ true,
 8746                /*weak*/ false, $res$$Register);
 8747   %}
 8748   ins_pipe(pipe_slow);
 8749 %}
 8750 
 8751 // This pattern is generated automatically from cas.m4.
 8752 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8753 instruct compareAndExchangeLAcq(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 8754   predicate(needs_acquiring_load_exclusive(n));
 8755   match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
 8756   ins_cost(VOLATILE_REF_COST);
 8757   effect(TEMP_DEF res, KILL cr);
 8758   format %{
 8759     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 8760   %}
 8761   ins_encode %{
 8762     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8763                Assembler::xword, /*acquire*/ true, /*release*/ true,
 8764                /*weak*/ false, $res$$Register);
 8765   %}
 8766   ins_pipe(pipe_slow);
 8767 %}
 8768 
 8769 // This pattern is generated automatically from cas.m4.
 8770 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8771 instruct compareAndExchangeNAcq(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 8772   predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
 8773   match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
 8774   ins_cost(VOLATILE_REF_COST);
 8775   effect(TEMP_DEF res, KILL cr);
 8776   format %{
 8777     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 8778   %}
 8779   ins_encode %{
 8780     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8781                Assembler::word, /*acquire*/ true, /*release*/ true,
 8782                /*weak*/ false, $res$$Register);
 8783   %}
 8784   ins_pipe(pipe_slow);
 8785 %}
 8786 
 8787 // This pattern is generated automatically from cas.m4.
 8788 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8789 instruct compareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 8790   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 8791   match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
 8792   ins_cost(VOLATILE_REF_COST);
 8793   effect(TEMP_DEF res, KILL cr);
 8794   format %{
 8795     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 8796   %}
 8797   ins_encode %{
 8798     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8799                Assembler::xword, /*acquire*/ true, /*release*/ true,
 8800                /*weak*/ false, $res$$Register);
 8801   %}
 8802   ins_pipe(pipe_slow);
 8803 %}
 8804 
 8805 // This pattern is generated automatically from cas.m4.
 8806 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8807 instruct weakCompareAndSwapB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 8808   match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
 8809   ins_cost(2 * VOLATILE_REF_COST);
 8810   effect(KILL cr);
 8811   format %{
 8812     "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 8813     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8814   %}
 8815   ins_encode %{
 8816     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8817                Assembler::byte, /*acquire*/ false, /*release*/ true,
 8818                /*weak*/ true, noreg);
 8819     __ csetw($res$$Register, Assembler::EQ);
 8820   %}
 8821   ins_pipe(pipe_slow);
 8822 %}
 8823 
 8824 // This pattern is generated automatically from cas.m4.
 8825 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8826 instruct weakCompareAndSwapS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 8827   match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
 8828   ins_cost(2 * VOLATILE_REF_COST);
 8829   effect(KILL cr);
 8830   format %{
 8831     "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 8832     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8833   %}
 8834   ins_encode %{
 8835     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8836                Assembler::halfword, /*acquire*/ false, /*release*/ true,
 8837                /*weak*/ true, noreg);
 8838     __ csetw($res$$Register, Assembler::EQ);
 8839   %}
 8840   ins_pipe(pipe_slow);
 8841 %}
 8842 
 8843 // This pattern is generated automatically from cas.m4.
 8844 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8845 instruct weakCompareAndSwapI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 8846   match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
 8847   ins_cost(2 * VOLATILE_REF_COST);
 8848   effect(KILL cr);
 8849   format %{
 8850     "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 8851     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8852   %}
 8853   ins_encode %{
 8854     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8855                Assembler::word, /*acquire*/ false, /*release*/ true,
 8856                /*weak*/ true, noreg);
 8857     __ csetw($res$$Register, Assembler::EQ);
 8858   %}
 8859   ins_pipe(pipe_slow);
 8860 %}
 8861 
 8862 // This pattern is generated automatically from cas.m4.
 8863 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8864 instruct weakCompareAndSwapL(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 8865   match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
 8866   ins_cost(2 * VOLATILE_REF_COST);
 8867   effect(KILL cr);
 8868   format %{
 8869     "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 8870     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8871   %}
 8872   ins_encode %{
 8873     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8874                Assembler::xword, /*acquire*/ false, /*release*/ true,
 8875                /*weak*/ true, noreg);
 8876     __ csetw($res$$Register, Assembler::EQ);
 8877   %}
 8878   ins_pipe(pipe_slow);
 8879 %}
 8880 
 8881 // This pattern is generated automatically from cas.m4.
 8882 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8883 instruct weakCompareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 8884   predicate(n->as_LoadStore()->barrier_data() == 0);
 8885   match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
 8886   ins_cost(2 * VOLATILE_REF_COST);
 8887   effect(KILL cr);
 8888   format %{
 8889     "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 8890     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8891   %}
 8892   ins_encode %{
 8893     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8894                Assembler::word, /*acquire*/ false, /*release*/ true,
 8895                /*weak*/ true, noreg);
 8896     __ csetw($res$$Register, Assembler::EQ);
 8897   %}
 8898   ins_pipe(pipe_slow);
 8899 %}
 8900 
 8901 // This pattern is generated automatically from cas.m4.
 8902 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8903 instruct weakCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 8904   predicate(n->as_LoadStore()->barrier_data() == 0);
 8905   match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
 8906   ins_cost(2 * VOLATILE_REF_COST);
 8907   effect(KILL cr);
 8908   format %{
 8909     "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 8910     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8911   %}
 8912   ins_encode %{
 8913     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8914                Assembler::xword, /*acquire*/ false, /*release*/ true,
 8915                /*weak*/ true, noreg);
 8916     __ csetw($res$$Register, Assembler::EQ);
 8917   %}
 8918   ins_pipe(pipe_slow);
 8919 %}
 8920 
 8921 // This pattern is generated automatically from cas.m4.
 8922 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8923 instruct weakCompareAndSwapBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 8924   predicate(needs_acquiring_load_exclusive(n));
 8925   match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
 8926   ins_cost(VOLATILE_REF_COST);
 8927   effect(KILL cr);
 8928   format %{
 8929     "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 8930     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8931   %}
 8932   ins_encode %{
 8933     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8934                Assembler::byte, /*acquire*/ true, /*release*/ true,
 8935                /*weak*/ true, noreg);
 8936     __ csetw($res$$Register, Assembler::EQ);
 8937   %}
 8938   ins_pipe(pipe_slow);
 8939 %}
 8940 
 8941 // This pattern is generated automatically from cas.m4.
 8942 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8943 instruct weakCompareAndSwapSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 8944   predicate(needs_acquiring_load_exclusive(n));
 8945   match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
 8946   ins_cost(VOLATILE_REF_COST);
 8947   effect(KILL cr);
 8948   format %{
 8949     "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 8950     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8951   %}
 8952   ins_encode %{
 8953     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8954                Assembler::halfword, /*acquire*/ true, /*release*/ true,
 8955                /*weak*/ true, noreg);
 8956     __ csetw($res$$Register, Assembler::EQ);
 8957   %}
 8958   ins_pipe(pipe_slow);
 8959 %}
 8960 
 8961 // This pattern is generated automatically from cas.m4.
 8962 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8963 instruct weakCompareAndSwapIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 8964   predicate(needs_acquiring_load_exclusive(n));
 8965   match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
 8966   ins_cost(VOLATILE_REF_COST);
 8967   effect(KILL cr);
 8968   format %{
 8969     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 8970     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8971   %}
 8972   ins_encode %{
 8973     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8974                Assembler::word, /*acquire*/ true, /*release*/ true,
 8975                /*weak*/ true, noreg);
 8976     __ csetw($res$$Register, Assembler::EQ);
 8977   %}
 8978   ins_pipe(pipe_slow);
 8979 %}
 8980 
 8981 // This pattern is generated automatically from cas.m4.
 8982 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8983 instruct weakCompareAndSwapLAcq(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 8984   predicate(needs_acquiring_load_exclusive(n));
 8985   match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
 8986   ins_cost(VOLATILE_REF_COST);
 8987   effect(KILL cr);
 8988   format %{
 8989     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 8990     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8991   %}
 8992   ins_encode %{
 8993     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8994                Assembler::xword, /*acquire*/ true, /*release*/ true,
 8995                /*weak*/ true, noreg);
 8996     __ csetw($res$$Register, Assembler::EQ);
 8997   %}
 8998   ins_pipe(pipe_slow);
 8999 %}
 9000 
 9001 // This pattern is generated automatically from cas.m4.
 9002 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9003 instruct weakCompareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 9004   predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
 9005   match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
 9006   ins_cost(VOLATILE_REF_COST);
 9007   effect(KILL cr);
 9008   format %{
 9009     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 9010     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9011   %}
 9012   ins_encode %{
 9013     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9014                Assembler::word, /*acquire*/ true, /*release*/ true,
 9015                /*weak*/ true, noreg);
 9016     __ csetw($res$$Register, Assembler::EQ);
 9017   %}
 9018   ins_pipe(pipe_slow);
 9019 %}
 9020 
 9021 // This pattern is generated automatically from cas.m4.
 9022 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9023 instruct weakCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9024   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 9025   match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
 9026   ins_cost(VOLATILE_REF_COST);
 9027   effect(KILL cr);
 9028   format %{
 9029     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 9030     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9031   %}
 9032   ins_encode %{
 9033     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9034                Assembler::xword, /*acquire*/ true, /*release*/ true,
 9035                /*weak*/ true, noreg);
 9036     __ csetw($res$$Register, Assembler::EQ);
 9037   %}
 9038   ins_pipe(pipe_slow);
 9039 %}
 9040 
 9041 // END This section of the file is automatically generated. Do not edit --------------
 9042 // ---------------------------------------------------------------------
 9043 
 9044 instruct get_and_setI(indirect mem, iRegI newv, iRegINoSp prev) %{
 9045   match(Set prev (GetAndSetI mem newv));
 9046   ins_cost(2 * VOLATILE_REF_COST);
 9047   format %{ "atomic_xchgw  $prev, $newv, [$mem]" %}
 9048   ins_encode %{
 9049     __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9050   %}
 9051   ins_pipe(pipe_serial);
 9052 %}
 9053 
 9054 instruct get_and_setL(indirect mem, iRegL newv, iRegLNoSp prev) %{
 9055   match(Set prev (GetAndSetL mem newv));
 9056   ins_cost(2 * VOLATILE_REF_COST);
 9057   format %{ "atomic_xchg  $prev, $newv, [$mem]" %}
 9058   ins_encode %{
 9059     __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9060   %}
 9061   ins_pipe(pipe_serial);
 9062 %}
 9063 
 9064 instruct get_and_setN(indirect mem, iRegN newv, iRegINoSp prev) %{
 9065   predicate(n->as_LoadStore()->barrier_data() == 0);
 9066   match(Set prev (GetAndSetN mem newv));
 9067   ins_cost(2 * VOLATILE_REF_COST);
 9068   format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
 9069   ins_encode %{
 9070     __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9071   %}
 9072   ins_pipe(pipe_serial);
 9073 %}
 9074 
 9075 instruct get_and_setP(indirect mem, iRegP newv, iRegPNoSp prev) %{
 9076   predicate(n->as_LoadStore()->barrier_data() == 0);
 9077   match(Set prev (GetAndSetP mem newv));
 9078   ins_cost(2 * VOLATILE_REF_COST);
 9079   format %{ "atomic_xchg  $prev, $newv, [$mem]" %}
 9080   ins_encode %{
 9081     __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9082   %}
 9083   ins_pipe(pipe_serial);
 9084 %}
 9085 
 9086 instruct get_and_setIAcq(indirect mem, iRegI newv, iRegINoSp prev) %{
 9087   predicate(needs_acquiring_load_exclusive(n));
 9088   match(Set prev (GetAndSetI mem newv));
 9089   ins_cost(VOLATILE_REF_COST);
 9090   format %{ "atomic_xchgw_acq  $prev, $newv, [$mem]" %}
 9091   ins_encode %{
 9092     __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9093   %}
 9094   ins_pipe(pipe_serial);
 9095 %}
 9096 
 9097 instruct get_and_setLAcq(indirect mem, iRegL newv, iRegLNoSp prev) %{
 9098   predicate(needs_acquiring_load_exclusive(n));
 9099   match(Set prev (GetAndSetL mem newv));
 9100   ins_cost(VOLATILE_REF_COST);
 9101   format %{ "atomic_xchg_acq  $prev, $newv, [$mem]" %}
 9102   ins_encode %{
 9103     __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9104   %}
 9105   ins_pipe(pipe_serial);
 9106 %}
 9107 
 9108 instruct get_and_setNAcq(indirect mem, iRegN newv, iRegINoSp prev) %{
 9109   predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
 9110   match(Set prev (GetAndSetN mem newv));
 9111   ins_cost(VOLATILE_REF_COST);
 9112   format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
 9113   ins_encode %{
 9114     __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9115   %}
 9116   ins_pipe(pipe_serial);
 9117 %}
 9118 
 9119 instruct get_and_setPAcq(indirect mem, iRegP newv, iRegPNoSp prev) %{
 9120   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 9121   match(Set prev (GetAndSetP mem newv));
 9122   ins_cost(VOLATILE_REF_COST);
 9123   format %{ "atomic_xchg_acq  $prev, $newv, [$mem]" %}
 9124   ins_encode %{
 9125     __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9126   %}
 9127   ins_pipe(pipe_serial);
 9128 %}
 9129 
 9130 
 9131 instruct get_and_addL(indirect mem, iRegLNoSp newval, iRegL incr) %{
 9132   match(Set newval (GetAndAddL mem incr));
 9133   ins_cost(2 * VOLATILE_REF_COST + 1);
 9134   format %{ "get_and_addL $newval, [$mem], $incr" %}
 9135   ins_encode %{
 9136     __ atomic_add($newval$$Register, $incr$$Register, as_Register($mem$$base));
 9137   %}
 9138   ins_pipe(pipe_serial);
 9139 %}
 9140 
 9141 instruct get_and_addL_no_res(indirect mem, Universe dummy, iRegL incr) %{
 9142   predicate(n->as_LoadStore()->result_not_used());
 9143   match(Set dummy (GetAndAddL mem incr));
 9144   ins_cost(2 * VOLATILE_REF_COST);
 9145   format %{ "get_and_addL [$mem], $incr" %}
 9146   ins_encode %{
 9147     __ atomic_add(noreg, $incr$$Register, as_Register($mem$$base));
 9148   %}
 9149   ins_pipe(pipe_serial);
 9150 %}
 9151 
 9152 instruct get_and_addLi(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
 9153   match(Set newval (GetAndAddL mem incr));
 9154   ins_cost(2 * VOLATILE_REF_COST + 1);
 9155   format %{ "get_and_addL $newval, [$mem], $incr" %}
 9156   ins_encode %{
 9157     __ atomic_add($newval$$Register, $incr$$constant, as_Register($mem$$base));
 9158   %}
 9159   ins_pipe(pipe_serial);
 9160 %}
 9161 
 9162 instruct get_and_addLi_no_res(indirect mem, Universe dummy, immLAddSub incr) %{
 9163   predicate(n->as_LoadStore()->result_not_used());
 9164   match(Set dummy (GetAndAddL mem incr));
 9165   ins_cost(2 * VOLATILE_REF_COST);
 9166   format %{ "get_and_addL [$mem], $incr" %}
 9167   ins_encode %{
 9168     __ atomic_add(noreg, $incr$$constant, as_Register($mem$$base));
 9169   %}
 9170   ins_pipe(pipe_serial);
 9171 %}
 9172 
 9173 instruct get_and_addI(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
 9174   match(Set newval (GetAndAddI mem incr));
 9175   ins_cost(2 * VOLATILE_REF_COST + 1);
 9176   format %{ "get_and_addI $newval, [$mem], $incr" %}
 9177   ins_encode %{
 9178     __ atomic_addw($newval$$Register, $incr$$Register, as_Register($mem$$base));
 9179   %}
 9180   ins_pipe(pipe_serial);
 9181 %}
 9182 
 9183 instruct get_and_addI_no_res(indirect mem, Universe dummy, iRegIorL2I incr) %{
 9184   predicate(n->as_LoadStore()->result_not_used());
 9185   match(Set dummy (GetAndAddI mem incr));
 9186   ins_cost(2 * VOLATILE_REF_COST);
 9187   format %{ "get_and_addI [$mem], $incr" %}
 9188   ins_encode %{
 9189     __ atomic_addw(noreg, $incr$$Register, as_Register($mem$$base));
 9190   %}
 9191   ins_pipe(pipe_serial);
 9192 %}
 9193 
 9194 instruct get_and_addIi(indirect mem, iRegINoSp newval, immIAddSub incr) %{
 9195   match(Set newval (GetAndAddI mem incr));
 9196   ins_cost(2 * VOLATILE_REF_COST + 1);
 9197   format %{ "get_and_addI $newval, [$mem], $incr" %}
 9198   ins_encode %{
 9199     __ atomic_addw($newval$$Register, $incr$$constant, as_Register($mem$$base));
 9200   %}
 9201   ins_pipe(pipe_serial);
 9202 %}
 9203 
 9204 instruct get_and_addIi_no_res(indirect mem, Universe dummy, immIAddSub incr) %{
 9205   predicate(n->as_LoadStore()->result_not_used());
 9206   match(Set dummy (GetAndAddI mem incr));
 9207   ins_cost(2 * VOLATILE_REF_COST);
 9208   format %{ "get_and_addI [$mem], $incr" %}
 9209   ins_encode %{
 9210     __ atomic_addw(noreg, $incr$$constant, as_Register($mem$$base));
 9211   %}
 9212   ins_pipe(pipe_serial);
 9213 %}
 9214 
 9215 instruct get_and_addLAcq(indirect mem, iRegLNoSp newval, iRegL incr) %{
 9216   predicate(needs_acquiring_load_exclusive(n));
 9217   match(Set newval (GetAndAddL mem incr));
 9218   ins_cost(VOLATILE_REF_COST + 1);
 9219   format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
 9220   ins_encode %{
 9221     __ atomic_addal($newval$$Register, $incr$$Register, as_Register($mem$$base));
 9222   %}
 9223   ins_pipe(pipe_serial);
 9224 %}
 9225 
 9226 instruct get_and_addL_no_resAcq(indirect mem, Universe dummy, iRegL incr) %{
 9227   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
 9228   match(Set dummy (GetAndAddL mem incr));
 9229   ins_cost(VOLATILE_REF_COST);
 9230   format %{ "get_and_addL_acq [$mem], $incr" %}
 9231   ins_encode %{
 9232     __ atomic_addal(noreg, $incr$$Register, as_Register($mem$$base));
 9233   %}
 9234   ins_pipe(pipe_serial);
 9235 %}
 9236 
 9237 instruct get_and_addLiAcq(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
 9238   predicate(needs_acquiring_load_exclusive(n));
 9239   match(Set newval (GetAndAddL mem incr));
 9240   ins_cost(VOLATILE_REF_COST + 1);
 9241   format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
 9242   ins_encode %{
 9243     __ atomic_addal($newval$$Register, $incr$$constant, as_Register($mem$$base));
 9244   %}
 9245   ins_pipe(pipe_serial);
 9246 %}
 9247 
 9248 instruct get_and_addLi_no_resAcq(indirect mem, Universe dummy, immLAddSub incr) %{
 9249   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
 9250   match(Set dummy (GetAndAddL mem incr));
 9251   ins_cost(VOLATILE_REF_COST);
 9252   format %{ "get_and_addL_acq [$mem], $incr" %}
 9253   ins_encode %{
 9254     __ atomic_addal(noreg, $incr$$constant, as_Register($mem$$base));
 9255   %}
 9256   ins_pipe(pipe_serial);
 9257 %}
 9258 
 9259 instruct get_and_addIAcq(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
 9260   predicate(needs_acquiring_load_exclusive(n));
 9261   match(Set newval (GetAndAddI mem incr));
 9262   ins_cost(VOLATILE_REF_COST + 1);
 9263   format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
 9264   ins_encode %{
 9265     __ atomic_addalw($newval$$Register, $incr$$Register, as_Register($mem$$base));
 9266   %}
 9267   ins_pipe(pipe_serial);
 9268 %}
 9269 
 9270 instruct get_and_addI_no_resAcq(indirect mem, Universe dummy, iRegIorL2I incr) %{
 9271   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
 9272   match(Set dummy (GetAndAddI mem incr));
 9273   ins_cost(VOLATILE_REF_COST);
 9274   format %{ "get_and_addI_acq [$mem], $incr" %}
 9275   ins_encode %{
 9276     __ atomic_addalw(noreg, $incr$$Register, as_Register($mem$$base));
 9277   %}
 9278   ins_pipe(pipe_serial);
 9279 %}
 9280 
 9281 instruct get_and_addIiAcq(indirect mem, iRegINoSp newval, immIAddSub incr) %{
 9282   predicate(needs_acquiring_load_exclusive(n));
 9283   match(Set newval (GetAndAddI mem incr));
 9284   ins_cost(VOLATILE_REF_COST + 1);
 9285   format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
 9286   ins_encode %{
 9287     __ atomic_addalw($newval$$Register, $incr$$constant, as_Register($mem$$base));
 9288   %}
 9289   ins_pipe(pipe_serial);
 9290 %}
 9291 
 9292 instruct get_and_addIi_no_resAcq(indirect mem, Universe dummy, immIAddSub incr) %{
 9293   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
 9294   match(Set dummy (GetAndAddI mem incr));
 9295   ins_cost(VOLATILE_REF_COST);
 9296   format %{ "get_and_addI_acq [$mem], $incr" %}
 9297   ins_encode %{
 9298     __ atomic_addalw(noreg, $incr$$constant, as_Register($mem$$base));
 9299   %}
 9300   ins_pipe(pipe_serial);
 9301 %}
 9302 
 9303 // Manifest a CmpU result in an integer register.
 9304 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
 9305 instruct cmpU3_reg_reg(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg flags)
 9306 %{
 9307   match(Set dst (CmpU3 src1 src2));
 9308   effect(KILL flags);
 9309 
 9310   ins_cost(INSN_COST * 3);
 9311   format %{
 9312       "cmpw $src1, $src2\n\t"
 9313       "csetw $dst, ne\n\t"
 9314       "cnegw $dst, lo\t# CmpU3(reg)"
 9315   %}
 9316   ins_encode %{
 9317     __ cmpw($src1$$Register, $src2$$Register);
 9318     __ csetw($dst$$Register, Assembler::NE);
 9319     __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
 9320   %}
 9321 
 9322   ins_pipe(pipe_class_default);
 9323 %}
 9324 
 9325 instruct cmpU3_reg_imm(iRegINoSp dst, iRegI src1, immIAddSub src2, rFlagsReg flags)
 9326 %{
 9327   match(Set dst (CmpU3 src1 src2));
 9328   effect(KILL flags);
 9329 
 9330   ins_cost(INSN_COST * 3);
 9331   format %{
 9332       "subsw zr, $src1, $src2\n\t"
 9333       "csetw $dst, ne\n\t"
 9334       "cnegw $dst, lo\t# CmpU3(imm)"
 9335   %}
 9336   ins_encode %{
 9337     __ subsw(zr, $src1$$Register, (int32_t)$src2$$constant);
 9338     __ csetw($dst$$Register, Assembler::NE);
 9339     __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
 9340   %}
 9341 
 9342   ins_pipe(pipe_class_default);
 9343 %}
 9344 
 9345 // Manifest a CmpUL result in an integer register.
 9346 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
 9347 instruct cmpUL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
 9348 %{
 9349   match(Set dst (CmpUL3 src1 src2));
 9350   effect(KILL flags);
 9351 
 9352   ins_cost(INSN_COST * 3);
 9353   format %{
 9354       "cmp $src1, $src2\n\t"
 9355       "csetw $dst, ne\n\t"
 9356       "cnegw $dst, lo\t# CmpUL3(reg)"
 9357   %}
 9358   ins_encode %{
 9359     __ cmp($src1$$Register, $src2$$Register);
 9360     __ csetw($dst$$Register, Assembler::NE);
 9361     __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
 9362   %}
 9363 
 9364   ins_pipe(pipe_class_default);
 9365 %}
 9366 
 9367 instruct cmpUL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
 9368 %{
 9369   match(Set dst (CmpUL3 src1 src2));
 9370   effect(KILL flags);
 9371 
 9372   ins_cost(INSN_COST * 3);
 9373   format %{
 9374       "subs zr, $src1, $src2\n\t"
 9375       "csetw $dst, ne\n\t"
 9376       "cnegw $dst, lo\t# CmpUL3(imm)"
 9377   %}
 9378   ins_encode %{
 9379     __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
 9380     __ csetw($dst$$Register, Assembler::NE);
 9381     __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
 9382   %}
 9383 
 9384   ins_pipe(pipe_class_default);
 9385 %}
 9386 
 9387 // Manifest a CmpL result in an integer register.
 9388 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
 9389 instruct cmpL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
 9390 %{
 9391   match(Set dst (CmpL3 src1 src2));
 9392   effect(KILL flags);
 9393 
 9394   ins_cost(INSN_COST * 3);
 9395   format %{
 9396       "cmp $src1, $src2\n\t"
 9397       "csetw $dst, ne\n\t"
 9398       "cnegw $dst, lt\t# CmpL3(reg)"
 9399   %}
 9400   ins_encode %{
 9401     __ cmp($src1$$Register, $src2$$Register);
 9402     __ csetw($dst$$Register, Assembler::NE);
 9403     __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
 9404   %}
 9405 
 9406   ins_pipe(pipe_class_default);
 9407 %}
 9408 
 9409 instruct cmpL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
 9410 %{
 9411   match(Set dst (CmpL3 src1 src2));
 9412   effect(KILL flags);
 9413 
 9414   ins_cost(INSN_COST * 3);
 9415   format %{
 9416       "subs zr, $src1, $src2\n\t"
 9417       "csetw $dst, ne\n\t"
 9418       "cnegw $dst, lt\t# CmpL3(imm)"
 9419   %}
 9420   ins_encode %{
 9421     __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
 9422     __ csetw($dst$$Register, Assembler::NE);
 9423     __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
 9424   %}
 9425 
 9426   ins_pipe(pipe_class_default);
 9427 %}
 9428 
 9429 // ============================================================================
 9430 // Conditional Move Instructions
 9431 
 9432 // n.b. we have identical rules for both a signed compare op (cmpOp)
 9433 // and an unsigned compare op (cmpOpU). it would be nice if we could
 9434 // define an op class which merged both inputs and use it to type the
 9435 // argument to a single rule. unfortunatelyt his fails because the
 9436 // opclass does not live up to the COND_INTER interface of its
 9437 // component operands. When the generic code tries to negate the
 9438 // operand it ends up running the generci Machoper::negate method
 9439 // which throws a ShouldNotHappen. So, we have to provide two flavours
 9440 // of each rule, one for a cmpOp and a second for a cmpOpU (sigh).
 9441 
 9442 instruct cmovI_reg_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
 9443   match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
 9444 
 9445   ins_cost(INSN_COST * 2);
 9446   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, int"  %}
 9447 
 9448   ins_encode %{
 9449     __ cselw(as_Register($dst$$reg),
 9450              as_Register($src2$$reg),
 9451              as_Register($src1$$reg),
 9452              (Assembler::Condition)$cmp$$cmpcode);
 9453   %}
 9454 
 9455   ins_pipe(icond_reg_reg);
 9456 %}
 9457 
 9458 instruct cmovUI_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
 9459   match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
 9460 
 9461   ins_cost(INSN_COST * 2);
 9462   format %{ "cselw $dst, $src2, $src1 $cmp\t# unsigned, int"  %}
 9463 
 9464   ins_encode %{
 9465     __ cselw(as_Register($dst$$reg),
 9466              as_Register($src2$$reg),
 9467              as_Register($src1$$reg),
 9468              (Assembler::Condition)$cmp$$cmpcode);
 9469   %}
 9470 
 9471   ins_pipe(icond_reg_reg);
 9472 %}
 9473 
 9474 // special cases where one arg is zero
 9475 
 9476 // n.b. this is selected in preference to the rule above because it
 9477 // avoids loading constant 0 into a source register
 9478 
 9479 // TODO
 9480 // we ought only to be able to cull one of these variants as the ideal
 9481 // transforms ought always to order the zero consistently (to left/right?)
 9482 
 9483 instruct cmovI_zero_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
 9484   match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
 9485 
 9486   ins_cost(INSN_COST * 2);
 9487   format %{ "cselw $dst, $src, zr $cmp\t# signed, int"  %}
 9488 
 9489   ins_encode %{
 9490     __ cselw(as_Register($dst$$reg),
 9491              as_Register($src$$reg),
 9492              zr,
 9493              (Assembler::Condition)$cmp$$cmpcode);
 9494   %}
 9495 
 9496   ins_pipe(icond_reg);
 9497 %}
 9498 
 9499 instruct cmovUI_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
 9500   match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
 9501 
 9502   ins_cost(INSN_COST * 2);
 9503   format %{ "cselw $dst, $src, zr $cmp\t# unsigned, int"  %}
 9504 
 9505   ins_encode %{
 9506     __ cselw(as_Register($dst$$reg),
 9507              as_Register($src$$reg),
 9508              zr,
 9509              (Assembler::Condition)$cmp$$cmpcode);
 9510   %}
 9511 
 9512   ins_pipe(icond_reg);
 9513 %}
 9514 
 9515 instruct cmovI_reg_zero(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
 9516   match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
 9517 
 9518   ins_cost(INSN_COST * 2);
 9519   format %{ "cselw $dst, zr, $src $cmp\t# signed, int"  %}
 9520 
 9521   ins_encode %{
 9522     __ cselw(as_Register($dst$$reg),
 9523              zr,
 9524              as_Register($src$$reg),
 9525              (Assembler::Condition)$cmp$$cmpcode);
 9526   %}
 9527 
 9528   ins_pipe(icond_reg);
 9529 %}
 9530 
 9531 instruct cmovUI_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
 9532   match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
 9533 
 9534   ins_cost(INSN_COST * 2);
 9535   format %{ "cselw $dst, zr, $src $cmp\t# unsigned, int"  %}
 9536 
 9537   ins_encode %{
 9538     __ cselw(as_Register($dst$$reg),
 9539              zr,
 9540              as_Register($src$$reg),
 9541              (Assembler::Condition)$cmp$$cmpcode);
 9542   %}
 9543 
 9544   ins_pipe(icond_reg);
 9545 %}
 9546 
 9547 // special case for creating a boolean 0 or 1
 9548 
 9549 // n.b. this is selected in preference to the rule above because it
 9550 // avoids loading constants 0 and 1 into a source register
 9551 
 9552 instruct cmovI_reg_zero_one(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
 9553   match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
 9554 
 9555   ins_cost(INSN_COST * 2);
 9556   format %{ "csincw $dst, zr, zr $cmp\t# signed, int"  %}
 9557 
 9558   ins_encode %{
 9559     // equivalently
 9560     // cset(as_Register($dst$$reg),
 9561     //      negate_condition((Assembler::Condition)$cmp$$cmpcode));
 9562     __ csincw(as_Register($dst$$reg),
 9563              zr,
 9564              zr,
 9565              (Assembler::Condition)$cmp$$cmpcode);
 9566   %}
 9567 
 9568   ins_pipe(icond_none);
 9569 %}
 9570 
 9571 instruct cmovUI_reg_zero_one(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
 9572   match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
 9573 
 9574   ins_cost(INSN_COST * 2);
 9575   format %{ "csincw $dst, zr, zr $cmp\t# unsigned, int"  %}
 9576 
 9577   ins_encode %{
 9578     // equivalently
 9579     // cset(as_Register($dst$$reg),
 9580     //      negate_condition((Assembler::Condition)$cmp$$cmpcode));
 9581     __ csincw(as_Register($dst$$reg),
 9582              zr,
 9583              zr,
 9584              (Assembler::Condition)$cmp$$cmpcode);
 9585   %}
 9586 
 9587   ins_pipe(icond_none);
 9588 %}
 9589 
 9590 instruct cmovL_reg_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
 9591   match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
 9592 
 9593   ins_cost(INSN_COST * 2);
 9594   format %{ "csel $dst, $src2, $src1 $cmp\t# signed, long"  %}
 9595 
 9596   ins_encode %{
 9597     __ csel(as_Register($dst$$reg),
 9598             as_Register($src2$$reg),
 9599             as_Register($src1$$reg),
 9600             (Assembler::Condition)$cmp$$cmpcode);
 9601   %}
 9602 
 9603   ins_pipe(icond_reg_reg);
 9604 %}
 9605 
 9606 instruct cmovUL_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
 9607   match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
 9608 
 9609   ins_cost(INSN_COST * 2);
 9610   format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, long"  %}
 9611 
 9612   ins_encode %{
 9613     __ csel(as_Register($dst$$reg),
 9614             as_Register($src2$$reg),
 9615             as_Register($src1$$reg),
 9616             (Assembler::Condition)$cmp$$cmpcode);
 9617   %}
 9618 
 9619   ins_pipe(icond_reg_reg);
 9620 %}
 9621 
 9622 // special cases where one arg is zero
 9623 
 9624 instruct cmovL_reg_zero(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
 9625   match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
 9626 
 9627   ins_cost(INSN_COST * 2);
 9628   format %{ "csel $dst, zr, $src $cmp\t# signed, long"  %}
 9629 
 9630   ins_encode %{
 9631     __ csel(as_Register($dst$$reg),
 9632             zr,
 9633             as_Register($src$$reg),
 9634             (Assembler::Condition)$cmp$$cmpcode);
 9635   %}
 9636 
 9637   ins_pipe(icond_reg);
 9638 %}
 9639 
 9640 instruct cmovUL_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
 9641   match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
 9642 
 9643   ins_cost(INSN_COST * 2);
 9644   format %{ "csel $dst, zr, $src $cmp\t# unsigned, long"  %}
 9645 
 9646   ins_encode %{
 9647     __ csel(as_Register($dst$$reg),
 9648             zr,
 9649             as_Register($src$$reg),
 9650             (Assembler::Condition)$cmp$$cmpcode);
 9651   %}
 9652 
 9653   ins_pipe(icond_reg);
 9654 %}
 9655 
 9656 instruct cmovL_zero_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
 9657   match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
 9658 
 9659   ins_cost(INSN_COST * 2);
 9660   format %{ "csel $dst, $src, zr $cmp\t# signed, long"  %}
 9661 
 9662   ins_encode %{
 9663     __ csel(as_Register($dst$$reg),
 9664             as_Register($src$$reg),
 9665             zr,
 9666             (Assembler::Condition)$cmp$$cmpcode);
 9667   %}
 9668 
 9669   ins_pipe(icond_reg);
 9670 %}
 9671 
 9672 instruct cmovUL_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
 9673   match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
 9674 
 9675   ins_cost(INSN_COST * 2);
 9676   format %{ "csel $dst, $src, zr $cmp\t# unsigned, long"  %}
 9677 
 9678   ins_encode %{
 9679     __ csel(as_Register($dst$$reg),
 9680             as_Register($src$$reg),
 9681             zr,
 9682             (Assembler::Condition)$cmp$$cmpcode);
 9683   %}
 9684 
 9685   ins_pipe(icond_reg);
 9686 %}
 9687 
 9688 instruct cmovP_reg_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
 9689   match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
 9690 
 9691   ins_cost(INSN_COST * 2);
 9692   format %{ "csel $dst, $src2, $src1 $cmp\t# signed, ptr"  %}
 9693 
 9694   ins_encode %{
 9695     __ csel(as_Register($dst$$reg),
 9696             as_Register($src2$$reg),
 9697             as_Register($src1$$reg),
 9698             (Assembler::Condition)$cmp$$cmpcode);
 9699   %}
 9700 
 9701   ins_pipe(icond_reg_reg);
 9702 %}
 9703 
 9704 instruct cmovUP_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
 9705   match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
 9706 
 9707   ins_cost(INSN_COST * 2);
 9708   format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, ptr"  %}
 9709 
 9710   ins_encode %{
 9711     __ csel(as_Register($dst$$reg),
 9712             as_Register($src2$$reg),
 9713             as_Register($src1$$reg),
 9714             (Assembler::Condition)$cmp$$cmpcode);
 9715   %}
 9716 
 9717   ins_pipe(icond_reg_reg);
 9718 %}
 9719 
 9720 // special cases where one arg is zero
 9721 
 9722 instruct cmovP_reg_zero(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
 9723   match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
 9724 
 9725   ins_cost(INSN_COST * 2);
 9726   format %{ "csel $dst, zr, $src $cmp\t# signed, ptr"  %}
 9727 
 9728   ins_encode %{
 9729     __ csel(as_Register($dst$$reg),
 9730             zr,
 9731             as_Register($src$$reg),
 9732             (Assembler::Condition)$cmp$$cmpcode);
 9733   %}
 9734 
 9735   ins_pipe(icond_reg);
 9736 %}
 9737 
 9738 instruct cmovUP_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
 9739   match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
 9740 
 9741   ins_cost(INSN_COST * 2);
 9742   format %{ "csel $dst, zr, $src $cmp\t# unsigned, ptr"  %}
 9743 
 9744   ins_encode %{
 9745     __ csel(as_Register($dst$$reg),
 9746             zr,
 9747             as_Register($src$$reg),
 9748             (Assembler::Condition)$cmp$$cmpcode);
 9749   %}
 9750 
 9751   ins_pipe(icond_reg);
 9752 %}
 9753 
 9754 instruct cmovP_zero_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
 9755   match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
 9756 
 9757   ins_cost(INSN_COST * 2);
 9758   format %{ "csel $dst, $src, zr $cmp\t# signed, ptr"  %}
 9759 
 9760   ins_encode %{
 9761     __ csel(as_Register($dst$$reg),
 9762             as_Register($src$$reg),
 9763             zr,
 9764             (Assembler::Condition)$cmp$$cmpcode);
 9765   %}
 9766 
 9767   ins_pipe(icond_reg);
 9768 %}
 9769 
 9770 instruct cmovUP_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
 9771   match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
 9772 
 9773   ins_cost(INSN_COST * 2);
 9774   format %{ "csel $dst, $src, zr $cmp\t# unsigned, ptr"  %}
 9775 
 9776   ins_encode %{
 9777     __ csel(as_Register($dst$$reg),
 9778             as_Register($src$$reg),
 9779             zr,
 9780             (Assembler::Condition)$cmp$$cmpcode);
 9781   %}
 9782 
 9783   ins_pipe(icond_reg);
 9784 %}
 9785 
 9786 instruct cmovN_reg_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
 9787   match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
 9788 
 9789   ins_cost(INSN_COST * 2);
 9790   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr"  %}
 9791 
 9792   ins_encode %{
 9793     __ cselw(as_Register($dst$$reg),
 9794              as_Register($src2$$reg),
 9795              as_Register($src1$$reg),
 9796              (Assembler::Condition)$cmp$$cmpcode);
 9797   %}
 9798 
 9799   ins_pipe(icond_reg_reg);
 9800 %}
 9801 
 9802 instruct cmovUN_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
 9803   match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
 9804 
 9805   ins_cost(INSN_COST * 2);
 9806   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr"  %}
 9807 
 9808   ins_encode %{
 9809     __ cselw(as_Register($dst$$reg),
 9810              as_Register($src2$$reg),
 9811              as_Register($src1$$reg),
 9812              (Assembler::Condition)$cmp$$cmpcode);
 9813   %}
 9814 
 9815   ins_pipe(icond_reg_reg);
 9816 %}
 9817 
 9818 // special cases where one arg is zero
 9819 
 9820 instruct cmovN_reg_zero(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
 9821   match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
 9822 
 9823   ins_cost(INSN_COST * 2);
 9824   format %{ "cselw $dst, zr, $src $cmp\t# signed, compressed ptr"  %}
 9825 
 9826   ins_encode %{
 9827     __ cselw(as_Register($dst$$reg),
 9828              zr,
 9829              as_Register($src$$reg),
 9830              (Assembler::Condition)$cmp$$cmpcode);
 9831   %}
 9832 
 9833   ins_pipe(icond_reg);
 9834 %}
 9835 
 9836 instruct cmovUN_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
 9837   match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
 9838 
 9839   ins_cost(INSN_COST * 2);
 9840   format %{ "cselw $dst, zr, $src $cmp\t# unsigned, compressed ptr"  %}
 9841 
 9842   ins_encode %{
 9843     __ cselw(as_Register($dst$$reg),
 9844              zr,
 9845              as_Register($src$$reg),
 9846              (Assembler::Condition)$cmp$$cmpcode);
 9847   %}
 9848 
 9849   ins_pipe(icond_reg);
 9850 %}
 9851 
 9852 instruct cmovN_zero_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
 9853   match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
 9854 
 9855   ins_cost(INSN_COST * 2);
 9856   format %{ "cselw $dst, $src, zr $cmp\t# signed, compressed ptr"  %}
 9857 
 9858   ins_encode %{
 9859     __ cselw(as_Register($dst$$reg),
 9860              as_Register($src$$reg),
 9861              zr,
 9862              (Assembler::Condition)$cmp$$cmpcode);
 9863   %}
 9864 
 9865   ins_pipe(icond_reg);
 9866 %}
 9867 
 9868 instruct cmovUN_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
 9869   match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
 9870 
 9871   ins_cost(INSN_COST * 2);
 9872   format %{ "cselw $dst, $src, zr $cmp\t# unsigned, compressed ptr"  %}
 9873 
 9874   ins_encode %{
 9875     __ cselw(as_Register($dst$$reg),
 9876              as_Register($src$$reg),
 9877              zr,
 9878              (Assembler::Condition)$cmp$$cmpcode);
 9879   %}
 9880 
 9881   ins_pipe(icond_reg);
 9882 %}
 9883 
 9884 instruct cmovF_reg(cmpOp cmp, rFlagsReg cr, vRegF dst, vRegF src1,  vRegF src2)
 9885 %{
 9886   match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
 9887 
 9888   ins_cost(INSN_COST * 3);
 9889 
 9890   format %{ "fcsels $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
 9891   ins_encode %{
 9892     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
 9893     __ fcsels(as_FloatRegister($dst$$reg),
 9894               as_FloatRegister($src2$$reg),
 9895               as_FloatRegister($src1$$reg),
 9896               cond);
 9897   %}
 9898 
 9899   ins_pipe(fp_cond_reg_reg_s);
 9900 %}
 9901 
 9902 instruct cmovUF_reg(cmpOpU cmp, rFlagsRegU cr, vRegF dst, vRegF src1,  vRegF src2)
 9903 %{
 9904   match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
 9905 
 9906   ins_cost(INSN_COST * 3);
 9907 
 9908   format %{ "fcsels $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
 9909   ins_encode %{
 9910     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
 9911     __ fcsels(as_FloatRegister($dst$$reg),
 9912               as_FloatRegister($src2$$reg),
 9913               as_FloatRegister($src1$$reg),
 9914               cond);
 9915   %}
 9916 
 9917   ins_pipe(fp_cond_reg_reg_s);
 9918 %}
 9919 
 9920 instruct cmovD_reg(cmpOp cmp, rFlagsReg cr, vRegD dst, vRegD src1,  vRegD src2)
 9921 %{
 9922   match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
 9923 
 9924   ins_cost(INSN_COST * 3);
 9925 
 9926   format %{ "fcseld $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
 9927   ins_encode %{
 9928     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
 9929     __ fcseld(as_FloatRegister($dst$$reg),
 9930               as_FloatRegister($src2$$reg),
 9931               as_FloatRegister($src1$$reg),
 9932               cond);
 9933   %}
 9934 
 9935   ins_pipe(fp_cond_reg_reg_d);
 9936 %}
 9937 
 9938 instruct cmovUD_reg(cmpOpU cmp, rFlagsRegU cr, vRegD dst, vRegD src1,  vRegD src2)
 9939 %{
 9940   match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
 9941 
 9942   ins_cost(INSN_COST * 3);
 9943 
 9944   format %{ "fcseld $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
 9945   ins_encode %{
 9946     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
 9947     __ fcseld(as_FloatRegister($dst$$reg),
 9948               as_FloatRegister($src2$$reg),
 9949               as_FloatRegister($src1$$reg),
 9950               cond);
 9951   %}
 9952 
 9953   ins_pipe(fp_cond_reg_reg_d);
 9954 %}
 9955 
 9956 // ============================================================================
 9957 // Arithmetic Instructions
 9958 //
 9959 
 9960 // Integer Addition
 9961 
 9962 // TODO
 9963 // these currently employ operations which do not set CR and hence are
 9964 // not flagged as killing CR but we would like to isolate the cases
 9965 // where we want to set flags from those where we don't. need to work
 9966 // out how to do that.
 9967 
 9968 instruct addI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
 9969   match(Set dst (AddI src1 src2));
 9970 
 9971   ins_cost(INSN_COST);
 9972   format %{ "addw  $dst, $src1, $src2" %}
 9973 
 9974   ins_encode %{
 9975     __ addw(as_Register($dst$$reg),
 9976             as_Register($src1$$reg),
 9977             as_Register($src2$$reg));
 9978   %}
 9979 
 9980   ins_pipe(ialu_reg_reg);
 9981 %}
 9982 
 9983 instruct addI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
 9984   match(Set dst (AddI src1 src2));
 9985 
 9986   ins_cost(INSN_COST);
 9987   format %{ "addw $dst, $src1, $src2" %}
 9988 
 9989   // use opcode to indicate that this is an add not a sub
 9990   opcode(0x0);
 9991 
 9992   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
 9993 
 9994   ins_pipe(ialu_reg_imm);
 9995 %}
 9996 
 9997 instruct addI_reg_imm_i2l(iRegINoSp dst, iRegL src1, immIAddSub src2) %{
 9998   match(Set dst (AddI (ConvL2I src1) src2));
 9999 
10000   ins_cost(INSN_COST);
10001   format %{ "addw $dst, $src1, $src2" %}
10002 
10003   // use opcode to indicate that this is an add not a sub
10004   opcode(0x0);
10005 
10006   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10007 
10008   ins_pipe(ialu_reg_imm);
10009 %}
10010 
10011 // Pointer Addition
10012 instruct addP_reg_reg(iRegPNoSp dst, iRegPorL2P src1, iRegL src2) %{
10013   match(Set dst (AddP src1 src2));
10014 
10015   ins_cost(INSN_COST);
10016   format %{ "add $dst, $src1, $src2\t# ptr" %}
10017 
10018   ins_encode %{
10019     __ add(as_Register($dst$$reg),
10020            as_Register($src1$$reg),
10021            as_Register($src2$$reg));
10022   %}
10023 
10024   ins_pipe(ialu_reg_reg);
10025 %}
10026 
10027 instruct addP_reg_reg_ext(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2) %{
10028   match(Set dst (AddP src1 (ConvI2L src2)));
10029 
10030   ins_cost(1.9 * INSN_COST);
10031   format %{ "add $dst, $src1, $src2, sxtw\t# ptr" %}
10032 
10033   ins_encode %{
10034     __ add(as_Register($dst$$reg),
10035            as_Register($src1$$reg),
10036            as_Register($src2$$reg), ext::sxtw);
10037   %}
10038 
10039   ins_pipe(ialu_reg_reg);
10040 %}
10041 
10042 instruct addP_reg_reg_lsl(iRegPNoSp dst, iRegPorL2P src1, iRegL src2, immIScale scale) %{
10043   match(Set dst (AddP src1 (LShiftL src2 scale)));
10044 
10045   ins_cost(1.9 * INSN_COST);
10046   format %{ "add $dst, $src1, $src2, LShiftL $scale\t# ptr" %}
10047 
10048   ins_encode %{
10049     __ lea(as_Register($dst$$reg),
10050            Address(as_Register($src1$$reg), as_Register($src2$$reg),
10051                    Address::lsl($scale$$constant)));
10052   %}
10053 
10054   ins_pipe(ialu_reg_reg_shift);
10055 %}
10056 
10057 instruct addP_reg_reg_ext_shift(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2, immIScale scale) %{
10058   match(Set dst (AddP src1 (LShiftL (ConvI2L src2) scale)));
10059 
10060   ins_cost(1.9 * INSN_COST);
10061   format %{ "add $dst, $src1, $src2, I2L $scale\t# ptr" %}
10062 
10063   ins_encode %{
10064     __ lea(as_Register($dst$$reg),
10065            Address(as_Register($src1$$reg), as_Register($src2$$reg),
10066                    Address::sxtw($scale$$constant)));
10067   %}
10068 
10069   ins_pipe(ialu_reg_reg_shift);
10070 %}
10071 
10072 instruct lshift_ext(iRegLNoSp dst, iRegIorL2I src, immI scale, rFlagsReg cr) %{
10073   match(Set dst (LShiftL (ConvI2L src) scale));
10074 
10075   ins_cost(INSN_COST);
10076   format %{ "sbfiz $dst, $src, $scale & 63, -$scale & 63\t" %}
10077 
10078   ins_encode %{
10079     __ sbfiz(as_Register($dst$$reg),
10080           as_Register($src$$reg),
10081           $scale$$constant & 63, MIN2(32, (int)((-$scale$$constant) & 63)));
10082   %}
10083 
10084   ins_pipe(ialu_reg_shift);
10085 %}
10086 
10087 // Pointer Immediate Addition
10088 // n.b. this needs to be more expensive than using an indirect memory
10089 // operand
10090 instruct addP_reg_imm(iRegPNoSp dst, iRegPorL2P src1, immLAddSub src2) %{
10091   match(Set dst (AddP src1 src2));
10092 
10093   ins_cost(INSN_COST);
10094   format %{ "add $dst, $src1, $src2\t# ptr" %}
10095 
10096   // use opcode to indicate that this is an add not a sub
10097   opcode(0x0);
10098 
10099   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10100 
10101   ins_pipe(ialu_reg_imm);
10102 %}
10103 
10104 // Long Addition
10105 instruct addL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10106 
10107   match(Set dst (AddL src1 src2));
10108 
10109   ins_cost(INSN_COST);
10110   format %{ "add  $dst, $src1, $src2" %}
10111 
10112   ins_encode %{
10113     __ add(as_Register($dst$$reg),
10114            as_Register($src1$$reg),
10115            as_Register($src2$$reg));
10116   %}
10117 
10118   ins_pipe(ialu_reg_reg);
10119 %}
10120 
10121 // No constant pool entries requiredLong Immediate Addition.
10122 instruct addL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10123   match(Set dst (AddL src1 src2));
10124 
10125   ins_cost(INSN_COST);
10126   format %{ "add $dst, $src1, $src2" %}
10127 
10128   // use opcode to indicate that this is an add not a sub
10129   opcode(0x0);
10130 
10131   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10132 
10133   ins_pipe(ialu_reg_imm);
10134 %}
10135 
10136 // Integer Subtraction
10137 instruct subI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10138   match(Set dst (SubI src1 src2));
10139 
10140   ins_cost(INSN_COST);
10141   format %{ "subw  $dst, $src1, $src2" %}
10142 
10143   ins_encode %{
10144     __ subw(as_Register($dst$$reg),
10145             as_Register($src1$$reg),
10146             as_Register($src2$$reg));
10147   %}
10148 
10149   ins_pipe(ialu_reg_reg);
10150 %}
10151 
10152 // Immediate Subtraction
10153 instruct subI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10154   match(Set dst (SubI src1 src2));
10155 
10156   ins_cost(INSN_COST);
10157   format %{ "subw $dst, $src1, $src2" %}
10158 
10159   // use opcode to indicate that this is a sub not an add
10160   opcode(0x1);
10161 
10162   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10163 
10164   ins_pipe(ialu_reg_imm);
10165 %}
10166 
10167 // Long Subtraction
10168 instruct subL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10169 
10170   match(Set dst (SubL src1 src2));
10171 
10172   ins_cost(INSN_COST);
10173   format %{ "sub  $dst, $src1, $src2" %}
10174 
10175   ins_encode %{
10176     __ sub(as_Register($dst$$reg),
10177            as_Register($src1$$reg),
10178            as_Register($src2$$reg));
10179   %}
10180 
10181   ins_pipe(ialu_reg_reg);
10182 %}
10183 
10184 // No constant pool entries requiredLong Immediate Subtraction.
10185 instruct subL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10186   match(Set dst (SubL src1 src2));
10187 
10188   ins_cost(INSN_COST);
10189   format %{ "sub$dst, $src1, $src2" %}
10190 
10191   // use opcode to indicate that this is a sub not an add
10192   opcode(0x1);
10193 
10194   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10195 
10196   ins_pipe(ialu_reg_imm);
10197 %}
10198 
10199 // Integer Negation (special case for sub)
10200 
10201 instruct negI_reg(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr) %{
10202   match(Set dst (SubI zero src));
10203 
10204   ins_cost(INSN_COST);
10205   format %{ "negw $dst, $src\t# int" %}
10206 
10207   ins_encode %{
10208     __ negw(as_Register($dst$$reg),
10209             as_Register($src$$reg));
10210   %}
10211 
10212   ins_pipe(ialu_reg);
10213 %}
10214 
10215 // Long Negation
10216 
10217 instruct negL_reg(iRegLNoSp dst, iRegL src, immL0 zero, rFlagsReg cr) %{
10218   match(Set dst (SubL zero src));
10219 
10220   ins_cost(INSN_COST);
10221   format %{ "neg $dst, $src\t# long" %}
10222 
10223   ins_encode %{
10224     __ neg(as_Register($dst$$reg),
10225            as_Register($src$$reg));
10226   %}
10227 
10228   ins_pipe(ialu_reg);
10229 %}
10230 
10231 // Integer Multiply
10232 
10233 instruct mulI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10234   match(Set dst (MulI src1 src2));
10235 
10236   ins_cost(INSN_COST * 3);
10237   format %{ "mulw  $dst, $src1, $src2" %}
10238 
10239   ins_encode %{
10240     __ mulw(as_Register($dst$$reg),
10241             as_Register($src1$$reg),
10242             as_Register($src2$$reg));
10243   %}
10244 
10245   ins_pipe(imul_reg_reg);
10246 %}
10247 
10248 instruct smulI(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10249   match(Set dst (MulL (ConvI2L src1) (ConvI2L src2)));
10250 
10251   ins_cost(INSN_COST * 3);
10252   format %{ "smull  $dst, $src1, $src2" %}
10253 
10254   ins_encode %{
10255     __ smull(as_Register($dst$$reg),
10256              as_Register($src1$$reg),
10257              as_Register($src2$$reg));
10258   %}
10259 
10260   ins_pipe(imul_reg_reg);
10261 %}
10262 
10263 // Long Multiply
10264 
10265 instruct mulL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10266   match(Set dst (MulL src1 src2));
10267 
10268   ins_cost(INSN_COST * 5);
10269   format %{ "mul  $dst, $src1, $src2" %}
10270 
10271   ins_encode %{
10272     __ mul(as_Register($dst$$reg),
10273            as_Register($src1$$reg),
10274            as_Register($src2$$reg));
10275   %}
10276 
10277   ins_pipe(lmul_reg_reg);
10278 %}
10279 
10280 instruct mulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10281 %{
10282   match(Set dst (MulHiL src1 src2));
10283 
10284   ins_cost(INSN_COST * 7);
10285   format %{ "smulh   $dst, $src1, $src2\t# mulhi" %}
10286 
10287   ins_encode %{
10288     __ smulh(as_Register($dst$$reg),
10289              as_Register($src1$$reg),
10290              as_Register($src2$$reg));
10291   %}
10292 
10293   ins_pipe(lmul_reg_reg);
10294 %}
10295 
10296 instruct umulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10297 %{
10298   match(Set dst (UMulHiL src1 src2));
10299 
10300   ins_cost(INSN_COST * 7);
10301   format %{ "umulh   $dst, $src1, $src2\t# umulhi" %}
10302 
10303   ins_encode %{
10304     __ umulh(as_Register($dst$$reg),
10305              as_Register($src1$$reg),
10306              as_Register($src2$$reg));
10307   %}
10308 
10309   ins_pipe(lmul_reg_reg);
10310 %}
10311 
10312 // Combined Integer Multiply & Add/Sub
10313 
10314 instruct maddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10315   match(Set dst (AddI src3 (MulI src1 src2)));
10316 
10317   ins_cost(INSN_COST * 3);
10318   format %{ "madd  $dst, $src1, $src2, $src3" %}
10319 
10320   ins_encode %{
10321     __ maddw(as_Register($dst$$reg),
10322              as_Register($src1$$reg),
10323              as_Register($src2$$reg),
10324              as_Register($src3$$reg));
10325   %}
10326 
10327   ins_pipe(imac_reg_reg);
10328 %}
10329 
10330 instruct msubI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10331   match(Set dst (SubI src3 (MulI src1 src2)));
10332 
10333   ins_cost(INSN_COST * 3);
10334   format %{ "msub  $dst, $src1, $src2, $src3" %}
10335 
10336   ins_encode %{
10337     __ msubw(as_Register($dst$$reg),
10338              as_Register($src1$$reg),
10339              as_Register($src2$$reg),
10340              as_Register($src3$$reg));
10341   %}
10342 
10343   ins_pipe(imac_reg_reg);
10344 %}
10345 
10346 // Combined Integer Multiply & Neg
10347 
10348 instruct mnegI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI0 zero) %{
10349   match(Set dst (MulI (SubI zero src1) src2));
10350 
10351   ins_cost(INSN_COST * 3);
10352   format %{ "mneg  $dst, $src1, $src2" %}
10353 
10354   ins_encode %{
10355     __ mnegw(as_Register($dst$$reg),
10356              as_Register($src1$$reg),
10357              as_Register($src2$$reg));
10358   %}
10359 
10360   ins_pipe(imac_reg_reg);
10361 %}
10362 
10363 // Combined Long Multiply & Add/Sub
10364 
10365 instruct maddL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10366   match(Set dst (AddL src3 (MulL src1 src2)));
10367 
10368   ins_cost(INSN_COST * 5);
10369   format %{ "madd  $dst, $src1, $src2, $src3" %}
10370 
10371   ins_encode %{
10372     __ madd(as_Register($dst$$reg),
10373             as_Register($src1$$reg),
10374             as_Register($src2$$reg),
10375             as_Register($src3$$reg));
10376   %}
10377 
10378   ins_pipe(lmac_reg_reg);
10379 %}
10380 
10381 instruct msubL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10382   match(Set dst (SubL src3 (MulL src1 src2)));
10383 
10384   ins_cost(INSN_COST * 5);
10385   format %{ "msub  $dst, $src1, $src2, $src3" %}
10386 
10387   ins_encode %{
10388     __ msub(as_Register($dst$$reg),
10389             as_Register($src1$$reg),
10390             as_Register($src2$$reg),
10391             as_Register($src3$$reg));
10392   %}
10393 
10394   ins_pipe(lmac_reg_reg);
10395 %}
10396 
10397 // Combined Long Multiply & Neg
10398 
10399 instruct mnegL(iRegLNoSp dst, iRegL src1, iRegL src2, immL0 zero) %{
10400   match(Set dst (MulL (SubL zero src1) src2));
10401 
10402   ins_cost(INSN_COST * 5);
10403   format %{ "mneg  $dst, $src1, $src2" %}
10404 
10405   ins_encode %{
10406     __ mneg(as_Register($dst$$reg),
10407             as_Register($src1$$reg),
10408             as_Register($src2$$reg));
10409   %}
10410 
10411   ins_pipe(lmac_reg_reg);
10412 %}
10413 
10414 // Combine Integer Signed Multiply & Add/Sub/Neg Long
10415 
10416 instruct smaddL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10417   match(Set dst (AddL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10418 
10419   ins_cost(INSN_COST * 3);
10420   format %{ "smaddl  $dst, $src1, $src2, $src3" %}
10421 
10422   ins_encode %{
10423     __ smaddl(as_Register($dst$$reg),
10424               as_Register($src1$$reg),
10425               as_Register($src2$$reg),
10426               as_Register($src3$$reg));
10427   %}
10428 
10429   ins_pipe(imac_reg_reg);
10430 %}
10431 
10432 instruct smsubL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10433   match(Set dst (SubL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10434 
10435   ins_cost(INSN_COST * 3);
10436   format %{ "smsubl  $dst, $src1, $src2, $src3" %}
10437 
10438   ins_encode %{
10439     __ smsubl(as_Register($dst$$reg),
10440               as_Register($src1$$reg),
10441               as_Register($src2$$reg),
10442               as_Register($src3$$reg));
10443   %}
10444 
10445   ins_pipe(imac_reg_reg);
10446 %}
10447 
10448 instruct smnegL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, immL0 zero) %{
10449   match(Set dst (MulL (SubL zero (ConvI2L src1)) (ConvI2L src2)));
10450 
10451   ins_cost(INSN_COST * 3);
10452   format %{ "smnegl  $dst, $src1, $src2" %}
10453 
10454   ins_encode %{
10455     __ smnegl(as_Register($dst$$reg),
10456               as_Register($src1$$reg),
10457               as_Register($src2$$reg));
10458   %}
10459 
10460   ins_pipe(imac_reg_reg);
10461 %}
10462 
10463 // Combined Multiply-Add Shorts into Integer (dst = src1 * src2 + src3 * src4)
10464 
10465 instruct muladdS2I(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3, iRegIorL2I src4) %{
10466   match(Set dst (MulAddS2I (Binary src1 src2) (Binary src3 src4)));
10467 
10468   ins_cost(INSN_COST * 5);
10469   format %{ "mulw  rscratch1, $src1, $src2\n\t"
10470             "maddw $dst, $src3, $src4, rscratch1" %}
10471 
10472   ins_encode %{
10473     __ mulw(rscratch1, as_Register($src1$$reg), as_Register($src2$$reg));
10474     __ maddw(as_Register($dst$$reg), as_Register($src3$$reg), as_Register($src4$$reg), rscratch1); %}
10475 
10476   ins_pipe(imac_reg_reg);
10477 %}
10478 
10479 // Integer Divide
10480 
10481 instruct divI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10482   match(Set dst (DivI src1 src2));
10483 
10484   ins_cost(INSN_COST * 19);
10485   format %{ "sdivw  $dst, $src1, $src2" %}
10486 
10487   ins_encode(aarch64_enc_divw(dst, src1, src2));
10488   ins_pipe(idiv_reg_reg);
10489 %}
10490 
10491 // Long Divide
10492 
10493 instruct divL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10494   match(Set dst (DivL src1 src2));
10495 
10496   ins_cost(INSN_COST * 35);
10497   format %{ "sdiv   $dst, $src1, $src2" %}
10498 
10499   ins_encode(aarch64_enc_div(dst, src1, src2));
10500   ins_pipe(ldiv_reg_reg);
10501 %}
10502 
10503 // Integer Remainder
10504 
10505 instruct modI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10506   match(Set dst (ModI src1 src2));
10507 
10508   ins_cost(INSN_COST * 22);
10509   format %{ "sdivw  rscratch1, $src1, $src2\n\t"
10510             "msubw  $dst, rscratch1, $src2, $src1" %}
10511 
10512   ins_encode(aarch64_enc_modw(dst, src1, src2));
10513   ins_pipe(idiv_reg_reg);
10514 %}
10515 
10516 // Long Remainder
10517 
10518 instruct modL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10519   match(Set dst (ModL src1 src2));
10520 
10521   ins_cost(INSN_COST * 38);
10522   format %{ "sdiv   rscratch1, $src1, $src2\n"
10523             "msub   $dst, rscratch1, $src2, $src1" %}
10524 
10525   ins_encode(aarch64_enc_mod(dst, src1, src2));
10526   ins_pipe(ldiv_reg_reg);
10527 %}
10528 
10529 // Unsigned Integer Divide
10530 
10531 instruct UdivI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10532   match(Set dst (UDivI src1 src2));
10533 
10534   ins_cost(INSN_COST * 19);
10535   format %{ "udivw  $dst, $src1, $src2" %}
10536 
10537   ins_encode %{
10538     __ udivw($dst$$Register, $src1$$Register, $src2$$Register);
10539   %}
10540 
10541   ins_pipe(idiv_reg_reg);
10542 %}
10543 
10544 //  Unsigned Long Divide
10545 
10546 instruct UdivL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10547   match(Set dst (UDivL src1 src2));
10548 
10549   ins_cost(INSN_COST * 35);
10550   format %{ "udiv   $dst, $src1, $src2" %}
10551 
10552   ins_encode %{
10553     __ udiv($dst$$Register, $src1$$Register, $src2$$Register);
10554   %}
10555 
10556   ins_pipe(ldiv_reg_reg);
10557 %}
10558 
10559 // Unsigned Integer Remainder
10560 
10561 instruct UmodI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10562   match(Set dst (UModI src1 src2));
10563 
10564   ins_cost(INSN_COST * 22);
10565   format %{ "udivw  rscratch1, $src1, $src2\n\t"
10566             "msubw  $dst, rscratch1, $src2, $src1" %}
10567 
10568   ins_encode %{
10569     __ udivw(rscratch1, $src1$$Register, $src2$$Register);
10570     __ msubw($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
10571   %}
10572 
10573   ins_pipe(idiv_reg_reg);
10574 %}
10575 
10576 // Unsigned Long Remainder
10577 
10578 instruct UModL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10579   match(Set dst (UModL src1 src2));
10580 
10581   ins_cost(INSN_COST * 38);
10582   format %{ "udiv   rscratch1, $src1, $src2\n"
10583             "msub   $dst, rscratch1, $src2, $src1" %}
10584 
10585   ins_encode %{
10586     __ udiv(rscratch1, $src1$$Register, $src2$$Register);
10587     __ msub($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
10588   %}
10589 
10590   ins_pipe(ldiv_reg_reg);
10591 %}
10592 
10593 // Integer Shifts
10594 
10595 // Shift Left Register
10596 instruct lShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10597   match(Set dst (LShiftI src1 src2));
10598 
10599   ins_cost(INSN_COST * 2);
10600   format %{ "lslvw  $dst, $src1, $src2" %}
10601 
10602   ins_encode %{
10603     __ lslvw(as_Register($dst$$reg),
10604              as_Register($src1$$reg),
10605              as_Register($src2$$reg));
10606   %}
10607 
10608   ins_pipe(ialu_reg_reg_vshift);
10609 %}
10610 
10611 // Shift Left Immediate
10612 instruct lShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10613   match(Set dst (LShiftI src1 src2));
10614 
10615   ins_cost(INSN_COST);
10616   format %{ "lslw $dst, $src1, ($src2 & 0x1f)" %}
10617 
10618   ins_encode %{
10619     __ lslw(as_Register($dst$$reg),
10620             as_Register($src1$$reg),
10621             $src2$$constant & 0x1f);
10622   %}
10623 
10624   ins_pipe(ialu_reg_shift);
10625 %}
10626 
10627 // Shift Right Logical Register
10628 instruct urShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10629   match(Set dst (URShiftI src1 src2));
10630 
10631   ins_cost(INSN_COST * 2);
10632   format %{ "lsrvw  $dst, $src1, $src2" %}
10633 
10634   ins_encode %{
10635     __ lsrvw(as_Register($dst$$reg),
10636              as_Register($src1$$reg),
10637              as_Register($src2$$reg));
10638   %}
10639 
10640   ins_pipe(ialu_reg_reg_vshift);
10641 %}
10642 
10643 // Shift Right Logical Immediate
10644 instruct urShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10645   match(Set dst (URShiftI src1 src2));
10646 
10647   ins_cost(INSN_COST);
10648   format %{ "lsrw $dst, $src1, ($src2 & 0x1f)" %}
10649 
10650   ins_encode %{
10651     __ lsrw(as_Register($dst$$reg),
10652             as_Register($src1$$reg),
10653             $src2$$constant & 0x1f);
10654   %}
10655 
10656   ins_pipe(ialu_reg_shift);
10657 %}
10658 
10659 // Shift Right Arithmetic Register
10660 instruct rShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10661   match(Set dst (RShiftI src1 src2));
10662 
10663   ins_cost(INSN_COST * 2);
10664   format %{ "asrvw  $dst, $src1, $src2" %}
10665 
10666   ins_encode %{
10667     __ asrvw(as_Register($dst$$reg),
10668              as_Register($src1$$reg),
10669              as_Register($src2$$reg));
10670   %}
10671 
10672   ins_pipe(ialu_reg_reg_vshift);
10673 %}
10674 
10675 // Shift Right Arithmetic Immediate
10676 instruct rShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10677   match(Set dst (RShiftI src1 src2));
10678 
10679   ins_cost(INSN_COST);
10680   format %{ "asrw $dst, $src1, ($src2 & 0x1f)" %}
10681 
10682   ins_encode %{
10683     __ asrw(as_Register($dst$$reg),
10684             as_Register($src1$$reg),
10685             $src2$$constant & 0x1f);
10686   %}
10687 
10688   ins_pipe(ialu_reg_shift);
10689 %}
10690 
10691 // Combined Int Mask and Right Shift (using UBFM)
10692 // TODO
10693 
10694 // Long Shifts
10695 
10696 // Shift Left Register
10697 instruct lShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10698   match(Set dst (LShiftL src1 src2));
10699 
10700   ins_cost(INSN_COST * 2);
10701   format %{ "lslv  $dst, $src1, $src2" %}
10702 
10703   ins_encode %{
10704     __ lslv(as_Register($dst$$reg),
10705             as_Register($src1$$reg),
10706             as_Register($src2$$reg));
10707   %}
10708 
10709   ins_pipe(ialu_reg_reg_vshift);
10710 %}
10711 
10712 // Shift Left Immediate
10713 instruct lShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10714   match(Set dst (LShiftL src1 src2));
10715 
10716   ins_cost(INSN_COST);
10717   format %{ "lsl $dst, $src1, ($src2 & 0x3f)" %}
10718 
10719   ins_encode %{
10720     __ lsl(as_Register($dst$$reg),
10721             as_Register($src1$$reg),
10722             $src2$$constant & 0x3f);
10723   %}
10724 
10725   ins_pipe(ialu_reg_shift);
10726 %}
10727 
10728 // Shift Right Logical Register
10729 instruct urShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10730   match(Set dst (URShiftL src1 src2));
10731 
10732   ins_cost(INSN_COST * 2);
10733   format %{ "lsrv  $dst, $src1, $src2" %}
10734 
10735   ins_encode %{
10736     __ lsrv(as_Register($dst$$reg),
10737             as_Register($src1$$reg),
10738             as_Register($src2$$reg));
10739   %}
10740 
10741   ins_pipe(ialu_reg_reg_vshift);
10742 %}
10743 
10744 // Shift Right Logical Immediate
10745 instruct urShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10746   match(Set dst (URShiftL src1 src2));
10747 
10748   ins_cost(INSN_COST);
10749   format %{ "lsr $dst, $src1, ($src2 & 0x3f)" %}
10750 
10751   ins_encode %{
10752     __ lsr(as_Register($dst$$reg),
10753            as_Register($src1$$reg),
10754            $src2$$constant & 0x3f);
10755   %}
10756 
10757   ins_pipe(ialu_reg_shift);
10758 %}
10759 
10760 // A special-case pattern for card table stores.
10761 instruct urShiftP_reg_imm(iRegLNoSp dst, iRegP src1, immI src2) %{
10762   match(Set dst (URShiftL (CastP2X src1) src2));
10763 
10764   ins_cost(INSN_COST);
10765   format %{ "lsr $dst, p2x($src1), ($src2 & 0x3f)" %}
10766 
10767   ins_encode %{
10768     __ lsr(as_Register($dst$$reg),
10769            as_Register($src1$$reg),
10770            $src2$$constant & 0x3f);
10771   %}
10772 
10773   ins_pipe(ialu_reg_shift);
10774 %}
10775 
10776 // Shift Right Arithmetic Register
10777 instruct rShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10778   match(Set dst (RShiftL src1 src2));
10779 
10780   ins_cost(INSN_COST * 2);
10781   format %{ "asrv  $dst, $src1, $src2" %}
10782 
10783   ins_encode %{
10784     __ asrv(as_Register($dst$$reg),
10785             as_Register($src1$$reg),
10786             as_Register($src2$$reg));
10787   %}
10788 
10789   ins_pipe(ialu_reg_reg_vshift);
10790 %}
10791 
10792 // Shift Right Arithmetic Immediate
10793 instruct rShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10794   match(Set dst (RShiftL src1 src2));
10795 
10796   ins_cost(INSN_COST);
10797   format %{ "asr $dst, $src1, ($src2 & 0x3f)" %}
10798 
10799   ins_encode %{
10800     __ asr(as_Register($dst$$reg),
10801            as_Register($src1$$reg),
10802            $src2$$constant & 0x3f);
10803   %}
10804 
10805   ins_pipe(ialu_reg_shift);
10806 %}
10807 
10808 // BEGIN This section of the file is automatically generated. Do not edit --------------
10809 // This section is generated from aarch64_ad.m4
10810 
10811 // This pattern is automatically generated from aarch64_ad.m4
10812 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10813 instruct regL_not_reg(iRegLNoSp dst,
10814                          iRegL src1, immL_M1 m1,
10815                          rFlagsReg cr) %{
10816   match(Set dst (XorL src1 m1));
10817   ins_cost(INSN_COST);
10818   format %{ "eon  $dst, $src1, zr" %}
10819 
10820   ins_encode %{
10821     __ eon(as_Register($dst$$reg),
10822               as_Register($src1$$reg),
10823               zr,
10824               Assembler::LSL, 0);
10825   %}
10826 
10827   ins_pipe(ialu_reg);
10828 %}
10829 
10830 // This pattern is automatically generated from aarch64_ad.m4
10831 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10832 instruct regI_not_reg(iRegINoSp dst,
10833                          iRegIorL2I src1, immI_M1 m1,
10834                          rFlagsReg cr) %{
10835   match(Set dst (XorI src1 m1));
10836   ins_cost(INSN_COST);
10837   format %{ "eonw  $dst, $src1, zr" %}
10838 
10839   ins_encode %{
10840     __ eonw(as_Register($dst$$reg),
10841               as_Register($src1$$reg),
10842               zr,
10843               Assembler::LSL, 0);
10844   %}
10845 
10846   ins_pipe(ialu_reg);
10847 %}
10848 
10849 // This pattern is automatically generated from aarch64_ad.m4
10850 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10851 instruct NegI_reg_URShift_reg(iRegINoSp dst,
10852                               immI0 zero, iRegIorL2I src1, immI src2) %{
10853   match(Set dst (SubI zero (URShiftI src1 src2)));
10854 
10855   ins_cost(1.9 * INSN_COST);
10856   format %{ "negw  $dst, $src1, LSR $src2" %}
10857 
10858   ins_encode %{
10859     __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
10860             Assembler::LSR, $src2$$constant & 0x1f);
10861   %}
10862 
10863   ins_pipe(ialu_reg_shift);
10864 %}
10865 
10866 // This pattern is automatically generated from aarch64_ad.m4
10867 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10868 instruct NegI_reg_RShift_reg(iRegINoSp dst,
10869                               immI0 zero, iRegIorL2I src1, immI src2) %{
10870   match(Set dst (SubI zero (RShiftI src1 src2)));
10871 
10872   ins_cost(1.9 * INSN_COST);
10873   format %{ "negw  $dst, $src1, ASR $src2" %}
10874 
10875   ins_encode %{
10876     __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
10877             Assembler::ASR, $src2$$constant & 0x1f);
10878   %}
10879 
10880   ins_pipe(ialu_reg_shift);
10881 %}
10882 
10883 // This pattern is automatically generated from aarch64_ad.m4
10884 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10885 instruct NegI_reg_LShift_reg(iRegINoSp dst,
10886                               immI0 zero, iRegIorL2I src1, immI src2) %{
10887   match(Set dst (SubI zero (LShiftI src1 src2)));
10888 
10889   ins_cost(1.9 * INSN_COST);
10890   format %{ "negw  $dst, $src1, LSL $src2" %}
10891 
10892   ins_encode %{
10893     __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
10894             Assembler::LSL, $src2$$constant & 0x1f);
10895   %}
10896 
10897   ins_pipe(ialu_reg_shift);
10898 %}
10899 
10900 // This pattern is automatically generated from aarch64_ad.m4
10901 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10902 instruct NegL_reg_URShift_reg(iRegLNoSp dst,
10903                               immL0 zero, iRegL src1, immI src2) %{
10904   match(Set dst (SubL zero (URShiftL src1 src2)));
10905 
10906   ins_cost(1.9 * INSN_COST);
10907   format %{ "neg  $dst, $src1, LSR $src2" %}
10908 
10909   ins_encode %{
10910     __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10911             Assembler::LSR, $src2$$constant & 0x3f);
10912   %}
10913 
10914   ins_pipe(ialu_reg_shift);
10915 %}
10916 
10917 // This pattern is automatically generated from aarch64_ad.m4
10918 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10919 instruct NegL_reg_RShift_reg(iRegLNoSp dst,
10920                               immL0 zero, iRegL src1, immI src2) %{
10921   match(Set dst (SubL zero (RShiftL src1 src2)));
10922 
10923   ins_cost(1.9 * INSN_COST);
10924   format %{ "neg  $dst, $src1, ASR $src2" %}
10925 
10926   ins_encode %{
10927     __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10928             Assembler::ASR, $src2$$constant & 0x3f);
10929   %}
10930 
10931   ins_pipe(ialu_reg_shift);
10932 %}
10933 
10934 // This pattern is automatically generated from aarch64_ad.m4
10935 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10936 instruct NegL_reg_LShift_reg(iRegLNoSp dst,
10937                               immL0 zero, iRegL src1, immI src2) %{
10938   match(Set dst (SubL zero (LShiftL src1 src2)));
10939 
10940   ins_cost(1.9 * INSN_COST);
10941   format %{ "neg  $dst, $src1, LSL $src2" %}
10942 
10943   ins_encode %{
10944     __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10945             Assembler::LSL, $src2$$constant & 0x3f);
10946   %}
10947 
10948   ins_pipe(ialu_reg_shift);
10949 %}
10950 
10951 // This pattern is automatically generated from aarch64_ad.m4
10952 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10953 instruct AndI_reg_not_reg(iRegINoSp dst,
10954                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10955   match(Set dst (AndI src1 (XorI src2 m1)));
10956   ins_cost(INSN_COST);
10957   format %{ "bicw  $dst, $src1, $src2" %}
10958 
10959   ins_encode %{
10960     __ bicw(as_Register($dst$$reg),
10961               as_Register($src1$$reg),
10962               as_Register($src2$$reg),
10963               Assembler::LSL, 0);
10964   %}
10965 
10966   ins_pipe(ialu_reg_reg);
10967 %}
10968 
10969 // This pattern is automatically generated from aarch64_ad.m4
10970 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10971 instruct AndL_reg_not_reg(iRegLNoSp dst,
10972                          iRegL src1, iRegL src2, immL_M1 m1) %{
10973   match(Set dst (AndL src1 (XorL src2 m1)));
10974   ins_cost(INSN_COST);
10975   format %{ "bic  $dst, $src1, $src2" %}
10976 
10977   ins_encode %{
10978     __ bic(as_Register($dst$$reg),
10979               as_Register($src1$$reg),
10980               as_Register($src2$$reg),
10981               Assembler::LSL, 0);
10982   %}
10983 
10984   ins_pipe(ialu_reg_reg);
10985 %}
10986 
10987 // This pattern is automatically generated from aarch64_ad.m4
10988 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10989 instruct OrI_reg_not_reg(iRegINoSp dst,
10990                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10991   match(Set dst (OrI src1 (XorI src2 m1)));
10992   ins_cost(INSN_COST);
10993   format %{ "ornw  $dst, $src1, $src2" %}
10994 
10995   ins_encode %{
10996     __ ornw(as_Register($dst$$reg),
10997               as_Register($src1$$reg),
10998               as_Register($src2$$reg),
10999               Assembler::LSL, 0);
11000   %}
11001 
11002   ins_pipe(ialu_reg_reg);
11003 %}
11004 
11005 // This pattern is automatically generated from aarch64_ad.m4
11006 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11007 instruct OrL_reg_not_reg(iRegLNoSp dst,
11008                          iRegL src1, iRegL src2, immL_M1 m1) %{
11009   match(Set dst (OrL src1 (XorL src2 m1)));
11010   ins_cost(INSN_COST);
11011   format %{ "orn  $dst, $src1, $src2" %}
11012 
11013   ins_encode %{
11014     __ orn(as_Register($dst$$reg),
11015               as_Register($src1$$reg),
11016               as_Register($src2$$reg),
11017               Assembler::LSL, 0);
11018   %}
11019 
11020   ins_pipe(ialu_reg_reg);
11021 %}
11022 
11023 // This pattern is automatically generated from aarch64_ad.m4
11024 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11025 instruct XorI_reg_not_reg(iRegINoSp dst,
11026                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11027   match(Set dst (XorI m1 (XorI src2 src1)));
11028   ins_cost(INSN_COST);
11029   format %{ "eonw  $dst, $src1, $src2" %}
11030 
11031   ins_encode %{
11032     __ eonw(as_Register($dst$$reg),
11033               as_Register($src1$$reg),
11034               as_Register($src2$$reg),
11035               Assembler::LSL, 0);
11036   %}
11037 
11038   ins_pipe(ialu_reg_reg);
11039 %}
11040 
11041 // This pattern is automatically generated from aarch64_ad.m4
11042 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11043 instruct XorL_reg_not_reg(iRegLNoSp dst,
11044                          iRegL src1, iRegL src2, immL_M1 m1) %{
11045   match(Set dst (XorL m1 (XorL src2 src1)));
11046   ins_cost(INSN_COST);
11047   format %{ "eon  $dst, $src1, $src2" %}
11048 
11049   ins_encode %{
11050     __ eon(as_Register($dst$$reg),
11051               as_Register($src1$$reg),
11052               as_Register($src2$$reg),
11053               Assembler::LSL, 0);
11054   %}
11055 
11056   ins_pipe(ialu_reg_reg);
11057 %}
11058 
11059 // This pattern is automatically generated from aarch64_ad.m4
11060 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11061 // val & (-1 ^ (val >>> shift)) ==> bicw
11062 instruct AndI_reg_URShift_not_reg(iRegINoSp dst,
11063                          iRegIorL2I src1, iRegIorL2I src2,
11064                          immI src3, immI_M1 src4) %{
11065   match(Set dst (AndI src1 (XorI(URShiftI src2 src3) src4)));
11066   ins_cost(1.9 * INSN_COST);
11067   format %{ "bicw  $dst, $src1, $src2, LSR $src3" %}
11068 
11069   ins_encode %{
11070     __ bicw(as_Register($dst$$reg),
11071               as_Register($src1$$reg),
11072               as_Register($src2$$reg),
11073               Assembler::LSR,
11074               $src3$$constant & 0x1f);
11075   %}
11076 
11077   ins_pipe(ialu_reg_reg_shift);
11078 %}
11079 
11080 // This pattern is automatically generated from aarch64_ad.m4
11081 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11082 // val & (-1 ^ (val >>> shift)) ==> bic
11083 instruct AndL_reg_URShift_not_reg(iRegLNoSp dst,
11084                          iRegL src1, iRegL src2,
11085                          immI src3, immL_M1 src4) %{
11086   match(Set dst (AndL src1 (XorL(URShiftL src2 src3) src4)));
11087   ins_cost(1.9 * INSN_COST);
11088   format %{ "bic  $dst, $src1, $src2, LSR $src3" %}
11089 
11090   ins_encode %{
11091     __ bic(as_Register($dst$$reg),
11092               as_Register($src1$$reg),
11093               as_Register($src2$$reg),
11094               Assembler::LSR,
11095               $src3$$constant & 0x3f);
11096   %}
11097 
11098   ins_pipe(ialu_reg_reg_shift);
11099 %}
11100 
11101 // This pattern is automatically generated from aarch64_ad.m4
11102 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11103 // val & (-1 ^ (val >> shift)) ==> bicw
11104 instruct AndI_reg_RShift_not_reg(iRegINoSp dst,
11105                          iRegIorL2I src1, iRegIorL2I src2,
11106                          immI src3, immI_M1 src4) %{
11107   match(Set dst (AndI src1 (XorI(RShiftI src2 src3) src4)));
11108   ins_cost(1.9 * INSN_COST);
11109   format %{ "bicw  $dst, $src1, $src2, ASR $src3" %}
11110 
11111   ins_encode %{
11112     __ bicw(as_Register($dst$$reg),
11113               as_Register($src1$$reg),
11114               as_Register($src2$$reg),
11115               Assembler::ASR,
11116               $src3$$constant & 0x1f);
11117   %}
11118 
11119   ins_pipe(ialu_reg_reg_shift);
11120 %}
11121 
11122 // This pattern is automatically generated from aarch64_ad.m4
11123 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11124 // val & (-1 ^ (val >> shift)) ==> bic
11125 instruct AndL_reg_RShift_not_reg(iRegLNoSp dst,
11126                          iRegL src1, iRegL src2,
11127                          immI src3, immL_M1 src4) %{
11128   match(Set dst (AndL src1 (XorL(RShiftL src2 src3) src4)));
11129   ins_cost(1.9 * INSN_COST);
11130   format %{ "bic  $dst, $src1, $src2, ASR $src3" %}
11131 
11132   ins_encode %{
11133     __ bic(as_Register($dst$$reg),
11134               as_Register($src1$$reg),
11135               as_Register($src2$$reg),
11136               Assembler::ASR,
11137               $src3$$constant & 0x3f);
11138   %}
11139 
11140   ins_pipe(ialu_reg_reg_shift);
11141 %}
11142 
11143 // This pattern is automatically generated from aarch64_ad.m4
11144 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11145 // val & (-1 ^ (val ror shift)) ==> bicw
11146 instruct AndI_reg_RotateRight_not_reg(iRegINoSp dst,
11147                          iRegIorL2I src1, iRegIorL2I src2,
11148                          immI src3, immI_M1 src4) %{
11149   match(Set dst (AndI src1 (XorI(RotateRight src2 src3) src4)));
11150   ins_cost(1.9 * INSN_COST);
11151   format %{ "bicw  $dst, $src1, $src2, ROR $src3" %}
11152 
11153   ins_encode %{
11154     __ bicw(as_Register($dst$$reg),
11155               as_Register($src1$$reg),
11156               as_Register($src2$$reg),
11157               Assembler::ROR,
11158               $src3$$constant & 0x1f);
11159   %}
11160 
11161   ins_pipe(ialu_reg_reg_shift);
11162 %}
11163 
11164 // This pattern is automatically generated from aarch64_ad.m4
11165 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11166 // val & (-1 ^ (val ror shift)) ==> bic
11167 instruct AndL_reg_RotateRight_not_reg(iRegLNoSp dst,
11168                          iRegL src1, iRegL src2,
11169                          immI src3, immL_M1 src4) %{
11170   match(Set dst (AndL src1 (XorL(RotateRight src2 src3) src4)));
11171   ins_cost(1.9 * INSN_COST);
11172   format %{ "bic  $dst, $src1, $src2, ROR $src3" %}
11173 
11174   ins_encode %{
11175     __ bic(as_Register($dst$$reg),
11176               as_Register($src1$$reg),
11177               as_Register($src2$$reg),
11178               Assembler::ROR,
11179               $src3$$constant & 0x3f);
11180   %}
11181 
11182   ins_pipe(ialu_reg_reg_shift);
11183 %}
11184 
11185 // This pattern is automatically generated from aarch64_ad.m4
11186 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11187 // val & (-1 ^ (val << shift)) ==> bicw
11188 instruct AndI_reg_LShift_not_reg(iRegINoSp dst,
11189                          iRegIorL2I src1, iRegIorL2I src2,
11190                          immI src3, immI_M1 src4) %{
11191   match(Set dst (AndI src1 (XorI(LShiftI src2 src3) src4)));
11192   ins_cost(1.9 * INSN_COST);
11193   format %{ "bicw  $dst, $src1, $src2, LSL $src3" %}
11194 
11195   ins_encode %{
11196     __ bicw(as_Register($dst$$reg),
11197               as_Register($src1$$reg),
11198               as_Register($src2$$reg),
11199               Assembler::LSL,
11200               $src3$$constant & 0x1f);
11201   %}
11202 
11203   ins_pipe(ialu_reg_reg_shift);
11204 %}
11205 
11206 // This pattern is automatically generated from aarch64_ad.m4
11207 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11208 // val & (-1 ^ (val << shift)) ==> bic
11209 instruct AndL_reg_LShift_not_reg(iRegLNoSp dst,
11210                          iRegL src1, iRegL src2,
11211                          immI src3, immL_M1 src4) %{
11212   match(Set dst (AndL src1 (XorL(LShiftL src2 src3) src4)));
11213   ins_cost(1.9 * INSN_COST);
11214   format %{ "bic  $dst, $src1, $src2, LSL $src3" %}
11215 
11216   ins_encode %{
11217     __ bic(as_Register($dst$$reg),
11218               as_Register($src1$$reg),
11219               as_Register($src2$$reg),
11220               Assembler::LSL,
11221               $src3$$constant & 0x3f);
11222   %}
11223 
11224   ins_pipe(ialu_reg_reg_shift);
11225 %}
11226 
11227 // This pattern is automatically generated from aarch64_ad.m4
11228 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11229 // val ^ (-1 ^ (val >>> shift)) ==> eonw
11230 instruct XorI_reg_URShift_not_reg(iRegINoSp dst,
11231                          iRegIorL2I src1, iRegIorL2I src2,
11232                          immI src3, immI_M1 src4) %{
11233   match(Set dst (XorI src4 (XorI(URShiftI src2 src3) src1)));
11234   ins_cost(1.9 * INSN_COST);
11235   format %{ "eonw  $dst, $src1, $src2, LSR $src3" %}
11236 
11237   ins_encode %{
11238     __ eonw(as_Register($dst$$reg),
11239               as_Register($src1$$reg),
11240               as_Register($src2$$reg),
11241               Assembler::LSR,
11242               $src3$$constant & 0x1f);
11243   %}
11244 
11245   ins_pipe(ialu_reg_reg_shift);
11246 %}
11247 
11248 // This pattern is automatically generated from aarch64_ad.m4
11249 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11250 // val ^ (-1 ^ (val >>> shift)) ==> eon
11251 instruct XorL_reg_URShift_not_reg(iRegLNoSp dst,
11252                          iRegL src1, iRegL src2,
11253                          immI src3, immL_M1 src4) %{
11254   match(Set dst (XorL src4 (XorL(URShiftL src2 src3) src1)));
11255   ins_cost(1.9 * INSN_COST);
11256   format %{ "eon  $dst, $src1, $src2, LSR $src3" %}
11257 
11258   ins_encode %{
11259     __ eon(as_Register($dst$$reg),
11260               as_Register($src1$$reg),
11261               as_Register($src2$$reg),
11262               Assembler::LSR,
11263               $src3$$constant & 0x3f);
11264   %}
11265 
11266   ins_pipe(ialu_reg_reg_shift);
11267 %}
11268 
11269 // This pattern is automatically generated from aarch64_ad.m4
11270 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11271 // val ^ (-1 ^ (val >> shift)) ==> eonw
11272 instruct XorI_reg_RShift_not_reg(iRegINoSp dst,
11273                          iRegIorL2I src1, iRegIorL2I src2,
11274                          immI src3, immI_M1 src4) %{
11275   match(Set dst (XorI src4 (XorI(RShiftI src2 src3) src1)));
11276   ins_cost(1.9 * INSN_COST);
11277   format %{ "eonw  $dst, $src1, $src2, ASR $src3" %}
11278 
11279   ins_encode %{
11280     __ eonw(as_Register($dst$$reg),
11281               as_Register($src1$$reg),
11282               as_Register($src2$$reg),
11283               Assembler::ASR,
11284               $src3$$constant & 0x1f);
11285   %}
11286 
11287   ins_pipe(ialu_reg_reg_shift);
11288 %}
11289 
11290 // This pattern is automatically generated from aarch64_ad.m4
11291 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11292 // val ^ (-1 ^ (val >> shift)) ==> eon
11293 instruct XorL_reg_RShift_not_reg(iRegLNoSp dst,
11294                          iRegL src1, iRegL src2,
11295                          immI src3, immL_M1 src4) %{
11296   match(Set dst (XorL src4 (XorL(RShiftL src2 src3) src1)));
11297   ins_cost(1.9 * INSN_COST);
11298   format %{ "eon  $dst, $src1, $src2, ASR $src3" %}
11299 
11300   ins_encode %{
11301     __ eon(as_Register($dst$$reg),
11302               as_Register($src1$$reg),
11303               as_Register($src2$$reg),
11304               Assembler::ASR,
11305               $src3$$constant & 0x3f);
11306   %}
11307 
11308   ins_pipe(ialu_reg_reg_shift);
11309 %}
11310 
11311 // This pattern is automatically generated from aarch64_ad.m4
11312 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11313 // val ^ (-1 ^ (val ror shift)) ==> eonw
11314 instruct XorI_reg_RotateRight_not_reg(iRegINoSp dst,
11315                          iRegIorL2I src1, iRegIorL2I src2,
11316                          immI src3, immI_M1 src4) %{
11317   match(Set dst (XorI src4 (XorI(RotateRight src2 src3) src1)));
11318   ins_cost(1.9 * INSN_COST);
11319   format %{ "eonw  $dst, $src1, $src2, ROR $src3" %}
11320 
11321   ins_encode %{
11322     __ eonw(as_Register($dst$$reg),
11323               as_Register($src1$$reg),
11324               as_Register($src2$$reg),
11325               Assembler::ROR,
11326               $src3$$constant & 0x1f);
11327   %}
11328 
11329   ins_pipe(ialu_reg_reg_shift);
11330 %}
11331 
11332 // This pattern is automatically generated from aarch64_ad.m4
11333 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11334 // val ^ (-1 ^ (val ror shift)) ==> eon
11335 instruct XorL_reg_RotateRight_not_reg(iRegLNoSp dst,
11336                          iRegL src1, iRegL src2,
11337                          immI src3, immL_M1 src4) %{
11338   match(Set dst (XorL src4 (XorL(RotateRight src2 src3) src1)));
11339   ins_cost(1.9 * INSN_COST);
11340   format %{ "eon  $dst, $src1, $src2, ROR $src3" %}
11341 
11342   ins_encode %{
11343     __ eon(as_Register($dst$$reg),
11344               as_Register($src1$$reg),
11345               as_Register($src2$$reg),
11346               Assembler::ROR,
11347               $src3$$constant & 0x3f);
11348   %}
11349 
11350   ins_pipe(ialu_reg_reg_shift);
11351 %}
11352 
11353 // This pattern is automatically generated from aarch64_ad.m4
11354 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11355 // val ^ (-1 ^ (val << shift)) ==> eonw
11356 instruct XorI_reg_LShift_not_reg(iRegINoSp dst,
11357                          iRegIorL2I src1, iRegIorL2I src2,
11358                          immI src3, immI_M1 src4) %{
11359   match(Set dst (XorI src4 (XorI(LShiftI src2 src3) src1)));
11360   ins_cost(1.9 * INSN_COST);
11361   format %{ "eonw  $dst, $src1, $src2, LSL $src3" %}
11362 
11363   ins_encode %{
11364     __ eonw(as_Register($dst$$reg),
11365               as_Register($src1$$reg),
11366               as_Register($src2$$reg),
11367               Assembler::LSL,
11368               $src3$$constant & 0x1f);
11369   %}
11370 
11371   ins_pipe(ialu_reg_reg_shift);
11372 %}
11373 
11374 // This pattern is automatically generated from aarch64_ad.m4
11375 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11376 // val ^ (-1 ^ (val << shift)) ==> eon
11377 instruct XorL_reg_LShift_not_reg(iRegLNoSp dst,
11378                          iRegL src1, iRegL src2,
11379                          immI src3, immL_M1 src4) %{
11380   match(Set dst (XorL src4 (XorL(LShiftL src2 src3) src1)));
11381   ins_cost(1.9 * INSN_COST);
11382   format %{ "eon  $dst, $src1, $src2, LSL $src3" %}
11383 
11384   ins_encode %{
11385     __ eon(as_Register($dst$$reg),
11386               as_Register($src1$$reg),
11387               as_Register($src2$$reg),
11388               Assembler::LSL,
11389               $src3$$constant & 0x3f);
11390   %}
11391 
11392   ins_pipe(ialu_reg_reg_shift);
11393 %}
11394 
11395 // This pattern is automatically generated from aarch64_ad.m4
11396 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11397 // val | (-1 ^ (val >>> shift)) ==> ornw
11398 instruct OrI_reg_URShift_not_reg(iRegINoSp dst,
11399                          iRegIorL2I src1, iRegIorL2I src2,
11400                          immI src3, immI_M1 src4) %{
11401   match(Set dst (OrI src1 (XorI(URShiftI src2 src3) src4)));
11402   ins_cost(1.9 * INSN_COST);
11403   format %{ "ornw  $dst, $src1, $src2, LSR $src3" %}
11404 
11405   ins_encode %{
11406     __ ornw(as_Register($dst$$reg),
11407               as_Register($src1$$reg),
11408               as_Register($src2$$reg),
11409               Assembler::LSR,
11410               $src3$$constant & 0x1f);
11411   %}
11412 
11413   ins_pipe(ialu_reg_reg_shift);
11414 %}
11415 
11416 // This pattern is automatically generated from aarch64_ad.m4
11417 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11418 // val | (-1 ^ (val >>> shift)) ==> orn
11419 instruct OrL_reg_URShift_not_reg(iRegLNoSp dst,
11420                          iRegL src1, iRegL src2,
11421                          immI src3, immL_M1 src4) %{
11422   match(Set dst (OrL src1 (XorL(URShiftL src2 src3) src4)));
11423   ins_cost(1.9 * INSN_COST);
11424   format %{ "orn  $dst, $src1, $src2, LSR $src3" %}
11425 
11426   ins_encode %{
11427     __ orn(as_Register($dst$$reg),
11428               as_Register($src1$$reg),
11429               as_Register($src2$$reg),
11430               Assembler::LSR,
11431               $src3$$constant & 0x3f);
11432   %}
11433 
11434   ins_pipe(ialu_reg_reg_shift);
11435 %}
11436 
11437 // This pattern is automatically generated from aarch64_ad.m4
11438 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11439 // val | (-1 ^ (val >> shift)) ==> ornw
11440 instruct OrI_reg_RShift_not_reg(iRegINoSp dst,
11441                          iRegIorL2I src1, iRegIorL2I src2,
11442                          immI src3, immI_M1 src4) %{
11443   match(Set dst (OrI src1 (XorI(RShiftI src2 src3) src4)));
11444   ins_cost(1.9 * INSN_COST);
11445   format %{ "ornw  $dst, $src1, $src2, ASR $src3" %}
11446 
11447   ins_encode %{
11448     __ ornw(as_Register($dst$$reg),
11449               as_Register($src1$$reg),
11450               as_Register($src2$$reg),
11451               Assembler::ASR,
11452               $src3$$constant & 0x1f);
11453   %}
11454 
11455   ins_pipe(ialu_reg_reg_shift);
11456 %}
11457 
11458 // This pattern is automatically generated from aarch64_ad.m4
11459 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11460 // val | (-1 ^ (val >> shift)) ==> orn
11461 instruct OrL_reg_RShift_not_reg(iRegLNoSp dst,
11462                          iRegL src1, iRegL src2,
11463                          immI src3, immL_M1 src4) %{
11464   match(Set dst (OrL src1 (XorL(RShiftL src2 src3) src4)));
11465   ins_cost(1.9 * INSN_COST);
11466   format %{ "orn  $dst, $src1, $src2, ASR $src3" %}
11467 
11468   ins_encode %{
11469     __ orn(as_Register($dst$$reg),
11470               as_Register($src1$$reg),
11471               as_Register($src2$$reg),
11472               Assembler::ASR,
11473               $src3$$constant & 0x3f);
11474   %}
11475 
11476   ins_pipe(ialu_reg_reg_shift);
11477 %}
11478 
11479 // This pattern is automatically generated from aarch64_ad.m4
11480 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11481 // val | (-1 ^ (val ror shift)) ==> ornw
11482 instruct OrI_reg_RotateRight_not_reg(iRegINoSp dst,
11483                          iRegIorL2I src1, iRegIorL2I src2,
11484                          immI src3, immI_M1 src4) %{
11485   match(Set dst (OrI src1 (XorI(RotateRight src2 src3) src4)));
11486   ins_cost(1.9 * INSN_COST);
11487   format %{ "ornw  $dst, $src1, $src2, ROR $src3" %}
11488 
11489   ins_encode %{
11490     __ ornw(as_Register($dst$$reg),
11491               as_Register($src1$$reg),
11492               as_Register($src2$$reg),
11493               Assembler::ROR,
11494               $src3$$constant & 0x1f);
11495   %}
11496 
11497   ins_pipe(ialu_reg_reg_shift);
11498 %}
11499 
11500 // This pattern is automatically generated from aarch64_ad.m4
11501 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11502 // val | (-1 ^ (val ror shift)) ==> orn
11503 instruct OrL_reg_RotateRight_not_reg(iRegLNoSp dst,
11504                          iRegL src1, iRegL src2,
11505                          immI src3, immL_M1 src4) %{
11506   match(Set dst (OrL src1 (XorL(RotateRight src2 src3) src4)));
11507   ins_cost(1.9 * INSN_COST);
11508   format %{ "orn  $dst, $src1, $src2, ROR $src3" %}
11509 
11510   ins_encode %{
11511     __ orn(as_Register($dst$$reg),
11512               as_Register($src1$$reg),
11513               as_Register($src2$$reg),
11514               Assembler::ROR,
11515               $src3$$constant & 0x3f);
11516   %}
11517 
11518   ins_pipe(ialu_reg_reg_shift);
11519 %}
11520 
11521 // This pattern is automatically generated from aarch64_ad.m4
11522 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11523 // val | (-1 ^ (val << shift)) ==> ornw
11524 instruct OrI_reg_LShift_not_reg(iRegINoSp dst,
11525                          iRegIorL2I src1, iRegIorL2I src2,
11526                          immI src3, immI_M1 src4) %{
11527   match(Set dst (OrI src1 (XorI(LShiftI src2 src3) src4)));
11528   ins_cost(1.9 * INSN_COST);
11529   format %{ "ornw  $dst, $src1, $src2, LSL $src3" %}
11530 
11531   ins_encode %{
11532     __ ornw(as_Register($dst$$reg),
11533               as_Register($src1$$reg),
11534               as_Register($src2$$reg),
11535               Assembler::LSL,
11536               $src3$$constant & 0x1f);
11537   %}
11538 
11539   ins_pipe(ialu_reg_reg_shift);
11540 %}
11541 
11542 // This pattern is automatically generated from aarch64_ad.m4
11543 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11544 // val | (-1 ^ (val << shift)) ==> orn
11545 instruct OrL_reg_LShift_not_reg(iRegLNoSp dst,
11546                          iRegL src1, iRegL src2,
11547                          immI src3, immL_M1 src4) %{
11548   match(Set dst (OrL src1 (XorL(LShiftL src2 src3) src4)));
11549   ins_cost(1.9 * INSN_COST);
11550   format %{ "orn  $dst, $src1, $src2, LSL $src3" %}
11551 
11552   ins_encode %{
11553     __ orn(as_Register($dst$$reg),
11554               as_Register($src1$$reg),
11555               as_Register($src2$$reg),
11556               Assembler::LSL,
11557               $src3$$constant & 0x3f);
11558   %}
11559 
11560   ins_pipe(ialu_reg_reg_shift);
11561 %}
11562 
11563 // This pattern is automatically generated from aarch64_ad.m4
11564 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11565 instruct AndI_reg_URShift_reg(iRegINoSp dst,
11566                          iRegIorL2I src1, iRegIorL2I src2,
11567                          immI src3) %{
11568   match(Set dst (AndI src1 (URShiftI src2 src3)));
11569 
11570   ins_cost(1.9 * INSN_COST);
11571   format %{ "andw  $dst, $src1, $src2, LSR $src3" %}
11572 
11573   ins_encode %{
11574     __ andw(as_Register($dst$$reg),
11575               as_Register($src1$$reg),
11576               as_Register($src2$$reg),
11577               Assembler::LSR,
11578               $src3$$constant & 0x1f);
11579   %}
11580 
11581   ins_pipe(ialu_reg_reg_shift);
11582 %}
11583 
11584 // This pattern is automatically generated from aarch64_ad.m4
11585 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11586 instruct AndL_reg_URShift_reg(iRegLNoSp dst,
11587                          iRegL src1, iRegL src2,
11588                          immI src3) %{
11589   match(Set dst (AndL src1 (URShiftL src2 src3)));
11590 
11591   ins_cost(1.9 * INSN_COST);
11592   format %{ "andr  $dst, $src1, $src2, LSR $src3" %}
11593 
11594   ins_encode %{
11595     __ andr(as_Register($dst$$reg),
11596               as_Register($src1$$reg),
11597               as_Register($src2$$reg),
11598               Assembler::LSR,
11599               $src3$$constant & 0x3f);
11600   %}
11601 
11602   ins_pipe(ialu_reg_reg_shift);
11603 %}
11604 
11605 // This pattern is automatically generated from aarch64_ad.m4
11606 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11607 instruct AndI_reg_RShift_reg(iRegINoSp dst,
11608                          iRegIorL2I src1, iRegIorL2I src2,
11609                          immI src3) %{
11610   match(Set dst (AndI src1 (RShiftI src2 src3)));
11611 
11612   ins_cost(1.9 * INSN_COST);
11613   format %{ "andw  $dst, $src1, $src2, ASR $src3" %}
11614 
11615   ins_encode %{
11616     __ andw(as_Register($dst$$reg),
11617               as_Register($src1$$reg),
11618               as_Register($src2$$reg),
11619               Assembler::ASR,
11620               $src3$$constant & 0x1f);
11621   %}
11622 
11623   ins_pipe(ialu_reg_reg_shift);
11624 %}
11625 
11626 // This pattern is automatically generated from aarch64_ad.m4
11627 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11628 instruct AndL_reg_RShift_reg(iRegLNoSp dst,
11629                          iRegL src1, iRegL src2,
11630                          immI src3) %{
11631   match(Set dst (AndL src1 (RShiftL src2 src3)));
11632 
11633   ins_cost(1.9 * INSN_COST);
11634   format %{ "andr  $dst, $src1, $src2, ASR $src3" %}
11635 
11636   ins_encode %{
11637     __ andr(as_Register($dst$$reg),
11638               as_Register($src1$$reg),
11639               as_Register($src2$$reg),
11640               Assembler::ASR,
11641               $src3$$constant & 0x3f);
11642   %}
11643 
11644   ins_pipe(ialu_reg_reg_shift);
11645 %}
11646 
11647 // This pattern is automatically generated from aarch64_ad.m4
11648 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11649 instruct AndI_reg_LShift_reg(iRegINoSp dst,
11650                          iRegIorL2I src1, iRegIorL2I src2,
11651                          immI src3) %{
11652   match(Set dst (AndI src1 (LShiftI src2 src3)));
11653 
11654   ins_cost(1.9 * INSN_COST);
11655   format %{ "andw  $dst, $src1, $src2, LSL $src3" %}
11656 
11657   ins_encode %{
11658     __ andw(as_Register($dst$$reg),
11659               as_Register($src1$$reg),
11660               as_Register($src2$$reg),
11661               Assembler::LSL,
11662               $src3$$constant & 0x1f);
11663   %}
11664 
11665   ins_pipe(ialu_reg_reg_shift);
11666 %}
11667 
11668 // This pattern is automatically generated from aarch64_ad.m4
11669 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11670 instruct AndL_reg_LShift_reg(iRegLNoSp dst,
11671                          iRegL src1, iRegL src2,
11672                          immI src3) %{
11673   match(Set dst (AndL src1 (LShiftL src2 src3)));
11674 
11675   ins_cost(1.9 * INSN_COST);
11676   format %{ "andr  $dst, $src1, $src2, LSL $src3" %}
11677 
11678   ins_encode %{
11679     __ andr(as_Register($dst$$reg),
11680               as_Register($src1$$reg),
11681               as_Register($src2$$reg),
11682               Assembler::LSL,
11683               $src3$$constant & 0x3f);
11684   %}
11685 
11686   ins_pipe(ialu_reg_reg_shift);
11687 %}
11688 
11689 // This pattern is automatically generated from aarch64_ad.m4
11690 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11691 instruct AndI_reg_RotateRight_reg(iRegINoSp dst,
11692                          iRegIorL2I src1, iRegIorL2I src2,
11693                          immI src3) %{
11694   match(Set dst (AndI src1 (RotateRight src2 src3)));
11695 
11696   ins_cost(1.9 * INSN_COST);
11697   format %{ "andw  $dst, $src1, $src2, ROR $src3" %}
11698 
11699   ins_encode %{
11700     __ andw(as_Register($dst$$reg),
11701               as_Register($src1$$reg),
11702               as_Register($src2$$reg),
11703               Assembler::ROR,
11704               $src3$$constant & 0x1f);
11705   %}
11706 
11707   ins_pipe(ialu_reg_reg_shift);
11708 %}
11709 
11710 // This pattern is automatically generated from aarch64_ad.m4
11711 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11712 instruct AndL_reg_RotateRight_reg(iRegLNoSp dst,
11713                          iRegL src1, iRegL src2,
11714                          immI src3) %{
11715   match(Set dst (AndL src1 (RotateRight src2 src3)));
11716 
11717   ins_cost(1.9 * INSN_COST);
11718   format %{ "andr  $dst, $src1, $src2, ROR $src3" %}
11719 
11720   ins_encode %{
11721     __ andr(as_Register($dst$$reg),
11722               as_Register($src1$$reg),
11723               as_Register($src2$$reg),
11724               Assembler::ROR,
11725               $src3$$constant & 0x3f);
11726   %}
11727 
11728   ins_pipe(ialu_reg_reg_shift);
11729 %}
11730 
11731 // This pattern is automatically generated from aarch64_ad.m4
11732 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11733 instruct XorI_reg_URShift_reg(iRegINoSp dst,
11734                          iRegIorL2I src1, iRegIorL2I src2,
11735                          immI src3) %{
11736   match(Set dst (XorI src1 (URShiftI src2 src3)));
11737 
11738   ins_cost(1.9 * INSN_COST);
11739   format %{ "eorw  $dst, $src1, $src2, LSR $src3" %}
11740 
11741   ins_encode %{
11742     __ eorw(as_Register($dst$$reg),
11743               as_Register($src1$$reg),
11744               as_Register($src2$$reg),
11745               Assembler::LSR,
11746               $src3$$constant & 0x1f);
11747   %}
11748 
11749   ins_pipe(ialu_reg_reg_shift);
11750 %}
11751 
11752 // This pattern is automatically generated from aarch64_ad.m4
11753 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11754 instruct XorL_reg_URShift_reg(iRegLNoSp dst,
11755                          iRegL src1, iRegL src2,
11756                          immI src3) %{
11757   match(Set dst (XorL src1 (URShiftL src2 src3)));
11758 
11759   ins_cost(1.9 * INSN_COST);
11760   format %{ "eor  $dst, $src1, $src2, LSR $src3" %}
11761 
11762   ins_encode %{
11763     __ eor(as_Register($dst$$reg),
11764               as_Register($src1$$reg),
11765               as_Register($src2$$reg),
11766               Assembler::LSR,
11767               $src3$$constant & 0x3f);
11768   %}
11769 
11770   ins_pipe(ialu_reg_reg_shift);
11771 %}
11772 
11773 // This pattern is automatically generated from aarch64_ad.m4
11774 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11775 instruct XorI_reg_RShift_reg(iRegINoSp dst,
11776                          iRegIorL2I src1, iRegIorL2I src2,
11777                          immI src3) %{
11778   match(Set dst (XorI src1 (RShiftI src2 src3)));
11779 
11780   ins_cost(1.9 * INSN_COST);
11781   format %{ "eorw  $dst, $src1, $src2, ASR $src3" %}
11782 
11783   ins_encode %{
11784     __ eorw(as_Register($dst$$reg),
11785               as_Register($src1$$reg),
11786               as_Register($src2$$reg),
11787               Assembler::ASR,
11788               $src3$$constant & 0x1f);
11789   %}
11790 
11791   ins_pipe(ialu_reg_reg_shift);
11792 %}
11793 
11794 // This pattern is automatically generated from aarch64_ad.m4
11795 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11796 instruct XorL_reg_RShift_reg(iRegLNoSp dst,
11797                          iRegL src1, iRegL src2,
11798                          immI src3) %{
11799   match(Set dst (XorL src1 (RShiftL src2 src3)));
11800 
11801   ins_cost(1.9 * INSN_COST);
11802   format %{ "eor  $dst, $src1, $src2, ASR $src3" %}
11803 
11804   ins_encode %{
11805     __ eor(as_Register($dst$$reg),
11806               as_Register($src1$$reg),
11807               as_Register($src2$$reg),
11808               Assembler::ASR,
11809               $src3$$constant & 0x3f);
11810   %}
11811 
11812   ins_pipe(ialu_reg_reg_shift);
11813 %}
11814 
11815 // This pattern is automatically generated from aarch64_ad.m4
11816 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11817 instruct XorI_reg_LShift_reg(iRegINoSp dst,
11818                          iRegIorL2I src1, iRegIorL2I src2,
11819                          immI src3) %{
11820   match(Set dst (XorI src1 (LShiftI src2 src3)));
11821 
11822   ins_cost(1.9 * INSN_COST);
11823   format %{ "eorw  $dst, $src1, $src2, LSL $src3" %}
11824 
11825   ins_encode %{
11826     __ eorw(as_Register($dst$$reg),
11827               as_Register($src1$$reg),
11828               as_Register($src2$$reg),
11829               Assembler::LSL,
11830               $src3$$constant & 0x1f);
11831   %}
11832 
11833   ins_pipe(ialu_reg_reg_shift);
11834 %}
11835 
11836 // This pattern is automatically generated from aarch64_ad.m4
11837 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11838 instruct XorL_reg_LShift_reg(iRegLNoSp dst,
11839                          iRegL src1, iRegL src2,
11840                          immI src3) %{
11841   match(Set dst (XorL src1 (LShiftL src2 src3)));
11842 
11843   ins_cost(1.9 * INSN_COST);
11844   format %{ "eor  $dst, $src1, $src2, LSL $src3" %}
11845 
11846   ins_encode %{
11847     __ eor(as_Register($dst$$reg),
11848               as_Register($src1$$reg),
11849               as_Register($src2$$reg),
11850               Assembler::LSL,
11851               $src3$$constant & 0x3f);
11852   %}
11853 
11854   ins_pipe(ialu_reg_reg_shift);
11855 %}
11856 
11857 // This pattern is automatically generated from aarch64_ad.m4
11858 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11859 instruct XorI_reg_RotateRight_reg(iRegINoSp dst,
11860                          iRegIorL2I src1, iRegIorL2I src2,
11861                          immI src3) %{
11862   match(Set dst (XorI src1 (RotateRight src2 src3)));
11863 
11864   ins_cost(1.9 * INSN_COST);
11865   format %{ "eorw  $dst, $src1, $src2, ROR $src3" %}
11866 
11867   ins_encode %{
11868     __ eorw(as_Register($dst$$reg),
11869               as_Register($src1$$reg),
11870               as_Register($src2$$reg),
11871               Assembler::ROR,
11872               $src3$$constant & 0x1f);
11873   %}
11874 
11875   ins_pipe(ialu_reg_reg_shift);
11876 %}
11877 
11878 // This pattern is automatically generated from aarch64_ad.m4
11879 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11880 instruct XorL_reg_RotateRight_reg(iRegLNoSp dst,
11881                          iRegL src1, iRegL src2,
11882                          immI src3) %{
11883   match(Set dst (XorL src1 (RotateRight src2 src3)));
11884 
11885   ins_cost(1.9 * INSN_COST);
11886   format %{ "eor  $dst, $src1, $src2, ROR $src3" %}
11887 
11888   ins_encode %{
11889     __ eor(as_Register($dst$$reg),
11890               as_Register($src1$$reg),
11891               as_Register($src2$$reg),
11892               Assembler::ROR,
11893               $src3$$constant & 0x3f);
11894   %}
11895 
11896   ins_pipe(ialu_reg_reg_shift);
11897 %}
11898 
11899 // This pattern is automatically generated from aarch64_ad.m4
11900 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11901 instruct OrI_reg_URShift_reg(iRegINoSp dst,
11902                          iRegIorL2I src1, iRegIorL2I src2,
11903                          immI src3) %{
11904   match(Set dst (OrI src1 (URShiftI src2 src3)));
11905 
11906   ins_cost(1.9 * INSN_COST);
11907   format %{ "orrw  $dst, $src1, $src2, LSR $src3" %}
11908 
11909   ins_encode %{
11910     __ orrw(as_Register($dst$$reg),
11911               as_Register($src1$$reg),
11912               as_Register($src2$$reg),
11913               Assembler::LSR,
11914               $src3$$constant & 0x1f);
11915   %}
11916 
11917   ins_pipe(ialu_reg_reg_shift);
11918 %}
11919 
11920 // This pattern is automatically generated from aarch64_ad.m4
11921 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11922 instruct OrL_reg_URShift_reg(iRegLNoSp dst,
11923                          iRegL src1, iRegL src2,
11924                          immI src3) %{
11925   match(Set dst (OrL src1 (URShiftL src2 src3)));
11926 
11927   ins_cost(1.9 * INSN_COST);
11928   format %{ "orr  $dst, $src1, $src2, LSR $src3" %}
11929 
11930   ins_encode %{
11931     __ orr(as_Register($dst$$reg),
11932               as_Register($src1$$reg),
11933               as_Register($src2$$reg),
11934               Assembler::LSR,
11935               $src3$$constant & 0x3f);
11936   %}
11937 
11938   ins_pipe(ialu_reg_reg_shift);
11939 %}
11940 
11941 // This pattern is automatically generated from aarch64_ad.m4
11942 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11943 instruct OrI_reg_RShift_reg(iRegINoSp dst,
11944                          iRegIorL2I src1, iRegIorL2I src2,
11945                          immI src3) %{
11946   match(Set dst (OrI src1 (RShiftI src2 src3)));
11947 
11948   ins_cost(1.9 * INSN_COST);
11949   format %{ "orrw  $dst, $src1, $src2, ASR $src3" %}
11950 
11951   ins_encode %{
11952     __ orrw(as_Register($dst$$reg),
11953               as_Register($src1$$reg),
11954               as_Register($src2$$reg),
11955               Assembler::ASR,
11956               $src3$$constant & 0x1f);
11957   %}
11958 
11959   ins_pipe(ialu_reg_reg_shift);
11960 %}
11961 
11962 // This pattern is automatically generated from aarch64_ad.m4
11963 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11964 instruct OrL_reg_RShift_reg(iRegLNoSp dst,
11965                          iRegL src1, iRegL src2,
11966                          immI src3) %{
11967   match(Set dst (OrL src1 (RShiftL src2 src3)));
11968 
11969   ins_cost(1.9 * INSN_COST);
11970   format %{ "orr  $dst, $src1, $src2, ASR $src3" %}
11971 
11972   ins_encode %{
11973     __ orr(as_Register($dst$$reg),
11974               as_Register($src1$$reg),
11975               as_Register($src2$$reg),
11976               Assembler::ASR,
11977               $src3$$constant & 0x3f);
11978   %}
11979 
11980   ins_pipe(ialu_reg_reg_shift);
11981 %}
11982 
11983 // This pattern is automatically generated from aarch64_ad.m4
11984 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11985 instruct OrI_reg_LShift_reg(iRegINoSp dst,
11986                          iRegIorL2I src1, iRegIorL2I src2,
11987                          immI src3) %{
11988   match(Set dst (OrI src1 (LShiftI src2 src3)));
11989 
11990   ins_cost(1.9 * INSN_COST);
11991   format %{ "orrw  $dst, $src1, $src2, LSL $src3" %}
11992 
11993   ins_encode %{
11994     __ orrw(as_Register($dst$$reg),
11995               as_Register($src1$$reg),
11996               as_Register($src2$$reg),
11997               Assembler::LSL,
11998               $src3$$constant & 0x1f);
11999   %}
12000 
12001   ins_pipe(ialu_reg_reg_shift);
12002 %}
12003 
12004 // This pattern is automatically generated from aarch64_ad.m4
12005 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12006 instruct OrL_reg_LShift_reg(iRegLNoSp dst,
12007                          iRegL src1, iRegL src2,
12008                          immI src3) %{
12009   match(Set dst (OrL src1 (LShiftL src2 src3)));
12010 
12011   ins_cost(1.9 * INSN_COST);
12012   format %{ "orr  $dst, $src1, $src2, LSL $src3" %}
12013 
12014   ins_encode %{
12015     __ orr(as_Register($dst$$reg),
12016               as_Register($src1$$reg),
12017               as_Register($src2$$reg),
12018               Assembler::LSL,
12019               $src3$$constant & 0x3f);
12020   %}
12021 
12022   ins_pipe(ialu_reg_reg_shift);
12023 %}
12024 
12025 // This pattern is automatically generated from aarch64_ad.m4
12026 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12027 instruct OrI_reg_RotateRight_reg(iRegINoSp dst,
12028                          iRegIorL2I src1, iRegIorL2I src2,
12029                          immI src3) %{
12030   match(Set dst (OrI src1 (RotateRight src2 src3)));
12031 
12032   ins_cost(1.9 * INSN_COST);
12033   format %{ "orrw  $dst, $src1, $src2, ROR $src3" %}
12034 
12035   ins_encode %{
12036     __ orrw(as_Register($dst$$reg),
12037               as_Register($src1$$reg),
12038               as_Register($src2$$reg),
12039               Assembler::ROR,
12040               $src3$$constant & 0x1f);
12041   %}
12042 
12043   ins_pipe(ialu_reg_reg_shift);
12044 %}
12045 
12046 // This pattern is automatically generated from aarch64_ad.m4
12047 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12048 instruct OrL_reg_RotateRight_reg(iRegLNoSp dst,
12049                          iRegL src1, iRegL src2,
12050                          immI src3) %{
12051   match(Set dst (OrL src1 (RotateRight src2 src3)));
12052 
12053   ins_cost(1.9 * INSN_COST);
12054   format %{ "orr  $dst, $src1, $src2, ROR $src3" %}
12055 
12056   ins_encode %{
12057     __ orr(as_Register($dst$$reg),
12058               as_Register($src1$$reg),
12059               as_Register($src2$$reg),
12060               Assembler::ROR,
12061               $src3$$constant & 0x3f);
12062   %}
12063 
12064   ins_pipe(ialu_reg_reg_shift);
12065 %}
12066 
12067 // This pattern is automatically generated from aarch64_ad.m4
12068 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12069 instruct AddI_reg_URShift_reg(iRegINoSp dst,
12070                          iRegIorL2I src1, iRegIorL2I src2,
12071                          immI src3) %{
12072   match(Set dst (AddI src1 (URShiftI src2 src3)));
12073 
12074   ins_cost(1.9 * INSN_COST);
12075   format %{ "addw  $dst, $src1, $src2, LSR $src3" %}
12076 
12077   ins_encode %{
12078     __ addw(as_Register($dst$$reg),
12079               as_Register($src1$$reg),
12080               as_Register($src2$$reg),
12081               Assembler::LSR,
12082               $src3$$constant & 0x1f);
12083   %}
12084 
12085   ins_pipe(ialu_reg_reg_shift);
12086 %}
12087 
12088 // This pattern is automatically generated from aarch64_ad.m4
12089 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12090 instruct AddL_reg_URShift_reg(iRegLNoSp dst,
12091                          iRegL src1, iRegL src2,
12092                          immI src3) %{
12093   match(Set dst (AddL src1 (URShiftL src2 src3)));
12094 
12095   ins_cost(1.9 * INSN_COST);
12096   format %{ "add  $dst, $src1, $src2, LSR $src3" %}
12097 
12098   ins_encode %{
12099     __ add(as_Register($dst$$reg),
12100               as_Register($src1$$reg),
12101               as_Register($src2$$reg),
12102               Assembler::LSR,
12103               $src3$$constant & 0x3f);
12104   %}
12105 
12106   ins_pipe(ialu_reg_reg_shift);
12107 %}
12108 
12109 // This pattern is automatically generated from aarch64_ad.m4
12110 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12111 instruct AddI_reg_RShift_reg(iRegINoSp dst,
12112                          iRegIorL2I src1, iRegIorL2I src2,
12113                          immI src3) %{
12114   match(Set dst (AddI src1 (RShiftI src2 src3)));
12115 
12116   ins_cost(1.9 * INSN_COST);
12117   format %{ "addw  $dst, $src1, $src2, ASR $src3" %}
12118 
12119   ins_encode %{
12120     __ addw(as_Register($dst$$reg),
12121               as_Register($src1$$reg),
12122               as_Register($src2$$reg),
12123               Assembler::ASR,
12124               $src3$$constant & 0x1f);
12125   %}
12126 
12127   ins_pipe(ialu_reg_reg_shift);
12128 %}
12129 
12130 // This pattern is automatically generated from aarch64_ad.m4
12131 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12132 instruct AddL_reg_RShift_reg(iRegLNoSp dst,
12133                          iRegL src1, iRegL src2,
12134                          immI src3) %{
12135   match(Set dst (AddL src1 (RShiftL src2 src3)));
12136 
12137   ins_cost(1.9 * INSN_COST);
12138   format %{ "add  $dst, $src1, $src2, ASR $src3" %}
12139 
12140   ins_encode %{
12141     __ add(as_Register($dst$$reg),
12142               as_Register($src1$$reg),
12143               as_Register($src2$$reg),
12144               Assembler::ASR,
12145               $src3$$constant & 0x3f);
12146   %}
12147 
12148   ins_pipe(ialu_reg_reg_shift);
12149 %}
12150 
12151 // This pattern is automatically generated from aarch64_ad.m4
12152 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12153 instruct AddI_reg_LShift_reg(iRegINoSp dst,
12154                          iRegIorL2I src1, iRegIorL2I src2,
12155                          immI src3) %{
12156   match(Set dst (AddI src1 (LShiftI src2 src3)));
12157 
12158   ins_cost(1.9 * INSN_COST);
12159   format %{ "addw  $dst, $src1, $src2, LSL $src3" %}
12160 
12161   ins_encode %{
12162     __ addw(as_Register($dst$$reg),
12163               as_Register($src1$$reg),
12164               as_Register($src2$$reg),
12165               Assembler::LSL,
12166               $src3$$constant & 0x1f);
12167   %}
12168 
12169   ins_pipe(ialu_reg_reg_shift);
12170 %}
12171 
12172 // This pattern is automatically generated from aarch64_ad.m4
12173 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12174 instruct AddL_reg_LShift_reg(iRegLNoSp dst,
12175                          iRegL src1, iRegL src2,
12176                          immI src3) %{
12177   match(Set dst (AddL src1 (LShiftL src2 src3)));
12178 
12179   ins_cost(1.9 * INSN_COST);
12180   format %{ "add  $dst, $src1, $src2, LSL $src3" %}
12181 
12182   ins_encode %{
12183     __ add(as_Register($dst$$reg),
12184               as_Register($src1$$reg),
12185               as_Register($src2$$reg),
12186               Assembler::LSL,
12187               $src3$$constant & 0x3f);
12188   %}
12189 
12190   ins_pipe(ialu_reg_reg_shift);
12191 %}
12192 
12193 // This pattern is automatically generated from aarch64_ad.m4
12194 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12195 instruct SubI_reg_URShift_reg(iRegINoSp dst,
12196                          iRegIorL2I src1, iRegIorL2I src2,
12197                          immI src3) %{
12198   match(Set dst (SubI src1 (URShiftI src2 src3)));
12199 
12200   ins_cost(1.9 * INSN_COST);
12201   format %{ "subw  $dst, $src1, $src2, LSR $src3" %}
12202 
12203   ins_encode %{
12204     __ subw(as_Register($dst$$reg),
12205               as_Register($src1$$reg),
12206               as_Register($src2$$reg),
12207               Assembler::LSR,
12208               $src3$$constant & 0x1f);
12209   %}
12210 
12211   ins_pipe(ialu_reg_reg_shift);
12212 %}
12213 
12214 // This pattern is automatically generated from aarch64_ad.m4
12215 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12216 instruct SubL_reg_URShift_reg(iRegLNoSp dst,
12217                          iRegL src1, iRegL src2,
12218                          immI src3) %{
12219   match(Set dst (SubL src1 (URShiftL src2 src3)));
12220 
12221   ins_cost(1.9 * INSN_COST);
12222   format %{ "sub  $dst, $src1, $src2, LSR $src3" %}
12223 
12224   ins_encode %{
12225     __ sub(as_Register($dst$$reg),
12226               as_Register($src1$$reg),
12227               as_Register($src2$$reg),
12228               Assembler::LSR,
12229               $src3$$constant & 0x3f);
12230   %}
12231 
12232   ins_pipe(ialu_reg_reg_shift);
12233 %}
12234 
12235 // This pattern is automatically generated from aarch64_ad.m4
12236 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12237 instruct SubI_reg_RShift_reg(iRegINoSp dst,
12238                          iRegIorL2I src1, iRegIorL2I src2,
12239                          immI src3) %{
12240   match(Set dst (SubI src1 (RShiftI src2 src3)));
12241 
12242   ins_cost(1.9 * INSN_COST);
12243   format %{ "subw  $dst, $src1, $src2, ASR $src3" %}
12244 
12245   ins_encode %{
12246     __ subw(as_Register($dst$$reg),
12247               as_Register($src1$$reg),
12248               as_Register($src2$$reg),
12249               Assembler::ASR,
12250               $src3$$constant & 0x1f);
12251   %}
12252 
12253   ins_pipe(ialu_reg_reg_shift);
12254 %}
12255 
12256 // This pattern is automatically generated from aarch64_ad.m4
12257 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12258 instruct SubL_reg_RShift_reg(iRegLNoSp dst,
12259                          iRegL src1, iRegL src2,
12260                          immI src3) %{
12261   match(Set dst (SubL src1 (RShiftL src2 src3)));
12262 
12263   ins_cost(1.9 * INSN_COST);
12264   format %{ "sub  $dst, $src1, $src2, ASR $src3" %}
12265 
12266   ins_encode %{
12267     __ sub(as_Register($dst$$reg),
12268               as_Register($src1$$reg),
12269               as_Register($src2$$reg),
12270               Assembler::ASR,
12271               $src3$$constant & 0x3f);
12272   %}
12273 
12274   ins_pipe(ialu_reg_reg_shift);
12275 %}
12276 
12277 // This pattern is automatically generated from aarch64_ad.m4
12278 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12279 instruct SubI_reg_LShift_reg(iRegINoSp dst,
12280                          iRegIorL2I src1, iRegIorL2I src2,
12281                          immI src3) %{
12282   match(Set dst (SubI src1 (LShiftI src2 src3)));
12283 
12284   ins_cost(1.9 * INSN_COST);
12285   format %{ "subw  $dst, $src1, $src2, LSL $src3" %}
12286 
12287   ins_encode %{
12288     __ subw(as_Register($dst$$reg),
12289               as_Register($src1$$reg),
12290               as_Register($src2$$reg),
12291               Assembler::LSL,
12292               $src3$$constant & 0x1f);
12293   %}
12294 
12295   ins_pipe(ialu_reg_reg_shift);
12296 %}
12297 
12298 // This pattern is automatically generated from aarch64_ad.m4
12299 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12300 instruct SubL_reg_LShift_reg(iRegLNoSp dst,
12301                          iRegL src1, iRegL src2,
12302                          immI src3) %{
12303   match(Set dst (SubL src1 (LShiftL src2 src3)));
12304 
12305   ins_cost(1.9 * INSN_COST);
12306   format %{ "sub  $dst, $src1, $src2, LSL $src3" %}
12307 
12308   ins_encode %{
12309     __ sub(as_Register($dst$$reg),
12310               as_Register($src1$$reg),
12311               as_Register($src2$$reg),
12312               Assembler::LSL,
12313               $src3$$constant & 0x3f);
12314   %}
12315 
12316   ins_pipe(ialu_reg_reg_shift);
12317 %}
12318 
12319 // This pattern is automatically generated from aarch64_ad.m4
12320 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12321 
12322 // Shift Left followed by Shift Right.
12323 // This idiom is used by the compiler for the i2b bytecode etc.
12324 instruct sbfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12325 %{
12326   match(Set dst (RShiftL (LShiftL src lshift_count) rshift_count));
12327   ins_cost(INSN_COST * 2);
12328   format %{ "sbfm  $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12329   ins_encode %{
12330     int lshift = $lshift_count$$constant & 63;
12331     int rshift = $rshift_count$$constant & 63;
12332     int s = 63 - lshift;
12333     int r = (rshift - lshift) & 63;
12334     __ sbfm(as_Register($dst$$reg),
12335             as_Register($src$$reg),
12336             r, s);
12337   %}
12338 
12339   ins_pipe(ialu_reg_shift);
12340 %}
12341 
12342 // This pattern is automatically generated from aarch64_ad.m4
12343 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12344 
12345 // Shift Left followed by Shift Right.
12346 // This idiom is used by the compiler for the i2b bytecode etc.
12347 instruct sbfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12348 %{
12349   match(Set dst (RShiftI (LShiftI src lshift_count) rshift_count));
12350   ins_cost(INSN_COST * 2);
12351   format %{ "sbfmw  $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12352   ins_encode %{
12353     int lshift = $lshift_count$$constant & 31;
12354     int rshift = $rshift_count$$constant & 31;
12355     int s = 31 - lshift;
12356     int r = (rshift - lshift) & 31;
12357     __ sbfmw(as_Register($dst$$reg),
12358             as_Register($src$$reg),
12359             r, s);
12360   %}
12361 
12362   ins_pipe(ialu_reg_shift);
12363 %}
12364 
12365 // This pattern is automatically generated from aarch64_ad.m4
12366 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12367 
12368 // Shift Left followed by Shift Right.
12369 // This idiom is used by the compiler for the i2b bytecode etc.
12370 instruct ubfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12371 %{
12372   match(Set dst (URShiftL (LShiftL src lshift_count) rshift_count));
12373   ins_cost(INSN_COST * 2);
12374   format %{ "ubfm  $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12375   ins_encode %{
12376     int lshift = $lshift_count$$constant & 63;
12377     int rshift = $rshift_count$$constant & 63;
12378     int s = 63 - lshift;
12379     int r = (rshift - lshift) & 63;
12380     __ ubfm(as_Register($dst$$reg),
12381             as_Register($src$$reg),
12382             r, s);
12383   %}
12384 
12385   ins_pipe(ialu_reg_shift);
12386 %}
12387 
12388 // This pattern is automatically generated from aarch64_ad.m4
12389 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12390 
12391 // Shift Left followed by Shift Right.
12392 // This idiom is used by the compiler for the i2b bytecode etc.
12393 instruct ubfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12394 %{
12395   match(Set dst (URShiftI (LShiftI src lshift_count) rshift_count));
12396   ins_cost(INSN_COST * 2);
12397   format %{ "ubfmw  $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12398   ins_encode %{
12399     int lshift = $lshift_count$$constant & 31;
12400     int rshift = $rshift_count$$constant & 31;
12401     int s = 31 - lshift;
12402     int r = (rshift - lshift) & 31;
12403     __ ubfmw(as_Register($dst$$reg),
12404             as_Register($src$$reg),
12405             r, s);
12406   %}
12407 
12408   ins_pipe(ialu_reg_shift);
12409 %}
12410 
12411 // Bitfield extract with shift & mask
12412 
12413 // This pattern is automatically generated from aarch64_ad.m4
12414 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12415 instruct ubfxwI(iRegINoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12416 %{
12417   match(Set dst (AndI (URShiftI src rshift) mask));
12418   // Make sure we are not going to exceed what ubfxw can do.
12419   predicate((exact_log2(n->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12420 
12421   ins_cost(INSN_COST);
12422   format %{ "ubfxw $dst, $src, $rshift, $mask" %}
12423   ins_encode %{
12424     int rshift = $rshift$$constant & 31;
12425     intptr_t mask = $mask$$constant;
12426     int width = exact_log2(mask+1);
12427     __ ubfxw(as_Register($dst$$reg),
12428             as_Register($src$$reg), rshift, width);
12429   %}
12430   ins_pipe(ialu_reg_shift);
12431 %}
12432 
12433 // This pattern is automatically generated from aarch64_ad.m4
12434 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12435 instruct ubfxL(iRegLNoSp dst, iRegL src, immI rshift, immL_bitmask mask)
12436 %{
12437   match(Set dst (AndL (URShiftL src rshift) mask));
12438   // Make sure we are not going to exceed what ubfx can do.
12439   predicate((exact_log2_long(n->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= (63 + 1));
12440 
12441   ins_cost(INSN_COST);
12442   format %{ "ubfx $dst, $src, $rshift, $mask" %}
12443   ins_encode %{
12444     int rshift = $rshift$$constant & 63;
12445     intptr_t mask = $mask$$constant;
12446     int width = exact_log2_long(mask+1);
12447     __ ubfx(as_Register($dst$$reg),
12448             as_Register($src$$reg), rshift, width);
12449   %}
12450   ins_pipe(ialu_reg_shift);
12451 %}
12452 
12453 
12454 // This pattern is automatically generated from aarch64_ad.m4
12455 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12456 
12457 // We can use ubfx when extending an And with a mask when we know mask
12458 // is positive.  We know that because immI_bitmask guarantees it.
12459 instruct ubfxIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12460 %{
12461   match(Set dst (ConvI2L (AndI (URShiftI src rshift) mask)));
12462   // Make sure we are not going to exceed what ubfxw can do.
12463   predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12464 
12465   ins_cost(INSN_COST * 2);
12466   format %{ "ubfx $dst, $src, $rshift, $mask" %}
12467   ins_encode %{
12468     int rshift = $rshift$$constant & 31;
12469     intptr_t mask = $mask$$constant;
12470     int width = exact_log2(mask+1);
12471     __ ubfx(as_Register($dst$$reg),
12472             as_Register($src$$reg), rshift, width);
12473   %}
12474   ins_pipe(ialu_reg_shift);
12475 %}
12476 
12477 
12478 // This pattern is automatically generated from aarch64_ad.m4
12479 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12480 
12481 // We can use ubfiz when masking by a positive number and then left shifting the result.
12482 // We know that the mask is positive because immI_bitmask guarantees it.
12483 instruct ubfizwI(iRegINoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12484 %{
12485   match(Set dst (LShiftI (AndI src mask) lshift));
12486   predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 31)) <= (31 + 1));
12487 
12488   ins_cost(INSN_COST);
12489   format %{ "ubfizw $dst, $src, $lshift, $mask" %}
12490   ins_encode %{
12491     int lshift = $lshift$$constant & 31;
12492     intptr_t mask = $mask$$constant;
12493     int width = exact_log2(mask+1);
12494     __ ubfizw(as_Register($dst$$reg),
12495           as_Register($src$$reg), lshift, width);
12496   %}
12497   ins_pipe(ialu_reg_shift);
12498 %}
12499 
12500 // This pattern is automatically generated from aarch64_ad.m4
12501 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12502 
12503 // We can use ubfiz when masking by a positive number and then left shifting the result.
12504 // We know that the mask is positive because immL_bitmask guarantees it.
12505 instruct ubfizL(iRegLNoSp dst, iRegL src, immI lshift, immL_bitmask mask)
12506 %{
12507   match(Set dst (LShiftL (AndL src mask) lshift));
12508   predicate((exact_log2_long(n->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
12509 
12510   ins_cost(INSN_COST);
12511   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12512   ins_encode %{
12513     int lshift = $lshift$$constant & 63;
12514     intptr_t mask = $mask$$constant;
12515     int width = exact_log2_long(mask+1);
12516     __ ubfiz(as_Register($dst$$reg),
12517           as_Register($src$$reg), lshift, width);
12518   %}
12519   ins_pipe(ialu_reg_shift);
12520 %}
12521 
12522 // This pattern is automatically generated from aarch64_ad.m4
12523 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12524 
12525 // We can use ubfiz when masking by a positive number and then left shifting the result.
12526 // We know that the mask is positive because immI_bitmask guarantees it.
12527 instruct ubfizwIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12528 %{
12529   match(Set dst (ConvI2L (LShiftI (AndI src mask) lshift)));
12530   predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= 31);
12531 
12532   ins_cost(INSN_COST);
12533   format %{ "ubfizw $dst, $src, $lshift, $mask" %}
12534   ins_encode %{
12535     int lshift = $lshift$$constant & 31;
12536     intptr_t mask = $mask$$constant;
12537     int width = exact_log2(mask+1);
12538     __ ubfizw(as_Register($dst$$reg),
12539           as_Register($src$$reg), lshift, width);
12540   %}
12541   ins_pipe(ialu_reg_shift);
12542 %}
12543 
12544 // This pattern is automatically generated from aarch64_ad.m4
12545 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12546 
12547 // We can use ubfiz when masking by a positive number and then left shifting the result.
12548 // We know that the mask is positive because immL_bitmask guarantees it.
12549 instruct ubfizLConvL2I(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
12550 %{
12551   match(Set dst (ConvL2I (LShiftL (AndL src mask) lshift)));
12552   predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= 31);
12553 
12554   ins_cost(INSN_COST);
12555   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12556   ins_encode %{
12557     int lshift = $lshift$$constant & 63;
12558     intptr_t mask = $mask$$constant;
12559     int width = exact_log2_long(mask+1);
12560     __ ubfiz(as_Register($dst$$reg),
12561           as_Register($src$$reg), lshift, width);
12562   %}
12563   ins_pipe(ialu_reg_shift);
12564 %}
12565 
12566 
12567 // This pattern is automatically generated from aarch64_ad.m4
12568 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12569 
12570 // If there is a convert I to L block between and AndI and a LShiftL, we can also match ubfiz
12571 instruct ubfizIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12572 %{
12573   match(Set dst (LShiftL (ConvI2L (AndI src mask)) lshift));
12574   predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
12575 
12576   ins_cost(INSN_COST);
12577   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12578   ins_encode %{
12579     int lshift = $lshift$$constant & 63;
12580     intptr_t mask = $mask$$constant;
12581     int width = exact_log2(mask+1);
12582     __ ubfiz(as_Register($dst$$reg),
12583              as_Register($src$$reg), lshift, width);
12584   %}
12585   ins_pipe(ialu_reg_shift);
12586 %}
12587 
12588 // This pattern is automatically generated from aarch64_ad.m4
12589 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12590 
12591 // If there is a convert L to I block between and AndL and a LShiftI, we can also match ubfiz
12592 instruct ubfizLConvL2Ix(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
12593 %{
12594   match(Set dst (LShiftI (ConvL2I (AndL src mask)) lshift));
12595   predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 31)) <= 31);
12596 
12597   ins_cost(INSN_COST);
12598   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12599   ins_encode %{
12600     int lshift = $lshift$$constant & 31;
12601     intptr_t mask = $mask$$constant;
12602     int width = exact_log2(mask+1);
12603     __ ubfiz(as_Register($dst$$reg),
12604              as_Register($src$$reg), lshift, width);
12605   %}
12606   ins_pipe(ialu_reg_shift);
12607 %}
12608 
12609 // This pattern is automatically generated from aarch64_ad.m4
12610 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12611 
12612 // Can skip int2long conversions after AND with small bitmask
12613 instruct ubfizIConvI2LAndI(iRegLNoSp dst, iRegI src, immI_bitmask msk)
12614 %{
12615   match(Set dst (ConvI2L (AndI src msk)));
12616   ins_cost(INSN_COST);
12617   format %{ "ubfiz $dst, $src, 0, exact_log2($msk + 1) " %}
12618   ins_encode %{
12619     __ ubfiz(as_Register($dst$$reg), as_Register($src$$reg), 0, exact_log2($msk$$constant + 1));
12620   %}
12621   ins_pipe(ialu_reg_shift);
12622 %}
12623 
12624 
12625 // Rotations
12626 
12627 // This pattern is automatically generated from aarch64_ad.m4
12628 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12629 instruct extrOrL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
12630 %{
12631   match(Set dst (OrL (LShiftL src1 lshift) (URShiftL src2 rshift)));
12632   predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
12633 
12634   ins_cost(INSN_COST);
12635   format %{ "extr $dst, $src1, $src2, #$rshift" %}
12636 
12637   ins_encode %{
12638     __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12639             $rshift$$constant & 63);
12640   %}
12641   ins_pipe(ialu_reg_reg_extr);
12642 %}
12643 
12644 
12645 // This pattern is automatically generated from aarch64_ad.m4
12646 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12647 instruct extrOrI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
12648 %{
12649   match(Set dst (OrI (LShiftI src1 lshift) (URShiftI src2 rshift)));
12650   predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
12651 
12652   ins_cost(INSN_COST);
12653   format %{ "extr $dst, $src1, $src2, #$rshift" %}
12654 
12655   ins_encode %{
12656     __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12657             $rshift$$constant & 31);
12658   %}
12659   ins_pipe(ialu_reg_reg_extr);
12660 %}
12661 
12662 
12663 // This pattern is automatically generated from aarch64_ad.m4
12664 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12665 instruct extrAddL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
12666 %{
12667   match(Set dst (AddL (LShiftL src1 lshift) (URShiftL src2 rshift)));
12668   predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
12669 
12670   ins_cost(INSN_COST);
12671   format %{ "extr $dst, $src1, $src2, #$rshift" %}
12672 
12673   ins_encode %{
12674     __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12675             $rshift$$constant & 63);
12676   %}
12677   ins_pipe(ialu_reg_reg_extr);
12678 %}
12679 
12680 
12681 // This pattern is automatically generated from aarch64_ad.m4
12682 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12683 instruct extrAddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
12684 %{
12685   match(Set dst (AddI (LShiftI src1 lshift) (URShiftI src2 rshift)));
12686   predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
12687 
12688   ins_cost(INSN_COST);
12689   format %{ "extr $dst, $src1, $src2, #$rshift" %}
12690 
12691   ins_encode %{
12692     __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12693             $rshift$$constant & 31);
12694   %}
12695   ins_pipe(ialu_reg_reg_extr);
12696 %}
12697 
12698 // This pattern is automatically generated from aarch64_ad.m4
12699 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12700 instruct rorI_imm(iRegINoSp dst, iRegI src, immI shift)
12701 %{
12702   match(Set dst (RotateRight src shift));
12703 
12704   ins_cost(INSN_COST);
12705   format %{ "ror    $dst, $src, $shift" %}
12706 
12707   ins_encode %{
12708      __ extrw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
12709                $shift$$constant & 0x1f);
12710   %}
12711   ins_pipe(ialu_reg_reg_vshift);
12712 %}
12713 
12714 // This pattern is automatically generated from aarch64_ad.m4
12715 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12716 instruct rorL_imm(iRegLNoSp dst, iRegL src, immI shift)
12717 %{
12718   match(Set dst (RotateRight src shift));
12719 
12720   ins_cost(INSN_COST);
12721   format %{ "ror    $dst, $src, $shift" %}
12722 
12723   ins_encode %{
12724      __ extr(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
12725                $shift$$constant & 0x3f);
12726   %}
12727   ins_pipe(ialu_reg_reg_vshift);
12728 %}
12729 
12730 // This pattern is automatically generated from aarch64_ad.m4
12731 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12732 instruct rorI_reg(iRegINoSp dst, iRegI src, iRegI shift)
12733 %{
12734   match(Set dst (RotateRight src shift));
12735 
12736   ins_cost(INSN_COST);
12737   format %{ "ror    $dst, $src, $shift" %}
12738 
12739   ins_encode %{
12740      __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
12741   %}
12742   ins_pipe(ialu_reg_reg_vshift);
12743 %}
12744 
12745 // This pattern is automatically generated from aarch64_ad.m4
12746 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12747 instruct rorL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
12748 %{
12749   match(Set dst (RotateRight src shift));
12750 
12751   ins_cost(INSN_COST);
12752   format %{ "ror    $dst, $src, $shift" %}
12753 
12754   ins_encode %{
12755      __ rorv(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
12756   %}
12757   ins_pipe(ialu_reg_reg_vshift);
12758 %}
12759 
12760 // This pattern is automatically generated from aarch64_ad.m4
12761 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12762 instruct rolI_reg(iRegINoSp dst, iRegI src, iRegI shift)
12763 %{
12764   match(Set dst (RotateLeft src shift));
12765 
12766   ins_cost(INSN_COST);
12767   format %{ "rol    $dst, $src, $shift" %}
12768 
12769   ins_encode %{
12770      __ subw(rscratch1, zr, as_Register($shift$$reg));
12771      __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
12772   %}
12773   ins_pipe(ialu_reg_reg_vshift);
12774 %}
12775 
12776 // This pattern is automatically generated from aarch64_ad.m4
12777 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12778 instruct rolL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
12779 %{
12780   match(Set dst (RotateLeft src shift));
12781 
12782   ins_cost(INSN_COST);
12783   format %{ "rol    $dst, $src, $shift" %}
12784 
12785   ins_encode %{
12786      __ subw(rscratch1, zr, as_Register($shift$$reg));
12787      __ rorv(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
12788   %}
12789   ins_pipe(ialu_reg_reg_vshift);
12790 %}
12791 
12792 
12793 // Add/subtract (extended)
12794 
12795 // This pattern is automatically generated from aarch64_ad.m4
12796 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12797 instruct AddExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
12798 %{
12799   match(Set dst (AddL src1 (ConvI2L src2)));
12800   ins_cost(INSN_COST);
12801   format %{ "add  $dst, $src1, $src2, sxtw" %}
12802 
12803    ins_encode %{
12804      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12805             as_Register($src2$$reg), ext::sxtw);
12806    %}
12807   ins_pipe(ialu_reg_reg);
12808 %}
12809 
12810 // This pattern is automatically generated from aarch64_ad.m4
12811 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12812 instruct SubExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
12813 %{
12814   match(Set dst (SubL src1 (ConvI2L src2)));
12815   ins_cost(INSN_COST);
12816   format %{ "sub  $dst, $src1, $src2, sxtw" %}
12817 
12818    ins_encode %{
12819      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12820             as_Register($src2$$reg), ext::sxtw);
12821    %}
12822   ins_pipe(ialu_reg_reg);
12823 %}
12824 
12825 // This pattern is automatically generated from aarch64_ad.m4
12826 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12827 instruct AddExtI_sxth(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_16 lshift, immI_16 rshift, rFlagsReg cr)
12828 %{
12829   match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
12830   ins_cost(INSN_COST);
12831   format %{ "add  $dst, $src1, $src2, sxth" %}
12832 
12833    ins_encode %{
12834      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12835             as_Register($src2$$reg), ext::sxth);
12836    %}
12837   ins_pipe(ialu_reg_reg);
12838 %}
12839 
12840 // This pattern is automatically generated from aarch64_ad.m4
12841 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12842 instruct AddExtI_sxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
12843 %{
12844   match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
12845   ins_cost(INSN_COST);
12846   format %{ "add  $dst, $src1, $src2, sxtb" %}
12847 
12848    ins_encode %{
12849      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12850             as_Register($src2$$reg), ext::sxtb);
12851    %}
12852   ins_pipe(ialu_reg_reg);
12853 %}
12854 
12855 // This pattern is automatically generated from aarch64_ad.m4
12856 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12857 instruct AddExtI_uxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
12858 %{
12859   match(Set dst (AddI src1 (URShiftI (LShiftI src2 lshift) rshift)));
12860   ins_cost(INSN_COST);
12861   format %{ "add  $dst, $src1, $src2, uxtb" %}
12862 
12863    ins_encode %{
12864      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12865             as_Register($src2$$reg), ext::uxtb);
12866    %}
12867   ins_pipe(ialu_reg_reg);
12868 %}
12869 
12870 // This pattern is automatically generated from aarch64_ad.m4
12871 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12872 instruct AddExtL_sxth(iRegLNoSp dst, iRegL src1, iRegL src2, immI_48 lshift, immI_48 rshift, rFlagsReg cr)
12873 %{
12874   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12875   ins_cost(INSN_COST);
12876   format %{ "add  $dst, $src1, $src2, sxth" %}
12877 
12878    ins_encode %{
12879      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12880             as_Register($src2$$reg), ext::sxth);
12881    %}
12882   ins_pipe(ialu_reg_reg);
12883 %}
12884 
12885 // This pattern is automatically generated from aarch64_ad.m4
12886 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12887 instruct AddExtL_sxtw(iRegLNoSp dst, iRegL src1, iRegL src2, immI_32 lshift, immI_32 rshift, rFlagsReg cr)
12888 %{
12889   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12890   ins_cost(INSN_COST);
12891   format %{ "add  $dst, $src1, $src2, sxtw" %}
12892 
12893    ins_encode %{
12894      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12895             as_Register($src2$$reg), ext::sxtw);
12896    %}
12897   ins_pipe(ialu_reg_reg);
12898 %}
12899 
12900 // This pattern is automatically generated from aarch64_ad.m4
12901 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12902 instruct AddExtL_sxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
12903 %{
12904   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12905   ins_cost(INSN_COST);
12906   format %{ "add  $dst, $src1, $src2, sxtb" %}
12907 
12908    ins_encode %{
12909      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12910             as_Register($src2$$reg), ext::sxtb);
12911    %}
12912   ins_pipe(ialu_reg_reg);
12913 %}
12914 
12915 // This pattern is automatically generated from aarch64_ad.m4
12916 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12917 instruct AddExtL_uxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
12918 %{
12919   match(Set dst (AddL src1 (URShiftL (LShiftL src2 lshift) rshift)));
12920   ins_cost(INSN_COST);
12921   format %{ "add  $dst, $src1, $src2, uxtb" %}
12922 
12923    ins_encode %{
12924      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12925             as_Register($src2$$reg), ext::uxtb);
12926    %}
12927   ins_pipe(ialu_reg_reg);
12928 %}
12929 
12930 // This pattern is automatically generated from aarch64_ad.m4
12931 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12932 instruct AddExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
12933 %{
12934   match(Set dst (AddI src1 (AndI src2 mask)));
12935   ins_cost(INSN_COST);
12936   format %{ "addw  $dst, $src1, $src2, uxtb" %}
12937 
12938    ins_encode %{
12939      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12940             as_Register($src2$$reg), ext::uxtb);
12941    %}
12942   ins_pipe(ialu_reg_reg);
12943 %}
12944 
12945 // This pattern is automatically generated from aarch64_ad.m4
12946 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12947 instruct AddExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
12948 %{
12949   match(Set dst (AddI src1 (AndI src2 mask)));
12950   ins_cost(INSN_COST);
12951   format %{ "addw  $dst, $src1, $src2, uxth" %}
12952 
12953    ins_encode %{
12954      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12955             as_Register($src2$$reg), ext::uxth);
12956    %}
12957   ins_pipe(ialu_reg_reg);
12958 %}
12959 
12960 // This pattern is automatically generated from aarch64_ad.m4
12961 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12962 instruct AddExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
12963 %{
12964   match(Set dst (AddL src1 (AndL src2 mask)));
12965   ins_cost(INSN_COST);
12966   format %{ "add  $dst, $src1, $src2, uxtb" %}
12967 
12968    ins_encode %{
12969      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12970             as_Register($src2$$reg), ext::uxtb);
12971    %}
12972   ins_pipe(ialu_reg_reg);
12973 %}
12974 
12975 // This pattern is automatically generated from aarch64_ad.m4
12976 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12977 instruct AddExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
12978 %{
12979   match(Set dst (AddL src1 (AndL src2 mask)));
12980   ins_cost(INSN_COST);
12981   format %{ "add  $dst, $src1, $src2, uxth" %}
12982 
12983    ins_encode %{
12984      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12985             as_Register($src2$$reg), ext::uxth);
12986    %}
12987   ins_pipe(ialu_reg_reg);
12988 %}
12989 
12990 // This pattern is automatically generated from aarch64_ad.m4
12991 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12992 instruct AddExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
12993 %{
12994   match(Set dst (AddL src1 (AndL src2 mask)));
12995   ins_cost(INSN_COST);
12996   format %{ "add  $dst, $src1, $src2, uxtw" %}
12997 
12998    ins_encode %{
12999      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13000             as_Register($src2$$reg), ext::uxtw);
13001    %}
13002   ins_pipe(ialu_reg_reg);
13003 %}
13004 
13005 // This pattern is automatically generated from aarch64_ad.m4
13006 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13007 instruct SubExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
13008 %{
13009   match(Set dst (SubI src1 (AndI src2 mask)));
13010   ins_cost(INSN_COST);
13011   format %{ "subw  $dst, $src1, $src2, uxtb" %}
13012 
13013    ins_encode %{
13014      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13015             as_Register($src2$$reg), ext::uxtb);
13016    %}
13017   ins_pipe(ialu_reg_reg);
13018 %}
13019 
13020 // This pattern is automatically generated from aarch64_ad.m4
13021 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13022 instruct SubExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
13023 %{
13024   match(Set dst (SubI src1 (AndI src2 mask)));
13025   ins_cost(INSN_COST);
13026   format %{ "subw  $dst, $src1, $src2, uxth" %}
13027 
13028    ins_encode %{
13029      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13030             as_Register($src2$$reg), ext::uxth);
13031    %}
13032   ins_pipe(ialu_reg_reg);
13033 %}
13034 
13035 // This pattern is automatically generated from aarch64_ad.m4
13036 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13037 instruct SubExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
13038 %{
13039   match(Set dst (SubL src1 (AndL src2 mask)));
13040   ins_cost(INSN_COST);
13041   format %{ "sub  $dst, $src1, $src2, uxtb" %}
13042 
13043    ins_encode %{
13044      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13045             as_Register($src2$$reg), ext::uxtb);
13046    %}
13047   ins_pipe(ialu_reg_reg);
13048 %}
13049 
13050 // This pattern is automatically generated from aarch64_ad.m4
13051 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13052 instruct SubExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
13053 %{
13054   match(Set dst (SubL src1 (AndL src2 mask)));
13055   ins_cost(INSN_COST);
13056   format %{ "sub  $dst, $src1, $src2, uxth" %}
13057 
13058    ins_encode %{
13059      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13060             as_Register($src2$$reg), ext::uxth);
13061    %}
13062   ins_pipe(ialu_reg_reg);
13063 %}
13064 
13065 // This pattern is automatically generated from aarch64_ad.m4
13066 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13067 instruct SubExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
13068 %{
13069   match(Set dst (SubL src1 (AndL src2 mask)));
13070   ins_cost(INSN_COST);
13071   format %{ "sub  $dst, $src1, $src2, uxtw" %}
13072 
13073    ins_encode %{
13074      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13075             as_Register($src2$$reg), ext::uxtw);
13076    %}
13077   ins_pipe(ialu_reg_reg);
13078 %}
13079 
13080 
13081 // This pattern is automatically generated from aarch64_ad.m4
13082 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13083 instruct AddExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13084 %{
13085   match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13086   ins_cost(1.9 * INSN_COST);
13087   format %{ "add  $dst, $src1, $src2, sxtb #lshift2" %}
13088 
13089    ins_encode %{
13090      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13091             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13092    %}
13093   ins_pipe(ialu_reg_reg_shift);
13094 %}
13095 
13096 // This pattern is automatically generated from aarch64_ad.m4
13097 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13098 instruct AddExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13099 %{
13100   match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13101   ins_cost(1.9 * INSN_COST);
13102   format %{ "add  $dst, $src1, $src2, sxth #lshift2" %}
13103 
13104    ins_encode %{
13105      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13106             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13107    %}
13108   ins_pipe(ialu_reg_reg_shift);
13109 %}
13110 
13111 // This pattern is automatically generated from aarch64_ad.m4
13112 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13113 instruct AddExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13114 %{
13115   match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13116   ins_cost(1.9 * INSN_COST);
13117   format %{ "add  $dst, $src1, $src2, sxtw #lshift2" %}
13118 
13119    ins_encode %{
13120      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13121             as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13122    %}
13123   ins_pipe(ialu_reg_reg_shift);
13124 %}
13125 
13126 // This pattern is automatically generated from aarch64_ad.m4
13127 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13128 instruct SubExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13129 %{
13130   match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13131   ins_cost(1.9 * INSN_COST);
13132   format %{ "sub  $dst, $src1, $src2, sxtb #lshift2" %}
13133 
13134    ins_encode %{
13135      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13136             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13137    %}
13138   ins_pipe(ialu_reg_reg_shift);
13139 %}
13140 
13141 // This pattern is automatically generated from aarch64_ad.m4
13142 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13143 instruct SubExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13144 %{
13145   match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13146   ins_cost(1.9 * INSN_COST);
13147   format %{ "sub  $dst, $src1, $src2, sxth #lshift2" %}
13148 
13149    ins_encode %{
13150      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13151             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13152    %}
13153   ins_pipe(ialu_reg_reg_shift);
13154 %}
13155 
13156 // This pattern is automatically generated from aarch64_ad.m4
13157 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13158 instruct SubExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13159 %{
13160   match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13161   ins_cost(1.9 * INSN_COST);
13162   format %{ "sub  $dst, $src1, $src2, sxtw #lshift2" %}
13163 
13164    ins_encode %{
13165      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13166             as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13167    %}
13168   ins_pipe(ialu_reg_reg_shift);
13169 %}
13170 
13171 // This pattern is automatically generated from aarch64_ad.m4
13172 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13173 instruct AddExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13174 %{
13175   match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13176   ins_cost(1.9 * INSN_COST);
13177   format %{ "addw  $dst, $src1, $src2, sxtb #lshift2" %}
13178 
13179    ins_encode %{
13180      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13181             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13182    %}
13183   ins_pipe(ialu_reg_reg_shift);
13184 %}
13185 
13186 // This pattern is automatically generated from aarch64_ad.m4
13187 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13188 instruct AddExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13189 %{
13190   match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13191   ins_cost(1.9 * INSN_COST);
13192   format %{ "addw  $dst, $src1, $src2, sxth #lshift2" %}
13193 
13194    ins_encode %{
13195      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13196             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13197    %}
13198   ins_pipe(ialu_reg_reg_shift);
13199 %}
13200 
13201 // This pattern is automatically generated from aarch64_ad.m4
13202 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13203 instruct SubExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13204 %{
13205   match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13206   ins_cost(1.9 * INSN_COST);
13207   format %{ "subw  $dst, $src1, $src2, sxtb #lshift2" %}
13208 
13209    ins_encode %{
13210      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13211             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13212    %}
13213   ins_pipe(ialu_reg_reg_shift);
13214 %}
13215 
13216 // This pattern is automatically generated from aarch64_ad.m4
13217 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13218 instruct SubExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13219 %{
13220   match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13221   ins_cost(1.9 * INSN_COST);
13222   format %{ "subw  $dst, $src1, $src2, sxth #lshift2" %}
13223 
13224    ins_encode %{
13225      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13226             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13227    %}
13228   ins_pipe(ialu_reg_reg_shift);
13229 %}
13230 
13231 // This pattern is automatically generated from aarch64_ad.m4
13232 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13233 instruct AddExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13234 %{
13235   match(Set dst (AddL src1 (LShiftL (ConvI2L src2) lshift)));
13236   ins_cost(1.9 * INSN_COST);
13237   format %{ "add  $dst, $src1, $src2, sxtw #lshift" %}
13238 
13239    ins_encode %{
13240      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13241             as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13242    %}
13243   ins_pipe(ialu_reg_reg_shift);
13244 %}
13245 
13246 // This pattern is automatically generated from aarch64_ad.m4
13247 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13248 instruct SubExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13249 %{
13250   match(Set dst (SubL src1 (LShiftL (ConvI2L src2) lshift)));
13251   ins_cost(1.9 * INSN_COST);
13252   format %{ "sub  $dst, $src1, $src2, sxtw #lshift" %}
13253 
13254    ins_encode %{
13255      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13256             as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13257    %}
13258   ins_pipe(ialu_reg_reg_shift);
13259 %}
13260 
13261 // This pattern is automatically generated from aarch64_ad.m4
13262 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13263 instruct AddExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13264 %{
13265   match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13266   ins_cost(1.9 * INSN_COST);
13267   format %{ "add  $dst, $src1, $src2, uxtb #lshift" %}
13268 
13269    ins_encode %{
13270      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13271             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13272    %}
13273   ins_pipe(ialu_reg_reg_shift);
13274 %}
13275 
13276 // This pattern is automatically generated from aarch64_ad.m4
13277 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13278 instruct AddExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13279 %{
13280   match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13281   ins_cost(1.9 * INSN_COST);
13282   format %{ "add  $dst, $src1, $src2, uxth #lshift" %}
13283 
13284    ins_encode %{
13285      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13286             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13287    %}
13288   ins_pipe(ialu_reg_reg_shift);
13289 %}
13290 
13291 // This pattern is automatically generated from aarch64_ad.m4
13292 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13293 instruct AddExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13294 %{
13295   match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13296   ins_cost(1.9 * INSN_COST);
13297   format %{ "add  $dst, $src1, $src2, uxtw #lshift" %}
13298 
13299    ins_encode %{
13300      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13301             as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13302    %}
13303   ins_pipe(ialu_reg_reg_shift);
13304 %}
13305 
13306 // This pattern is automatically generated from aarch64_ad.m4
13307 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13308 instruct SubExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13309 %{
13310   match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13311   ins_cost(1.9 * INSN_COST);
13312   format %{ "sub  $dst, $src1, $src2, uxtb #lshift" %}
13313 
13314    ins_encode %{
13315      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13316             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13317    %}
13318   ins_pipe(ialu_reg_reg_shift);
13319 %}
13320 
13321 // This pattern is automatically generated from aarch64_ad.m4
13322 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13323 instruct SubExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13324 %{
13325   match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13326   ins_cost(1.9 * INSN_COST);
13327   format %{ "sub  $dst, $src1, $src2, uxth #lshift" %}
13328 
13329    ins_encode %{
13330      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13331             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13332    %}
13333   ins_pipe(ialu_reg_reg_shift);
13334 %}
13335 
13336 // This pattern is automatically generated from aarch64_ad.m4
13337 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13338 instruct SubExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13339 %{
13340   match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13341   ins_cost(1.9 * INSN_COST);
13342   format %{ "sub  $dst, $src1, $src2, uxtw #lshift" %}
13343 
13344    ins_encode %{
13345      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13346             as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13347    %}
13348   ins_pipe(ialu_reg_reg_shift);
13349 %}
13350 
13351 // This pattern is automatically generated from aarch64_ad.m4
13352 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13353 instruct AddExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13354 %{
13355   match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13356   ins_cost(1.9 * INSN_COST);
13357   format %{ "addw  $dst, $src1, $src2, uxtb #lshift" %}
13358 
13359    ins_encode %{
13360      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13361             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13362    %}
13363   ins_pipe(ialu_reg_reg_shift);
13364 %}
13365 
13366 // This pattern is automatically generated from aarch64_ad.m4
13367 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13368 instruct AddExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13369 %{
13370   match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13371   ins_cost(1.9 * INSN_COST);
13372   format %{ "addw  $dst, $src1, $src2, uxth #lshift" %}
13373 
13374    ins_encode %{
13375      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13376             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13377    %}
13378   ins_pipe(ialu_reg_reg_shift);
13379 %}
13380 
13381 // This pattern is automatically generated from aarch64_ad.m4
13382 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13383 instruct SubExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13384 %{
13385   match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13386   ins_cost(1.9 * INSN_COST);
13387   format %{ "subw  $dst, $src1, $src2, uxtb #lshift" %}
13388 
13389    ins_encode %{
13390      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13391             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13392    %}
13393   ins_pipe(ialu_reg_reg_shift);
13394 %}
13395 
13396 // This pattern is automatically generated from aarch64_ad.m4
13397 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13398 instruct SubExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13399 %{
13400   match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13401   ins_cost(1.9 * INSN_COST);
13402   format %{ "subw  $dst, $src1, $src2, uxth #lshift" %}
13403 
13404    ins_encode %{
13405      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13406             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13407    %}
13408   ins_pipe(ialu_reg_reg_shift);
13409 %}
13410 
13411 // This pattern is automatically generated from aarch64_ad.m4
13412 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13413 instruct cmovI_reg_reg_lt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
13414 %{
13415   effect(DEF dst, USE src1, USE src2, USE cr);
13416   ins_cost(INSN_COST * 2);
13417   format %{ "cselw $dst, $src1, $src2 lt\t"  %}
13418 
13419   ins_encode %{
13420     __ cselw($dst$$Register,
13421              $src1$$Register,
13422              $src2$$Register,
13423              Assembler::LT);
13424   %}
13425   ins_pipe(icond_reg_reg);
13426 %}
13427 
13428 // This pattern is automatically generated from aarch64_ad.m4
13429 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13430 instruct cmovI_reg_reg_gt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
13431 %{
13432   effect(DEF dst, USE src1, USE src2, USE cr);
13433   ins_cost(INSN_COST * 2);
13434   format %{ "cselw $dst, $src1, $src2 gt\t"  %}
13435 
13436   ins_encode %{
13437     __ cselw($dst$$Register,
13438              $src1$$Register,
13439              $src2$$Register,
13440              Assembler::GT);
13441   %}
13442   ins_pipe(icond_reg_reg);
13443 %}
13444 
13445 // This pattern is automatically generated from aarch64_ad.m4
13446 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13447 instruct cmovI_reg_imm0_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13448 %{
13449   effect(DEF dst, USE src1, USE cr);
13450   ins_cost(INSN_COST * 2);
13451   format %{ "cselw $dst, $src1, zr lt\t"  %}
13452 
13453   ins_encode %{
13454     __ cselw($dst$$Register,
13455              $src1$$Register,
13456              zr,
13457              Assembler::LT);
13458   %}
13459   ins_pipe(icond_reg);
13460 %}
13461 
13462 // This pattern is automatically generated from aarch64_ad.m4
13463 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13464 instruct cmovI_reg_imm0_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13465 %{
13466   effect(DEF dst, USE src1, USE cr);
13467   ins_cost(INSN_COST * 2);
13468   format %{ "cselw $dst, $src1, zr gt\t"  %}
13469 
13470   ins_encode %{
13471     __ cselw($dst$$Register,
13472              $src1$$Register,
13473              zr,
13474              Assembler::GT);
13475   %}
13476   ins_pipe(icond_reg);
13477 %}
13478 
13479 // This pattern is automatically generated from aarch64_ad.m4
13480 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13481 instruct cmovI_reg_imm1_le(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13482 %{
13483   effect(DEF dst, USE src1, USE cr);
13484   ins_cost(INSN_COST * 2);
13485   format %{ "csincw $dst, $src1, zr le\t"  %}
13486 
13487   ins_encode %{
13488     __ csincw($dst$$Register,
13489              $src1$$Register,
13490              zr,
13491              Assembler::LE);
13492   %}
13493   ins_pipe(icond_reg);
13494 %}
13495 
13496 // This pattern is automatically generated from aarch64_ad.m4
13497 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13498 instruct cmovI_reg_imm1_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13499 %{
13500   effect(DEF dst, USE src1, USE cr);
13501   ins_cost(INSN_COST * 2);
13502   format %{ "csincw $dst, $src1, zr gt\t"  %}
13503 
13504   ins_encode %{
13505     __ csincw($dst$$Register,
13506              $src1$$Register,
13507              zr,
13508              Assembler::GT);
13509   %}
13510   ins_pipe(icond_reg);
13511 %}
13512 
13513 // This pattern is automatically generated from aarch64_ad.m4
13514 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13515 instruct cmovI_reg_immM1_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13516 %{
13517   effect(DEF dst, USE src1, USE cr);
13518   ins_cost(INSN_COST * 2);
13519   format %{ "csinvw $dst, $src1, zr lt\t"  %}
13520 
13521   ins_encode %{
13522     __ csinvw($dst$$Register,
13523              $src1$$Register,
13524              zr,
13525              Assembler::LT);
13526   %}
13527   ins_pipe(icond_reg);
13528 %}
13529 
13530 // This pattern is automatically generated from aarch64_ad.m4
13531 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13532 instruct cmovI_reg_immM1_ge(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13533 %{
13534   effect(DEF dst, USE src1, USE cr);
13535   ins_cost(INSN_COST * 2);
13536   format %{ "csinvw $dst, $src1, zr ge\t"  %}
13537 
13538   ins_encode %{
13539     __ csinvw($dst$$Register,
13540              $src1$$Register,
13541              zr,
13542              Assembler::GE);
13543   %}
13544   ins_pipe(icond_reg);
13545 %}
13546 
13547 // This pattern is automatically generated from aarch64_ad.m4
13548 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13549 instruct minI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
13550 %{
13551   match(Set dst (MinI src imm));
13552   ins_cost(INSN_COST * 3);
13553   expand %{
13554     rFlagsReg cr;
13555     compI_reg_imm0(cr, src);
13556     cmovI_reg_imm0_lt(dst, src, cr);
13557   %}
13558 %}
13559 
13560 // This pattern is automatically generated from aarch64_ad.m4
13561 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13562 instruct minI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
13563 %{
13564   match(Set dst (MinI imm src));
13565   ins_cost(INSN_COST * 3);
13566   expand %{
13567     rFlagsReg cr;
13568     compI_reg_imm0(cr, src);
13569     cmovI_reg_imm0_lt(dst, src, cr);
13570   %}
13571 %}
13572 
13573 // This pattern is automatically generated from aarch64_ad.m4
13574 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13575 instruct minI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
13576 %{
13577   match(Set dst (MinI src imm));
13578   ins_cost(INSN_COST * 3);
13579   expand %{
13580     rFlagsReg cr;
13581     compI_reg_imm0(cr, src);
13582     cmovI_reg_imm1_le(dst, src, cr);
13583   %}
13584 %}
13585 
13586 // This pattern is automatically generated from aarch64_ad.m4
13587 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13588 instruct minI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
13589 %{
13590   match(Set dst (MinI imm src));
13591   ins_cost(INSN_COST * 3);
13592   expand %{
13593     rFlagsReg cr;
13594     compI_reg_imm0(cr, src);
13595     cmovI_reg_imm1_le(dst, src, cr);
13596   %}
13597 %}
13598 
13599 // This pattern is automatically generated from aarch64_ad.m4
13600 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13601 instruct minI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
13602 %{
13603   match(Set dst (MinI src imm));
13604   ins_cost(INSN_COST * 3);
13605   expand %{
13606     rFlagsReg cr;
13607     compI_reg_imm0(cr, src);
13608     cmovI_reg_immM1_lt(dst, src, cr);
13609   %}
13610 %}
13611 
13612 // This pattern is automatically generated from aarch64_ad.m4
13613 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13614 instruct minI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
13615 %{
13616   match(Set dst (MinI imm src));
13617   ins_cost(INSN_COST * 3);
13618   expand %{
13619     rFlagsReg cr;
13620     compI_reg_imm0(cr, src);
13621     cmovI_reg_immM1_lt(dst, src, cr);
13622   %}
13623 %}
13624 
13625 // This pattern is automatically generated from aarch64_ad.m4
13626 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13627 instruct maxI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
13628 %{
13629   match(Set dst (MaxI src imm));
13630   ins_cost(INSN_COST * 3);
13631   expand %{
13632     rFlagsReg cr;
13633     compI_reg_imm0(cr, src);
13634     cmovI_reg_imm0_gt(dst, src, cr);
13635   %}
13636 %}
13637 
13638 // This pattern is automatically generated from aarch64_ad.m4
13639 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13640 instruct maxI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
13641 %{
13642   match(Set dst (MaxI imm src));
13643   ins_cost(INSN_COST * 3);
13644   expand %{
13645     rFlagsReg cr;
13646     compI_reg_imm0(cr, src);
13647     cmovI_reg_imm0_gt(dst, src, cr);
13648   %}
13649 %}
13650 
13651 // This pattern is automatically generated from aarch64_ad.m4
13652 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13653 instruct maxI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
13654 %{
13655   match(Set dst (MaxI src imm));
13656   ins_cost(INSN_COST * 3);
13657   expand %{
13658     rFlagsReg cr;
13659     compI_reg_imm0(cr, src);
13660     cmovI_reg_imm1_gt(dst, src, cr);
13661   %}
13662 %}
13663 
13664 // This pattern is automatically generated from aarch64_ad.m4
13665 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13666 instruct maxI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
13667 %{
13668   match(Set dst (MaxI imm src));
13669   ins_cost(INSN_COST * 3);
13670   expand %{
13671     rFlagsReg cr;
13672     compI_reg_imm0(cr, src);
13673     cmovI_reg_imm1_gt(dst, src, cr);
13674   %}
13675 %}
13676 
13677 // This pattern is automatically generated from aarch64_ad.m4
13678 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13679 instruct maxI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
13680 %{
13681   match(Set dst (MaxI src imm));
13682   ins_cost(INSN_COST * 3);
13683   expand %{
13684     rFlagsReg cr;
13685     compI_reg_imm0(cr, src);
13686     cmovI_reg_immM1_ge(dst, src, cr);
13687   %}
13688 %}
13689 
13690 // This pattern is automatically generated from aarch64_ad.m4
13691 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13692 instruct maxI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
13693 %{
13694   match(Set dst (MaxI imm src));
13695   ins_cost(INSN_COST * 3);
13696   expand %{
13697     rFlagsReg cr;
13698     compI_reg_imm0(cr, src);
13699     cmovI_reg_immM1_ge(dst, src, cr);
13700   %}
13701 %}
13702 
13703 // This pattern is automatically generated from aarch64_ad.m4
13704 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13705 instruct bits_reverse_I(iRegINoSp dst, iRegIorL2I src)
13706 %{
13707   match(Set dst (ReverseI src));
13708   ins_cost(INSN_COST);
13709   format %{ "rbitw  $dst, $src" %}
13710   ins_encode %{
13711     __ rbitw($dst$$Register, $src$$Register);
13712   %}
13713   ins_pipe(ialu_reg);
13714 %}
13715 
13716 // This pattern is automatically generated from aarch64_ad.m4
13717 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13718 instruct bits_reverse_L(iRegLNoSp dst, iRegL src)
13719 %{
13720   match(Set dst (ReverseL src));
13721   ins_cost(INSN_COST);
13722   format %{ "rbit  $dst, $src" %}
13723   ins_encode %{
13724     __ rbit($dst$$Register, $src$$Register);
13725   %}
13726   ins_pipe(ialu_reg);
13727 %}
13728 
13729 
13730 // END This section of the file is automatically generated. Do not edit --------------
13731 
13732 
13733 // ============================================================================
13734 // Floating Point Arithmetic Instructions
13735 
13736 instruct addHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13737   match(Set dst (AddHF src1 src2));
13738   format %{ "faddh $dst, $src1, $src2" %}
13739   ins_encode %{
13740     __ faddh($dst$$FloatRegister,
13741              $src1$$FloatRegister,
13742              $src2$$FloatRegister);
13743   %}
13744   ins_pipe(fp_dop_reg_reg_s);
13745 %}
13746 
13747 instruct addF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13748   match(Set dst (AddF src1 src2));
13749 
13750   ins_cost(INSN_COST * 5);
13751   format %{ "fadds   $dst, $src1, $src2" %}
13752 
13753   ins_encode %{
13754     __ fadds(as_FloatRegister($dst$$reg),
13755              as_FloatRegister($src1$$reg),
13756              as_FloatRegister($src2$$reg));
13757   %}
13758 
13759   ins_pipe(fp_dop_reg_reg_s);
13760 %}
13761 
13762 instruct addD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13763   match(Set dst (AddD src1 src2));
13764 
13765   ins_cost(INSN_COST * 5);
13766   format %{ "faddd   $dst, $src1, $src2" %}
13767 
13768   ins_encode %{
13769     __ faddd(as_FloatRegister($dst$$reg),
13770              as_FloatRegister($src1$$reg),
13771              as_FloatRegister($src2$$reg));
13772   %}
13773 
13774   ins_pipe(fp_dop_reg_reg_d);
13775 %}
13776 
13777 instruct subHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13778   match(Set dst (SubHF src1 src2));
13779   format %{ "fsubh $dst, $src1, $src2" %}
13780   ins_encode %{
13781     __ fsubh($dst$$FloatRegister,
13782              $src1$$FloatRegister,
13783              $src2$$FloatRegister);
13784   %}
13785   ins_pipe(fp_dop_reg_reg_s);
13786 %}
13787 
13788 instruct subF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13789   match(Set dst (SubF src1 src2));
13790 
13791   ins_cost(INSN_COST * 5);
13792   format %{ "fsubs   $dst, $src1, $src2" %}
13793 
13794   ins_encode %{
13795     __ fsubs(as_FloatRegister($dst$$reg),
13796              as_FloatRegister($src1$$reg),
13797              as_FloatRegister($src2$$reg));
13798   %}
13799 
13800   ins_pipe(fp_dop_reg_reg_s);
13801 %}
13802 
13803 instruct subD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13804   match(Set dst (SubD src1 src2));
13805 
13806   ins_cost(INSN_COST * 5);
13807   format %{ "fsubd   $dst, $src1, $src2" %}
13808 
13809   ins_encode %{
13810     __ fsubd(as_FloatRegister($dst$$reg),
13811              as_FloatRegister($src1$$reg),
13812              as_FloatRegister($src2$$reg));
13813   %}
13814 
13815   ins_pipe(fp_dop_reg_reg_d);
13816 %}
13817 
13818 instruct mulHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13819   match(Set dst (MulHF src1 src2));
13820   format %{ "fmulh $dst, $src1, $src2" %}
13821   ins_encode %{
13822     __ fmulh($dst$$FloatRegister,
13823              $src1$$FloatRegister,
13824              $src2$$FloatRegister);
13825   %}
13826   ins_pipe(fp_dop_reg_reg_s);
13827 %}
13828 
13829 instruct mulF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13830   match(Set dst (MulF src1 src2));
13831 
13832   ins_cost(INSN_COST * 6);
13833   format %{ "fmuls   $dst, $src1, $src2" %}
13834 
13835   ins_encode %{
13836     __ fmuls(as_FloatRegister($dst$$reg),
13837              as_FloatRegister($src1$$reg),
13838              as_FloatRegister($src2$$reg));
13839   %}
13840 
13841   ins_pipe(fp_dop_reg_reg_s);
13842 %}
13843 
13844 instruct mulD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13845   match(Set dst (MulD src1 src2));
13846 
13847   ins_cost(INSN_COST * 6);
13848   format %{ "fmuld   $dst, $src1, $src2" %}
13849 
13850   ins_encode %{
13851     __ fmuld(as_FloatRegister($dst$$reg),
13852              as_FloatRegister($src1$$reg),
13853              as_FloatRegister($src2$$reg));
13854   %}
13855 
13856   ins_pipe(fp_dop_reg_reg_d);
13857 %}
13858 
13859 // src1 * src2 + src3 (half-precision float)
13860 instruct maddHF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13861   match(Set dst (FmaHF src3 (Binary src1 src2)));
13862   format %{ "fmaddh $dst, $src1, $src2, $src3" %}
13863   ins_encode %{
13864     assert(UseFMA, "Needs FMA instructions support.");
13865     __ fmaddh($dst$$FloatRegister,
13866               $src1$$FloatRegister,
13867               $src2$$FloatRegister,
13868               $src3$$FloatRegister);
13869   %}
13870   ins_pipe(pipe_class_default);
13871 %}
13872 
13873 // src1 * src2 + src3
13874 instruct maddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13875   match(Set dst (FmaF src3 (Binary src1 src2)));
13876 
13877   format %{ "fmadds   $dst, $src1, $src2, $src3" %}
13878 
13879   ins_encode %{
13880     assert(UseFMA, "Needs FMA instructions support.");
13881     __ fmadds(as_FloatRegister($dst$$reg),
13882              as_FloatRegister($src1$$reg),
13883              as_FloatRegister($src2$$reg),
13884              as_FloatRegister($src3$$reg));
13885   %}
13886 
13887   ins_pipe(pipe_class_default);
13888 %}
13889 
13890 // src1 * src2 + src3
13891 instruct maddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13892   match(Set dst (FmaD src3 (Binary src1 src2)));
13893 
13894   format %{ "fmaddd   $dst, $src1, $src2, $src3" %}
13895 
13896   ins_encode %{
13897     assert(UseFMA, "Needs FMA instructions support.");
13898     __ fmaddd(as_FloatRegister($dst$$reg),
13899              as_FloatRegister($src1$$reg),
13900              as_FloatRegister($src2$$reg),
13901              as_FloatRegister($src3$$reg));
13902   %}
13903 
13904   ins_pipe(pipe_class_default);
13905 %}
13906 
13907 // src1 * (-src2) + src3
13908 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
13909 instruct msubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13910   match(Set dst (FmaF src3 (Binary src1 (NegF src2))));
13911 
13912   format %{ "fmsubs   $dst, $src1, $src2, $src3" %}
13913 
13914   ins_encode %{
13915     assert(UseFMA, "Needs FMA instructions support.");
13916     __ fmsubs(as_FloatRegister($dst$$reg),
13917               as_FloatRegister($src1$$reg),
13918               as_FloatRegister($src2$$reg),
13919               as_FloatRegister($src3$$reg));
13920   %}
13921 
13922   ins_pipe(pipe_class_default);
13923 %}
13924 
13925 // src1 * (-src2) + src3
13926 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
13927 instruct msubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13928   match(Set dst (FmaD src3 (Binary src1 (NegD src2))));
13929 
13930   format %{ "fmsubd   $dst, $src1, $src2, $src3" %}
13931 
13932   ins_encode %{
13933     assert(UseFMA, "Needs FMA instructions support.");
13934     __ fmsubd(as_FloatRegister($dst$$reg),
13935               as_FloatRegister($src1$$reg),
13936               as_FloatRegister($src2$$reg),
13937               as_FloatRegister($src3$$reg));
13938   %}
13939 
13940   ins_pipe(pipe_class_default);
13941 %}
13942 
13943 // src1 * (-src2) - src3
13944 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
13945 instruct mnaddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13946   match(Set dst (FmaF (NegF src3) (Binary src1 (NegF src2))));
13947 
13948   format %{ "fnmadds  $dst, $src1, $src2, $src3" %}
13949 
13950   ins_encode %{
13951     assert(UseFMA, "Needs FMA instructions support.");
13952     __ fnmadds(as_FloatRegister($dst$$reg),
13953                as_FloatRegister($src1$$reg),
13954                as_FloatRegister($src2$$reg),
13955                as_FloatRegister($src3$$reg));
13956   %}
13957 
13958   ins_pipe(pipe_class_default);
13959 %}
13960 
13961 // src1 * (-src2) - src3
13962 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
13963 instruct mnaddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13964   match(Set dst (FmaD (NegD src3) (Binary src1 (NegD src2))));
13965 
13966   format %{ "fnmaddd   $dst, $src1, $src2, $src3" %}
13967 
13968   ins_encode %{
13969     assert(UseFMA, "Needs FMA instructions support.");
13970     __ fnmaddd(as_FloatRegister($dst$$reg),
13971                as_FloatRegister($src1$$reg),
13972                as_FloatRegister($src2$$reg),
13973                as_FloatRegister($src3$$reg));
13974   %}
13975 
13976   ins_pipe(pipe_class_default);
13977 %}
13978 
13979 // src1 * src2 - src3
13980 instruct mnsubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3, immF0 zero) %{
13981   match(Set dst (FmaF (NegF src3) (Binary src1 src2)));
13982 
13983   format %{ "fnmsubs  $dst, $src1, $src2, $src3" %}
13984 
13985   ins_encode %{
13986     assert(UseFMA, "Needs FMA instructions support.");
13987     __ fnmsubs(as_FloatRegister($dst$$reg),
13988                as_FloatRegister($src1$$reg),
13989                as_FloatRegister($src2$$reg),
13990                as_FloatRegister($src3$$reg));
13991   %}
13992 
13993   ins_pipe(pipe_class_default);
13994 %}
13995 
13996 // src1 * src2 - src3
13997 instruct mnsubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3, immD0 zero) %{
13998   match(Set dst (FmaD (NegD src3) (Binary src1 src2)));
13999 
14000   format %{ "fnmsubd   $dst, $src1, $src2, $src3" %}
14001 
14002   ins_encode %{
14003     assert(UseFMA, "Needs FMA instructions support.");
14004     // n.b. insn name should be fnmsubd
14005     __ fnmsub(as_FloatRegister($dst$$reg),
14006               as_FloatRegister($src1$$reg),
14007               as_FloatRegister($src2$$reg),
14008               as_FloatRegister($src3$$reg));
14009   %}
14010 
14011   ins_pipe(pipe_class_default);
14012 %}
14013 
14014 // Math.max(HH)H (half-precision float)
14015 instruct maxHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14016   match(Set dst (MaxHF src1 src2));
14017   format %{ "fmaxh $dst, $src1, $src2" %}
14018   ins_encode %{
14019     __ fmaxh($dst$$FloatRegister,
14020              $src1$$FloatRegister,
14021              $src2$$FloatRegister);
14022   %}
14023   ins_pipe(fp_dop_reg_reg_s);
14024 %}
14025 
14026 // Math.min(HH)H (half-precision float)
14027 instruct minHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14028   match(Set dst (MinHF src1 src2));
14029   format %{ "fminh $dst, $src1, $src2" %}
14030   ins_encode %{
14031     __ fminh($dst$$FloatRegister,
14032              $src1$$FloatRegister,
14033              $src2$$FloatRegister);
14034   %}
14035   ins_pipe(fp_dop_reg_reg_s);
14036 %}
14037 
14038 // Math.max(FF)F
14039 instruct maxF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14040   match(Set dst (MaxF src1 src2));
14041 
14042   format %{ "fmaxs   $dst, $src1, $src2" %}
14043   ins_encode %{
14044     __ fmaxs(as_FloatRegister($dst$$reg),
14045              as_FloatRegister($src1$$reg),
14046              as_FloatRegister($src2$$reg));
14047   %}
14048 
14049   ins_pipe(fp_dop_reg_reg_s);
14050 %}
14051 
14052 // Math.min(FF)F
14053 instruct minF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14054   match(Set dst (MinF src1 src2));
14055 
14056   format %{ "fmins   $dst, $src1, $src2" %}
14057   ins_encode %{
14058     __ fmins(as_FloatRegister($dst$$reg),
14059              as_FloatRegister($src1$$reg),
14060              as_FloatRegister($src2$$reg));
14061   %}
14062 
14063   ins_pipe(fp_dop_reg_reg_s);
14064 %}
14065 
14066 // Math.max(DD)D
14067 instruct maxD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14068   match(Set dst (MaxD src1 src2));
14069 
14070   format %{ "fmaxd   $dst, $src1, $src2" %}
14071   ins_encode %{
14072     __ fmaxd(as_FloatRegister($dst$$reg),
14073              as_FloatRegister($src1$$reg),
14074              as_FloatRegister($src2$$reg));
14075   %}
14076 
14077   ins_pipe(fp_dop_reg_reg_d);
14078 %}
14079 
14080 // Math.min(DD)D
14081 instruct minD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14082   match(Set dst (MinD src1 src2));
14083 
14084   format %{ "fmind   $dst, $src1, $src2" %}
14085   ins_encode %{
14086     __ fmind(as_FloatRegister($dst$$reg),
14087              as_FloatRegister($src1$$reg),
14088              as_FloatRegister($src2$$reg));
14089   %}
14090 
14091   ins_pipe(fp_dop_reg_reg_d);
14092 %}
14093 
14094 instruct divHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14095   match(Set dst (DivHF src1  src2));
14096   format %{ "fdivh $dst, $src1, $src2" %}
14097   ins_encode %{
14098     __ fdivh($dst$$FloatRegister,
14099              $src1$$FloatRegister,
14100              $src2$$FloatRegister);
14101   %}
14102   ins_pipe(fp_div_s);
14103 %}
14104 
14105 instruct divF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14106   match(Set dst (DivF src1  src2));
14107 
14108   ins_cost(INSN_COST * 18);
14109   format %{ "fdivs   $dst, $src1, $src2" %}
14110 
14111   ins_encode %{
14112     __ fdivs(as_FloatRegister($dst$$reg),
14113              as_FloatRegister($src1$$reg),
14114              as_FloatRegister($src2$$reg));
14115   %}
14116 
14117   ins_pipe(fp_div_s);
14118 %}
14119 
14120 instruct divD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14121   match(Set dst (DivD src1  src2));
14122 
14123   ins_cost(INSN_COST * 32);
14124   format %{ "fdivd   $dst, $src1, $src2" %}
14125 
14126   ins_encode %{
14127     __ fdivd(as_FloatRegister($dst$$reg),
14128              as_FloatRegister($src1$$reg),
14129              as_FloatRegister($src2$$reg));
14130   %}
14131 
14132   ins_pipe(fp_div_d);
14133 %}
14134 
14135 instruct negF_reg_reg(vRegF dst, vRegF src) %{
14136   match(Set dst (NegF src));
14137 
14138   ins_cost(INSN_COST * 3);
14139   format %{ "fneg   $dst, $src" %}
14140 
14141   ins_encode %{
14142     __ fnegs(as_FloatRegister($dst$$reg),
14143              as_FloatRegister($src$$reg));
14144   %}
14145 
14146   ins_pipe(fp_uop_s);
14147 %}
14148 
14149 instruct negD_reg_reg(vRegD dst, vRegD src) %{
14150   match(Set dst (NegD src));
14151 
14152   ins_cost(INSN_COST * 3);
14153   format %{ "fnegd   $dst, $src" %}
14154 
14155   ins_encode %{
14156     __ fnegd(as_FloatRegister($dst$$reg),
14157              as_FloatRegister($src$$reg));
14158   %}
14159 
14160   ins_pipe(fp_uop_d);
14161 %}
14162 
14163 instruct absI_reg(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
14164 %{
14165   match(Set dst (AbsI src));
14166 
14167   effect(KILL cr);
14168   ins_cost(INSN_COST * 2);
14169   format %{ "cmpw  $src, zr\n\t"
14170             "cnegw $dst, $src, Assembler::LT\t# int abs"
14171   %}
14172 
14173   ins_encode %{
14174     __ cmpw(as_Register($src$$reg), zr);
14175     __ cnegw(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14176   %}
14177   ins_pipe(pipe_class_default);
14178 %}
14179 
14180 instruct absL_reg(iRegLNoSp dst, iRegL src, rFlagsReg cr)
14181 %{
14182   match(Set dst (AbsL src));
14183 
14184   effect(KILL cr);
14185   ins_cost(INSN_COST * 2);
14186   format %{ "cmp  $src, zr\n\t"
14187             "cneg $dst, $src, Assembler::LT\t# long abs"
14188   %}
14189 
14190   ins_encode %{
14191     __ cmp(as_Register($src$$reg), zr);
14192     __ cneg(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14193   %}
14194   ins_pipe(pipe_class_default);
14195 %}
14196 
14197 instruct absF_reg(vRegF dst, vRegF src) %{
14198   match(Set dst (AbsF src));
14199 
14200   ins_cost(INSN_COST * 3);
14201   format %{ "fabss   $dst, $src" %}
14202   ins_encode %{
14203     __ fabss(as_FloatRegister($dst$$reg),
14204              as_FloatRegister($src$$reg));
14205   %}
14206 
14207   ins_pipe(fp_uop_s);
14208 %}
14209 
14210 instruct absD_reg(vRegD dst, vRegD src) %{
14211   match(Set dst (AbsD src));
14212 
14213   ins_cost(INSN_COST * 3);
14214   format %{ "fabsd   $dst, $src" %}
14215   ins_encode %{
14216     __ fabsd(as_FloatRegister($dst$$reg),
14217              as_FloatRegister($src$$reg));
14218   %}
14219 
14220   ins_pipe(fp_uop_d);
14221 %}
14222 
14223 instruct absdF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14224   match(Set dst (AbsF (SubF src1 src2)));
14225 
14226   ins_cost(INSN_COST * 3);
14227   format %{ "fabds   $dst, $src1, $src2" %}
14228   ins_encode %{
14229     __ fabds(as_FloatRegister($dst$$reg),
14230              as_FloatRegister($src1$$reg),
14231              as_FloatRegister($src2$$reg));
14232   %}
14233 
14234   ins_pipe(fp_uop_s);
14235 %}
14236 
14237 instruct absdD_reg(vRegD dst, vRegD src1, vRegD src2) %{
14238   match(Set dst (AbsD (SubD src1 src2)));
14239 
14240   ins_cost(INSN_COST * 3);
14241   format %{ "fabdd   $dst, $src1, $src2" %}
14242   ins_encode %{
14243     __ fabdd(as_FloatRegister($dst$$reg),
14244              as_FloatRegister($src1$$reg),
14245              as_FloatRegister($src2$$reg));
14246   %}
14247 
14248   ins_pipe(fp_uop_d);
14249 %}
14250 
14251 instruct sqrtD_reg(vRegD dst, vRegD src) %{
14252   match(Set dst (SqrtD src));
14253 
14254   ins_cost(INSN_COST * 50);
14255   format %{ "fsqrtd  $dst, $src" %}
14256   ins_encode %{
14257     __ fsqrtd(as_FloatRegister($dst$$reg),
14258              as_FloatRegister($src$$reg));
14259   %}
14260 
14261   ins_pipe(fp_div_s);
14262 %}
14263 
14264 instruct sqrtF_reg(vRegF dst, vRegF src) %{
14265   match(Set dst (SqrtF src));
14266 
14267   ins_cost(INSN_COST * 50);
14268   format %{ "fsqrts  $dst, $src" %}
14269   ins_encode %{
14270     __ fsqrts(as_FloatRegister($dst$$reg),
14271              as_FloatRegister($src$$reg));
14272   %}
14273 
14274   ins_pipe(fp_div_d);
14275 %}
14276 
14277 instruct sqrtHF_reg(vRegF dst, vRegF src) %{
14278   match(Set dst (SqrtHF src));
14279   format %{ "fsqrth $dst, $src" %}
14280   ins_encode %{
14281     __ fsqrth($dst$$FloatRegister,
14282               $src$$FloatRegister);
14283   %}
14284   ins_pipe(fp_div_s);
14285 %}
14286 
14287 // Math.rint, floor, ceil
14288 instruct roundD_reg(vRegD dst, vRegD src, immI rmode) %{
14289   match(Set dst (RoundDoubleMode src rmode));
14290   format %{ "frint  $dst, $src, $rmode" %}
14291   ins_encode %{
14292     switch ($rmode$$constant) {
14293       case RoundDoubleModeNode::rmode_rint:
14294         __ frintnd(as_FloatRegister($dst$$reg),
14295                    as_FloatRegister($src$$reg));
14296         break;
14297       case RoundDoubleModeNode::rmode_floor:
14298         __ frintmd(as_FloatRegister($dst$$reg),
14299                    as_FloatRegister($src$$reg));
14300         break;
14301       case RoundDoubleModeNode::rmode_ceil:
14302         __ frintpd(as_FloatRegister($dst$$reg),
14303                    as_FloatRegister($src$$reg));
14304         break;
14305     }
14306   %}
14307   ins_pipe(fp_uop_d);
14308 %}
14309 
14310 instruct copySignD_reg(vRegD dst, vRegD src1, vRegD src2, vRegD zero) %{
14311   match(Set dst (CopySignD src1 (Binary src2 zero)));
14312   effect(TEMP_DEF dst, USE src1, USE src2, USE zero);
14313   format %{ "CopySignD  $dst $src1 $src2" %}
14314   ins_encode %{
14315     FloatRegister dst = as_FloatRegister($dst$$reg),
14316                   src1 = as_FloatRegister($src1$$reg),
14317                   src2 = as_FloatRegister($src2$$reg),
14318                   zero = as_FloatRegister($zero$$reg);
14319     __ fnegd(dst, zero);
14320     __ bsl(dst, __ T8B, src2, src1);
14321   %}
14322   ins_pipe(fp_uop_d);
14323 %}
14324 
14325 instruct copySignF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14326   match(Set dst (CopySignF src1 src2));
14327   effect(TEMP_DEF dst, USE src1, USE src2);
14328   format %{ "CopySignF  $dst $src1 $src2" %}
14329   ins_encode %{
14330     FloatRegister dst = as_FloatRegister($dst$$reg),
14331                   src1 = as_FloatRegister($src1$$reg),
14332                   src2 = as_FloatRegister($src2$$reg);
14333     __ movi(dst, __ T2S, 0x80, 24);
14334     __ bsl(dst, __ T8B, src2, src1);
14335   %}
14336   ins_pipe(fp_uop_d);
14337 %}
14338 
14339 instruct signumD_reg(vRegD dst, vRegD src, vRegD zero, vRegD one) %{
14340   match(Set dst (SignumD src (Binary zero one)));
14341   effect(TEMP_DEF dst, USE src, USE zero, USE one);
14342   format %{ "signumD  $dst, $src" %}
14343   ins_encode %{
14344     FloatRegister src = as_FloatRegister($src$$reg),
14345                   dst = as_FloatRegister($dst$$reg),
14346                   zero = as_FloatRegister($zero$$reg),
14347                   one = as_FloatRegister($one$$reg);
14348     __ facgtd(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14349     __ ushrd(dst, dst, 1);     // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14350     // Bit selection instruction gets bit from "one" for each enabled bit in
14351     // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14352     // NaN the whole "src" will be copied because "dst" is zero. For all other
14353     // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14354     // from "src", and all other bits are copied from 1.0.
14355     __ bsl(dst, __ T8B, one, src);
14356   %}
14357   ins_pipe(fp_uop_d);
14358 %}
14359 
14360 instruct signumF_reg(vRegF dst, vRegF src, vRegF zero, vRegF one) %{
14361   match(Set dst (SignumF src (Binary zero one)));
14362   effect(TEMP_DEF dst, USE src, USE zero, USE one);
14363   format %{ "signumF  $dst, $src" %}
14364   ins_encode %{
14365     FloatRegister src = as_FloatRegister($src$$reg),
14366                   dst = as_FloatRegister($dst$$reg),
14367                   zero = as_FloatRegister($zero$$reg),
14368                   one = as_FloatRegister($one$$reg);
14369     __ facgts(dst, src, zero);    // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14370     __ ushr(dst, __ T2S, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14371     // Bit selection instruction gets bit from "one" for each enabled bit in
14372     // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14373     // NaN the whole "src" will be copied because "dst" is zero. For all other
14374     // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14375     // from "src", and all other bits are copied from 1.0.
14376     __ bsl(dst, __ T8B, one, src);
14377   %}
14378   ins_pipe(fp_uop_d);
14379 %}
14380 
14381 instruct onspinwait() %{
14382   match(OnSpinWait);
14383   ins_cost(INSN_COST);
14384 
14385   format %{ "onspinwait" %}
14386 
14387   ins_encode %{
14388     __ spin_wait();
14389   %}
14390   ins_pipe(pipe_class_empty);
14391 %}
14392 
14393 // ============================================================================
14394 // Logical Instructions
14395 
14396 // Integer Logical Instructions
14397 
14398 // And Instructions
14399 
14400 
14401 instruct andI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, rFlagsReg cr) %{
14402   match(Set dst (AndI src1 src2));
14403 
14404   format %{ "andw  $dst, $src1, $src2\t# int" %}
14405 
14406   ins_cost(INSN_COST);
14407   ins_encode %{
14408     __ andw(as_Register($dst$$reg),
14409             as_Register($src1$$reg),
14410             as_Register($src2$$reg));
14411   %}
14412 
14413   ins_pipe(ialu_reg_reg);
14414 %}
14415 
14416 instruct andI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2, rFlagsReg cr) %{
14417   match(Set dst (AndI src1 src2));
14418 
14419   format %{ "andsw  $dst, $src1, $src2\t# int" %}
14420 
14421   ins_cost(INSN_COST);
14422   ins_encode %{
14423     __ andw(as_Register($dst$$reg),
14424             as_Register($src1$$reg),
14425             (uint64_t)($src2$$constant));
14426   %}
14427 
14428   ins_pipe(ialu_reg_imm);
14429 %}
14430 
14431 // Or Instructions
14432 
14433 instruct orI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14434   match(Set dst (OrI src1 src2));
14435 
14436   format %{ "orrw  $dst, $src1, $src2\t# int" %}
14437 
14438   ins_cost(INSN_COST);
14439   ins_encode %{
14440     __ orrw(as_Register($dst$$reg),
14441             as_Register($src1$$reg),
14442             as_Register($src2$$reg));
14443   %}
14444 
14445   ins_pipe(ialu_reg_reg);
14446 %}
14447 
14448 instruct orI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14449   match(Set dst (OrI src1 src2));
14450 
14451   format %{ "orrw  $dst, $src1, $src2\t# int" %}
14452 
14453   ins_cost(INSN_COST);
14454   ins_encode %{
14455     __ orrw(as_Register($dst$$reg),
14456             as_Register($src1$$reg),
14457             (uint64_t)($src2$$constant));
14458   %}
14459 
14460   ins_pipe(ialu_reg_imm);
14461 %}
14462 
14463 // Xor Instructions
14464 
14465 instruct xorI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14466   match(Set dst (XorI src1 src2));
14467 
14468   format %{ "eorw  $dst, $src1, $src2\t# int" %}
14469 
14470   ins_cost(INSN_COST);
14471   ins_encode %{
14472     __ eorw(as_Register($dst$$reg),
14473             as_Register($src1$$reg),
14474             as_Register($src2$$reg));
14475   %}
14476 
14477   ins_pipe(ialu_reg_reg);
14478 %}
14479 
14480 instruct xorI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14481   match(Set dst (XorI src1 src2));
14482 
14483   format %{ "eorw  $dst, $src1, $src2\t# int" %}
14484 
14485   ins_cost(INSN_COST);
14486   ins_encode %{
14487     __ eorw(as_Register($dst$$reg),
14488             as_Register($src1$$reg),
14489             (uint64_t)($src2$$constant));
14490   %}
14491 
14492   ins_pipe(ialu_reg_imm);
14493 %}
14494 
14495 // Long Logical Instructions
14496 // TODO
14497 
14498 instruct andL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr) %{
14499   match(Set dst (AndL src1 src2));
14500 
14501   format %{ "and  $dst, $src1, $src2\t# int" %}
14502 
14503   ins_cost(INSN_COST);
14504   ins_encode %{
14505     __ andr(as_Register($dst$$reg),
14506             as_Register($src1$$reg),
14507             as_Register($src2$$reg));
14508   %}
14509 
14510   ins_pipe(ialu_reg_reg);
14511 %}
14512 
14513 instruct andL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2, rFlagsReg cr) %{
14514   match(Set dst (AndL src1 src2));
14515 
14516   format %{ "and  $dst, $src1, $src2\t# int" %}
14517 
14518   ins_cost(INSN_COST);
14519   ins_encode %{
14520     __ andr(as_Register($dst$$reg),
14521             as_Register($src1$$reg),
14522             (uint64_t)($src2$$constant));
14523   %}
14524 
14525   ins_pipe(ialu_reg_imm);
14526 %}
14527 
14528 // Or Instructions
14529 
14530 instruct orL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14531   match(Set dst (OrL src1 src2));
14532 
14533   format %{ "orr  $dst, $src1, $src2\t# int" %}
14534 
14535   ins_cost(INSN_COST);
14536   ins_encode %{
14537     __ orr(as_Register($dst$$reg),
14538            as_Register($src1$$reg),
14539            as_Register($src2$$reg));
14540   %}
14541 
14542   ins_pipe(ialu_reg_reg);
14543 %}
14544 
14545 instruct orL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14546   match(Set dst (OrL src1 src2));
14547 
14548   format %{ "orr  $dst, $src1, $src2\t# int" %}
14549 
14550   ins_cost(INSN_COST);
14551   ins_encode %{
14552     __ orr(as_Register($dst$$reg),
14553            as_Register($src1$$reg),
14554            (uint64_t)($src2$$constant));
14555   %}
14556 
14557   ins_pipe(ialu_reg_imm);
14558 %}
14559 
14560 // Xor Instructions
14561 
14562 instruct xorL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14563   match(Set dst (XorL src1 src2));
14564 
14565   format %{ "eor  $dst, $src1, $src2\t# int" %}
14566 
14567   ins_cost(INSN_COST);
14568   ins_encode %{
14569     __ eor(as_Register($dst$$reg),
14570            as_Register($src1$$reg),
14571            as_Register($src2$$reg));
14572   %}
14573 
14574   ins_pipe(ialu_reg_reg);
14575 %}
14576 
14577 instruct xorL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14578   match(Set dst (XorL src1 src2));
14579 
14580   ins_cost(INSN_COST);
14581   format %{ "eor  $dst, $src1, $src2\t# int" %}
14582 
14583   ins_encode %{
14584     __ eor(as_Register($dst$$reg),
14585            as_Register($src1$$reg),
14586            (uint64_t)($src2$$constant));
14587   %}
14588 
14589   ins_pipe(ialu_reg_imm);
14590 %}
14591 
14592 instruct convI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src)
14593 %{
14594   match(Set dst (ConvI2L src));
14595 
14596   ins_cost(INSN_COST);
14597   format %{ "sxtw  $dst, $src\t# i2l" %}
14598   ins_encode %{
14599     __ sbfm($dst$$Register, $src$$Register, 0, 31);
14600   %}
14601   ins_pipe(ialu_reg_shift);
14602 %}
14603 
14604 // this pattern occurs in bigmath arithmetic
14605 instruct convUI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src, immL_32bits mask)
14606 %{
14607   match(Set dst (AndL (ConvI2L src) mask));
14608 
14609   ins_cost(INSN_COST);
14610   format %{ "ubfm  $dst, $src, 0, 31\t# ui2l" %}
14611   ins_encode %{
14612     __ ubfm($dst$$Register, $src$$Register, 0, 31);
14613   %}
14614 
14615   ins_pipe(ialu_reg_shift);
14616 %}
14617 
14618 instruct convL2I_reg(iRegINoSp dst, iRegL src) %{
14619   match(Set dst (ConvL2I src));
14620 
14621   ins_cost(INSN_COST);
14622   format %{ "movw  $dst, $src \t// l2i" %}
14623 
14624   ins_encode %{
14625     __ movw(as_Register($dst$$reg), as_Register($src$$reg));
14626   %}
14627 
14628   ins_pipe(ialu_reg);
14629 %}
14630 
14631 instruct convD2F_reg(vRegF dst, vRegD src) %{
14632   match(Set dst (ConvD2F src));
14633 
14634   ins_cost(INSN_COST * 5);
14635   format %{ "fcvtd  $dst, $src \t// d2f" %}
14636 
14637   ins_encode %{
14638     __ fcvtd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14639   %}
14640 
14641   ins_pipe(fp_d2f);
14642 %}
14643 
14644 instruct convF2D_reg(vRegD dst, vRegF src) %{
14645   match(Set dst (ConvF2D src));
14646 
14647   ins_cost(INSN_COST * 5);
14648   format %{ "fcvts  $dst, $src \t// f2d" %}
14649 
14650   ins_encode %{
14651     __ fcvts(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14652   %}
14653 
14654   ins_pipe(fp_f2d);
14655 %}
14656 
14657 instruct convF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14658   match(Set dst (ConvF2I src));
14659 
14660   ins_cost(INSN_COST * 5);
14661   format %{ "fcvtzsw  $dst, $src \t// f2i" %}
14662 
14663   ins_encode %{
14664     __ fcvtzsw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14665   %}
14666 
14667   ins_pipe(fp_f2i);
14668 %}
14669 
14670 instruct convF2L_reg_reg(iRegLNoSp dst, vRegF src) %{
14671   match(Set dst (ConvF2L src));
14672 
14673   ins_cost(INSN_COST * 5);
14674   format %{ "fcvtzs  $dst, $src \t// f2l" %}
14675 
14676   ins_encode %{
14677     __ fcvtzs(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14678   %}
14679 
14680   ins_pipe(fp_f2l);
14681 %}
14682 
14683 instruct convF2HF_reg_reg(iRegINoSp dst, vRegF src, vRegF tmp) %{
14684   match(Set dst (ConvF2HF src));
14685   format %{ "fcvt $tmp, $src\t# convert single to half precision\n\t"
14686             "smov $dst, $tmp\t# move result from $tmp to $dst"
14687   %}
14688   effect(TEMP tmp);
14689   ins_encode %{
14690       __ flt_to_flt16($dst$$Register, $src$$FloatRegister, $tmp$$FloatRegister);
14691   %}
14692   ins_pipe(pipe_slow);
14693 %}
14694 
14695 instruct convHF2F_reg_reg(vRegF dst, iRegINoSp src, vRegF tmp) %{
14696   match(Set dst (ConvHF2F src));
14697   format %{ "mov $tmp, $src\t# move source from $src to $tmp\n\t"
14698             "fcvt $dst, $tmp\t# convert half to single precision"
14699   %}
14700   effect(TEMP tmp);
14701   ins_encode %{
14702       __ flt16_to_flt($dst$$FloatRegister, $src$$Register, $tmp$$FloatRegister);
14703   %}
14704   ins_pipe(pipe_slow);
14705 %}
14706 
14707 instruct convI2F_reg_reg(vRegF dst, iRegIorL2I src) %{
14708   match(Set dst (ConvI2F src));
14709 
14710   ins_cost(INSN_COST * 5);
14711   format %{ "scvtfws  $dst, $src \t// i2f" %}
14712 
14713   ins_encode %{
14714     __ scvtfws(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14715   %}
14716 
14717   ins_pipe(fp_i2f);
14718 %}
14719 
14720 instruct convL2F_reg_reg(vRegF dst, iRegL src) %{
14721   match(Set dst (ConvL2F src));
14722 
14723   ins_cost(INSN_COST * 5);
14724   format %{ "scvtfs  $dst, $src \t// l2f" %}
14725 
14726   ins_encode %{
14727     __ scvtfs(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14728   %}
14729 
14730   ins_pipe(fp_l2f);
14731 %}
14732 
14733 instruct convD2I_reg_reg(iRegINoSp dst, vRegD src) %{
14734   match(Set dst (ConvD2I src));
14735 
14736   ins_cost(INSN_COST * 5);
14737   format %{ "fcvtzdw  $dst, $src \t// d2i" %}
14738 
14739   ins_encode %{
14740     __ fcvtzdw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14741   %}
14742 
14743   ins_pipe(fp_d2i);
14744 %}
14745 
14746 instruct convD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14747   match(Set dst (ConvD2L src));
14748 
14749   ins_cost(INSN_COST * 5);
14750   format %{ "fcvtzd  $dst, $src \t// d2l" %}
14751 
14752   ins_encode %{
14753     __ fcvtzd(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14754   %}
14755 
14756   ins_pipe(fp_d2l);
14757 %}
14758 
14759 instruct convI2D_reg_reg(vRegD dst, iRegIorL2I src) %{
14760   match(Set dst (ConvI2D src));
14761 
14762   ins_cost(INSN_COST * 5);
14763   format %{ "scvtfwd  $dst, $src \t// i2d" %}
14764 
14765   ins_encode %{
14766     __ scvtfwd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14767   %}
14768 
14769   ins_pipe(fp_i2d);
14770 %}
14771 
14772 instruct convL2D_reg_reg(vRegD dst, iRegL src) %{
14773   match(Set dst (ConvL2D src));
14774 
14775   ins_cost(INSN_COST * 5);
14776   format %{ "scvtfd  $dst, $src \t// l2d" %}
14777 
14778   ins_encode %{
14779     __ scvtfd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14780   %}
14781 
14782   ins_pipe(fp_l2d);
14783 %}
14784 
14785 instruct round_double_reg(iRegLNoSp dst, vRegD src, vRegD ftmp, rFlagsReg cr)
14786 %{
14787   match(Set dst (RoundD src));
14788   effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
14789   format %{ "java_round_double $dst,$src"%}
14790   ins_encode %{
14791     __ java_round_double($dst$$Register, as_FloatRegister($src$$reg),
14792                          as_FloatRegister($ftmp$$reg));
14793   %}
14794   ins_pipe(pipe_slow);
14795 %}
14796 
14797 instruct round_float_reg(iRegINoSp dst, vRegF src, vRegF ftmp, rFlagsReg cr)
14798 %{
14799   match(Set dst (RoundF src));
14800   effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
14801   format %{ "java_round_float $dst,$src"%}
14802   ins_encode %{
14803     __ java_round_float($dst$$Register, as_FloatRegister($src$$reg),
14804                         as_FloatRegister($ftmp$$reg));
14805   %}
14806   ins_pipe(pipe_slow);
14807 %}
14808 
14809 // stack <-> reg and reg <-> reg shuffles with no conversion
14810 
14811 instruct MoveF2I_stack_reg(iRegINoSp dst, stackSlotF src) %{
14812 
14813   match(Set dst (MoveF2I src));
14814 
14815   effect(DEF dst, USE src);
14816 
14817   ins_cost(4 * INSN_COST);
14818 
14819   format %{ "ldrw $dst, $src\t# MoveF2I_stack_reg" %}
14820 
14821   ins_encode %{
14822     __ ldrw($dst$$Register, Address(sp, $src$$disp));
14823   %}
14824 
14825   ins_pipe(iload_reg_reg);
14826 
14827 %}
14828 
14829 instruct MoveI2F_stack_reg(vRegF dst, stackSlotI src) %{
14830 
14831   match(Set dst (MoveI2F src));
14832 
14833   effect(DEF dst, USE src);
14834 
14835   ins_cost(4 * INSN_COST);
14836 
14837   format %{ "ldrs $dst, $src\t# MoveI2F_stack_reg" %}
14838 
14839   ins_encode %{
14840     __ ldrs(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
14841   %}
14842 
14843   ins_pipe(pipe_class_memory);
14844 
14845 %}
14846 
14847 instruct MoveD2L_stack_reg(iRegLNoSp dst, stackSlotD src) %{
14848 
14849   match(Set dst (MoveD2L src));
14850 
14851   effect(DEF dst, USE src);
14852 
14853   ins_cost(4 * INSN_COST);
14854 
14855   format %{ "ldr $dst, $src\t# MoveD2L_stack_reg" %}
14856 
14857   ins_encode %{
14858     __ ldr($dst$$Register, Address(sp, $src$$disp));
14859   %}
14860 
14861   ins_pipe(iload_reg_reg);
14862 
14863 %}
14864 
14865 instruct MoveL2D_stack_reg(vRegD dst, stackSlotL src) %{
14866 
14867   match(Set dst (MoveL2D src));
14868 
14869   effect(DEF dst, USE src);
14870 
14871   ins_cost(4 * INSN_COST);
14872 
14873   format %{ "ldrd $dst, $src\t# MoveL2D_stack_reg" %}
14874 
14875   ins_encode %{
14876     __ ldrd(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
14877   %}
14878 
14879   ins_pipe(pipe_class_memory);
14880 
14881 %}
14882 
14883 instruct MoveF2I_reg_stack(stackSlotI dst, vRegF src) %{
14884 
14885   match(Set dst (MoveF2I src));
14886 
14887   effect(DEF dst, USE src);
14888 
14889   ins_cost(INSN_COST);
14890 
14891   format %{ "strs $src, $dst\t# MoveF2I_reg_stack" %}
14892 
14893   ins_encode %{
14894     __ strs(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
14895   %}
14896 
14897   ins_pipe(pipe_class_memory);
14898 
14899 %}
14900 
14901 instruct MoveI2F_reg_stack(stackSlotF dst, iRegI src) %{
14902 
14903   match(Set dst (MoveI2F src));
14904 
14905   effect(DEF dst, USE src);
14906 
14907   ins_cost(INSN_COST);
14908 
14909   format %{ "strw $src, $dst\t# MoveI2F_reg_stack" %}
14910 
14911   ins_encode %{
14912     __ strw($src$$Register, Address(sp, $dst$$disp));
14913   %}
14914 
14915   ins_pipe(istore_reg_reg);
14916 
14917 %}
14918 
14919 instruct MoveD2L_reg_stack(stackSlotL dst, vRegD src) %{
14920 
14921   match(Set dst (MoveD2L src));
14922 
14923   effect(DEF dst, USE src);
14924 
14925   ins_cost(INSN_COST);
14926 
14927   format %{ "strd $dst, $src\t# MoveD2L_reg_stack" %}
14928 
14929   ins_encode %{
14930     __ strd(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
14931   %}
14932 
14933   ins_pipe(pipe_class_memory);
14934 
14935 %}
14936 
14937 instruct MoveL2D_reg_stack(stackSlotD dst, iRegL src) %{
14938 
14939   match(Set dst (MoveL2D src));
14940 
14941   effect(DEF dst, USE src);
14942 
14943   ins_cost(INSN_COST);
14944 
14945   format %{ "str $src, $dst\t# MoveL2D_reg_stack" %}
14946 
14947   ins_encode %{
14948     __ str($src$$Register, Address(sp, $dst$$disp));
14949   %}
14950 
14951   ins_pipe(istore_reg_reg);
14952 
14953 %}
14954 
14955 instruct MoveF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14956 
14957   match(Set dst (MoveF2I src));
14958 
14959   effect(DEF dst, USE src);
14960 
14961   ins_cost(INSN_COST);
14962 
14963   format %{ "fmovs $dst, $src\t# MoveF2I_reg_reg" %}
14964 
14965   ins_encode %{
14966     __ fmovs($dst$$Register, as_FloatRegister($src$$reg));
14967   %}
14968 
14969   ins_pipe(fp_f2i);
14970 
14971 %}
14972 
14973 instruct MoveI2F_reg_reg(vRegF dst, iRegI src) %{
14974 
14975   match(Set dst (MoveI2F src));
14976 
14977   effect(DEF dst, USE src);
14978 
14979   ins_cost(INSN_COST);
14980 
14981   format %{ "fmovs $dst, $src\t# MoveI2F_reg_reg" %}
14982 
14983   ins_encode %{
14984     __ fmovs(as_FloatRegister($dst$$reg), $src$$Register);
14985   %}
14986 
14987   ins_pipe(fp_i2f);
14988 
14989 %}
14990 
14991 instruct MoveD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14992 
14993   match(Set dst (MoveD2L src));
14994 
14995   effect(DEF dst, USE src);
14996 
14997   ins_cost(INSN_COST);
14998 
14999   format %{ "fmovd $dst, $src\t# MoveD2L_reg_reg" %}
15000 
15001   ins_encode %{
15002     __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
15003   %}
15004 
15005   ins_pipe(fp_d2l);
15006 
15007 %}
15008 
15009 instruct MoveL2D_reg_reg(vRegD dst, iRegL src) %{
15010 
15011   match(Set dst (MoveL2D src));
15012 
15013   effect(DEF dst, USE src);
15014 
15015   ins_cost(INSN_COST);
15016 
15017   format %{ "fmovd $dst, $src\t# MoveL2D_reg_reg" %}
15018 
15019   ins_encode %{
15020     __ fmovd(as_FloatRegister($dst$$reg), $src$$Register);
15021   %}
15022 
15023   ins_pipe(fp_l2d);
15024 
15025 %}
15026 
15027 // ============================================================================
15028 // clearing of an array
15029 
15030 instruct clearArray_reg_reg_immL0(iRegL_R11 cnt, iRegP_R10 base, immL0 zero, Universe dummy, rFlagsReg cr)
15031 %{
15032   match(Set dummy (ClearArray (Binary cnt base) zero));
15033   effect(USE_KILL cnt, USE_KILL base, KILL cr);
15034 
15035   ins_cost(4 * INSN_COST);
15036   format %{ "ClearArray $cnt, $base" %}
15037 
15038   ins_encode %{
15039     address tpc = __ zero_words($base$$Register, $cnt$$Register);
15040     if (tpc == nullptr) {
15041       ciEnv::current()->record_failure("CodeCache is full");
15042       return;
15043     }
15044   %}
15045 
15046   ins_pipe(pipe_class_memory);
15047 %}
15048 
15049 instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, iRegL val, Universe dummy, rFlagsReg cr)
15050 %{
15051   predicate(((ClearArrayNode*)n)->word_copy_only());
15052   match(Set dummy (ClearArray (Binary cnt base) val));
15053   effect(USE_KILL cnt, USE_KILL base, KILL cr);
15054 
15055   ins_cost(4 * INSN_COST);
15056   format %{ "ClearArray $cnt, $base, $val" %}
15057 
15058   ins_encode %{
15059     __ fill_words($base$$Register, $cnt$$Register, $val$$Register);
15060   %}
15061 
15062   ins_pipe(pipe_class_memory);
15063 %}
15064 
15065 instruct clearArray_imm_reg(immL cnt, iRegP_R10 base, iRegL_R11 temp, Universe dummy, rFlagsReg cr)
15066 %{
15067   predicate((uint64_t)n->in(2)->get_long()
15068             < (uint64_t)(BlockZeroingLowLimit >> LogBytesPerWord)
15069             && !((ClearArrayNode*)n)->word_copy_only());
15070   match(Set dummy (ClearArray cnt base));
15071   effect(TEMP temp, USE_KILL base, KILL cr);
15072 
15073   ins_cost(4 * INSN_COST);
15074   format %{ "ClearArray $cnt, $base" %}
15075 
15076   ins_encode %{
15077     address tpc = __ zero_words($base$$Register, (uint64_t)$cnt$$constant);
15078     if (tpc == nullptr) {
15079       ciEnv::current()->record_failure("CodeCache is full");
15080       return;
15081     }
15082   %}
15083 
15084   ins_pipe(pipe_class_memory);
15085 %}
15086 
15087 // ============================================================================
15088 // Overflow Math Instructions
15089 
15090 instruct overflowAddI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15091 %{
15092   match(Set cr (OverflowAddI op1 op2));
15093 
15094   format %{ "cmnw  $op1, $op2\t# overflow check int" %}
15095   ins_cost(INSN_COST);
15096   ins_encode %{
15097     __ cmnw($op1$$Register, $op2$$Register);
15098   %}
15099 
15100   ins_pipe(icmp_reg_reg);
15101 %}
15102 
15103 instruct overflowAddI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
15104 %{
15105   match(Set cr (OverflowAddI op1 op2));
15106 
15107   format %{ "cmnw  $op1, $op2\t# overflow check int" %}
15108   ins_cost(INSN_COST);
15109   ins_encode %{
15110     __ cmnw($op1$$Register, $op2$$constant);
15111   %}
15112 
15113   ins_pipe(icmp_reg_imm);
15114 %}
15115 
15116 instruct overflowAddL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15117 %{
15118   match(Set cr (OverflowAddL op1 op2));
15119 
15120   format %{ "cmn   $op1, $op2\t# overflow check long" %}
15121   ins_cost(INSN_COST);
15122   ins_encode %{
15123     __ cmn($op1$$Register, $op2$$Register);
15124   %}
15125 
15126   ins_pipe(icmp_reg_reg);
15127 %}
15128 
15129 instruct overflowAddL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
15130 %{
15131   match(Set cr (OverflowAddL op1 op2));
15132 
15133   format %{ "adds  zr, $op1, $op2\t# overflow check long" %}
15134   ins_cost(INSN_COST);
15135   ins_encode %{
15136     __ adds(zr, $op1$$Register, $op2$$constant);
15137   %}
15138 
15139   ins_pipe(icmp_reg_imm);
15140 %}
15141 
15142 instruct overflowSubI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15143 %{
15144   match(Set cr (OverflowSubI op1 op2));
15145 
15146   format %{ "cmpw  $op1, $op2\t# overflow check int" %}
15147   ins_cost(INSN_COST);
15148   ins_encode %{
15149     __ cmpw($op1$$Register, $op2$$Register);
15150   %}
15151 
15152   ins_pipe(icmp_reg_reg);
15153 %}
15154 
15155 instruct overflowSubI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
15156 %{
15157   match(Set cr (OverflowSubI op1 op2));
15158 
15159   format %{ "cmpw  $op1, $op2\t# overflow check int" %}
15160   ins_cost(INSN_COST);
15161   ins_encode %{
15162     __ cmpw($op1$$Register, $op2$$constant);
15163   %}
15164 
15165   ins_pipe(icmp_reg_imm);
15166 %}
15167 
15168 instruct overflowSubL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15169 %{
15170   match(Set cr (OverflowSubL op1 op2));
15171 
15172   format %{ "cmp   $op1, $op2\t# overflow check long" %}
15173   ins_cost(INSN_COST);
15174   ins_encode %{
15175     __ cmp($op1$$Register, $op2$$Register);
15176   %}
15177 
15178   ins_pipe(icmp_reg_reg);
15179 %}
15180 
15181 instruct overflowSubL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
15182 %{
15183   match(Set cr (OverflowSubL op1 op2));
15184 
15185   format %{ "cmp   $op1, $op2\t# overflow check long" %}
15186   ins_cost(INSN_COST);
15187   ins_encode %{
15188     __ subs(zr, $op1$$Register, $op2$$constant);
15189   %}
15190 
15191   ins_pipe(icmp_reg_imm);
15192 %}
15193 
15194 instruct overflowNegI_reg(rFlagsReg cr, immI0 zero, iRegIorL2I op1)
15195 %{
15196   match(Set cr (OverflowSubI zero op1));
15197 
15198   format %{ "cmpw  zr, $op1\t# overflow check int" %}
15199   ins_cost(INSN_COST);
15200   ins_encode %{
15201     __ cmpw(zr, $op1$$Register);
15202   %}
15203 
15204   ins_pipe(icmp_reg_imm);
15205 %}
15206 
15207 instruct overflowNegL_reg(rFlagsReg cr, immI0 zero, iRegL op1)
15208 %{
15209   match(Set cr (OverflowSubL zero op1));
15210 
15211   format %{ "cmp   zr, $op1\t# overflow check long" %}
15212   ins_cost(INSN_COST);
15213   ins_encode %{
15214     __ cmp(zr, $op1$$Register);
15215   %}
15216 
15217   ins_pipe(icmp_reg_imm);
15218 %}
15219 
15220 instruct overflowMulI_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15221 %{
15222   match(Set cr (OverflowMulI op1 op2));
15223 
15224   format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15225             "cmp   rscratch1, rscratch1, sxtw\n\t"
15226             "movw  rscratch1, #0x80000000\n\t"
15227             "cselw rscratch1, rscratch1, zr, NE\n\t"
15228             "cmpw  rscratch1, #1" %}
15229   ins_cost(5 * INSN_COST);
15230   ins_encode %{
15231     __ smull(rscratch1, $op1$$Register, $op2$$Register);
15232     __ subs(zr, rscratch1, rscratch1, ext::sxtw);      // NE => overflow
15233     __ movw(rscratch1, 0x80000000);                    // Develop 0 (EQ),
15234     __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15235     __ cmpw(rscratch1, 1);                             // 0x80000000 - 1 => VS
15236   %}
15237 
15238   ins_pipe(pipe_slow);
15239 %}
15240 
15241 instruct overflowMulI_reg_branch(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, label labl, rFlagsReg cr)
15242 %{
15243   match(If cmp (OverflowMulI op1 op2));
15244   predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15245             || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15246   effect(USE labl, KILL cr);
15247 
15248   format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15249             "cmp   rscratch1, rscratch1, sxtw\n\t"
15250             "b$cmp   $labl" %}
15251   ins_cost(3 * INSN_COST); // Branch is rare so treat as INSN_COST
15252   ins_encode %{
15253     Label* L = $labl$$label;
15254     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15255     __ smull(rscratch1, $op1$$Register, $op2$$Register);
15256     __ subs(zr, rscratch1, rscratch1, ext::sxtw);      // NE => overflow
15257     __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15258   %}
15259 
15260   ins_pipe(pipe_serial);
15261 %}
15262 
15263 instruct overflowMulL_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15264 %{
15265   match(Set cr (OverflowMulL op1 op2));
15266 
15267   format %{ "mul   rscratch1, $op1, $op2\t#overflow check long\n\t"
15268             "smulh rscratch2, $op1, $op2\n\t"
15269             "cmp   rscratch2, rscratch1, ASR #63\n\t"
15270             "movw  rscratch1, #0x80000000\n\t"
15271             "cselw rscratch1, rscratch1, zr, NE\n\t"
15272             "cmpw  rscratch1, #1" %}
15273   ins_cost(6 * INSN_COST);
15274   ins_encode %{
15275     __ mul(rscratch1, $op1$$Register, $op2$$Register);   // Result bits 0..63
15276     __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15277     __ cmp(rscratch2, rscratch1, Assembler::ASR, 63);    // Top is pure sign ext
15278     __ movw(rscratch1, 0x80000000);                    // Develop 0 (EQ),
15279     __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15280     __ cmpw(rscratch1, 1);                             // 0x80000000 - 1 => VS
15281   %}
15282 
15283   ins_pipe(pipe_slow);
15284 %}
15285 
15286 instruct overflowMulL_reg_branch(cmpOp cmp, iRegL op1, iRegL op2, label labl, rFlagsReg cr)
15287 %{
15288   match(If cmp (OverflowMulL op1 op2));
15289   predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15290             || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15291   effect(USE labl, KILL cr);
15292 
15293   format %{ "mul   rscratch1, $op1, $op2\t#overflow check long\n\t"
15294             "smulh rscratch2, $op1, $op2\n\t"
15295             "cmp   rscratch2, rscratch1, ASR #63\n\t"
15296             "b$cmp $labl" %}
15297   ins_cost(4 * INSN_COST); // Branch is rare so treat as INSN_COST
15298   ins_encode %{
15299     Label* L = $labl$$label;
15300     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15301     __ mul(rscratch1, $op1$$Register, $op2$$Register);   // Result bits 0..63
15302     __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15303     __ cmp(rscratch2, rscratch1, Assembler::ASR, 63);    // Top is pure sign ext
15304     __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15305   %}
15306 
15307   ins_pipe(pipe_serial);
15308 %}
15309 
15310 // ============================================================================
15311 // Compare Instructions
15312 
15313 instruct compI_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
15314 %{
15315   match(Set cr (CmpI op1 op2));
15316 
15317   effect(DEF cr, USE op1, USE op2);
15318 
15319   ins_cost(INSN_COST);
15320   format %{ "cmpw  $op1, $op2" %}
15321 
15322   ins_encode(aarch64_enc_cmpw(op1, op2));
15323 
15324   ins_pipe(icmp_reg_reg);
15325 %}
15326 
15327 instruct compI_reg_immI0(rFlagsReg cr, iRegI op1, immI0 zero)
15328 %{
15329   match(Set cr (CmpI op1 zero));
15330 
15331   effect(DEF cr, USE op1);
15332 
15333   ins_cost(INSN_COST);
15334   format %{ "cmpw $op1, 0" %}
15335 
15336   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15337 
15338   ins_pipe(icmp_reg_imm);
15339 %}
15340 
15341 instruct compI_reg_immIAddSub(rFlagsReg cr, iRegI op1, immIAddSub op2)
15342 %{
15343   match(Set cr (CmpI op1 op2));
15344 
15345   effect(DEF cr, USE op1);
15346 
15347   ins_cost(INSN_COST);
15348   format %{ "cmpw  $op1, $op2" %}
15349 
15350   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15351 
15352   ins_pipe(icmp_reg_imm);
15353 %}
15354 
15355 instruct compI_reg_immI(rFlagsReg cr, iRegI op1, immI op2)
15356 %{
15357   match(Set cr (CmpI op1 op2));
15358 
15359   effect(DEF cr, USE op1);
15360 
15361   ins_cost(INSN_COST * 2);
15362   format %{ "cmpw  $op1, $op2" %}
15363 
15364   ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15365 
15366   ins_pipe(icmp_reg_imm);
15367 %}
15368 
15369 // Unsigned compare Instructions; really, same as signed compare
15370 // except it should only be used to feed an If or a CMovI which takes a
15371 // cmpOpU.
15372 
15373 instruct compU_reg_reg(rFlagsRegU cr, iRegI op1, iRegI op2)
15374 %{
15375   match(Set cr (CmpU op1 op2));
15376 
15377   effect(DEF cr, USE op1, USE op2);
15378 
15379   ins_cost(INSN_COST);
15380   format %{ "cmpw  $op1, $op2\t# unsigned" %}
15381 
15382   ins_encode(aarch64_enc_cmpw(op1, op2));
15383 
15384   ins_pipe(icmp_reg_reg);
15385 %}
15386 
15387 instruct compU_reg_immI0(rFlagsRegU cr, iRegI op1, immI0 zero)
15388 %{
15389   match(Set cr (CmpU op1 zero));
15390 
15391   effect(DEF cr, USE op1);
15392 
15393   ins_cost(INSN_COST);
15394   format %{ "cmpw $op1, #0\t# unsigned" %}
15395 
15396   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15397 
15398   ins_pipe(icmp_reg_imm);
15399 %}
15400 
15401 instruct compU_reg_immIAddSub(rFlagsRegU cr, iRegI op1, immIAddSub op2)
15402 %{
15403   match(Set cr (CmpU op1 op2));
15404 
15405   effect(DEF cr, USE op1);
15406 
15407   ins_cost(INSN_COST);
15408   format %{ "cmpw  $op1, $op2\t# unsigned" %}
15409 
15410   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15411 
15412   ins_pipe(icmp_reg_imm);
15413 %}
15414 
15415 instruct compU_reg_immI(rFlagsRegU cr, iRegI op1, immI op2)
15416 %{
15417   match(Set cr (CmpU op1 op2));
15418 
15419   effect(DEF cr, USE op1);
15420 
15421   ins_cost(INSN_COST * 2);
15422   format %{ "cmpw  $op1, $op2\t# unsigned" %}
15423 
15424   ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15425 
15426   ins_pipe(icmp_reg_imm);
15427 %}
15428 
15429 instruct compL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15430 %{
15431   match(Set cr (CmpL op1 op2));
15432 
15433   effect(DEF cr, USE op1, USE op2);
15434 
15435   ins_cost(INSN_COST);
15436   format %{ "cmp  $op1, $op2" %}
15437 
15438   ins_encode(aarch64_enc_cmp(op1, op2));
15439 
15440   ins_pipe(icmp_reg_reg);
15441 %}
15442 
15443 instruct compL_reg_immL0(rFlagsReg cr, iRegL op1, immL0 zero)
15444 %{
15445   match(Set cr (CmpL op1 zero));
15446 
15447   effect(DEF cr, USE op1);
15448 
15449   ins_cost(INSN_COST);
15450   format %{ "tst  $op1" %}
15451 
15452   ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15453 
15454   ins_pipe(icmp_reg_imm);
15455 %}
15456 
15457 instruct compL_reg_immLAddSub(rFlagsReg cr, iRegL op1, immLAddSub op2)
15458 %{
15459   match(Set cr (CmpL op1 op2));
15460 
15461   effect(DEF cr, USE op1);
15462 
15463   ins_cost(INSN_COST);
15464   format %{ "cmp  $op1, $op2" %}
15465 
15466   ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15467 
15468   ins_pipe(icmp_reg_imm);
15469 %}
15470 
15471 instruct compL_reg_immL(rFlagsReg cr, iRegL op1, immL op2)
15472 %{
15473   match(Set cr (CmpL op1 op2));
15474 
15475   effect(DEF cr, USE op1);
15476 
15477   ins_cost(INSN_COST * 2);
15478   format %{ "cmp  $op1, $op2" %}
15479 
15480   ins_encode(aarch64_enc_cmp_imm(op1, op2));
15481 
15482   ins_pipe(icmp_reg_imm);
15483 %}
15484 
15485 instruct compUL_reg_reg(rFlagsRegU cr, iRegL op1, iRegL op2)
15486 %{
15487   match(Set cr (CmpUL op1 op2));
15488 
15489   effect(DEF cr, USE op1, USE op2);
15490 
15491   ins_cost(INSN_COST);
15492   format %{ "cmp  $op1, $op2" %}
15493 
15494   ins_encode(aarch64_enc_cmp(op1, op2));
15495 
15496   ins_pipe(icmp_reg_reg);
15497 %}
15498 
15499 instruct compUL_reg_immL0(rFlagsRegU cr, iRegL op1, immL0 zero)
15500 %{
15501   match(Set cr (CmpUL op1 zero));
15502 
15503   effect(DEF cr, USE op1);
15504 
15505   ins_cost(INSN_COST);
15506   format %{ "tst  $op1" %}
15507 
15508   ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15509 
15510   ins_pipe(icmp_reg_imm);
15511 %}
15512 
15513 instruct compUL_reg_immLAddSub(rFlagsRegU cr, iRegL op1, immLAddSub op2)
15514 %{
15515   match(Set cr (CmpUL op1 op2));
15516 
15517   effect(DEF cr, USE op1);
15518 
15519   ins_cost(INSN_COST);
15520   format %{ "cmp  $op1, $op2" %}
15521 
15522   ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15523 
15524   ins_pipe(icmp_reg_imm);
15525 %}
15526 
15527 instruct compUL_reg_immL(rFlagsRegU cr, iRegL op1, immL op2)
15528 %{
15529   match(Set cr (CmpUL op1 op2));
15530 
15531   effect(DEF cr, USE op1);
15532 
15533   ins_cost(INSN_COST * 2);
15534   format %{ "cmp  $op1, $op2" %}
15535 
15536   ins_encode(aarch64_enc_cmp_imm(op1, op2));
15537 
15538   ins_pipe(icmp_reg_imm);
15539 %}
15540 
15541 instruct compP_reg_reg(rFlagsRegU cr, iRegP op1, iRegP op2)
15542 %{
15543   match(Set cr (CmpP op1 op2));
15544 
15545   effect(DEF cr, USE op1, USE op2);
15546 
15547   ins_cost(INSN_COST);
15548   format %{ "cmp  $op1, $op2\t // ptr" %}
15549 
15550   ins_encode(aarch64_enc_cmpp(op1, op2));
15551 
15552   ins_pipe(icmp_reg_reg);
15553 %}
15554 
15555 instruct compN_reg_reg(rFlagsRegU cr, iRegN op1, iRegN op2)
15556 %{
15557   match(Set cr (CmpN op1 op2));
15558 
15559   effect(DEF cr, USE op1, USE op2);
15560 
15561   ins_cost(INSN_COST);
15562   format %{ "cmp  $op1, $op2\t // compressed ptr" %}
15563 
15564   ins_encode(aarch64_enc_cmpn(op1, op2));
15565 
15566   ins_pipe(icmp_reg_reg);
15567 %}
15568 
15569 instruct testP_reg(rFlagsRegU cr, iRegP op1, immP0 zero)
15570 %{
15571   match(Set cr (CmpP op1 zero));
15572 
15573   effect(DEF cr, USE op1, USE zero);
15574 
15575   ins_cost(INSN_COST);
15576   format %{ "cmp  $op1, 0\t // ptr" %}
15577 
15578   ins_encode(aarch64_enc_testp(op1));
15579 
15580   ins_pipe(icmp_reg_imm);
15581 %}
15582 
15583 instruct testN_reg(rFlagsRegU cr, iRegN op1, immN0 zero)
15584 %{
15585   match(Set cr (CmpN op1 zero));
15586 
15587   effect(DEF cr, USE op1, USE zero);
15588 
15589   ins_cost(INSN_COST);
15590   format %{ "cmp  $op1, 0\t // compressed ptr" %}
15591 
15592   ins_encode(aarch64_enc_testn(op1));
15593 
15594   ins_pipe(icmp_reg_imm);
15595 %}
15596 
15597 // FP comparisons
15598 //
15599 // n.b. CmpF/CmpD set a normal flags reg which then gets compared
15600 // using normal cmpOp. See declaration of rFlagsReg for details.
15601 
15602 instruct compF_reg_reg(rFlagsReg cr, vRegF src1, vRegF src2)
15603 %{
15604   match(Set cr (CmpF src1 src2));
15605 
15606   ins_cost(3 * INSN_COST);
15607   format %{ "fcmps $src1, $src2" %}
15608 
15609   ins_encode %{
15610     __ fcmps(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15611   %}
15612 
15613   ins_pipe(pipe_class_compare);
15614 %}
15615 
15616 instruct compF_reg_zero(rFlagsReg cr, vRegF src1, immF0 src2)
15617 %{
15618   match(Set cr (CmpF src1 src2));
15619 
15620   ins_cost(3 * INSN_COST);
15621   format %{ "fcmps $src1, 0.0" %}
15622 
15623   ins_encode %{
15624     __ fcmps(as_FloatRegister($src1$$reg), 0.0);
15625   %}
15626 
15627   ins_pipe(pipe_class_compare);
15628 %}
15629 // FROM HERE
15630 
15631 instruct compD_reg_reg(rFlagsReg cr, vRegD src1, vRegD src2)
15632 %{
15633   match(Set cr (CmpD src1 src2));
15634 
15635   ins_cost(3 * INSN_COST);
15636   format %{ "fcmpd $src1, $src2" %}
15637 
15638   ins_encode %{
15639     __ fcmpd(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15640   %}
15641 
15642   ins_pipe(pipe_class_compare);
15643 %}
15644 
15645 instruct compD_reg_zero(rFlagsReg cr, vRegD src1, immD0 src2)
15646 %{
15647   match(Set cr (CmpD src1 src2));
15648 
15649   ins_cost(3 * INSN_COST);
15650   format %{ "fcmpd $src1, 0.0" %}
15651 
15652   ins_encode %{
15653     __ fcmpd(as_FloatRegister($src1$$reg), 0.0);
15654   %}
15655 
15656   ins_pipe(pipe_class_compare);
15657 %}
15658 
15659 instruct compF3_reg_reg(iRegINoSp dst, vRegF src1, vRegF src2, rFlagsReg cr)
15660 %{
15661   match(Set dst (CmpF3 src1 src2));
15662   effect(KILL cr);
15663 
15664   ins_cost(5 * INSN_COST);
15665   format %{ "fcmps $src1, $src2\n\t"
15666             "csinvw($dst, zr, zr, eq\n\t"
15667             "csnegw($dst, $dst, $dst, lt)"
15668   %}
15669 
15670   ins_encode %{
15671     Label done;
15672     FloatRegister s1 = as_FloatRegister($src1$$reg);
15673     FloatRegister s2 = as_FloatRegister($src2$$reg);
15674     Register d = as_Register($dst$$reg);
15675     __ fcmps(s1, s2);
15676     // installs 0 if EQ else -1
15677     __ csinvw(d, zr, zr, Assembler::EQ);
15678     // keeps -1 if less or unordered else installs 1
15679     __ csnegw(d, d, d, Assembler::LT);
15680     __ bind(done);
15681   %}
15682 
15683   ins_pipe(pipe_class_default);
15684 
15685 %}
15686 
15687 instruct compD3_reg_reg(iRegINoSp dst, vRegD src1, vRegD src2, rFlagsReg cr)
15688 %{
15689   match(Set dst (CmpD3 src1 src2));
15690   effect(KILL cr);
15691 
15692   ins_cost(5 * INSN_COST);
15693   format %{ "fcmpd $src1, $src2\n\t"
15694             "csinvw($dst, zr, zr, eq\n\t"
15695             "csnegw($dst, $dst, $dst, lt)"
15696   %}
15697 
15698   ins_encode %{
15699     Label done;
15700     FloatRegister s1 = as_FloatRegister($src1$$reg);
15701     FloatRegister s2 = as_FloatRegister($src2$$reg);
15702     Register d = as_Register($dst$$reg);
15703     __ fcmpd(s1, s2);
15704     // installs 0 if EQ else -1
15705     __ csinvw(d, zr, zr, Assembler::EQ);
15706     // keeps -1 if less or unordered else installs 1
15707     __ csnegw(d, d, d, Assembler::LT);
15708     __ bind(done);
15709   %}
15710   ins_pipe(pipe_class_default);
15711 
15712 %}
15713 
15714 instruct compF3_reg_immF0(iRegINoSp dst, vRegF src1, immF0 zero, rFlagsReg cr)
15715 %{
15716   match(Set dst (CmpF3 src1 zero));
15717   effect(KILL cr);
15718 
15719   ins_cost(5 * INSN_COST);
15720   format %{ "fcmps $src1, 0.0\n\t"
15721             "csinvw($dst, zr, zr, eq\n\t"
15722             "csnegw($dst, $dst, $dst, lt)"
15723   %}
15724 
15725   ins_encode %{
15726     Label done;
15727     FloatRegister s1 = as_FloatRegister($src1$$reg);
15728     Register d = as_Register($dst$$reg);
15729     __ fcmps(s1, 0.0);
15730     // installs 0 if EQ else -1
15731     __ csinvw(d, zr, zr, Assembler::EQ);
15732     // keeps -1 if less or unordered else installs 1
15733     __ csnegw(d, d, d, Assembler::LT);
15734     __ bind(done);
15735   %}
15736 
15737   ins_pipe(pipe_class_default);
15738 
15739 %}
15740 
15741 instruct compD3_reg_immD0(iRegINoSp dst, vRegD src1, immD0 zero, rFlagsReg cr)
15742 %{
15743   match(Set dst (CmpD3 src1 zero));
15744   effect(KILL cr);
15745 
15746   ins_cost(5 * INSN_COST);
15747   format %{ "fcmpd $src1, 0.0\n\t"
15748             "csinvw($dst, zr, zr, eq\n\t"
15749             "csnegw($dst, $dst, $dst, lt)"
15750   %}
15751 
15752   ins_encode %{
15753     Label done;
15754     FloatRegister s1 = as_FloatRegister($src1$$reg);
15755     Register d = as_Register($dst$$reg);
15756     __ fcmpd(s1, 0.0);
15757     // installs 0 if EQ else -1
15758     __ csinvw(d, zr, zr, Assembler::EQ);
15759     // keeps -1 if less or unordered else installs 1
15760     __ csnegw(d, d, d, Assembler::LT);
15761     __ bind(done);
15762   %}
15763   ins_pipe(pipe_class_default);
15764 
15765 %}
15766 
15767 instruct cmpLTMask_reg_reg(iRegINoSp dst, iRegIorL2I p, iRegIorL2I q, rFlagsReg cr)
15768 %{
15769   match(Set dst (CmpLTMask p q));
15770   effect(KILL cr);
15771 
15772   ins_cost(3 * INSN_COST);
15773 
15774   format %{ "cmpw $p, $q\t# cmpLTMask\n\t"
15775             "csetw $dst, lt\n\t"
15776             "subw $dst, zr, $dst"
15777   %}
15778 
15779   ins_encode %{
15780     __ cmpw(as_Register($p$$reg), as_Register($q$$reg));
15781     __ csetw(as_Register($dst$$reg), Assembler::LT);
15782     __ subw(as_Register($dst$$reg), zr, as_Register($dst$$reg));
15783   %}
15784 
15785   ins_pipe(ialu_reg_reg);
15786 %}
15787 
15788 instruct cmpLTMask_reg_zero(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr)
15789 %{
15790   match(Set dst (CmpLTMask src zero));
15791   effect(KILL cr);
15792 
15793   ins_cost(INSN_COST);
15794 
15795   format %{ "asrw $dst, $src, #31\t# cmpLTMask0" %}
15796 
15797   ins_encode %{
15798     __ asrw(as_Register($dst$$reg), as_Register($src$$reg), 31);
15799   %}
15800 
15801   ins_pipe(ialu_reg_shift);
15802 %}
15803 
15804 // ============================================================================
15805 // Max and Min
15806 
15807 // Like compI_reg_reg or compI_reg_immI0 but without match rule and second zero parameter.
15808 
15809 instruct compI_reg_imm0(rFlagsReg cr, iRegI src)
15810 %{
15811   effect(DEF cr, USE src);
15812   ins_cost(INSN_COST);
15813   format %{ "cmpw $src, 0" %}
15814 
15815   ins_encode %{
15816     __ cmpw($src$$Register, 0);
15817   %}
15818   ins_pipe(icmp_reg_imm);
15819 %}
15820 
15821 instruct minI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
15822 %{
15823   match(Set dst (MinI src1 src2));
15824   ins_cost(INSN_COST * 3);
15825 
15826   expand %{
15827     rFlagsReg cr;
15828     compI_reg_reg(cr, src1, src2);
15829     cmovI_reg_reg_lt(dst, src1, src2, cr);
15830   %}
15831 %}
15832 
15833 instruct maxI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
15834 %{
15835   match(Set dst (MaxI src1 src2));
15836   ins_cost(INSN_COST * 3);
15837 
15838   expand %{
15839     rFlagsReg cr;
15840     compI_reg_reg(cr, src1, src2);
15841     cmovI_reg_reg_gt(dst, src1, src2, cr);
15842   %}
15843 %}
15844 
15845 
15846 // ============================================================================
15847 // Branch Instructions
15848 
15849 // Direct Branch.
15850 instruct branch(label lbl)
15851 %{
15852   match(Goto);
15853 
15854   effect(USE lbl);
15855 
15856   ins_cost(BRANCH_COST);
15857   format %{ "b  $lbl" %}
15858 
15859   ins_encode(aarch64_enc_b(lbl));
15860 
15861   ins_pipe(pipe_branch);
15862 %}
15863 
15864 // Conditional Near Branch
15865 instruct branchCon(cmpOp cmp, rFlagsReg cr, label lbl)
15866 %{
15867   // Same match rule as `branchConFar'.
15868   match(If cmp cr);
15869 
15870   effect(USE lbl);
15871 
15872   ins_cost(BRANCH_COST);
15873   // If set to 1 this indicates that the current instruction is a
15874   // short variant of a long branch. This avoids using this
15875   // instruction in first-pass matching. It will then only be used in
15876   // the `Shorten_branches' pass.
15877   // ins_short_branch(1);
15878   format %{ "b$cmp  $lbl" %}
15879 
15880   ins_encode(aarch64_enc_br_con(cmp, lbl));
15881 
15882   ins_pipe(pipe_branch_cond);
15883 %}
15884 
15885 // Conditional Near Branch Unsigned
15886 instruct branchConU(cmpOpU cmp, rFlagsRegU cr, label lbl)
15887 %{
15888   // Same match rule as `branchConFar'.
15889   match(If cmp cr);
15890 
15891   effect(USE lbl);
15892 
15893   ins_cost(BRANCH_COST);
15894   // If set to 1 this indicates that the current instruction is a
15895   // short variant of a long branch. This avoids using this
15896   // instruction in first-pass matching. It will then only be used in
15897   // the `Shorten_branches' pass.
15898   // ins_short_branch(1);
15899   format %{ "b$cmp  $lbl\t# unsigned" %}
15900 
15901   ins_encode(aarch64_enc_br_conU(cmp, lbl));
15902 
15903   ins_pipe(pipe_branch_cond);
15904 %}
15905 
15906 // Make use of CBZ and CBNZ.  These instructions, as well as being
15907 // shorter than (cmp; branch), have the additional benefit of not
15908 // killing the flags.
15909 
15910 instruct cmpI_imm0_branch(cmpOpEqNe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsReg cr) %{
15911   match(If cmp (CmpI op1 op2));
15912   effect(USE labl);
15913 
15914   ins_cost(BRANCH_COST);
15915   format %{ "cbw$cmp   $op1, $labl" %}
15916   ins_encode %{
15917     Label* L = $labl$$label;
15918     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15919     if (cond == Assembler::EQ)
15920       __ cbzw($op1$$Register, *L);
15921     else
15922       __ cbnzw($op1$$Register, *L);
15923   %}
15924   ins_pipe(pipe_cmp_branch);
15925 %}
15926 
15927 instruct cmpL_imm0_branch(cmpOpEqNe cmp, iRegL op1, immL0 op2, label labl, rFlagsReg cr) %{
15928   match(If cmp (CmpL op1 op2));
15929   effect(USE labl);
15930 
15931   ins_cost(BRANCH_COST);
15932   format %{ "cb$cmp   $op1, $labl" %}
15933   ins_encode %{
15934     Label* L = $labl$$label;
15935     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15936     if (cond == Assembler::EQ)
15937       __ cbz($op1$$Register, *L);
15938     else
15939       __ cbnz($op1$$Register, *L);
15940   %}
15941   ins_pipe(pipe_cmp_branch);
15942 %}
15943 
15944 instruct cmpP_imm0_branch(cmpOpEqNe cmp, iRegP op1, immP0 op2, label labl, rFlagsReg cr) %{
15945   match(If cmp (CmpP op1 op2));
15946   effect(USE labl);
15947 
15948   ins_cost(BRANCH_COST);
15949   format %{ "cb$cmp   $op1, $labl" %}
15950   ins_encode %{
15951     Label* L = $labl$$label;
15952     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15953     if (cond == Assembler::EQ)
15954       __ cbz($op1$$Register, *L);
15955     else
15956       __ cbnz($op1$$Register, *L);
15957   %}
15958   ins_pipe(pipe_cmp_branch);
15959 %}
15960 
15961 instruct cmpN_imm0_branch(cmpOpEqNe cmp, iRegN op1, immN0 op2, label labl, rFlagsReg cr) %{
15962   match(If cmp (CmpN op1 op2));
15963   effect(USE labl);
15964 
15965   ins_cost(BRANCH_COST);
15966   format %{ "cbw$cmp   $op1, $labl" %}
15967   ins_encode %{
15968     Label* L = $labl$$label;
15969     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15970     if (cond == Assembler::EQ)
15971       __ cbzw($op1$$Register, *L);
15972     else
15973       __ cbnzw($op1$$Register, *L);
15974   %}
15975   ins_pipe(pipe_cmp_branch);
15976 %}
15977 
15978 instruct cmpP_narrowOop_imm0_branch(cmpOpEqNe cmp, iRegN oop, immP0 zero, label labl, rFlagsReg cr) %{
15979   match(If cmp (CmpP (DecodeN oop) zero));
15980   effect(USE labl);
15981 
15982   ins_cost(BRANCH_COST);
15983   format %{ "cb$cmp   $oop, $labl" %}
15984   ins_encode %{
15985     Label* L = $labl$$label;
15986     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15987     if (cond == Assembler::EQ)
15988       __ cbzw($oop$$Register, *L);
15989     else
15990       __ cbnzw($oop$$Register, *L);
15991   %}
15992   ins_pipe(pipe_cmp_branch);
15993 %}
15994 
15995 instruct cmpUI_imm0_branch(cmpOpUEqNeLeGt cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15996   match(If cmp (CmpU op1 op2));
15997   effect(USE labl);
15998 
15999   ins_cost(BRANCH_COST);
16000   format %{ "cbw$cmp   $op1, $labl" %}
16001   ins_encode %{
16002     Label* L = $labl$$label;
16003     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16004     if (cond == Assembler::EQ || cond == Assembler::LS) {
16005       __ cbzw($op1$$Register, *L);
16006     } else {
16007       assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
16008       __ cbnzw($op1$$Register, *L);
16009     }
16010   %}
16011   ins_pipe(pipe_cmp_branch);
16012 %}
16013 
16014 instruct cmpUL_imm0_branch(cmpOpUEqNeLeGt cmp, iRegL op1, immL0 op2, label labl) %{
16015   match(If cmp (CmpUL op1 op2));
16016   effect(USE labl);
16017 
16018   ins_cost(BRANCH_COST);
16019   format %{ "cb$cmp   $op1, $labl" %}
16020   ins_encode %{
16021     Label* L = $labl$$label;
16022     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16023     if (cond == Assembler::EQ || cond == Assembler::LS) {
16024       __ cbz($op1$$Register, *L);
16025     } else {
16026       assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
16027       __ cbnz($op1$$Register, *L);
16028     }
16029   %}
16030   ins_pipe(pipe_cmp_branch);
16031 %}
16032 
16033 // Test bit and Branch
16034 
16035 // Patterns for short (< 32KiB) variants
16036 instruct cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
16037   match(If cmp (CmpL op1 op2));
16038   effect(USE labl);
16039 
16040   ins_cost(BRANCH_COST);
16041   format %{ "cb$cmp   $op1, $labl # long" %}
16042   ins_encode %{
16043     Label* L = $labl$$label;
16044     Assembler::Condition cond =
16045       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16046     __ tbr(cond, $op1$$Register, 63, *L);
16047   %}
16048   ins_pipe(pipe_cmp_branch);
16049   ins_short_branch(1);
16050 %}
16051 
16052 instruct cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16053   match(If cmp (CmpI op1 op2));
16054   effect(USE labl);
16055 
16056   ins_cost(BRANCH_COST);
16057   format %{ "cb$cmp   $op1, $labl # int" %}
16058   ins_encode %{
16059     Label* L = $labl$$label;
16060     Assembler::Condition cond =
16061       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16062     __ tbr(cond, $op1$$Register, 31, *L);
16063   %}
16064   ins_pipe(pipe_cmp_branch);
16065   ins_short_branch(1);
16066 %}
16067 
16068 instruct cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
16069   match(If cmp (CmpL (AndL op1 op2) op3));
16070   predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
16071   effect(USE labl);
16072 
16073   ins_cost(BRANCH_COST);
16074   format %{ "tb$cmp   $op1, $op2, $labl" %}
16075   ins_encode %{
16076     Label* L = $labl$$label;
16077     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16078     int bit = exact_log2_long($op2$$constant);
16079     __ tbr(cond, $op1$$Register, bit, *L);
16080   %}
16081   ins_pipe(pipe_cmp_branch);
16082   ins_short_branch(1);
16083 %}
16084 
16085 instruct cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
16086   match(If cmp (CmpI (AndI op1 op2) op3));
16087   predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
16088   effect(USE labl);
16089 
16090   ins_cost(BRANCH_COST);
16091   format %{ "tb$cmp   $op1, $op2, $labl" %}
16092   ins_encode %{
16093     Label* L = $labl$$label;
16094     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16095     int bit = exact_log2((juint)$op2$$constant);
16096     __ tbr(cond, $op1$$Register, bit, *L);
16097   %}
16098   ins_pipe(pipe_cmp_branch);
16099   ins_short_branch(1);
16100 %}
16101 
16102 // And far variants
16103 instruct far_cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
16104   match(If cmp (CmpL op1 op2));
16105   effect(USE labl);
16106 
16107   ins_cost(BRANCH_COST);
16108   format %{ "cb$cmp   $op1, $labl # long" %}
16109   ins_encode %{
16110     Label* L = $labl$$label;
16111     Assembler::Condition cond =
16112       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16113     __ tbr(cond, $op1$$Register, 63, *L, /*far*/true);
16114   %}
16115   ins_pipe(pipe_cmp_branch);
16116 %}
16117 
16118 instruct far_cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16119   match(If cmp (CmpI op1 op2));
16120   effect(USE labl);
16121 
16122   ins_cost(BRANCH_COST);
16123   format %{ "cb$cmp   $op1, $labl # int" %}
16124   ins_encode %{
16125     Label* L = $labl$$label;
16126     Assembler::Condition cond =
16127       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16128     __ tbr(cond, $op1$$Register, 31, *L, /*far*/true);
16129   %}
16130   ins_pipe(pipe_cmp_branch);
16131 %}
16132 
16133 instruct far_cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
16134   match(If cmp (CmpL (AndL op1 op2) op3));
16135   predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
16136   effect(USE labl);
16137 
16138   ins_cost(BRANCH_COST);
16139   format %{ "tb$cmp   $op1, $op2, $labl" %}
16140   ins_encode %{
16141     Label* L = $labl$$label;
16142     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16143     int bit = exact_log2_long($op2$$constant);
16144     __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
16145   %}
16146   ins_pipe(pipe_cmp_branch);
16147 %}
16148 
16149 instruct far_cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
16150   match(If cmp (CmpI (AndI op1 op2) op3));
16151   predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
16152   effect(USE labl);
16153 
16154   ins_cost(BRANCH_COST);
16155   format %{ "tb$cmp   $op1, $op2, $labl" %}
16156   ins_encode %{
16157     Label* L = $labl$$label;
16158     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16159     int bit = exact_log2((juint)$op2$$constant);
16160     __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
16161   %}
16162   ins_pipe(pipe_cmp_branch);
16163 %}
16164 
16165 // Test bits
16166 
16167 instruct cmpL_and(cmpOp cmp, iRegL op1, immL op2, immL0 op3, rFlagsReg cr) %{
16168   match(Set cr (CmpL (AndL op1 op2) op3));
16169   predicate(Assembler::operand_valid_for_logical_immediate
16170             (/*is_32*/false, n->in(1)->in(2)->get_long()));
16171 
16172   ins_cost(INSN_COST);
16173   format %{ "tst $op1, $op2 # long" %}
16174   ins_encode %{
16175     __ tst($op1$$Register, $op2$$constant);
16176   %}
16177   ins_pipe(ialu_reg_reg);
16178 %}
16179 
16180 instruct cmpI_and(cmpOp cmp, iRegIorL2I op1, immI op2, immI0 op3, rFlagsReg cr) %{
16181   match(Set cr (CmpI (AndI op1 op2) op3));
16182   predicate(Assembler::operand_valid_for_logical_immediate
16183             (/*is_32*/true, n->in(1)->in(2)->get_int()));
16184 
16185   ins_cost(INSN_COST);
16186   format %{ "tst $op1, $op2 # int" %}
16187   ins_encode %{
16188     __ tstw($op1$$Register, $op2$$constant);
16189   %}
16190   ins_pipe(ialu_reg_reg);
16191 %}
16192 
16193 instruct cmpL_and_reg(cmpOp cmp, iRegL op1, iRegL op2, immL0 op3, rFlagsReg cr) %{
16194   match(Set cr (CmpL (AndL op1 op2) op3));
16195 
16196   ins_cost(INSN_COST);
16197   format %{ "tst $op1, $op2 # long" %}
16198   ins_encode %{
16199     __ tst($op1$$Register, $op2$$Register);
16200   %}
16201   ins_pipe(ialu_reg_reg);
16202 %}
16203 
16204 instruct cmpI_and_reg(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, immI0 op3, rFlagsReg cr) %{
16205   match(Set cr (CmpI (AndI op1 op2) op3));
16206 
16207   ins_cost(INSN_COST);
16208   format %{ "tstw $op1, $op2 # int" %}
16209   ins_encode %{
16210     __ tstw($op1$$Register, $op2$$Register);
16211   %}
16212   ins_pipe(ialu_reg_reg);
16213 %}
16214 
16215 
16216 // Conditional Far Branch
16217 // Conditional Far Branch Unsigned
16218 // TODO: fixme
16219 
16220 // counted loop end branch near
16221 instruct branchLoopEnd(cmpOp cmp, rFlagsReg cr, label lbl)
16222 %{
16223   match(CountedLoopEnd cmp cr);
16224 
16225   effect(USE lbl);
16226 
16227   ins_cost(BRANCH_COST);
16228   // short variant.
16229   // ins_short_branch(1);
16230   format %{ "b$cmp $lbl \t// counted loop end" %}
16231 
16232   ins_encode(aarch64_enc_br_con(cmp, lbl));
16233 
16234   ins_pipe(pipe_branch);
16235 %}
16236 
16237 // counted loop end branch far
16238 // TODO: fixme
16239 
16240 // ============================================================================
16241 // inlined locking and unlocking
16242 
16243 instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
16244 %{
16245   predicate(LockingMode != LM_LIGHTWEIGHT);
16246   match(Set cr (FastLock object box));
16247   effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
16248 
16249   ins_cost(5 * INSN_COST);
16250   format %{ "fastlock $object,$box\t! kills $tmp,$tmp2,$tmp3" %}
16251 
16252   ins_encode %{
16253     __ fast_lock($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
16254   %}
16255 
16256   ins_pipe(pipe_serial);
16257 %}
16258 
16259 instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
16260 %{
16261   predicate(LockingMode != LM_LIGHTWEIGHT);
16262   match(Set cr (FastUnlock object box));
16263   effect(TEMP tmp, TEMP tmp2);
16264 
16265   ins_cost(5 * INSN_COST);
16266   format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2" %}
16267 
16268   ins_encode %{
16269     __ fast_unlock($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register);
16270   %}
16271 
16272   ins_pipe(pipe_serial);
16273 %}
16274 
16275 instruct cmpFastLockLightweight(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
16276 %{
16277   predicate(LockingMode == LM_LIGHTWEIGHT);
16278   match(Set cr (FastLock object box));
16279   effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
16280 
16281   ins_cost(5 * INSN_COST);
16282   format %{ "fastlock $object,$box\t! kills $tmp,$tmp2,$tmp3" %}
16283 
16284   ins_encode %{
16285     __ fast_lock_lightweight($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
16286   %}
16287 
16288   ins_pipe(pipe_serial);
16289 %}
16290 
16291 instruct cmpFastUnlockLightweight(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
16292 %{
16293   predicate(LockingMode == LM_LIGHTWEIGHT);
16294   match(Set cr (FastUnlock object box));
16295   effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
16296 
16297   ins_cost(5 * INSN_COST);
16298   format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2, $tmp3" %}
16299 
16300   ins_encode %{
16301     __ fast_unlock_lightweight($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
16302   %}
16303 
16304   ins_pipe(pipe_serial);
16305 %}
16306 
16307 // ============================================================================
16308 // Safepoint Instructions
16309 
16310 // TODO
16311 // provide a near and far version of this code
16312 
16313 instruct safePoint(rFlagsReg cr, iRegP poll)
16314 %{
16315   match(SafePoint poll);
16316   effect(KILL cr);
16317 
16318   format %{
16319     "ldrw zr, [$poll]\t# Safepoint: poll for GC"
16320   %}
16321   ins_encode %{
16322     __ read_polling_page(as_Register($poll$$reg), relocInfo::poll_type);
16323   %}
16324   ins_pipe(pipe_serial); // ins_pipe(iload_reg_mem);
16325 %}
16326 
16327 
16328 // ============================================================================
16329 // Procedure Call/Return Instructions
16330 
16331 // Call Java Static Instruction
16332 
16333 instruct CallStaticJavaDirect(method meth)
16334 %{
16335   match(CallStaticJava);
16336 
16337   effect(USE meth);
16338 
16339   ins_cost(CALL_COST);
16340 
16341   format %{ "call,static $meth \t// ==> " %}
16342 
16343   ins_encode(aarch64_enc_java_static_call(meth),
16344              aarch64_enc_call_epilog);
16345 
16346   ins_pipe(pipe_class_call);
16347 %}
16348 
16349 // TO HERE
16350 
16351 // Call Java Dynamic Instruction
16352 instruct CallDynamicJavaDirect(method meth)
16353 %{
16354   match(CallDynamicJava);
16355 
16356   effect(USE meth);
16357 
16358   ins_cost(CALL_COST);
16359 
16360   format %{ "CALL,dynamic $meth \t// ==> " %}
16361 
16362   ins_encode(aarch64_enc_java_dynamic_call(meth),
16363              aarch64_enc_call_epilog);
16364 
16365   ins_pipe(pipe_class_call);
16366 %}
16367 
16368 // Call Runtime Instruction
16369 
16370 instruct CallRuntimeDirect(method meth)
16371 %{
16372   match(CallRuntime);
16373 
16374   effect(USE meth);
16375 
16376   ins_cost(CALL_COST);
16377 
16378   format %{ "CALL, runtime $meth" %}
16379 
16380   ins_encode( aarch64_enc_java_to_runtime(meth) );
16381 
16382   ins_pipe(pipe_class_call);
16383 %}
16384 
16385 // Call Runtime Instruction
16386 
16387 instruct CallLeafDirect(method meth)
16388 %{
16389   match(CallLeaf);
16390 
16391   effect(USE meth);
16392 
16393   ins_cost(CALL_COST);
16394 
16395   format %{ "CALL, runtime leaf $meth" %}
16396 
16397   ins_encode( aarch64_enc_java_to_runtime(meth) );
16398 
16399   ins_pipe(pipe_class_call);
16400 %}
16401 
16402 // Call Runtime Instruction without safepoint and with vector arguments
16403 instruct CallLeafDirectVector(method meth)
16404 %{
16405   match(CallLeafVector);
16406 
16407   effect(USE meth);
16408 
16409   ins_cost(CALL_COST);
16410 
16411   format %{ "CALL, runtime leaf vector $meth" %}
16412 
16413   ins_encode(aarch64_enc_java_to_runtime(meth));
16414 
16415   ins_pipe(pipe_class_call);
16416 %}
16417 
16418 // Call Runtime Instruction
16419 
16420 // entry point is null, target holds the address to call
16421 instruct CallLeafNoFPIndirect(iRegP target)
16422 %{
16423   predicate(n->as_Call()->entry_point() == nullptr);
16424 
16425   match(CallLeafNoFP target);
16426 
16427   ins_cost(CALL_COST);
16428 
16429   format %{ "CALL, runtime leaf nofp indirect $target" %}
16430 
16431   ins_encode %{
16432     __ blr($target$$Register);
16433   %}
16434 
16435   ins_pipe(pipe_class_call);
16436 %}
16437 
16438 instruct CallLeafNoFPDirect(method meth)
16439 %{
16440   predicate(n->as_Call()->entry_point() != nullptr);
16441 
16442   match(CallLeafNoFP);
16443 
16444   effect(USE meth);
16445 
16446   ins_cost(CALL_COST);
16447 
16448   format %{ "CALL, runtime leaf nofp $meth" %}
16449 
16450   ins_encode( aarch64_enc_java_to_runtime(meth) );
16451 
16452   ins_pipe(pipe_class_call);
16453 %}
16454 
16455 // Tail Call; Jump from runtime stub to Java code.
16456 // Also known as an 'interprocedural jump'.
16457 // Target of jump will eventually return to caller.
16458 // TailJump below removes the return address.
16459 // Don't use rfp for 'jump_target' because a MachEpilogNode has already been
16460 // emitted just above the TailCall which has reset rfp to the caller state.
16461 instruct TailCalljmpInd(iRegPNoSpNoRfp jump_target, inline_cache_RegP method_ptr)
16462 %{
16463   match(TailCall jump_target method_ptr);
16464 
16465   ins_cost(CALL_COST);
16466 
16467   format %{ "br $jump_target\t# $method_ptr holds method" %}
16468 
16469   ins_encode(aarch64_enc_tail_call(jump_target));
16470 
16471   ins_pipe(pipe_class_call);
16472 %}
16473 
16474 instruct TailjmpInd(iRegPNoSpNoRfp jump_target, iRegP_R0 ex_oop)
16475 %{
16476   match(TailJump jump_target ex_oop);
16477 
16478   ins_cost(CALL_COST);
16479 
16480   format %{ "br $jump_target\t# $ex_oop holds exception oop" %}
16481 
16482   ins_encode(aarch64_enc_tail_jmp(jump_target));
16483 
16484   ins_pipe(pipe_class_call);
16485 %}
16486 
16487 // Forward exception.
16488 instruct ForwardExceptionjmp()
16489 %{
16490   match(ForwardException);
16491   ins_cost(CALL_COST);
16492 
16493   format %{ "b forward_exception_stub" %}
16494   ins_encode %{
16495     __ far_jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
16496   %}
16497   ins_pipe(pipe_class_call);
16498 %}
16499 
16500 // Create exception oop: created by stack-crawling runtime code.
16501 // Created exception is now available to this handler, and is setup
16502 // just prior to jumping to this handler. No code emitted.
16503 // TODO check
16504 // should ex_oop be in r0? intel uses rax, ppc cannot use r0 so uses rarg1
16505 instruct CreateException(iRegP_R0 ex_oop)
16506 %{
16507   match(Set ex_oop (CreateEx));
16508 
16509   format %{ " -- \t// exception oop; no code emitted" %}
16510 
16511   size(0);
16512 
16513   ins_encode( /*empty*/ );
16514 
16515   ins_pipe(pipe_class_empty);
16516 %}
16517 
16518 // Rethrow exception: The exception oop will come in the first
16519 // argument position. Then JUMP (not call) to the rethrow stub code.
16520 instruct RethrowException() %{
16521   match(Rethrow);
16522   ins_cost(CALL_COST);
16523 
16524   format %{ "b rethrow_stub" %}
16525 
16526   ins_encode( aarch64_enc_rethrow() );
16527 
16528   ins_pipe(pipe_class_call);
16529 %}
16530 
16531 
16532 // Return Instruction
16533 // epilog node loads ret address into lr as part of frame pop
16534 instruct Ret()
16535 %{
16536   match(Return);
16537 
16538   format %{ "ret\t// return register" %}
16539 
16540   ins_encode( aarch64_enc_ret() );
16541 
16542   ins_pipe(pipe_branch);
16543 %}
16544 
16545 // Die now.
16546 instruct ShouldNotReachHere() %{
16547   match(Halt);
16548 
16549   ins_cost(CALL_COST);
16550   format %{ "ShouldNotReachHere" %}
16551 
16552   ins_encode %{
16553     if (is_reachable()) {
16554       const char* str = __ code_string(_halt_reason);
16555       __ stop(str);
16556     }
16557   %}
16558 
16559   ins_pipe(pipe_class_default);
16560 %}
16561 
16562 // ============================================================================
16563 // Partial Subtype Check
16564 //
16565 // superklass array for an instance of the superklass.  Set a hidden
16566 // internal cache on a hit (cache is checked with exposed code in
16567 // gen_subtype_check()).  Return NZ for a miss or zero for a hit.  The
16568 // encoding ALSO sets flags.
16569 
16570 instruct partialSubtypeCheck(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, rFlagsReg cr)
16571 %{
16572   match(Set result (PartialSubtypeCheck sub super));
16573   predicate(!UseSecondarySupersTable);
16574   effect(KILL cr, KILL temp);
16575 
16576   ins_cost(20 * INSN_COST);  // slightly larger than the next version
16577   format %{ "partialSubtypeCheck $result, $sub, $super" %}
16578 
16579   ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
16580 
16581   opcode(0x1); // Force zero of result reg on hit
16582 
16583   ins_pipe(pipe_class_memory);
16584 %}
16585 
16586 // Two versions of partialSubtypeCheck, both used when we need to
16587 // search for a super class in the secondary supers array. The first
16588 // is used when we don't know _a priori_ the class being searched
16589 // for. The second, far more common, is used when we do know: this is
16590 // used for instanceof, checkcast, and any case where C2 can determine
16591 // it by constant propagation.
16592 
16593 instruct partialSubtypeCheckVarSuper(iRegP_R4 sub, iRegP_R0 super, vRegD_V0 vtemp, iRegP_R5 result,
16594                                      iRegP_R1 tempR1, iRegP_R2 tempR2, iRegP_R3 tempR3,
16595                                      rFlagsReg cr)
16596 %{
16597   match(Set result (PartialSubtypeCheck sub super));
16598   predicate(UseSecondarySupersTable);
16599   effect(KILL cr, TEMP tempR1, TEMP tempR2, TEMP tempR3, TEMP vtemp);
16600 
16601   ins_cost(10 * INSN_COST);  // slightly larger than the next version
16602   format %{ "partialSubtypeCheck $result, $sub, $super" %}
16603 
16604   ins_encode %{
16605     __ lookup_secondary_supers_table_var($sub$$Register, $super$$Register,
16606                                          $tempR1$$Register, $tempR2$$Register, $tempR3$$Register,
16607                                          $vtemp$$FloatRegister,
16608                                          $result$$Register, /*L_success*/nullptr);
16609   %}
16610 
16611   ins_pipe(pipe_class_memory);
16612 %}
16613 
16614 instruct partialSubtypeCheckConstSuper(iRegP_R4 sub, iRegP_R0 super_reg, immP super_con, vRegD_V0 vtemp, iRegP_R5 result,
16615                                        iRegP_R1 tempR1, iRegP_R2 tempR2, iRegP_R3 tempR3,
16616                                        rFlagsReg cr)
16617 %{
16618   match(Set result (PartialSubtypeCheck sub (Binary super_reg super_con)));
16619   predicate(UseSecondarySupersTable);
16620   effect(KILL cr, TEMP tempR1, TEMP tempR2, TEMP tempR3, TEMP vtemp);
16621 
16622   ins_cost(5 * INSN_COST);  // smaller than the next version
16623   format %{ "partialSubtypeCheck $result, $sub, $super_reg, $super_con" %}
16624 
16625   ins_encode %{
16626     bool success = false;
16627     u1 super_klass_slot = ((Klass*)$super_con$$constant)->hash_slot();
16628     if (InlineSecondarySupersTest) {
16629       success =
16630         __ lookup_secondary_supers_table_const($sub$$Register, $super_reg$$Register,
16631                                                $tempR1$$Register, $tempR2$$Register, $tempR3$$Register,
16632                                                $vtemp$$FloatRegister,
16633                                                $result$$Register,
16634                                                super_klass_slot);
16635     } else {
16636       address call = __ trampoline_call(RuntimeAddress(StubRoutines::lookup_secondary_supers_table_stub(super_klass_slot)));
16637       success = (call != nullptr);
16638     }
16639     if (!success) {
16640       ciEnv::current()->record_failure("CodeCache is full");
16641       return;
16642     }
16643   %}
16644 
16645   ins_pipe(pipe_class_memory);
16646 %}
16647 
16648 // Intrisics for String.compareTo()
16649 
16650 instruct string_compareU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16651                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16652 %{
16653   predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
16654   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16655   effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16656 
16657   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1" %}
16658   ins_encode %{
16659     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16660     __ string_compare($str1$$Register, $str2$$Register,
16661                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16662                       $tmp1$$Register, $tmp2$$Register,
16663                       fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::UU);
16664   %}
16665   ins_pipe(pipe_class_memory);
16666 %}
16667 
16668 instruct string_compareL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16669                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16670 %{
16671   predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
16672   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16673   effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16674 
16675   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1" %}
16676   ins_encode %{
16677     __ string_compare($str1$$Register, $str2$$Register,
16678                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16679                       $tmp1$$Register, $tmp2$$Register,
16680                       fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::LL);
16681   %}
16682   ins_pipe(pipe_class_memory);
16683 %}
16684 
16685 instruct string_compareUL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16686                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16687                         vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16688 %{
16689   predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
16690   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16691   effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16692          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16693 
16694   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16695   ins_encode %{
16696     __ string_compare($str1$$Register, $str2$$Register,
16697                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16698                       $tmp1$$Register, $tmp2$$Register,
16699                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16700                       $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::UL);
16701   %}
16702   ins_pipe(pipe_class_memory);
16703 %}
16704 
16705 instruct string_compareLU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16706                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16707                         vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16708 %{
16709   predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
16710   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16711   effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16712          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16713 
16714   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16715   ins_encode %{
16716     __ string_compare($str1$$Register, $str2$$Register,
16717                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16718                       $tmp1$$Register, $tmp2$$Register,
16719                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16720                       $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::LU);
16721   %}
16722   ins_pipe(pipe_class_memory);
16723 %}
16724 
16725 // Note that Z registers alias the corresponding NEON registers, we declare the vector operands of
16726 // these string_compare variants as NEON register type for convenience so that the prototype of
16727 // string_compare can be shared with all variants.
16728 
16729 instruct string_compareLL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16730                               iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16731                               vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16732                               pRegGov_P1 pgtmp2, rFlagsReg cr)
16733 %{
16734   predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
16735   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16736   effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16737          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16738 
16739   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # USE sve" %}
16740   ins_encode %{
16741     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16742     __ string_compare($str1$$Register, $str2$$Register,
16743                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16744                       $tmp1$$Register, $tmp2$$Register,
16745                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16746                       as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16747                       StrIntrinsicNode::LL);
16748   %}
16749   ins_pipe(pipe_class_memory);
16750 %}
16751 
16752 instruct string_compareLU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16753                               iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16754                               vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16755                               pRegGov_P1 pgtmp2, rFlagsReg cr)
16756 %{
16757   predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
16758   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16759   effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16760          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16761 
16762   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # USE sve" %}
16763   ins_encode %{
16764     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16765     __ string_compare($str1$$Register, $str2$$Register,
16766                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16767                       $tmp1$$Register, $tmp2$$Register,
16768                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16769                       as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16770                       StrIntrinsicNode::LU);
16771   %}
16772   ins_pipe(pipe_class_memory);
16773 %}
16774 
16775 instruct string_compareUL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16776                               iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16777                               vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16778                               pRegGov_P1 pgtmp2, rFlagsReg cr)
16779 %{
16780   predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
16781   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16782   effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16783          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16784 
16785   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # USE sve" %}
16786   ins_encode %{
16787     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16788     __ string_compare($str1$$Register, $str2$$Register,
16789                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16790                       $tmp1$$Register, $tmp2$$Register,
16791                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16792                       as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16793                       StrIntrinsicNode::UL);
16794   %}
16795   ins_pipe(pipe_class_memory);
16796 %}
16797 
16798 instruct string_compareUU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16799                               iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16800                               vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16801                               pRegGov_P1 pgtmp2, rFlagsReg cr)
16802 %{
16803   predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
16804   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16805   effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16806          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16807 
16808   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # USE sve" %}
16809   ins_encode %{
16810     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16811     __ string_compare($str1$$Register, $str2$$Register,
16812                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16813                       $tmp1$$Register, $tmp2$$Register,
16814                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16815                       as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16816                       StrIntrinsicNode::UU);
16817   %}
16818   ins_pipe(pipe_class_memory);
16819 %}
16820 
16821 instruct string_indexofUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16822                           iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16823                           iRegINoSp tmp3, iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
16824                           vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
16825 %{
16826   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16827   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16828   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16829          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
16830          TEMP vtmp0, TEMP vtmp1, KILL cr);
16831   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UU) "
16832             "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
16833 
16834   ins_encode %{
16835     __ string_indexof($str1$$Register, $str2$$Register,
16836                       $cnt1$$Register, $cnt2$$Register,
16837                       $tmp1$$Register, $tmp2$$Register,
16838                       $tmp3$$Register, $tmp4$$Register,
16839                       $tmp5$$Register, $tmp6$$Register,
16840                       -1, $result$$Register, StrIntrinsicNode::UU);
16841   %}
16842   ins_pipe(pipe_class_memory);
16843 %}
16844 
16845 instruct string_indexofLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16846                           iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
16847                           iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
16848                           vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
16849 %{
16850   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16851   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16852   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16853          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
16854          TEMP vtmp0, TEMP vtmp1, KILL cr);
16855   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (LL) "
16856             "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
16857 
16858   ins_encode %{
16859     __ string_indexof($str1$$Register, $str2$$Register,
16860                       $cnt1$$Register, $cnt2$$Register,
16861                       $tmp1$$Register, $tmp2$$Register,
16862                       $tmp3$$Register, $tmp4$$Register,
16863                       $tmp5$$Register, $tmp6$$Register,
16864                       -1, $result$$Register, StrIntrinsicNode::LL);
16865   %}
16866   ins_pipe(pipe_class_memory);
16867 %}
16868 
16869 instruct string_indexofUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16870                           iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,iRegINoSp tmp3,
16871                           iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
16872                           vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
16873 %{
16874   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16875   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16876   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16877          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5,
16878          TEMP tmp6, TEMP vtmp0, TEMP vtmp1, KILL cr);
16879   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UL) "
16880             "# KILL $str1 cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
16881 
16882   ins_encode %{
16883     __ string_indexof($str1$$Register, $str2$$Register,
16884                       $cnt1$$Register, $cnt2$$Register,
16885                       $tmp1$$Register, $tmp2$$Register,
16886                       $tmp3$$Register, $tmp4$$Register,
16887                       $tmp5$$Register, $tmp6$$Register,
16888                       -1, $result$$Register, StrIntrinsicNode::UL);
16889   %}
16890   ins_pipe(pipe_class_memory);
16891 %}
16892 
16893 instruct string_indexof_conUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16894                               immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16895                               iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16896 %{
16897   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16898   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16899   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16900          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16901   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UU) "
16902             "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16903 
16904   ins_encode %{
16905     int icnt2 = (int)$int_cnt2$$constant;
16906     __ string_indexof($str1$$Register, $str2$$Register,
16907                       $cnt1$$Register, zr,
16908                       $tmp1$$Register, $tmp2$$Register,
16909                       $tmp3$$Register, $tmp4$$Register, zr, zr,
16910                       icnt2, $result$$Register, StrIntrinsicNode::UU);
16911   %}
16912   ins_pipe(pipe_class_memory);
16913 %}
16914 
16915 instruct string_indexof_conLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16916                               immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16917                               iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16918 %{
16919   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16920   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16921   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16922          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16923   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (LL) "
16924             "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16925 
16926   ins_encode %{
16927     int icnt2 = (int)$int_cnt2$$constant;
16928     __ string_indexof($str1$$Register, $str2$$Register,
16929                       $cnt1$$Register, zr,
16930                       $tmp1$$Register, $tmp2$$Register,
16931                       $tmp3$$Register, $tmp4$$Register, zr, zr,
16932                       icnt2, $result$$Register, StrIntrinsicNode::LL);
16933   %}
16934   ins_pipe(pipe_class_memory);
16935 %}
16936 
16937 instruct string_indexof_conUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16938                               immI_1 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16939                               iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16940 %{
16941   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16942   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16943   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16944          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16945   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UL) "
16946             "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16947 
16948   ins_encode %{
16949     int icnt2 = (int)$int_cnt2$$constant;
16950     __ string_indexof($str1$$Register, $str2$$Register,
16951                       $cnt1$$Register, zr,
16952                       $tmp1$$Register, $tmp2$$Register,
16953                       $tmp3$$Register, $tmp4$$Register, zr, zr,
16954                       icnt2, $result$$Register, StrIntrinsicNode::UL);
16955   %}
16956   ins_pipe(pipe_class_memory);
16957 %}
16958 
16959 instruct string_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16960                              iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16961                              iRegINoSp tmp3, rFlagsReg cr)
16962 %{
16963   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16964   predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
16965   effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16966          TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16967 
16968   format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16969 
16970   ins_encode %{
16971     __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16972                            $result$$Register, $tmp1$$Register, $tmp2$$Register,
16973                            $tmp3$$Register);
16974   %}
16975   ins_pipe(pipe_class_memory);
16976 %}
16977 
16978 instruct stringL_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16979                               iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16980                               iRegINoSp tmp3, rFlagsReg cr)
16981 %{
16982   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16983   predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
16984   effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16985          TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16986 
16987   format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16988 
16989   ins_encode %{
16990     __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16991                             $result$$Register, $tmp1$$Register, $tmp2$$Register,
16992                             $tmp3$$Register);
16993   %}
16994   ins_pipe(pipe_class_memory);
16995 %}
16996 
16997 instruct stringL_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16998                                   iRegI_R0 result, vecA ztmp1, vecA ztmp2,
16999                                   pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
17000   predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L);
17001   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
17002   effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
17003   format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
17004   ins_encode %{
17005     __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
17006                                $result$$Register, $ztmp1$$FloatRegister,
17007                                $ztmp2$$FloatRegister, $pgtmp$$PRegister,
17008                                $ptmp$$PRegister, true /* isL */);
17009   %}
17010   ins_pipe(pipe_class_memory);
17011 %}
17012 
17013 instruct stringU_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
17014                                   iRegI_R0 result, vecA ztmp1, vecA ztmp2,
17015                                   pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
17016   predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U);
17017   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
17018   effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
17019   format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
17020   ins_encode %{
17021     __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
17022                                $result$$Register, $ztmp1$$FloatRegister,
17023                                $ztmp2$$FloatRegister, $pgtmp$$PRegister,
17024                                $ptmp$$PRegister, false /* isL */);
17025   %}
17026   ins_pipe(pipe_class_memory);
17027 %}
17028 
17029 instruct string_equalsL(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
17030                         iRegI_R0 result, rFlagsReg cr)
17031 %{
17032   predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::LL);
17033   match(Set result (StrEquals (Binary str1 str2) cnt));
17034   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
17035 
17036   format %{ "String Equals $str1,$str2,$cnt -> $result" %}
17037   ins_encode %{
17038     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
17039     __ string_equals($str1$$Register, $str2$$Register,
17040                      $result$$Register, $cnt$$Register);
17041   %}
17042   ins_pipe(pipe_class_memory);
17043 %}
17044 
17045 instruct array_equalsB(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
17046                        iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
17047                        vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17048                        vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
17049                        iRegP_R10 tmp, rFlagsReg cr)
17050 %{
17051   predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
17052   match(Set result (AryEq ary1 ary2));
17053   effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
17054          TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
17055          TEMP vtmp6, TEMP vtmp7, KILL cr);
17056 
17057   format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
17058   ins_encode %{
17059     address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
17060                                    $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
17061                                    $result$$Register, $tmp$$Register, 1);
17062     if (tpc == nullptr) {
17063       ciEnv::current()->record_failure("CodeCache is full");
17064       return;
17065     }
17066   %}
17067   ins_pipe(pipe_class_memory);
17068 %}
17069 
17070 instruct array_equalsC(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
17071                        iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
17072                        vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17073                        vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
17074                        iRegP_R10 tmp, rFlagsReg cr)
17075 %{
17076   predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
17077   match(Set result (AryEq ary1 ary2));
17078   effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
17079          TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
17080          TEMP vtmp6, TEMP vtmp7, KILL cr);
17081 
17082   format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
17083   ins_encode %{
17084     address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
17085                                    $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
17086                                    $result$$Register, $tmp$$Register, 2);
17087     if (tpc == nullptr) {
17088       ciEnv::current()->record_failure("CodeCache is full");
17089       return;
17090     }
17091   %}
17092   ins_pipe(pipe_class_memory);
17093 %}
17094 
17095 instruct arrays_hashcode(iRegP_R1 ary, iRegI_R2 cnt, iRegI_R0 result, immI basic_type,
17096                          vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17097                          vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
17098                          vRegD_V12 vtmp8, vRegD_V13 vtmp9, rFlagsReg cr)
17099 %{
17100   match(Set result (VectorizedHashCode (Binary ary cnt) (Binary result basic_type)));
17101   effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5, TEMP vtmp6,
17102          TEMP vtmp7, TEMP vtmp8, TEMP vtmp9, USE_KILL ary, USE_KILL cnt, USE basic_type, KILL cr);
17103 
17104   format %{ "Array HashCode array[] $ary,$cnt,$result,$basic_type -> $result   // KILL all" %}
17105   ins_encode %{
17106     address tpc = __ arrays_hashcode($ary$$Register, $cnt$$Register, $result$$Register,
17107                                      $vtmp3$$FloatRegister, $vtmp2$$FloatRegister,
17108                                      $vtmp1$$FloatRegister, $vtmp0$$FloatRegister,
17109                                      $vtmp4$$FloatRegister, $vtmp5$$FloatRegister,
17110                                      $vtmp6$$FloatRegister, $vtmp7$$FloatRegister,
17111                                      $vtmp8$$FloatRegister, $vtmp9$$FloatRegister,
17112                                      (BasicType)$basic_type$$constant);
17113     if (tpc == nullptr) {
17114       ciEnv::current()->record_failure("CodeCache is full");
17115       return;
17116     }
17117   %}
17118   ins_pipe(pipe_class_memory);
17119 %}
17120 
17121 instruct count_positives(iRegP_R1 ary1, iRegI_R2 len, iRegI_R0 result, rFlagsReg cr)
17122 %{
17123   match(Set result (CountPositives ary1 len));
17124   effect(USE_KILL ary1, USE_KILL len, KILL cr);
17125   format %{ "count positives byte[] $ary1,$len -> $result" %}
17126   ins_encode %{
17127     address tpc = __ count_positives($ary1$$Register, $len$$Register, $result$$Register);
17128     if (tpc == nullptr) {
17129       ciEnv::current()->record_failure("CodeCache is full");
17130       return;
17131     }
17132   %}
17133   ins_pipe( pipe_slow );
17134 %}
17135 
17136 // fast char[] to byte[] compression
17137 instruct string_compress(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17138                          vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
17139                          vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
17140                          iRegI_R0 result, rFlagsReg cr)
17141 %{
17142   match(Set result (StrCompressedCopy src (Binary dst len)));
17143   effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
17144          USE_KILL src, USE_KILL dst, USE len, KILL cr);
17145 
17146   format %{ "String Compress $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
17147   ins_encode %{
17148     __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
17149                            $result$$Register, $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17150                            $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
17151                            $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
17152   %}
17153   ins_pipe(pipe_slow);
17154 %}
17155 
17156 // fast byte[] to char[] inflation
17157 instruct string_inflate(Universe dummy, iRegP_R0 src, iRegP_R1 dst, iRegI_R2 len, iRegP_R3 tmp,
17158                         vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17159                         vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, rFlagsReg cr)
17160 %{
17161   match(Set dummy (StrInflatedCopy src (Binary dst len)));
17162   effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3,
17163          TEMP vtmp4, TEMP vtmp5, TEMP vtmp6, TEMP tmp,
17164          USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
17165 
17166   format %{ "String Inflate $src,$dst # KILL $tmp $src $dst $len V0-V6 cr" %}
17167   ins_encode %{
17168     address tpc = __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
17169                                         $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17170                                         $vtmp2$$FloatRegister, $tmp$$Register);
17171     if (tpc == nullptr) {
17172       ciEnv::current()->record_failure("CodeCache is full");
17173       return;
17174     }
17175   %}
17176   ins_pipe(pipe_class_memory);
17177 %}
17178 
17179 // encode char[] to byte[] in ISO_8859_1
17180 instruct encode_iso_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17181                           vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
17182                           vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
17183                           iRegI_R0 result, rFlagsReg cr)
17184 %{
17185   predicate(!((EncodeISOArrayNode*)n)->is_ascii());
17186   match(Set result (EncodeISOArray src (Binary dst len)));
17187   effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
17188          KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
17189 
17190   format %{ "Encode ISO array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
17191   ins_encode %{
17192     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
17193                         $result$$Register, false,
17194                         $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17195                         $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
17196                         $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
17197   %}
17198   ins_pipe(pipe_class_memory);
17199 %}
17200 
17201 instruct encode_ascii_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17202                             vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
17203                             vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
17204                             iRegI_R0 result, rFlagsReg cr)
17205 %{
17206   predicate(((EncodeISOArrayNode*)n)->is_ascii());
17207   match(Set result (EncodeISOArray src (Binary dst len)));
17208   effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
17209          KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
17210 
17211   format %{ "Encode ASCII array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
17212   ins_encode %{
17213     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
17214                         $result$$Register, true,
17215                         $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17216                         $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
17217                         $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
17218   %}
17219   ins_pipe(pipe_class_memory);
17220 %}
17221 
17222 //----------------------------- CompressBits/ExpandBits ------------------------
17223 
17224 instruct compressBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
17225                            vRegF tdst, vRegF tsrc, vRegF tmask) %{
17226   match(Set dst (CompressBits src mask));
17227   effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17228   format %{ "mov    $tsrc, $src\n\t"
17229             "mov    $tmask, $mask\n\t"
17230             "bext   $tdst, $tsrc, $tmask\n\t"
17231             "mov    $dst, $tdst"
17232           %}
17233   ins_encode %{
17234     __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
17235     __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
17236     __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17237     __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17238   %}
17239   ins_pipe(pipe_slow);
17240 %}
17241 
17242 instruct compressBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
17243                            vRegF tdst, vRegF tsrc, vRegF tmask) %{
17244   match(Set dst (CompressBits (LoadI mem) mask));
17245   effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17246   format %{ "ldrs   $tsrc, $mem\n\t"
17247             "ldrs   $tmask, $mask\n\t"
17248             "bext   $tdst, $tsrc, $tmask\n\t"
17249             "mov    $dst, $tdst"
17250           %}
17251   ins_encode %{
17252     loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
17253               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
17254     __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
17255     __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17256     __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17257   %}
17258   ins_pipe(pipe_slow);
17259 %}
17260 
17261 instruct compressBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
17262                            vRegD tdst, vRegD tsrc, vRegD tmask) %{
17263   match(Set dst (CompressBits src mask));
17264   effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17265   format %{ "mov    $tsrc, $src\n\t"
17266             "mov    $tmask, $mask\n\t"
17267             "bext   $tdst, $tsrc, $tmask\n\t"
17268             "mov    $dst, $tdst"
17269           %}
17270   ins_encode %{
17271     __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
17272     __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
17273     __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17274     __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17275   %}
17276   ins_pipe(pipe_slow);
17277 %}
17278 
17279 instruct compressBitsL_memcon(iRegLNoSp dst, memory8 mem, immL mask,
17280                            vRegF tdst, vRegF tsrc, vRegF tmask) %{
17281   match(Set dst (CompressBits (LoadL mem) mask));
17282   effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17283   format %{ "ldrd   $tsrc, $mem\n\t"
17284             "ldrd   $tmask, $mask\n\t"
17285             "bext   $tdst, $tsrc, $tmask\n\t"
17286             "mov    $dst, $tdst"
17287           %}
17288   ins_encode %{
17289     loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
17290               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
17291     __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
17292     __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17293     __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17294   %}
17295   ins_pipe(pipe_slow);
17296 %}
17297 
17298 instruct expandBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
17299                          vRegF tdst, vRegF tsrc, vRegF tmask) %{
17300   match(Set dst (ExpandBits src mask));
17301   effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17302   format %{ "mov    $tsrc, $src\n\t"
17303             "mov    $tmask, $mask\n\t"
17304             "bdep   $tdst, $tsrc, $tmask\n\t"
17305             "mov    $dst, $tdst"
17306           %}
17307   ins_encode %{
17308     __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
17309     __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
17310     __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17311     __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17312   %}
17313   ins_pipe(pipe_slow);
17314 %}
17315 
17316 instruct expandBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
17317                          vRegF tdst, vRegF tsrc, vRegF tmask) %{
17318   match(Set dst (ExpandBits (LoadI mem) mask));
17319   effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17320   format %{ "ldrs   $tsrc, $mem\n\t"
17321             "ldrs   $tmask, $mask\n\t"
17322             "bdep   $tdst, $tsrc, $tmask\n\t"
17323             "mov    $dst, $tdst"
17324           %}
17325   ins_encode %{
17326     loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
17327               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
17328     __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
17329     __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17330     __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17331   %}
17332   ins_pipe(pipe_slow);
17333 %}
17334 
17335 instruct expandBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
17336                          vRegD tdst, vRegD tsrc, vRegD tmask) %{
17337   match(Set dst (ExpandBits src mask));
17338   effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17339   format %{ "mov    $tsrc, $src\n\t"
17340             "mov    $tmask, $mask\n\t"
17341             "bdep   $tdst, $tsrc, $tmask\n\t"
17342             "mov    $dst, $tdst"
17343           %}
17344   ins_encode %{
17345     __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
17346     __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
17347     __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17348     __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17349   %}
17350   ins_pipe(pipe_slow);
17351 %}
17352 
17353 
17354 instruct expandBitsL_memcon(iRegINoSp dst, memory8 mem, immL mask,
17355                          vRegF tdst, vRegF tsrc, vRegF tmask) %{
17356   match(Set dst (ExpandBits (LoadL mem) mask));
17357   effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17358   format %{ "ldrd   $tsrc, $mem\n\t"
17359             "ldrd   $tmask, $mask\n\t"
17360             "bdep   $tdst, $tsrc, $tmask\n\t"
17361             "mov    $dst, $tdst"
17362           %}
17363   ins_encode %{
17364     loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
17365               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
17366     __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
17367     __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17368     __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17369   %}
17370   ins_pipe(pipe_slow);
17371 %}
17372 
17373 //----------------------------- Reinterpret ----------------------------------
17374 // Reinterpret a half-precision float value in a floating point register to a general purpose register
17375 instruct reinterpretHF2S(iRegINoSp dst, vRegF src) %{
17376   match(Set dst (ReinterpretHF2S src));
17377   format %{ "reinterpretHF2S $dst, $src" %}
17378   ins_encode %{
17379     __ smov($dst$$Register, $src$$FloatRegister, __ H, 0);
17380   %}
17381   ins_pipe(pipe_slow);
17382 %}
17383 
17384 // Reinterpret a half-precision float value in a general purpose register to a floating point register
17385 instruct reinterpretS2HF(vRegF dst, iRegINoSp src) %{
17386   match(Set dst (ReinterpretS2HF src));
17387   format %{ "reinterpretS2HF $dst, $src" %}
17388   ins_encode %{
17389     __ mov($dst$$FloatRegister, __ H, 0, $src$$Register);
17390   %}
17391   ins_pipe(pipe_slow);
17392 %}
17393 
17394 // Without this optimization, ReinterpretS2HF (ConvF2HF src) would result in the following
17395 // instructions (the first two are for ConvF2HF and the last instruction is for ReinterpretS2HF) -
17396 // fcvt $tmp1_fpr, $src_fpr    // Convert float to half-precision float
17397 // mov  $tmp2_gpr, $tmp1_fpr   // Move half-precision float in FPR to a GPR
17398 // mov  $dst_fpr,  $tmp2_gpr   // Move the result from a GPR to an FPR
17399 // The move from FPR to GPR in ConvF2HF and the move from GPR to FPR in ReinterpretS2HF
17400 // can be omitted in this pattern, resulting in -
17401 // fcvt $dst, $src  // Convert float to half-precision float
17402 instruct convF2HFAndS2HF(vRegF dst, vRegF src)
17403 %{
17404   match(Set dst (ReinterpretS2HF (ConvF2HF src)));
17405   format %{ "convF2HFAndS2HF $dst, $src" %}
17406   ins_encode %{
17407     __ fcvtsh($dst$$FloatRegister, $src$$FloatRegister);
17408   %}
17409   ins_pipe(pipe_slow);
17410 %}
17411 
17412 // Without this optimization, ConvHF2F (ReinterpretHF2S src) would result in the following
17413 // instructions (the first one is for ReinterpretHF2S and the last two are for ConvHF2F) -
17414 // mov  $tmp1_gpr, $src_fpr  // Move the half-precision float from an FPR to a GPR
17415 // mov  $tmp2_fpr, $tmp1_gpr // Move the same value from GPR to an FPR
17416 // fcvt $dst_fpr,  $tmp2_fpr // Convert the half-precision float to 32-bit float
17417 // The move from FPR to GPR in ReinterpretHF2S and the move from GPR to FPR in ConvHF2F
17418 // can be omitted as the input (src) is already in an FPR required for the fcvths instruction
17419 // resulting in -
17420 // fcvt $dst, $src  // Convert half-precision float to a 32-bit float
17421 instruct convHF2SAndHF2F(vRegF dst, vRegF src)
17422 %{
17423   match(Set dst (ConvHF2F (ReinterpretHF2S src)));
17424   format %{ "convHF2SAndHF2F $dst, $src" %}
17425   ins_encode %{
17426     __ fcvths($dst$$FloatRegister, $src$$FloatRegister);
17427   %}
17428   ins_pipe(pipe_slow);
17429 %}
17430 
17431 // ============================================================================
17432 // This name is KNOWN by the ADLC and cannot be changed.
17433 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
17434 // for this guy.
17435 instruct tlsLoadP(thread_RegP dst)
17436 %{
17437   match(Set dst (ThreadLocal));
17438 
17439   ins_cost(0);
17440 
17441   format %{ " -- \t// $dst=Thread::current(), empty" %}
17442 
17443   size(0);
17444 
17445   ins_encode( /*empty*/ );
17446 
17447   ins_pipe(pipe_class_empty);
17448 %}
17449 
17450 //----------PEEPHOLE RULES-----------------------------------------------------
17451 // These must follow all instruction definitions as they use the names
17452 // defined in the instructions definitions.
17453 //
17454 // peepmatch ( root_instr_name [preceding_instruction]* );
17455 //
17456 // peepconstraint %{
17457 // (instruction_number.operand_name relational_op instruction_number.operand_name
17458 //  [, ...] );
17459 // // instruction numbers are zero-based using left to right order in peepmatch
17460 //
17461 // peepreplace ( instr_name  ( [instruction_number.operand_name]* ) );
17462 // // provide an instruction_number.operand_name for each operand that appears
17463 // // in the replacement instruction's match rule
17464 //
17465 // ---------VM FLAGS---------------------------------------------------------
17466 //
17467 // All peephole optimizations can be turned off using -XX:-OptoPeephole
17468 //
17469 // Each peephole rule is given an identifying number starting with zero and
17470 // increasing by one in the order seen by the parser.  An individual peephole
17471 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
17472 // on the command-line.
17473 //
17474 // ---------CURRENT LIMITATIONS----------------------------------------------
17475 //
17476 // Only match adjacent instructions in same basic block
17477 // Only equality constraints
17478 // Only constraints between operands, not (0.dest_reg == RAX_enc)
17479 // Only one replacement instruction
17480 //
17481 // ---------EXAMPLE----------------------------------------------------------
17482 //
17483 // // pertinent parts of existing instructions in architecture description
17484 // instruct movI(iRegINoSp dst, iRegI src)
17485 // %{
17486 //   match(Set dst (CopyI src));
17487 // %}
17488 //
17489 // instruct incI_iReg(iRegINoSp dst, immI1 src, rFlagsReg cr)
17490 // %{
17491 //   match(Set dst (AddI dst src));
17492 //   effect(KILL cr);
17493 // %}
17494 //
17495 // // Change (inc mov) to lea
17496 // peephole %{
17497 //   // increment preceded by register-register move
17498 //   peepmatch ( incI_iReg movI );
17499 //   // require that the destination register of the increment
17500 //   // match the destination register of the move
17501 //   peepconstraint ( 0.dst == 1.dst );
17502 //   // construct a replacement instruction that sets
17503 //   // the destination to ( move's source register + one )
17504 //   peepreplace ( leaI_iReg_immI( 0.dst 1.src 0.src ) );
17505 // %}
17506 //
17507 
17508 // Implementation no longer uses movX instructions since
17509 // machine-independent system no longer uses CopyX nodes.
17510 //
17511 // peephole
17512 // %{
17513 //   peepmatch (incI_iReg movI);
17514 //   peepconstraint (0.dst == 1.dst);
17515 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17516 // %}
17517 
17518 // peephole
17519 // %{
17520 //   peepmatch (decI_iReg movI);
17521 //   peepconstraint (0.dst == 1.dst);
17522 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17523 // %}
17524 
17525 // peephole
17526 // %{
17527 //   peepmatch (addI_iReg_imm movI);
17528 //   peepconstraint (0.dst == 1.dst);
17529 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17530 // %}
17531 
17532 // peephole
17533 // %{
17534 //   peepmatch (incL_iReg movL);
17535 //   peepconstraint (0.dst == 1.dst);
17536 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17537 // %}
17538 
17539 // peephole
17540 // %{
17541 //   peepmatch (decL_iReg movL);
17542 //   peepconstraint (0.dst == 1.dst);
17543 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17544 // %}
17545 
17546 // peephole
17547 // %{
17548 //   peepmatch (addL_iReg_imm movL);
17549 //   peepconstraint (0.dst == 1.dst);
17550 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17551 // %}
17552 
17553 // peephole
17554 // %{
17555 //   peepmatch (addP_iReg_imm movP);
17556 //   peepconstraint (0.dst == 1.dst);
17557 //   peepreplace (leaP_iReg_imm(0.dst 1.src 0.src));
17558 // %}
17559 
17560 // // Change load of spilled value to only a spill
17561 // instruct storeI(memory mem, iRegI src)
17562 // %{
17563 //   match(Set mem (StoreI mem src));
17564 // %}
17565 //
17566 // instruct loadI(iRegINoSp dst, memory mem)
17567 // %{
17568 //   match(Set dst (LoadI mem));
17569 // %}
17570 //
17571 
17572 //----------SMARTSPILL RULES---------------------------------------------------
17573 // These must follow all instruction definitions as they use the names
17574 // defined in the instructions definitions.
17575 
17576 // Local Variables:
17577 // mode: c++
17578 // End: