1 //
    2 // Copyright (c) 2003, 2025, Oracle and/or its affiliates. All rights reserved.
    3 // Copyright (c) 2014, 2024, Red Hat, Inc. All rights reserved.
    4 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
    5 //
    6 // This code is free software; you can redistribute it and/or modify it
    7 // under the terms of the GNU General Public License version 2 only, as
    8 // published by the Free Software Foundation.
    9 //
   10 // This code is distributed in the hope that it will be useful, but WITHOUT
   11 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
   12 // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
   13 // version 2 for more details (a copy is included in the LICENSE file that
   14 // accompanied this code).
   15 //
   16 // You should have received a copy of the GNU General Public License version
   17 // 2 along with this work; if not, write to the Free Software Foundation,
   18 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
   19 //
   20 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
   21 // or visit www.oracle.com if you need additional information or have any
   22 // questions.
   23 //
   24 //
   25 
   26 // AArch64 Architecture Description File
   27 
   28 //----------REGISTER DEFINITION BLOCK------------------------------------------
   29 // This information is used by the matcher and the register allocator to
   30 // describe individual registers and classes of registers within the target
   31 // architecture.
   32 
   33 register %{
   34 //----------Architecture Description Register Definitions----------------------
   35 // General Registers
   36 // "reg_def"  name ( register save type, C convention save type,
   37 //                   ideal register type, encoding );
   38 // Register Save Types:
   39 //
   40 // NS  = No-Save:       The register allocator assumes that these registers
   41 //                      can be used without saving upon entry to the method, &
   42 //                      that they do not need to be saved at call sites.
   43 //
   44 // SOC = Save-On-Call:  The register allocator assumes that these registers
   45 //                      can be used without saving upon entry to the method,
   46 //                      but that they must be saved at call sites.
   47 //
   48 // SOE = Save-On-Entry: The register allocator assumes that these registers
   49 //                      must be saved before using them upon entry to the
   50 //                      method, but they do not need to be saved at call
   51 //                      sites.
   52 //
   53 // AS  = Always-Save:   The register allocator assumes that these registers
   54 //                      must be saved before using them upon entry to the
   55 //                      method, & that they must be saved at call sites.
   56 //
   57 // Ideal Register Type is used to determine how to save & restore a
   58 // register.  Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
   59 // spilled with LoadP/StoreP.  If the register supports both, use Op_RegI.
   60 //
   61 // The encoding number is the actual bit-pattern placed into the opcodes.
   62 
   63 // We must define the 64 bit int registers in two 32 bit halves, the
   64 // real lower register and a virtual upper half register. upper halves
   65 // are used by the register allocator but are not actually supplied as
   66 // operands to memory ops.
   67 //
   68 // follow the C1 compiler in making registers
   69 //
   70 //   r0-r7,r10-r26 volatile (caller save)
   71 //   r27-r32 system (no save, no allocate)
   72 //   r8-r9 non-allocatable (so we can use them as scratch regs)
   73 //
   74 // as regards Java usage. we don't use any callee save registers
   75 // because this makes it difficult to de-optimise a frame (see comment
   76 // in x86 implementation of Deoptimization::unwind_callee_save_values)
   77 //
   78 
   79 // General Registers
   80 
   81 reg_def R0      ( SOC, SOC, Op_RegI,  0, r0->as_VMReg()         );
   82 reg_def R0_H    ( SOC, SOC, Op_RegI,  0, r0->as_VMReg()->next() );
   83 reg_def R1      ( SOC, SOC, Op_RegI,  1, r1->as_VMReg()         );
   84 reg_def R1_H    ( SOC, SOC, Op_RegI,  1, r1->as_VMReg()->next() );
   85 reg_def R2      ( SOC, SOC, Op_RegI,  2, r2->as_VMReg()         );
   86 reg_def R2_H    ( SOC, SOC, Op_RegI,  2, r2->as_VMReg()->next() );
   87 reg_def R3      ( SOC, SOC, Op_RegI,  3, r3->as_VMReg()         );
   88 reg_def R3_H    ( SOC, SOC, Op_RegI,  3, r3->as_VMReg()->next() );
   89 reg_def R4      ( SOC, SOC, Op_RegI,  4, r4->as_VMReg()         );
   90 reg_def R4_H    ( SOC, SOC, Op_RegI,  4, r4->as_VMReg()->next() );
   91 reg_def R5      ( SOC, SOC, Op_RegI,  5, r5->as_VMReg()         );
   92 reg_def R5_H    ( SOC, SOC, Op_RegI,  5, r5->as_VMReg()->next() );
   93 reg_def R6      ( SOC, SOC, Op_RegI,  6, r6->as_VMReg()         );
   94 reg_def R6_H    ( SOC, SOC, Op_RegI,  6, r6->as_VMReg()->next() );
   95 reg_def R7      ( SOC, SOC, Op_RegI,  7, r7->as_VMReg()         );
   96 reg_def R7_H    ( SOC, SOC, Op_RegI,  7, r7->as_VMReg()->next() );
   97 reg_def R8      ( NS,  SOC, Op_RegI,  8, r8->as_VMReg()         ); // rscratch1, non-allocatable
   98 reg_def R8_H    ( NS,  SOC, Op_RegI,  8, r8->as_VMReg()->next() );
   99 reg_def R9      ( NS,  SOC, Op_RegI,  9, r9->as_VMReg()         ); // rscratch2, non-allocatable
  100 reg_def R9_H    ( NS,  SOC, Op_RegI,  9, r9->as_VMReg()->next() );
  101 reg_def R10     ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()        );
  102 reg_def R10_H   ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
  103 reg_def R11     ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()        );
  104 reg_def R11_H   ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
  105 reg_def R12     ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()        );
  106 reg_def R12_H   ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()->next());
  107 reg_def R13     ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()        );
  108 reg_def R13_H   ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()->next());
  109 reg_def R14     ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()        );
  110 reg_def R14_H   ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()->next());
  111 reg_def R15     ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()        );
  112 reg_def R15_H   ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()->next());
  113 reg_def R16     ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()        );
  114 reg_def R16_H   ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
  115 reg_def R17     ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()        );
  116 reg_def R17_H   ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
  117 reg_def R18     ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()        );
  118 reg_def R18_H   ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()->next());
  119 reg_def R19     ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()        );
  120 reg_def R19_H   ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()->next());
  121 reg_def R20     ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()        ); // caller esp
  122 reg_def R20_H   ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()->next());
  123 reg_def R21     ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()        );
  124 reg_def R21_H   ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()->next());
  125 reg_def R22     ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()        );
  126 reg_def R22_H   ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()->next());
  127 reg_def R23     ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()        );
  128 reg_def R23_H   ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()->next());
  129 reg_def R24     ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()        );
  130 reg_def R24_H   ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()->next());
  131 reg_def R25     ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()        );
  132 reg_def R25_H   ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()->next());
  133 reg_def R26     ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()        );
  134 reg_def R26_H   ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()->next());
  135 reg_def R27     ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()        ); // heapbase
  136 reg_def R27_H   ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()->next());
  137 reg_def R28     (  NS, SOE, Op_RegI, 28, r28->as_VMReg()        ); // thread
  138 reg_def R28_H   (  NS, SOE, Op_RegI, 28, r28->as_VMReg()->next());
  139 reg_def R29     (  NS,  NS, Op_RegI, 29, r29->as_VMReg()        ); // fp
  140 reg_def R29_H   (  NS,  NS, Op_RegI, 29, r29->as_VMReg()->next());
  141 reg_def R30     (  NS,  NS, Op_RegI, 30, r30->as_VMReg()        ); // lr
  142 reg_def R30_H   (  NS,  NS, Op_RegI, 30, r30->as_VMReg()->next());
  143 reg_def R31     (  NS,  NS, Op_RegI, 31, r31_sp->as_VMReg()     ); // sp
  144 reg_def R31_H   (  NS,  NS, Op_RegI, 31, r31_sp->as_VMReg()->next());
  145 
  146 // ----------------------------
  147 // Float/Double/Vector Registers
  148 // ----------------------------
  149 
  150 // Double Registers
  151 
  152 // The rules of ADL require that double registers be defined in pairs.
  153 // Each pair must be two 32-bit values, but not necessarily a pair of
  154 // single float registers. In each pair, ADLC-assigned register numbers
  155 // must be adjacent, with the lower number even. Finally, when the
  156 // CPU stores such a register pair to memory, the word associated with
  157 // the lower ADLC-assigned number must be stored to the lower address.
  158 
  159 // AArch64 has 32 floating-point registers. Each can store a vector of
  160 // single or double precision floating-point values up to 8 * 32
  161 // floats, 4 * 64 bit floats or 2 * 128 bit floats.  We currently only
  162 // use the first float or double element of the vector.
  163 
  164 // for Java use float registers v0-v15 are always save on call whereas
  165 // the platform ABI treats v8-v15 as callee save). float registers
  166 // v16-v31 are SOC as per the platform spec
  167 
  168 // For SVE vector registers, we simply extend vector register size to 8
  169 // 'logical' slots. This is nominally 256 bits but it actually covers
  170 // all possible 'physical' SVE vector register lengths from 128 ~ 2048
  171 // bits. The 'physical' SVE vector register length is detected during
  172 // startup, so the register allocator is able to identify the correct
  173 // number of bytes needed for an SVE spill/unspill.
  174 // Note that a vector register with 4 slots denotes a 128-bit NEON
  175 // register allowing it to be distinguished from the corresponding SVE
  176 // vector register when the SVE vector length is 128 bits.
  177 
  178   reg_def V0   ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()          );
  179   reg_def V0_H ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next()  );
  180   reg_def V0_J ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(2) );
  181   reg_def V0_K ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(3) );
  182 
  183   reg_def V1   ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()          );
  184   reg_def V1_H ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next()  );
  185   reg_def V1_J ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(2) );
  186   reg_def V1_K ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(3) );
  187 
  188   reg_def V2   ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()          );
  189   reg_def V2_H ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next()  );
  190   reg_def V2_J ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(2) );
  191   reg_def V2_K ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(3) );
  192 
  193   reg_def V3   ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()          );
  194   reg_def V3_H ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next()  );
  195   reg_def V3_J ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(2) );
  196   reg_def V3_K ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(3) );
  197 
  198   reg_def V4   ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()          );
  199   reg_def V4_H ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next()  );
  200   reg_def V4_J ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(2) );
  201   reg_def V4_K ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(3) );
  202 
  203   reg_def V5   ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()          );
  204   reg_def V5_H ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next()  );
  205   reg_def V5_J ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(2) );
  206   reg_def V5_K ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(3) );
  207 
  208   reg_def V6   ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()          );
  209   reg_def V6_H ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next()  );
  210   reg_def V6_J ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(2) );
  211   reg_def V6_K ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(3) );
  212 
  213   reg_def V7   ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()          );
  214   reg_def V7_H ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next()  );
  215   reg_def V7_J ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(2) );
  216   reg_def V7_K ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(3) );
  217 
  218   reg_def V8   ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()          );
  219   reg_def V8_H ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()->next()  );
  220   reg_def V8_J ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(2) );
  221   reg_def V8_K ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(3) );
  222 
  223   reg_def V9   ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()          );
  224   reg_def V9_H ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()->next()  );
  225   reg_def V9_J ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(2) );
  226   reg_def V9_K ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(3) );
  227 
  228   reg_def V10   ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()          );
  229   reg_def V10_H ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()->next()  );
  230   reg_def V10_J ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(2) );
  231   reg_def V10_K ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(3) );
  232 
  233   reg_def V11   ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()          );
  234   reg_def V11_H ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()->next()  );
  235   reg_def V11_J ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(2) );
  236   reg_def V11_K ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(3) );
  237 
  238   reg_def V12   ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()          );
  239   reg_def V12_H ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()->next()  );
  240   reg_def V12_J ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(2) );
  241   reg_def V12_K ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(3) );
  242 
  243   reg_def V13   ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()          );
  244   reg_def V13_H ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()->next()  );
  245   reg_def V13_J ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(2) );
  246   reg_def V13_K ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(3) );
  247 
  248   reg_def V14   ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()          );
  249   reg_def V14_H ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()->next()  );
  250   reg_def V14_J ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(2) );
  251   reg_def V14_K ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(3) );
  252 
  253   reg_def V15   ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()          );
  254   reg_def V15_H ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()->next()  );
  255   reg_def V15_J ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(2) );
  256   reg_def V15_K ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(3) );
  257 
  258   reg_def V16   ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()          );
  259   reg_def V16_H ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next()  );
  260   reg_def V16_J ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(2) );
  261   reg_def V16_K ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(3) );
  262 
  263   reg_def V17   ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()          );
  264   reg_def V17_H ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next()  );
  265   reg_def V17_J ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(2) );
  266   reg_def V17_K ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(3) );
  267 
  268   reg_def V18   ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()          );
  269   reg_def V18_H ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next()  );
  270   reg_def V18_J ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(2) );
  271   reg_def V18_K ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(3) );
  272 
  273   reg_def V19   ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()          );
  274   reg_def V19_H ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next()  );
  275   reg_def V19_J ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(2) );
  276   reg_def V19_K ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(3) );
  277 
  278   reg_def V20   ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()          );
  279   reg_def V20_H ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next()  );
  280   reg_def V20_J ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(2) );
  281   reg_def V20_K ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(3) );
  282 
  283   reg_def V21   ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()          );
  284   reg_def V21_H ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next()  );
  285   reg_def V21_J ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(2) );
  286   reg_def V21_K ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(3) );
  287 
  288   reg_def V22   ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()          );
  289   reg_def V22_H ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next()  );
  290   reg_def V22_J ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(2) );
  291   reg_def V22_K ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(3) );
  292 
  293   reg_def V23   ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()          );
  294   reg_def V23_H ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next()  );
  295   reg_def V23_J ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(2) );
  296   reg_def V23_K ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(3) );
  297 
  298   reg_def V24   ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()          );
  299   reg_def V24_H ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next()  );
  300   reg_def V24_J ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(2) );
  301   reg_def V24_K ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(3) );
  302 
  303   reg_def V25   ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()          );
  304   reg_def V25_H ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next()  );
  305   reg_def V25_J ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(2) );
  306   reg_def V25_K ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(3) );
  307 
  308   reg_def V26   ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()          );
  309   reg_def V26_H ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next()  );
  310   reg_def V26_J ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(2) );
  311   reg_def V26_K ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(3) );
  312 
  313   reg_def V27   ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()          );
  314   reg_def V27_H ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next()  );
  315   reg_def V27_J ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(2) );
  316   reg_def V27_K ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(3) );
  317 
  318   reg_def V28   ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()          );
  319   reg_def V28_H ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next()  );
  320   reg_def V28_J ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(2) );
  321   reg_def V28_K ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(3) );
  322 
  323   reg_def V29   ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()          );
  324   reg_def V29_H ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next()  );
  325   reg_def V29_J ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(2) );
  326   reg_def V29_K ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(3) );
  327 
  328   reg_def V30   ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()          );
  329   reg_def V30_H ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next()  );
  330   reg_def V30_J ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(2) );
  331   reg_def V30_K ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(3) );
  332 
  333   reg_def V31   ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()          );
  334   reg_def V31_H ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next()  );
  335   reg_def V31_J ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(2) );
  336   reg_def V31_K ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(3) );
  337 
  338 // ----------------------------
  339 // SVE Predicate Registers
  340 // ----------------------------
  341   reg_def P0 (SOC, SOC, Op_RegVectMask, 0, p0->as_VMReg());
  342   reg_def P1 (SOC, SOC, Op_RegVectMask, 1, p1->as_VMReg());
  343   reg_def P2 (SOC, SOC, Op_RegVectMask, 2, p2->as_VMReg());
  344   reg_def P3 (SOC, SOC, Op_RegVectMask, 3, p3->as_VMReg());
  345   reg_def P4 (SOC, SOC, Op_RegVectMask, 4, p4->as_VMReg());
  346   reg_def P5 (SOC, SOC, Op_RegVectMask, 5, p5->as_VMReg());
  347   reg_def P6 (SOC, SOC, Op_RegVectMask, 6, p6->as_VMReg());
  348   reg_def P7 (SOC, SOC, Op_RegVectMask, 7, p7->as_VMReg());
  349   reg_def P8 (SOC, SOC, Op_RegVectMask, 8, p8->as_VMReg());
  350   reg_def P9 (SOC, SOC, Op_RegVectMask, 9, p9->as_VMReg());
  351   reg_def P10 (SOC, SOC, Op_RegVectMask, 10, p10->as_VMReg());
  352   reg_def P11 (SOC, SOC, Op_RegVectMask, 11, p11->as_VMReg());
  353   reg_def P12 (SOC, SOC, Op_RegVectMask, 12, p12->as_VMReg());
  354   reg_def P13 (SOC, SOC, Op_RegVectMask, 13, p13->as_VMReg());
  355   reg_def P14 (SOC, SOC, Op_RegVectMask, 14, p14->as_VMReg());
  356   reg_def P15 (SOC, SOC, Op_RegVectMask, 15, p15->as_VMReg());
  357 
  358 // ----------------------------
  359 // Special Registers
  360 // ----------------------------
  361 
  362 // the AArch64 CSPR status flag register is not directly accessible as
  363 // instruction operand. the FPSR status flag register is a system
  364 // register which can be written/read using MSR/MRS but again does not
  365 // appear as an operand (a code identifying the FSPR occurs as an
  366 // immediate value in the instruction).
  367 
  368 reg_def RFLAGS(SOC, SOC, 0, 32, VMRegImpl::Bad());
  369 
  370 // Specify priority of register selection within phases of register
  371 // allocation.  Highest priority is first.  A useful heuristic is to
  372 // give registers a low priority when they are required by machine
  373 // instructions, like EAX and EDX on I486, and choose no-save registers
  374 // before save-on-call, & save-on-call before save-on-entry.  Registers
  375 // which participate in fixed calling sequences should come last.
  376 // Registers which are used as pairs must fall on an even boundary.
  377 
  378 alloc_class chunk0(
  379     // volatiles
  380     R10, R10_H,
  381     R11, R11_H,
  382     R12, R12_H,
  383     R13, R13_H,
  384     R14, R14_H,
  385     R15, R15_H,
  386     R16, R16_H,
  387     R17, R17_H,
  388     R18, R18_H,
  389 
  390     // arg registers
  391     R0, R0_H,
  392     R1, R1_H,
  393     R2, R2_H,
  394     R3, R3_H,
  395     R4, R4_H,
  396     R5, R5_H,
  397     R6, R6_H,
  398     R7, R7_H,
  399 
  400     // non-volatiles
  401     R19, R19_H,
  402     R20, R20_H,
  403     R21, R21_H,
  404     R22, R22_H,
  405     R23, R23_H,
  406     R24, R24_H,
  407     R25, R25_H,
  408     R26, R26_H,
  409 
  410     // non-allocatable registers
  411 
  412     R27, R27_H, // heapbase
  413     R28, R28_H, // thread
  414     R29, R29_H, // fp
  415     R30, R30_H, // lr
  416     R31, R31_H, // sp
  417     R8, R8_H,   // rscratch1
  418     R9, R9_H,   // rscratch2
  419 );
  420 
  421 alloc_class chunk1(
  422 
  423     // no save
  424     V16, V16_H, V16_J, V16_K,
  425     V17, V17_H, V17_J, V17_K,
  426     V18, V18_H, V18_J, V18_K,
  427     V19, V19_H, V19_J, V19_K,
  428     V20, V20_H, V20_J, V20_K,
  429     V21, V21_H, V21_J, V21_K,
  430     V22, V22_H, V22_J, V22_K,
  431     V23, V23_H, V23_J, V23_K,
  432     V24, V24_H, V24_J, V24_K,
  433     V25, V25_H, V25_J, V25_K,
  434     V26, V26_H, V26_J, V26_K,
  435     V27, V27_H, V27_J, V27_K,
  436     V28, V28_H, V28_J, V28_K,
  437     V29, V29_H, V29_J, V29_K,
  438     V30, V30_H, V30_J, V30_K,
  439     V31, V31_H, V31_J, V31_K,
  440 
  441     // arg registers
  442     V0, V0_H, V0_J, V0_K,
  443     V1, V1_H, V1_J, V1_K,
  444     V2, V2_H, V2_J, V2_K,
  445     V3, V3_H, V3_J, V3_K,
  446     V4, V4_H, V4_J, V4_K,
  447     V5, V5_H, V5_J, V5_K,
  448     V6, V6_H, V6_J, V6_K,
  449     V7, V7_H, V7_J, V7_K,
  450 
  451     // non-volatiles
  452     V8, V8_H, V8_J, V8_K,
  453     V9, V9_H, V9_J, V9_K,
  454     V10, V10_H, V10_J, V10_K,
  455     V11, V11_H, V11_J, V11_K,
  456     V12, V12_H, V12_J, V12_K,
  457     V13, V13_H, V13_J, V13_K,
  458     V14, V14_H, V14_J, V14_K,
  459     V15, V15_H, V15_J, V15_K,
  460 );
  461 
  462 alloc_class chunk2 (
  463     // Governing predicates for load/store and arithmetic
  464     P0,
  465     P1,
  466     P2,
  467     P3,
  468     P4,
  469     P5,
  470     P6,
  471 
  472     // Extra predicates
  473     P8,
  474     P9,
  475     P10,
  476     P11,
  477     P12,
  478     P13,
  479     P14,
  480     P15,
  481 
  482     // Preserved for all-true predicate
  483     P7,
  484 );
  485 
  486 alloc_class chunk3(RFLAGS);
  487 
  488 //----------Architecture Description Register Classes--------------------------
  489 // Several register classes are automatically defined based upon information in
  490 // this architecture description.
  491 // 1) reg_class inline_cache_reg           ( /* as def'd in frame section */ )
  492 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
  493 //
  494 
  495 // Class for all 32 bit general purpose registers
  496 reg_class all_reg32(
  497     R0,
  498     R1,
  499     R2,
  500     R3,
  501     R4,
  502     R5,
  503     R6,
  504     R7,
  505     R10,
  506     R11,
  507     R12,
  508     R13,
  509     R14,
  510     R15,
  511     R16,
  512     R17,
  513     R18,
  514     R19,
  515     R20,
  516     R21,
  517     R22,
  518     R23,
  519     R24,
  520     R25,
  521     R26,
  522     R27,
  523     R28,
  524     R29,
  525     R30,
  526     R31
  527 );
  528 
  529 
  530 // Class for all 32 bit integer registers (excluding SP which
  531 // will never be used as an integer register)
  532 reg_class any_reg32 %{
  533   return _ANY_REG32_mask;
  534 %}
  535 
  536 // Singleton class for R0 int register
  537 reg_class int_r0_reg(R0);
  538 
  539 // Singleton class for R2 int register
  540 reg_class int_r2_reg(R2);
  541 
  542 // Singleton class for R3 int register
  543 reg_class int_r3_reg(R3);
  544 
  545 // Singleton class for R4 int register
  546 reg_class int_r4_reg(R4);
  547 
  548 // Singleton class for R31 int register
  549 reg_class int_r31_reg(R31);
  550 
  551 // Class for all 64 bit general purpose registers
  552 reg_class all_reg(
  553     R0, R0_H,
  554     R1, R1_H,
  555     R2, R2_H,
  556     R3, R3_H,
  557     R4, R4_H,
  558     R5, R5_H,
  559     R6, R6_H,
  560     R7, R7_H,
  561     R10, R10_H,
  562     R11, R11_H,
  563     R12, R12_H,
  564     R13, R13_H,
  565     R14, R14_H,
  566     R15, R15_H,
  567     R16, R16_H,
  568     R17, R17_H,
  569     R18, R18_H,
  570     R19, R19_H,
  571     R20, R20_H,
  572     R21, R21_H,
  573     R22, R22_H,
  574     R23, R23_H,
  575     R24, R24_H,
  576     R25, R25_H,
  577     R26, R26_H,
  578     R27, R27_H,
  579     R28, R28_H,
  580     R29, R29_H,
  581     R30, R30_H,
  582     R31, R31_H
  583 );
  584 
  585 // Class for all long integer registers (including SP)
  586 reg_class any_reg %{
  587   return _ANY_REG_mask;
  588 %}
  589 
  590 // Class for non-allocatable 32 bit registers
  591 reg_class non_allocatable_reg32(
  592 #ifdef R18_RESERVED
  593     // See comment in register_aarch64.hpp
  594     R18,                        // tls on Windows
  595 #endif
  596     R28,                        // thread
  597     R30,                        // lr
  598     R31                         // sp
  599 );
  600 
  601 // Class for non-allocatable 64 bit registers
  602 reg_class non_allocatable_reg(
  603 #ifdef R18_RESERVED
  604     // See comment in register_aarch64.hpp
  605     R18, R18_H,                 // tls on Windows, platform register on macOS
  606 #endif
  607     R28, R28_H,                 // thread
  608     R30, R30_H,                 // lr
  609     R31, R31_H                  // sp
  610 );
  611 
  612 // Class for all non-special integer registers
  613 reg_class no_special_reg32 %{
  614   return _NO_SPECIAL_REG32_mask;
  615 %}
  616 
  617 // Class for all non-special long integer registers
  618 reg_class no_special_reg %{
  619   return _NO_SPECIAL_REG_mask;
  620 %}
  621 
  622 // Class for 64 bit register r0
  623 reg_class r0_reg(
  624     R0, R0_H
  625 );
  626 
  627 // Class for 64 bit register r1
  628 reg_class r1_reg(
  629     R1, R1_H
  630 );
  631 
  632 // Class for 64 bit register r2
  633 reg_class r2_reg(
  634     R2, R2_H
  635 );
  636 
  637 // Class for 64 bit register r3
  638 reg_class r3_reg(
  639     R3, R3_H
  640 );
  641 
  642 // Class for 64 bit register r4
  643 reg_class r4_reg(
  644     R4, R4_H
  645 );
  646 
  647 // Class for 64 bit register r5
  648 reg_class r5_reg(
  649     R5, R5_H
  650 );
  651 
  652 // Class for 64 bit register r10
  653 reg_class r10_reg(
  654     R10, R10_H
  655 );
  656 
  657 // Class for 64 bit register r11
  658 reg_class r11_reg(
  659     R11, R11_H
  660 );
  661 
  662 // Class for method register
  663 reg_class method_reg(
  664     R12, R12_H
  665 );
  666 
  667 // Class for thread register
  668 reg_class thread_reg(
  669     R28, R28_H
  670 );
  671 
  672 // Class for frame pointer register
  673 reg_class fp_reg(
  674     R29, R29_H
  675 );
  676 
  677 // Class for link register
  678 reg_class lr_reg(
  679     R30, R30_H
  680 );
  681 
  682 // Class for long sp register
  683 reg_class sp_reg(
  684   R31, R31_H
  685 );
  686 
  687 // Class for all pointer registers
  688 reg_class ptr_reg %{
  689   return _PTR_REG_mask;
  690 %}
  691 
  692 // Class for all non_special pointer registers
  693 reg_class no_special_ptr_reg %{
  694   return _NO_SPECIAL_PTR_REG_mask;
  695 %}
  696 
  697 // Class for all non_special pointer registers (excluding rfp)
  698 reg_class no_special_no_rfp_ptr_reg %{
  699   return _NO_SPECIAL_NO_RFP_PTR_REG_mask;
  700 %}
  701 
  702 // Class for all float registers
  703 reg_class float_reg(
  704     V0,
  705     V1,
  706     V2,
  707     V3,
  708     V4,
  709     V5,
  710     V6,
  711     V7,
  712     V8,
  713     V9,
  714     V10,
  715     V11,
  716     V12,
  717     V13,
  718     V14,
  719     V15,
  720     V16,
  721     V17,
  722     V18,
  723     V19,
  724     V20,
  725     V21,
  726     V22,
  727     V23,
  728     V24,
  729     V25,
  730     V26,
  731     V27,
  732     V28,
  733     V29,
  734     V30,
  735     V31
  736 );
  737 
  738 // Double precision float registers have virtual `high halves' that
  739 // are needed by the allocator.
  740 // Class for all double registers
  741 reg_class double_reg(
  742     V0, V0_H,
  743     V1, V1_H,
  744     V2, V2_H,
  745     V3, V3_H,
  746     V4, V4_H,
  747     V5, V5_H,
  748     V6, V6_H,
  749     V7, V7_H,
  750     V8, V8_H,
  751     V9, V9_H,
  752     V10, V10_H,
  753     V11, V11_H,
  754     V12, V12_H,
  755     V13, V13_H,
  756     V14, V14_H,
  757     V15, V15_H,
  758     V16, V16_H,
  759     V17, V17_H,
  760     V18, V18_H,
  761     V19, V19_H,
  762     V20, V20_H,
  763     V21, V21_H,
  764     V22, V22_H,
  765     V23, V23_H,
  766     V24, V24_H,
  767     V25, V25_H,
  768     V26, V26_H,
  769     V27, V27_H,
  770     V28, V28_H,
  771     V29, V29_H,
  772     V30, V30_H,
  773     V31, V31_H
  774 );
  775 
  776 // Class for all SVE vector registers.
  777 reg_class vectora_reg (
  778     V0, V0_H, V0_J, V0_K,
  779     V1, V1_H, V1_J, V1_K,
  780     V2, V2_H, V2_J, V2_K,
  781     V3, V3_H, V3_J, V3_K,
  782     V4, V4_H, V4_J, V4_K,
  783     V5, V5_H, V5_J, V5_K,
  784     V6, V6_H, V6_J, V6_K,
  785     V7, V7_H, V7_J, V7_K,
  786     V8, V8_H, V8_J, V8_K,
  787     V9, V9_H, V9_J, V9_K,
  788     V10, V10_H, V10_J, V10_K,
  789     V11, V11_H, V11_J, V11_K,
  790     V12, V12_H, V12_J, V12_K,
  791     V13, V13_H, V13_J, V13_K,
  792     V14, V14_H, V14_J, V14_K,
  793     V15, V15_H, V15_J, V15_K,
  794     V16, V16_H, V16_J, V16_K,
  795     V17, V17_H, V17_J, V17_K,
  796     V18, V18_H, V18_J, V18_K,
  797     V19, V19_H, V19_J, V19_K,
  798     V20, V20_H, V20_J, V20_K,
  799     V21, V21_H, V21_J, V21_K,
  800     V22, V22_H, V22_J, V22_K,
  801     V23, V23_H, V23_J, V23_K,
  802     V24, V24_H, V24_J, V24_K,
  803     V25, V25_H, V25_J, V25_K,
  804     V26, V26_H, V26_J, V26_K,
  805     V27, V27_H, V27_J, V27_K,
  806     V28, V28_H, V28_J, V28_K,
  807     V29, V29_H, V29_J, V29_K,
  808     V30, V30_H, V30_J, V30_K,
  809     V31, V31_H, V31_J, V31_K,
  810 );
  811 
  812 // Class for all 64bit vector registers
  813 reg_class vectord_reg(
  814     V0, V0_H,
  815     V1, V1_H,
  816     V2, V2_H,
  817     V3, V3_H,
  818     V4, V4_H,
  819     V5, V5_H,
  820     V6, V6_H,
  821     V7, V7_H,
  822     V8, V8_H,
  823     V9, V9_H,
  824     V10, V10_H,
  825     V11, V11_H,
  826     V12, V12_H,
  827     V13, V13_H,
  828     V14, V14_H,
  829     V15, V15_H,
  830     V16, V16_H,
  831     V17, V17_H,
  832     V18, V18_H,
  833     V19, V19_H,
  834     V20, V20_H,
  835     V21, V21_H,
  836     V22, V22_H,
  837     V23, V23_H,
  838     V24, V24_H,
  839     V25, V25_H,
  840     V26, V26_H,
  841     V27, V27_H,
  842     V28, V28_H,
  843     V29, V29_H,
  844     V30, V30_H,
  845     V31, V31_H
  846 );
  847 
  848 // Class for all 128bit vector registers
  849 reg_class vectorx_reg(
  850     V0, V0_H, V0_J, V0_K,
  851     V1, V1_H, V1_J, V1_K,
  852     V2, V2_H, V2_J, V2_K,
  853     V3, V3_H, V3_J, V3_K,
  854     V4, V4_H, V4_J, V4_K,
  855     V5, V5_H, V5_J, V5_K,
  856     V6, V6_H, V6_J, V6_K,
  857     V7, V7_H, V7_J, V7_K,
  858     V8, V8_H, V8_J, V8_K,
  859     V9, V9_H, V9_J, V9_K,
  860     V10, V10_H, V10_J, V10_K,
  861     V11, V11_H, V11_J, V11_K,
  862     V12, V12_H, V12_J, V12_K,
  863     V13, V13_H, V13_J, V13_K,
  864     V14, V14_H, V14_J, V14_K,
  865     V15, V15_H, V15_J, V15_K,
  866     V16, V16_H, V16_J, V16_K,
  867     V17, V17_H, V17_J, V17_K,
  868     V18, V18_H, V18_J, V18_K,
  869     V19, V19_H, V19_J, V19_K,
  870     V20, V20_H, V20_J, V20_K,
  871     V21, V21_H, V21_J, V21_K,
  872     V22, V22_H, V22_J, V22_K,
  873     V23, V23_H, V23_J, V23_K,
  874     V24, V24_H, V24_J, V24_K,
  875     V25, V25_H, V25_J, V25_K,
  876     V26, V26_H, V26_J, V26_K,
  877     V27, V27_H, V27_J, V27_K,
  878     V28, V28_H, V28_J, V28_K,
  879     V29, V29_H, V29_J, V29_K,
  880     V30, V30_H, V30_J, V30_K,
  881     V31, V31_H, V31_J, V31_K
  882 );
  883 
  884 // Class for vector register V10
  885 reg_class v10_veca_reg(
  886     V10, V10_H, V10_J, V10_K
  887 );
  888 
  889 // Class for vector register V11
  890 reg_class v11_veca_reg(
  891     V11, V11_H, V11_J, V11_K
  892 );
  893 
  894 // Class for vector register V12
  895 reg_class v12_veca_reg(
  896     V12, V12_H, V12_J, V12_K
  897 );
  898 
  899 // Class for vector register V13
  900 reg_class v13_veca_reg(
  901     V13, V13_H, V13_J, V13_K
  902 );
  903 
  904 // Class for vector register V17
  905 reg_class v17_veca_reg(
  906     V17, V17_H, V17_J, V17_K
  907 );
  908 
  909 // Class for vector register V18
  910 reg_class v18_veca_reg(
  911     V18, V18_H, V18_J, V18_K
  912 );
  913 
  914 // Class for vector register V23
  915 reg_class v23_veca_reg(
  916     V23, V23_H, V23_J, V23_K
  917 );
  918 
  919 // Class for vector register V24
  920 reg_class v24_veca_reg(
  921     V24, V24_H, V24_J, V24_K
  922 );
  923 
  924 // Class for 128 bit register v0
  925 reg_class v0_reg(
  926     V0, V0_H
  927 );
  928 
  929 // Class for 128 bit register v1
  930 reg_class v1_reg(
  931     V1, V1_H
  932 );
  933 
  934 // Class for 128 bit register v2
  935 reg_class v2_reg(
  936     V2, V2_H
  937 );
  938 
  939 // Class for 128 bit register v3
  940 reg_class v3_reg(
  941     V3, V3_H
  942 );
  943 
  944 // Class for 128 bit register v4
  945 reg_class v4_reg(
  946     V4, V4_H
  947 );
  948 
  949 // Class for 128 bit register v5
  950 reg_class v5_reg(
  951     V5, V5_H
  952 );
  953 
  954 // Class for 128 bit register v6
  955 reg_class v6_reg(
  956     V6, V6_H
  957 );
  958 
  959 // Class for 128 bit register v7
  960 reg_class v7_reg(
  961     V7, V7_H
  962 );
  963 
  964 // Class for 128 bit register v8
  965 reg_class v8_reg(
  966     V8, V8_H
  967 );
  968 
  969 // Class for 128 bit register v9
  970 reg_class v9_reg(
  971     V9, V9_H
  972 );
  973 
  974 // Class for 128 bit register v10
  975 reg_class v10_reg(
  976     V10, V10_H
  977 );
  978 
  979 // Class for 128 bit register v11
  980 reg_class v11_reg(
  981     V11, V11_H
  982 );
  983 
  984 // Class for 128 bit register v12
  985 reg_class v12_reg(
  986     V12, V12_H
  987 );
  988 
  989 // Class for 128 bit register v13
  990 reg_class v13_reg(
  991     V13, V13_H
  992 );
  993 
  994 // Class for 128 bit register v14
  995 reg_class v14_reg(
  996     V14, V14_H
  997 );
  998 
  999 // Class for 128 bit register v15
 1000 reg_class v15_reg(
 1001     V15, V15_H
 1002 );
 1003 
 1004 // Class for 128 bit register v16
 1005 reg_class v16_reg(
 1006     V16, V16_H
 1007 );
 1008 
 1009 // Class for 128 bit register v17
 1010 reg_class v17_reg(
 1011     V17, V17_H
 1012 );
 1013 
 1014 // Class for 128 bit register v18
 1015 reg_class v18_reg(
 1016     V18, V18_H
 1017 );
 1018 
 1019 // Class for 128 bit register v19
 1020 reg_class v19_reg(
 1021     V19, V19_H
 1022 );
 1023 
 1024 // Class for 128 bit register v20
 1025 reg_class v20_reg(
 1026     V20, V20_H
 1027 );
 1028 
 1029 // Class for 128 bit register v21
 1030 reg_class v21_reg(
 1031     V21, V21_H
 1032 );
 1033 
 1034 // Class for 128 bit register v22
 1035 reg_class v22_reg(
 1036     V22, V22_H
 1037 );
 1038 
 1039 // Class for 128 bit register v23
 1040 reg_class v23_reg(
 1041     V23, V23_H
 1042 );
 1043 
 1044 // Class for 128 bit register v24
 1045 reg_class v24_reg(
 1046     V24, V24_H
 1047 );
 1048 
 1049 // Class for 128 bit register v25
 1050 reg_class v25_reg(
 1051     V25, V25_H
 1052 );
 1053 
 1054 // Class for 128 bit register v26
 1055 reg_class v26_reg(
 1056     V26, V26_H
 1057 );
 1058 
 1059 // Class for 128 bit register v27
 1060 reg_class v27_reg(
 1061     V27, V27_H
 1062 );
 1063 
 1064 // Class for 128 bit register v28
 1065 reg_class v28_reg(
 1066     V28, V28_H
 1067 );
 1068 
 1069 // Class for 128 bit register v29
 1070 reg_class v29_reg(
 1071     V29, V29_H
 1072 );
 1073 
 1074 // Class for 128 bit register v30
 1075 reg_class v30_reg(
 1076     V30, V30_H
 1077 );
 1078 
 1079 // Class for 128 bit register v31
 1080 reg_class v31_reg(
 1081     V31, V31_H
 1082 );
 1083 
 1084 // Class for all SVE predicate registers.
 1085 reg_class pr_reg (
 1086     P0,
 1087     P1,
 1088     P2,
 1089     P3,
 1090     P4,
 1091     P5,
 1092     P6,
 1093     // P7, non-allocatable, preserved with all elements preset to TRUE.
 1094     P8,
 1095     P9,
 1096     P10,
 1097     P11,
 1098     P12,
 1099     P13,
 1100     P14,
 1101     P15
 1102 );
 1103 
 1104 // Class for SVE governing predicate registers, which are used
 1105 // to determine the active elements of a predicated instruction.
 1106 reg_class gov_pr (
 1107     P0,
 1108     P1,
 1109     P2,
 1110     P3,
 1111     P4,
 1112     P5,
 1113     P6,
 1114     // P7, non-allocatable, preserved with all elements preset to TRUE.
 1115 );
 1116 
 1117 reg_class p0_reg(P0);
 1118 reg_class p1_reg(P1);
 1119 
 1120 // Singleton class for condition codes
 1121 reg_class int_flags(RFLAGS);
 1122 
 1123 %}
 1124 
 1125 //----------DEFINITION BLOCK---------------------------------------------------
 1126 // Define name --> value mappings to inform the ADLC of an integer valued name
 1127 // Current support includes integer values in the range [0, 0x7FFFFFFF]
 1128 // Format:
 1129 //        int_def  <name>         ( <int_value>, <expression>);
 1130 // Generated Code in ad_<arch>.hpp
 1131 //        #define  <name>   (<expression>)
 1132 //        // value == <int_value>
 1133 // Generated code in ad_<arch>.cpp adlc_verification()
 1134 //        assert( <name> == <int_value>, "Expect (<expression>) to equal <int_value>");
 1135 //
 1136 
 1137 // we follow the ppc-aix port in using a simple cost model which ranks
 1138 // register operations as cheap, memory ops as more expensive and
 1139 // branches as most expensive. the first two have a low as well as a
 1140 // normal cost. huge cost appears to be a way of saying don't do
 1141 // something
 1142 
 1143 definitions %{
 1144   // The default cost (of a register move instruction).
 1145   int_def INSN_COST            (    100,     100);
 1146   int_def BRANCH_COST          (    200,     2 * INSN_COST);
 1147   int_def CALL_COST            (    200,     2 * INSN_COST);
 1148   int_def VOLATILE_REF_COST    (   1000,     10 * INSN_COST);
 1149 %}
 1150 
 1151 
 1152 //----------SOURCE BLOCK-------------------------------------------------------
 1153 // This is a block of C++ code which provides values, functions, and
 1154 // definitions necessary in the rest of the architecture description
 1155 
 1156 source_hpp %{
 1157 
 1158 #include "asm/macroAssembler.hpp"
 1159 #include "gc/shared/barrierSetAssembler.hpp"
 1160 #include "gc/shared/cardTable.hpp"
 1161 #include "gc/shared/cardTableBarrierSet.hpp"
 1162 #include "gc/shared/collectedHeap.hpp"
 1163 #include "opto/addnode.hpp"
 1164 #include "opto/convertnode.hpp"
 1165 #include "runtime/objectMonitor.hpp"
 1166 
 1167 extern RegMask _ANY_REG32_mask;
 1168 extern RegMask _ANY_REG_mask;
 1169 extern RegMask _PTR_REG_mask;
 1170 extern RegMask _NO_SPECIAL_REG32_mask;
 1171 extern RegMask _NO_SPECIAL_REG_mask;
 1172 extern RegMask _NO_SPECIAL_PTR_REG_mask;
 1173 extern RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
 1174 
 1175 class CallStubImpl {
 1176 
 1177   //--------------------------------------------------------------
 1178   //---<  Used for optimization in Compile::shorten_branches  >---
 1179   //--------------------------------------------------------------
 1180 
 1181  public:
 1182   // Size of call trampoline stub.
 1183   static uint size_call_trampoline() {
 1184     return 0; // no call trampolines on this platform
 1185   }
 1186 
 1187   // number of relocations needed by a call trampoline stub
 1188   static uint reloc_call_trampoline() {
 1189     return 0; // no call trampolines on this platform
 1190   }
 1191 };
 1192 
 1193 class HandlerImpl {
 1194 
 1195  public:
 1196 
 1197   static int emit_exception_handler(C2_MacroAssembler *masm);
 1198   static int emit_deopt_handler(C2_MacroAssembler* masm);
 1199 
 1200   static uint size_exception_handler() {
 1201     return MacroAssembler::far_codestub_branch_size();
 1202   }
 1203 
 1204   static uint size_deopt_handler() {
 1205     // count one adr and one far branch instruction
 1206     return NativeInstruction::instruction_size + MacroAssembler::far_codestub_branch_size();
 1207   }
 1208 };
 1209 
 1210 class Node::PD {
 1211 public:
 1212   enum NodeFlags {
 1213     _last_flag = Node::_last_flag
 1214   };
 1215 };
 1216 
 1217   bool is_CAS(int opcode, bool maybe_volatile);
 1218 
 1219   // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
 1220 
 1221   bool unnecessary_acquire(const Node *barrier);
 1222   bool needs_acquiring_load(const Node *load);
 1223 
 1224   // predicates controlling emit of str<x>/stlr<x> and associated dmbs
 1225 
 1226   bool unnecessary_release(const Node *barrier);
 1227   bool unnecessary_volatile(const Node *barrier);
 1228   bool needs_releasing_store(const Node *store);
 1229 
 1230   // predicate controlling translation of CompareAndSwapX
 1231   bool needs_acquiring_load_exclusive(const Node *load);
 1232 
 1233   // predicate controlling addressing modes
 1234   bool size_fits_all_mem_uses(AddPNode* addp, int shift);
 1235 
 1236   // Convert BootTest condition to Assembler condition.
 1237   // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
 1238   Assembler::Condition to_assembler_cond(BoolTest::mask cond);
 1239 %}
 1240 
 1241 source %{
 1242 
 1243   // Derived RegMask with conditionally allocatable registers
 1244 
 1245   void PhaseOutput::pd_perform_mach_node_analysis() {
 1246   }
 1247 
 1248   int MachNode::pd_alignment_required() const {
 1249     return 1;
 1250   }
 1251 
 1252   int MachNode::compute_padding(int current_offset) const {
 1253     return 0;
 1254   }
 1255 
 1256   RegMask _ANY_REG32_mask;
 1257   RegMask _ANY_REG_mask;
 1258   RegMask _PTR_REG_mask;
 1259   RegMask _NO_SPECIAL_REG32_mask;
 1260   RegMask _NO_SPECIAL_REG_mask;
 1261   RegMask _NO_SPECIAL_PTR_REG_mask;
 1262   RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
 1263 
 1264   void reg_mask_init() {
 1265     // We derive below RegMask(s) from the ones which are auto-generated from
 1266     // adlc register classes to make AArch64 rheapbase (r27) and rfp (r29)
 1267     // registers conditionally reserved.
 1268 
 1269     _ANY_REG32_mask = _ALL_REG32_mask;
 1270     _ANY_REG32_mask.Remove(OptoReg::as_OptoReg(r31_sp->as_VMReg()));
 1271 
 1272     _ANY_REG_mask = _ALL_REG_mask;
 1273 
 1274     _PTR_REG_mask = _ALL_REG_mask;
 1275 
 1276     _NO_SPECIAL_REG32_mask = _ALL_REG32_mask;
 1277     _NO_SPECIAL_REG32_mask.SUBTRACT(_NON_ALLOCATABLE_REG32_mask);
 1278 
 1279     _NO_SPECIAL_REG_mask = _ALL_REG_mask;
 1280     _NO_SPECIAL_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
 1281 
 1282     _NO_SPECIAL_PTR_REG_mask = _ALL_REG_mask;
 1283     _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
 1284 
 1285     // r27 is not allocatable when compressed oops is on and heapbase is not
 1286     // zero, compressed klass pointers doesn't use r27 after JDK-8234794
 1287     if (UseCompressedOops && (CompressedOops::base() != nullptr)) {
 1288       _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
 1289       _NO_SPECIAL_REG_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
 1290       _NO_SPECIAL_PTR_REG_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
 1291     }
 1292 
 1293     // r29 is not allocatable when PreserveFramePointer is on
 1294     if (PreserveFramePointer) {
 1295       _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
 1296       _NO_SPECIAL_REG_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
 1297       _NO_SPECIAL_PTR_REG_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
 1298     }
 1299 
 1300     _NO_SPECIAL_NO_RFP_PTR_REG_mask = _NO_SPECIAL_PTR_REG_mask;
 1301     _NO_SPECIAL_NO_RFP_PTR_REG_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
 1302   }
 1303 
 1304   // Optimizaton of volatile gets and puts
 1305   // -------------------------------------
 1306   //
 1307   // AArch64 has ldar<x> and stlr<x> instructions which we can safely
 1308   // use to implement volatile reads and writes. For a volatile read
 1309   // we simply need
 1310   //
 1311   //   ldar<x>
 1312   //
 1313   // and for a volatile write we need
 1314   //
 1315   //   stlr<x>
 1316   //
 1317   // Alternatively, we can implement them by pairing a normal
 1318   // load/store with a memory barrier. For a volatile read we need
 1319   //
 1320   //   ldr<x>
 1321   //   dmb ishld
 1322   //
 1323   // for a volatile write
 1324   //
 1325   //   dmb ish
 1326   //   str<x>
 1327   //   dmb ish
 1328   //
 1329   // We can also use ldaxr and stlxr to implement compare and swap CAS
 1330   // sequences. These are normally translated to an instruction
 1331   // sequence like the following
 1332   //
 1333   //   dmb      ish
 1334   // retry:
 1335   //   ldxr<x>   rval raddr
 1336   //   cmp       rval rold
 1337   //   b.ne done
 1338   //   stlxr<x>  rval, rnew, rold
 1339   //   cbnz      rval retry
 1340   // done:
 1341   //   cset      r0, eq
 1342   //   dmb ishld
 1343   //
 1344   // Note that the exclusive store is already using an stlxr
 1345   // instruction. That is required to ensure visibility to other
 1346   // threads of the exclusive write (assuming it succeeds) before that
 1347   // of any subsequent writes.
 1348   //
 1349   // The following instruction sequence is an improvement on the above
 1350   //
 1351   // retry:
 1352   //   ldaxr<x>  rval raddr
 1353   //   cmp       rval rold
 1354   //   b.ne done
 1355   //   stlxr<x>  rval, rnew, rold
 1356   //   cbnz      rval retry
 1357   // done:
 1358   //   cset      r0, eq
 1359   //
 1360   // We don't need the leading dmb ish since the stlxr guarantees
 1361   // visibility of prior writes in the case that the swap is
 1362   // successful. Crucially we don't have to worry about the case where
 1363   // the swap is not successful since no valid program should be
 1364   // relying on visibility of prior changes by the attempting thread
 1365   // in the case where the CAS fails.
 1366   //
 1367   // Similarly, we don't need the trailing dmb ishld if we substitute
 1368   // an ldaxr instruction since that will provide all the guarantees we
 1369   // require regarding observation of changes made by other threads
 1370   // before any change to the CAS address observed by the load.
 1371   //
 1372   // In order to generate the desired instruction sequence we need to
 1373   // be able to identify specific 'signature' ideal graph node
 1374   // sequences which i) occur as a translation of a volatile reads or
 1375   // writes or CAS operations and ii) do not occur through any other
 1376   // translation or graph transformation. We can then provide
 1377   // alternative aldc matching rules which translate these node
 1378   // sequences to the desired machine code sequences. Selection of the
 1379   // alternative rules can be implemented by predicates which identify
 1380   // the relevant node sequences.
 1381   //
 1382   // The ideal graph generator translates a volatile read to the node
 1383   // sequence
 1384   //
 1385   //   LoadX[mo_acquire]
 1386   //   MemBarAcquire
 1387   //
 1388   // As a special case when using the compressed oops optimization we
 1389   // may also see this variant
 1390   //
 1391   //   LoadN[mo_acquire]
 1392   //   DecodeN
 1393   //   MemBarAcquire
 1394   //
 1395   // A volatile write is translated to the node sequence
 1396   //
 1397   //   MemBarRelease
 1398   //   StoreX[mo_release] {CardMark}-optional
 1399   //   MemBarVolatile
 1400   //
 1401   // n.b. the above node patterns are generated with a strict
 1402   // 'signature' configuration of input and output dependencies (see
 1403   // the predicates below for exact details). The card mark may be as
 1404   // simple as a few extra nodes or, in a few GC configurations, may
 1405   // include more complex control flow between the leading and
 1406   // trailing memory barriers. However, whatever the card mark
 1407   // configuration these signatures are unique to translated volatile
 1408   // reads/stores -- they will not appear as a result of any other
 1409   // bytecode translation or inlining nor as a consequence of
 1410   // optimizing transforms.
 1411   //
 1412   // We also want to catch inlined unsafe volatile gets and puts and
 1413   // be able to implement them using either ldar<x>/stlr<x> or some
 1414   // combination of ldr<x>/stlr<x> and dmb instructions.
 1415   //
 1416   // Inlined unsafe volatiles puts manifest as a minor variant of the
 1417   // normal volatile put node sequence containing an extra cpuorder
 1418   // membar
 1419   //
 1420   //   MemBarRelease
 1421   //   MemBarCPUOrder
 1422   //   StoreX[mo_release] {CardMark}-optional
 1423   //   MemBarCPUOrder
 1424   //   MemBarVolatile
 1425   //
 1426   // n.b. as an aside, a cpuorder membar is not itself subject to
 1427   // matching and translation by adlc rules.  However, the rule
 1428   // predicates need to detect its presence in order to correctly
 1429   // select the desired adlc rules.
 1430   //
 1431   // Inlined unsafe volatile gets manifest as a slightly different
 1432   // node sequence to a normal volatile get because of the
 1433   // introduction of some CPUOrder memory barriers to bracket the
 1434   // Load. However, but the same basic skeleton of a LoadX feeding a
 1435   // MemBarAcquire, possibly through an optional DecodeN, is still
 1436   // present
 1437   //
 1438   //   MemBarCPUOrder
 1439   //        ||       \\
 1440   //   MemBarCPUOrder LoadX[mo_acquire]
 1441   //        ||            |
 1442   //        ||       {DecodeN} optional
 1443   //        ||       /
 1444   //     MemBarAcquire
 1445   //
 1446   // In this case the acquire membar does not directly depend on the
 1447   // load. However, we can be sure that the load is generated from an
 1448   // inlined unsafe volatile get if we see it dependent on this unique
 1449   // sequence of membar nodes. Similarly, given an acquire membar we
 1450   // can know that it was added because of an inlined unsafe volatile
 1451   // get if it is fed and feeds a cpuorder membar and if its feed
 1452   // membar also feeds an acquiring load.
 1453   //
 1454   // Finally an inlined (Unsafe) CAS operation is translated to the
 1455   // following ideal graph
 1456   //
 1457   //   MemBarRelease
 1458   //   MemBarCPUOrder
 1459   //   CompareAndSwapX {CardMark}-optional
 1460   //   MemBarCPUOrder
 1461   //   MemBarAcquire
 1462   //
 1463   // So, where we can identify these volatile read and write
 1464   // signatures we can choose to plant either of the above two code
 1465   // sequences. For a volatile read we can simply plant a normal
 1466   // ldr<x> and translate the MemBarAcquire to a dmb. However, we can
 1467   // also choose to inhibit translation of the MemBarAcquire and
 1468   // inhibit planting of the ldr<x>, instead planting an ldar<x>.
 1469   //
 1470   // When we recognise a volatile store signature we can choose to
 1471   // plant at a dmb ish as a translation for the MemBarRelease, a
 1472   // normal str<x> and then a dmb ish for the MemBarVolatile.
 1473   // Alternatively, we can inhibit translation of the MemBarRelease
 1474   // and MemBarVolatile and instead plant a simple stlr<x>
 1475   // instruction.
 1476   //
 1477   // when we recognise a CAS signature we can choose to plant a dmb
 1478   // ish as a translation for the MemBarRelease, the conventional
 1479   // macro-instruction sequence for the CompareAndSwap node (which
 1480   // uses ldxr<x>) and then a dmb ishld for the MemBarAcquire.
 1481   // Alternatively, we can elide generation of the dmb instructions
 1482   // and plant the alternative CompareAndSwap macro-instruction
 1483   // sequence (which uses ldaxr<x>).
 1484   //
 1485   // Of course, the above only applies when we see these signature
 1486   // configurations. We still want to plant dmb instructions in any
 1487   // other cases where we may see a MemBarAcquire, MemBarRelease or
 1488   // MemBarVolatile. For example, at the end of a constructor which
 1489   // writes final/volatile fields we will see a MemBarRelease
 1490   // instruction and this needs a 'dmb ish' lest we risk the
 1491   // constructed object being visible without making the
 1492   // final/volatile field writes visible.
 1493   //
 1494   // n.b. the translation rules below which rely on detection of the
 1495   // volatile signatures and insert ldar<x> or stlr<x> are failsafe.
 1496   // If we see anything other than the signature configurations we
 1497   // always just translate the loads and stores to ldr<x> and str<x>
 1498   // and translate acquire, release and volatile membars to the
 1499   // relevant dmb instructions.
 1500   //
 1501 
 1502   // is_CAS(int opcode, bool maybe_volatile)
 1503   //
 1504   // return true if opcode is one of the possible CompareAndSwapX
 1505   // values otherwise false.
 1506 
 1507   bool is_CAS(int opcode, bool maybe_volatile)
 1508   {
 1509     switch(opcode) {
 1510       // We handle these
 1511     case Op_CompareAndSwapI:
 1512     case Op_CompareAndSwapL:
 1513     case Op_CompareAndSwapP:
 1514     case Op_CompareAndSwapN:
 1515     case Op_ShenandoahCompareAndSwapP:
 1516     case Op_ShenandoahCompareAndSwapN:
 1517     case Op_CompareAndSwapB:
 1518     case Op_CompareAndSwapS:
 1519     case Op_GetAndSetI:
 1520     case Op_GetAndSetL:
 1521     case Op_GetAndSetP:
 1522     case Op_GetAndSetN:
 1523     case Op_GetAndAddI:
 1524     case Op_GetAndAddL:
 1525       return true;
 1526     case Op_CompareAndExchangeI:
 1527     case Op_CompareAndExchangeN:
 1528     case Op_CompareAndExchangeB:
 1529     case Op_CompareAndExchangeS:
 1530     case Op_CompareAndExchangeL:
 1531     case Op_CompareAndExchangeP:
 1532     case Op_WeakCompareAndSwapB:
 1533     case Op_WeakCompareAndSwapS:
 1534     case Op_WeakCompareAndSwapI:
 1535     case Op_WeakCompareAndSwapL:
 1536     case Op_WeakCompareAndSwapP:
 1537     case Op_WeakCompareAndSwapN:
 1538     case Op_ShenandoahWeakCompareAndSwapP:
 1539     case Op_ShenandoahWeakCompareAndSwapN:
 1540     case Op_ShenandoahCompareAndExchangeP:
 1541     case Op_ShenandoahCompareAndExchangeN:
 1542       return maybe_volatile;
 1543     default:
 1544       return false;
 1545     }
 1546   }
 1547 
 1548   // helper to determine the maximum number of Phi nodes we may need to
 1549   // traverse when searching from a card mark membar for the merge mem
 1550   // feeding a trailing membar or vice versa
 1551 
 1552 // predicates controlling emit of ldr<x>/ldar<x>
 1553 
 1554 bool unnecessary_acquire(const Node *barrier)
 1555 {
 1556   assert(barrier->is_MemBar(), "expecting a membar");
 1557 
 1558   MemBarNode* mb = barrier->as_MemBar();
 1559 
 1560   if (mb->trailing_load()) {
 1561     return true;
 1562   }
 1563 
 1564   if (mb->trailing_load_store()) {
 1565     Node* load_store = mb->in(MemBarNode::Precedent);
 1566     assert(load_store->is_LoadStore(), "unexpected graph shape");
 1567     return is_CAS(load_store->Opcode(), true);
 1568   }
 1569 
 1570   return false;
 1571 }
 1572 
 1573 bool needs_acquiring_load(const Node *n)
 1574 {
 1575   assert(n->is_Load(), "expecting a load");
 1576   LoadNode *ld = n->as_Load();
 1577   return ld->is_acquire();
 1578 }
 1579 
 1580 bool unnecessary_release(const Node *n)
 1581 {
 1582   assert((n->is_MemBar() &&
 1583           n->Opcode() == Op_MemBarRelease),
 1584          "expecting a release membar");
 1585 
 1586   MemBarNode *barrier = n->as_MemBar();
 1587   if (!barrier->leading()) {
 1588     return false;
 1589   } else {
 1590     Node* trailing = barrier->trailing_membar();
 1591     MemBarNode* trailing_mb = trailing->as_MemBar();
 1592     assert(trailing_mb->trailing(), "Not a trailing membar?");
 1593     assert(trailing_mb->leading_membar() == n, "inconsistent leading/trailing membars");
 1594 
 1595     Node* mem = trailing_mb->in(MemBarNode::Precedent);
 1596     if (mem->is_Store()) {
 1597       assert(mem->as_Store()->is_release(), "");
 1598       assert(trailing_mb->Opcode() == Op_MemBarVolatile, "");
 1599       return true;
 1600     } else {
 1601       assert(mem->is_LoadStore(), "");
 1602       assert(trailing_mb->Opcode() == Op_MemBarAcquire, "");
 1603       return is_CAS(mem->Opcode(), true);
 1604     }
 1605   }
 1606   return false;
 1607 }
 1608 
 1609 bool unnecessary_volatile(const Node *n)
 1610 {
 1611   // assert n->is_MemBar();
 1612   MemBarNode *mbvol = n->as_MemBar();
 1613 
 1614   bool release = mbvol->trailing_store();
 1615   assert(!release || (mbvol->in(MemBarNode::Precedent)->is_Store() && mbvol->in(MemBarNode::Precedent)->as_Store()->is_release()), "");
 1616 #ifdef ASSERT
 1617   if (release) {
 1618     Node* leading = mbvol->leading_membar();
 1619     assert(leading->Opcode() == Op_MemBarRelease, "");
 1620     assert(leading->as_MemBar()->leading_store(), "");
 1621     assert(leading->as_MemBar()->trailing_membar() == mbvol, "");
 1622   }
 1623 #endif
 1624 
 1625   return release;
 1626 }
 1627 
 1628 // predicates controlling emit of str<x>/stlr<x>
 1629 
 1630 bool needs_releasing_store(const Node *n)
 1631 {
 1632   // assert n->is_Store();
 1633   StoreNode *st = n->as_Store();
 1634   return st->trailing_membar() != nullptr;
 1635 }
 1636 
 1637 // predicate controlling translation of CAS
 1638 //
 1639 // returns true if CAS needs to use an acquiring load otherwise false
 1640 
 1641 bool needs_acquiring_load_exclusive(const Node *n)
 1642 {
 1643   assert(is_CAS(n->Opcode(), true), "expecting a compare and swap");
 1644   LoadStoreNode* ldst = n->as_LoadStore();
 1645   if (is_CAS(n->Opcode(), false)) {
 1646     assert(ldst->trailing_membar() != nullptr, "expected trailing membar");
 1647   } else {
 1648     return ldst->trailing_membar() != nullptr;
 1649   }
 1650 
 1651   // so we can just return true here
 1652   return true;
 1653 }
 1654 
 1655 #define __ masm->
 1656 
 1657 // advance declarations for helper functions to convert register
 1658 // indices to register objects
 1659 
 1660 // the ad file has to provide implementations of certain methods
 1661 // expected by the generic code
 1662 //
 1663 // REQUIRED FUNCTIONALITY
 1664 
 1665 //=============================================================================
 1666 
 1667 // !!!!! Special hack to get all types of calls to specify the byte offset
 1668 //       from the start of the call to the point where the return address
 1669 //       will point.
 1670 
 1671 int MachCallStaticJavaNode::ret_addr_offset()
 1672 {
 1673   // call should be a simple bl
 1674   int off = 4;
 1675   return off;
 1676 }
 1677 
 1678 int MachCallDynamicJavaNode::ret_addr_offset()
 1679 {
 1680   return 16; // movz, movk, movk, bl
 1681 }
 1682 
 1683 int MachCallRuntimeNode::ret_addr_offset() {
 1684   // for generated stubs the call will be
 1685   //   bl(addr)
 1686   // or with far branches
 1687   //   bl(trampoline_stub)
 1688   // for real runtime callouts it will be six instructions
 1689   // see aarch64_enc_java_to_runtime
 1690   //   adr(rscratch2, retaddr)
 1691   //   str(rscratch2, Address(rthread, JavaThread::last_Java_pc_offset()));
 1692   //   lea(rscratch1, RuntimeAddress(addr)
 1693   //   blr(rscratch1)
 1694   CodeBlob *cb = CodeCache::find_blob(_entry_point);
 1695   if (cb) {
 1696     return 1 * NativeInstruction::instruction_size;
 1697   } else if (_entry_point == nullptr) {
 1698     // See CallLeafNoFPIndirect
 1699     return 1 * NativeInstruction::instruction_size;
 1700   } else {
 1701     return 6 * NativeInstruction::instruction_size;
 1702   }
 1703 }
 1704 
 1705 //=============================================================================
 1706 
 1707 #ifndef PRODUCT
 1708 void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 1709   st->print("BREAKPOINT");
 1710 }
 1711 #endif
 1712 
 1713 void MachBreakpointNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
 1714   __ brk(0);
 1715 }
 1716 
 1717 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
 1718   return MachNode::size(ra_);
 1719 }
 1720 
 1721 //=============================================================================
 1722 
 1723 #ifndef PRODUCT
 1724   void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
 1725     st->print("nop \t# %d bytes pad for loops and calls", _count);
 1726   }
 1727 #endif
 1728 
 1729   void MachNopNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc*) const {
 1730     for (int i = 0; i < _count; i++) {
 1731       __ nop();
 1732     }
 1733   }
 1734 
 1735   uint MachNopNode::size(PhaseRegAlloc*) const {
 1736     return _count * NativeInstruction::instruction_size;
 1737   }
 1738 
 1739 //=============================================================================
 1740 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
 1741 
 1742 int ConstantTable::calculate_table_base_offset() const {
 1743   return 0;  // absolute addressing, no offset
 1744 }
 1745 
 1746 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
 1747 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
 1748   ShouldNotReachHere();
 1749 }
 1750 
 1751 void MachConstantBaseNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const {
 1752   // Empty encoding
 1753 }
 1754 
 1755 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
 1756   return 0;
 1757 }
 1758 
 1759 #ifndef PRODUCT
 1760 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
 1761   st->print("-- \t// MachConstantBaseNode (empty encoding)");
 1762 }
 1763 #endif
 1764 
 1765 #ifndef PRODUCT
 1766 void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 1767   Compile* C = ra_->C;
 1768 
 1769   int framesize = C->output()->frame_slots() << LogBytesPerInt;
 1770 
 1771   if (C->output()->need_stack_bang(framesize))
 1772     st->print("# stack bang size=%d\n\t", framesize);
 1773 
 1774   if (VM_Version::use_rop_protection()) {
 1775     st->print("ldr  zr, [lr]\n\t");
 1776     st->print("paciaz\n\t");
 1777   }
 1778   if (framesize < ((1 << 9) + 2 * wordSize)) {
 1779     st->print("sub  sp, sp, #%d\n\t", framesize);
 1780     st->print("stp  rfp, lr, [sp, #%d]", framesize - 2 * wordSize);
 1781     if (PreserveFramePointer) st->print("\n\tadd  rfp, sp, #%d", framesize - 2 * wordSize);
 1782   } else {
 1783     st->print("stp  lr, rfp, [sp, #%d]!\n\t", -(2 * wordSize));
 1784     if (PreserveFramePointer) st->print("mov  rfp, sp\n\t");
 1785     st->print("mov  rscratch1, #%d\n\t", framesize - 2 * wordSize);
 1786     st->print("sub  sp, sp, rscratch1");
 1787   }
 1788   if (C->stub_function() == nullptr) {
 1789     st->print("\n\t");
 1790     st->print("ldr  rscratch1, [guard]\n\t");
 1791     st->print("dmb ishld\n\t");
 1792     st->print("ldr  rscratch2, [rthread, #thread_disarmed_guard_value_offset]\n\t");
 1793     st->print("cmp  rscratch1, rscratch2\n\t");
 1794     st->print("b.eq skip");
 1795     st->print("\n\t");
 1796     st->print("blr #nmethod_entry_barrier_stub\n\t");
 1797     st->print("b skip\n\t");
 1798     st->print("guard: int\n\t");
 1799     st->print("\n\t");
 1800     st->print("skip:\n\t");
 1801   }
 1802 }
 1803 #endif
 1804 
 1805 void MachPrologNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
 1806   Compile* C = ra_->C;
 1807 
 1808 
 1809   __ verified_entry(C, 0);
 1810 
 1811   if (C->stub_function() == nullptr) {
 1812     __ entry_barrier();
 1813   }
 1814 
 1815   if (!Compile::current()->output()->in_scratch_emit_size()) {
 1816     __ bind(*_verified_entry);
 1817   }
 1818 
 1819   if (VerifyStackAtCalls) {
 1820     Unimplemented();
 1821   }
 1822 
 1823   C->output()->set_frame_complete(__ offset());
 1824 
 1825   if (C->has_mach_constant_base_node()) {
 1826     // NOTE: We set the table base offset here because users might be
 1827     // emitted before MachConstantBaseNode.
 1828     ConstantTable& constant_table = C->output()->constant_table();
 1829     constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
 1830   }
 1831 }
 1832 
 1833 int MachPrologNode::reloc() const
 1834 {
 1835   return 0;
 1836 }
 1837 
 1838 //=============================================================================
 1839 
 1840 #ifndef PRODUCT
 1841 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 1842   Compile* C = ra_->C;
 1843   int framesize = C->output()->frame_slots() << LogBytesPerInt;
 1844 
 1845   st->print("# pop frame %d\n\t",framesize);
 1846 
 1847   if (framesize == 0) {
 1848     st->print("ldp  lr, rfp, [sp],#%d\n\t", (2 * wordSize));
 1849   } else if (framesize < ((1 << 9) + 2 * wordSize)) {
 1850     st->print("ldp  lr, rfp, [sp,#%d]\n\t", framesize - 2 * wordSize);
 1851     st->print("add  sp, sp, #%d\n\t", framesize);
 1852   } else {
 1853     st->print("mov  rscratch1, #%d\n\t", framesize - 2 * wordSize);
 1854     st->print("add  sp, sp, rscratch1\n\t");
 1855     st->print("ldp  lr, rfp, [sp],#%d\n\t", (2 * wordSize));
 1856   }
 1857   if (VM_Version::use_rop_protection()) {
 1858     st->print("autiaz\n\t");
 1859     st->print("ldr  zr, [lr]\n\t");
 1860   }
 1861 
 1862   if (do_polling() && C->is_method_compilation()) {
 1863     st->print("# test polling word\n\t");
 1864     st->print("ldr  rscratch1, [rthread],#%d\n\t", in_bytes(JavaThread::polling_word_offset()));
 1865     st->print("cmp  sp, rscratch1\n\t");
 1866     st->print("bhi #slow_path");
 1867   }
 1868 }
 1869 #endif
 1870 
 1871 void MachEpilogNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
 1872   Compile* C = ra_->C;
 1873   int framesize = C->output()->frame_slots() << LogBytesPerInt;
 1874 
 1875   __ remove_frame(framesize, C->needs_stack_repair());
 1876 
 1877   if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
 1878     __ reserved_stack_check();
 1879   }
 1880 
 1881   if (do_polling() && C->is_method_compilation()) {
 1882     Label dummy_label;
 1883     Label* code_stub = &dummy_label;
 1884     if (!C->output()->in_scratch_emit_size()) {
 1885       C2SafepointPollStub* stub = new (C->comp_arena()) C2SafepointPollStub(__ offset());
 1886       C->output()->add_stub(stub);
 1887       code_stub = &stub->entry();
 1888     }
 1889     __ relocate(relocInfo::poll_return_type);
 1890     __ safepoint_poll(*code_stub, true /* at_return */, true /* in_nmethod */);
 1891   }
 1892 }
 1893 
 1894 int MachEpilogNode::reloc() const {
 1895   // Return number of relocatable values contained in this instruction.
 1896   return 1; // 1 for polling page.
 1897 }
 1898 
 1899 const Pipeline * MachEpilogNode::pipeline() const {
 1900   return MachNode::pipeline_class();
 1901 }
 1902 
 1903 //=============================================================================
 1904 
 1905 static enum RC rc_class(OptoReg::Name reg) {
 1906 
 1907   if (reg == OptoReg::Bad) {
 1908     return rc_bad;
 1909   }
 1910 
 1911   // we have 32 int registers * 2 halves
 1912   int slots_of_int_registers = Register::number_of_registers * Register::max_slots_per_register;
 1913 
 1914   if (reg < slots_of_int_registers) {
 1915     return rc_int;
 1916   }
 1917 
 1918   // we have 32 float register * 8 halves
 1919   int slots_of_float_registers = FloatRegister::number_of_registers * FloatRegister::max_slots_per_register;
 1920   if (reg < slots_of_int_registers + slots_of_float_registers) {
 1921     return rc_float;
 1922   }
 1923 
 1924   int slots_of_predicate_registers = PRegister::number_of_registers * PRegister::max_slots_per_register;
 1925   if (reg < slots_of_int_registers + slots_of_float_registers + slots_of_predicate_registers) {
 1926     return rc_predicate;
 1927   }
 1928 
 1929   // Between predicate regs & stack is the flags.
 1930   assert(OptoReg::is_stack(reg), "blow up if spilling flags");
 1931 
 1932   return rc_stack;
 1933 }
 1934 
 1935 uint MachSpillCopyNode::implementation(C2_MacroAssembler *masm, PhaseRegAlloc *ra_, bool do_size, outputStream *st) const {
 1936   Compile* C = ra_->C;
 1937 
 1938   // Get registers to move.
 1939   OptoReg::Name src_hi = ra_->get_reg_second(in(1));
 1940   OptoReg::Name src_lo = ra_->get_reg_first(in(1));
 1941   OptoReg::Name dst_hi = ra_->get_reg_second(this);
 1942   OptoReg::Name dst_lo = ra_->get_reg_first(this);
 1943 
 1944   enum RC src_hi_rc = rc_class(src_hi);
 1945   enum RC src_lo_rc = rc_class(src_lo);
 1946   enum RC dst_hi_rc = rc_class(dst_hi);
 1947   enum RC dst_lo_rc = rc_class(dst_lo);
 1948 
 1949   assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
 1950 
 1951   if (src_hi != OptoReg::Bad && !bottom_type()->isa_vectmask()) {
 1952     assert((src_lo&1)==0 && src_lo+1==src_hi &&
 1953            (dst_lo&1)==0 && dst_lo+1==dst_hi,
 1954            "expected aligned-adjacent pairs");
 1955   }
 1956 
 1957   if (src_lo == dst_lo && src_hi == dst_hi) {
 1958     return 0;            // Self copy, no move.
 1959   }
 1960 
 1961   bool is64 = (src_lo & 1) == 0 && src_lo + 1 == src_hi &&
 1962               (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi;
 1963   int src_offset = ra_->reg2offset(src_lo);
 1964   int dst_offset = ra_->reg2offset(dst_lo);
 1965 
 1966   if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
 1967     uint ireg = ideal_reg();
 1968     if (ireg == Op_VecA && masm) {
 1969       int sve_vector_reg_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
 1970       if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
 1971         // stack->stack
 1972         __ spill_copy_sve_vector_stack_to_stack(src_offset, dst_offset,
 1973                                                 sve_vector_reg_size_in_bytes);
 1974       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
 1975         __ spill_sve_vector(as_FloatRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
 1976                             sve_vector_reg_size_in_bytes);
 1977       } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
 1978         __ unspill_sve_vector(as_FloatRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
 1979                               sve_vector_reg_size_in_bytes);
 1980       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
 1981         __ sve_orr(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 1982                    as_FloatRegister(Matcher::_regEncode[src_lo]),
 1983                    as_FloatRegister(Matcher::_regEncode[src_lo]));
 1984       } else {
 1985         ShouldNotReachHere();
 1986       }
 1987     } else if (masm) {
 1988       assert(ireg == Op_VecD || ireg == Op_VecX, "must be 64 bit or 128 bit vector");
 1989       assert((src_lo_rc != rc_int && dst_lo_rc != rc_int), "sanity");
 1990       if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
 1991         // stack->stack
 1992         assert((src_offset & 7) == 0 && (dst_offset & 7) == 0, "unaligned stack offset");
 1993         if (ireg == Op_VecD) {
 1994           __ unspill(rscratch1, true, src_offset);
 1995           __ spill(rscratch1, true, dst_offset);
 1996         } else {
 1997           __ spill_copy128(src_offset, dst_offset);
 1998         }
 1999       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
 2000         __ mov(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2001                ireg == Op_VecD ? __ T8B : __ T16B,
 2002                as_FloatRegister(Matcher::_regEncode[src_lo]));
 2003       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
 2004         __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
 2005                  ireg == Op_VecD ? __ D : __ Q,
 2006                  ra_->reg2offset(dst_lo));
 2007       } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
 2008         __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2009                    ireg == Op_VecD ? __ D : __ Q,
 2010                    ra_->reg2offset(src_lo));
 2011       } else {
 2012         ShouldNotReachHere();
 2013       }
 2014     }
 2015   } else if (masm) {
 2016     switch (src_lo_rc) {
 2017     case rc_int:
 2018       if (dst_lo_rc == rc_int) {  // gpr --> gpr copy
 2019         if (is64) {
 2020             __ mov(as_Register(Matcher::_regEncode[dst_lo]),
 2021                    as_Register(Matcher::_regEncode[src_lo]));
 2022         } else {
 2023             __ movw(as_Register(Matcher::_regEncode[dst_lo]),
 2024                     as_Register(Matcher::_regEncode[src_lo]));
 2025         }
 2026       } else if (dst_lo_rc == rc_float) { // gpr --> fpr copy
 2027         if (is64) {
 2028             __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2029                      as_Register(Matcher::_regEncode[src_lo]));
 2030         } else {
 2031             __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2032                      as_Register(Matcher::_regEncode[src_lo]));
 2033         }
 2034       } else {                    // gpr --> stack spill
 2035         assert(dst_lo_rc == rc_stack, "spill to bad register class");
 2036         __ spill(as_Register(Matcher::_regEncode[src_lo]), is64, dst_offset);
 2037       }
 2038       break;
 2039     case rc_float:
 2040       if (dst_lo_rc == rc_int) {  // fpr --> gpr copy
 2041         if (is64) {
 2042             __ fmovd(as_Register(Matcher::_regEncode[dst_lo]),
 2043                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2044         } else {
 2045             __ fmovs(as_Register(Matcher::_regEncode[dst_lo]),
 2046                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2047         }
 2048       } else if (dst_lo_rc == rc_float) { // fpr --> fpr copy
 2049         if (is64) {
 2050             __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2051                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2052         } else {
 2053             __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2054                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2055         }
 2056       } else {                    // fpr --> stack spill
 2057         assert(dst_lo_rc == rc_stack, "spill to bad register class");
 2058         __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
 2059                  is64 ? __ D : __ S, dst_offset);
 2060       }
 2061       break;
 2062     case rc_stack:
 2063       if (dst_lo_rc == rc_int) {  // stack --> gpr load
 2064         __ unspill(as_Register(Matcher::_regEncode[dst_lo]), is64, src_offset);
 2065       } else if (dst_lo_rc == rc_float) { // stack --> fpr load
 2066         __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2067                    is64 ? __ D : __ S, src_offset);
 2068       } else if (dst_lo_rc == rc_predicate) {
 2069         __ unspill_sve_predicate(as_PRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
 2070                                  Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
 2071       } else {                    // stack --> stack copy
 2072         assert(dst_lo_rc == rc_stack, "spill to bad register class");
 2073         if (ideal_reg() == Op_RegVectMask) {
 2074           __ spill_copy_sve_predicate_stack_to_stack(src_offset, dst_offset,
 2075                                                      Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
 2076         } else {
 2077           __ unspill(rscratch1, is64, src_offset);
 2078           __ spill(rscratch1, is64, dst_offset);
 2079         }
 2080       }
 2081       break;
 2082     case rc_predicate:
 2083       if (dst_lo_rc == rc_predicate) {
 2084         __ sve_mov(as_PRegister(Matcher::_regEncode[dst_lo]), as_PRegister(Matcher::_regEncode[src_lo]));
 2085       } else if (dst_lo_rc == rc_stack) {
 2086         __ spill_sve_predicate(as_PRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
 2087                                Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
 2088       } else {
 2089         assert(false, "bad src and dst rc_class combination.");
 2090         ShouldNotReachHere();
 2091       }
 2092       break;
 2093     default:
 2094       assert(false, "bad rc_class for spill");
 2095       ShouldNotReachHere();
 2096     }
 2097   }
 2098 
 2099   if (st) {
 2100     st->print("spill ");
 2101     if (src_lo_rc == rc_stack) {
 2102       st->print("[sp, #%d] -> ", ra_->reg2offset(src_lo));
 2103     } else {
 2104       st->print("%s -> ", Matcher::regName[src_lo]);
 2105     }
 2106     if (dst_lo_rc == rc_stack) {
 2107       st->print("[sp, #%d]", ra_->reg2offset(dst_lo));
 2108     } else {
 2109       st->print("%s", Matcher::regName[dst_lo]);
 2110     }
 2111     if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
 2112       int vsize = 0;
 2113       switch (ideal_reg()) {
 2114       case Op_VecD:
 2115         vsize = 64;
 2116         break;
 2117       case Op_VecX:
 2118         vsize = 128;
 2119         break;
 2120       case Op_VecA:
 2121         vsize = Matcher::scalable_vector_reg_size(T_BYTE) * 8;
 2122         break;
 2123       default:
 2124         assert(false, "bad register type for spill");
 2125         ShouldNotReachHere();
 2126       }
 2127       st->print("\t# vector spill size = %d", vsize);
 2128     } else if (ideal_reg() == Op_RegVectMask) {
 2129       assert(Matcher::supports_scalable_vector(), "bad register type for spill");
 2130       int vsize = Matcher::scalable_predicate_reg_slots() * 32;
 2131       st->print("\t# predicate spill size = %d", vsize);
 2132     } else {
 2133       st->print("\t# spill size = %d", is64 ? 64 : 32);
 2134     }
 2135   }
 2136 
 2137   return 0;
 2138 
 2139 }
 2140 
 2141 #ifndef PRODUCT
 2142 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 2143   if (!ra_)
 2144     st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
 2145   else
 2146     implementation(nullptr, ra_, false, st);
 2147 }
 2148 #endif
 2149 
 2150 void MachSpillCopyNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
 2151   implementation(masm, ra_, false, nullptr);
 2152 }
 2153 
 2154 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
 2155   return MachNode::size(ra_);
 2156 }
 2157 
 2158 //=============================================================================
 2159 
 2160 #ifndef PRODUCT
 2161 void BoxLockNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 2162   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 2163   int reg = ra_->get_reg_first(this);
 2164   st->print("add %s, rsp, #%d]\t# box lock",
 2165             Matcher::regName[reg], offset);
 2166 }
 2167 #endif
 2168 
 2169 void BoxLockNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
 2170   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 2171   int reg    = ra_->get_encode(this);
 2172 
 2173   // This add will handle any 24-bit signed offset. 24 bits allows an
 2174   // 8 megabyte stack frame.
 2175   __ add(as_Register(reg), sp, offset);
 2176 }
 2177 
 2178 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
 2179   // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_).
 2180   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 2181 
 2182   if (Assembler::operand_valid_for_add_sub_immediate(offset)) {
 2183     return NativeInstruction::instruction_size;
 2184   } else {
 2185     return 2 * NativeInstruction::instruction_size;
 2186   }
 2187 }
 2188 
 2189 ///=============================================================================
 2190 #ifndef PRODUCT
 2191 void MachVEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 2192 {
 2193   st->print_cr("# MachVEPNode");
 2194   if (!_verified) {
 2195     st->print_cr("\t load_class");
 2196   } else {
 2197     st->print_cr("\t unpack_inline_arg");
 2198   }
 2199 }
 2200 #endif
 2201 
 2202 void MachVEPNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc* ra_) const
 2203 {
 2204   if (!_verified) {
 2205     __ ic_check(1);
 2206   } else {
 2207     // TODO 8284443 Avoid creation of temporary frame
 2208     if (ra_->C->stub_function() == nullptr) {
 2209       __ verified_entry(ra_->C, 0);
 2210       __ entry_barrier();
 2211       int framesize = ra_->C->output()->frame_slots() << LogBytesPerInt;
 2212       __ remove_frame(framesize, false);
 2213     }
 2214     // Unpack inline type args passed as oop and then jump to
 2215     // the verified entry point (skipping the unverified entry).
 2216     int sp_inc = __ unpack_inline_args(ra_->C, _receiver_only);
 2217     // Emit code for verified entry and save increment for stack repair on return
 2218     __ verified_entry(ra_->C, sp_inc);
 2219     if (Compile::current()->output()->in_scratch_emit_size()) {
 2220       Label dummy_verified_entry;
 2221       __ b(dummy_verified_entry);
 2222     } else {
 2223       __ b(*_verified_entry);
 2224     }
 2225   }
 2226 }
 2227 
 2228 //=============================================================================
 2229 #ifndef PRODUCT
 2230 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 2231 {
 2232   st->print_cr("# MachUEPNode");
 2233   if (UseCompressedClassPointers) {
 2234     st->print_cr("\tldrw rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
 2235     st->print_cr("\tldrw r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
 2236     st->print_cr("\tcmpw rscratch1, r10");
 2237   } else {
 2238     st->print_cr("\tldr rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
 2239     st->print_cr("\tldr r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
 2240     st->print_cr("\tcmp rscratch1, r10");
 2241   }
 2242   st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
 2243 }
 2244 #endif
 2245 
 2246 void MachUEPNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
 2247 {
 2248   __ ic_check(InteriorEntryAlignment);
 2249 }
 2250 
 2251 // REQUIRED EMIT CODE
 2252 
 2253 //=============================================================================
 2254 
 2255 // Emit exception handler code.
 2256 int HandlerImpl::emit_exception_handler(C2_MacroAssembler* masm)
 2257 {
 2258   // mov rscratch1 #exception_blob_entry_point
 2259   // br rscratch1
 2260   // Note that the code buffer's insts_mark is always relative to insts.
 2261   // That's why we must use the macroassembler to generate a handler.
 2262   address base = __ start_a_stub(size_exception_handler());
 2263   if (base == nullptr) {
 2264     ciEnv::current()->record_failure("CodeCache is full");
 2265     return 0;  // CodeBuffer::expand failed
 2266   }
 2267   int offset = __ offset();
 2268   __ far_jump(RuntimeAddress(OptoRuntime::exception_blob()->entry_point()));
 2269   assert(__ offset() - offset <= (int) size_exception_handler(), "overflow");
 2270   __ end_a_stub();
 2271   return offset;
 2272 }
 2273 
 2274 // Emit deopt handler code.
 2275 int HandlerImpl::emit_deopt_handler(C2_MacroAssembler* masm)
 2276 {
 2277   // Note that the code buffer's insts_mark is always relative to insts.
 2278   // That's why we must use the macroassembler to generate a handler.
 2279   address base = __ start_a_stub(size_deopt_handler());
 2280   if (base == nullptr) {
 2281     ciEnv::current()->record_failure("CodeCache is full");
 2282     return 0;  // CodeBuffer::expand failed
 2283   }
 2284   int offset = __ offset();
 2285 
 2286   __ adr(lr, __ pc());
 2287   __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
 2288 
 2289   assert(__ offset() - offset == (int) size_deopt_handler(), "overflow");
 2290   __ end_a_stub();
 2291   return offset;
 2292 }
 2293 
 2294 // REQUIRED MATCHER CODE
 2295 
 2296 //=============================================================================
 2297 
 2298 bool Matcher::match_rule_supported(int opcode) {
 2299   if (!has_match_rule(opcode))
 2300     return false;
 2301 
 2302   switch (opcode) {
 2303     case Op_OnSpinWait:
 2304       return VM_Version::supports_on_spin_wait();
 2305     case Op_CacheWB:
 2306     case Op_CacheWBPreSync:
 2307     case Op_CacheWBPostSync:
 2308       if (!VM_Version::supports_data_cache_line_flush()) {
 2309         return false;
 2310       }
 2311       break;
 2312     case Op_ExpandBits:
 2313     case Op_CompressBits:
 2314       if (!VM_Version::supports_svebitperm()) {
 2315         return false;
 2316       }
 2317       break;
 2318     case Op_FmaF:
 2319     case Op_FmaD:
 2320     case Op_FmaVF:
 2321     case Op_FmaVD:
 2322       if (!UseFMA) {
 2323         return false;
 2324       }
 2325       break;
 2326     case Op_FmaHF:
 2327       // UseFMA flag also needs to be checked along with FEAT_FP16
 2328       if (!UseFMA || !is_feat_fp16_supported()) {
 2329         return false;
 2330       }
 2331       break;
 2332     case Op_AddHF:
 2333     case Op_SubHF:
 2334     case Op_MulHF:
 2335     case Op_DivHF:
 2336     case Op_MinHF:
 2337     case Op_MaxHF:
 2338     case Op_SqrtHF:
 2339       // Half-precision floating point scalar operations require FEAT_FP16
 2340       // to be available. FEAT_FP16 is enabled if both "fphp" and "asimdhp"
 2341       // features are supported.
 2342       if (!is_feat_fp16_supported()) {
 2343         return false;
 2344       }
 2345       break;
 2346   }
 2347 
 2348   return true; // Per default match rules are supported.
 2349 }
 2350 
 2351 const RegMask* Matcher::predicate_reg_mask(void) {
 2352   return &_PR_REG_mask;
 2353 }
 2354 
 2355 bool Matcher::supports_vector_calling_convention(void) {
 2356   return EnableVectorSupport;
 2357 }
 2358 
 2359 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
 2360   assert(EnableVectorSupport, "sanity");
 2361   int lo = V0_num;
 2362   int hi = V0_H_num;
 2363   if (ideal_reg == Op_VecX || ideal_reg == Op_VecA) {
 2364     hi = V0_K_num;
 2365   }
 2366   return OptoRegPair(hi, lo);
 2367 }
 2368 
 2369 // Is this branch offset short enough that a short branch can be used?
 2370 //
 2371 // NOTE: If the platform does not provide any short branch variants, then
 2372 //       this method should return false for offset 0.
 2373 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
 2374   // The passed offset is relative to address of the branch.
 2375 
 2376   return (-32768 <= offset && offset < 32768);
 2377 }
 2378 
 2379 // Vector width in bytes.
 2380 int Matcher::vector_width_in_bytes(BasicType bt) {
 2381   // The MaxVectorSize should have been set by detecting SVE max vector register size.
 2382   int size = MIN2((UseSVE > 0) ? (int)FloatRegister::sve_vl_max : (int)FloatRegister::neon_vl, (int)MaxVectorSize);
 2383   // Minimum 2 values in vector
 2384   if (size < 2*type2aelembytes(bt)) size = 0;
 2385   // But never < 4
 2386   if (size < 4) size = 0;
 2387   return size;
 2388 }
 2389 
 2390 // Limits on vector size (number of elements) loaded into vector.
 2391 int Matcher::max_vector_size(const BasicType bt) {
 2392   return vector_width_in_bytes(bt)/type2aelembytes(bt);
 2393 }
 2394 
 2395 int Matcher::min_vector_size(const BasicType bt) {
 2396   // Usually, the shortest vector length supported by AArch64 ISA and
 2397   // Vector API species is 64 bits. However, we allow 32-bit or 16-bit
 2398   // vectors in a few special cases.
 2399   int size;
 2400   switch(bt) {
 2401     case T_BOOLEAN:
 2402       // Load/store a vector mask with only 2 elements for vector types
 2403       // such as "2I/2F/2L/2D".
 2404       size = 2;
 2405       break;
 2406     case T_BYTE:
 2407       // Generate a "4B" vector, to support vector cast between "8B/16B"
 2408       // and "4S/4I/4L/4F/4D".
 2409       size = 4;
 2410       break;
 2411     case T_SHORT:
 2412       // Generate a "2S" vector, to support vector cast between "4S/8S"
 2413       // and "2I/2L/2F/2D".
 2414       size = 2;
 2415       break;
 2416     default:
 2417       // Limit the min vector length to 64-bit.
 2418       size = 8 / type2aelembytes(bt);
 2419       // The number of elements in a vector should be at least 2.
 2420       size = MAX2(size, 2);
 2421   }
 2422 
 2423   int max_size = max_vector_size(bt);
 2424   return MIN2(size, max_size);
 2425 }
 2426 
 2427 int Matcher::max_vector_size_auto_vectorization(const BasicType bt) {
 2428   return Matcher::max_vector_size(bt);
 2429 }
 2430 
 2431 // Actual max scalable vector register length.
 2432 int Matcher::scalable_vector_reg_size(const BasicType bt) {
 2433   return Matcher::max_vector_size(bt);
 2434 }
 2435 
 2436 // Vector ideal reg.
 2437 uint Matcher::vector_ideal_reg(int len) {
 2438   if (UseSVE > 0 && FloatRegister::neon_vl < len && len <= FloatRegister::sve_vl_max) {
 2439     return Op_VecA;
 2440   }
 2441   switch(len) {
 2442     // For 16-bit/32-bit mask vector, reuse VecD.
 2443     case  2:
 2444     case  4:
 2445     case  8: return Op_VecD;
 2446     case 16: return Op_VecX;
 2447   }
 2448   ShouldNotReachHere();
 2449   return 0;
 2450 }
 2451 
 2452 MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* generic_opnd, uint ideal_reg, bool is_temp) {
 2453   assert(Matcher::is_generic_vector(generic_opnd), "not generic");
 2454   switch (ideal_reg) {
 2455     case Op_VecA: return new vecAOper();
 2456     case Op_VecD: return new vecDOper();
 2457     case Op_VecX: return new vecXOper();
 2458   }
 2459   ShouldNotReachHere();
 2460   return nullptr;
 2461 }
 2462 
 2463 bool Matcher::is_reg2reg_move(MachNode* m) {
 2464   return false;
 2465 }
 2466 
 2467 bool Matcher::is_generic_vector(MachOper* opnd)  {
 2468   return opnd->opcode() == VREG;
 2469 }
 2470 
 2471 // Return whether or not this register is ever used as an argument.
 2472 // This function is used on startup to build the trampoline stubs in
 2473 // generateOptoStub.  Registers not mentioned will be killed by the VM
 2474 // call in the trampoline, and arguments in those registers not be
 2475 // available to the callee.
 2476 bool Matcher::can_be_java_arg(int reg)
 2477 {
 2478   return
 2479     reg ==  R0_num || reg == R0_H_num ||
 2480     reg ==  R1_num || reg == R1_H_num ||
 2481     reg ==  R2_num || reg == R2_H_num ||
 2482     reg ==  R3_num || reg == R3_H_num ||
 2483     reg ==  R4_num || reg == R4_H_num ||
 2484     reg ==  R5_num || reg == R5_H_num ||
 2485     reg ==  R6_num || reg == R6_H_num ||
 2486     reg ==  R7_num || reg == R7_H_num ||
 2487     reg ==  V0_num || reg == V0_H_num ||
 2488     reg ==  V1_num || reg == V1_H_num ||
 2489     reg ==  V2_num || reg == V2_H_num ||
 2490     reg ==  V3_num || reg == V3_H_num ||
 2491     reg ==  V4_num || reg == V4_H_num ||
 2492     reg ==  V5_num || reg == V5_H_num ||
 2493     reg ==  V6_num || reg == V6_H_num ||
 2494     reg ==  V7_num || reg == V7_H_num;
 2495 }
 2496 
 2497 bool Matcher::is_spillable_arg(int reg)
 2498 {
 2499   return can_be_java_arg(reg);
 2500 }
 2501 
 2502 uint Matcher::int_pressure_limit()
 2503 {
 2504   // JDK-8183543: When taking the number of available registers as int
 2505   // register pressure threshold, the jtreg test:
 2506   // test/hotspot/jtreg/compiler/regalloc/TestC2IntPressure.java
 2507   // failed due to C2 compilation failure with
 2508   // "COMPILE SKIPPED: failed spill-split-recycle sanity check".
 2509   //
 2510   // A derived pointer is live at CallNode and then is flagged by RA
 2511   // as a spilled LRG. Spilling heuristics(Spill-USE) explicitly skip
 2512   // derived pointers and lastly fail to spill after reaching maximum
 2513   // number of iterations. Lowering the default pressure threshold to
 2514   // (_NO_SPECIAL_REG32_mask.Size() minus 1) forces CallNode to become
 2515   // a high register pressure area of the code so that split_DEF can
 2516   // generate DefinitionSpillCopy for the derived pointer.
 2517   uint default_int_pressure_threshold = _NO_SPECIAL_REG32_mask.Size() - 1;
 2518   if (!PreserveFramePointer) {
 2519     // When PreserveFramePointer is off, frame pointer is allocatable,
 2520     // but different from other SOC registers, it is excluded from
 2521     // fatproj's mask because its save type is No-Save. Decrease 1 to
 2522     // ensure high pressure at fatproj when PreserveFramePointer is off.
 2523     // See check_pressure_at_fatproj().
 2524     default_int_pressure_threshold--;
 2525   }
 2526   return (INTPRESSURE == -1) ? default_int_pressure_threshold : INTPRESSURE;
 2527 }
 2528 
 2529 uint Matcher::float_pressure_limit()
 2530 {
 2531   // _FLOAT_REG_mask is generated by adlc from the float_reg register class.
 2532   return (FLOATPRESSURE == -1) ? _FLOAT_REG_mask.Size() : FLOATPRESSURE;
 2533 }
 2534 
 2535 bool Matcher::use_asm_for_ldiv_by_con(jlong divisor) {
 2536   return false;
 2537 }
 2538 
 2539 RegMask Matcher::divI_proj_mask() {
 2540   ShouldNotReachHere();
 2541   return RegMask();
 2542 }
 2543 
 2544 // Register for MODI projection of divmodI.
 2545 RegMask Matcher::modI_proj_mask() {
 2546   ShouldNotReachHere();
 2547   return RegMask();
 2548 }
 2549 
 2550 // Register for DIVL projection of divmodL.
 2551 RegMask Matcher::divL_proj_mask() {
 2552   ShouldNotReachHere();
 2553   return RegMask();
 2554 }
 2555 
 2556 // Register for MODL projection of divmodL.
 2557 RegMask Matcher::modL_proj_mask() {
 2558   ShouldNotReachHere();
 2559   return RegMask();
 2560 }
 2561 
 2562 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
 2563   return FP_REG_mask();
 2564 }
 2565 
 2566 bool size_fits_all_mem_uses(AddPNode* addp, int shift) {
 2567   for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) {
 2568     Node* u = addp->fast_out(i);
 2569     if (u->is_LoadStore()) {
 2570       // On AArch64, LoadStoreNodes (i.e. compare and swap
 2571       // instructions) only take register indirect as an operand, so
 2572       // any attempt to use an AddPNode as an input to a LoadStoreNode
 2573       // must fail.
 2574       return false;
 2575     }
 2576     if (u->is_Mem()) {
 2577       int opsize = u->as_Mem()->memory_size();
 2578       assert(opsize > 0, "unexpected memory operand size");
 2579       if (u->as_Mem()->memory_size() != (1<<shift)) {
 2580         return false;
 2581       }
 2582     }
 2583   }
 2584   return true;
 2585 }
 2586 
 2587 // Convert BootTest condition to Assembler condition.
 2588 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
 2589 Assembler::Condition to_assembler_cond(BoolTest::mask cond) {
 2590   Assembler::Condition result;
 2591   switch(cond) {
 2592     case BoolTest::eq:
 2593       result = Assembler::EQ; break;
 2594     case BoolTest::ne:
 2595       result = Assembler::NE; break;
 2596     case BoolTest::le:
 2597       result = Assembler::LE; break;
 2598     case BoolTest::ge:
 2599       result = Assembler::GE; break;
 2600     case BoolTest::lt:
 2601       result = Assembler::LT; break;
 2602     case BoolTest::gt:
 2603       result = Assembler::GT; break;
 2604     case BoolTest::ule:
 2605       result = Assembler::LS; break;
 2606     case BoolTest::uge:
 2607       result = Assembler::HS; break;
 2608     case BoolTest::ult:
 2609       result = Assembler::LO; break;
 2610     case BoolTest::ugt:
 2611       result = Assembler::HI; break;
 2612     case BoolTest::overflow:
 2613       result = Assembler::VS; break;
 2614     case BoolTest::no_overflow:
 2615       result = Assembler::VC; break;
 2616     default:
 2617       ShouldNotReachHere();
 2618       return Assembler::Condition(-1);
 2619   }
 2620 
 2621   // Check conversion
 2622   if (cond & BoolTest::unsigned_compare) {
 2623     assert(cmpOpUOper((BoolTest::mask)((int)cond & ~(BoolTest::unsigned_compare))).ccode() == result, "Invalid conversion");
 2624   } else {
 2625     assert(cmpOpOper(cond).ccode() == result, "Invalid conversion");
 2626   }
 2627 
 2628   return result;
 2629 }
 2630 
 2631 // Binary src (Replicate con)
 2632 static bool is_valid_sve_arith_imm_pattern(Node* n, Node* m) {
 2633   if (n == nullptr || m == nullptr) {
 2634     return false;
 2635   }
 2636 
 2637   if (UseSVE == 0 || m->Opcode() != Op_Replicate) {
 2638     return false;
 2639   }
 2640 
 2641   Node* imm_node = m->in(1);
 2642   if (!imm_node->is_Con()) {
 2643     return false;
 2644   }
 2645 
 2646   const Type* t = imm_node->bottom_type();
 2647   if (!(t->isa_int() || t->isa_long())) {
 2648     return false;
 2649   }
 2650 
 2651   switch (n->Opcode()) {
 2652   case Op_AndV:
 2653   case Op_OrV:
 2654   case Op_XorV: {
 2655     Assembler::SIMD_RegVariant T = Assembler::elemType_to_regVariant(Matcher::vector_element_basic_type(n));
 2656     uint64_t value = t->isa_long() ? (uint64_t)imm_node->get_long() : (uint64_t)imm_node->get_int();
 2657     return Assembler::operand_valid_for_sve_logical_immediate(Assembler::regVariant_to_elemBits(T), value);
 2658   }
 2659   case Op_AddVB:
 2660     return (imm_node->get_int() <= 255 && imm_node->get_int() >= -255);
 2661   case Op_AddVS:
 2662   case Op_AddVI:
 2663     return Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)imm_node->get_int());
 2664   case Op_AddVL:
 2665     return Assembler::operand_valid_for_sve_add_sub_immediate(imm_node->get_long());
 2666   default:
 2667     return false;
 2668   }
 2669 }
 2670 
 2671 // (XorV src (Replicate m1))
 2672 // (XorVMask src (MaskAll m1))
 2673 static bool is_vector_bitwise_not_pattern(Node* n, Node* m) {
 2674   if (n != nullptr && m != nullptr) {
 2675     return (n->Opcode() == Op_XorV || n->Opcode() == Op_XorVMask) &&
 2676            VectorNode::is_all_ones_vector(m);
 2677   }
 2678   return false;
 2679 }
 2680 
 2681 // Should the matcher clone input 'm' of node 'n'?
 2682 bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
 2683   if (is_vshift_con_pattern(n, m) ||
 2684       is_vector_bitwise_not_pattern(n, m) ||
 2685       is_valid_sve_arith_imm_pattern(n, m) ||
 2686       is_encode_and_store_pattern(n, m)) {
 2687     mstack.push(m, Visit);
 2688     return true;
 2689   }
 2690   return false;
 2691 }
 2692 
 2693 // Should the Matcher clone shifts on addressing modes, expecting them
 2694 // to be subsumed into complex addressing expressions or compute them
 2695 // into registers?
 2696 bool Matcher::pd_clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
 2697 
 2698   // Loads and stores with indirect memory input (e.g., volatile loads and
 2699   // stores) do not subsume the input into complex addressing expressions. If
 2700   // the addressing expression is input to at least one such load or store, do
 2701   // not clone the addressing expression. Query needs_acquiring_load and
 2702   // needs_releasing_store as a proxy for indirect memory input, as it is not
 2703   // possible to directly query for indirect memory input at this stage.
 2704   for (DUIterator_Fast imax, i = m->fast_outs(imax); i < imax; i++) {
 2705     Node* n = m->fast_out(i);
 2706     if (n->is_Load() && needs_acquiring_load(n)) {
 2707       return false;
 2708     }
 2709     if (n->is_Store() && needs_releasing_store(n)) {
 2710       return false;
 2711     }
 2712   }
 2713 
 2714   if (clone_base_plus_offset_address(m, mstack, address_visited)) {
 2715     return true;
 2716   }
 2717 
 2718   Node *off = m->in(AddPNode::Offset);
 2719   if (off->Opcode() == Op_LShiftL && off->in(2)->is_Con() &&
 2720       size_fits_all_mem_uses(m, off->in(2)->get_int()) &&
 2721       // Are there other uses besides address expressions?
 2722       !is_visited(off)) {
 2723     address_visited.set(off->_idx); // Flag as address_visited
 2724     mstack.push(off->in(2), Visit);
 2725     Node *conv = off->in(1);
 2726     if (conv->Opcode() == Op_ConvI2L &&
 2727         // Are there other uses besides address expressions?
 2728         !is_visited(conv)) {
 2729       address_visited.set(conv->_idx); // Flag as address_visited
 2730       mstack.push(conv->in(1), Pre_Visit);
 2731     } else {
 2732       mstack.push(conv, Pre_Visit);
 2733     }
 2734     address_visited.test_set(m->_idx); // Flag as address_visited
 2735     mstack.push(m->in(AddPNode::Address), Pre_Visit);
 2736     mstack.push(m->in(AddPNode::Base), Pre_Visit);
 2737     return true;
 2738   } else if (off->Opcode() == Op_ConvI2L &&
 2739              // Are there other uses besides address expressions?
 2740              !is_visited(off)) {
 2741     address_visited.test_set(m->_idx); // Flag as address_visited
 2742     address_visited.set(off->_idx); // Flag as address_visited
 2743     mstack.push(off->in(1), Pre_Visit);
 2744     mstack.push(m->in(AddPNode::Address), Pre_Visit);
 2745     mstack.push(m->in(AddPNode::Base), Pre_Visit);
 2746     return true;
 2747   }
 2748   return false;
 2749 }
 2750 
 2751 #define MOV_VOLATILE(REG, BASE, INDEX, SCALE, DISP, SCRATCH, INSN)      \
 2752   {                                                                     \
 2753     guarantee(INDEX == -1, "mode not permitted for volatile");          \
 2754     guarantee(DISP == 0, "mode not permitted for volatile");            \
 2755     guarantee(SCALE == 0, "mode not permitted for volatile");           \
 2756     __ INSN(REG, as_Register(BASE));                                    \
 2757   }
 2758 
 2759 
 2760 static Address mem2address(int opcode, Register base, int index, int size, int disp)
 2761   {
 2762     Address::extend scale;
 2763 
 2764     // Hooboy, this is fugly.  We need a way to communicate to the
 2765     // encoder that the index needs to be sign extended, so we have to
 2766     // enumerate all the cases.
 2767     switch (opcode) {
 2768     case INDINDEXSCALEDI2L:
 2769     case INDINDEXSCALEDI2LN:
 2770     case INDINDEXI2L:
 2771     case INDINDEXI2LN:
 2772       scale = Address::sxtw(size);
 2773       break;
 2774     default:
 2775       scale = Address::lsl(size);
 2776     }
 2777 
 2778     if (index == -1) {
 2779       return Address(base, disp);
 2780     } else {
 2781       assert(disp == 0, "unsupported address mode: disp = %d", disp);
 2782       return Address(base, as_Register(index), scale);
 2783     }
 2784   }
 2785 
 2786 
 2787 typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr);
 2788 typedef void (MacroAssembler::* mem_insn2)(Register Rt, Register adr);
 2789 typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr);
 2790 typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt,
 2791                                   MacroAssembler::SIMD_RegVariant T, const Address &adr);
 2792 
 2793   // Used for all non-volatile memory accesses.  The use of
 2794   // $mem->opcode() to discover whether this pattern uses sign-extended
 2795   // offsets is something of a kludge.
 2796   static void loadStore(C2_MacroAssembler* masm, mem_insn insn,
 2797                         Register reg, int opcode,
 2798                         Register base, int index, int scale, int disp,
 2799                         int size_in_memory)
 2800   {
 2801     Address addr = mem2address(opcode, base, index, scale, disp);
 2802     if (addr.getMode() == Address::base_plus_offset) {
 2803       /* Fix up any out-of-range offsets. */
 2804       assert_different_registers(rscratch1, base);
 2805       assert_different_registers(rscratch1, reg);
 2806       addr = __ legitimize_address(addr, size_in_memory, rscratch1);
 2807     }
 2808     (masm->*insn)(reg, addr);
 2809   }
 2810 
 2811   static void loadStore(C2_MacroAssembler* masm, mem_float_insn insn,
 2812                         FloatRegister reg, int opcode,
 2813                         Register base, int index, int size, int disp,
 2814                         int size_in_memory)
 2815   {
 2816     Address::extend scale;
 2817 
 2818     switch (opcode) {
 2819     case INDINDEXSCALEDI2L:
 2820     case INDINDEXSCALEDI2LN:
 2821       scale = Address::sxtw(size);
 2822       break;
 2823     default:
 2824       scale = Address::lsl(size);
 2825     }
 2826 
 2827     if (index == -1) {
 2828       // Fix up any out-of-range offsets.
 2829       assert_different_registers(rscratch1, base);
 2830       Address addr = Address(base, disp);
 2831       addr = __ legitimize_address(addr, size_in_memory, rscratch1);
 2832       (masm->*insn)(reg, addr);
 2833     } else {
 2834       assert(disp == 0, "unsupported address mode: disp = %d", disp);
 2835       (masm->*insn)(reg, Address(base, as_Register(index), scale));
 2836     }
 2837   }
 2838 
 2839   static void loadStore(C2_MacroAssembler* masm, mem_vector_insn insn,
 2840                         FloatRegister reg, MacroAssembler::SIMD_RegVariant T,
 2841                         int opcode, Register base, int index, int size, int disp)
 2842   {
 2843     if (index == -1) {
 2844       (masm->*insn)(reg, T, Address(base, disp));
 2845     } else {
 2846       assert(disp == 0, "unsupported address mode");
 2847       (masm->*insn)(reg, T, Address(base, as_Register(index), Address::lsl(size)));
 2848     }
 2849   }
 2850 
 2851 %}
 2852 
 2853 
 2854 
 2855 //----------ENCODING BLOCK-----------------------------------------------------
 2856 // This block specifies the encoding classes used by the compiler to
 2857 // output byte streams.  Encoding classes are parameterized macros
 2858 // used by Machine Instruction Nodes in order to generate the bit
 2859 // encoding of the instruction.  Operands specify their base encoding
 2860 // interface with the interface keyword.  There are currently
 2861 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
 2862 // COND_INTER.  REG_INTER causes an operand to generate a function
 2863 // which returns its register number when queried.  CONST_INTER causes
 2864 // an operand to generate a function which returns the value of the
 2865 // constant when queried.  MEMORY_INTER causes an operand to generate
 2866 // four functions which return the Base Register, the Index Register,
 2867 // the Scale Value, and the Offset Value of the operand when queried.
 2868 // COND_INTER causes an operand to generate six functions which return
 2869 // the encoding code (ie - encoding bits for the instruction)
 2870 // associated with each basic boolean condition for a conditional
 2871 // instruction.
 2872 //
 2873 // Instructions specify two basic values for encoding.  Again, a
 2874 // function is available to check if the constant displacement is an
 2875 // oop. They use the ins_encode keyword to specify their encoding
 2876 // classes (which must be a sequence of enc_class names, and their
 2877 // parameters, specified in the encoding block), and they use the
 2878 // opcode keyword to specify, in order, their primary, secondary, and
 2879 // tertiary opcode.  Only the opcode sections which a particular
 2880 // instruction needs for encoding need to be specified.
 2881 encode %{
 2882   // Build emit functions for each basic byte or larger field in the
 2883   // intel encoding scheme (opcode, rm, sib, immediate), and call them
 2884   // from C++ code in the enc_class source block.  Emit functions will
 2885   // live in the main source block for now.  In future, we can
 2886   // generalize this by adding a syntax that specifies the sizes of
 2887   // fields in an order, so that the adlc can build the emit functions
 2888   // automagically
 2889 
 2890   // catch all for unimplemented encodings
 2891   enc_class enc_unimplemented %{
 2892     __ unimplemented("C2 catch all");
 2893   %}
 2894 
 2895   // BEGIN Non-volatile memory access
 2896 
 2897   // This encoding class is generated automatically from ad_encode.m4.
 2898   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2899   enc_class aarch64_enc_ldrsbw(iRegI dst, memory1 mem) %{
 2900     Register dst_reg = as_Register($dst$$reg);
 2901     loadStore(masm, &MacroAssembler::ldrsbw, dst_reg, $mem->opcode(),
 2902                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2903   %}
 2904 
 2905   // This encoding class is generated automatically from ad_encode.m4.
 2906   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2907   enc_class aarch64_enc_ldrsb(iRegI dst, memory1 mem) %{
 2908     Register dst_reg = as_Register($dst$$reg);
 2909     loadStore(masm, &MacroAssembler::ldrsb, dst_reg, $mem->opcode(),
 2910                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2911   %}
 2912 
 2913   // This encoding class is generated automatically from ad_encode.m4.
 2914   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2915   enc_class aarch64_enc_ldrb(iRegI dst, memory1 mem) %{
 2916     Register dst_reg = as_Register($dst$$reg);
 2917     loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
 2918                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2919   %}
 2920 
 2921   // This encoding class is generated automatically from ad_encode.m4.
 2922   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2923   enc_class aarch64_enc_ldrb(iRegL dst, memory1 mem) %{
 2924     Register dst_reg = as_Register($dst$$reg);
 2925     loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
 2926                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2927   %}
 2928 
 2929   // This encoding class is generated automatically from ad_encode.m4.
 2930   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2931   enc_class aarch64_enc_ldrshw(iRegI dst, memory2 mem) %{
 2932     Register dst_reg = as_Register($dst$$reg);
 2933     loadStore(masm, &MacroAssembler::ldrshw, dst_reg, $mem->opcode(),
 2934                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2935   %}
 2936 
 2937   // This encoding class is generated automatically from ad_encode.m4.
 2938   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2939   enc_class aarch64_enc_ldrsh(iRegI dst, memory2 mem) %{
 2940     Register dst_reg = as_Register($dst$$reg);
 2941     loadStore(masm, &MacroAssembler::ldrsh, dst_reg, $mem->opcode(),
 2942                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2943   %}
 2944 
 2945   // This encoding class is generated automatically from ad_encode.m4.
 2946   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2947   enc_class aarch64_enc_ldrh(iRegI dst, memory2 mem) %{
 2948     Register dst_reg = as_Register($dst$$reg);
 2949     loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
 2950                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2951   %}
 2952 
 2953   // This encoding class is generated automatically from ad_encode.m4.
 2954   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2955   enc_class aarch64_enc_ldrh(iRegL dst, memory2 mem) %{
 2956     Register dst_reg = as_Register($dst$$reg);
 2957     loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
 2958                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2959   %}
 2960 
 2961   // This encoding class is generated automatically from ad_encode.m4.
 2962   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2963   enc_class aarch64_enc_ldrw(iRegI dst, memory4 mem) %{
 2964     Register dst_reg = as_Register($dst$$reg);
 2965     loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
 2966                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2967   %}
 2968 
 2969   // This encoding class is generated automatically from ad_encode.m4.
 2970   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2971   enc_class aarch64_enc_ldrw(iRegL dst, memory4 mem) %{
 2972     Register dst_reg = as_Register($dst$$reg);
 2973     loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
 2974                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2975   %}
 2976 
 2977   // This encoding class is generated automatically from ad_encode.m4.
 2978   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2979   enc_class aarch64_enc_ldrsw(iRegL dst, memory4 mem) %{
 2980     Register dst_reg = as_Register($dst$$reg);
 2981     loadStore(masm, &MacroAssembler::ldrsw, dst_reg, $mem->opcode(),
 2982                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2983   %}
 2984 
 2985   // This encoding class is generated automatically from ad_encode.m4.
 2986   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2987   enc_class aarch64_enc_ldr(iRegL dst, memory8 mem) %{
 2988     Register dst_reg = as_Register($dst$$reg);
 2989     loadStore(masm, &MacroAssembler::ldr, dst_reg, $mem->opcode(),
 2990                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 2991   %}
 2992 
 2993   // This encoding class is generated automatically from ad_encode.m4.
 2994   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2995   enc_class aarch64_enc_ldrs(vRegF dst, memory4 mem) %{
 2996     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 2997     loadStore(masm, &MacroAssembler::ldrs, dst_reg, $mem->opcode(),
 2998                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2999   %}
 3000 
 3001   // This encoding class is generated automatically from ad_encode.m4.
 3002   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3003   enc_class aarch64_enc_ldrd(vRegD dst, memory8 mem) %{
 3004     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3005     loadStore(masm, &MacroAssembler::ldrd, dst_reg, $mem->opcode(),
 3006                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 3007   %}
 3008 
 3009   // This encoding class is generated automatically from ad_encode.m4.
 3010   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3011   enc_class aarch64_enc_strb(iRegI src, memory1 mem) %{
 3012     Register src_reg = as_Register($src$$reg);
 3013     loadStore(masm, &MacroAssembler::strb, src_reg, $mem->opcode(),
 3014                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 3015   %}
 3016 
 3017   // This encoding class is generated automatically from ad_encode.m4.
 3018   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3019   enc_class aarch64_enc_strb0(memory1 mem) %{
 3020     loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
 3021                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 3022   %}
 3023 
 3024   // This encoding class is generated automatically from ad_encode.m4.
 3025   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3026   enc_class aarch64_enc_strh(iRegI src, memory2 mem) %{
 3027     Register src_reg = as_Register($src$$reg);
 3028     loadStore(masm, &MacroAssembler::strh, src_reg, $mem->opcode(),
 3029                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 3030   %}
 3031 
 3032   // This encoding class is generated automatically from ad_encode.m4.
 3033   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3034   enc_class aarch64_enc_strh0(memory2 mem) %{
 3035     loadStore(masm, &MacroAssembler::strh, zr, $mem->opcode(),
 3036                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 3037   %}
 3038 
 3039   // This encoding class is generated automatically from ad_encode.m4.
 3040   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3041   enc_class aarch64_enc_strw(iRegI src, memory4 mem) %{
 3042     Register src_reg = as_Register($src$$reg);
 3043     loadStore(masm, &MacroAssembler::strw, src_reg, $mem->opcode(),
 3044                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 3045   %}
 3046 
 3047   // This encoding class is generated automatically from ad_encode.m4.
 3048   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3049   enc_class aarch64_enc_strw0(memory4 mem) %{
 3050     loadStore(masm, &MacroAssembler::strw, zr, $mem->opcode(),
 3051                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 3052   %}
 3053 
 3054   // This encoding class is generated automatically from ad_encode.m4.
 3055   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3056   enc_class aarch64_enc_str(iRegL src, memory8 mem) %{
 3057     Register src_reg = as_Register($src$$reg);
 3058     // we sometimes get asked to store the stack pointer into the
 3059     // current thread -- we cannot do that directly on AArch64
 3060     if (src_reg == r31_sp) {
 3061       assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
 3062       __ mov(rscratch2, sp);
 3063       src_reg = rscratch2;
 3064     }
 3065     loadStore(masm, &MacroAssembler::str, src_reg, $mem->opcode(),
 3066                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 3067   %}
 3068 
 3069   // This encoding class is generated automatically from ad_encode.m4.
 3070   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3071   enc_class aarch64_enc_str0(memory8 mem) %{
 3072     loadStore(masm, &MacroAssembler::str, zr, $mem->opcode(),
 3073                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 3074   %}
 3075 
 3076   // This encoding class is generated automatically from ad_encode.m4.
 3077   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3078   enc_class aarch64_enc_strs(vRegF src, memory4 mem) %{
 3079     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3080     loadStore(masm, &MacroAssembler::strs, src_reg, $mem->opcode(),
 3081                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 3082   %}
 3083 
 3084   // This encoding class is generated automatically from ad_encode.m4.
 3085   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3086   enc_class aarch64_enc_strd(vRegD src, memory8 mem) %{
 3087     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3088     loadStore(masm, &MacroAssembler::strd, src_reg, $mem->opcode(),
 3089                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 3090   %}
 3091 
 3092   // This encoding class is generated automatically from ad_encode.m4.
 3093   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3094   enc_class aarch64_enc_strb0_ordered(memory4 mem) %{
 3095       __ membar(Assembler::StoreStore);
 3096       loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
 3097                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 3098   %}
 3099 
 3100   // END Non-volatile memory access
 3101 
 3102   // Vector loads and stores
 3103   enc_class aarch64_enc_ldrvH(vReg dst, memory mem) %{
 3104     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3105     loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::H,
 3106        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3107   %}
 3108 
 3109   enc_class aarch64_enc_ldrvS(vReg dst, memory mem) %{
 3110     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3111     loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::S,
 3112        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3113   %}
 3114 
 3115   enc_class aarch64_enc_ldrvD(vReg dst, memory mem) %{
 3116     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3117     loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::D,
 3118        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3119   %}
 3120 
 3121   enc_class aarch64_enc_ldrvQ(vReg dst, memory mem) %{
 3122     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3123     loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::Q,
 3124        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3125   %}
 3126 
 3127   enc_class aarch64_enc_strvH(vReg src, memory mem) %{
 3128     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3129     loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::H,
 3130        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3131   %}
 3132 
 3133   enc_class aarch64_enc_strvS(vReg src, memory mem) %{
 3134     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3135     loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::S,
 3136        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3137   %}
 3138 
 3139   enc_class aarch64_enc_strvD(vReg src, memory mem) %{
 3140     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3141     loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::D,
 3142        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3143   %}
 3144 
 3145   enc_class aarch64_enc_strvQ(vReg src, memory mem) %{
 3146     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3147     loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::Q,
 3148        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3149   %}
 3150 
 3151   // volatile loads and stores
 3152 
 3153   enc_class aarch64_enc_stlrb(iRegI src, memory mem) %{
 3154     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3155                  rscratch1, stlrb);
 3156   %}
 3157 
 3158   enc_class aarch64_enc_stlrb0(memory mem) %{
 3159     MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3160                  rscratch1, stlrb);
 3161   %}
 3162 
 3163   enc_class aarch64_enc_stlrh(iRegI src, memory mem) %{
 3164     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3165                  rscratch1, stlrh);
 3166   %}
 3167 
 3168   enc_class aarch64_enc_stlrh0(memory mem) %{
 3169     MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3170                  rscratch1, stlrh);
 3171   %}
 3172 
 3173   enc_class aarch64_enc_stlrw(iRegI src, memory mem) %{
 3174     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3175                  rscratch1, stlrw);
 3176   %}
 3177 
 3178   enc_class aarch64_enc_stlrw0(memory mem) %{
 3179     MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3180                  rscratch1, stlrw);
 3181   %}
 3182 
 3183   enc_class aarch64_enc_ldarsbw(iRegI dst, memory mem) %{
 3184     Register dst_reg = as_Register($dst$$reg);
 3185     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3186              rscratch1, ldarb);
 3187     __ sxtbw(dst_reg, dst_reg);
 3188   %}
 3189 
 3190   enc_class aarch64_enc_ldarsb(iRegL dst, memory mem) %{
 3191     Register dst_reg = as_Register($dst$$reg);
 3192     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3193              rscratch1, ldarb);
 3194     __ sxtb(dst_reg, dst_reg);
 3195   %}
 3196 
 3197   enc_class aarch64_enc_ldarbw(iRegI dst, memory mem) %{
 3198     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3199              rscratch1, ldarb);
 3200   %}
 3201 
 3202   enc_class aarch64_enc_ldarb(iRegL dst, memory mem) %{
 3203     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3204              rscratch1, ldarb);
 3205   %}
 3206 
 3207   enc_class aarch64_enc_ldarshw(iRegI dst, memory mem) %{
 3208     Register dst_reg = as_Register($dst$$reg);
 3209     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3210              rscratch1, ldarh);
 3211     __ sxthw(dst_reg, dst_reg);
 3212   %}
 3213 
 3214   enc_class aarch64_enc_ldarsh(iRegL dst, memory mem) %{
 3215     Register dst_reg = as_Register($dst$$reg);
 3216     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3217              rscratch1, ldarh);
 3218     __ sxth(dst_reg, dst_reg);
 3219   %}
 3220 
 3221   enc_class aarch64_enc_ldarhw(iRegI dst, memory mem) %{
 3222     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3223              rscratch1, ldarh);
 3224   %}
 3225 
 3226   enc_class aarch64_enc_ldarh(iRegL dst, memory mem) %{
 3227     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3228              rscratch1, ldarh);
 3229   %}
 3230 
 3231   enc_class aarch64_enc_ldarw(iRegI dst, memory mem) %{
 3232     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3233              rscratch1, ldarw);
 3234   %}
 3235 
 3236   enc_class aarch64_enc_ldarw(iRegL dst, memory mem) %{
 3237     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3238              rscratch1, ldarw);
 3239   %}
 3240 
 3241   enc_class aarch64_enc_ldar(iRegL dst, memory mem) %{
 3242     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3243              rscratch1, ldar);
 3244   %}
 3245 
 3246   enc_class aarch64_enc_fldars(vRegF dst, memory mem) %{
 3247     MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3248              rscratch1, ldarw);
 3249     __ fmovs(as_FloatRegister($dst$$reg), rscratch1);
 3250   %}
 3251 
 3252   enc_class aarch64_enc_fldard(vRegD dst, memory mem) %{
 3253     MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3254              rscratch1, ldar);
 3255     __ fmovd(as_FloatRegister($dst$$reg), rscratch1);
 3256   %}
 3257 
 3258   enc_class aarch64_enc_stlr(iRegL src, memory mem) %{
 3259     Register src_reg = as_Register($src$$reg);
 3260     // we sometimes get asked to store the stack pointer into the
 3261     // current thread -- we cannot do that directly on AArch64
 3262     if (src_reg == r31_sp) {
 3263       assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
 3264       __ mov(rscratch2, sp);
 3265       src_reg = rscratch2;
 3266     }
 3267     MOV_VOLATILE(src_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3268                  rscratch1, stlr);
 3269   %}
 3270 
 3271   enc_class aarch64_enc_stlr0(memory mem) %{
 3272     MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3273                  rscratch1, stlr);
 3274   %}
 3275 
 3276   enc_class aarch64_enc_fstlrs(vRegF src, memory mem) %{
 3277     {
 3278       FloatRegister src_reg = as_FloatRegister($src$$reg);
 3279       __ fmovs(rscratch2, src_reg);
 3280     }
 3281     MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3282                  rscratch1, stlrw);
 3283   %}
 3284 
 3285   enc_class aarch64_enc_fstlrd(vRegD src, memory mem) %{
 3286     {
 3287       FloatRegister src_reg = as_FloatRegister($src$$reg);
 3288       __ fmovd(rscratch2, src_reg);
 3289     }
 3290     MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3291                  rscratch1, stlr);
 3292   %}
 3293 
 3294   // synchronized read/update encodings
 3295 
 3296   enc_class aarch64_enc_ldaxr(iRegL dst, memory8 mem) %{
 3297     Register dst_reg = as_Register($dst$$reg);
 3298     Register base = as_Register($mem$$base);
 3299     int index = $mem$$index;
 3300     int scale = $mem$$scale;
 3301     int disp = $mem$$disp;
 3302     if (index == -1) {
 3303        if (disp != 0) {
 3304         __ lea(rscratch1, Address(base, disp));
 3305         __ ldaxr(dst_reg, rscratch1);
 3306       } else {
 3307         // TODO
 3308         // should we ever get anything other than this case?
 3309         __ ldaxr(dst_reg, base);
 3310       }
 3311     } else {
 3312       Register index_reg = as_Register(index);
 3313       if (disp == 0) {
 3314         __ lea(rscratch1, Address(base, index_reg, Address::lsl(scale)));
 3315         __ ldaxr(dst_reg, rscratch1);
 3316       } else {
 3317         __ lea(rscratch1, Address(base, disp));
 3318         __ lea(rscratch1, Address(rscratch1, index_reg, Address::lsl(scale)));
 3319         __ ldaxr(dst_reg, rscratch1);
 3320       }
 3321     }
 3322   %}
 3323 
 3324   enc_class aarch64_enc_stlxr(iRegLNoSp src, memory8 mem) %{
 3325     Register src_reg = as_Register($src$$reg);
 3326     Register base = as_Register($mem$$base);
 3327     int index = $mem$$index;
 3328     int scale = $mem$$scale;
 3329     int disp = $mem$$disp;
 3330     if (index == -1) {
 3331        if (disp != 0) {
 3332         __ lea(rscratch2, Address(base, disp));
 3333         __ stlxr(rscratch1, src_reg, rscratch2);
 3334       } else {
 3335         // TODO
 3336         // should we ever get anything other than this case?
 3337         __ stlxr(rscratch1, src_reg, base);
 3338       }
 3339     } else {
 3340       Register index_reg = as_Register(index);
 3341       if (disp == 0) {
 3342         __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
 3343         __ stlxr(rscratch1, src_reg, rscratch2);
 3344       } else {
 3345         __ lea(rscratch2, Address(base, disp));
 3346         __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
 3347         __ stlxr(rscratch1, src_reg, rscratch2);
 3348       }
 3349     }
 3350     __ cmpw(rscratch1, zr);
 3351   %}
 3352 
 3353   enc_class aarch64_enc_cmpxchg(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
 3354     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3355     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3356                Assembler::xword, /*acquire*/ false, /*release*/ true,
 3357                /*weak*/ false, noreg);
 3358   %}
 3359 
 3360   enc_class aarch64_enc_cmpxchgw(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3361     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3362     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3363                Assembler::word, /*acquire*/ false, /*release*/ true,
 3364                /*weak*/ false, noreg);
 3365   %}
 3366 
 3367   enc_class aarch64_enc_cmpxchgs(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3368     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3369     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3370                Assembler::halfword, /*acquire*/ false, /*release*/ true,
 3371                /*weak*/ false, noreg);
 3372   %}
 3373 
 3374   enc_class aarch64_enc_cmpxchgb(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3375     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3376     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3377                Assembler::byte, /*acquire*/ false, /*release*/ true,
 3378                /*weak*/ false, noreg);
 3379   %}
 3380 
 3381 
 3382   // The only difference between aarch64_enc_cmpxchg and
 3383   // aarch64_enc_cmpxchg_acq is that we use load-acquire in the
 3384   // CompareAndSwap sequence to serve as a barrier on acquiring a
 3385   // lock.
 3386   enc_class aarch64_enc_cmpxchg_acq(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
 3387     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3388     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3389                Assembler::xword, /*acquire*/ true, /*release*/ true,
 3390                /*weak*/ false, noreg);
 3391   %}
 3392 
 3393   enc_class aarch64_enc_cmpxchgw_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3394     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3395     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3396                Assembler::word, /*acquire*/ true, /*release*/ true,
 3397                /*weak*/ false, noreg);
 3398   %}
 3399 
 3400   enc_class aarch64_enc_cmpxchgs_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3401     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3402     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3403                Assembler::halfword, /*acquire*/ true, /*release*/ true,
 3404                /*weak*/ false, noreg);
 3405   %}
 3406 
 3407   enc_class aarch64_enc_cmpxchgb_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3408     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3409     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3410                Assembler::byte, /*acquire*/ true, /*release*/ true,
 3411                /*weak*/ false, noreg);
 3412   %}
 3413 
 3414   // auxiliary used for CompareAndSwapX to set result register
 3415   enc_class aarch64_enc_cset_eq(iRegINoSp res) %{
 3416     Register res_reg = as_Register($res$$reg);
 3417     __ cset(res_reg, Assembler::EQ);
 3418   %}
 3419 
 3420   // prefetch encodings
 3421 
 3422   enc_class aarch64_enc_prefetchw(memory mem) %{
 3423     Register base = as_Register($mem$$base);
 3424     int index = $mem$$index;
 3425     int scale = $mem$$scale;
 3426     int disp = $mem$$disp;
 3427     if (index == -1) {
 3428       // Fix up any out-of-range offsets.
 3429       assert_different_registers(rscratch1, base);
 3430       Address addr = Address(base, disp);
 3431       addr = __ legitimize_address(addr, 8, rscratch1);
 3432       __ prfm(addr, PSTL1KEEP);
 3433     } else {
 3434       Register index_reg = as_Register(index);
 3435       if (disp == 0) {
 3436         __ prfm(Address(base, index_reg, Address::lsl(scale)), PSTL1KEEP);
 3437       } else {
 3438         __ lea(rscratch1, Address(base, disp));
 3439 	__ prfm(Address(rscratch1, index_reg, Address::lsl(scale)), PSTL1KEEP);
 3440       }
 3441     }
 3442   %}
 3443 
 3444   // mov encodings
 3445 
 3446   enc_class aarch64_enc_movw_imm(iRegI dst, immI src) %{
 3447     uint32_t con = (uint32_t)$src$$constant;
 3448     Register dst_reg = as_Register($dst$$reg);
 3449     if (con == 0) {
 3450       __ movw(dst_reg, zr);
 3451     } else {
 3452       __ movw(dst_reg, con);
 3453     }
 3454   %}
 3455 
 3456   enc_class aarch64_enc_mov_imm(iRegL dst, immL src) %{
 3457     Register dst_reg = as_Register($dst$$reg);
 3458     uint64_t con = (uint64_t)$src$$constant;
 3459     if (con == 0) {
 3460       __ mov(dst_reg, zr);
 3461     } else {
 3462       __ mov(dst_reg, con);
 3463     }
 3464   %}
 3465 
 3466   enc_class aarch64_enc_mov_p(iRegP dst, immP src) %{
 3467     Register dst_reg = as_Register($dst$$reg);
 3468     address con = (address)$src$$constant;
 3469     if (con == nullptr || con == (address)1) {
 3470       ShouldNotReachHere();
 3471     } else {
 3472       relocInfo::relocType rtype = $src->constant_reloc();
 3473       if (rtype == relocInfo::oop_type) {
 3474         __ movoop(dst_reg, (jobject)con);
 3475       } else if (rtype == relocInfo::metadata_type) {
 3476         __ mov_metadata(dst_reg, (Metadata*)con);
 3477       } else {
 3478         assert(rtype == relocInfo::none, "unexpected reloc type");
 3479         if (! __ is_valid_AArch64_address(con) ||
 3480             con < (address)(uintptr_t)os::vm_page_size()) {
 3481           __ mov(dst_reg, con);
 3482         } else {
 3483           uint64_t offset;
 3484           __ adrp(dst_reg, con, offset);
 3485           __ add(dst_reg, dst_reg, offset);
 3486         }
 3487       }
 3488     }
 3489   %}
 3490 
 3491   enc_class aarch64_enc_mov_p0(iRegP dst, immP0 src) %{
 3492     Register dst_reg = as_Register($dst$$reg);
 3493     __ mov(dst_reg, zr);
 3494   %}
 3495 
 3496   enc_class aarch64_enc_mov_p1(iRegP dst, immP_1 src) %{
 3497     Register dst_reg = as_Register($dst$$reg);
 3498     __ mov(dst_reg, (uint64_t)1);
 3499   %}
 3500 
 3501   enc_class aarch64_enc_mov_n(iRegN dst, immN src) %{
 3502     Register dst_reg = as_Register($dst$$reg);
 3503     address con = (address)$src$$constant;
 3504     if (con == nullptr) {
 3505       ShouldNotReachHere();
 3506     } else {
 3507       relocInfo::relocType rtype = $src->constant_reloc();
 3508       assert(rtype == relocInfo::oop_type, "unexpected reloc type");
 3509       __ set_narrow_oop(dst_reg, (jobject)con);
 3510     }
 3511   %}
 3512 
 3513   enc_class aarch64_enc_mov_n0(iRegN dst, immN0 src) %{
 3514     Register dst_reg = as_Register($dst$$reg);
 3515     __ mov(dst_reg, zr);
 3516   %}
 3517 
 3518   enc_class aarch64_enc_mov_nk(iRegN dst, immNKlass src) %{
 3519     Register dst_reg = as_Register($dst$$reg);
 3520     address con = (address)$src$$constant;
 3521     if (con == nullptr) {
 3522       ShouldNotReachHere();
 3523     } else {
 3524       relocInfo::relocType rtype = $src->constant_reloc();
 3525       assert(rtype == relocInfo::metadata_type, "unexpected reloc type");
 3526       __ set_narrow_klass(dst_reg, (Klass *)con);
 3527     }
 3528   %}
 3529 
 3530   // arithmetic encodings
 3531 
 3532   enc_class aarch64_enc_addsubw_imm(iRegI dst, iRegI src1, immIAddSub src2) %{
 3533     Register dst_reg = as_Register($dst$$reg);
 3534     Register src_reg = as_Register($src1$$reg);
 3535     int32_t con = (int32_t)$src2$$constant;
 3536     // add has primary == 0, subtract has primary == 1
 3537     if ($primary) { con = -con; }
 3538     if (con < 0) {
 3539       __ subw(dst_reg, src_reg, -con);
 3540     } else {
 3541       __ addw(dst_reg, src_reg, con);
 3542     }
 3543   %}
 3544 
 3545   enc_class aarch64_enc_addsub_imm(iRegL dst, iRegL src1, immLAddSub src2) %{
 3546     Register dst_reg = as_Register($dst$$reg);
 3547     Register src_reg = as_Register($src1$$reg);
 3548     int32_t con = (int32_t)$src2$$constant;
 3549     // add has primary == 0, subtract has primary == 1
 3550     if ($primary) { con = -con; }
 3551     if (con < 0) {
 3552       __ sub(dst_reg, src_reg, -con);
 3553     } else {
 3554       __ add(dst_reg, src_reg, con);
 3555     }
 3556   %}
 3557 
 3558   enc_class aarch64_enc_divw(iRegI dst, iRegI src1, iRegI src2) %{
 3559    Register dst_reg = as_Register($dst$$reg);
 3560    Register src1_reg = as_Register($src1$$reg);
 3561    Register src2_reg = as_Register($src2$$reg);
 3562     __ corrected_idivl(dst_reg, src1_reg, src2_reg, false, rscratch1);
 3563   %}
 3564 
 3565   enc_class aarch64_enc_div(iRegI dst, iRegI src1, iRegI src2) %{
 3566    Register dst_reg = as_Register($dst$$reg);
 3567    Register src1_reg = as_Register($src1$$reg);
 3568    Register src2_reg = as_Register($src2$$reg);
 3569     __ corrected_idivq(dst_reg, src1_reg, src2_reg, false, rscratch1);
 3570   %}
 3571 
 3572   enc_class aarch64_enc_modw(iRegI dst, iRegI src1, iRegI src2) %{
 3573    Register dst_reg = as_Register($dst$$reg);
 3574    Register src1_reg = as_Register($src1$$reg);
 3575    Register src2_reg = as_Register($src2$$reg);
 3576     __ corrected_idivl(dst_reg, src1_reg, src2_reg, true, rscratch1);
 3577   %}
 3578 
 3579   enc_class aarch64_enc_mod(iRegI dst, iRegI src1, iRegI src2) %{
 3580    Register dst_reg = as_Register($dst$$reg);
 3581    Register src1_reg = as_Register($src1$$reg);
 3582    Register src2_reg = as_Register($src2$$reg);
 3583     __ corrected_idivq(dst_reg, src1_reg, src2_reg, true, rscratch1);
 3584   %}
 3585 
 3586   // compare instruction encodings
 3587 
 3588   enc_class aarch64_enc_cmpw(iRegI src1, iRegI src2) %{
 3589     Register reg1 = as_Register($src1$$reg);
 3590     Register reg2 = as_Register($src2$$reg);
 3591     __ cmpw(reg1, reg2);
 3592   %}
 3593 
 3594   enc_class aarch64_enc_cmpw_imm_addsub(iRegI src1, immIAddSub src2) %{
 3595     Register reg = as_Register($src1$$reg);
 3596     int32_t val = $src2$$constant;
 3597     if (val >= 0) {
 3598       __ subsw(zr, reg, val);
 3599     } else {
 3600       __ addsw(zr, reg, -val);
 3601     }
 3602   %}
 3603 
 3604   enc_class aarch64_enc_cmpw_imm(iRegI src1, immI src2) %{
 3605     Register reg1 = as_Register($src1$$reg);
 3606     uint32_t val = (uint32_t)$src2$$constant;
 3607     __ movw(rscratch1, val);
 3608     __ cmpw(reg1, rscratch1);
 3609   %}
 3610 
 3611   enc_class aarch64_enc_cmp(iRegL src1, iRegL src2) %{
 3612     Register reg1 = as_Register($src1$$reg);
 3613     Register reg2 = as_Register($src2$$reg);
 3614     __ cmp(reg1, reg2);
 3615   %}
 3616 
 3617   enc_class aarch64_enc_cmp_imm_addsub(iRegL src1, immL12 src2) %{
 3618     Register reg = as_Register($src1$$reg);
 3619     int64_t val = $src2$$constant;
 3620     if (val >= 0) {
 3621       __ subs(zr, reg, val);
 3622     } else if (val != -val) {
 3623       __ adds(zr, reg, -val);
 3624     } else {
 3625     // aargh, Long.MIN_VALUE is a special case
 3626       __ orr(rscratch1, zr, (uint64_t)val);
 3627       __ subs(zr, reg, rscratch1);
 3628     }
 3629   %}
 3630 
 3631   enc_class aarch64_enc_cmp_imm(iRegL src1, immL src2) %{
 3632     Register reg1 = as_Register($src1$$reg);
 3633     uint64_t val = (uint64_t)$src2$$constant;
 3634     __ mov(rscratch1, val);
 3635     __ cmp(reg1, rscratch1);
 3636   %}
 3637 
 3638   enc_class aarch64_enc_cmpp(iRegP src1, iRegP src2) %{
 3639     Register reg1 = as_Register($src1$$reg);
 3640     Register reg2 = as_Register($src2$$reg);
 3641     __ cmp(reg1, reg2);
 3642   %}
 3643 
 3644   enc_class aarch64_enc_cmpn(iRegN src1, iRegN src2) %{
 3645     Register reg1 = as_Register($src1$$reg);
 3646     Register reg2 = as_Register($src2$$reg);
 3647     __ cmpw(reg1, reg2);
 3648   %}
 3649 
 3650   enc_class aarch64_enc_testp(iRegP src) %{
 3651     Register reg = as_Register($src$$reg);
 3652     __ cmp(reg, zr);
 3653   %}
 3654 
 3655   enc_class aarch64_enc_testn(iRegN src) %{
 3656     Register reg = as_Register($src$$reg);
 3657     __ cmpw(reg, zr);
 3658   %}
 3659 
 3660   enc_class aarch64_enc_b(label lbl) %{
 3661     Label *L = $lbl$$label;
 3662     __ b(*L);
 3663   %}
 3664 
 3665   enc_class aarch64_enc_br_con(cmpOp cmp, label lbl) %{
 3666     Label *L = $lbl$$label;
 3667     __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
 3668   %}
 3669 
 3670   enc_class aarch64_enc_br_conU(cmpOpU cmp, label lbl) %{
 3671     Label *L = $lbl$$label;
 3672     __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
 3673   %}
 3674 
 3675   enc_class aarch64_enc_partial_subtype_check(iRegP sub, iRegP super, iRegP temp, iRegP result)
 3676   %{
 3677      Register sub_reg = as_Register($sub$$reg);
 3678      Register super_reg = as_Register($super$$reg);
 3679      Register temp_reg = as_Register($temp$$reg);
 3680      Register result_reg = as_Register($result$$reg);
 3681 
 3682      Label miss;
 3683      __ check_klass_subtype_slow_path(sub_reg, super_reg, temp_reg, result_reg,
 3684                                      nullptr, &miss,
 3685                                      /*set_cond_codes:*/ true);
 3686      if ($primary) {
 3687        __ mov(result_reg, zr);
 3688      }
 3689      __ bind(miss);
 3690   %}
 3691 
 3692   enc_class aarch64_enc_java_static_call(method meth) %{
 3693     address addr = (address)$meth$$method;
 3694     address call;
 3695     if (!_method) {
 3696       // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
 3697       call = __ trampoline_call(Address(addr, relocInfo::runtime_call_type));
 3698       if (call == nullptr) {
 3699         ciEnv::current()->record_failure("CodeCache is full");
 3700         return;
 3701       }
 3702     } else if (_method->intrinsic_id() == vmIntrinsicID::_ensureMaterializedForStackWalk) {
 3703       // The NOP here is purely to ensure that eliding a call to
 3704       // JVM_EnsureMaterializedForStackWalk doesn't change the code size.
 3705       __ nop();
 3706       __ block_comment("call JVM_EnsureMaterializedForStackWalk (elided)");
 3707     } else {
 3708       int method_index = resolved_method_index(masm);
 3709       RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
 3710                                                   : static_call_Relocation::spec(method_index);
 3711       call = __ trampoline_call(Address(addr, rspec));
 3712       if (call == nullptr) {
 3713         ciEnv::current()->record_failure("CodeCache is full");
 3714         return;
 3715       }
 3716       if (CodeBuffer::supports_shared_stubs() && _method->can_be_statically_bound()) {
 3717         // Calls of the same statically bound method can share
 3718         // a stub to the interpreter.
 3719         __ code()->shared_stub_to_interp_for(_method, call - __ begin());
 3720       } else {
 3721         // Emit stub for static call
 3722         address stub = CompiledDirectCall::emit_to_interp_stub(masm, call);
 3723         if (stub == nullptr) {
 3724           ciEnv::current()->record_failure("CodeCache is full");
 3725           return;
 3726         }
 3727       }
 3728     }
 3729 
 3730     __ post_call_nop();
 3731 
 3732     // Only non uncommon_trap calls need to reinitialize ptrue.
 3733     if (Compile::current()->max_vector_size() > 0 && uncommon_trap_request() == 0) {
 3734       __ reinitialize_ptrue();
 3735     }
 3736   %}
 3737 
 3738   enc_class aarch64_enc_java_dynamic_call(method meth) %{
 3739     int method_index = resolved_method_index(masm);
 3740     address call = __ ic_call((address)$meth$$method, method_index);
 3741     if (call == nullptr) {
 3742       ciEnv::current()->record_failure("CodeCache is full");
 3743       return;
 3744     }
 3745     __ post_call_nop();
 3746     if (Compile::current()->max_vector_size() > 0) {
 3747       __ reinitialize_ptrue();
 3748     }
 3749   %}
 3750 
 3751   enc_class aarch64_enc_call_epilog() %{
 3752     if (VerifyStackAtCalls) {
 3753       // Check that stack depth is unchanged: find majik cookie on stack
 3754       __ call_Unimplemented();
 3755     }
 3756     if (tf()->returns_inline_type_as_fields() && !_method->is_method_handle_intrinsic() && _method->return_type()->is_loaded()) {
 3757       // The last return value is not set by the callee but used to pass the null marker to compiled code.
 3758       // Search for the corresponding projection, get the register and emit code that initialized it.
 3759       uint con = (tf()->range_cc()->cnt() - 1);
 3760       for (DUIterator_Fast imax, i = fast_outs(imax); i < imax; i++) {
 3761         ProjNode* proj = fast_out(i)->as_Proj();
 3762         if (proj->_con == con) {
 3763           // Set null marker if r0 is non-null (a non-null value is returned buffered or scalarized)
 3764           OptoReg::Name optoReg = ra_->get_reg_first(proj);
 3765           VMReg reg = OptoReg::as_VMReg(optoReg, ra_->_framesize, OptoReg::reg2stack(ra_->_matcher._new_SP));
 3766           Register toReg = reg->is_reg() ? reg->as_Register() : rscratch1;
 3767           __ cmp(r0, zr);
 3768           __ cset(toReg, Assembler::NE);
 3769           if (reg->is_stack()) {
 3770             int st_off = reg->reg2stack() * VMRegImpl::stack_slot_size;
 3771             __ str(toReg, Address(sp, st_off));
 3772           }
 3773           break;
 3774         }
 3775       }
 3776       if (return_value_is_used()) {
 3777         // An inline type is returned as fields in multiple registers.
 3778         // R0 either contains an oop if the inline type is buffered or a pointer
 3779         // to the corresponding InlineKlass with the lowest bit set to 1. Zero r0
 3780         // if the lowest bit is set to allow C2 to use the oop after null checking.
 3781         // r0 &= (r0 & 1) - 1
 3782         __ andr(rscratch1, r0, 0x1);
 3783         __ sub(rscratch1, rscratch1, 0x1);
 3784         __ andr(r0, r0, rscratch1);
 3785       }
 3786     }
 3787   %}
 3788 
 3789   enc_class aarch64_enc_java_to_runtime(method meth) %{
 3790     // some calls to generated routines (arraycopy code) are scheduled
 3791     // by C2 as runtime calls. if so we can call them using a br (they
 3792     // will be in a reachable segment) otherwise we have to use a blr
 3793     // which loads the absolute address into a register.
 3794     address entry = (address)$meth$$method;
 3795     CodeBlob *cb = CodeCache::find_blob(entry);
 3796     if (cb) {
 3797       address call = __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
 3798       if (call == nullptr) {
 3799         ciEnv::current()->record_failure("CodeCache is full");
 3800         return;
 3801       }
 3802       __ post_call_nop();
 3803     } else {
 3804       Label retaddr;
 3805       // Make the anchor frame walkable
 3806       __ adr(rscratch2, retaddr);
 3807       __ str(rscratch2, Address(rthread, JavaThread::last_Java_pc_offset()));
 3808       __ lea(rscratch1, RuntimeAddress(entry));
 3809       __ blr(rscratch1);
 3810       __ bind(retaddr);
 3811       __ post_call_nop();
 3812     }
 3813     if (Compile::current()->max_vector_size() > 0) {
 3814       __ reinitialize_ptrue();
 3815     }
 3816   %}
 3817 
 3818   enc_class aarch64_enc_rethrow() %{
 3819     __ far_jump(RuntimeAddress(OptoRuntime::rethrow_stub()));
 3820   %}
 3821 
 3822   enc_class aarch64_enc_ret() %{
 3823 #ifdef ASSERT
 3824     if (Compile::current()->max_vector_size() > 0) {
 3825       __ verify_ptrue();
 3826     }
 3827 #endif
 3828     __ ret(lr);
 3829   %}
 3830 
 3831   enc_class aarch64_enc_tail_call(iRegP jump_target) %{
 3832     Register target_reg = as_Register($jump_target$$reg);
 3833     __ br(target_reg);
 3834   %}
 3835 
 3836   enc_class aarch64_enc_tail_jmp(iRegP jump_target) %{
 3837     Register target_reg = as_Register($jump_target$$reg);
 3838     // exception oop should be in r0
 3839     // ret addr has been popped into lr
 3840     // callee expects it in r3
 3841     __ mov(r3, lr);
 3842     __ br(target_reg);
 3843   %}
 3844 
 3845 %}
 3846 
 3847 //----------FRAME--------------------------------------------------------------
 3848 // Definition of frame structure and management information.
 3849 //
 3850 //  S T A C K   L A Y O U T    Allocators stack-slot number
 3851 //                             |   (to get allocators register number
 3852 //  G  Owned by    |        |  v    add OptoReg::stack0())
 3853 //  r   CALLER     |        |
 3854 //  o     |        +--------+      pad to even-align allocators stack-slot
 3855 //  w     V        |  pad0  |        numbers; owned by CALLER
 3856 //  t   -----------+--------+----> Matcher::_in_arg_limit, unaligned
 3857 //  h     ^        |   in   |  5
 3858 //        |        |  args  |  4   Holes in incoming args owned by SELF
 3859 //  |     |        |        |  3
 3860 //  |     |        +--------+
 3861 //  V     |        | old out|      Empty on Intel, window on Sparc
 3862 //        |    old |preserve|      Must be even aligned.
 3863 //        |     SP-+--------+----> Matcher::_old_SP, even aligned
 3864 //        |        |   in   |  3   area for Intel ret address
 3865 //     Owned by    |preserve|      Empty on Sparc.
 3866 //       SELF      +--------+
 3867 //        |        |  pad2  |  2   pad to align old SP
 3868 //        |        +--------+  1
 3869 //        |        | locks  |  0
 3870 //        |        +--------+----> OptoReg::stack0(), even aligned
 3871 //        |        |  pad1  | 11   pad to align new SP
 3872 //        |        +--------+
 3873 //        |        |        | 10
 3874 //        |        | spills |  9   spills
 3875 //        V        |        |  8   (pad0 slot for callee)
 3876 //      -----------+--------+----> Matcher::_out_arg_limit, unaligned
 3877 //        ^        |  out   |  7
 3878 //        |        |  args  |  6   Holes in outgoing args owned by CALLEE
 3879 //     Owned by    +--------+
 3880 //      CALLEE     | new out|  6   Empty on Intel, window on Sparc
 3881 //        |    new |preserve|      Must be even-aligned.
 3882 //        |     SP-+--------+----> Matcher::_new_SP, even aligned
 3883 //        |        |        |
 3884 //
 3885 // Note 1: Only region 8-11 is determined by the allocator.  Region 0-5 is
 3886 //         known from SELF's arguments and the Java calling convention.
 3887 //         Region 6-7 is determined per call site.
 3888 // Note 2: If the calling convention leaves holes in the incoming argument
 3889 //         area, those holes are owned by SELF.  Holes in the outgoing area
 3890 //         are owned by the CALLEE.  Holes should not be necessary in the
 3891 //         incoming area, as the Java calling convention is completely under
 3892 //         the control of the AD file.  Doubles can be sorted and packed to
 3893 //         avoid holes.  Holes in the outgoing arguments may be necessary for
 3894 //         varargs C calling conventions.
 3895 // Note 3: Region 0-3 is even aligned, with pad2 as needed.  Region 3-5 is
 3896 //         even aligned with pad0 as needed.
 3897 //         Region 6 is even aligned.  Region 6-7 is NOT even aligned;
 3898 //           (the latter is true on Intel but is it false on AArch64?)
 3899 //         region 6-11 is even aligned; it may be padded out more so that
 3900 //         the region from SP to FP meets the minimum stack alignment.
 3901 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
 3902 //         alignment.  Region 11, pad1, may be dynamically extended so that
 3903 //         SP meets the minimum alignment.
 3904 
 3905 frame %{
 3906   // These three registers define part of the calling convention
 3907   // between compiled code and the interpreter.
 3908 
 3909   // Inline Cache Register or Method for I2C.
 3910   inline_cache_reg(R12);
 3911 
 3912   // Number of stack slots consumed by locking an object
 3913   sync_stack_slots(2);
 3914 
 3915   // Compiled code's Frame Pointer
 3916   frame_pointer(R31);
 3917 
 3918   // Interpreter stores its frame pointer in a register which is
 3919   // stored to the stack by I2CAdaptors.
 3920   // I2CAdaptors convert from interpreted java to compiled java.
 3921   interpreter_frame_pointer(R29);
 3922 
 3923   // Stack alignment requirement
 3924   stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
 3925 
 3926   // Number of outgoing stack slots killed above the out_preserve_stack_slots
 3927   // for calls to C.  Supports the var-args backing area for register parms.
 3928   varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
 3929 
 3930   // The after-PROLOG location of the return address.  Location of
 3931   // return address specifies a type (REG or STACK) and a number
 3932   // representing the register number (i.e. - use a register name) or
 3933   // stack slot.
 3934   // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
 3935   // Otherwise, it is above the locks and verification slot and alignment word
 3936   // TODO this may well be correct but need to check why that - 2 is there
 3937   // ppc port uses 0 but we definitely need to allow for fixed_slots
 3938   // which folds in the space used for monitors
 3939   return_addr(STACK - 2 +
 3940               align_up((Compile::current()->in_preserve_stack_slots() +
 3941                         Compile::current()->fixed_slots()),
 3942                        stack_alignment_in_slots()));
 3943 
 3944   // Location of compiled Java return values.  Same as C for now.
 3945   return_value
 3946   %{
 3947     // TODO do we allow ideal_reg == Op_RegN???
 3948     assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
 3949            "only return normal values");
 3950 
 3951     static const int lo[Op_RegL + 1] = { // enum name
 3952       0,                                 // Op_Node
 3953       0,                                 // Op_Set
 3954       R0_num,                            // Op_RegN
 3955       R0_num,                            // Op_RegI
 3956       R0_num,                            // Op_RegP
 3957       V0_num,                            // Op_RegF
 3958       V0_num,                            // Op_RegD
 3959       R0_num                             // Op_RegL
 3960     };
 3961 
 3962     static const int hi[Op_RegL + 1] = { // enum name
 3963       0,                                 // Op_Node
 3964       0,                                 // Op_Set
 3965       OptoReg::Bad,                      // Op_RegN
 3966       OptoReg::Bad,                      // Op_RegI
 3967       R0_H_num,                          // Op_RegP
 3968       OptoReg::Bad,                      // Op_RegF
 3969       V0_H_num,                          // Op_RegD
 3970       R0_H_num                           // Op_RegL
 3971     };
 3972 
 3973     return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
 3974   %}
 3975 %}
 3976 
 3977 //----------ATTRIBUTES---------------------------------------------------------
 3978 //----------Operand Attributes-------------------------------------------------
 3979 op_attrib op_cost(1);        // Required cost attribute
 3980 
 3981 //----------Instruction Attributes---------------------------------------------
 3982 ins_attrib ins_cost(INSN_COST); // Required cost attribute
 3983 ins_attrib ins_size(32);        // Required size attribute (in bits)
 3984 ins_attrib ins_short_branch(0); // Required flag: is this instruction
 3985                                 // a non-matching short branch variant
 3986                                 // of some long branch?
 3987 ins_attrib ins_alignment(4);    // Required alignment attribute (must
 3988                                 // be a power of 2) specifies the
 3989                                 // alignment that some part of the
 3990                                 // instruction (not necessarily the
 3991                                 // start) requires.  If > 1, a
 3992                                 // compute_padding() function must be
 3993                                 // provided for the instruction
 3994 
 3995 // Whether this node is expanded during code emission into a sequence of
 3996 // instructions and the first instruction can perform an implicit null check.
 3997 ins_attrib ins_is_late_expanded_null_check_candidate(false);
 3998 
 3999 //----------OPERANDS-----------------------------------------------------------
 4000 // Operand definitions must precede instruction definitions for correct parsing
 4001 // in the ADLC because operands constitute user defined types which are used in
 4002 // instruction definitions.
 4003 
 4004 //----------Simple Operands----------------------------------------------------
 4005 
 4006 // Integer operands 32 bit
 4007 // 32 bit immediate
 4008 operand immI()
 4009 %{
 4010   match(ConI);
 4011 
 4012   op_cost(0);
 4013   format %{ %}
 4014   interface(CONST_INTER);
 4015 %}
 4016 
 4017 // 32 bit zero
 4018 operand immI0()
 4019 %{
 4020   predicate(n->get_int() == 0);
 4021   match(ConI);
 4022 
 4023   op_cost(0);
 4024   format %{ %}
 4025   interface(CONST_INTER);
 4026 %}
 4027 
 4028 // 32 bit unit increment
 4029 operand immI_1()
 4030 %{
 4031   predicate(n->get_int() == 1);
 4032   match(ConI);
 4033 
 4034   op_cost(0);
 4035   format %{ %}
 4036   interface(CONST_INTER);
 4037 %}
 4038 
 4039 // 32 bit unit decrement
 4040 operand immI_M1()
 4041 %{
 4042   predicate(n->get_int() == -1);
 4043   match(ConI);
 4044 
 4045   op_cost(0);
 4046   format %{ %}
 4047   interface(CONST_INTER);
 4048 %}
 4049 
 4050 // Shift values for add/sub extension shift
 4051 operand immIExt()
 4052 %{
 4053   predicate(0 <= n->get_int() && (n->get_int() <= 4));
 4054   match(ConI);
 4055 
 4056   op_cost(0);
 4057   format %{ %}
 4058   interface(CONST_INTER);
 4059 %}
 4060 
 4061 operand immI_gt_1()
 4062 %{
 4063   predicate(n->get_int() > 1);
 4064   match(ConI);
 4065 
 4066   op_cost(0);
 4067   format %{ %}
 4068   interface(CONST_INTER);
 4069 %}
 4070 
 4071 operand immI_le_4()
 4072 %{
 4073   predicate(n->get_int() <= 4);
 4074   match(ConI);
 4075 
 4076   op_cost(0);
 4077   format %{ %}
 4078   interface(CONST_INTER);
 4079 %}
 4080 
 4081 operand immI_16()
 4082 %{
 4083   predicate(n->get_int() == 16);
 4084   match(ConI);
 4085 
 4086   op_cost(0);
 4087   format %{ %}
 4088   interface(CONST_INTER);
 4089 %}
 4090 
 4091 operand immI_24()
 4092 %{
 4093   predicate(n->get_int() == 24);
 4094   match(ConI);
 4095 
 4096   op_cost(0);
 4097   format %{ %}
 4098   interface(CONST_INTER);
 4099 %}
 4100 
 4101 operand immI_32()
 4102 %{
 4103   predicate(n->get_int() == 32);
 4104   match(ConI);
 4105 
 4106   op_cost(0);
 4107   format %{ %}
 4108   interface(CONST_INTER);
 4109 %}
 4110 
 4111 operand immI_48()
 4112 %{
 4113   predicate(n->get_int() == 48);
 4114   match(ConI);
 4115 
 4116   op_cost(0);
 4117   format %{ %}
 4118   interface(CONST_INTER);
 4119 %}
 4120 
 4121 operand immI_56()
 4122 %{
 4123   predicate(n->get_int() == 56);
 4124   match(ConI);
 4125 
 4126   op_cost(0);
 4127   format %{ %}
 4128   interface(CONST_INTER);
 4129 %}
 4130 
 4131 operand immI_255()
 4132 %{
 4133   predicate(n->get_int() == 255);
 4134   match(ConI);
 4135 
 4136   op_cost(0);
 4137   format %{ %}
 4138   interface(CONST_INTER);
 4139 %}
 4140 
 4141 operand immI_65535()
 4142 %{
 4143   predicate(n->get_int() == 65535);
 4144   match(ConI);
 4145 
 4146   op_cost(0);
 4147   format %{ %}
 4148   interface(CONST_INTER);
 4149 %}
 4150 
 4151 operand immI_positive()
 4152 %{
 4153   predicate(n->get_int() > 0);
 4154   match(ConI);
 4155 
 4156   op_cost(0);
 4157   format %{ %}
 4158   interface(CONST_INTER);
 4159 %}
 4160 
 4161 // BoolTest condition for signed compare
 4162 operand immI_cmp_cond()
 4163 %{
 4164   predicate(!Matcher::is_unsigned_booltest_pred(n->get_int()));
 4165   match(ConI);
 4166 
 4167   op_cost(0);
 4168   format %{ %}
 4169   interface(CONST_INTER);
 4170 %}
 4171 
 4172 // BoolTest condition for unsigned compare
 4173 operand immI_cmpU_cond()
 4174 %{
 4175   predicate(Matcher::is_unsigned_booltest_pred(n->get_int()));
 4176   match(ConI);
 4177 
 4178   op_cost(0);
 4179   format %{ %}
 4180   interface(CONST_INTER);
 4181 %}
 4182 
 4183 operand immL_255()
 4184 %{
 4185   predicate(n->get_long() == 255L);
 4186   match(ConL);
 4187 
 4188   op_cost(0);
 4189   format %{ %}
 4190   interface(CONST_INTER);
 4191 %}
 4192 
 4193 operand immL_65535()
 4194 %{
 4195   predicate(n->get_long() == 65535L);
 4196   match(ConL);
 4197 
 4198   op_cost(0);
 4199   format %{ %}
 4200   interface(CONST_INTER);
 4201 %}
 4202 
 4203 operand immL_4294967295()
 4204 %{
 4205   predicate(n->get_long() == 4294967295L);
 4206   match(ConL);
 4207 
 4208   op_cost(0);
 4209   format %{ %}
 4210   interface(CONST_INTER);
 4211 %}
 4212 
 4213 operand immL_bitmask()
 4214 %{
 4215   predicate((n->get_long() != 0)
 4216             && ((n->get_long() & 0xc000000000000000l) == 0)
 4217             && is_power_of_2(n->get_long() + 1));
 4218   match(ConL);
 4219 
 4220   op_cost(0);
 4221   format %{ %}
 4222   interface(CONST_INTER);
 4223 %}
 4224 
 4225 operand immI_bitmask()
 4226 %{
 4227   predicate((n->get_int() != 0)
 4228             && ((n->get_int() & 0xc0000000) == 0)
 4229             && is_power_of_2(n->get_int() + 1));
 4230   match(ConI);
 4231 
 4232   op_cost(0);
 4233   format %{ %}
 4234   interface(CONST_INTER);
 4235 %}
 4236 
 4237 operand immL_positive_bitmaskI()
 4238 %{
 4239   predicate((n->get_long() != 0)
 4240             && ((julong)n->get_long() < 0x80000000ULL)
 4241             && is_power_of_2(n->get_long() + 1));
 4242   match(ConL);
 4243 
 4244   op_cost(0);
 4245   format %{ %}
 4246   interface(CONST_INTER);
 4247 %}
 4248 
 4249 // Scale values for scaled offset addressing modes (up to long but not quad)
 4250 operand immIScale()
 4251 %{
 4252   predicate(0 <= n->get_int() && (n->get_int() <= 3));
 4253   match(ConI);
 4254 
 4255   op_cost(0);
 4256   format %{ %}
 4257   interface(CONST_INTER);
 4258 %}
 4259 
 4260 // 5 bit signed integer
 4261 operand immI5()
 4262 %{
 4263   predicate(Assembler::is_simm(n->get_int(), 5));
 4264   match(ConI);
 4265 
 4266   op_cost(0);
 4267   format %{ %}
 4268   interface(CONST_INTER);
 4269 %}
 4270 
 4271 // 7 bit unsigned integer
 4272 operand immIU7()
 4273 %{
 4274   predicate(Assembler::is_uimm(n->get_int(), 7));
 4275   match(ConI);
 4276 
 4277   op_cost(0);
 4278   format %{ %}
 4279   interface(CONST_INTER);
 4280 %}
 4281 
 4282 // Offset for scaled or unscaled immediate loads and stores
 4283 operand immIOffset()
 4284 %{
 4285   predicate(Address::offset_ok_for_immed(n->get_int(), 0));
 4286   match(ConI);
 4287 
 4288   op_cost(0);
 4289   format %{ %}
 4290   interface(CONST_INTER);
 4291 %}
 4292 
 4293 operand immIOffset1()
 4294 %{
 4295   predicate(Address::offset_ok_for_immed(n->get_int(), 0));
 4296   match(ConI);
 4297 
 4298   op_cost(0);
 4299   format %{ %}
 4300   interface(CONST_INTER);
 4301 %}
 4302 
 4303 operand immIOffset2()
 4304 %{
 4305   predicate(Address::offset_ok_for_immed(n->get_int(), 1));
 4306   match(ConI);
 4307 
 4308   op_cost(0);
 4309   format %{ %}
 4310   interface(CONST_INTER);
 4311 %}
 4312 
 4313 operand immIOffset4()
 4314 %{
 4315   predicate(Address::offset_ok_for_immed(n->get_int(), 2));
 4316   match(ConI);
 4317 
 4318   op_cost(0);
 4319   format %{ %}
 4320   interface(CONST_INTER);
 4321 %}
 4322 
 4323 operand immIOffset8()
 4324 %{
 4325   predicate(Address::offset_ok_for_immed(n->get_int(), 3));
 4326   match(ConI);
 4327 
 4328   op_cost(0);
 4329   format %{ %}
 4330   interface(CONST_INTER);
 4331 %}
 4332 
 4333 operand immIOffset16()
 4334 %{
 4335   predicate(Address::offset_ok_for_immed(n->get_int(), 4));
 4336   match(ConI);
 4337 
 4338   op_cost(0);
 4339   format %{ %}
 4340   interface(CONST_INTER);
 4341 %}
 4342 
 4343 operand immLOffset()
 4344 %{
 4345   predicate(n->get_long() >= -256 && n->get_long() <= 65520);
 4346   match(ConL);
 4347 
 4348   op_cost(0);
 4349   format %{ %}
 4350   interface(CONST_INTER);
 4351 %}
 4352 
 4353 operand immLoffset1()
 4354 %{
 4355   predicate(Address::offset_ok_for_immed(n->get_long(), 0));
 4356   match(ConL);
 4357 
 4358   op_cost(0);
 4359   format %{ %}
 4360   interface(CONST_INTER);
 4361 %}
 4362 
 4363 operand immLoffset2()
 4364 %{
 4365   predicate(Address::offset_ok_for_immed(n->get_long(), 1));
 4366   match(ConL);
 4367 
 4368   op_cost(0);
 4369   format %{ %}
 4370   interface(CONST_INTER);
 4371 %}
 4372 
 4373 operand immLoffset4()
 4374 %{
 4375   predicate(Address::offset_ok_for_immed(n->get_long(), 2));
 4376   match(ConL);
 4377 
 4378   op_cost(0);
 4379   format %{ %}
 4380   interface(CONST_INTER);
 4381 %}
 4382 
 4383 operand immLoffset8()
 4384 %{
 4385   predicate(Address::offset_ok_for_immed(n->get_long(), 3));
 4386   match(ConL);
 4387 
 4388   op_cost(0);
 4389   format %{ %}
 4390   interface(CONST_INTER);
 4391 %}
 4392 
 4393 operand immLoffset16()
 4394 %{
 4395   predicate(Address::offset_ok_for_immed(n->get_long(), 4));
 4396   match(ConL);
 4397 
 4398   op_cost(0);
 4399   format %{ %}
 4400   interface(CONST_INTER);
 4401 %}
 4402 
 4403 // 5 bit signed long integer
 4404 operand immL5()
 4405 %{
 4406   predicate(Assembler::is_simm(n->get_long(), 5));
 4407   match(ConL);
 4408 
 4409   op_cost(0);
 4410   format %{ %}
 4411   interface(CONST_INTER);
 4412 %}
 4413 
 4414 // 7 bit unsigned long integer
 4415 operand immLU7()
 4416 %{
 4417   predicate(Assembler::is_uimm(n->get_long(), 7));
 4418   match(ConL);
 4419 
 4420   op_cost(0);
 4421   format %{ %}
 4422   interface(CONST_INTER);
 4423 %}
 4424 
 4425 // 8 bit signed value.
 4426 operand immI8()
 4427 %{
 4428   predicate(n->get_int() <= 127 && n->get_int() >= -128);
 4429   match(ConI);
 4430 
 4431   op_cost(0);
 4432   format %{ %}
 4433   interface(CONST_INTER);
 4434 %}
 4435 
 4436 // 8 bit signed value (simm8), or #simm8 LSL 8.
 4437 operand immI8_shift8()
 4438 %{
 4439   predicate((n->get_int() <= 127 && n->get_int() >= -128) ||
 4440             (n->get_int() <= 32512 && n->get_int() >= -32768 && (n->get_int() & 0xff) == 0));
 4441   match(ConI);
 4442 
 4443   op_cost(0);
 4444   format %{ %}
 4445   interface(CONST_INTER);
 4446 %}
 4447 
 4448 // 8 bit signed value (simm8), or #simm8 LSL 8.
 4449 operand immL8_shift8()
 4450 %{
 4451   predicate((n->get_long() <= 127 && n->get_long() >= -128) ||
 4452             (n->get_long() <= 32512 && n->get_long() >= -32768 && (n->get_long() & 0xff) == 0));
 4453   match(ConL);
 4454 
 4455   op_cost(0);
 4456   format %{ %}
 4457   interface(CONST_INTER);
 4458 %}
 4459 
 4460 // 8 bit integer valid for vector add sub immediate
 4461 operand immBAddSubV()
 4462 %{
 4463   predicate(n->get_int() <= 255 && n->get_int() >= -255);
 4464   match(ConI);
 4465 
 4466   op_cost(0);
 4467   format %{ %}
 4468   interface(CONST_INTER);
 4469 %}
 4470 
 4471 // 32 bit integer valid for add sub immediate
 4472 operand immIAddSub()
 4473 %{
 4474   predicate(Assembler::operand_valid_for_add_sub_immediate((int64_t)n->get_int()));
 4475   match(ConI);
 4476   op_cost(0);
 4477   format %{ %}
 4478   interface(CONST_INTER);
 4479 %}
 4480 
 4481 // 32 bit integer valid for vector add sub immediate
 4482 operand immIAddSubV()
 4483 %{
 4484   predicate(Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)n->get_int()));
 4485   match(ConI);
 4486 
 4487   op_cost(0);
 4488   format %{ %}
 4489   interface(CONST_INTER);
 4490 %}
 4491 
 4492 // 32 bit unsigned integer valid for logical immediate
 4493 
 4494 operand immBLog()
 4495 %{
 4496   predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerByte, (uint64_t)n->get_int()));
 4497   match(ConI);
 4498 
 4499   op_cost(0);
 4500   format %{ %}
 4501   interface(CONST_INTER);
 4502 %}
 4503 
 4504 operand immSLog()
 4505 %{
 4506   predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerShort, (uint64_t)n->get_int()));
 4507   match(ConI);
 4508 
 4509   op_cost(0);
 4510   format %{ %}
 4511   interface(CONST_INTER);
 4512 %}
 4513 
 4514 operand immILog()
 4515 %{
 4516   predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/true, (uint64_t)n->get_int()));
 4517   match(ConI);
 4518 
 4519   op_cost(0);
 4520   format %{ %}
 4521   interface(CONST_INTER);
 4522 %}
 4523 
 4524 // Integer operands 64 bit
 4525 // 64 bit immediate
 4526 operand immL()
 4527 %{
 4528   match(ConL);
 4529 
 4530   op_cost(0);
 4531   format %{ %}
 4532   interface(CONST_INTER);
 4533 %}
 4534 
 4535 // 64 bit zero
 4536 operand immL0()
 4537 %{
 4538   predicate(n->get_long() == 0);
 4539   match(ConL);
 4540 
 4541   op_cost(0);
 4542   format %{ %}
 4543   interface(CONST_INTER);
 4544 %}
 4545 
 4546 // 64 bit unit decrement
 4547 operand immL_M1()
 4548 %{
 4549   predicate(n->get_long() == -1);
 4550   match(ConL);
 4551 
 4552   op_cost(0);
 4553   format %{ %}
 4554   interface(CONST_INTER);
 4555 %}
 4556 
 4557 // 64 bit integer valid for add sub immediate
 4558 operand immLAddSub()
 4559 %{
 4560   predicate(Assembler::operand_valid_for_add_sub_immediate(n->get_long()));
 4561   match(ConL);
 4562   op_cost(0);
 4563   format %{ %}
 4564   interface(CONST_INTER);
 4565 %}
 4566 
 4567 // 64 bit integer valid for addv subv immediate
 4568 operand immLAddSubV()
 4569 %{
 4570   predicate(Assembler::operand_valid_for_sve_add_sub_immediate(n->get_long()));
 4571   match(ConL);
 4572 
 4573   op_cost(0);
 4574   format %{ %}
 4575   interface(CONST_INTER);
 4576 %}
 4577 
 4578 // 64 bit integer valid for logical immediate
 4579 operand immLLog()
 4580 %{
 4581   predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/false, (uint64_t)n->get_long()));
 4582   match(ConL);
 4583   op_cost(0);
 4584   format %{ %}
 4585   interface(CONST_INTER);
 4586 %}
 4587 
 4588 // Long Immediate: low 32-bit mask
 4589 operand immL_32bits()
 4590 %{
 4591   predicate(n->get_long() == 0xFFFFFFFFL);
 4592   match(ConL);
 4593   op_cost(0);
 4594   format %{ %}
 4595   interface(CONST_INTER);
 4596 %}
 4597 
 4598 // Pointer operands
 4599 // Pointer Immediate
 4600 operand immP()
 4601 %{
 4602   match(ConP);
 4603 
 4604   op_cost(0);
 4605   format %{ %}
 4606   interface(CONST_INTER);
 4607 %}
 4608 
 4609 // nullptr Pointer Immediate
 4610 operand immP0()
 4611 %{
 4612   predicate(n->get_ptr() == 0);
 4613   match(ConP);
 4614 
 4615   op_cost(0);
 4616   format %{ %}
 4617   interface(CONST_INTER);
 4618 %}
 4619 
 4620 // Pointer Immediate One
 4621 // this is used in object initialization (initial object header)
 4622 operand immP_1()
 4623 %{
 4624   predicate(n->get_ptr() == 1);
 4625   match(ConP);
 4626 
 4627   op_cost(0);
 4628   format %{ %}
 4629   interface(CONST_INTER);
 4630 %}
 4631 
 4632 // Float and Double operands
 4633 // Double Immediate
 4634 operand immD()
 4635 %{
 4636   match(ConD);
 4637   op_cost(0);
 4638   format %{ %}
 4639   interface(CONST_INTER);
 4640 %}
 4641 
 4642 // Double Immediate: +0.0d
 4643 operand immD0()
 4644 %{
 4645   predicate(jlong_cast(n->getd()) == 0);
 4646   match(ConD);
 4647 
 4648   op_cost(0);
 4649   format %{ %}
 4650   interface(CONST_INTER);
 4651 %}
 4652 
 4653 // constant 'double +0.0'.
 4654 operand immDPacked()
 4655 %{
 4656   predicate(Assembler::operand_valid_for_float_immediate(n->getd()));
 4657   match(ConD);
 4658   op_cost(0);
 4659   format %{ %}
 4660   interface(CONST_INTER);
 4661 %}
 4662 
 4663 // Float Immediate
 4664 operand immF()
 4665 %{
 4666   match(ConF);
 4667   op_cost(0);
 4668   format %{ %}
 4669   interface(CONST_INTER);
 4670 %}
 4671 
 4672 // Float Immediate: +0.0f.
 4673 operand immF0()
 4674 %{
 4675   predicate(jint_cast(n->getf()) == 0);
 4676   match(ConF);
 4677 
 4678   op_cost(0);
 4679   format %{ %}
 4680   interface(CONST_INTER);
 4681 %}
 4682 
 4683 // Half Float (FP16) Immediate
 4684 operand immH()
 4685 %{
 4686   match(ConH);
 4687   op_cost(0);
 4688   format %{ %}
 4689   interface(CONST_INTER);
 4690 %}
 4691 
 4692 //
 4693 operand immFPacked()
 4694 %{
 4695   predicate(Assembler::operand_valid_for_float_immediate((double)n->getf()));
 4696   match(ConF);
 4697   op_cost(0);
 4698   format %{ %}
 4699   interface(CONST_INTER);
 4700 %}
 4701 
 4702 // Narrow pointer operands
 4703 // Narrow Pointer Immediate
 4704 operand immN()
 4705 %{
 4706   match(ConN);
 4707 
 4708   op_cost(0);
 4709   format %{ %}
 4710   interface(CONST_INTER);
 4711 %}
 4712 
 4713 // Narrow nullptr Pointer Immediate
 4714 operand immN0()
 4715 %{
 4716   predicate(n->get_narrowcon() == 0);
 4717   match(ConN);
 4718 
 4719   op_cost(0);
 4720   format %{ %}
 4721   interface(CONST_INTER);
 4722 %}
 4723 
 4724 operand immNKlass()
 4725 %{
 4726   match(ConNKlass);
 4727 
 4728   op_cost(0);
 4729   format %{ %}
 4730   interface(CONST_INTER);
 4731 %}
 4732 
 4733 // Integer 32 bit Register Operands
 4734 // Integer 32 bitRegister (excludes SP)
 4735 operand iRegI()
 4736 %{
 4737   constraint(ALLOC_IN_RC(any_reg32));
 4738   match(RegI);
 4739   match(iRegINoSp);
 4740   op_cost(0);
 4741   format %{ %}
 4742   interface(REG_INTER);
 4743 %}
 4744 
 4745 // Integer 32 bit Register not Special
 4746 operand iRegINoSp()
 4747 %{
 4748   constraint(ALLOC_IN_RC(no_special_reg32));
 4749   match(RegI);
 4750   op_cost(0);
 4751   format %{ %}
 4752   interface(REG_INTER);
 4753 %}
 4754 
 4755 // Integer 64 bit Register Operands
 4756 // Integer 64 bit Register (includes SP)
 4757 operand iRegL()
 4758 %{
 4759   constraint(ALLOC_IN_RC(any_reg));
 4760   match(RegL);
 4761   match(iRegLNoSp);
 4762   op_cost(0);
 4763   format %{ %}
 4764   interface(REG_INTER);
 4765 %}
 4766 
 4767 // Integer 64 bit Register not Special
 4768 operand iRegLNoSp()
 4769 %{
 4770   constraint(ALLOC_IN_RC(no_special_reg));
 4771   match(RegL);
 4772   match(iRegL_R0);
 4773   format %{ %}
 4774   interface(REG_INTER);
 4775 %}
 4776 
 4777 // Pointer Register Operands
 4778 // Pointer Register
 4779 operand iRegP()
 4780 %{
 4781   constraint(ALLOC_IN_RC(ptr_reg));
 4782   match(RegP);
 4783   match(iRegPNoSp);
 4784   match(iRegP_R0);
 4785   //match(iRegP_R2);
 4786   //match(iRegP_R4);
 4787   match(iRegP_R5);
 4788   match(thread_RegP);
 4789   op_cost(0);
 4790   format %{ %}
 4791   interface(REG_INTER);
 4792 %}
 4793 
 4794 // Pointer 64 bit Register not Special
 4795 operand iRegPNoSp()
 4796 %{
 4797   constraint(ALLOC_IN_RC(no_special_ptr_reg));
 4798   match(RegP);
 4799   // match(iRegP);
 4800   // match(iRegP_R0);
 4801   // match(iRegP_R2);
 4802   // match(iRegP_R4);
 4803   // match(iRegP_R5);
 4804   // match(thread_RegP);
 4805   op_cost(0);
 4806   format %{ %}
 4807   interface(REG_INTER);
 4808 %}
 4809 
 4810 // This operand is not allowed to use rfp even if
 4811 // rfp is not used to hold the frame pointer.
 4812 operand iRegPNoSpNoRfp()
 4813 %{
 4814   constraint(ALLOC_IN_RC(no_special_no_rfp_ptr_reg));
 4815   match(RegP);
 4816   match(iRegPNoSp);
 4817   op_cost(0);
 4818   format %{ %}
 4819   interface(REG_INTER);
 4820 %}
 4821 
 4822 // Pointer 64 bit Register R0 only
 4823 operand iRegP_R0()
 4824 %{
 4825   constraint(ALLOC_IN_RC(r0_reg));
 4826   match(RegP);
 4827   // match(iRegP);
 4828   match(iRegPNoSp);
 4829   op_cost(0);
 4830   format %{ %}
 4831   interface(REG_INTER);
 4832 %}
 4833 
 4834 // Pointer 64 bit Register R1 only
 4835 operand iRegP_R1()
 4836 %{
 4837   constraint(ALLOC_IN_RC(r1_reg));
 4838   match(RegP);
 4839   // match(iRegP);
 4840   match(iRegPNoSp);
 4841   op_cost(0);
 4842   format %{ %}
 4843   interface(REG_INTER);
 4844 %}
 4845 
 4846 // Pointer 64 bit Register R2 only
 4847 operand iRegP_R2()
 4848 %{
 4849   constraint(ALLOC_IN_RC(r2_reg));
 4850   match(RegP);
 4851   // match(iRegP);
 4852   match(iRegPNoSp);
 4853   op_cost(0);
 4854   format %{ %}
 4855   interface(REG_INTER);
 4856 %}
 4857 
 4858 // Pointer 64 bit Register R3 only
 4859 operand iRegP_R3()
 4860 %{
 4861   constraint(ALLOC_IN_RC(r3_reg));
 4862   match(RegP);
 4863   // match(iRegP);
 4864   match(iRegPNoSp);
 4865   op_cost(0);
 4866   format %{ %}
 4867   interface(REG_INTER);
 4868 %}
 4869 
 4870 // Pointer 64 bit Register R4 only
 4871 operand iRegP_R4()
 4872 %{
 4873   constraint(ALLOC_IN_RC(r4_reg));
 4874   match(RegP);
 4875   // match(iRegP);
 4876   match(iRegPNoSp);
 4877   op_cost(0);
 4878   format %{ %}
 4879   interface(REG_INTER);
 4880 %}
 4881 
 4882 // Pointer 64 bit Register R5 only
 4883 operand iRegP_R5()
 4884 %{
 4885   constraint(ALLOC_IN_RC(r5_reg));
 4886   match(RegP);
 4887   // match(iRegP);
 4888   match(iRegPNoSp);
 4889   op_cost(0);
 4890   format %{ %}
 4891   interface(REG_INTER);
 4892 %}
 4893 
 4894 // Pointer 64 bit Register R10 only
 4895 operand iRegP_R10()
 4896 %{
 4897   constraint(ALLOC_IN_RC(r10_reg));
 4898   match(RegP);
 4899   // match(iRegP);
 4900   match(iRegPNoSp);
 4901   op_cost(0);
 4902   format %{ %}
 4903   interface(REG_INTER);
 4904 %}
 4905 
 4906 // Long 64 bit Register R0 only
 4907 operand iRegL_R0()
 4908 %{
 4909   constraint(ALLOC_IN_RC(r0_reg));
 4910   match(RegL);
 4911   match(iRegLNoSp);
 4912   op_cost(0);
 4913   format %{ %}
 4914   interface(REG_INTER);
 4915 %}
 4916 
 4917 // Long 64 bit Register R11 only
 4918 operand iRegL_R11()
 4919 %{
 4920   constraint(ALLOC_IN_RC(r11_reg));
 4921   match(RegL);
 4922   match(iRegLNoSp);
 4923   op_cost(0);
 4924   format %{ %}
 4925   interface(REG_INTER);
 4926 %}
 4927 
 4928 // Register R0 only
 4929 operand iRegI_R0()
 4930 %{
 4931   constraint(ALLOC_IN_RC(int_r0_reg));
 4932   match(RegI);
 4933   match(iRegINoSp);
 4934   op_cost(0);
 4935   format %{ %}
 4936   interface(REG_INTER);
 4937 %}
 4938 
 4939 // Register R2 only
 4940 operand iRegI_R2()
 4941 %{
 4942   constraint(ALLOC_IN_RC(int_r2_reg));
 4943   match(RegI);
 4944   match(iRegINoSp);
 4945   op_cost(0);
 4946   format %{ %}
 4947   interface(REG_INTER);
 4948 %}
 4949 
 4950 // Register R3 only
 4951 operand iRegI_R3()
 4952 %{
 4953   constraint(ALLOC_IN_RC(int_r3_reg));
 4954   match(RegI);
 4955   match(iRegINoSp);
 4956   op_cost(0);
 4957   format %{ %}
 4958   interface(REG_INTER);
 4959 %}
 4960 
 4961 
 4962 // Register R4 only
 4963 operand iRegI_R4()
 4964 %{
 4965   constraint(ALLOC_IN_RC(int_r4_reg));
 4966   match(RegI);
 4967   match(iRegINoSp);
 4968   op_cost(0);
 4969   format %{ %}
 4970   interface(REG_INTER);
 4971 %}
 4972 
 4973 
 4974 // Pointer Register Operands
 4975 // Narrow Pointer Register
 4976 operand iRegN()
 4977 %{
 4978   constraint(ALLOC_IN_RC(any_reg32));
 4979   match(RegN);
 4980   match(iRegNNoSp);
 4981   op_cost(0);
 4982   format %{ %}
 4983   interface(REG_INTER);
 4984 %}
 4985 
 4986 // Integer 64 bit Register not Special
 4987 operand iRegNNoSp()
 4988 %{
 4989   constraint(ALLOC_IN_RC(no_special_reg32));
 4990   match(RegN);
 4991   op_cost(0);
 4992   format %{ %}
 4993   interface(REG_INTER);
 4994 %}
 4995 
 4996 // Float Register
 4997 // Float register operands
 4998 operand vRegF()
 4999 %{
 5000   constraint(ALLOC_IN_RC(float_reg));
 5001   match(RegF);
 5002 
 5003   op_cost(0);
 5004   format %{ %}
 5005   interface(REG_INTER);
 5006 %}
 5007 
 5008 // Double Register
 5009 // Double register operands
 5010 operand vRegD()
 5011 %{
 5012   constraint(ALLOC_IN_RC(double_reg));
 5013   match(RegD);
 5014 
 5015   op_cost(0);
 5016   format %{ %}
 5017   interface(REG_INTER);
 5018 %}
 5019 
 5020 // Generic vector class. This will be used for
 5021 // all vector operands, including NEON and SVE.
 5022 operand vReg()
 5023 %{
 5024   constraint(ALLOC_IN_RC(dynamic));
 5025   match(VecA);
 5026   match(VecD);
 5027   match(VecX);
 5028 
 5029   op_cost(0);
 5030   format %{ %}
 5031   interface(REG_INTER);
 5032 %}
 5033 
 5034 operand vReg_V10()
 5035 %{
 5036   constraint(ALLOC_IN_RC(v10_veca_reg));
 5037   match(vReg);
 5038 
 5039   op_cost(0);
 5040   format %{ %}
 5041   interface(REG_INTER);
 5042 %}
 5043 
 5044 operand vReg_V11()
 5045 %{
 5046   constraint(ALLOC_IN_RC(v11_veca_reg));
 5047   match(vReg);
 5048 
 5049   op_cost(0);
 5050   format %{ %}
 5051   interface(REG_INTER);
 5052 %}
 5053 
 5054 operand vReg_V12()
 5055 %{
 5056   constraint(ALLOC_IN_RC(v12_veca_reg));
 5057   match(vReg);
 5058 
 5059   op_cost(0);
 5060   format %{ %}
 5061   interface(REG_INTER);
 5062 %}
 5063 
 5064 operand vReg_V13()
 5065 %{
 5066   constraint(ALLOC_IN_RC(v13_veca_reg));
 5067   match(vReg);
 5068 
 5069   op_cost(0);
 5070   format %{ %}
 5071   interface(REG_INTER);
 5072 %}
 5073 
 5074 operand vReg_V17()
 5075 %{
 5076   constraint(ALLOC_IN_RC(v17_veca_reg));
 5077   match(vReg);
 5078 
 5079   op_cost(0);
 5080   format %{ %}
 5081   interface(REG_INTER);
 5082 %}
 5083 
 5084 operand vReg_V18()
 5085 %{
 5086   constraint(ALLOC_IN_RC(v18_veca_reg));
 5087   match(vReg);
 5088 
 5089   op_cost(0);
 5090   format %{ %}
 5091   interface(REG_INTER);
 5092 %}
 5093 
 5094 operand vReg_V23()
 5095 %{
 5096   constraint(ALLOC_IN_RC(v23_veca_reg));
 5097   match(vReg);
 5098 
 5099   op_cost(0);
 5100   format %{ %}
 5101   interface(REG_INTER);
 5102 %}
 5103 
 5104 operand vReg_V24()
 5105 %{
 5106   constraint(ALLOC_IN_RC(v24_veca_reg));
 5107   match(vReg);
 5108 
 5109   op_cost(0);
 5110   format %{ %}
 5111   interface(REG_INTER);
 5112 %}
 5113 
 5114 operand vecA()
 5115 %{
 5116   constraint(ALLOC_IN_RC(vectora_reg));
 5117   match(VecA);
 5118 
 5119   op_cost(0);
 5120   format %{ %}
 5121   interface(REG_INTER);
 5122 %}
 5123 
 5124 operand vecD()
 5125 %{
 5126   constraint(ALLOC_IN_RC(vectord_reg));
 5127   match(VecD);
 5128 
 5129   op_cost(0);
 5130   format %{ %}
 5131   interface(REG_INTER);
 5132 %}
 5133 
 5134 operand vecX()
 5135 %{
 5136   constraint(ALLOC_IN_RC(vectorx_reg));
 5137   match(VecX);
 5138 
 5139   op_cost(0);
 5140   format %{ %}
 5141   interface(REG_INTER);
 5142 %}
 5143 
 5144 operand vRegD_V0()
 5145 %{
 5146   constraint(ALLOC_IN_RC(v0_reg));
 5147   match(RegD);
 5148   op_cost(0);
 5149   format %{ %}
 5150   interface(REG_INTER);
 5151 %}
 5152 
 5153 operand vRegD_V1()
 5154 %{
 5155   constraint(ALLOC_IN_RC(v1_reg));
 5156   match(RegD);
 5157   op_cost(0);
 5158   format %{ %}
 5159   interface(REG_INTER);
 5160 %}
 5161 
 5162 operand vRegD_V2()
 5163 %{
 5164   constraint(ALLOC_IN_RC(v2_reg));
 5165   match(RegD);
 5166   op_cost(0);
 5167   format %{ %}
 5168   interface(REG_INTER);
 5169 %}
 5170 
 5171 operand vRegD_V3()
 5172 %{
 5173   constraint(ALLOC_IN_RC(v3_reg));
 5174   match(RegD);
 5175   op_cost(0);
 5176   format %{ %}
 5177   interface(REG_INTER);
 5178 %}
 5179 
 5180 operand vRegD_V4()
 5181 %{
 5182   constraint(ALLOC_IN_RC(v4_reg));
 5183   match(RegD);
 5184   op_cost(0);
 5185   format %{ %}
 5186   interface(REG_INTER);
 5187 %}
 5188 
 5189 operand vRegD_V5()
 5190 %{
 5191   constraint(ALLOC_IN_RC(v5_reg));
 5192   match(RegD);
 5193   op_cost(0);
 5194   format %{ %}
 5195   interface(REG_INTER);
 5196 %}
 5197 
 5198 operand vRegD_V6()
 5199 %{
 5200   constraint(ALLOC_IN_RC(v6_reg));
 5201   match(RegD);
 5202   op_cost(0);
 5203   format %{ %}
 5204   interface(REG_INTER);
 5205 %}
 5206 
 5207 operand vRegD_V7()
 5208 %{
 5209   constraint(ALLOC_IN_RC(v7_reg));
 5210   match(RegD);
 5211   op_cost(0);
 5212   format %{ %}
 5213   interface(REG_INTER);
 5214 %}
 5215 
 5216 operand vRegD_V12()
 5217 %{
 5218   constraint(ALLOC_IN_RC(v12_reg));
 5219   match(RegD);
 5220   op_cost(0);
 5221   format %{ %}
 5222   interface(REG_INTER);
 5223 %}
 5224 
 5225 operand vRegD_V13()
 5226 %{
 5227   constraint(ALLOC_IN_RC(v13_reg));
 5228   match(RegD);
 5229   op_cost(0);
 5230   format %{ %}
 5231   interface(REG_INTER);
 5232 %}
 5233 
 5234 operand pReg()
 5235 %{
 5236   constraint(ALLOC_IN_RC(pr_reg));
 5237   match(RegVectMask);
 5238   match(pRegGov);
 5239   op_cost(0);
 5240   format %{ %}
 5241   interface(REG_INTER);
 5242 %}
 5243 
 5244 operand pRegGov()
 5245 %{
 5246   constraint(ALLOC_IN_RC(gov_pr));
 5247   match(RegVectMask);
 5248   match(pReg);
 5249   op_cost(0);
 5250   format %{ %}
 5251   interface(REG_INTER);
 5252 %}
 5253 
 5254 operand pRegGov_P0()
 5255 %{
 5256   constraint(ALLOC_IN_RC(p0_reg));
 5257   match(RegVectMask);
 5258   op_cost(0);
 5259   format %{ %}
 5260   interface(REG_INTER);
 5261 %}
 5262 
 5263 operand pRegGov_P1()
 5264 %{
 5265   constraint(ALLOC_IN_RC(p1_reg));
 5266   match(RegVectMask);
 5267   op_cost(0);
 5268   format %{ %}
 5269   interface(REG_INTER);
 5270 %}
 5271 
 5272 // Flags register, used as output of signed compare instructions
 5273 
 5274 // note that on AArch64 we also use this register as the output for
 5275 // for floating point compare instructions (CmpF CmpD). this ensures
 5276 // that ordered inequality tests use GT, GE, LT or LE none of which
 5277 // pass through cases where the result is unordered i.e. one or both
 5278 // inputs to the compare is a NaN. this means that the ideal code can
 5279 // replace e.g. a GT with an LE and not end up capturing the NaN case
 5280 // (where the comparison should always fail). EQ and NE tests are
 5281 // always generated in ideal code so that unordered folds into the NE
 5282 // case, matching the behaviour of AArch64 NE.
 5283 //
 5284 // This differs from x86 where the outputs of FP compares use a
 5285 // special FP flags registers and where compares based on this
 5286 // register are distinguished into ordered inequalities (cmpOpUCF) and
 5287 // EQ/NEQ tests (cmpOpUCF2). x86 has to special case the latter tests
 5288 // to explicitly handle the unordered case in branches. x86 also has
 5289 // to include extra CMoveX rules to accept a cmpOpUCF input.
 5290 
 5291 operand rFlagsReg()
 5292 %{
 5293   constraint(ALLOC_IN_RC(int_flags));
 5294   match(RegFlags);
 5295 
 5296   op_cost(0);
 5297   format %{ "RFLAGS" %}
 5298   interface(REG_INTER);
 5299 %}
 5300 
 5301 // Flags register, used as output of unsigned compare instructions
 5302 operand rFlagsRegU()
 5303 %{
 5304   constraint(ALLOC_IN_RC(int_flags));
 5305   match(RegFlags);
 5306 
 5307   op_cost(0);
 5308   format %{ "RFLAGSU" %}
 5309   interface(REG_INTER);
 5310 %}
 5311 
 5312 // Special Registers
 5313 
 5314 // Method Register
 5315 operand inline_cache_RegP(iRegP reg)
 5316 %{
 5317   constraint(ALLOC_IN_RC(method_reg)); // inline_cache_reg
 5318   match(reg);
 5319   match(iRegPNoSp);
 5320   op_cost(0);
 5321   format %{ %}
 5322   interface(REG_INTER);
 5323 %}
 5324 
 5325 // Thread Register
 5326 operand thread_RegP(iRegP reg)
 5327 %{
 5328   constraint(ALLOC_IN_RC(thread_reg)); // link_reg
 5329   match(reg);
 5330   op_cost(0);
 5331   format %{ %}
 5332   interface(REG_INTER);
 5333 %}
 5334 
 5335 //----------Memory Operands----------------------------------------------------
 5336 
 5337 operand indirect(iRegP reg)
 5338 %{
 5339   constraint(ALLOC_IN_RC(ptr_reg));
 5340   match(reg);
 5341   op_cost(0);
 5342   format %{ "[$reg]" %}
 5343   interface(MEMORY_INTER) %{
 5344     base($reg);
 5345     index(0xffffffff);
 5346     scale(0x0);
 5347     disp(0x0);
 5348   %}
 5349 %}
 5350 
 5351 operand indIndexScaledI2L(iRegP reg, iRegI ireg, immIScale scale)
 5352 %{
 5353   constraint(ALLOC_IN_RC(ptr_reg));
 5354   predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 5355   match(AddP reg (LShiftL (ConvI2L ireg) scale));
 5356   op_cost(0);
 5357   format %{ "$reg, $ireg sxtw($scale), 0, I2L" %}
 5358   interface(MEMORY_INTER) %{
 5359     base($reg);
 5360     index($ireg);
 5361     scale($scale);
 5362     disp(0x0);
 5363   %}
 5364 %}
 5365 
 5366 operand indIndexScaled(iRegP reg, iRegL lreg, immIScale scale)
 5367 %{
 5368   constraint(ALLOC_IN_RC(ptr_reg));
 5369   predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 5370   match(AddP reg (LShiftL lreg scale));
 5371   op_cost(0);
 5372   format %{ "$reg, $lreg lsl($scale)" %}
 5373   interface(MEMORY_INTER) %{
 5374     base($reg);
 5375     index($lreg);
 5376     scale($scale);
 5377     disp(0x0);
 5378   %}
 5379 %}
 5380 
 5381 operand indIndexI2L(iRegP reg, iRegI ireg)
 5382 %{
 5383   constraint(ALLOC_IN_RC(ptr_reg));
 5384   match(AddP reg (ConvI2L ireg));
 5385   op_cost(0);
 5386   format %{ "$reg, $ireg, 0, I2L" %}
 5387   interface(MEMORY_INTER) %{
 5388     base($reg);
 5389     index($ireg);
 5390     scale(0x0);
 5391     disp(0x0);
 5392   %}
 5393 %}
 5394 
 5395 operand indIndex(iRegP reg, iRegL lreg)
 5396 %{
 5397   constraint(ALLOC_IN_RC(ptr_reg));
 5398   match(AddP reg lreg);
 5399   op_cost(0);
 5400   format %{ "$reg, $lreg" %}
 5401   interface(MEMORY_INTER) %{
 5402     base($reg);
 5403     index($lreg);
 5404     scale(0x0);
 5405     disp(0x0);
 5406   %}
 5407 %}
 5408 
 5409 operand indOffI1(iRegP reg, immIOffset1 off)
 5410 %{
 5411   constraint(ALLOC_IN_RC(ptr_reg));
 5412   match(AddP reg off);
 5413   op_cost(0);
 5414   format %{ "[$reg, $off]" %}
 5415   interface(MEMORY_INTER) %{
 5416     base($reg);
 5417     index(0xffffffff);
 5418     scale(0x0);
 5419     disp($off);
 5420   %}
 5421 %}
 5422 
 5423 operand indOffI2(iRegP reg, immIOffset2 off)
 5424 %{
 5425   constraint(ALLOC_IN_RC(ptr_reg));
 5426   match(AddP reg off);
 5427   op_cost(0);
 5428   format %{ "[$reg, $off]" %}
 5429   interface(MEMORY_INTER) %{
 5430     base($reg);
 5431     index(0xffffffff);
 5432     scale(0x0);
 5433     disp($off);
 5434   %}
 5435 %}
 5436 
 5437 operand indOffI4(iRegP reg, immIOffset4 off)
 5438 %{
 5439   constraint(ALLOC_IN_RC(ptr_reg));
 5440   match(AddP reg off);
 5441   op_cost(0);
 5442   format %{ "[$reg, $off]" %}
 5443   interface(MEMORY_INTER) %{
 5444     base($reg);
 5445     index(0xffffffff);
 5446     scale(0x0);
 5447     disp($off);
 5448   %}
 5449 %}
 5450 
 5451 operand indOffI8(iRegP reg, immIOffset8 off)
 5452 %{
 5453   constraint(ALLOC_IN_RC(ptr_reg));
 5454   match(AddP reg off);
 5455   op_cost(0);
 5456   format %{ "[$reg, $off]" %}
 5457   interface(MEMORY_INTER) %{
 5458     base($reg);
 5459     index(0xffffffff);
 5460     scale(0x0);
 5461     disp($off);
 5462   %}
 5463 %}
 5464 
 5465 operand indOffI16(iRegP reg, immIOffset16 off)
 5466 %{
 5467   constraint(ALLOC_IN_RC(ptr_reg));
 5468   match(AddP reg off);
 5469   op_cost(0);
 5470   format %{ "[$reg, $off]" %}
 5471   interface(MEMORY_INTER) %{
 5472     base($reg);
 5473     index(0xffffffff);
 5474     scale(0x0);
 5475     disp($off);
 5476   %}
 5477 %}
 5478 
 5479 operand indOffL1(iRegP reg, immLoffset1 off)
 5480 %{
 5481   constraint(ALLOC_IN_RC(ptr_reg));
 5482   match(AddP reg off);
 5483   op_cost(0);
 5484   format %{ "[$reg, $off]" %}
 5485   interface(MEMORY_INTER) %{
 5486     base($reg);
 5487     index(0xffffffff);
 5488     scale(0x0);
 5489     disp($off);
 5490   %}
 5491 %}
 5492 
 5493 operand indOffL2(iRegP reg, immLoffset2 off)
 5494 %{
 5495   constraint(ALLOC_IN_RC(ptr_reg));
 5496   match(AddP reg off);
 5497   op_cost(0);
 5498   format %{ "[$reg, $off]" %}
 5499   interface(MEMORY_INTER) %{
 5500     base($reg);
 5501     index(0xffffffff);
 5502     scale(0x0);
 5503     disp($off);
 5504   %}
 5505 %}
 5506 
 5507 operand indOffL4(iRegP reg, immLoffset4 off)
 5508 %{
 5509   constraint(ALLOC_IN_RC(ptr_reg));
 5510   match(AddP reg off);
 5511   op_cost(0);
 5512   format %{ "[$reg, $off]" %}
 5513   interface(MEMORY_INTER) %{
 5514     base($reg);
 5515     index(0xffffffff);
 5516     scale(0x0);
 5517     disp($off);
 5518   %}
 5519 %}
 5520 
 5521 operand indOffL8(iRegP reg, immLoffset8 off)
 5522 %{
 5523   constraint(ALLOC_IN_RC(ptr_reg));
 5524   match(AddP reg off);
 5525   op_cost(0);
 5526   format %{ "[$reg, $off]" %}
 5527   interface(MEMORY_INTER) %{
 5528     base($reg);
 5529     index(0xffffffff);
 5530     scale(0x0);
 5531     disp($off);
 5532   %}
 5533 %}
 5534 
 5535 operand indOffL16(iRegP reg, immLoffset16 off)
 5536 %{
 5537   constraint(ALLOC_IN_RC(ptr_reg));
 5538   match(AddP reg off);
 5539   op_cost(0);
 5540   format %{ "[$reg, $off]" %}
 5541   interface(MEMORY_INTER) %{
 5542     base($reg);
 5543     index(0xffffffff);
 5544     scale(0x0);
 5545     disp($off);
 5546   %}
 5547 %}
 5548 
 5549 operand indirectX2P(iRegL reg)
 5550 %{
 5551   constraint(ALLOC_IN_RC(ptr_reg));
 5552   match(CastX2P reg);
 5553   op_cost(0);
 5554   format %{ "[$reg]\t# long -> ptr" %}
 5555   interface(MEMORY_INTER) %{
 5556     base($reg);
 5557     index(0xffffffff);
 5558     scale(0x0);
 5559     disp(0x0);
 5560   %}
 5561 %}
 5562 
 5563 operand indOffX2P(iRegL reg, immLOffset off)
 5564 %{
 5565   constraint(ALLOC_IN_RC(ptr_reg));
 5566   match(AddP (CastX2P reg) off);
 5567   op_cost(0);
 5568   format %{ "[$reg, $off]\t# long -> ptr" %}
 5569   interface(MEMORY_INTER) %{
 5570     base($reg);
 5571     index(0xffffffff);
 5572     scale(0x0);
 5573     disp($off);
 5574   %}
 5575 %}
 5576 
 5577 operand indirectN(iRegN reg)
 5578 %{
 5579   predicate(CompressedOops::shift() == 0);
 5580   constraint(ALLOC_IN_RC(ptr_reg));
 5581   match(DecodeN reg);
 5582   op_cost(0);
 5583   format %{ "[$reg]\t# narrow" %}
 5584   interface(MEMORY_INTER) %{
 5585     base($reg);
 5586     index(0xffffffff);
 5587     scale(0x0);
 5588     disp(0x0);
 5589   %}
 5590 %}
 5591 
 5592 operand indIndexScaledI2LN(iRegN reg, iRegI ireg, immIScale scale)
 5593 %{
 5594   predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 5595   constraint(ALLOC_IN_RC(ptr_reg));
 5596   match(AddP (DecodeN reg) (LShiftL (ConvI2L ireg) scale));
 5597   op_cost(0);
 5598   format %{ "$reg, $ireg sxtw($scale), 0, I2L\t# narrow" %}
 5599   interface(MEMORY_INTER) %{
 5600     base($reg);
 5601     index($ireg);
 5602     scale($scale);
 5603     disp(0x0);
 5604   %}
 5605 %}
 5606 
 5607 operand indIndexScaledN(iRegN reg, iRegL lreg, immIScale scale)
 5608 %{
 5609   predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 5610   constraint(ALLOC_IN_RC(ptr_reg));
 5611   match(AddP (DecodeN reg) (LShiftL lreg scale));
 5612   op_cost(0);
 5613   format %{ "$reg, $lreg lsl($scale)\t# narrow" %}
 5614   interface(MEMORY_INTER) %{
 5615     base($reg);
 5616     index($lreg);
 5617     scale($scale);
 5618     disp(0x0);
 5619   %}
 5620 %}
 5621 
 5622 operand indIndexI2LN(iRegN reg, iRegI ireg)
 5623 %{
 5624   predicate(CompressedOops::shift() == 0);
 5625   constraint(ALLOC_IN_RC(ptr_reg));
 5626   match(AddP (DecodeN reg) (ConvI2L ireg));
 5627   op_cost(0);
 5628   format %{ "$reg, $ireg, 0, I2L\t# narrow" %}
 5629   interface(MEMORY_INTER) %{
 5630     base($reg);
 5631     index($ireg);
 5632     scale(0x0);
 5633     disp(0x0);
 5634   %}
 5635 %}
 5636 
 5637 operand indIndexN(iRegN reg, iRegL lreg)
 5638 %{
 5639   predicate(CompressedOops::shift() == 0);
 5640   constraint(ALLOC_IN_RC(ptr_reg));
 5641   match(AddP (DecodeN reg) lreg);
 5642   op_cost(0);
 5643   format %{ "$reg, $lreg\t# narrow" %}
 5644   interface(MEMORY_INTER) %{
 5645     base($reg);
 5646     index($lreg);
 5647     scale(0x0);
 5648     disp(0x0);
 5649   %}
 5650 %}
 5651 
 5652 operand indOffIN(iRegN reg, immIOffset off)
 5653 %{
 5654   predicate(CompressedOops::shift() == 0);
 5655   constraint(ALLOC_IN_RC(ptr_reg));
 5656   match(AddP (DecodeN reg) off);
 5657   op_cost(0);
 5658   format %{ "[$reg, $off]\t# narrow" %}
 5659   interface(MEMORY_INTER) %{
 5660     base($reg);
 5661     index(0xffffffff);
 5662     scale(0x0);
 5663     disp($off);
 5664   %}
 5665 %}
 5666 
 5667 operand indOffLN(iRegN reg, immLOffset off)
 5668 %{
 5669   predicate(CompressedOops::shift() == 0);
 5670   constraint(ALLOC_IN_RC(ptr_reg));
 5671   match(AddP (DecodeN reg) off);
 5672   op_cost(0);
 5673   format %{ "[$reg, $off]\t# narrow" %}
 5674   interface(MEMORY_INTER) %{
 5675     base($reg);
 5676     index(0xffffffff);
 5677     scale(0x0);
 5678     disp($off);
 5679   %}
 5680 %}
 5681 
 5682 
 5683 //----------Special Memory Operands--------------------------------------------
 5684 // Stack Slot Operand - This operand is used for loading and storing temporary
 5685 //                      values on the stack where a match requires a value to
 5686 //                      flow through memory.
 5687 operand stackSlotP(sRegP reg)
 5688 %{
 5689   constraint(ALLOC_IN_RC(stack_slots));
 5690   op_cost(100);
 5691   // No match rule because this operand is only generated in matching
 5692   // match(RegP);
 5693   format %{ "[$reg]" %}
 5694   interface(MEMORY_INTER) %{
 5695     base(0x1e);  // RSP
 5696     index(0x0);  // No Index
 5697     scale(0x0);  // No Scale
 5698     disp($reg);  // Stack Offset
 5699   %}
 5700 %}
 5701 
 5702 operand stackSlotI(sRegI reg)
 5703 %{
 5704   constraint(ALLOC_IN_RC(stack_slots));
 5705   // No match rule because this operand is only generated in matching
 5706   // match(RegI);
 5707   format %{ "[$reg]" %}
 5708   interface(MEMORY_INTER) %{
 5709     base(0x1e);  // RSP
 5710     index(0x0);  // No Index
 5711     scale(0x0);  // No Scale
 5712     disp($reg);  // Stack Offset
 5713   %}
 5714 %}
 5715 
 5716 operand stackSlotF(sRegF reg)
 5717 %{
 5718   constraint(ALLOC_IN_RC(stack_slots));
 5719   // No match rule because this operand is only generated in matching
 5720   // match(RegF);
 5721   format %{ "[$reg]" %}
 5722   interface(MEMORY_INTER) %{
 5723     base(0x1e);  // RSP
 5724     index(0x0);  // No Index
 5725     scale(0x0);  // No Scale
 5726     disp($reg);  // Stack Offset
 5727   %}
 5728 %}
 5729 
 5730 operand stackSlotD(sRegD reg)
 5731 %{
 5732   constraint(ALLOC_IN_RC(stack_slots));
 5733   // No match rule because this operand is only generated in matching
 5734   // match(RegD);
 5735   format %{ "[$reg]" %}
 5736   interface(MEMORY_INTER) %{
 5737     base(0x1e);  // RSP
 5738     index(0x0);  // No Index
 5739     scale(0x0);  // No Scale
 5740     disp($reg);  // Stack Offset
 5741   %}
 5742 %}
 5743 
 5744 operand stackSlotL(sRegL reg)
 5745 %{
 5746   constraint(ALLOC_IN_RC(stack_slots));
 5747   // No match rule because this operand is only generated in matching
 5748   // match(RegL);
 5749   format %{ "[$reg]" %}
 5750   interface(MEMORY_INTER) %{
 5751     base(0x1e);  // RSP
 5752     index(0x0);  // No Index
 5753     scale(0x0);  // No Scale
 5754     disp($reg);  // Stack Offset
 5755   %}
 5756 %}
 5757 
 5758 // Operands for expressing Control Flow
 5759 // NOTE: Label is a predefined operand which should not be redefined in
 5760 //       the AD file. It is generically handled within the ADLC.
 5761 
 5762 //----------Conditional Branch Operands----------------------------------------
 5763 // Comparison Op  - This is the operation of the comparison, and is limited to
 5764 //                  the following set of codes:
 5765 //                  L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
 5766 //
 5767 // Other attributes of the comparison, such as unsignedness, are specified
 5768 // by the comparison instruction that sets a condition code flags register.
 5769 // That result is represented by a flags operand whose subtype is appropriate
 5770 // to the unsignedness (etc.) of the comparison.
 5771 //
 5772 // Later, the instruction which matches both the Comparison Op (a Bool) and
 5773 // the flags (produced by the Cmp) specifies the coding of the comparison op
 5774 // by matching a specific subtype of Bool operand below, such as cmpOpU.
 5775 
 5776 // used for signed integral comparisons and fp comparisons
 5777 
 5778 operand cmpOp()
 5779 %{
 5780   match(Bool);
 5781 
 5782   format %{ "" %}
 5783   interface(COND_INTER) %{
 5784     equal(0x0, "eq");
 5785     not_equal(0x1, "ne");
 5786     less(0xb, "lt");
 5787     greater_equal(0xa, "ge");
 5788     less_equal(0xd, "le");
 5789     greater(0xc, "gt");
 5790     overflow(0x6, "vs");
 5791     no_overflow(0x7, "vc");
 5792   %}
 5793 %}
 5794 
 5795 // used for unsigned integral comparisons
 5796 
 5797 operand cmpOpU()
 5798 %{
 5799   match(Bool);
 5800 
 5801   format %{ "" %}
 5802   interface(COND_INTER) %{
 5803     equal(0x0, "eq");
 5804     not_equal(0x1, "ne");
 5805     less(0x3, "lo");
 5806     greater_equal(0x2, "hs");
 5807     less_equal(0x9, "ls");
 5808     greater(0x8, "hi");
 5809     overflow(0x6, "vs");
 5810     no_overflow(0x7, "vc");
 5811   %}
 5812 %}
 5813 
 5814 // used for certain integral comparisons which can be
 5815 // converted to cbxx or tbxx instructions
 5816 
 5817 operand cmpOpEqNe()
 5818 %{
 5819   match(Bool);
 5820   op_cost(0);
 5821   predicate(n->as_Bool()->_test._test == BoolTest::ne
 5822             || n->as_Bool()->_test._test == BoolTest::eq);
 5823 
 5824   format %{ "" %}
 5825   interface(COND_INTER) %{
 5826     equal(0x0, "eq");
 5827     not_equal(0x1, "ne");
 5828     less(0xb, "lt");
 5829     greater_equal(0xa, "ge");
 5830     less_equal(0xd, "le");
 5831     greater(0xc, "gt");
 5832     overflow(0x6, "vs");
 5833     no_overflow(0x7, "vc");
 5834   %}
 5835 %}
 5836 
 5837 // used for certain integral comparisons which can be
 5838 // converted to cbxx or tbxx instructions
 5839 
 5840 operand cmpOpLtGe()
 5841 %{
 5842   match(Bool);
 5843   op_cost(0);
 5844 
 5845   predicate(n->as_Bool()->_test._test == BoolTest::lt
 5846             || n->as_Bool()->_test._test == BoolTest::ge);
 5847 
 5848   format %{ "" %}
 5849   interface(COND_INTER) %{
 5850     equal(0x0, "eq");
 5851     not_equal(0x1, "ne");
 5852     less(0xb, "lt");
 5853     greater_equal(0xa, "ge");
 5854     less_equal(0xd, "le");
 5855     greater(0xc, "gt");
 5856     overflow(0x6, "vs");
 5857     no_overflow(0x7, "vc");
 5858   %}
 5859 %}
 5860 
 5861 // used for certain unsigned integral comparisons which can be
 5862 // converted to cbxx or tbxx instructions
 5863 
 5864 operand cmpOpUEqNeLeGt()
 5865 %{
 5866   match(Bool);
 5867   op_cost(0);
 5868 
 5869   predicate(n->as_Bool()->_test._test == BoolTest::eq ||
 5870             n->as_Bool()->_test._test == BoolTest::ne ||
 5871             n->as_Bool()->_test._test == BoolTest::le ||
 5872             n->as_Bool()->_test._test == BoolTest::gt);
 5873 
 5874   format %{ "" %}
 5875   interface(COND_INTER) %{
 5876     equal(0x0, "eq");
 5877     not_equal(0x1, "ne");
 5878     less(0x3, "lo");
 5879     greater_equal(0x2, "hs");
 5880     less_equal(0x9, "ls");
 5881     greater(0x8, "hi");
 5882     overflow(0x6, "vs");
 5883     no_overflow(0x7, "vc");
 5884   %}
 5885 %}
 5886 
 5887 // Special operand allowing long args to int ops to be truncated for free
 5888 
 5889 operand iRegL2I(iRegL reg) %{
 5890 
 5891   op_cost(0);
 5892 
 5893   match(ConvL2I reg);
 5894 
 5895   format %{ "l2i($reg)" %}
 5896 
 5897   interface(REG_INTER)
 5898 %}
 5899 
 5900 operand iRegL2P(iRegL reg) %{
 5901 
 5902   op_cost(0);
 5903 
 5904   match(CastX2P reg);
 5905 
 5906   format %{ "l2p($reg)" %}
 5907 
 5908   interface(REG_INTER)
 5909 %}
 5910 
 5911 opclass vmem2(indirect, indIndex, indOffI2, indOffL2);
 5912 opclass vmem4(indirect, indIndex, indOffI4, indOffL4);
 5913 opclass vmem8(indirect, indIndex, indOffI8, indOffL8);
 5914 opclass vmem16(indirect, indIndex, indOffI16, indOffL16);
 5915 
 5916 //----------OPERAND CLASSES----------------------------------------------------
 5917 // Operand Classes are groups of operands that are used as to simplify
 5918 // instruction definitions by not requiring the AD writer to specify
 5919 // separate instructions for every form of operand when the
 5920 // instruction accepts multiple operand types with the same basic
 5921 // encoding and format. The classic case of this is memory operands.
 5922 
 5923 // memory is used to define read/write location for load/store
 5924 // instruction defs. we can turn a memory op into an Address
 5925 
 5926 opclass memory1(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI1, indOffL1,
 5927                 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
 5928 
 5929 opclass memory2(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI2, indOffL2,
 5930                 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
 5931 
 5932 opclass memory4(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI4, indOffL4,
 5933                 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
 5934 
 5935 opclass memory8(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI8, indOffL8,
 5936                 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
 5937 
 5938 // All of the memory operands. For the pipeline description.
 5939 opclass memory(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex,
 5940                indOffI1, indOffL1, indOffI2, indOffL2, indOffI4, indOffL4, indOffI8, indOffL8,
 5941                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
 5942 
 5943 
 5944 // iRegIorL2I is used for src inputs in rules for 32 bit int (I)
 5945 // operations. it allows the src to be either an iRegI or a (ConvL2I
 5946 // iRegL). in the latter case the l2i normally planted for a ConvL2I
 5947 // can be elided because the 32-bit instruction will just employ the
 5948 // lower 32 bits anyway.
 5949 //
 5950 // n.b. this does not elide all L2I conversions. if the truncated
 5951 // value is consumed by more than one operation then the ConvL2I
 5952 // cannot be bundled into the consuming nodes so an l2i gets planted
 5953 // (actually a movw $dst $src) and the downstream instructions consume
 5954 // the result of the l2i as an iRegI input. That's a shame since the
 5955 // movw is actually redundant but its not too costly.
 5956 
 5957 opclass iRegIorL2I(iRegI, iRegL2I);
 5958 opclass iRegPorL2P(iRegP, iRegL2P);
 5959 
 5960 //----------PIPELINE-----------------------------------------------------------
 5961 // Rules which define the behavior of the target architectures pipeline.
 5962 
 5963 // For specific pipelines, eg A53, define the stages of that pipeline
 5964 //pipe_desc(ISS, EX1, EX2, WR);
 5965 #define ISS S0
 5966 #define EX1 S1
 5967 #define EX2 S2
 5968 #define WR  S3
 5969 
 5970 // Integer ALU reg operation
 5971 pipeline %{
 5972 
 5973 attributes %{
 5974   // ARM instructions are of fixed length
 5975   fixed_size_instructions;        // Fixed size instructions TODO does
 5976   max_instructions_per_bundle = 4;   // A53 = 2, A57 = 4
 5977   // ARM instructions come in 32-bit word units
 5978   instruction_unit_size = 4;         // An instruction is 4 bytes long
 5979   instruction_fetch_unit_size = 64;  // The processor fetches one line
 5980   instruction_fetch_units = 1;       // of 64 bytes
 5981 
 5982   // List of nop instructions
 5983   nops( MachNop );
 5984 %}
 5985 
 5986 // We don't use an actual pipeline model so don't care about resources
 5987 // or description. we do use pipeline classes to introduce fixed
 5988 // latencies
 5989 
 5990 //----------RESOURCES----------------------------------------------------------
 5991 // Resources are the functional units available to the machine
 5992 
 5993 resources( INS0, INS1, INS01 = INS0 | INS1,
 5994            ALU0, ALU1, ALU = ALU0 | ALU1,
 5995            MAC,
 5996            DIV,
 5997            BRANCH,
 5998            LDST,
 5999            NEON_FP);
 6000 
 6001 //----------PIPELINE DESCRIPTION-----------------------------------------------
 6002 // Pipeline Description specifies the stages in the machine's pipeline
 6003 
 6004 // Define the pipeline as a generic 6 stage pipeline
 6005 pipe_desc(S0, S1, S2, S3, S4, S5);
 6006 
 6007 //----------PIPELINE CLASSES---------------------------------------------------
 6008 // Pipeline Classes describe the stages in which input and output are
 6009 // referenced by the hardware pipeline.
 6010 
 6011 pipe_class fp_dop_reg_reg_s(vRegF dst, vRegF src1, vRegF src2)
 6012 %{
 6013   single_instruction;
 6014   src1   : S1(read);
 6015   src2   : S2(read);
 6016   dst    : S5(write);
 6017   INS01  : ISS;
 6018   NEON_FP : S5;
 6019 %}
 6020 
 6021 pipe_class fp_dop_reg_reg_d(vRegD dst, vRegD src1, vRegD src2)
 6022 %{
 6023   single_instruction;
 6024   src1   : S1(read);
 6025   src2   : S2(read);
 6026   dst    : S5(write);
 6027   INS01  : ISS;
 6028   NEON_FP : S5;
 6029 %}
 6030 
 6031 pipe_class fp_uop_s(vRegF dst, vRegF src)
 6032 %{
 6033   single_instruction;
 6034   src    : S1(read);
 6035   dst    : S5(write);
 6036   INS01  : ISS;
 6037   NEON_FP : S5;
 6038 %}
 6039 
 6040 pipe_class fp_uop_d(vRegD dst, vRegD src)
 6041 %{
 6042   single_instruction;
 6043   src    : S1(read);
 6044   dst    : S5(write);
 6045   INS01  : ISS;
 6046   NEON_FP : S5;
 6047 %}
 6048 
 6049 pipe_class fp_d2f(vRegF dst, vRegD src)
 6050 %{
 6051   single_instruction;
 6052   src    : S1(read);
 6053   dst    : S5(write);
 6054   INS01  : ISS;
 6055   NEON_FP : S5;
 6056 %}
 6057 
 6058 pipe_class fp_f2d(vRegD dst, vRegF src)
 6059 %{
 6060   single_instruction;
 6061   src    : S1(read);
 6062   dst    : S5(write);
 6063   INS01  : ISS;
 6064   NEON_FP : S5;
 6065 %}
 6066 
 6067 pipe_class fp_f2i(iRegINoSp dst, vRegF src)
 6068 %{
 6069   single_instruction;
 6070   src    : S1(read);
 6071   dst    : S5(write);
 6072   INS01  : ISS;
 6073   NEON_FP : S5;
 6074 %}
 6075 
 6076 pipe_class fp_f2l(iRegLNoSp dst, vRegF src)
 6077 %{
 6078   single_instruction;
 6079   src    : S1(read);
 6080   dst    : S5(write);
 6081   INS01  : ISS;
 6082   NEON_FP : S5;
 6083 %}
 6084 
 6085 pipe_class fp_i2f(vRegF dst, iRegIorL2I src)
 6086 %{
 6087   single_instruction;
 6088   src    : S1(read);
 6089   dst    : S5(write);
 6090   INS01  : ISS;
 6091   NEON_FP : S5;
 6092 %}
 6093 
 6094 pipe_class fp_l2f(vRegF dst, iRegL src)
 6095 %{
 6096   single_instruction;
 6097   src    : S1(read);
 6098   dst    : S5(write);
 6099   INS01  : ISS;
 6100   NEON_FP : S5;
 6101 %}
 6102 
 6103 pipe_class fp_d2i(iRegINoSp dst, vRegD src)
 6104 %{
 6105   single_instruction;
 6106   src    : S1(read);
 6107   dst    : S5(write);
 6108   INS01  : ISS;
 6109   NEON_FP : S5;
 6110 %}
 6111 
 6112 pipe_class fp_d2l(iRegLNoSp dst, vRegD src)
 6113 %{
 6114   single_instruction;
 6115   src    : S1(read);
 6116   dst    : S5(write);
 6117   INS01  : ISS;
 6118   NEON_FP : S5;
 6119 %}
 6120 
 6121 pipe_class fp_i2d(vRegD dst, iRegIorL2I src)
 6122 %{
 6123   single_instruction;
 6124   src    : S1(read);
 6125   dst    : S5(write);
 6126   INS01  : ISS;
 6127   NEON_FP : S5;
 6128 %}
 6129 
 6130 pipe_class fp_l2d(vRegD dst, iRegIorL2I src)
 6131 %{
 6132   single_instruction;
 6133   src    : S1(read);
 6134   dst    : S5(write);
 6135   INS01  : ISS;
 6136   NEON_FP : S5;
 6137 %}
 6138 
 6139 pipe_class fp_div_s(vRegF dst, vRegF src1, vRegF src2)
 6140 %{
 6141   single_instruction;
 6142   src1   : S1(read);
 6143   src2   : S2(read);
 6144   dst    : S5(write);
 6145   INS0   : ISS;
 6146   NEON_FP : S5;
 6147 %}
 6148 
 6149 pipe_class fp_div_d(vRegD dst, vRegD src1, vRegD src2)
 6150 %{
 6151   single_instruction;
 6152   src1   : S1(read);
 6153   src2   : S2(read);
 6154   dst    : S5(write);
 6155   INS0   : ISS;
 6156   NEON_FP : S5;
 6157 %}
 6158 
 6159 pipe_class fp_cond_reg_reg_s(vRegF dst, vRegF src1, vRegF src2, rFlagsReg cr)
 6160 %{
 6161   single_instruction;
 6162   cr     : S1(read);
 6163   src1   : S1(read);
 6164   src2   : S1(read);
 6165   dst    : S3(write);
 6166   INS01  : ISS;
 6167   NEON_FP : S3;
 6168 %}
 6169 
 6170 pipe_class fp_cond_reg_reg_d(vRegD dst, vRegD src1, vRegD src2, rFlagsReg cr)
 6171 %{
 6172   single_instruction;
 6173   cr     : S1(read);
 6174   src1   : S1(read);
 6175   src2   : S1(read);
 6176   dst    : S3(write);
 6177   INS01  : ISS;
 6178   NEON_FP : S3;
 6179 %}
 6180 
 6181 pipe_class fp_imm_s(vRegF dst)
 6182 %{
 6183   single_instruction;
 6184   dst    : S3(write);
 6185   INS01  : ISS;
 6186   NEON_FP : S3;
 6187 %}
 6188 
 6189 pipe_class fp_imm_d(vRegD dst)
 6190 %{
 6191   single_instruction;
 6192   dst    : S3(write);
 6193   INS01  : ISS;
 6194   NEON_FP : S3;
 6195 %}
 6196 
 6197 pipe_class fp_load_constant_s(vRegF dst)
 6198 %{
 6199   single_instruction;
 6200   dst    : S4(write);
 6201   INS01  : ISS;
 6202   NEON_FP : S4;
 6203 %}
 6204 
 6205 pipe_class fp_load_constant_d(vRegD dst)
 6206 %{
 6207   single_instruction;
 6208   dst    : S4(write);
 6209   INS01  : ISS;
 6210   NEON_FP : S4;
 6211 %}
 6212 
 6213 //------- Integer ALU operations --------------------------
 6214 
 6215 // Integer ALU reg-reg operation
 6216 // Operands needed in EX1, result generated in EX2
 6217 // Eg.  ADD     x0, x1, x2
 6218 pipe_class ialu_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 6219 %{
 6220   single_instruction;
 6221   dst    : EX2(write);
 6222   src1   : EX1(read);
 6223   src2   : EX1(read);
 6224   INS01  : ISS; // Dual issue as instruction 0 or 1
 6225   ALU    : EX2;
 6226 %}
 6227 
 6228 // Integer ALU reg-reg operation with constant shift
 6229 // Shifted register must be available in LATE_ISS instead of EX1
 6230 // Eg.  ADD     x0, x1, x2, LSL #2
 6231 pipe_class ialu_reg_reg_shift(iRegI dst, iRegI src1, iRegI src2, immI shift)
 6232 %{
 6233   single_instruction;
 6234   dst    : EX2(write);
 6235   src1   : EX1(read);
 6236   src2   : ISS(read);
 6237   INS01  : ISS;
 6238   ALU    : EX2;
 6239 %}
 6240 
 6241 // Integer ALU reg operation with constant shift
 6242 // Eg.  LSL     x0, x1, #shift
 6243 pipe_class ialu_reg_shift(iRegI dst, iRegI src1)
 6244 %{
 6245   single_instruction;
 6246   dst    : EX2(write);
 6247   src1   : ISS(read);
 6248   INS01  : ISS;
 6249   ALU    : EX2;
 6250 %}
 6251 
 6252 // Integer ALU reg-reg operation with variable shift
 6253 // Both operands must be available in LATE_ISS instead of EX1
 6254 // Result is available in EX1 instead of EX2
 6255 // Eg.  LSLV    x0, x1, x2
 6256 pipe_class ialu_reg_reg_vshift(iRegI dst, iRegI src1, iRegI src2)
 6257 %{
 6258   single_instruction;
 6259   dst    : EX1(write);
 6260   src1   : ISS(read);
 6261   src2   : ISS(read);
 6262   INS01  : ISS;
 6263   ALU    : EX1;
 6264 %}
 6265 
 6266 // Integer ALU reg-reg operation with extract
 6267 // As for _vshift above, but result generated in EX2
 6268 // Eg.  EXTR    x0, x1, x2, #N
 6269 pipe_class ialu_reg_reg_extr(iRegI dst, iRegI src1, iRegI src2)
 6270 %{
 6271   single_instruction;
 6272   dst    : EX2(write);
 6273   src1   : ISS(read);
 6274   src2   : ISS(read);
 6275   INS1   : ISS; // Can only dual issue as Instruction 1
 6276   ALU    : EX1;
 6277 %}
 6278 
 6279 // Integer ALU reg operation
 6280 // Eg.  NEG     x0, x1
 6281 pipe_class ialu_reg(iRegI dst, iRegI src)
 6282 %{
 6283   single_instruction;
 6284   dst    : EX2(write);
 6285   src    : EX1(read);
 6286   INS01  : ISS;
 6287   ALU    : EX2;
 6288 %}
 6289 
 6290 // Integer ALU reg mmediate operation
 6291 // Eg.  ADD     x0, x1, #N
 6292 pipe_class ialu_reg_imm(iRegI dst, iRegI src1)
 6293 %{
 6294   single_instruction;
 6295   dst    : EX2(write);
 6296   src1   : EX1(read);
 6297   INS01  : ISS;
 6298   ALU    : EX2;
 6299 %}
 6300 
 6301 // Integer ALU immediate operation (no source operands)
 6302 // Eg.  MOV     x0, #N
 6303 pipe_class ialu_imm(iRegI dst)
 6304 %{
 6305   single_instruction;
 6306   dst    : EX1(write);
 6307   INS01  : ISS;
 6308   ALU    : EX1;
 6309 %}
 6310 
 6311 //------- Compare operation -------------------------------
 6312 
 6313 // Compare reg-reg
 6314 // Eg.  CMP     x0, x1
 6315 pipe_class icmp_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
 6316 %{
 6317   single_instruction;
 6318 //  fixed_latency(16);
 6319   cr     : EX2(write);
 6320   op1    : EX1(read);
 6321   op2    : EX1(read);
 6322   INS01  : ISS;
 6323   ALU    : EX2;
 6324 %}
 6325 
 6326 // Compare reg-reg
 6327 // Eg.  CMP     x0, #N
 6328 pipe_class icmp_reg_imm(rFlagsReg cr, iRegI op1)
 6329 %{
 6330   single_instruction;
 6331 //  fixed_latency(16);
 6332   cr     : EX2(write);
 6333   op1    : EX1(read);
 6334   INS01  : ISS;
 6335   ALU    : EX2;
 6336 %}
 6337 
 6338 //------- Conditional instructions ------------------------
 6339 
 6340 // Conditional no operands
 6341 // Eg.  CSINC   x0, zr, zr, <cond>
 6342 pipe_class icond_none(iRegI dst, rFlagsReg cr)
 6343 %{
 6344   single_instruction;
 6345   cr     : EX1(read);
 6346   dst    : EX2(write);
 6347   INS01  : ISS;
 6348   ALU    : EX2;
 6349 %}
 6350 
 6351 // Conditional 2 operand
 6352 // EG.  CSEL    X0, X1, X2, <cond>
 6353 pipe_class icond_reg_reg(iRegI dst, iRegI src1, iRegI src2, rFlagsReg cr)
 6354 %{
 6355   single_instruction;
 6356   cr     : EX1(read);
 6357   src1   : EX1(read);
 6358   src2   : EX1(read);
 6359   dst    : EX2(write);
 6360   INS01  : ISS;
 6361   ALU    : EX2;
 6362 %}
 6363 
 6364 // Conditional 2 operand
 6365 // EG.  CSEL    X0, X1, X2, <cond>
 6366 pipe_class icond_reg(iRegI dst, iRegI src, rFlagsReg cr)
 6367 %{
 6368   single_instruction;
 6369   cr     : EX1(read);
 6370   src    : EX1(read);
 6371   dst    : EX2(write);
 6372   INS01  : ISS;
 6373   ALU    : EX2;
 6374 %}
 6375 
 6376 //------- Multiply pipeline operations --------------------
 6377 
 6378 // Multiply reg-reg
 6379 // Eg.  MUL     w0, w1, w2
 6380 pipe_class imul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 6381 %{
 6382   single_instruction;
 6383   dst    : WR(write);
 6384   src1   : ISS(read);
 6385   src2   : ISS(read);
 6386   INS01  : ISS;
 6387   MAC    : WR;
 6388 %}
 6389 
 6390 // Multiply accumulate
 6391 // Eg.  MADD    w0, w1, w2, w3
 6392 pipe_class imac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
 6393 %{
 6394   single_instruction;
 6395   dst    : WR(write);
 6396   src1   : ISS(read);
 6397   src2   : ISS(read);
 6398   src3   : ISS(read);
 6399   INS01  : ISS;
 6400   MAC    : WR;
 6401 %}
 6402 
 6403 // Eg.  MUL     w0, w1, w2
 6404 pipe_class lmul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 6405 %{
 6406   single_instruction;
 6407   fixed_latency(3); // Maximum latency for 64 bit mul
 6408   dst    : WR(write);
 6409   src1   : ISS(read);
 6410   src2   : ISS(read);
 6411   INS01  : ISS;
 6412   MAC    : WR;
 6413 %}
 6414 
 6415 // Multiply accumulate
 6416 // Eg.  MADD    w0, w1, w2, w3
 6417 pipe_class lmac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
 6418 %{
 6419   single_instruction;
 6420   fixed_latency(3); // Maximum latency for 64 bit mul
 6421   dst    : WR(write);
 6422   src1   : ISS(read);
 6423   src2   : ISS(read);
 6424   src3   : ISS(read);
 6425   INS01  : ISS;
 6426   MAC    : WR;
 6427 %}
 6428 
 6429 //------- Divide pipeline operations --------------------
 6430 
 6431 // Eg.  SDIV    w0, w1, w2
 6432 pipe_class idiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 6433 %{
 6434   single_instruction;
 6435   fixed_latency(8); // Maximum latency for 32 bit divide
 6436   dst    : WR(write);
 6437   src1   : ISS(read);
 6438   src2   : ISS(read);
 6439   INS0   : ISS; // Can only dual issue as instruction 0
 6440   DIV    : WR;
 6441 %}
 6442 
 6443 // Eg.  SDIV    x0, x1, x2
 6444 pipe_class ldiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 6445 %{
 6446   single_instruction;
 6447   fixed_latency(16); // Maximum latency for 64 bit divide
 6448   dst    : WR(write);
 6449   src1   : ISS(read);
 6450   src2   : ISS(read);
 6451   INS0   : ISS; // Can only dual issue as instruction 0
 6452   DIV    : WR;
 6453 %}
 6454 
 6455 //------- Load pipeline operations ------------------------
 6456 
 6457 // Load - prefetch
 6458 // Eg.  PFRM    <mem>
 6459 pipe_class iload_prefetch(memory mem)
 6460 %{
 6461   single_instruction;
 6462   mem    : ISS(read);
 6463   INS01  : ISS;
 6464   LDST   : WR;
 6465 %}
 6466 
 6467 // Load - reg, mem
 6468 // Eg.  LDR     x0, <mem>
 6469 pipe_class iload_reg_mem(iRegI dst, memory mem)
 6470 %{
 6471   single_instruction;
 6472   dst    : WR(write);
 6473   mem    : ISS(read);
 6474   INS01  : ISS;
 6475   LDST   : WR;
 6476 %}
 6477 
 6478 // Load - reg, reg
 6479 // Eg.  LDR     x0, [sp, x1]
 6480 pipe_class iload_reg_reg(iRegI dst, iRegI src)
 6481 %{
 6482   single_instruction;
 6483   dst    : WR(write);
 6484   src    : ISS(read);
 6485   INS01  : ISS;
 6486   LDST   : WR;
 6487 %}
 6488 
 6489 //------- Store pipeline operations -----------------------
 6490 
 6491 // Store - zr, mem
 6492 // Eg.  STR     zr, <mem>
 6493 pipe_class istore_mem(memory mem)
 6494 %{
 6495   single_instruction;
 6496   mem    : ISS(read);
 6497   INS01  : ISS;
 6498   LDST   : WR;
 6499 %}
 6500 
 6501 // Store - reg, mem
 6502 // Eg.  STR     x0, <mem>
 6503 pipe_class istore_reg_mem(iRegI src, memory mem)
 6504 %{
 6505   single_instruction;
 6506   mem    : ISS(read);
 6507   src    : EX2(read);
 6508   INS01  : ISS;
 6509   LDST   : WR;
 6510 %}
 6511 
 6512 // Store - reg, reg
 6513 // Eg. STR      x0, [sp, x1]
 6514 pipe_class istore_reg_reg(iRegI dst, iRegI src)
 6515 %{
 6516   single_instruction;
 6517   dst    : ISS(read);
 6518   src    : EX2(read);
 6519   INS01  : ISS;
 6520   LDST   : WR;
 6521 %}
 6522 
 6523 //------- Store pipeline operations -----------------------
 6524 
 6525 // Branch
 6526 pipe_class pipe_branch()
 6527 %{
 6528   single_instruction;
 6529   INS01  : ISS;
 6530   BRANCH : EX1;
 6531 %}
 6532 
 6533 // Conditional branch
 6534 pipe_class pipe_branch_cond(rFlagsReg cr)
 6535 %{
 6536   single_instruction;
 6537   cr     : EX1(read);
 6538   INS01  : ISS;
 6539   BRANCH : EX1;
 6540 %}
 6541 
 6542 // Compare & Branch
 6543 // EG.  CBZ/CBNZ
 6544 pipe_class pipe_cmp_branch(iRegI op1)
 6545 %{
 6546   single_instruction;
 6547   op1    : EX1(read);
 6548   INS01  : ISS;
 6549   BRANCH : EX1;
 6550 %}
 6551 
 6552 //------- Synchronisation operations ----------------------
 6553 
 6554 // Any operation requiring serialization.
 6555 // EG.  DMB/Atomic Ops/Load Acquire/Str Release
 6556 pipe_class pipe_serial()
 6557 %{
 6558   single_instruction;
 6559   force_serialization;
 6560   fixed_latency(16);
 6561   INS01  : ISS(2); // Cannot dual issue with any other instruction
 6562   LDST   : WR;
 6563 %}
 6564 
 6565 // Generic big/slow expanded idiom - also serialized
 6566 pipe_class pipe_slow()
 6567 %{
 6568   instruction_count(10);
 6569   multiple_bundles;
 6570   force_serialization;
 6571   fixed_latency(16);
 6572   INS01  : ISS(2); // Cannot dual issue with any other instruction
 6573   LDST   : WR;
 6574 %}
 6575 
 6576 // Empty pipeline class
 6577 pipe_class pipe_class_empty()
 6578 %{
 6579   single_instruction;
 6580   fixed_latency(0);
 6581 %}
 6582 
 6583 // Default pipeline class.
 6584 pipe_class pipe_class_default()
 6585 %{
 6586   single_instruction;
 6587   fixed_latency(2);
 6588 %}
 6589 
 6590 // Pipeline class for compares.
 6591 pipe_class pipe_class_compare()
 6592 %{
 6593   single_instruction;
 6594   fixed_latency(16);
 6595 %}
 6596 
 6597 // Pipeline class for memory operations.
 6598 pipe_class pipe_class_memory()
 6599 %{
 6600   single_instruction;
 6601   fixed_latency(16);
 6602 %}
 6603 
 6604 // Pipeline class for call.
 6605 pipe_class pipe_class_call()
 6606 %{
 6607   single_instruction;
 6608   fixed_latency(100);
 6609 %}
 6610 
 6611 // Define the class for the Nop node.
 6612 define %{
 6613    MachNop = pipe_class_empty;
 6614 %}
 6615 
 6616 %}
 6617 //----------INSTRUCTIONS-------------------------------------------------------
 6618 //
 6619 // match      -- States which machine-independent subtree may be replaced
 6620 //               by this instruction.
 6621 // ins_cost   -- The estimated cost of this instruction is used by instruction
 6622 //               selection to identify a minimum cost tree of machine
 6623 //               instructions that matches a tree of machine-independent
 6624 //               instructions.
 6625 // format     -- A string providing the disassembly for this instruction.
 6626 //               The value of an instruction's operand may be inserted
 6627 //               by referring to it with a '$' prefix.
 6628 // opcode     -- Three instruction opcodes may be provided.  These are referred
 6629 //               to within an encode class as $primary, $secondary, and $tertiary
 6630 //               rrspectively.  The primary opcode is commonly used to
 6631 //               indicate the type of machine instruction, while secondary
 6632 //               and tertiary are often used for prefix options or addressing
 6633 //               modes.
 6634 // ins_encode -- A list of encode classes with parameters. The encode class
 6635 //               name must have been defined in an 'enc_class' specification
 6636 //               in the encode section of the architecture description.
 6637 
 6638 // ============================================================================
 6639 // Memory (Load/Store) Instructions
 6640 
 6641 // Load Instructions
 6642 
 6643 // Load Byte (8 bit signed)
 6644 instruct loadB(iRegINoSp dst, memory1 mem)
 6645 %{
 6646   match(Set dst (LoadB mem));
 6647   predicate(!needs_acquiring_load(n));
 6648 
 6649   ins_cost(4 * INSN_COST);
 6650   format %{ "ldrsbw  $dst, $mem\t# byte" %}
 6651 
 6652   ins_encode(aarch64_enc_ldrsbw(dst, mem));
 6653 
 6654   ins_pipe(iload_reg_mem);
 6655 %}
 6656 
 6657 // Load Byte (8 bit signed) into long
 6658 instruct loadB2L(iRegLNoSp dst, memory1 mem)
 6659 %{
 6660   match(Set dst (ConvI2L (LoadB mem)));
 6661   predicate(!needs_acquiring_load(n->in(1)));
 6662 
 6663   ins_cost(4 * INSN_COST);
 6664   format %{ "ldrsb  $dst, $mem\t# byte" %}
 6665 
 6666   ins_encode(aarch64_enc_ldrsb(dst, mem));
 6667 
 6668   ins_pipe(iload_reg_mem);
 6669 %}
 6670 
 6671 // Load Byte (8 bit unsigned)
 6672 instruct loadUB(iRegINoSp dst, memory1 mem)
 6673 %{
 6674   match(Set dst (LoadUB mem));
 6675   predicate(!needs_acquiring_load(n));
 6676 
 6677   ins_cost(4 * INSN_COST);
 6678   format %{ "ldrbw  $dst, $mem\t# byte" %}
 6679 
 6680   ins_encode(aarch64_enc_ldrb(dst, mem));
 6681 
 6682   ins_pipe(iload_reg_mem);
 6683 %}
 6684 
 6685 // Load Byte (8 bit unsigned) into long
 6686 instruct loadUB2L(iRegLNoSp dst, memory1 mem)
 6687 %{
 6688   match(Set dst (ConvI2L (LoadUB mem)));
 6689   predicate(!needs_acquiring_load(n->in(1)));
 6690 
 6691   ins_cost(4 * INSN_COST);
 6692   format %{ "ldrb  $dst, $mem\t# byte" %}
 6693 
 6694   ins_encode(aarch64_enc_ldrb(dst, mem));
 6695 
 6696   ins_pipe(iload_reg_mem);
 6697 %}
 6698 
 6699 // Load Short (16 bit signed)
 6700 instruct loadS(iRegINoSp dst, memory2 mem)
 6701 %{
 6702   match(Set dst (LoadS mem));
 6703   predicate(!needs_acquiring_load(n));
 6704 
 6705   ins_cost(4 * INSN_COST);
 6706   format %{ "ldrshw  $dst, $mem\t# short" %}
 6707 
 6708   ins_encode(aarch64_enc_ldrshw(dst, mem));
 6709 
 6710   ins_pipe(iload_reg_mem);
 6711 %}
 6712 
 6713 // Load Short (16 bit signed) into long
 6714 instruct loadS2L(iRegLNoSp dst, memory2 mem)
 6715 %{
 6716   match(Set dst (ConvI2L (LoadS mem)));
 6717   predicate(!needs_acquiring_load(n->in(1)));
 6718 
 6719   ins_cost(4 * INSN_COST);
 6720   format %{ "ldrsh  $dst, $mem\t# short" %}
 6721 
 6722   ins_encode(aarch64_enc_ldrsh(dst, mem));
 6723 
 6724   ins_pipe(iload_reg_mem);
 6725 %}
 6726 
 6727 // Load Char (16 bit unsigned)
 6728 instruct loadUS(iRegINoSp dst, memory2 mem)
 6729 %{
 6730   match(Set dst (LoadUS mem));
 6731   predicate(!needs_acquiring_load(n));
 6732 
 6733   ins_cost(4 * INSN_COST);
 6734   format %{ "ldrh  $dst, $mem\t# short" %}
 6735 
 6736   ins_encode(aarch64_enc_ldrh(dst, mem));
 6737 
 6738   ins_pipe(iload_reg_mem);
 6739 %}
 6740 
 6741 // Load Short/Char (16 bit unsigned) into long
 6742 instruct loadUS2L(iRegLNoSp dst, memory2 mem)
 6743 %{
 6744   match(Set dst (ConvI2L (LoadUS mem)));
 6745   predicate(!needs_acquiring_load(n->in(1)));
 6746 
 6747   ins_cost(4 * INSN_COST);
 6748   format %{ "ldrh  $dst, $mem\t# short" %}
 6749 
 6750   ins_encode(aarch64_enc_ldrh(dst, mem));
 6751 
 6752   ins_pipe(iload_reg_mem);
 6753 %}
 6754 
 6755 // Load Integer (32 bit signed)
 6756 instruct loadI(iRegINoSp dst, memory4 mem)
 6757 %{
 6758   match(Set dst (LoadI mem));
 6759   predicate(!needs_acquiring_load(n));
 6760 
 6761   ins_cost(4 * INSN_COST);
 6762   format %{ "ldrw  $dst, $mem\t# int" %}
 6763 
 6764   ins_encode(aarch64_enc_ldrw(dst, mem));
 6765 
 6766   ins_pipe(iload_reg_mem);
 6767 %}
 6768 
 6769 // Load Integer (32 bit signed) into long
 6770 instruct loadI2L(iRegLNoSp dst, memory4 mem)
 6771 %{
 6772   match(Set dst (ConvI2L (LoadI mem)));
 6773   predicate(!needs_acquiring_load(n->in(1)));
 6774 
 6775   ins_cost(4 * INSN_COST);
 6776   format %{ "ldrsw  $dst, $mem\t# int" %}
 6777 
 6778   ins_encode(aarch64_enc_ldrsw(dst, mem));
 6779 
 6780   ins_pipe(iload_reg_mem);
 6781 %}
 6782 
 6783 // Load Integer (32 bit unsigned) into long
 6784 instruct loadUI2L(iRegLNoSp dst, memory4 mem, immL_32bits mask)
 6785 %{
 6786   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
 6787   predicate(!needs_acquiring_load(n->in(1)->in(1)->as_Load()));
 6788 
 6789   ins_cost(4 * INSN_COST);
 6790   format %{ "ldrw  $dst, $mem\t# int" %}
 6791 
 6792   ins_encode(aarch64_enc_ldrw(dst, mem));
 6793 
 6794   ins_pipe(iload_reg_mem);
 6795 %}
 6796 
 6797 // Load Long (64 bit signed)
 6798 instruct loadL(iRegLNoSp dst, memory8 mem)
 6799 %{
 6800   match(Set dst (LoadL mem));
 6801   predicate(!needs_acquiring_load(n));
 6802 
 6803   ins_cost(4 * INSN_COST);
 6804   format %{ "ldr  $dst, $mem\t# int" %}
 6805 
 6806   ins_encode(aarch64_enc_ldr(dst, mem));
 6807 
 6808   ins_pipe(iload_reg_mem);
 6809 %}
 6810 
 6811 // Load Range
 6812 instruct loadRange(iRegINoSp dst, memory4 mem)
 6813 %{
 6814   match(Set dst (LoadRange mem));
 6815 
 6816   ins_cost(4 * INSN_COST);
 6817   format %{ "ldrw  $dst, $mem\t# range" %}
 6818 
 6819   ins_encode(aarch64_enc_ldrw(dst, mem));
 6820 
 6821   ins_pipe(iload_reg_mem);
 6822 %}
 6823 
 6824 // Load Pointer
 6825 instruct loadP(iRegPNoSp dst, memory8 mem)
 6826 %{
 6827   match(Set dst (LoadP mem));
 6828   predicate(!needs_acquiring_load(n) && (n->as_Load()->barrier_data() == 0));
 6829 
 6830   ins_cost(4 * INSN_COST);
 6831   format %{ "ldr  $dst, $mem\t# ptr" %}
 6832 
 6833   ins_encode(aarch64_enc_ldr(dst, mem));
 6834 
 6835   ins_pipe(iload_reg_mem);
 6836 %}
 6837 
 6838 // Load Compressed Pointer
 6839 instruct loadN(iRegNNoSp dst, memory4 mem)
 6840 %{
 6841   match(Set dst (LoadN mem));
 6842   predicate(!needs_acquiring_load(n) && n->as_Load()->barrier_data() == 0);
 6843 
 6844   ins_cost(4 * INSN_COST);
 6845   format %{ "ldrw  $dst, $mem\t# compressed ptr" %}
 6846 
 6847   ins_encode(aarch64_enc_ldrw(dst, mem));
 6848 
 6849   ins_pipe(iload_reg_mem);
 6850 %}
 6851 
 6852 // Load Klass Pointer
 6853 instruct loadKlass(iRegPNoSp dst, memory8 mem)
 6854 %{
 6855   match(Set dst (LoadKlass mem));
 6856   predicate(!needs_acquiring_load(n));
 6857 
 6858   ins_cost(4 * INSN_COST);
 6859   format %{ "ldr  $dst, $mem\t# class" %}
 6860 
 6861   ins_encode(aarch64_enc_ldr(dst, mem));
 6862 
 6863   ins_pipe(iload_reg_mem);
 6864 %}
 6865 
 6866 // Load Narrow Klass Pointer
 6867 instruct loadNKlass(iRegNNoSp dst, memory4 mem)
 6868 %{
 6869   match(Set dst (LoadNKlass mem));
 6870   predicate(!needs_acquiring_load(n) && !UseCompactObjectHeaders);
 6871 
 6872   ins_cost(4 * INSN_COST);
 6873   format %{ "ldrw  $dst, $mem\t# compressed class ptr" %}
 6874 
 6875   ins_encode(aarch64_enc_ldrw(dst, mem));
 6876 
 6877   ins_pipe(iload_reg_mem);
 6878 %}
 6879 
 6880 instruct loadNKlassCompactHeaders(iRegNNoSp dst, memory4 mem)
 6881 %{
 6882   match(Set dst (LoadNKlass mem));
 6883   predicate(!needs_acquiring_load(n) && UseCompactObjectHeaders);
 6884 
 6885   ins_cost(4 * INSN_COST);
 6886   format %{
 6887     "ldrw  $dst, $mem\t# compressed class ptr, shifted\n\t"
 6888     "lsrw  $dst, $dst, markWord::klass_shift_at_offset"
 6889   %}
 6890   ins_encode %{
 6891     // inlined aarch64_enc_ldrw
 6892     loadStore(masm, &MacroAssembler::ldrw, $dst$$Register, $mem->opcode(),
 6893               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 6894     __ lsrw($dst$$Register, $dst$$Register, markWord::klass_shift_at_offset);
 6895   %}
 6896   ins_pipe(iload_reg_mem);
 6897 %}
 6898 
 6899 // Load Float
 6900 instruct loadF(vRegF dst, memory4 mem)
 6901 %{
 6902   match(Set dst (LoadF mem));
 6903   predicate(!needs_acquiring_load(n));
 6904 
 6905   ins_cost(4 * INSN_COST);
 6906   format %{ "ldrs  $dst, $mem\t# float" %}
 6907 
 6908   ins_encode( aarch64_enc_ldrs(dst, mem) );
 6909 
 6910   ins_pipe(pipe_class_memory);
 6911 %}
 6912 
 6913 // Load Double
 6914 instruct loadD(vRegD dst, memory8 mem)
 6915 %{
 6916   match(Set dst (LoadD mem));
 6917   predicate(!needs_acquiring_load(n));
 6918 
 6919   ins_cost(4 * INSN_COST);
 6920   format %{ "ldrd  $dst, $mem\t# double" %}
 6921 
 6922   ins_encode( aarch64_enc_ldrd(dst, mem) );
 6923 
 6924   ins_pipe(pipe_class_memory);
 6925 %}
 6926 
 6927 
 6928 // Load Int Constant
 6929 instruct loadConI(iRegINoSp dst, immI src)
 6930 %{
 6931   match(Set dst src);
 6932 
 6933   ins_cost(INSN_COST);
 6934   format %{ "mov $dst, $src\t# int" %}
 6935 
 6936   ins_encode( aarch64_enc_movw_imm(dst, src) );
 6937 
 6938   ins_pipe(ialu_imm);
 6939 %}
 6940 
 6941 // Load Long Constant
 6942 instruct loadConL(iRegLNoSp dst, immL src)
 6943 %{
 6944   match(Set dst src);
 6945 
 6946   ins_cost(INSN_COST);
 6947   format %{ "mov $dst, $src\t# long" %}
 6948 
 6949   ins_encode( aarch64_enc_mov_imm(dst, src) );
 6950 
 6951   ins_pipe(ialu_imm);
 6952 %}
 6953 
 6954 // Load Pointer Constant
 6955 
 6956 instruct loadConP(iRegPNoSp dst, immP con)
 6957 %{
 6958   match(Set dst con);
 6959 
 6960   ins_cost(INSN_COST * 4);
 6961   format %{
 6962     "mov  $dst, $con\t# ptr"
 6963   %}
 6964 
 6965   ins_encode(aarch64_enc_mov_p(dst, con));
 6966 
 6967   ins_pipe(ialu_imm);
 6968 %}
 6969 
 6970 // Load Null Pointer Constant
 6971 
 6972 instruct loadConP0(iRegPNoSp dst, immP0 con)
 6973 %{
 6974   match(Set dst con);
 6975 
 6976   ins_cost(INSN_COST);
 6977   format %{ "mov  $dst, $con\t# nullptr ptr" %}
 6978 
 6979   ins_encode(aarch64_enc_mov_p0(dst, con));
 6980 
 6981   ins_pipe(ialu_imm);
 6982 %}
 6983 
 6984 // Load Pointer Constant One
 6985 
 6986 instruct loadConP1(iRegPNoSp dst, immP_1 con)
 6987 %{
 6988   match(Set dst con);
 6989 
 6990   ins_cost(INSN_COST);
 6991   format %{ "mov  $dst, $con\t# nullptr ptr" %}
 6992 
 6993   ins_encode(aarch64_enc_mov_p1(dst, con));
 6994 
 6995   ins_pipe(ialu_imm);
 6996 %}
 6997 
 6998 // Load Narrow Pointer Constant
 6999 
 7000 instruct loadConN(iRegNNoSp dst, immN con)
 7001 %{
 7002   match(Set dst con);
 7003 
 7004   ins_cost(INSN_COST * 4);
 7005   format %{ "mov  $dst, $con\t# compressed ptr" %}
 7006 
 7007   ins_encode(aarch64_enc_mov_n(dst, con));
 7008 
 7009   ins_pipe(ialu_imm);
 7010 %}
 7011 
 7012 // Load Narrow Null Pointer Constant
 7013 
 7014 instruct loadConN0(iRegNNoSp dst, immN0 con)
 7015 %{
 7016   match(Set dst con);
 7017 
 7018   ins_cost(INSN_COST);
 7019   format %{ "mov  $dst, $con\t# compressed nullptr ptr" %}
 7020 
 7021   ins_encode(aarch64_enc_mov_n0(dst, con));
 7022 
 7023   ins_pipe(ialu_imm);
 7024 %}
 7025 
 7026 // Load Narrow Klass Constant
 7027 
 7028 instruct loadConNKlass(iRegNNoSp dst, immNKlass con)
 7029 %{
 7030   match(Set dst con);
 7031 
 7032   ins_cost(INSN_COST);
 7033   format %{ "mov  $dst, $con\t# compressed klass ptr" %}
 7034 
 7035   ins_encode(aarch64_enc_mov_nk(dst, con));
 7036 
 7037   ins_pipe(ialu_imm);
 7038 %}
 7039 
 7040 // Load Packed Float Constant
 7041 
 7042 instruct loadConF_packed(vRegF dst, immFPacked con) %{
 7043   match(Set dst con);
 7044   ins_cost(INSN_COST * 4);
 7045   format %{ "fmovs  $dst, $con"%}
 7046   ins_encode %{
 7047     __ fmovs(as_FloatRegister($dst$$reg), (double)$con$$constant);
 7048   %}
 7049 
 7050   ins_pipe(fp_imm_s);
 7051 %}
 7052 
 7053 // Load Float Constant
 7054 
 7055 instruct loadConF(vRegF dst, immF con) %{
 7056   match(Set dst con);
 7057 
 7058   ins_cost(INSN_COST * 4);
 7059 
 7060   format %{
 7061     "ldrs $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
 7062   %}
 7063 
 7064   ins_encode %{
 7065     __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
 7066   %}
 7067 
 7068   ins_pipe(fp_load_constant_s);
 7069 %}
 7070 
 7071 // Load Packed Double Constant
 7072 
 7073 instruct loadConD_packed(vRegD dst, immDPacked con) %{
 7074   match(Set dst con);
 7075   ins_cost(INSN_COST);
 7076   format %{ "fmovd  $dst, $con"%}
 7077   ins_encode %{
 7078     __ fmovd(as_FloatRegister($dst$$reg), $con$$constant);
 7079   %}
 7080 
 7081   ins_pipe(fp_imm_d);
 7082 %}
 7083 
 7084 // Load Double Constant
 7085 
 7086 instruct loadConD(vRegD dst, immD con) %{
 7087   match(Set dst con);
 7088 
 7089   ins_cost(INSN_COST * 5);
 7090   format %{
 7091     "ldrd $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
 7092   %}
 7093 
 7094   ins_encode %{
 7095     __ ldrd(as_FloatRegister($dst$$reg), $constantaddress($con));
 7096   %}
 7097 
 7098   ins_pipe(fp_load_constant_d);
 7099 %}
 7100 
 7101 // Load Half Float Constant
 7102 // The "ldr" instruction loads a 32-bit word from the constant pool into a
 7103 // 32-bit register but only the bottom half will be populated and the top
 7104 // 16 bits are zero.
 7105 instruct loadConH(vRegF dst, immH con) %{
 7106   match(Set dst con);
 7107   format %{
 7108     "ldrs $dst, [$constantaddress]\t# load from constant table: half float=$con\n\t"
 7109   %}
 7110   ins_encode %{
 7111     __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
 7112   %}
 7113   ins_pipe(fp_load_constant_s);
 7114 %}
 7115 
 7116 // Store Instructions
 7117 
 7118 // Store Byte
 7119 instruct storeB(iRegIorL2I src, memory1 mem)
 7120 %{
 7121   match(Set mem (StoreB mem src));
 7122   predicate(!needs_releasing_store(n));
 7123 
 7124   ins_cost(INSN_COST);
 7125   format %{ "strb  $src, $mem\t# byte" %}
 7126 
 7127   ins_encode(aarch64_enc_strb(src, mem));
 7128 
 7129   ins_pipe(istore_reg_mem);
 7130 %}
 7131 
 7132 
 7133 instruct storeimmB0(immI0 zero, memory1 mem)
 7134 %{
 7135   match(Set mem (StoreB mem zero));
 7136   predicate(!needs_releasing_store(n));
 7137 
 7138   ins_cost(INSN_COST);
 7139   format %{ "strb rscractch2, $mem\t# byte" %}
 7140 
 7141   ins_encode(aarch64_enc_strb0(mem));
 7142 
 7143   ins_pipe(istore_mem);
 7144 %}
 7145 
 7146 // Store Char/Short
 7147 instruct storeC(iRegIorL2I src, memory2 mem)
 7148 %{
 7149   match(Set mem (StoreC mem src));
 7150   predicate(!needs_releasing_store(n));
 7151 
 7152   ins_cost(INSN_COST);
 7153   format %{ "strh  $src, $mem\t# short" %}
 7154 
 7155   ins_encode(aarch64_enc_strh(src, mem));
 7156 
 7157   ins_pipe(istore_reg_mem);
 7158 %}
 7159 
 7160 instruct storeimmC0(immI0 zero, memory2 mem)
 7161 %{
 7162   match(Set mem (StoreC mem zero));
 7163   predicate(!needs_releasing_store(n));
 7164 
 7165   ins_cost(INSN_COST);
 7166   format %{ "strh  zr, $mem\t# short" %}
 7167 
 7168   ins_encode(aarch64_enc_strh0(mem));
 7169 
 7170   ins_pipe(istore_mem);
 7171 %}
 7172 
 7173 // Store Integer
 7174 
 7175 instruct storeI(iRegIorL2I src, memory4 mem)
 7176 %{
 7177   match(Set mem(StoreI mem src));
 7178   predicate(!needs_releasing_store(n));
 7179 
 7180   ins_cost(INSN_COST);
 7181   format %{ "strw  $src, $mem\t# int" %}
 7182 
 7183   ins_encode(aarch64_enc_strw(src, mem));
 7184 
 7185   ins_pipe(istore_reg_mem);
 7186 %}
 7187 
 7188 instruct storeimmI0(immI0 zero, memory4 mem)
 7189 %{
 7190   match(Set mem(StoreI mem zero));
 7191   predicate(!needs_releasing_store(n));
 7192 
 7193   ins_cost(INSN_COST);
 7194   format %{ "strw  zr, $mem\t# int" %}
 7195 
 7196   ins_encode(aarch64_enc_strw0(mem));
 7197 
 7198   ins_pipe(istore_mem);
 7199 %}
 7200 
 7201 // Store Long (64 bit signed)
 7202 instruct storeL(iRegL src, memory8 mem)
 7203 %{
 7204   match(Set mem (StoreL mem src));
 7205   predicate(!needs_releasing_store(n));
 7206 
 7207   ins_cost(INSN_COST);
 7208   format %{ "str  $src, $mem\t# int" %}
 7209 
 7210   ins_encode(aarch64_enc_str(src, mem));
 7211 
 7212   ins_pipe(istore_reg_mem);
 7213 %}
 7214 
 7215 // Store Long (64 bit signed)
 7216 instruct storeimmL0(immL0 zero, memory8 mem)
 7217 %{
 7218   match(Set mem (StoreL mem zero));
 7219   predicate(!needs_releasing_store(n));
 7220 
 7221   ins_cost(INSN_COST);
 7222   format %{ "str  zr, $mem\t# int" %}
 7223 
 7224   ins_encode(aarch64_enc_str0(mem));
 7225 
 7226   ins_pipe(istore_mem);
 7227 %}
 7228 
 7229 // Store Pointer
 7230 instruct storeP(iRegP src, memory8 mem)
 7231 %{
 7232   match(Set mem (StoreP mem src));
 7233   predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
 7234 
 7235   ins_cost(INSN_COST);
 7236   format %{ "str  $src, $mem\t# ptr" %}
 7237 
 7238   ins_encode(aarch64_enc_str(src, mem));
 7239 
 7240   ins_pipe(istore_reg_mem);
 7241 %}
 7242 
 7243 // Store Pointer
 7244 instruct storeimmP0(immP0 zero, memory8 mem)
 7245 %{
 7246   match(Set mem (StoreP mem zero));
 7247   predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
 7248 
 7249   ins_cost(INSN_COST);
 7250   format %{ "str zr, $mem\t# ptr" %}
 7251 
 7252   ins_encode(aarch64_enc_str0(mem));
 7253 
 7254   ins_pipe(istore_mem);
 7255 %}
 7256 
 7257 // Store Compressed Pointer
 7258 instruct storeN(iRegN src, memory4 mem)
 7259 %{
 7260   match(Set mem (StoreN mem src));
 7261   predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
 7262 
 7263   ins_cost(INSN_COST);
 7264   format %{ "strw  $src, $mem\t# compressed ptr" %}
 7265 
 7266   ins_encode(aarch64_enc_strw(src, mem));
 7267 
 7268   ins_pipe(istore_reg_mem);
 7269 %}
 7270 
 7271 instruct storeImmN0(immN0 zero, memory4 mem)
 7272 %{
 7273   match(Set mem (StoreN mem zero));
 7274   predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
 7275 
 7276   ins_cost(INSN_COST);
 7277   format %{ "strw  zr, $mem\t# compressed ptr" %}
 7278 
 7279   ins_encode(aarch64_enc_strw0(mem));
 7280 
 7281   ins_pipe(istore_mem);
 7282 %}
 7283 
 7284 // Store Float
 7285 instruct storeF(vRegF src, memory4 mem)
 7286 %{
 7287   match(Set mem (StoreF mem src));
 7288   predicate(!needs_releasing_store(n));
 7289 
 7290   ins_cost(INSN_COST);
 7291   format %{ "strs  $src, $mem\t# float" %}
 7292 
 7293   ins_encode( aarch64_enc_strs(src, mem) );
 7294 
 7295   ins_pipe(pipe_class_memory);
 7296 %}
 7297 
 7298 // TODO
 7299 // implement storeImmF0 and storeFImmPacked
 7300 
 7301 // Store Double
 7302 instruct storeD(vRegD src, memory8 mem)
 7303 %{
 7304   match(Set mem (StoreD mem src));
 7305   predicate(!needs_releasing_store(n));
 7306 
 7307   ins_cost(INSN_COST);
 7308   format %{ "strd  $src, $mem\t# double" %}
 7309 
 7310   ins_encode( aarch64_enc_strd(src, mem) );
 7311 
 7312   ins_pipe(pipe_class_memory);
 7313 %}
 7314 
 7315 // Store Compressed Klass Pointer
 7316 instruct storeNKlass(iRegN src, memory4 mem)
 7317 %{
 7318   predicate(!needs_releasing_store(n));
 7319   match(Set mem (StoreNKlass mem src));
 7320 
 7321   ins_cost(INSN_COST);
 7322   format %{ "strw  $src, $mem\t# compressed klass ptr" %}
 7323 
 7324   ins_encode(aarch64_enc_strw(src, mem));
 7325 
 7326   ins_pipe(istore_reg_mem);
 7327 %}
 7328 
 7329 // TODO
 7330 // implement storeImmD0 and storeDImmPacked
 7331 
 7332 // prefetch instructions
 7333 // Must be safe to execute with invalid address (cannot fault).
 7334 
 7335 instruct prefetchalloc( memory8 mem ) %{
 7336   match(PrefetchAllocation mem);
 7337 
 7338   ins_cost(INSN_COST);
 7339   format %{ "prfm $mem, PSTL1KEEP\t# Prefetch into level 1 cache write keep" %}
 7340 
 7341   ins_encode( aarch64_enc_prefetchw(mem) );
 7342 
 7343   ins_pipe(iload_prefetch);
 7344 %}
 7345 
 7346 //  ---------------- volatile loads and stores ----------------
 7347 
 7348 // Load Byte (8 bit signed)
 7349 instruct loadB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 7350 %{
 7351   match(Set dst (LoadB mem));
 7352 
 7353   ins_cost(VOLATILE_REF_COST);
 7354   format %{ "ldarsb  $dst, $mem\t# byte" %}
 7355 
 7356   ins_encode(aarch64_enc_ldarsb(dst, mem));
 7357 
 7358   ins_pipe(pipe_serial);
 7359 %}
 7360 
 7361 // Load Byte (8 bit signed) into long
 7362 instruct loadB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 7363 %{
 7364   match(Set dst (ConvI2L (LoadB mem)));
 7365 
 7366   ins_cost(VOLATILE_REF_COST);
 7367   format %{ "ldarsb  $dst, $mem\t# byte" %}
 7368 
 7369   ins_encode(aarch64_enc_ldarsb(dst, mem));
 7370 
 7371   ins_pipe(pipe_serial);
 7372 %}
 7373 
 7374 // Load Byte (8 bit unsigned)
 7375 instruct loadUB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 7376 %{
 7377   match(Set dst (LoadUB mem));
 7378 
 7379   ins_cost(VOLATILE_REF_COST);
 7380   format %{ "ldarb  $dst, $mem\t# byte" %}
 7381 
 7382   ins_encode(aarch64_enc_ldarb(dst, mem));
 7383 
 7384   ins_pipe(pipe_serial);
 7385 %}
 7386 
 7387 // Load Byte (8 bit unsigned) into long
 7388 instruct loadUB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 7389 %{
 7390   match(Set dst (ConvI2L (LoadUB mem)));
 7391 
 7392   ins_cost(VOLATILE_REF_COST);
 7393   format %{ "ldarb  $dst, $mem\t# byte" %}
 7394 
 7395   ins_encode(aarch64_enc_ldarb(dst, mem));
 7396 
 7397   ins_pipe(pipe_serial);
 7398 %}
 7399 
 7400 // Load Short (16 bit signed)
 7401 instruct loadS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 7402 %{
 7403   match(Set dst (LoadS mem));
 7404 
 7405   ins_cost(VOLATILE_REF_COST);
 7406   format %{ "ldarshw  $dst, $mem\t# short" %}
 7407 
 7408   ins_encode(aarch64_enc_ldarshw(dst, mem));
 7409 
 7410   ins_pipe(pipe_serial);
 7411 %}
 7412 
 7413 instruct loadUS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 7414 %{
 7415   match(Set dst (LoadUS mem));
 7416 
 7417   ins_cost(VOLATILE_REF_COST);
 7418   format %{ "ldarhw  $dst, $mem\t# short" %}
 7419 
 7420   ins_encode(aarch64_enc_ldarhw(dst, mem));
 7421 
 7422   ins_pipe(pipe_serial);
 7423 %}
 7424 
 7425 // Load Short/Char (16 bit unsigned) into long
 7426 instruct loadUS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 7427 %{
 7428   match(Set dst (ConvI2L (LoadUS mem)));
 7429 
 7430   ins_cost(VOLATILE_REF_COST);
 7431   format %{ "ldarh  $dst, $mem\t# short" %}
 7432 
 7433   ins_encode(aarch64_enc_ldarh(dst, mem));
 7434 
 7435   ins_pipe(pipe_serial);
 7436 %}
 7437 
 7438 // Load Short/Char (16 bit signed) into long
 7439 instruct loadS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 7440 %{
 7441   match(Set dst (ConvI2L (LoadS mem)));
 7442 
 7443   ins_cost(VOLATILE_REF_COST);
 7444   format %{ "ldarh  $dst, $mem\t# short" %}
 7445 
 7446   ins_encode(aarch64_enc_ldarsh(dst, mem));
 7447 
 7448   ins_pipe(pipe_serial);
 7449 %}
 7450 
 7451 // Load Integer (32 bit signed)
 7452 instruct loadI_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 7453 %{
 7454   match(Set dst (LoadI mem));
 7455 
 7456   ins_cost(VOLATILE_REF_COST);
 7457   format %{ "ldarw  $dst, $mem\t# int" %}
 7458 
 7459   ins_encode(aarch64_enc_ldarw(dst, mem));
 7460 
 7461   ins_pipe(pipe_serial);
 7462 %}
 7463 
 7464 // Load Integer (32 bit unsigned) into long
 7465 instruct loadUI2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem, immL_32bits mask)
 7466 %{
 7467   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
 7468 
 7469   ins_cost(VOLATILE_REF_COST);
 7470   format %{ "ldarw  $dst, $mem\t# int" %}
 7471 
 7472   ins_encode(aarch64_enc_ldarw(dst, mem));
 7473 
 7474   ins_pipe(pipe_serial);
 7475 %}
 7476 
 7477 // Load Long (64 bit signed)
 7478 instruct loadL_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 7479 %{
 7480   match(Set dst (LoadL mem));
 7481 
 7482   ins_cost(VOLATILE_REF_COST);
 7483   format %{ "ldar  $dst, $mem\t# int" %}
 7484 
 7485   ins_encode(aarch64_enc_ldar(dst, mem));
 7486 
 7487   ins_pipe(pipe_serial);
 7488 %}
 7489 
 7490 // Load Pointer
 7491 instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem)
 7492 %{
 7493   match(Set dst (LoadP mem));
 7494   predicate(n->as_Load()->barrier_data() == 0);
 7495 
 7496   ins_cost(VOLATILE_REF_COST);
 7497   format %{ "ldar  $dst, $mem\t# ptr" %}
 7498 
 7499   ins_encode(aarch64_enc_ldar(dst, mem));
 7500 
 7501   ins_pipe(pipe_serial);
 7502 %}
 7503 
 7504 // Load Compressed Pointer
 7505 instruct loadN_volatile(iRegNNoSp dst, /* sync_memory*/indirect mem)
 7506 %{
 7507   match(Set dst (LoadN mem));
 7508   predicate(n->as_Load()->barrier_data() == 0);
 7509 
 7510   ins_cost(VOLATILE_REF_COST);
 7511   format %{ "ldarw  $dst, $mem\t# compressed ptr" %}
 7512 
 7513   ins_encode(aarch64_enc_ldarw(dst, mem));
 7514 
 7515   ins_pipe(pipe_serial);
 7516 %}
 7517 
 7518 // Load Float
 7519 instruct loadF_volatile(vRegF dst, /* sync_memory*/indirect mem)
 7520 %{
 7521   match(Set dst (LoadF mem));
 7522 
 7523   ins_cost(VOLATILE_REF_COST);
 7524   format %{ "ldars  $dst, $mem\t# float" %}
 7525 
 7526   ins_encode( aarch64_enc_fldars(dst, mem) );
 7527 
 7528   ins_pipe(pipe_serial);
 7529 %}
 7530 
 7531 // Load Double
 7532 instruct loadD_volatile(vRegD dst, /* sync_memory*/indirect mem)
 7533 %{
 7534   match(Set dst (LoadD mem));
 7535 
 7536   ins_cost(VOLATILE_REF_COST);
 7537   format %{ "ldard  $dst, $mem\t# double" %}
 7538 
 7539   ins_encode( aarch64_enc_fldard(dst, mem) );
 7540 
 7541   ins_pipe(pipe_serial);
 7542 %}
 7543 
 7544 // Store Byte
 7545 instruct storeB_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
 7546 %{
 7547   match(Set mem (StoreB mem src));
 7548 
 7549   ins_cost(VOLATILE_REF_COST);
 7550   format %{ "stlrb  $src, $mem\t# byte" %}
 7551 
 7552   ins_encode(aarch64_enc_stlrb(src, mem));
 7553 
 7554   ins_pipe(pipe_class_memory);
 7555 %}
 7556 
 7557 instruct storeimmB0_volatile(immI0 zero, /* sync_memory*/indirect mem)
 7558 %{
 7559   match(Set mem (StoreB mem zero));
 7560 
 7561   ins_cost(VOLATILE_REF_COST);
 7562   format %{ "stlrb  zr, $mem\t# byte" %}
 7563 
 7564   ins_encode(aarch64_enc_stlrb0(mem));
 7565 
 7566   ins_pipe(pipe_class_memory);
 7567 %}
 7568 
 7569 // Store Char/Short
 7570 instruct storeC_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
 7571 %{
 7572   match(Set mem (StoreC mem src));
 7573 
 7574   ins_cost(VOLATILE_REF_COST);
 7575   format %{ "stlrh  $src, $mem\t# short" %}
 7576 
 7577   ins_encode(aarch64_enc_stlrh(src, mem));
 7578 
 7579   ins_pipe(pipe_class_memory);
 7580 %}
 7581 
 7582 instruct storeimmC0_volatile(immI0 zero, /* sync_memory*/indirect mem)
 7583 %{
 7584   match(Set mem (StoreC mem zero));
 7585 
 7586   ins_cost(VOLATILE_REF_COST);
 7587   format %{ "stlrh  zr, $mem\t# short" %}
 7588 
 7589   ins_encode(aarch64_enc_stlrh0(mem));
 7590 
 7591   ins_pipe(pipe_class_memory);
 7592 %}
 7593 
 7594 // Store Integer
 7595 
 7596 instruct storeI_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
 7597 %{
 7598   match(Set mem(StoreI mem src));
 7599 
 7600   ins_cost(VOLATILE_REF_COST);
 7601   format %{ "stlrw  $src, $mem\t# int" %}
 7602 
 7603   ins_encode(aarch64_enc_stlrw(src, mem));
 7604 
 7605   ins_pipe(pipe_class_memory);
 7606 %}
 7607 
 7608 instruct storeimmI0_volatile(immI0 zero, /* sync_memory*/indirect mem)
 7609 %{
 7610   match(Set mem(StoreI mem zero));
 7611 
 7612   ins_cost(VOLATILE_REF_COST);
 7613   format %{ "stlrw  zr, $mem\t# int" %}
 7614 
 7615   ins_encode(aarch64_enc_stlrw0(mem));
 7616 
 7617   ins_pipe(pipe_class_memory);
 7618 %}
 7619 
 7620 // Store Long (64 bit signed)
 7621 instruct storeL_volatile(iRegL src, /* sync_memory*/indirect mem)
 7622 %{
 7623   match(Set mem (StoreL mem src));
 7624 
 7625   ins_cost(VOLATILE_REF_COST);
 7626   format %{ "stlr  $src, $mem\t# int" %}
 7627 
 7628   ins_encode(aarch64_enc_stlr(src, mem));
 7629 
 7630   ins_pipe(pipe_class_memory);
 7631 %}
 7632 
 7633 instruct storeimmL0_volatile(immL0 zero, /* sync_memory*/indirect mem)
 7634 %{
 7635   match(Set mem (StoreL mem zero));
 7636 
 7637   ins_cost(VOLATILE_REF_COST);
 7638   format %{ "stlr  zr, $mem\t# int" %}
 7639 
 7640   ins_encode(aarch64_enc_stlr0(mem));
 7641 
 7642   ins_pipe(pipe_class_memory);
 7643 %}
 7644 
 7645 // Store Pointer
 7646 instruct storeP_volatile(iRegP src, /* sync_memory*/indirect mem)
 7647 %{
 7648   match(Set mem (StoreP mem src));
 7649   predicate(n->as_Store()->barrier_data() == 0);
 7650 
 7651   ins_cost(VOLATILE_REF_COST);
 7652   format %{ "stlr  $src, $mem\t# ptr" %}
 7653 
 7654   ins_encode(aarch64_enc_stlr(src, mem));
 7655 
 7656   ins_pipe(pipe_class_memory);
 7657 %}
 7658 
 7659 instruct storeimmP0_volatile(immP0 zero, /* sync_memory*/indirect mem)
 7660 %{
 7661   match(Set mem (StoreP mem zero));
 7662   predicate(n->as_Store()->barrier_data() == 0);
 7663 
 7664   ins_cost(VOLATILE_REF_COST);
 7665   format %{ "stlr  zr, $mem\t# ptr" %}
 7666 
 7667   ins_encode(aarch64_enc_stlr0(mem));
 7668 
 7669   ins_pipe(pipe_class_memory);
 7670 %}
 7671 
 7672 // Store Compressed Pointer
 7673 instruct storeN_volatile(iRegN src, /* sync_memory*/indirect mem)
 7674 %{
 7675   match(Set mem (StoreN mem src));
 7676   predicate(n->as_Store()->barrier_data() == 0);
 7677 
 7678   ins_cost(VOLATILE_REF_COST);
 7679   format %{ "stlrw  $src, $mem\t# compressed ptr" %}
 7680 
 7681   ins_encode(aarch64_enc_stlrw(src, mem));
 7682 
 7683   ins_pipe(pipe_class_memory);
 7684 %}
 7685 
 7686 instruct storeimmN0_volatile(immN0 zero, /* sync_memory*/indirect mem)
 7687 %{
 7688   match(Set mem (StoreN mem zero));
 7689   predicate(n->as_Store()->barrier_data() == 0);
 7690 
 7691   ins_cost(VOLATILE_REF_COST);
 7692   format %{ "stlrw  zr, $mem\t# compressed ptr" %}
 7693 
 7694   ins_encode(aarch64_enc_stlrw0(mem));
 7695 
 7696   ins_pipe(pipe_class_memory);
 7697 %}
 7698 
 7699 // Store Float
 7700 instruct storeF_volatile(vRegF src, /* sync_memory*/indirect mem)
 7701 %{
 7702   match(Set mem (StoreF mem src));
 7703 
 7704   ins_cost(VOLATILE_REF_COST);
 7705   format %{ "stlrs  $src, $mem\t# float" %}
 7706 
 7707   ins_encode( aarch64_enc_fstlrs(src, mem) );
 7708 
 7709   ins_pipe(pipe_class_memory);
 7710 %}
 7711 
 7712 // TODO
 7713 // implement storeImmF0 and storeFImmPacked
 7714 
 7715 // Store Double
 7716 instruct storeD_volatile(vRegD src, /* sync_memory*/indirect mem)
 7717 %{
 7718   match(Set mem (StoreD mem src));
 7719 
 7720   ins_cost(VOLATILE_REF_COST);
 7721   format %{ "stlrd  $src, $mem\t# double" %}
 7722 
 7723   ins_encode( aarch64_enc_fstlrd(src, mem) );
 7724 
 7725   ins_pipe(pipe_class_memory);
 7726 %}
 7727 
 7728 //  ---------------- end of volatile loads and stores ----------------
 7729 
 7730 instruct cacheWB(indirect addr)
 7731 %{
 7732   predicate(VM_Version::supports_data_cache_line_flush());
 7733   match(CacheWB addr);
 7734 
 7735   ins_cost(100);
 7736   format %{"cache wb $addr" %}
 7737   ins_encode %{
 7738     assert($addr->index_position() < 0, "should be");
 7739     assert($addr$$disp == 0, "should be");
 7740     __ cache_wb(Address($addr$$base$$Register, 0));
 7741   %}
 7742   ins_pipe(pipe_slow); // XXX
 7743 %}
 7744 
 7745 instruct cacheWBPreSync()
 7746 %{
 7747   predicate(VM_Version::supports_data_cache_line_flush());
 7748   match(CacheWBPreSync);
 7749 
 7750   ins_cost(100);
 7751   format %{"cache wb presync" %}
 7752   ins_encode %{
 7753     __ cache_wbsync(true);
 7754   %}
 7755   ins_pipe(pipe_slow); // XXX
 7756 %}
 7757 
 7758 instruct cacheWBPostSync()
 7759 %{
 7760   predicate(VM_Version::supports_data_cache_line_flush());
 7761   match(CacheWBPostSync);
 7762 
 7763   ins_cost(100);
 7764   format %{"cache wb postsync" %}
 7765   ins_encode %{
 7766     __ cache_wbsync(false);
 7767   %}
 7768   ins_pipe(pipe_slow); // XXX
 7769 %}
 7770 
 7771 // ============================================================================
 7772 // BSWAP Instructions
 7773 
 7774 instruct bytes_reverse_int(iRegINoSp dst, iRegIorL2I src) %{
 7775   match(Set dst (ReverseBytesI src));
 7776 
 7777   ins_cost(INSN_COST);
 7778   format %{ "revw  $dst, $src" %}
 7779 
 7780   ins_encode %{
 7781     __ revw(as_Register($dst$$reg), as_Register($src$$reg));
 7782   %}
 7783 
 7784   ins_pipe(ialu_reg);
 7785 %}
 7786 
 7787 instruct bytes_reverse_long(iRegLNoSp dst, iRegL src) %{
 7788   match(Set dst (ReverseBytesL src));
 7789 
 7790   ins_cost(INSN_COST);
 7791   format %{ "rev  $dst, $src" %}
 7792 
 7793   ins_encode %{
 7794     __ rev(as_Register($dst$$reg), as_Register($src$$reg));
 7795   %}
 7796 
 7797   ins_pipe(ialu_reg);
 7798 %}
 7799 
 7800 instruct bytes_reverse_unsigned_short(iRegINoSp dst, iRegIorL2I src) %{
 7801   match(Set dst (ReverseBytesUS src));
 7802 
 7803   ins_cost(INSN_COST);
 7804   format %{ "rev16w  $dst, $src" %}
 7805 
 7806   ins_encode %{
 7807     __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
 7808   %}
 7809 
 7810   ins_pipe(ialu_reg);
 7811 %}
 7812 
 7813 instruct bytes_reverse_short(iRegINoSp dst, iRegIorL2I src) %{
 7814   match(Set dst (ReverseBytesS src));
 7815 
 7816   ins_cost(INSN_COST);
 7817   format %{ "rev16w  $dst, $src\n\t"
 7818             "sbfmw $dst, $dst, #0, #15" %}
 7819 
 7820   ins_encode %{
 7821     __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
 7822     __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 15U);
 7823   %}
 7824 
 7825   ins_pipe(ialu_reg);
 7826 %}
 7827 
 7828 // ============================================================================
 7829 // Zero Count Instructions
 7830 
 7831 instruct countLeadingZerosI(iRegINoSp dst, iRegIorL2I src) %{
 7832   match(Set dst (CountLeadingZerosI src));
 7833 
 7834   ins_cost(INSN_COST);
 7835   format %{ "clzw  $dst, $src" %}
 7836   ins_encode %{
 7837     __ clzw(as_Register($dst$$reg), as_Register($src$$reg));
 7838   %}
 7839 
 7840   ins_pipe(ialu_reg);
 7841 %}
 7842 
 7843 instruct countLeadingZerosL(iRegINoSp dst, iRegL src) %{
 7844   match(Set dst (CountLeadingZerosL src));
 7845 
 7846   ins_cost(INSN_COST);
 7847   format %{ "clz   $dst, $src" %}
 7848   ins_encode %{
 7849     __ clz(as_Register($dst$$reg), as_Register($src$$reg));
 7850   %}
 7851 
 7852   ins_pipe(ialu_reg);
 7853 %}
 7854 
 7855 instruct countTrailingZerosI(iRegINoSp dst, iRegIorL2I src) %{
 7856   match(Set dst (CountTrailingZerosI src));
 7857 
 7858   ins_cost(INSN_COST * 2);
 7859   format %{ "rbitw  $dst, $src\n\t"
 7860             "clzw   $dst, $dst" %}
 7861   ins_encode %{
 7862     __ rbitw(as_Register($dst$$reg), as_Register($src$$reg));
 7863     __ clzw(as_Register($dst$$reg), as_Register($dst$$reg));
 7864   %}
 7865 
 7866   ins_pipe(ialu_reg);
 7867 %}
 7868 
 7869 instruct countTrailingZerosL(iRegINoSp dst, iRegL src) %{
 7870   match(Set dst (CountTrailingZerosL src));
 7871 
 7872   ins_cost(INSN_COST * 2);
 7873   format %{ "rbit   $dst, $src\n\t"
 7874             "clz    $dst, $dst" %}
 7875   ins_encode %{
 7876     __ rbit(as_Register($dst$$reg), as_Register($src$$reg));
 7877     __ clz(as_Register($dst$$reg), as_Register($dst$$reg));
 7878   %}
 7879 
 7880   ins_pipe(ialu_reg);
 7881 %}
 7882 
 7883 //---------- Population Count Instructions -------------------------------------
 7884 //
 7885 
 7886 instruct popCountI(iRegINoSp dst, iRegIorL2I src, vRegF tmp) %{
 7887   match(Set dst (PopCountI src));
 7888   effect(TEMP tmp);
 7889   ins_cost(INSN_COST * 13);
 7890 
 7891   format %{ "fmovs  $tmp, $src\t# vector (1S)\n\t"
 7892             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 7893             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 7894             "mov    $dst, $tmp\t# vector (1D)" %}
 7895   ins_encode %{
 7896     __ fmovs($tmp$$FloatRegister, $src$$Register);
 7897     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 7898     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 7899     __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
 7900   %}
 7901 
 7902   ins_pipe(pipe_class_default);
 7903 %}
 7904 
 7905 instruct popCountI_mem(iRegINoSp dst, memory4 mem, vRegF tmp) %{
 7906   match(Set dst (PopCountI (LoadI mem)));
 7907   effect(TEMP tmp);
 7908   ins_cost(INSN_COST * 13);
 7909 
 7910   format %{ "ldrs   $tmp, $mem\n\t"
 7911             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 7912             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 7913             "mov    $dst, $tmp\t# vector (1D)" %}
 7914   ins_encode %{
 7915     FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
 7916     loadStore(masm, &MacroAssembler::ldrs, tmp_reg, $mem->opcode(),
 7917               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 7918     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 7919     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 7920     __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
 7921   %}
 7922 
 7923   ins_pipe(pipe_class_default);
 7924 %}
 7925 
 7926 // Note: Long.bitCount(long) returns an int.
 7927 instruct popCountL(iRegINoSp dst, iRegL src, vRegD tmp) %{
 7928   match(Set dst (PopCountL src));
 7929   effect(TEMP tmp);
 7930   ins_cost(INSN_COST * 13);
 7931 
 7932   format %{ "mov    $tmp, $src\t# vector (1D)\n\t"
 7933             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 7934             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 7935             "mov    $dst, $tmp\t# vector (1D)" %}
 7936   ins_encode %{
 7937     __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
 7938     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 7939     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 7940     __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
 7941   %}
 7942 
 7943   ins_pipe(pipe_class_default);
 7944 %}
 7945 
 7946 instruct popCountL_mem(iRegINoSp dst, memory8 mem, vRegD tmp) %{
 7947   match(Set dst (PopCountL (LoadL mem)));
 7948   effect(TEMP tmp);
 7949   ins_cost(INSN_COST * 13);
 7950 
 7951   format %{ "ldrd   $tmp, $mem\n\t"
 7952             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 7953             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 7954             "mov    $dst, $tmp\t# vector (1D)" %}
 7955   ins_encode %{
 7956     FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
 7957     loadStore(masm, &MacroAssembler::ldrd, tmp_reg, $mem->opcode(),
 7958               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 7959     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 7960     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 7961     __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
 7962   %}
 7963 
 7964   ins_pipe(pipe_class_default);
 7965 %}
 7966 
 7967 // ============================================================================
 7968 // VerifyVectorAlignment Instruction
 7969 
 7970 instruct verify_vector_alignment(iRegP addr, immL_positive_bitmaskI mask, rFlagsReg cr) %{
 7971   match(Set addr (VerifyVectorAlignment addr mask));
 7972   effect(KILL cr);
 7973   format %{ "verify_vector_alignment $addr $mask \t! verify alignment" %}
 7974   ins_encode %{
 7975     Label Lskip;
 7976     // check if masked bits of addr are zero
 7977     __ tst($addr$$Register, $mask$$constant);
 7978     __ br(Assembler::EQ, Lskip);
 7979     __ stop("verify_vector_alignment found a misaligned vector memory access");
 7980     __ bind(Lskip);
 7981   %}
 7982   ins_pipe(pipe_slow);
 7983 %}
 7984 
 7985 // ============================================================================
 7986 // MemBar Instruction
 7987 
 7988 instruct load_fence() %{
 7989   match(LoadFence);
 7990   ins_cost(VOLATILE_REF_COST);
 7991 
 7992   format %{ "load_fence" %}
 7993 
 7994   ins_encode %{
 7995     __ membar(Assembler::LoadLoad|Assembler::LoadStore);
 7996   %}
 7997   ins_pipe(pipe_serial);
 7998 %}
 7999 
 8000 instruct unnecessary_membar_acquire() %{
 8001   predicate(unnecessary_acquire(n));
 8002   match(MemBarAcquire);
 8003   ins_cost(0);
 8004 
 8005   format %{ "membar_acquire (elided)" %}
 8006 
 8007   ins_encode %{
 8008     __ block_comment("membar_acquire (elided)");
 8009   %}
 8010 
 8011   ins_pipe(pipe_class_empty);
 8012 %}
 8013 
 8014 instruct membar_acquire() %{
 8015   match(MemBarAcquire);
 8016   ins_cost(VOLATILE_REF_COST);
 8017 
 8018   format %{ "membar_acquire\n\t"
 8019             "dmb ishld" %}
 8020 
 8021   ins_encode %{
 8022     __ block_comment("membar_acquire");
 8023     __ membar(Assembler::LoadLoad|Assembler::LoadStore);
 8024   %}
 8025 
 8026   ins_pipe(pipe_serial);
 8027 %}
 8028 
 8029 
 8030 instruct membar_acquire_lock() %{
 8031   match(MemBarAcquireLock);
 8032   ins_cost(VOLATILE_REF_COST);
 8033 
 8034   format %{ "membar_acquire_lock (elided)" %}
 8035 
 8036   ins_encode %{
 8037     __ block_comment("membar_acquire_lock (elided)");
 8038   %}
 8039 
 8040   ins_pipe(pipe_serial);
 8041 %}
 8042 
 8043 instruct store_fence() %{
 8044   match(StoreFence);
 8045   ins_cost(VOLATILE_REF_COST);
 8046 
 8047   format %{ "store_fence" %}
 8048 
 8049   ins_encode %{
 8050     __ membar(Assembler::LoadStore|Assembler::StoreStore);
 8051   %}
 8052   ins_pipe(pipe_serial);
 8053 %}
 8054 
 8055 instruct unnecessary_membar_release() %{
 8056   predicate(unnecessary_release(n));
 8057   match(MemBarRelease);
 8058   ins_cost(0);
 8059 
 8060   format %{ "membar_release (elided)" %}
 8061 
 8062   ins_encode %{
 8063     __ block_comment("membar_release (elided)");
 8064   %}
 8065   ins_pipe(pipe_serial);
 8066 %}
 8067 
 8068 instruct membar_release() %{
 8069   match(MemBarRelease);
 8070   ins_cost(VOLATILE_REF_COST);
 8071 
 8072   format %{ "membar_release\n\t"
 8073             "dmb ishst\n\tdmb ishld" %}
 8074 
 8075   ins_encode %{
 8076     __ block_comment("membar_release");
 8077     // These will be merged if AlwaysMergeDMB is enabled.
 8078     __ membar(Assembler::StoreStore);
 8079     __ membar(Assembler::LoadStore);
 8080   %}
 8081   ins_pipe(pipe_serial);
 8082 %}
 8083 
 8084 instruct membar_storestore() %{
 8085   match(MemBarStoreStore);
 8086   match(StoreStoreFence);
 8087   ins_cost(VOLATILE_REF_COST);
 8088 
 8089   format %{ "MEMBAR-store-store" %}
 8090 
 8091   ins_encode %{
 8092     __ membar(Assembler::StoreStore);
 8093   %}
 8094   ins_pipe(pipe_serial);
 8095 %}
 8096 
 8097 instruct membar_release_lock() %{
 8098   match(MemBarReleaseLock);
 8099   ins_cost(VOLATILE_REF_COST);
 8100 
 8101   format %{ "membar_release_lock (elided)" %}
 8102 
 8103   ins_encode %{
 8104     __ block_comment("membar_release_lock (elided)");
 8105   %}
 8106 
 8107   ins_pipe(pipe_serial);
 8108 %}
 8109 
 8110 instruct unnecessary_membar_volatile() %{
 8111   predicate(unnecessary_volatile(n));
 8112   match(MemBarVolatile);
 8113   ins_cost(0);
 8114 
 8115   format %{ "membar_volatile (elided)" %}
 8116 
 8117   ins_encode %{
 8118     __ block_comment("membar_volatile (elided)");
 8119   %}
 8120 
 8121   ins_pipe(pipe_serial);
 8122 %}
 8123 
 8124 instruct membar_volatile() %{
 8125   match(MemBarVolatile);
 8126   ins_cost(VOLATILE_REF_COST*100);
 8127 
 8128   format %{ "membar_volatile\n\t"
 8129              "dmb ish"%}
 8130 
 8131   ins_encode %{
 8132     __ block_comment("membar_volatile");
 8133     __ membar(Assembler::StoreLoad);
 8134   %}
 8135 
 8136   ins_pipe(pipe_serial);
 8137 %}
 8138 
 8139 // ============================================================================
 8140 // Cast/Convert Instructions
 8141 
 8142 instruct castX2P(iRegPNoSp dst, iRegL src) %{
 8143   match(Set dst (CastX2P src));
 8144 
 8145   ins_cost(INSN_COST);
 8146   format %{ "mov $dst, $src\t# long -> ptr" %}
 8147 
 8148   ins_encode %{
 8149     if ($dst$$reg != $src$$reg) {
 8150       __ mov(as_Register($dst$$reg), as_Register($src$$reg));
 8151     }
 8152   %}
 8153 
 8154   ins_pipe(ialu_reg);
 8155 %}
 8156 
 8157 instruct castI2N(iRegNNoSp dst, iRegI src) %{
 8158   match(Set dst (CastI2N src));
 8159 
 8160   ins_cost(INSN_COST);
 8161   format %{ "mov $dst, $src\t# int -> narrow ptr" %}
 8162 
 8163   ins_encode %{
 8164     if ($dst$$reg != $src$$reg) {
 8165       __ mov(as_Register($dst$$reg), as_Register($src$$reg));
 8166     }
 8167   %}
 8168 
 8169   ins_pipe(ialu_reg);
 8170 %}
 8171 
 8172 instruct castN2X(iRegLNoSp dst, iRegN src) %{
 8173   match(Set dst (CastP2X src));
 8174 
 8175   ins_cost(INSN_COST);
 8176   format %{ "mov $dst, $src\t# ptr -> long" %}
 8177 
 8178   ins_encode %{
 8179     if ($dst$$reg != $src$$reg) {
 8180       __ mov(as_Register($dst$$reg), as_Register($src$$reg));
 8181     }
 8182   %}
 8183 
 8184   ins_pipe(ialu_reg);
 8185 %}
 8186 
 8187 instruct castP2X(iRegLNoSp dst, iRegP src) %{
 8188   match(Set dst (CastP2X src));
 8189 
 8190   ins_cost(INSN_COST);
 8191   format %{ "mov $dst, $src\t# ptr -> long" %}
 8192 
 8193   ins_encode %{
 8194     if ($dst$$reg != $src$$reg) {
 8195       __ mov(as_Register($dst$$reg), as_Register($src$$reg));
 8196     }
 8197   %}
 8198 
 8199   ins_pipe(ialu_reg);
 8200 %}
 8201 
 8202 // Convert oop into int for vectors alignment masking
 8203 instruct convP2I(iRegINoSp dst, iRegP src) %{
 8204   match(Set dst (ConvL2I (CastP2X src)));
 8205 
 8206   ins_cost(INSN_COST);
 8207   format %{ "movw $dst, $src\t# ptr -> int" %}
 8208   ins_encode %{
 8209     __ movw($dst$$Register, $src$$Register);
 8210   %}
 8211 
 8212   ins_pipe(ialu_reg);
 8213 %}
 8214 
 8215 // Convert compressed oop into int for vectors alignment masking
 8216 // in case of 32bit oops (heap < 4Gb).
 8217 instruct convN2I(iRegINoSp dst, iRegN src)
 8218 %{
 8219   predicate(CompressedOops::shift() == 0);
 8220   match(Set dst (ConvL2I (CastP2X (DecodeN src))));
 8221 
 8222   ins_cost(INSN_COST);
 8223   format %{ "mov dst, $src\t# compressed ptr -> int" %}
 8224   ins_encode %{
 8225     __ movw($dst$$Register, $src$$Register);
 8226   %}
 8227 
 8228   ins_pipe(ialu_reg);
 8229 %}
 8230 
 8231 
 8232 // Convert oop pointer into compressed form
 8233 instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
 8234   predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
 8235   match(Set dst (EncodeP src));
 8236   effect(KILL cr);
 8237   ins_cost(INSN_COST * 3);
 8238   format %{ "encode_heap_oop $dst, $src" %}
 8239   ins_encode %{
 8240     Register s = $src$$Register;
 8241     Register d = $dst$$Register;
 8242     __ encode_heap_oop(d, s);
 8243   %}
 8244   ins_pipe(ialu_reg);
 8245 %}
 8246 
 8247 instruct encodeHeapOop_not_null(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
 8248   predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
 8249   match(Set dst (EncodeP src));
 8250   ins_cost(INSN_COST * 3);
 8251   format %{ "encode_heap_oop_not_null $dst, $src" %}
 8252   ins_encode %{
 8253     __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
 8254   %}
 8255   ins_pipe(ialu_reg);
 8256 %}
 8257 
 8258 instruct decodeHeapOop(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
 8259   predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
 8260             n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
 8261   match(Set dst (DecodeN src));
 8262   ins_cost(INSN_COST * 3);
 8263   format %{ "decode_heap_oop $dst, $src" %}
 8264   ins_encode %{
 8265     Register s = $src$$Register;
 8266     Register d = $dst$$Register;
 8267     __ decode_heap_oop(d, s);
 8268   %}
 8269   ins_pipe(ialu_reg);
 8270 %}
 8271 
 8272 instruct decodeHeapOop_not_null(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
 8273   predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
 8274             n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
 8275   match(Set dst (DecodeN src));
 8276   ins_cost(INSN_COST * 3);
 8277   format %{ "decode_heap_oop_not_null $dst, $src" %}
 8278   ins_encode %{
 8279     Register s = $src$$Register;
 8280     Register d = $dst$$Register;
 8281     __ decode_heap_oop_not_null(d, s);
 8282   %}
 8283   ins_pipe(ialu_reg);
 8284 %}
 8285 
 8286 // n.b. AArch64 implementations of encode_klass_not_null and
 8287 // decode_klass_not_null do not modify the flags register so, unlike
 8288 // Intel, we don't kill CR as a side effect here
 8289 
 8290 instruct encodeKlass_not_null(iRegNNoSp dst, iRegP src) %{
 8291   match(Set dst (EncodePKlass src));
 8292 
 8293   ins_cost(INSN_COST * 3);
 8294   format %{ "encode_klass_not_null $dst,$src" %}
 8295 
 8296   ins_encode %{
 8297     Register src_reg = as_Register($src$$reg);
 8298     Register dst_reg = as_Register($dst$$reg);
 8299     __ encode_klass_not_null(dst_reg, src_reg);
 8300   %}
 8301 
 8302    ins_pipe(ialu_reg);
 8303 %}
 8304 
 8305 instruct decodeKlass_not_null(iRegPNoSp dst, iRegN src) %{
 8306   match(Set dst (DecodeNKlass src));
 8307 
 8308   ins_cost(INSN_COST * 3);
 8309   format %{ "decode_klass_not_null $dst,$src" %}
 8310 
 8311   ins_encode %{
 8312     Register src_reg = as_Register($src$$reg);
 8313     Register dst_reg = as_Register($dst$$reg);
 8314     if (dst_reg != src_reg) {
 8315       __ decode_klass_not_null(dst_reg, src_reg);
 8316     } else {
 8317       __ decode_klass_not_null(dst_reg);
 8318     }
 8319   %}
 8320 
 8321    ins_pipe(ialu_reg);
 8322 %}
 8323 
 8324 instruct checkCastPP(iRegPNoSp dst)
 8325 %{
 8326   match(Set dst (CheckCastPP dst));
 8327 
 8328   size(0);
 8329   format %{ "# checkcastPP of $dst" %}
 8330   ins_encode(/* empty encoding */);
 8331   ins_pipe(pipe_class_empty);
 8332 %}
 8333 
 8334 instruct castPP(iRegPNoSp dst)
 8335 %{
 8336   match(Set dst (CastPP dst));
 8337 
 8338   size(0);
 8339   format %{ "# castPP of $dst" %}
 8340   ins_encode(/* empty encoding */);
 8341   ins_pipe(pipe_class_empty);
 8342 %}
 8343 
 8344 instruct castII(iRegI dst)
 8345 %{
 8346   predicate(VerifyConstraintCasts == 0);
 8347   match(Set dst (CastII dst));
 8348 
 8349   size(0);
 8350   format %{ "# castII of $dst" %}
 8351   ins_encode(/* empty encoding */);
 8352   ins_cost(0);
 8353   ins_pipe(pipe_class_empty);
 8354 %}
 8355 
 8356 instruct castII_checked(iRegI dst, rFlagsReg cr)
 8357 %{
 8358   predicate(VerifyConstraintCasts > 0);
 8359   match(Set dst (CastII dst));
 8360   effect(KILL cr);
 8361 
 8362   format %{ "# castII_checked of $dst" %}
 8363   ins_encode %{
 8364     __ verify_int_in_range(_idx, bottom_type()->is_int(), $dst$$Register, rscratch1);
 8365   %}
 8366   ins_pipe(pipe_slow);
 8367 %}
 8368 
 8369 instruct castLL(iRegL dst)
 8370 %{
 8371   predicate(VerifyConstraintCasts == 0);
 8372   match(Set dst (CastLL dst));
 8373 
 8374   size(0);
 8375   format %{ "# castLL of $dst" %}
 8376   ins_encode(/* empty encoding */);
 8377   ins_cost(0);
 8378   ins_pipe(pipe_class_empty);
 8379 %}
 8380 
 8381 instruct castLL_checked(iRegL dst, rFlagsReg cr)
 8382 %{
 8383   predicate(VerifyConstraintCasts > 0);
 8384   match(Set dst (CastLL dst));
 8385   effect(KILL cr);
 8386 
 8387   format %{ "# castLL_checked of $dst" %}
 8388   ins_encode %{
 8389     __ verify_long_in_range(_idx, bottom_type()->is_long(), $dst$$Register, rscratch1);
 8390   %}
 8391   ins_pipe(pipe_slow);
 8392 %}
 8393 
 8394 instruct castHH(vRegF dst)
 8395 %{
 8396   match(Set dst (CastHH dst));
 8397   size(0);
 8398   format %{ "# castHH of $dst" %}
 8399   ins_encode(/* empty encoding */);
 8400   ins_cost(0);
 8401   ins_pipe(pipe_class_empty);
 8402 %}
 8403 
 8404 instruct castFF(vRegF dst)
 8405 %{
 8406   match(Set dst (CastFF dst));
 8407 
 8408   size(0);
 8409   format %{ "# castFF of $dst" %}
 8410   ins_encode(/* empty encoding */);
 8411   ins_cost(0);
 8412   ins_pipe(pipe_class_empty);
 8413 %}
 8414 
 8415 instruct castDD(vRegD dst)
 8416 %{
 8417   match(Set dst (CastDD dst));
 8418 
 8419   size(0);
 8420   format %{ "# castDD of $dst" %}
 8421   ins_encode(/* empty encoding */);
 8422   ins_cost(0);
 8423   ins_pipe(pipe_class_empty);
 8424 %}
 8425 
 8426 instruct castVV(vReg dst)
 8427 %{
 8428   match(Set dst (CastVV dst));
 8429 
 8430   size(0);
 8431   format %{ "# castVV of $dst" %}
 8432   ins_encode(/* empty encoding */);
 8433   ins_cost(0);
 8434   ins_pipe(pipe_class_empty);
 8435 %}
 8436 
 8437 instruct castVVMask(pRegGov dst)
 8438 %{
 8439   match(Set dst (CastVV dst));
 8440 
 8441   size(0);
 8442   format %{ "# castVV of $dst" %}
 8443   ins_encode(/* empty encoding */);
 8444   ins_cost(0);
 8445   ins_pipe(pipe_class_empty);
 8446 %}
 8447 
 8448 // ============================================================================
 8449 // Atomic operation instructions
 8450 //
 8451 
 8452 // standard CompareAndSwapX when we are using barriers
 8453 // these have higher priority than the rules selected by a predicate
 8454 
 8455 // XXX No flag versions for CompareAndSwap{I,L,P,N} because matcher
 8456 // can't match them
 8457 
 8458 instruct compareAndSwapB(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 8459 
 8460   match(Set res (CompareAndSwapB mem (Binary oldval newval)));
 8461   ins_cost(2 * VOLATILE_REF_COST);
 8462 
 8463   effect(KILL cr);
 8464 
 8465   format %{
 8466     "cmpxchgb $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 8467     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8468   %}
 8469 
 8470   ins_encode(aarch64_enc_cmpxchgb(mem, oldval, newval),
 8471             aarch64_enc_cset_eq(res));
 8472 
 8473   ins_pipe(pipe_slow);
 8474 %}
 8475 
 8476 instruct compareAndSwapS(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 8477 
 8478   match(Set res (CompareAndSwapS mem (Binary oldval newval)));
 8479   ins_cost(2 * VOLATILE_REF_COST);
 8480 
 8481   effect(KILL cr);
 8482 
 8483   format %{
 8484     "cmpxchgs $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 8485     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8486   %}
 8487 
 8488   ins_encode(aarch64_enc_cmpxchgs(mem, oldval, newval),
 8489             aarch64_enc_cset_eq(res));
 8490 
 8491   ins_pipe(pipe_slow);
 8492 %}
 8493 
 8494 instruct compareAndSwapI(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 8495 
 8496   match(Set res (CompareAndSwapI mem (Binary oldval newval)));
 8497   ins_cost(2 * VOLATILE_REF_COST);
 8498 
 8499   effect(KILL cr);
 8500 
 8501  format %{
 8502     "cmpxchgw $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 8503     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8504  %}
 8505 
 8506  ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
 8507             aarch64_enc_cset_eq(res));
 8508 
 8509   ins_pipe(pipe_slow);
 8510 %}
 8511 
 8512 instruct compareAndSwapL(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
 8513 
 8514   match(Set res (CompareAndSwapL mem (Binary oldval newval)));
 8515   ins_cost(2 * VOLATILE_REF_COST);
 8516 
 8517   effect(KILL cr);
 8518 
 8519  format %{
 8520     "cmpxchg $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
 8521     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8522  %}
 8523 
 8524  ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
 8525             aarch64_enc_cset_eq(res));
 8526 
 8527   ins_pipe(pipe_slow);
 8528 %}
 8529 
 8530 instruct compareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 8531 
 8532   match(Set res (CompareAndSwapP mem (Binary oldval newval)));
 8533   predicate(n->as_LoadStore()->barrier_data() == 0);
 8534   ins_cost(2 * VOLATILE_REF_COST);
 8535 
 8536   effect(KILL cr);
 8537 
 8538  format %{
 8539     "cmpxchg $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
 8540     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8541  %}
 8542 
 8543  ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
 8544             aarch64_enc_cset_eq(res));
 8545 
 8546   ins_pipe(pipe_slow);
 8547 %}
 8548 
 8549 instruct compareAndSwapN(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
 8550 
 8551   match(Set res (CompareAndSwapN mem (Binary oldval newval)));
 8552   predicate(n->as_LoadStore()->barrier_data() == 0);
 8553   ins_cost(2 * VOLATILE_REF_COST);
 8554 
 8555   effect(KILL cr);
 8556 
 8557  format %{
 8558     "cmpxchgw $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
 8559     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8560  %}
 8561 
 8562  ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
 8563             aarch64_enc_cset_eq(res));
 8564 
 8565   ins_pipe(pipe_slow);
 8566 %}
 8567 
 8568 // alternative CompareAndSwapX when we are eliding barriers
 8569 
 8570 instruct compareAndSwapBAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 8571 
 8572   predicate(needs_acquiring_load_exclusive(n));
 8573   match(Set res (CompareAndSwapB mem (Binary oldval newval)));
 8574   ins_cost(VOLATILE_REF_COST);
 8575 
 8576   effect(KILL cr);
 8577 
 8578   format %{
 8579     "cmpxchgb_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 8580     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8581   %}
 8582 
 8583   ins_encode(aarch64_enc_cmpxchgb_acq(mem, oldval, newval),
 8584             aarch64_enc_cset_eq(res));
 8585 
 8586   ins_pipe(pipe_slow);
 8587 %}
 8588 
 8589 instruct compareAndSwapSAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 8590 
 8591   predicate(needs_acquiring_load_exclusive(n));
 8592   match(Set res (CompareAndSwapS mem (Binary oldval newval)));
 8593   ins_cost(VOLATILE_REF_COST);
 8594 
 8595   effect(KILL cr);
 8596 
 8597   format %{
 8598     "cmpxchgs_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 8599     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8600   %}
 8601 
 8602   ins_encode(aarch64_enc_cmpxchgs_acq(mem, oldval, newval),
 8603             aarch64_enc_cset_eq(res));
 8604 
 8605   ins_pipe(pipe_slow);
 8606 %}
 8607 
 8608 instruct compareAndSwapIAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 8609 
 8610   predicate(needs_acquiring_load_exclusive(n));
 8611   match(Set res (CompareAndSwapI mem (Binary oldval newval)));
 8612   ins_cost(VOLATILE_REF_COST);
 8613 
 8614   effect(KILL cr);
 8615 
 8616  format %{
 8617     "cmpxchgw_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 8618     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8619  %}
 8620 
 8621  ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
 8622             aarch64_enc_cset_eq(res));
 8623 
 8624   ins_pipe(pipe_slow);
 8625 %}
 8626 
 8627 instruct compareAndSwapLAcq(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
 8628 
 8629   predicate(needs_acquiring_load_exclusive(n));
 8630   match(Set res (CompareAndSwapL mem (Binary oldval newval)));
 8631   ins_cost(VOLATILE_REF_COST);
 8632 
 8633   effect(KILL cr);
 8634 
 8635  format %{
 8636     "cmpxchg_acq $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
 8637     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8638  %}
 8639 
 8640  ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
 8641             aarch64_enc_cset_eq(res));
 8642 
 8643   ins_pipe(pipe_slow);
 8644 %}
 8645 
 8646 instruct compareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 8647 
 8648   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 8649   match(Set res (CompareAndSwapP mem (Binary oldval newval)));
 8650   ins_cost(VOLATILE_REF_COST);
 8651 
 8652   effect(KILL cr);
 8653 
 8654  format %{
 8655     "cmpxchg_acq $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
 8656     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8657  %}
 8658 
 8659  ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
 8660             aarch64_enc_cset_eq(res));
 8661 
 8662   ins_pipe(pipe_slow);
 8663 %}
 8664 
 8665 instruct compareAndSwapNAcq(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
 8666 
 8667   predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
 8668   match(Set res (CompareAndSwapN mem (Binary oldval newval)));
 8669   ins_cost(VOLATILE_REF_COST);
 8670 
 8671   effect(KILL cr);
 8672 
 8673  format %{
 8674     "cmpxchgw_acq $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
 8675     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8676  %}
 8677 
 8678  ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
 8679             aarch64_enc_cset_eq(res));
 8680 
 8681   ins_pipe(pipe_slow);
 8682 %}
 8683 
 8684 
 8685 // ---------------------------------------------------------------------
 8686 
 8687 // BEGIN This section of the file is automatically generated. Do not edit --------------
 8688 
 8689 // Sundry CAS operations.  Note that release is always true,
 8690 // regardless of the memory ordering of the CAS.  This is because we
 8691 // need the volatile case to be sequentially consistent but there is
 8692 // no trailing StoreLoad barrier emitted by C2.  Unfortunately we
 8693 // can't check the type of memory ordering here, so we always emit a
 8694 // STLXR.
 8695 
 8696 // This section is generated from cas.m4
 8697 
 8698 
 8699 // This pattern is generated automatically from cas.m4.
 8700 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8701 instruct compareAndExchangeB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 8702   match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
 8703   ins_cost(2 * VOLATILE_REF_COST);
 8704   effect(TEMP_DEF res, KILL cr);
 8705   format %{
 8706     "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 8707   %}
 8708   ins_encode %{
 8709     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8710                Assembler::byte, /*acquire*/ false, /*release*/ true,
 8711                /*weak*/ false, $res$$Register);
 8712     __ sxtbw($res$$Register, $res$$Register);
 8713   %}
 8714   ins_pipe(pipe_slow);
 8715 %}
 8716 
 8717 // This pattern is generated automatically from cas.m4.
 8718 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8719 instruct compareAndExchangeS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 8720   match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
 8721   ins_cost(2 * VOLATILE_REF_COST);
 8722   effect(TEMP_DEF res, KILL cr);
 8723   format %{
 8724     "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 8725   %}
 8726   ins_encode %{
 8727     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8728                Assembler::halfword, /*acquire*/ false, /*release*/ true,
 8729                /*weak*/ false, $res$$Register);
 8730     __ sxthw($res$$Register, $res$$Register);
 8731   %}
 8732   ins_pipe(pipe_slow);
 8733 %}
 8734 
 8735 // This pattern is generated automatically from cas.m4.
 8736 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8737 instruct compareAndExchangeI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 8738   match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
 8739   ins_cost(2 * VOLATILE_REF_COST);
 8740   effect(TEMP_DEF res, KILL cr);
 8741   format %{
 8742     "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 8743   %}
 8744   ins_encode %{
 8745     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8746                Assembler::word, /*acquire*/ false, /*release*/ true,
 8747                /*weak*/ false, $res$$Register);
 8748   %}
 8749   ins_pipe(pipe_slow);
 8750 %}
 8751 
 8752 // This pattern is generated automatically from cas.m4.
 8753 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8754 instruct compareAndExchangeL(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 8755   match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
 8756   ins_cost(2 * VOLATILE_REF_COST);
 8757   effect(TEMP_DEF res, KILL cr);
 8758   format %{
 8759     "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 8760   %}
 8761   ins_encode %{
 8762     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8763                Assembler::xword, /*acquire*/ false, /*release*/ true,
 8764                /*weak*/ false, $res$$Register);
 8765   %}
 8766   ins_pipe(pipe_slow);
 8767 %}
 8768 
 8769 // This pattern is generated automatically from cas.m4.
 8770 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8771 instruct compareAndExchangeN(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 8772   predicate(n->as_LoadStore()->barrier_data() == 0);
 8773   match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
 8774   ins_cost(2 * VOLATILE_REF_COST);
 8775   effect(TEMP_DEF res, KILL cr);
 8776   format %{
 8777     "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 8778   %}
 8779   ins_encode %{
 8780     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8781                Assembler::word, /*acquire*/ false, /*release*/ true,
 8782                /*weak*/ false, $res$$Register);
 8783   %}
 8784   ins_pipe(pipe_slow);
 8785 %}
 8786 
 8787 // This pattern is generated automatically from cas.m4.
 8788 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8789 instruct compareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 8790   predicate(n->as_LoadStore()->barrier_data() == 0);
 8791   match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
 8792   ins_cost(2 * VOLATILE_REF_COST);
 8793   effect(TEMP_DEF res, KILL cr);
 8794   format %{
 8795     "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 8796   %}
 8797   ins_encode %{
 8798     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8799                Assembler::xword, /*acquire*/ false, /*release*/ true,
 8800                /*weak*/ false, $res$$Register);
 8801   %}
 8802   ins_pipe(pipe_slow);
 8803 %}
 8804 
 8805 // This pattern is generated automatically from cas.m4.
 8806 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8807 instruct compareAndExchangeBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 8808   predicate(needs_acquiring_load_exclusive(n));
 8809   match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
 8810   ins_cost(VOLATILE_REF_COST);
 8811   effect(TEMP_DEF res, KILL cr);
 8812   format %{
 8813     "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 8814   %}
 8815   ins_encode %{
 8816     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8817                Assembler::byte, /*acquire*/ true, /*release*/ true,
 8818                /*weak*/ false, $res$$Register);
 8819     __ sxtbw($res$$Register, $res$$Register);
 8820   %}
 8821   ins_pipe(pipe_slow);
 8822 %}
 8823 
 8824 // This pattern is generated automatically from cas.m4.
 8825 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8826 instruct compareAndExchangeSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 8827   predicate(needs_acquiring_load_exclusive(n));
 8828   match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
 8829   ins_cost(VOLATILE_REF_COST);
 8830   effect(TEMP_DEF res, KILL cr);
 8831   format %{
 8832     "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 8833   %}
 8834   ins_encode %{
 8835     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8836                Assembler::halfword, /*acquire*/ true, /*release*/ true,
 8837                /*weak*/ false, $res$$Register);
 8838     __ sxthw($res$$Register, $res$$Register);
 8839   %}
 8840   ins_pipe(pipe_slow);
 8841 %}
 8842 
 8843 // This pattern is generated automatically from cas.m4.
 8844 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8845 instruct compareAndExchangeIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 8846   predicate(needs_acquiring_load_exclusive(n));
 8847   match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
 8848   ins_cost(VOLATILE_REF_COST);
 8849   effect(TEMP_DEF res, KILL cr);
 8850   format %{
 8851     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 8852   %}
 8853   ins_encode %{
 8854     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8855                Assembler::word, /*acquire*/ true, /*release*/ true,
 8856                /*weak*/ false, $res$$Register);
 8857   %}
 8858   ins_pipe(pipe_slow);
 8859 %}
 8860 
 8861 // This pattern is generated automatically from cas.m4.
 8862 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8863 instruct compareAndExchangeLAcq(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 8864   predicate(needs_acquiring_load_exclusive(n));
 8865   match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
 8866   ins_cost(VOLATILE_REF_COST);
 8867   effect(TEMP_DEF res, KILL cr);
 8868   format %{
 8869     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 8870   %}
 8871   ins_encode %{
 8872     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8873                Assembler::xword, /*acquire*/ true, /*release*/ true,
 8874                /*weak*/ false, $res$$Register);
 8875   %}
 8876   ins_pipe(pipe_slow);
 8877 %}
 8878 
 8879 // This pattern is generated automatically from cas.m4.
 8880 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8881 instruct compareAndExchangeNAcq(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 8882   predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
 8883   match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
 8884   ins_cost(VOLATILE_REF_COST);
 8885   effect(TEMP_DEF res, KILL cr);
 8886   format %{
 8887     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 8888   %}
 8889   ins_encode %{
 8890     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8891                Assembler::word, /*acquire*/ true, /*release*/ true,
 8892                /*weak*/ false, $res$$Register);
 8893   %}
 8894   ins_pipe(pipe_slow);
 8895 %}
 8896 
 8897 // This pattern is generated automatically from cas.m4.
 8898 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8899 instruct compareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 8900   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 8901   match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
 8902   ins_cost(VOLATILE_REF_COST);
 8903   effect(TEMP_DEF res, KILL cr);
 8904   format %{
 8905     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 8906   %}
 8907   ins_encode %{
 8908     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8909                Assembler::xword, /*acquire*/ true, /*release*/ true,
 8910                /*weak*/ false, $res$$Register);
 8911   %}
 8912   ins_pipe(pipe_slow);
 8913 %}
 8914 
 8915 // This pattern is generated automatically from cas.m4.
 8916 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8917 instruct weakCompareAndSwapB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 8918   match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
 8919   ins_cost(2 * VOLATILE_REF_COST);
 8920   effect(KILL cr);
 8921   format %{
 8922     "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 8923     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8924   %}
 8925   ins_encode %{
 8926     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8927                Assembler::byte, /*acquire*/ false, /*release*/ true,
 8928                /*weak*/ true, noreg);
 8929     __ csetw($res$$Register, Assembler::EQ);
 8930   %}
 8931   ins_pipe(pipe_slow);
 8932 %}
 8933 
 8934 // This pattern is generated automatically from cas.m4.
 8935 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8936 instruct weakCompareAndSwapS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 8937   match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
 8938   ins_cost(2 * VOLATILE_REF_COST);
 8939   effect(KILL cr);
 8940   format %{
 8941     "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 8942     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8943   %}
 8944   ins_encode %{
 8945     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8946                Assembler::halfword, /*acquire*/ false, /*release*/ true,
 8947                /*weak*/ true, noreg);
 8948     __ csetw($res$$Register, Assembler::EQ);
 8949   %}
 8950   ins_pipe(pipe_slow);
 8951 %}
 8952 
 8953 // This pattern is generated automatically from cas.m4.
 8954 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8955 instruct weakCompareAndSwapI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 8956   match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
 8957   ins_cost(2 * VOLATILE_REF_COST);
 8958   effect(KILL cr);
 8959   format %{
 8960     "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 8961     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8962   %}
 8963   ins_encode %{
 8964     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8965                Assembler::word, /*acquire*/ false, /*release*/ true,
 8966                /*weak*/ true, noreg);
 8967     __ csetw($res$$Register, Assembler::EQ);
 8968   %}
 8969   ins_pipe(pipe_slow);
 8970 %}
 8971 
 8972 // This pattern is generated automatically from cas.m4.
 8973 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8974 instruct weakCompareAndSwapL(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 8975   match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
 8976   ins_cost(2 * VOLATILE_REF_COST);
 8977   effect(KILL cr);
 8978   format %{
 8979     "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 8980     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8981   %}
 8982   ins_encode %{
 8983     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8984                Assembler::xword, /*acquire*/ false, /*release*/ true,
 8985                /*weak*/ true, noreg);
 8986     __ csetw($res$$Register, Assembler::EQ);
 8987   %}
 8988   ins_pipe(pipe_slow);
 8989 %}
 8990 
 8991 // This pattern is generated automatically from cas.m4.
 8992 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8993 instruct weakCompareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 8994   predicate(n->as_LoadStore()->barrier_data() == 0);
 8995   match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
 8996   ins_cost(2 * VOLATILE_REF_COST);
 8997   effect(KILL cr);
 8998   format %{
 8999     "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 9000     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9001   %}
 9002   ins_encode %{
 9003     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9004                Assembler::word, /*acquire*/ false, /*release*/ true,
 9005                /*weak*/ true, noreg);
 9006     __ csetw($res$$Register, Assembler::EQ);
 9007   %}
 9008   ins_pipe(pipe_slow);
 9009 %}
 9010 
 9011 // This pattern is generated automatically from cas.m4.
 9012 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9013 instruct weakCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9014   predicate(n->as_LoadStore()->barrier_data() == 0);
 9015   match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
 9016   ins_cost(2 * VOLATILE_REF_COST);
 9017   effect(KILL cr);
 9018   format %{
 9019     "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 9020     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9021   %}
 9022   ins_encode %{
 9023     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9024                Assembler::xword, /*acquire*/ false, /*release*/ true,
 9025                /*weak*/ true, noreg);
 9026     __ csetw($res$$Register, Assembler::EQ);
 9027   %}
 9028   ins_pipe(pipe_slow);
 9029 %}
 9030 
 9031 // This pattern is generated automatically from cas.m4.
 9032 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9033 instruct weakCompareAndSwapBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9034   predicate(needs_acquiring_load_exclusive(n));
 9035   match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
 9036   ins_cost(VOLATILE_REF_COST);
 9037   effect(KILL cr);
 9038   format %{
 9039     "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 9040     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9041   %}
 9042   ins_encode %{
 9043     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9044                Assembler::byte, /*acquire*/ true, /*release*/ true,
 9045                /*weak*/ true, noreg);
 9046     __ csetw($res$$Register, Assembler::EQ);
 9047   %}
 9048   ins_pipe(pipe_slow);
 9049 %}
 9050 
 9051 // This pattern is generated automatically from cas.m4.
 9052 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9053 instruct weakCompareAndSwapSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9054   predicate(needs_acquiring_load_exclusive(n));
 9055   match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
 9056   ins_cost(VOLATILE_REF_COST);
 9057   effect(KILL cr);
 9058   format %{
 9059     "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 9060     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9061   %}
 9062   ins_encode %{
 9063     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9064                Assembler::halfword, /*acquire*/ true, /*release*/ true,
 9065                /*weak*/ true, noreg);
 9066     __ csetw($res$$Register, Assembler::EQ);
 9067   %}
 9068   ins_pipe(pipe_slow);
 9069 %}
 9070 
 9071 // This pattern is generated automatically from cas.m4.
 9072 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9073 instruct weakCompareAndSwapIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9074   predicate(needs_acquiring_load_exclusive(n));
 9075   match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
 9076   ins_cost(VOLATILE_REF_COST);
 9077   effect(KILL cr);
 9078   format %{
 9079     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 9080     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9081   %}
 9082   ins_encode %{
 9083     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9084                Assembler::word, /*acquire*/ true, /*release*/ true,
 9085                /*weak*/ true, noreg);
 9086     __ csetw($res$$Register, Assembler::EQ);
 9087   %}
 9088   ins_pipe(pipe_slow);
 9089 %}
 9090 
 9091 // This pattern is generated automatically from cas.m4.
 9092 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9093 instruct weakCompareAndSwapLAcq(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 9094   predicate(needs_acquiring_load_exclusive(n));
 9095   match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
 9096   ins_cost(VOLATILE_REF_COST);
 9097   effect(KILL cr);
 9098   format %{
 9099     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 9100     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9101   %}
 9102   ins_encode %{
 9103     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9104                Assembler::xword, /*acquire*/ true, /*release*/ true,
 9105                /*weak*/ true, noreg);
 9106     __ csetw($res$$Register, Assembler::EQ);
 9107   %}
 9108   ins_pipe(pipe_slow);
 9109 %}
 9110 
 9111 // This pattern is generated automatically from cas.m4.
 9112 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9113 instruct weakCompareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 9114   predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
 9115   match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
 9116   ins_cost(VOLATILE_REF_COST);
 9117   effect(KILL cr);
 9118   format %{
 9119     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 9120     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9121   %}
 9122   ins_encode %{
 9123     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9124                Assembler::word, /*acquire*/ true, /*release*/ true,
 9125                /*weak*/ true, noreg);
 9126     __ csetw($res$$Register, Assembler::EQ);
 9127   %}
 9128   ins_pipe(pipe_slow);
 9129 %}
 9130 
 9131 // This pattern is generated automatically from cas.m4.
 9132 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9133 instruct weakCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9134   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 9135   match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
 9136   ins_cost(VOLATILE_REF_COST);
 9137   effect(KILL cr);
 9138   format %{
 9139     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 9140     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9141   %}
 9142   ins_encode %{
 9143     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9144                Assembler::xword, /*acquire*/ true, /*release*/ true,
 9145                /*weak*/ true, noreg);
 9146     __ csetw($res$$Register, Assembler::EQ);
 9147   %}
 9148   ins_pipe(pipe_slow);
 9149 %}
 9150 
 9151 // END This section of the file is automatically generated. Do not edit --------------
 9152 // ---------------------------------------------------------------------
 9153 
 9154 instruct get_and_setI(indirect mem, iRegI newv, iRegINoSp prev) %{
 9155   match(Set prev (GetAndSetI mem newv));
 9156   ins_cost(2 * VOLATILE_REF_COST);
 9157   format %{ "atomic_xchgw  $prev, $newv, [$mem]" %}
 9158   ins_encode %{
 9159     __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9160   %}
 9161   ins_pipe(pipe_serial);
 9162 %}
 9163 
 9164 instruct get_and_setL(indirect mem, iRegL newv, iRegLNoSp prev) %{
 9165   match(Set prev (GetAndSetL mem newv));
 9166   ins_cost(2 * VOLATILE_REF_COST);
 9167   format %{ "atomic_xchg  $prev, $newv, [$mem]" %}
 9168   ins_encode %{
 9169     __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9170   %}
 9171   ins_pipe(pipe_serial);
 9172 %}
 9173 
 9174 instruct get_and_setN(indirect mem, iRegN newv, iRegINoSp prev) %{
 9175   predicate(n->as_LoadStore()->barrier_data() == 0);
 9176   match(Set prev (GetAndSetN mem newv));
 9177   ins_cost(2 * VOLATILE_REF_COST);
 9178   format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
 9179   ins_encode %{
 9180     __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9181   %}
 9182   ins_pipe(pipe_serial);
 9183 %}
 9184 
 9185 instruct get_and_setP(indirect mem, iRegP newv, iRegPNoSp prev) %{
 9186   predicate(n->as_LoadStore()->barrier_data() == 0);
 9187   match(Set prev (GetAndSetP mem newv));
 9188   ins_cost(2 * VOLATILE_REF_COST);
 9189   format %{ "atomic_xchg  $prev, $newv, [$mem]" %}
 9190   ins_encode %{
 9191     __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9192   %}
 9193   ins_pipe(pipe_serial);
 9194 %}
 9195 
 9196 instruct get_and_setIAcq(indirect mem, iRegI newv, iRegINoSp prev) %{
 9197   predicate(needs_acquiring_load_exclusive(n));
 9198   match(Set prev (GetAndSetI mem newv));
 9199   ins_cost(VOLATILE_REF_COST);
 9200   format %{ "atomic_xchgw_acq  $prev, $newv, [$mem]" %}
 9201   ins_encode %{
 9202     __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9203   %}
 9204   ins_pipe(pipe_serial);
 9205 %}
 9206 
 9207 instruct get_and_setLAcq(indirect mem, iRegL newv, iRegLNoSp prev) %{
 9208   predicate(needs_acquiring_load_exclusive(n));
 9209   match(Set prev (GetAndSetL mem newv));
 9210   ins_cost(VOLATILE_REF_COST);
 9211   format %{ "atomic_xchg_acq  $prev, $newv, [$mem]" %}
 9212   ins_encode %{
 9213     __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9214   %}
 9215   ins_pipe(pipe_serial);
 9216 %}
 9217 
 9218 instruct get_and_setNAcq(indirect mem, iRegN newv, iRegINoSp prev) %{
 9219   predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
 9220   match(Set prev (GetAndSetN mem newv));
 9221   ins_cost(VOLATILE_REF_COST);
 9222   format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
 9223   ins_encode %{
 9224     __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9225   %}
 9226   ins_pipe(pipe_serial);
 9227 %}
 9228 
 9229 instruct get_and_setPAcq(indirect mem, iRegP newv, iRegPNoSp prev) %{
 9230   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 9231   match(Set prev (GetAndSetP mem newv));
 9232   ins_cost(VOLATILE_REF_COST);
 9233   format %{ "atomic_xchg_acq  $prev, $newv, [$mem]" %}
 9234   ins_encode %{
 9235     __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9236   %}
 9237   ins_pipe(pipe_serial);
 9238 %}
 9239 
 9240 
 9241 instruct get_and_addL(indirect mem, iRegLNoSp newval, iRegL incr) %{
 9242   match(Set newval (GetAndAddL mem incr));
 9243   ins_cost(2 * VOLATILE_REF_COST + 1);
 9244   format %{ "get_and_addL $newval, [$mem], $incr" %}
 9245   ins_encode %{
 9246     __ atomic_add($newval$$Register, $incr$$Register, as_Register($mem$$base));
 9247   %}
 9248   ins_pipe(pipe_serial);
 9249 %}
 9250 
 9251 instruct get_and_addL_no_res(indirect mem, Universe dummy, iRegL incr) %{
 9252   predicate(n->as_LoadStore()->result_not_used());
 9253   match(Set dummy (GetAndAddL mem incr));
 9254   ins_cost(2 * VOLATILE_REF_COST);
 9255   format %{ "get_and_addL [$mem], $incr" %}
 9256   ins_encode %{
 9257     __ atomic_add(noreg, $incr$$Register, as_Register($mem$$base));
 9258   %}
 9259   ins_pipe(pipe_serial);
 9260 %}
 9261 
 9262 instruct get_and_addLi(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
 9263   match(Set newval (GetAndAddL mem incr));
 9264   ins_cost(2 * VOLATILE_REF_COST + 1);
 9265   format %{ "get_and_addL $newval, [$mem], $incr" %}
 9266   ins_encode %{
 9267     __ atomic_add($newval$$Register, $incr$$constant, as_Register($mem$$base));
 9268   %}
 9269   ins_pipe(pipe_serial);
 9270 %}
 9271 
 9272 instruct get_and_addLi_no_res(indirect mem, Universe dummy, immLAddSub incr) %{
 9273   predicate(n->as_LoadStore()->result_not_used());
 9274   match(Set dummy (GetAndAddL mem incr));
 9275   ins_cost(2 * VOLATILE_REF_COST);
 9276   format %{ "get_and_addL [$mem], $incr" %}
 9277   ins_encode %{
 9278     __ atomic_add(noreg, $incr$$constant, as_Register($mem$$base));
 9279   %}
 9280   ins_pipe(pipe_serial);
 9281 %}
 9282 
 9283 instruct get_and_addI(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
 9284   match(Set newval (GetAndAddI mem incr));
 9285   ins_cost(2 * VOLATILE_REF_COST + 1);
 9286   format %{ "get_and_addI $newval, [$mem], $incr" %}
 9287   ins_encode %{
 9288     __ atomic_addw($newval$$Register, $incr$$Register, as_Register($mem$$base));
 9289   %}
 9290   ins_pipe(pipe_serial);
 9291 %}
 9292 
 9293 instruct get_and_addI_no_res(indirect mem, Universe dummy, iRegIorL2I incr) %{
 9294   predicate(n->as_LoadStore()->result_not_used());
 9295   match(Set dummy (GetAndAddI mem incr));
 9296   ins_cost(2 * VOLATILE_REF_COST);
 9297   format %{ "get_and_addI [$mem], $incr" %}
 9298   ins_encode %{
 9299     __ atomic_addw(noreg, $incr$$Register, as_Register($mem$$base));
 9300   %}
 9301   ins_pipe(pipe_serial);
 9302 %}
 9303 
 9304 instruct get_and_addIi(indirect mem, iRegINoSp newval, immIAddSub incr) %{
 9305   match(Set newval (GetAndAddI mem incr));
 9306   ins_cost(2 * VOLATILE_REF_COST + 1);
 9307   format %{ "get_and_addI $newval, [$mem], $incr" %}
 9308   ins_encode %{
 9309     __ atomic_addw($newval$$Register, $incr$$constant, as_Register($mem$$base));
 9310   %}
 9311   ins_pipe(pipe_serial);
 9312 %}
 9313 
 9314 instruct get_and_addIi_no_res(indirect mem, Universe dummy, immIAddSub incr) %{
 9315   predicate(n->as_LoadStore()->result_not_used());
 9316   match(Set dummy (GetAndAddI mem incr));
 9317   ins_cost(2 * VOLATILE_REF_COST);
 9318   format %{ "get_and_addI [$mem], $incr" %}
 9319   ins_encode %{
 9320     __ atomic_addw(noreg, $incr$$constant, as_Register($mem$$base));
 9321   %}
 9322   ins_pipe(pipe_serial);
 9323 %}
 9324 
 9325 instruct get_and_addLAcq(indirect mem, iRegLNoSp newval, iRegL incr) %{
 9326   predicate(needs_acquiring_load_exclusive(n));
 9327   match(Set newval (GetAndAddL mem incr));
 9328   ins_cost(VOLATILE_REF_COST + 1);
 9329   format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
 9330   ins_encode %{
 9331     __ atomic_addal($newval$$Register, $incr$$Register, as_Register($mem$$base));
 9332   %}
 9333   ins_pipe(pipe_serial);
 9334 %}
 9335 
 9336 instruct get_and_addL_no_resAcq(indirect mem, Universe dummy, iRegL incr) %{
 9337   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
 9338   match(Set dummy (GetAndAddL mem incr));
 9339   ins_cost(VOLATILE_REF_COST);
 9340   format %{ "get_and_addL_acq [$mem], $incr" %}
 9341   ins_encode %{
 9342     __ atomic_addal(noreg, $incr$$Register, as_Register($mem$$base));
 9343   %}
 9344   ins_pipe(pipe_serial);
 9345 %}
 9346 
 9347 instruct get_and_addLiAcq(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
 9348   predicate(needs_acquiring_load_exclusive(n));
 9349   match(Set newval (GetAndAddL mem incr));
 9350   ins_cost(VOLATILE_REF_COST + 1);
 9351   format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
 9352   ins_encode %{
 9353     __ atomic_addal($newval$$Register, $incr$$constant, as_Register($mem$$base));
 9354   %}
 9355   ins_pipe(pipe_serial);
 9356 %}
 9357 
 9358 instruct get_and_addLi_no_resAcq(indirect mem, Universe dummy, immLAddSub incr) %{
 9359   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
 9360   match(Set dummy (GetAndAddL mem incr));
 9361   ins_cost(VOLATILE_REF_COST);
 9362   format %{ "get_and_addL_acq [$mem], $incr" %}
 9363   ins_encode %{
 9364     __ atomic_addal(noreg, $incr$$constant, as_Register($mem$$base));
 9365   %}
 9366   ins_pipe(pipe_serial);
 9367 %}
 9368 
 9369 instruct get_and_addIAcq(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
 9370   predicate(needs_acquiring_load_exclusive(n));
 9371   match(Set newval (GetAndAddI mem incr));
 9372   ins_cost(VOLATILE_REF_COST + 1);
 9373   format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
 9374   ins_encode %{
 9375     __ atomic_addalw($newval$$Register, $incr$$Register, as_Register($mem$$base));
 9376   %}
 9377   ins_pipe(pipe_serial);
 9378 %}
 9379 
 9380 instruct get_and_addI_no_resAcq(indirect mem, Universe dummy, iRegIorL2I incr) %{
 9381   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
 9382   match(Set dummy (GetAndAddI mem incr));
 9383   ins_cost(VOLATILE_REF_COST);
 9384   format %{ "get_and_addI_acq [$mem], $incr" %}
 9385   ins_encode %{
 9386     __ atomic_addalw(noreg, $incr$$Register, as_Register($mem$$base));
 9387   %}
 9388   ins_pipe(pipe_serial);
 9389 %}
 9390 
 9391 instruct get_and_addIiAcq(indirect mem, iRegINoSp newval, immIAddSub incr) %{
 9392   predicate(needs_acquiring_load_exclusive(n));
 9393   match(Set newval (GetAndAddI mem incr));
 9394   ins_cost(VOLATILE_REF_COST + 1);
 9395   format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
 9396   ins_encode %{
 9397     __ atomic_addalw($newval$$Register, $incr$$constant, as_Register($mem$$base));
 9398   %}
 9399   ins_pipe(pipe_serial);
 9400 %}
 9401 
 9402 instruct get_and_addIi_no_resAcq(indirect mem, Universe dummy, immIAddSub incr) %{
 9403   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
 9404   match(Set dummy (GetAndAddI mem incr));
 9405   ins_cost(VOLATILE_REF_COST);
 9406   format %{ "get_and_addI_acq [$mem], $incr" %}
 9407   ins_encode %{
 9408     __ atomic_addalw(noreg, $incr$$constant, as_Register($mem$$base));
 9409   %}
 9410   ins_pipe(pipe_serial);
 9411 %}
 9412 
 9413 // Manifest a CmpU result in an integer register.
 9414 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
 9415 instruct cmpU3_reg_reg(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg flags)
 9416 %{
 9417   match(Set dst (CmpU3 src1 src2));
 9418   effect(KILL flags);
 9419 
 9420   ins_cost(INSN_COST * 3);
 9421   format %{
 9422       "cmpw $src1, $src2\n\t"
 9423       "csetw $dst, ne\n\t"
 9424       "cnegw $dst, lo\t# CmpU3(reg)"
 9425   %}
 9426   ins_encode %{
 9427     __ cmpw($src1$$Register, $src2$$Register);
 9428     __ csetw($dst$$Register, Assembler::NE);
 9429     __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
 9430   %}
 9431 
 9432   ins_pipe(pipe_class_default);
 9433 %}
 9434 
 9435 instruct cmpU3_reg_imm(iRegINoSp dst, iRegI src1, immIAddSub src2, rFlagsReg flags)
 9436 %{
 9437   match(Set dst (CmpU3 src1 src2));
 9438   effect(KILL flags);
 9439 
 9440   ins_cost(INSN_COST * 3);
 9441   format %{
 9442       "subsw zr, $src1, $src2\n\t"
 9443       "csetw $dst, ne\n\t"
 9444       "cnegw $dst, lo\t# CmpU3(imm)"
 9445   %}
 9446   ins_encode %{
 9447     __ subsw(zr, $src1$$Register, (int32_t)$src2$$constant);
 9448     __ csetw($dst$$Register, Assembler::NE);
 9449     __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
 9450   %}
 9451 
 9452   ins_pipe(pipe_class_default);
 9453 %}
 9454 
 9455 // Manifest a CmpUL result in an integer register.
 9456 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
 9457 instruct cmpUL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
 9458 %{
 9459   match(Set dst (CmpUL3 src1 src2));
 9460   effect(KILL flags);
 9461 
 9462   ins_cost(INSN_COST * 3);
 9463   format %{
 9464       "cmp $src1, $src2\n\t"
 9465       "csetw $dst, ne\n\t"
 9466       "cnegw $dst, lo\t# CmpUL3(reg)"
 9467   %}
 9468   ins_encode %{
 9469     __ cmp($src1$$Register, $src2$$Register);
 9470     __ csetw($dst$$Register, Assembler::NE);
 9471     __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
 9472   %}
 9473 
 9474   ins_pipe(pipe_class_default);
 9475 %}
 9476 
 9477 instruct cmpUL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
 9478 %{
 9479   match(Set dst (CmpUL3 src1 src2));
 9480   effect(KILL flags);
 9481 
 9482   ins_cost(INSN_COST * 3);
 9483   format %{
 9484       "subs zr, $src1, $src2\n\t"
 9485       "csetw $dst, ne\n\t"
 9486       "cnegw $dst, lo\t# CmpUL3(imm)"
 9487   %}
 9488   ins_encode %{
 9489     __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
 9490     __ csetw($dst$$Register, Assembler::NE);
 9491     __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
 9492   %}
 9493 
 9494   ins_pipe(pipe_class_default);
 9495 %}
 9496 
 9497 // Manifest a CmpL result in an integer register.
 9498 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
 9499 instruct cmpL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
 9500 %{
 9501   match(Set dst (CmpL3 src1 src2));
 9502   effect(KILL flags);
 9503 
 9504   ins_cost(INSN_COST * 3);
 9505   format %{
 9506       "cmp $src1, $src2\n\t"
 9507       "csetw $dst, ne\n\t"
 9508       "cnegw $dst, lt\t# CmpL3(reg)"
 9509   %}
 9510   ins_encode %{
 9511     __ cmp($src1$$Register, $src2$$Register);
 9512     __ csetw($dst$$Register, Assembler::NE);
 9513     __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
 9514   %}
 9515 
 9516   ins_pipe(pipe_class_default);
 9517 %}
 9518 
 9519 instruct cmpL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
 9520 %{
 9521   match(Set dst (CmpL3 src1 src2));
 9522   effect(KILL flags);
 9523 
 9524   ins_cost(INSN_COST * 3);
 9525   format %{
 9526       "subs zr, $src1, $src2\n\t"
 9527       "csetw $dst, ne\n\t"
 9528       "cnegw $dst, lt\t# CmpL3(imm)"
 9529   %}
 9530   ins_encode %{
 9531     __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
 9532     __ csetw($dst$$Register, Assembler::NE);
 9533     __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
 9534   %}
 9535 
 9536   ins_pipe(pipe_class_default);
 9537 %}
 9538 
 9539 // ============================================================================
 9540 // Conditional Move Instructions
 9541 
 9542 // n.b. we have identical rules for both a signed compare op (cmpOp)
 9543 // and an unsigned compare op (cmpOpU). it would be nice if we could
 9544 // define an op class which merged both inputs and use it to type the
 9545 // argument to a single rule. unfortunatelyt his fails because the
 9546 // opclass does not live up to the COND_INTER interface of its
 9547 // component operands. When the generic code tries to negate the
 9548 // operand it ends up running the generci Machoper::negate method
 9549 // which throws a ShouldNotHappen. So, we have to provide two flavours
 9550 // of each rule, one for a cmpOp and a second for a cmpOpU (sigh).
 9551 
 9552 instruct cmovI_reg_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
 9553   match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
 9554 
 9555   ins_cost(INSN_COST * 2);
 9556   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, int"  %}
 9557 
 9558   ins_encode %{
 9559     __ cselw(as_Register($dst$$reg),
 9560              as_Register($src2$$reg),
 9561              as_Register($src1$$reg),
 9562              (Assembler::Condition)$cmp$$cmpcode);
 9563   %}
 9564 
 9565   ins_pipe(icond_reg_reg);
 9566 %}
 9567 
 9568 instruct cmovUI_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
 9569   match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
 9570 
 9571   ins_cost(INSN_COST * 2);
 9572   format %{ "cselw $dst, $src2, $src1 $cmp\t# unsigned, int"  %}
 9573 
 9574   ins_encode %{
 9575     __ cselw(as_Register($dst$$reg),
 9576              as_Register($src2$$reg),
 9577              as_Register($src1$$reg),
 9578              (Assembler::Condition)$cmp$$cmpcode);
 9579   %}
 9580 
 9581   ins_pipe(icond_reg_reg);
 9582 %}
 9583 
 9584 // special cases where one arg is zero
 9585 
 9586 // n.b. this is selected in preference to the rule above because it
 9587 // avoids loading constant 0 into a source register
 9588 
 9589 // TODO
 9590 // we ought only to be able to cull one of these variants as the ideal
 9591 // transforms ought always to order the zero consistently (to left/right?)
 9592 
 9593 instruct cmovI_zero_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
 9594   match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
 9595 
 9596   ins_cost(INSN_COST * 2);
 9597   format %{ "cselw $dst, $src, zr $cmp\t# signed, int"  %}
 9598 
 9599   ins_encode %{
 9600     __ cselw(as_Register($dst$$reg),
 9601              as_Register($src$$reg),
 9602              zr,
 9603              (Assembler::Condition)$cmp$$cmpcode);
 9604   %}
 9605 
 9606   ins_pipe(icond_reg);
 9607 %}
 9608 
 9609 instruct cmovUI_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
 9610   match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
 9611 
 9612   ins_cost(INSN_COST * 2);
 9613   format %{ "cselw $dst, $src, zr $cmp\t# unsigned, int"  %}
 9614 
 9615   ins_encode %{
 9616     __ cselw(as_Register($dst$$reg),
 9617              as_Register($src$$reg),
 9618              zr,
 9619              (Assembler::Condition)$cmp$$cmpcode);
 9620   %}
 9621 
 9622   ins_pipe(icond_reg);
 9623 %}
 9624 
 9625 instruct cmovI_reg_zero(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
 9626   match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
 9627 
 9628   ins_cost(INSN_COST * 2);
 9629   format %{ "cselw $dst, zr, $src $cmp\t# signed, int"  %}
 9630 
 9631   ins_encode %{
 9632     __ cselw(as_Register($dst$$reg),
 9633              zr,
 9634              as_Register($src$$reg),
 9635              (Assembler::Condition)$cmp$$cmpcode);
 9636   %}
 9637 
 9638   ins_pipe(icond_reg);
 9639 %}
 9640 
 9641 instruct cmovUI_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
 9642   match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
 9643 
 9644   ins_cost(INSN_COST * 2);
 9645   format %{ "cselw $dst, zr, $src $cmp\t# unsigned, int"  %}
 9646 
 9647   ins_encode %{
 9648     __ cselw(as_Register($dst$$reg),
 9649              zr,
 9650              as_Register($src$$reg),
 9651              (Assembler::Condition)$cmp$$cmpcode);
 9652   %}
 9653 
 9654   ins_pipe(icond_reg);
 9655 %}
 9656 
 9657 // special case for creating a boolean 0 or 1
 9658 
 9659 // n.b. this is selected in preference to the rule above because it
 9660 // avoids loading constants 0 and 1 into a source register
 9661 
 9662 instruct cmovI_reg_zero_one(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
 9663   match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
 9664 
 9665   ins_cost(INSN_COST * 2);
 9666   format %{ "csincw $dst, zr, zr $cmp\t# signed, int"  %}
 9667 
 9668   ins_encode %{
 9669     // equivalently
 9670     // cset(as_Register($dst$$reg),
 9671     //      negate_condition((Assembler::Condition)$cmp$$cmpcode));
 9672     __ csincw(as_Register($dst$$reg),
 9673              zr,
 9674              zr,
 9675              (Assembler::Condition)$cmp$$cmpcode);
 9676   %}
 9677 
 9678   ins_pipe(icond_none);
 9679 %}
 9680 
 9681 instruct cmovUI_reg_zero_one(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
 9682   match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
 9683 
 9684   ins_cost(INSN_COST * 2);
 9685   format %{ "csincw $dst, zr, zr $cmp\t# unsigned, int"  %}
 9686 
 9687   ins_encode %{
 9688     // equivalently
 9689     // cset(as_Register($dst$$reg),
 9690     //      negate_condition((Assembler::Condition)$cmp$$cmpcode));
 9691     __ csincw(as_Register($dst$$reg),
 9692              zr,
 9693              zr,
 9694              (Assembler::Condition)$cmp$$cmpcode);
 9695   %}
 9696 
 9697   ins_pipe(icond_none);
 9698 %}
 9699 
 9700 instruct cmovL_reg_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
 9701   match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
 9702 
 9703   ins_cost(INSN_COST * 2);
 9704   format %{ "csel $dst, $src2, $src1 $cmp\t# signed, long"  %}
 9705 
 9706   ins_encode %{
 9707     __ csel(as_Register($dst$$reg),
 9708             as_Register($src2$$reg),
 9709             as_Register($src1$$reg),
 9710             (Assembler::Condition)$cmp$$cmpcode);
 9711   %}
 9712 
 9713   ins_pipe(icond_reg_reg);
 9714 %}
 9715 
 9716 instruct cmovUL_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
 9717   match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
 9718 
 9719   ins_cost(INSN_COST * 2);
 9720   format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, long"  %}
 9721 
 9722   ins_encode %{
 9723     __ csel(as_Register($dst$$reg),
 9724             as_Register($src2$$reg),
 9725             as_Register($src1$$reg),
 9726             (Assembler::Condition)$cmp$$cmpcode);
 9727   %}
 9728 
 9729   ins_pipe(icond_reg_reg);
 9730 %}
 9731 
 9732 // special cases where one arg is zero
 9733 
 9734 instruct cmovL_reg_zero(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
 9735   match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
 9736 
 9737   ins_cost(INSN_COST * 2);
 9738   format %{ "csel $dst, zr, $src $cmp\t# signed, long"  %}
 9739 
 9740   ins_encode %{
 9741     __ csel(as_Register($dst$$reg),
 9742             zr,
 9743             as_Register($src$$reg),
 9744             (Assembler::Condition)$cmp$$cmpcode);
 9745   %}
 9746 
 9747   ins_pipe(icond_reg);
 9748 %}
 9749 
 9750 instruct cmovUL_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
 9751   match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
 9752 
 9753   ins_cost(INSN_COST * 2);
 9754   format %{ "csel $dst, zr, $src $cmp\t# unsigned, long"  %}
 9755 
 9756   ins_encode %{
 9757     __ csel(as_Register($dst$$reg),
 9758             zr,
 9759             as_Register($src$$reg),
 9760             (Assembler::Condition)$cmp$$cmpcode);
 9761   %}
 9762 
 9763   ins_pipe(icond_reg);
 9764 %}
 9765 
 9766 instruct cmovL_zero_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
 9767   match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
 9768 
 9769   ins_cost(INSN_COST * 2);
 9770   format %{ "csel $dst, $src, zr $cmp\t# signed, long"  %}
 9771 
 9772   ins_encode %{
 9773     __ csel(as_Register($dst$$reg),
 9774             as_Register($src$$reg),
 9775             zr,
 9776             (Assembler::Condition)$cmp$$cmpcode);
 9777   %}
 9778 
 9779   ins_pipe(icond_reg);
 9780 %}
 9781 
 9782 instruct cmovUL_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
 9783   match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
 9784 
 9785   ins_cost(INSN_COST * 2);
 9786   format %{ "csel $dst, $src, zr $cmp\t# unsigned, long"  %}
 9787 
 9788   ins_encode %{
 9789     __ csel(as_Register($dst$$reg),
 9790             as_Register($src$$reg),
 9791             zr,
 9792             (Assembler::Condition)$cmp$$cmpcode);
 9793   %}
 9794 
 9795   ins_pipe(icond_reg);
 9796 %}
 9797 
 9798 instruct cmovP_reg_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
 9799   match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
 9800 
 9801   ins_cost(INSN_COST * 2);
 9802   format %{ "csel $dst, $src2, $src1 $cmp\t# signed, ptr"  %}
 9803 
 9804   ins_encode %{
 9805     __ csel(as_Register($dst$$reg),
 9806             as_Register($src2$$reg),
 9807             as_Register($src1$$reg),
 9808             (Assembler::Condition)$cmp$$cmpcode);
 9809   %}
 9810 
 9811   ins_pipe(icond_reg_reg);
 9812 %}
 9813 
 9814 instruct cmovUP_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
 9815   match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
 9816 
 9817   ins_cost(INSN_COST * 2);
 9818   format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, ptr"  %}
 9819 
 9820   ins_encode %{
 9821     __ csel(as_Register($dst$$reg),
 9822             as_Register($src2$$reg),
 9823             as_Register($src1$$reg),
 9824             (Assembler::Condition)$cmp$$cmpcode);
 9825   %}
 9826 
 9827   ins_pipe(icond_reg_reg);
 9828 %}
 9829 
 9830 // special cases where one arg is zero
 9831 
 9832 instruct cmovP_reg_zero(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
 9833   match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
 9834 
 9835   ins_cost(INSN_COST * 2);
 9836   format %{ "csel $dst, zr, $src $cmp\t# signed, ptr"  %}
 9837 
 9838   ins_encode %{
 9839     __ csel(as_Register($dst$$reg),
 9840             zr,
 9841             as_Register($src$$reg),
 9842             (Assembler::Condition)$cmp$$cmpcode);
 9843   %}
 9844 
 9845   ins_pipe(icond_reg);
 9846 %}
 9847 
 9848 instruct cmovUP_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
 9849   match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
 9850 
 9851   ins_cost(INSN_COST * 2);
 9852   format %{ "csel $dst, zr, $src $cmp\t# unsigned, ptr"  %}
 9853 
 9854   ins_encode %{
 9855     __ csel(as_Register($dst$$reg),
 9856             zr,
 9857             as_Register($src$$reg),
 9858             (Assembler::Condition)$cmp$$cmpcode);
 9859   %}
 9860 
 9861   ins_pipe(icond_reg);
 9862 %}
 9863 
 9864 instruct cmovP_zero_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
 9865   match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
 9866 
 9867   ins_cost(INSN_COST * 2);
 9868   format %{ "csel $dst, $src, zr $cmp\t# signed, ptr"  %}
 9869 
 9870   ins_encode %{
 9871     __ csel(as_Register($dst$$reg),
 9872             as_Register($src$$reg),
 9873             zr,
 9874             (Assembler::Condition)$cmp$$cmpcode);
 9875   %}
 9876 
 9877   ins_pipe(icond_reg);
 9878 %}
 9879 
 9880 instruct cmovUP_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
 9881   match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
 9882 
 9883   ins_cost(INSN_COST * 2);
 9884   format %{ "csel $dst, $src, zr $cmp\t# unsigned, ptr"  %}
 9885 
 9886   ins_encode %{
 9887     __ csel(as_Register($dst$$reg),
 9888             as_Register($src$$reg),
 9889             zr,
 9890             (Assembler::Condition)$cmp$$cmpcode);
 9891   %}
 9892 
 9893   ins_pipe(icond_reg);
 9894 %}
 9895 
 9896 instruct cmovN_reg_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
 9897   match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
 9898 
 9899   ins_cost(INSN_COST * 2);
 9900   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr"  %}
 9901 
 9902   ins_encode %{
 9903     __ cselw(as_Register($dst$$reg),
 9904              as_Register($src2$$reg),
 9905              as_Register($src1$$reg),
 9906              (Assembler::Condition)$cmp$$cmpcode);
 9907   %}
 9908 
 9909   ins_pipe(icond_reg_reg);
 9910 %}
 9911 
 9912 instruct cmovUN_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
 9913   match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
 9914 
 9915   ins_cost(INSN_COST * 2);
 9916   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr"  %}
 9917 
 9918   ins_encode %{
 9919     __ cselw(as_Register($dst$$reg),
 9920              as_Register($src2$$reg),
 9921              as_Register($src1$$reg),
 9922              (Assembler::Condition)$cmp$$cmpcode);
 9923   %}
 9924 
 9925   ins_pipe(icond_reg_reg);
 9926 %}
 9927 
 9928 // special cases where one arg is zero
 9929 
 9930 instruct cmovN_reg_zero(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
 9931   match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
 9932 
 9933   ins_cost(INSN_COST * 2);
 9934   format %{ "cselw $dst, zr, $src $cmp\t# signed, compressed ptr"  %}
 9935 
 9936   ins_encode %{
 9937     __ cselw(as_Register($dst$$reg),
 9938              zr,
 9939              as_Register($src$$reg),
 9940              (Assembler::Condition)$cmp$$cmpcode);
 9941   %}
 9942 
 9943   ins_pipe(icond_reg);
 9944 %}
 9945 
 9946 instruct cmovUN_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
 9947   match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
 9948 
 9949   ins_cost(INSN_COST * 2);
 9950   format %{ "cselw $dst, zr, $src $cmp\t# unsigned, compressed ptr"  %}
 9951 
 9952   ins_encode %{
 9953     __ cselw(as_Register($dst$$reg),
 9954              zr,
 9955              as_Register($src$$reg),
 9956              (Assembler::Condition)$cmp$$cmpcode);
 9957   %}
 9958 
 9959   ins_pipe(icond_reg);
 9960 %}
 9961 
 9962 instruct cmovN_zero_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
 9963   match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
 9964 
 9965   ins_cost(INSN_COST * 2);
 9966   format %{ "cselw $dst, $src, zr $cmp\t# signed, compressed ptr"  %}
 9967 
 9968   ins_encode %{
 9969     __ cselw(as_Register($dst$$reg),
 9970              as_Register($src$$reg),
 9971              zr,
 9972              (Assembler::Condition)$cmp$$cmpcode);
 9973   %}
 9974 
 9975   ins_pipe(icond_reg);
 9976 %}
 9977 
 9978 instruct cmovUN_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
 9979   match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
 9980 
 9981   ins_cost(INSN_COST * 2);
 9982   format %{ "cselw $dst, $src, zr $cmp\t# unsigned, compressed ptr"  %}
 9983 
 9984   ins_encode %{
 9985     __ cselw(as_Register($dst$$reg),
 9986              as_Register($src$$reg),
 9987              zr,
 9988              (Assembler::Condition)$cmp$$cmpcode);
 9989   %}
 9990 
 9991   ins_pipe(icond_reg);
 9992 %}
 9993 
 9994 instruct cmovF_reg(cmpOp cmp, rFlagsReg cr, vRegF dst, vRegF src1,  vRegF src2)
 9995 %{
 9996   match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
 9997 
 9998   ins_cost(INSN_COST * 3);
 9999 
10000   format %{ "fcsels $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
10001   ins_encode %{
10002     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10003     __ fcsels(as_FloatRegister($dst$$reg),
10004               as_FloatRegister($src2$$reg),
10005               as_FloatRegister($src1$$reg),
10006               cond);
10007   %}
10008 
10009   ins_pipe(fp_cond_reg_reg_s);
10010 %}
10011 
10012 instruct cmovUF_reg(cmpOpU cmp, rFlagsRegU cr, vRegF dst, vRegF src1,  vRegF src2)
10013 %{
10014   match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
10015 
10016   ins_cost(INSN_COST * 3);
10017 
10018   format %{ "fcsels $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
10019   ins_encode %{
10020     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10021     __ fcsels(as_FloatRegister($dst$$reg),
10022               as_FloatRegister($src2$$reg),
10023               as_FloatRegister($src1$$reg),
10024               cond);
10025   %}
10026 
10027   ins_pipe(fp_cond_reg_reg_s);
10028 %}
10029 
10030 instruct cmovD_reg(cmpOp cmp, rFlagsReg cr, vRegD dst, vRegD src1,  vRegD src2)
10031 %{
10032   match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
10033 
10034   ins_cost(INSN_COST * 3);
10035 
10036   format %{ "fcseld $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
10037   ins_encode %{
10038     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10039     __ fcseld(as_FloatRegister($dst$$reg),
10040               as_FloatRegister($src2$$reg),
10041               as_FloatRegister($src1$$reg),
10042               cond);
10043   %}
10044 
10045   ins_pipe(fp_cond_reg_reg_d);
10046 %}
10047 
10048 instruct cmovUD_reg(cmpOpU cmp, rFlagsRegU cr, vRegD dst, vRegD src1,  vRegD src2)
10049 %{
10050   match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
10051 
10052   ins_cost(INSN_COST * 3);
10053 
10054   format %{ "fcseld $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
10055   ins_encode %{
10056     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10057     __ fcseld(as_FloatRegister($dst$$reg),
10058               as_FloatRegister($src2$$reg),
10059               as_FloatRegister($src1$$reg),
10060               cond);
10061   %}
10062 
10063   ins_pipe(fp_cond_reg_reg_d);
10064 %}
10065 
10066 // ============================================================================
10067 // Arithmetic Instructions
10068 //
10069 
10070 // Integer Addition
10071 
10072 // TODO
10073 // these currently employ operations which do not set CR and hence are
10074 // not flagged as killing CR but we would like to isolate the cases
10075 // where we want to set flags from those where we don't. need to work
10076 // out how to do that.
10077 
10078 instruct addI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10079   match(Set dst (AddI src1 src2));
10080 
10081   ins_cost(INSN_COST);
10082   format %{ "addw  $dst, $src1, $src2" %}
10083 
10084   ins_encode %{
10085     __ addw(as_Register($dst$$reg),
10086             as_Register($src1$$reg),
10087             as_Register($src2$$reg));
10088   %}
10089 
10090   ins_pipe(ialu_reg_reg);
10091 %}
10092 
10093 instruct addI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10094   match(Set dst (AddI src1 src2));
10095 
10096   ins_cost(INSN_COST);
10097   format %{ "addw $dst, $src1, $src2" %}
10098 
10099   // use opcode to indicate that this is an add not a sub
10100   opcode(0x0);
10101 
10102   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10103 
10104   ins_pipe(ialu_reg_imm);
10105 %}
10106 
10107 instruct addI_reg_imm_i2l(iRegINoSp dst, iRegL src1, immIAddSub src2) %{
10108   match(Set dst (AddI (ConvL2I src1) src2));
10109 
10110   ins_cost(INSN_COST);
10111   format %{ "addw $dst, $src1, $src2" %}
10112 
10113   // use opcode to indicate that this is an add not a sub
10114   opcode(0x0);
10115 
10116   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10117 
10118   ins_pipe(ialu_reg_imm);
10119 %}
10120 
10121 // Pointer Addition
10122 instruct addP_reg_reg(iRegPNoSp dst, iRegPorL2P src1, iRegL src2) %{
10123   match(Set dst (AddP src1 src2));
10124 
10125   ins_cost(INSN_COST);
10126   format %{ "add $dst, $src1, $src2\t# ptr" %}
10127 
10128   ins_encode %{
10129     __ add(as_Register($dst$$reg),
10130            as_Register($src1$$reg),
10131            as_Register($src2$$reg));
10132   %}
10133 
10134   ins_pipe(ialu_reg_reg);
10135 %}
10136 
10137 instruct addP_reg_reg_ext(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2) %{
10138   match(Set dst (AddP src1 (ConvI2L src2)));
10139 
10140   ins_cost(1.9 * INSN_COST);
10141   format %{ "add $dst, $src1, $src2, sxtw\t# ptr" %}
10142 
10143   ins_encode %{
10144     __ add(as_Register($dst$$reg),
10145            as_Register($src1$$reg),
10146            as_Register($src2$$reg), ext::sxtw);
10147   %}
10148 
10149   ins_pipe(ialu_reg_reg);
10150 %}
10151 
10152 instruct addP_reg_reg_lsl(iRegPNoSp dst, iRegPorL2P src1, iRegL src2, immIScale scale) %{
10153   match(Set dst (AddP src1 (LShiftL src2 scale)));
10154 
10155   ins_cost(1.9 * INSN_COST);
10156   format %{ "add $dst, $src1, $src2, LShiftL $scale\t# ptr" %}
10157 
10158   ins_encode %{
10159     __ lea(as_Register($dst$$reg),
10160            Address(as_Register($src1$$reg), as_Register($src2$$reg),
10161                    Address::lsl($scale$$constant)));
10162   %}
10163 
10164   ins_pipe(ialu_reg_reg_shift);
10165 %}
10166 
10167 instruct addP_reg_reg_ext_shift(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2, immIScale scale) %{
10168   match(Set dst (AddP src1 (LShiftL (ConvI2L src2) scale)));
10169 
10170   ins_cost(1.9 * INSN_COST);
10171   format %{ "add $dst, $src1, $src2, I2L $scale\t# ptr" %}
10172 
10173   ins_encode %{
10174     __ lea(as_Register($dst$$reg),
10175            Address(as_Register($src1$$reg), as_Register($src2$$reg),
10176                    Address::sxtw($scale$$constant)));
10177   %}
10178 
10179   ins_pipe(ialu_reg_reg_shift);
10180 %}
10181 
10182 instruct lshift_ext(iRegLNoSp dst, iRegIorL2I src, immI scale, rFlagsReg cr) %{
10183   match(Set dst (LShiftL (ConvI2L src) scale));
10184 
10185   ins_cost(INSN_COST);
10186   format %{ "sbfiz $dst, $src, $scale & 63, -$scale & 63\t" %}
10187 
10188   ins_encode %{
10189     __ sbfiz(as_Register($dst$$reg),
10190           as_Register($src$$reg),
10191           $scale$$constant & 63, MIN2(32, (int)((-$scale$$constant) & 63)));
10192   %}
10193 
10194   ins_pipe(ialu_reg_shift);
10195 %}
10196 
10197 // Pointer Immediate Addition
10198 // n.b. this needs to be more expensive than using an indirect memory
10199 // operand
10200 instruct addP_reg_imm(iRegPNoSp dst, iRegPorL2P src1, immLAddSub src2) %{
10201   match(Set dst (AddP src1 src2));
10202 
10203   ins_cost(INSN_COST);
10204   format %{ "add $dst, $src1, $src2\t# ptr" %}
10205 
10206   // use opcode to indicate that this is an add not a sub
10207   opcode(0x0);
10208 
10209   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10210 
10211   ins_pipe(ialu_reg_imm);
10212 %}
10213 
10214 // Long Addition
10215 instruct addL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10216 
10217   match(Set dst (AddL src1 src2));
10218 
10219   ins_cost(INSN_COST);
10220   format %{ "add  $dst, $src1, $src2" %}
10221 
10222   ins_encode %{
10223     __ add(as_Register($dst$$reg),
10224            as_Register($src1$$reg),
10225            as_Register($src2$$reg));
10226   %}
10227 
10228   ins_pipe(ialu_reg_reg);
10229 %}
10230 
10231 // No constant pool entries requiredLong Immediate Addition.
10232 instruct addL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10233   match(Set dst (AddL src1 src2));
10234 
10235   ins_cost(INSN_COST);
10236   format %{ "add $dst, $src1, $src2" %}
10237 
10238   // use opcode to indicate that this is an add not a sub
10239   opcode(0x0);
10240 
10241   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10242 
10243   ins_pipe(ialu_reg_imm);
10244 %}
10245 
10246 // Integer Subtraction
10247 instruct subI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10248   match(Set dst (SubI src1 src2));
10249 
10250   ins_cost(INSN_COST);
10251   format %{ "subw  $dst, $src1, $src2" %}
10252 
10253   ins_encode %{
10254     __ subw(as_Register($dst$$reg),
10255             as_Register($src1$$reg),
10256             as_Register($src2$$reg));
10257   %}
10258 
10259   ins_pipe(ialu_reg_reg);
10260 %}
10261 
10262 // Immediate Subtraction
10263 instruct subI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10264   match(Set dst (SubI src1 src2));
10265 
10266   ins_cost(INSN_COST);
10267   format %{ "subw $dst, $src1, $src2" %}
10268 
10269   // use opcode to indicate that this is a sub not an add
10270   opcode(0x1);
10271 
10272   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10273 
10274   ins_pipe(ialu_reg_imm);
10275 %}
10276 
10277 // Long Subtraction
10278 instruct subL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10279 
10280   match(Set dst (SubL src1 src2));
10281 
10282   ins_cost(INSN_COST);
10283   format %{ "sub  $dst, $src1, $src2" %}
10284 
10285   ins_encode %{
10286     __ sub(as_Register($dst$$reg),
10287            as_Register($src1$$reg),
10288            as_Register($src2$$reg));
10289   %}
10290 
10291   ins_pipe(ialu_reg_reg);
10292 %}
10293 
10294 // No constant pool entries requiredLong Immediate Subtraction.
10295 instruct subL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10296   match(Set dst (SubL src1 src2));
10297 
10298   ins_cost(INSN_COST);
10299   format %{ "sub$dst, $src1, $src2" %}
10300 
10301   // use opcode to indicate that this is a sub not an add
10302   opcode(0x1);
10303 
10304   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10305 
10306   ins_pipe(ialu_reg_imm);
10307 %}
10308 
10309 // Integer Negation (special case for sub)
10310 
10311 instruct negI_reg(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr) %{
10312   match(Set dst (SubI zero src));
10313 
10314   ins_cost(INSN_COST);
10315   format %{ "negw $dst, $src\t# int" %}
10316 
10317   ins_encode %{
10318     __ negw(as_Register($dst$$reg),
10319             as_Register($src$$reg));
10320   %}
10321 
10322   ins_pipe(ialu_reg);
10323 %}
10324 
10325 // Long Negation
10326 
10327 instruct negL_reg(iRegLNoSp dst, iRegL src, immL0 zero, rFlagsReg cr) %{
10328   match(Set dst (SubL zero src));
10329 
10330   ins_cost(INSN_COST);
10331   format %{ "neg $dst, $src\t# long" %}
10332 
10333   ins_encode %{
10334     __ neg(as_Register($dst$$reg),
10335            as_Register($src$$reg));
10336   %}
10337 
10338   ins_pipe(ialu_reg);
10339 %}
10340 
10341 // Integer Multiply
10342 
10343 instruct mulI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10344   match(Set dst (MulI src1 src2));
10345 
10346   ins_cost(INSN_COST * 3);
10347   format %{ "mulw  $dst, $src1, $src2" %}
10348 
10349   ins_encode %{
10350     __ mulw(as_Register($dst$$reg),
10351             as_Register($src1$$reg),
10352             as_Register($src2$$reg));
10353   %}
10354 
10355   ins_pipe(imul_reg_reg);
10356 %}
10357 
10358 instruct smulI(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10359   match(Set dst (MulL (ConvI2L src1) (ConvI2L src2)));
10360 
10361   ins_cost(INSN_COST * 3);
10362   format %{ "smull  $dst, $src1, $src2" %}
10363 
10364   ins_encode %{
10365     __ smull(as_Register($dst$$reg),
10366              as_Register($src1$$reg),
10367              as_Register($src2$$reg));
10368   %}
10369 
10370   ins_pipe(imul_reg_reg);
10371 %}
10372 
10373 // Long Multiply
10374 
10375 instruct mulL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10376   match(Set dst (MulL src1 src2));
10377 
10378   ins_cost(INSN_COST * 5);
10379   format %{ "mul  $dst, $src1, $src2" %}
10380 
10381   ins_encode %{
10382     __ mul(as_Register($dst$$reg),
10383            as_Register($src1$$reg),
10384            as_Register($src2$$reg));
10385   %}
10386 
10387   ins_pipe(lmul_reg_reg);
10388 %}
10389 
10390 instruct mulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10391 %{
10392   match(Set dst (MulHiL src1 src2));
10393 
10394   ins_cost(INSN_COST * 7);
10395   format %{ "smulh   $dst, $src1, $src2\t# mulhi" %}
10396 
10397   ins_encode %{
10398     __ smulh(as_Register($dst$$reg),
10399              as_Register($src1$$reg),
10400              as_Register($src2$$reg));
10401   %}
10402 
10403   ins_pipe(lmul_reg_reg);
10404 %}
10405 
10406 instruct umulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10407 %{
10408   match(Set dst (UMulHiL src1 src2));
10409 
10410   ins_cost(INSN_COST * 7);
10411   format %{ "umulh   $dst, $src1, $src2\t# umulhi" %}
10412 
10413   ins_encode %{
10414     __ umulh(as_Register($dst$$reg),
10415              as_Register($src1$$reg),
10416              as_Register($src2$$reg));
10417   %}
10418 
10419   ins_pipe(lmul_reg_reg);
10420 %}
10421 
10422 // Combined Integer Multiply & Add/Sub
10423 
10424 instruct maddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10425   match(Set dst (AddI src3 (MulI src1 src2)));
10426 
10427   ins_cost(INSN_COST * 3);
10428   format %{ "madd  $dst, $src1, $src2, $src3" %}
10429 
10430   ins_encode %{
10431     __ maddw(as_Register($dst$$reg),
10432              as_Register($src1$$reg),
10433              as_Register($src2$$reg),
10434              as_Register($src3$$reg));
10435   %}
10436 
10437   ins_pipe(imac_reg_reg);
10438 %}
10439 
10440 instruct msubI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10441   match(Set dst (SubI src3 (MulI src1 src2)));
10442 
10443   ins_cost(INSN_COST * 3);
10444   format %{ "msub  $dst, $src1, $src2, $src3" %}
10445 
10446   ins_encode %{
10447     __ msubw(as_Register($dst$$reg),
10448              as_Register($src1$$reg),
10449              as_Register($src2$$reg),
10450              as_Register($src3$$reg));
10451   %}
10452 
10453   ins_pipe(imac_reg_reg);
10454 %}
10455 
10456 // Combined Integer Multiply & Neg
10457 
10458 instruct mnegI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI0 zero) %{
10459   match(Set dst (MulI (SubI zero src1) src2));
10460 
10461   ins_cost(INSN_COST * 3);
10462   format %{ "mneg  $dst, $src1, $src2" %}
10463 
10464   ins_encode %{
10465     __ mnegw(as_Register($dst$$reg),
10466              as_Register($src1$$reg),
10467              as_Register($src2$$reg));
10468   %}
10469 
10470   ins_pipe(imac_reg_reg);
10471 %}
10472 
10473 // Combined Long Multiply & Add/Sub
10474 
10475 instruct maddL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10476   match(Set dst (AddL src3 (MulL src1 src2)));
10477 
10478   ins_cost(INSN_COST * 5);
10479   format %{ "madd  $dst, $src1, $src2, $src3" %}
10480 
10481   ins_encode %{
10482     __ madd(as_Register($dst$$reg),
10483             as_Register($src1$$reg),
10484             as_Register($src2$$reg),
10485             as_Register($src3$$reg));
10486   %}
10487 
10488   ins_pipe(lmac_reg_reg);
10489 %}
10490 
10491 instruct msubL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10492   match(Set dst (SubL src3 (MulL src1 src2)));
10493 
10494   ins_cost(INSN_COST * 5);
10495   format %{ "msub  $dst, $src1, $src2, $src3" %}
10496 
10497   ins_encode %{
10498     __ msub(as_Register($dst$$reg),
10499             as_Register($src1$$reg),
10500             as_Register($src2$$reg),
10501             as_Register($src3$$reg));
10502   %}
10503 
10504   ins_pipe(lmac_reg_reg);
10505 %}
10506 
10507 // Combined Long Multiply & Neg
10508 
10509 instruct mnegL(iRegLNoSp dst, iRegL src1, iRegL src2, immL0 zero) %{
10510   match(Set dst (MulL (SubL zero src1) src2));
10511 
10512   ins_cost(INSN_COST * 5);
10513   format %{ "mneg  $dst, $src1, $src2" %}
10514 
10515   ins_encode %{
10516     __ mneg(as_Register($dst$$reg),
10517             as_Register($src1$$reg),
10518             as_Register($src2$$reg));
10519   %}
10520 
10521   ins_pipe(lmac_reg_reg);
10522 %}
10523 
10524 // Combine Integer Signed Multiply & Add/Sub/Neg Long
10525 
10526 instruct smaddL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10527   match(Set dst (AddL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10528 
10529   ins_cost(INSN_COST * 3);
10530   format %{ "smaddl  $dst, $src1, $src2, $src3" %}
10531 
10532   ins_encode %{
10533     __ smaddl(as_Register($dst$$reg),
10534               as_Register($src1$$reg),
10535               as_Register($src2$$reg),
10536               as_Register($src3$$reg));
10537   %}
10538 
10539   ins_pipe(imac_reg_reg);
10540 %}
10541 
10542 instruct smsubL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10543   match(Set dst (SubL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10544 
10545   ins_cost(INSN_COST * 3);
10546   format %{ "smsubl  $dst, $src1, $src2, $src3" %}
10547 
10548   ins_encode %{
10549     __ smsubl(as_Register($dst$$reg),
10550               as_Register($src1$$reg),
10551               as_Register($src2$$reg),
10552               as_Register($src3$$reg));
10553   %}
10554 
10555   ins_pipe(imac_reg_reg);
10556 %}
10557 
10558 instruct smnegL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, immL0 zero) %{
10559   match(Set dst (MulL (SubL zero (ConvI2L src1)) (ConvI2L src2)));
10560 
10561   ins_cost(INSN_COST * 3);
10562   format %{ "smnegl  $dst, $src1, $src2" %}
10563 
10564   ins_encode %{
10565     __ smnegl(as_Register($dst$$reg),
10566               as_Register($src1$$reg),
10567               as_Register($src2$$reg));
10568   %}
10569 
10570   ins_pipe(imac_reg_reg);
10571 %}
10572 
10573 // Combined Multiply-Add Shorts into Integer (dst = src1 * src2 + src3 * src4)
10574 
10575 instruct muladdS2I(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3, iRegIorL2I src4) %{
10576   match(Set dst (MulAddS2I (Binary src1 src2) (Binary src3 src4)));
10577 
10578   ins_cost(INSN_COST * 5);
10579   format %{ "mulw  rscratch1, $src1, $src2\n\t"
10580             "maddw $dst, $src3, $src4, rscratch1" %}
10581 
10582   ins_encode %{
10583     __ mulw(rscratch1, as_Register($src1$$reg), as_Register($src2$$reg));
10584     __ maddw(as_Register($dst$$reg), as_Register($src3$$reg), as_Register($src4$$reg), rscratch1); %}
10585 
10586   ins_pipe(imac_reg_reg);
10587 %}
10588 
10589 // Integer Divide
10590 
10591 instruct divI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10592   match(Set dst (DivI src1 src2));
10593 
10594   ins_cost(INSN_COST * 19);
10595   format %{ "sdivw  $dst, $src1, $src2" %}
10596 
10597   ins_encode(aarch64_enc_divw(dst, src1, src2));
10598   ins_pipe(idiv_reg_reg);
10599 %}
10600 
10601 // Long Divide
10602 
10603 instruct divL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10604   match(Set dst (DivL src1 src2));
10605 
10606   ins_cost(INSN_COST * 35);
10607   format %{ "sdiv   $dst, $src1, $src2" %}
10608 
10609   ins_encode(aarch64_enc_div(dst, src1, src2));
10610   ins_pipe(ldiv_reg_reg);
10611 %}
10612 
10613 // Integer Remainder
10614 
10615 instruct modI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10616   match(Set dst (ModI src1 src2));
10617 
10618   ins_cost(INSN_COST * 22);
10619   format %{ "sdivw  rscratch1, $src1, $src2\n\t"
10620             "msubw  $dst, rscratch1, $src2, $src1" %}
10621 
10622   ins_encode(aarch64_enc_modw(dst, src1, src2));
10623   ins_pipe(idiv_reg_reg);
10624 %}
10625 
10626 // Long Remainder
10627 
10628 instruct modL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10629   match(Set dst (ModL src1 src2));
10630 
10631   ins_cost(INSN_COST * 38);
10632   format %{ "sdiv   rscratch1, $src1, $src2\n"
10633             "msub   $dst, rscratch1, $src2, $src1" %}
10634 
10635   ins_encode(aarch64_enc_mod(dst, src1, src2));
10636   ins_pipe(ldiv_reg_reg);
10637 %}
10638 
10639 // Unsigned Integer Divide
10640 
10641 instruct UdivI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10642   match(Set dst (UDivI src1 src2));
10643 
10644   ins_cost(INSN_COST * 19);
10645   format %{ "udivw  $dst, $src1, $src2" %}
10646 
10647   ins_encode %{
10648     __ udivw($dst$$Register, $src1$$Register, $src2$$Register);
10649   %}
10650 
10651   ins_pipe(idiv_reg_reg);
10652 %}
10653 
10654 //  Unsigned Long Divide
10655 
10656 instruct UdivL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10657   match(Set dst (UDivL src1 src2));
10658 
10659   ins_cost(INSN_COST * 35);
10660   format %{ "udiv   $dst, $src1, $src2" %}
10661 
10662   ins_encode %{
10663     __ udiv($dst$$Register, $src1$$Register, $src2$$Register);
10664   %}
10665 
10666   ins_pipe(ldiv_reg_reg);
10667 %}
10668 
10669 // Unsigned Integer Remainder
10670 
10671 instruct UmodI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10672   match(Set dst (UModI src1 src2));
10673 
10674   ins_cost(INSN_COST * 22);
10675   format %{ "udivw  rscratch1, $src1, $src2\n\t"
10676             "msubw  $dst, rscratch1, $src2, $src1" %}
10677 
10678   ins_encode %{
10679     __ udivw(rscratch1, $src1$$Register, $src2$$Register);
10680     __ msubw($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
10681   %}
10682 
10683   ins_pipe(idiv_reg_reg);
10684 %}
10685 
10686 // Unsigned Long Remainder
10687 
10688 instruct UModL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10689   match(Set dst (UModL src1 src2));
10690 
10691   ins_cost(INSN_COST * 38);
10692   format %{ "udiv   rscratch1, $src1, $src2\n"
10693             "msub   $dst, rscratch1, $src2, $src1" %}
10694 
10695   ins_encode %{
10696     __ udiv(rscratch1, $src1$$Register, $src2$$Register);
10697     __ msub($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
10698   %}
10699 
10700   ins_pipe(ldiv_reg_reg);
10701 %}
10702 
10703 // Integer Shifts
10704 
10705 // Shift Left Register
10706 instruct lShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10707   match(Set dst (LShiftI src1 src2));
10708 
10709   ins_cost(INSN_COST * 2);
10710   format %{ "lslvw  $dst, $src1, $src2" %}
10711 
10712   ins_encode %{
10713     __ lslvw(as_Register($dst$$reg),
10714              as_Register($src1$$reg),
10715              as_Register($src2$$reg));
10716   %}
10717 
10718   ins_pipe(ialu_reg_reg_vshift);
10719 %}
10720 
10721 // Shift Left Immediate
10722 instruct lShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10723   match(Set dst (LShiftI src1 src2));
10724 
10725   ins_cost(INSN_COST);
10726   format %{ "lslw $dst, $src1, ($src2 & 0x1f)" %}
10727 
10728   ins_encode %{
10729     __ lslw(as_Register($dst$$reg),
10730             as_Register($src1$$reg),
10731             $src2$$constant & 0x1f);
10732   %}
10733 
10734   ins_pipe(ialu_reg_shift);
10735 %}
10736 
10737 // Shift Right Logical Register
10738 instruct urShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10739   match(Set dst (URShiftI src1 src2));
10740 
10741   ins_cost(INSN_COST * 2);
10742   format %{ "lsrvw  $dst, $src1, $src2" %}
10743 
10744   ins_encode %{
10745     __ lsrvw(as_Register($dst$$reg),
10746              as_Register($src1$$reg),
10747              as_Register($src2$$reg));
10748   %}
10749 
10750   ins_pipe(ialu_reg_reg_vshift);
10751 %}
10752 
10753 // Shift Right Logical Immediate
10754 instruct urShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10755   match(Set dst (URShiftI src1 src2));
10756 
10757   ins_cost(INSN_COST);
10758   format %{ "lsrw $dst, $src1, ($src2 & 0x1f)" %}
10759 
10760   ins_encode %{
10761     __ lsrw(as_Register($dst$$reg),
10762             as_Register($src1$$reg),
10763             $src2$$constant & 0x1f);
10764   %}
10765 
10766   ins_pipe(ialu_reg_shift);
10767 %}
10768 
10769 // Shift Right Arithmetic Register
10770 instruct rShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10771   match(Set dst (RShiftI src1 src2));
10772 
10773   ins_cost(INSN_COST * 2);
10774   format %{ "asrvw  $dst, $src1, $src2" %}
10775 
10776   ins_encode %{
10777     __ asrvw(as_Register($dst$$reg),
10778              as_Register($src1$$reg),
10779              as_Register($src2$$reg));
10780   %}
10781 
10782   ins_pipe(ialu_reg_reg_vshift);
10783 %}
10784 
10785 // Shift Right Arithmetic Immediate
10786 instruct rShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10787   match(Set dst (RShiftI src1 src2));
10788 
10789   ins_cost(INSN_COST);
10790   format %{ "asrw $dst, $src1, ($src2 & 0x1f)" %}
10791 
10792   ins_encode %{
10793     __ asrw(as_Register($dst$$reg),
10794             as_Register($src1$$reg),
10795             $src2$$constant & 0x1f);
10796   %}
10797 
10798   ins_pipe(ialu_reg_shift);
10799 %}
10800 
10801 // Combined Int Mask and Right Shift (using UBFM)
10802 // TODO
10803 
10804 // Long Shifts
10805 
10806 // Shift Left Register
10807 instruct lShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10808   match(Set dst (LShiftL src1 src2));
10809 
10810   ins_cost(INSN_COST * 2);
10811   format %{ "lslv  $dst, $src1, $src2" %}
10812 
10813   ins_encode %{
10814     __ lslv(as_Register($dst$$reg),
10815             as_Register($src1$$reg),
10816             as_Register($src2$$reg));
10817   %}
10818 
10819   ins_pipe(ialu_reg_reg_vshift);
10820 %}
10821 
10822 // Shift Left Immediate
10823 instruct lShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10824   match(Set dst (LShiftL src1 src2));
10825 
10826   ins_cost(INSN_COST);
10827   format %{ "lsl $dst, $src1, ($src2 & 0x3f)" %}
10828 
10829   ins_encode %{
10830     __ lsl(as_Register($dst$$reg),
10831             as_Register($src1$$reg),
10832             $src2$$constant & 0x3f);
10833   %}
10834 
10835   ins_pipe(ialu_reg_shift);
10836 %}
10837 
10838 // Shift Right Logical Register
10839 instruct urShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10840   match(Set dst (URShiftL src1 src2));
10841 
10842   ins_cost(INSN_COST * 2);
10843   format %{ "lsrv  $dst, $src1, $src2" %}
10844 
10845   ins_encode %{
10846     __ lsrv(as_Register($dst$$reg),
10847             as_Register($src1$$reg),
10848             as_Register($src2$$reg));
10849   %}
10850 
10851   ins_pipe(ialu_reg_reg_vshift);
10852 %}
10853 
10854 // Shift Right Logical Immediate
10855 instruct urShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10856   match(Set dst (URShiftL src1 src2));
10857 
10858   ins_cost(INSN_COST);
10859   format %{ "lsr $dst, $src1, ($src2 & 0x3f)" %}
10860 
10861   ins_encode %{
10862     __ lsr(as_Register($dst$$reg),
10863            as_Register($src1$$reg),
10864            $src2$$constant & 0x3f);
10865   %}
10866 
10867   ins_pipe(ialu_reg_shift);
10868 %}
10869 
10870 // A special-case pattern for card table stores.
10871 instruct urShiftP_reg_imm(iRegLNoSp dst, iRegP src1, immI src2) %{
10872   match(Set dst (URShiftL (CastP2X src1) src2));
10873 
10874   ins_cost(INSN_COST);
10875   format %{ "lsr $dst, p2x($src1), ($src2 & 0x3f)" %}
10876 
10877   ins_encode %{
10878     __ lsr(as_Register($dst$$reg),
10879            as_Register($src1$$reg),
10880            $src2$$constant & 0x3f);
10881   %}
10882 
10883   ins_pipe(ialu_reg_shift);
10884 %}
10885 
10886 // Shift Right Arithmetic Register
10887 instruct rShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10888   match(Set dst (RShiftL src1 src2));
10889 
10890   ins_cost(INSN_COST * 2);
10891   format %{ "asrv  $dst, $src1, $src2" %}
10892 
10893   ins_encode %{
10894     __ asrv(as_Register($dst$$reg),
10895             as_Register($src1$$reg),
10896             as_Register($src2$$reg));
10897   %}
10898 
10899   ins_pipe(ialu_reg_reg_vshift);
10900 %}
10901 
10902 // Shift Right Arithmetic Immediate
10903 instruct rShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10904   match(Set dst (RShiftL src1 src2));
10905 
10906   ins_cost(INSN_COST);
10907   format %{ "asr $dst, $src1, ($src2 & 0x3f)" %}
10908 
10909   ins_encode %{
10910     __ asr(as_Register($dst$$reg),
10911            as_Register($src1$$reg),
10912            $src2$$constant & 0x3f);
10913   %}
10914 
10915   ins_pipe(ialu_reg_shift);
10916 %}
10917 
10918 // BEGIN This section of the file is automatically generated. Do not edit --------------
10919 // This section is generated from aarch64_ad.m4
10920 
10921 // This pattern is automatically generated from aarch64_ad.m4
10922 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10923 instruct regL_not_reg(iRegLNoSp dst,
10924                          iRegL src1, immL_M1 m1,
10925                          rFlagsReg cr) %{
10926   match(Set dst (XorL src1 m1));
10927   ins_cost(INSN_COST);
10928   format %{ "eon  $dst, $src1, zr" %}
10929 
10930   ins_encode %{
10931     __ eon(as_Register($dst$$reg),
10932               as_Register($src1$$reg),
10933               zr,
10934               Assembler::LSL, 0);
10935   %}
10936 
10937   ins_pipe(ialu_reg);
10938 %}
10939 
10940 // This pattern is automatically generated from aarch64_ad.m4
10941 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10942 instruct regI_not_reg(iRegINoSp dst,
10943                          iRegIorL2I src1, immI_M1 m1,
10944                          rFlagsReg cr) %{
10945   match(Set dst (XorI src1 m1));
10946   ins_cost(INSN_COST);
10947   format %{ "eonw  $dst, $src1, zr" %}
10948 
10949   ins_encode %{
10950     __ eonw(as_Register($dst$$reg),
10951               as_Register($src1$$reg),
10952               zr,
10953               Assembler::LSL, 0);
10954   %}
10955 
10956   ins_pipe(ialu_reg);
10957 %}
10958 
10959 // This pattern is automatically generated from aarch64_ad.m4
10960 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10961 instruct NegI_reg_URShift_reg(iRegINoSp dst,
10962                               immI0 zero, iRegIorL2I src1, immI src2) %{
10963   match(Set dst (SubI zero (URShiftI src1 src2)));
10964 
10965   ins_cost(1.9 * INSN_COST);
10966   format %{ "negw  $dst, $src1, LSR $src2" %}
10967 
10968   ins_encode %{
10969     __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
10970             Assembler::LSR, $src2$$constant & 0x1f);
10971   %}
10972 
10973   ins_pipe(ialu_reg_shift);
10974 %}
10975 
10976 // This pattern is automatically generated from aarch64_ad.m4
10977 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10978 instruct NegI_reg_RShift_reg(iRegINoSp dst,
10979                               immI0 zero, iRegIorL2I src1, immI src2) %{
10980   match(Set dst (SubI zero (RShiftI src1 src2)));
10981 
10982   ins_cost(1.9 * INSN_COST);
10983   format %{ "negw  $dst, $src1, ASR $src2" %}
10984 
10985   ins_encode %{
10986     __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
10987             Assembler::ASR, $src2$$constant & 0x1f);
10988   %}
10989 
10990   ins_pipe(ialu_reg_shift);
10991 %}
10992 
10993 // This pattern is automatically generated from aarch64_ad.m4
10994 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10995 instruct NegI_reg_LShift_reg(iRegINoSp dst,
10996                               immI0 zero, iRegIorL2I src1, immI src2) %{
10997   match(Set dst (SubI zero (LShiftI src1 src2)));
10998 
10999   ins_cost(1.9 * INSN_COST);
11000   format %{ "negw  $dst, $src1, LSL $src2" %}
11001 
11002   ins_encode %{
11003     __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
11004             Assembler::LSL, $src2$$constant & 0x1f);
11005   %}
11006 
11007   ins_pipe(ialu_reg_shift);
11008 %}
11009 
11010 // This pattern is automatically generated from aarch64_ad.m4
11011 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11012 instruct NegL_reg_URShift_reg(iRegLNoSp dst,
11013                               immL0 zero, iRegL src1, immI src2) %{
11014   match(Set dst (SubL zero (URShiftL src1 src2)));
11015 
11016   ins_cost(1.9 * INSN_COST);
11017   format %{ "neg  $dst, $src1, LSR $src2" %}
11018 
11019   ins_encode %{
11020     __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
11021             Assembler::LSR, $src2$$constant & 0x3f);
11022   %}
11023 
11024   ins_pipe(ialu_reg_shift);
11025 %}
11026 
11027 // This pattern is automatically generated from aarch64_ad.m4
11028 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11029 instruct NegL_reg_RShift_reg(iRegLNoSp dst,
11030                               immL0 zero, iRegL src1, immI src2) %{
11031   match(Set dst (SubL zero (RShiftL src1 src2)));
11032 
11033   ins_cost(1.9 * INSN_COST);
11034   format %{ "neg  $dst, $src1, ASR $src2" %}
11035 
11036   ins_encode %{
11037     __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
11038             Assembler::ASR, $src2$$constant & 0x3f);
11039   %}
11040 
11041   ins_pipe(ialu_reg_shift);
11042 %}
11043 
11044 // This pattern is automatically generated from aarch64_ad.m4
11045 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11046 instruct NegL_reg_LShift_reg(iRegLNoSp dst,
11047                               immL0 zero, iRegL src1, immI src2) %{
11048   match(Set dst (SubL zero (LShiftL src1 src2)));
11049 
11050   ins_cost(1.9 * INSN_COST);
11051   format %{ "neg  $dst, $src1, LSL $src2" %}
11052 
11053   ins_encode %{
11054     __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
11055             Assembler::LSL, $src2$$constant & 0x3f);
11056   %}
11057 
11058   ins_pipe(ialu_reg_shift);
11059 %}
11060 
11061 // This pattern is automatically generated from aarch64_ad.m4
11062 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11063 instruct AndI_reg_not_reg(iRegINoSp dst,
11064                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11065   match(Set dst (AndI src1 (XorI src2 m1)));
11066   ins_cost(INSN_COST);
11067   format %{ "bicw  $dst, $src1, $src2" %}
11068 
11069   ins_encode %{
11070     __ bicw(as_Register($dst$$reg),
11071               as_Register($src1$$reg),
11072               as_Register($src2$$reg),
11073               Assembler::LSL, 0);
11074   %}
11075 
11076   ins_pipe(ialu_reg_reg);
11077 %}
11078 
11079 // This pattern is automatically generated from aarch64_ad.m4
11080 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11081 instruct AndL_reg_not_reg(iRegLNoSp dst,
11082                          iRegL src1, iRegL src2, immL_M1 m1) %{
11083   match(Set dst (AndL src1 (XorL src2 m1)));
11084   ins_cost(INSN_COST);
11085   format %{ "bic  $dst, $src1, $src2" %}
11086 
11087   ins_encode %{
11088     __ bic(as_Register($dst$$reg),
11089               as_Register($src1$$reg),
11090               as_Register($src2$$reg),
11091               Assembler::LSL, 0);
11092   %}
11093 
11094   ins_pipe(ialu_reg_reg);
11095 %}
11096 
11097 // This pattern is automatically generated from aarch64_ad.m4
11098 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11099 instruct OrI_reg_not_reg(iRegINoSp dst,
11100                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11101   match(Set dst (OrI src1 (XorI src2 m1)));
11102   ins_cost(INSN_COST);
11103   format %{ "ornw  $dst, $src1, $src2" %}
11104 
11105   ins_encode %{
11106     __ ornw(as_Register($dst$$reg),
11107               as_Register($src1$$reg),
11108               as_Register($src2$$reg),
11109               Assembler::LSL, 0);
11110   %}
11111 
11112   ins_pipe(ialu_reg_reg);
11113 %}
11114 
11115 // This pattern is automatically generated from aarch64_ad.m4
11116 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11117 instruct OrL_reg_not_reg(iRegLNoSp dst,
11118                          iRegL src1, iRegL src2, immL_M1 m1) %{
11119   match(Set dst (OrL src1 (XorL src2 m1)));
11120   ins_cost(INSN_COST);
11121   format %{ "orn  $dst, $src1, $src2" %}
11122 
11123   ins_encode %{
11124     __ orn(as_Register($dst$$reg),
11125               as_Register($src1$$reg),
11126               as_Register($src2$$reg),
11127               Assembler::LSL, 0);
11128   %}
11129 
11130   ins_pipe(ialu_reg_reg);
11131 %}
11132 
11133 // This pattern is automatically generated from aarch64_ad.m4
11134 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11135 instruct XorI_reg_not_reg(iRegINoSp dst,
11136                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11137   match(Set dst (XorI m1 (XorI src2 src1)));
11138   ins_cost(INSN_COST);
11139   format %{ "eonw  $dst, $src1, $src2" %}
11140 
11141   ins_encode %{
11142     __ eonw(as_Register($dst$$reg),
11143               as_Register($src1$$reg),
11144               as_Register($src2$$reg),
11145               Assembler::LSL, 0);
11146   %}
11147 
11148   ins_pipe(ialu_reg_reg);
11149 %}
11150 
11151 // This pattern is automatically generated from aarch64_ad.m4
11152 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11153 instruct XorL_reg_not_reg(iRegLNoSp dst,
11154                          iRegL src1, iRegL src2, immL_M1 m1) %{
11155   match(Set dst (XorL m1 (XorL src2 src1)));
11156   ins_cost(INSN_COST);
11157   format %{ "eon  $dst, $src1, $src2" %}
11158 
11159   ins_encode %{
11160     __ eon(as_Register($dst$$reg),
11161               as_Register($src1$$reg),
11162               as_Register($src2$$reg),
11163               Assembler::LSL, 0);
11164   %}
11165 
11166   ins_pipe(ialu_reg_reg);
11167 %}
11168 
11169 // This pattern is automatically generated from aarch64_ad.m4
11170 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11171 // val & (-1 ^ (val >>> shift)) ==> bicw
11172 instruct AndI_reg_URShift_not_reg(iRegINoSp dst,
11173                          iRegIorL2I src1, iRegIorL2I src2,
11174                          immI src3, immI_M1 src4) %{
11175   match(Set dst (AndI src1 (XorI(URShiftI src2 src3) src4)));
11176   ins_cost(1.9 * INSN_COST);
11177   format %{ "bicw  $dst, $src1, $src2, LSR $src3" %}
11178 
11179   ins_encode %{
11180     __ bicw(as_Register($dst$$reg),
11181               as_Register($src1$$reg),
11182               as_Register($src2$$reg),
11183               Assembler::LSR,
11184               $src3$$constant & 0x1f);
11185   %}
11186 
11187   ins_pipe(ialu_reg_reg_shift);
11188 %}
11189 
11190 // This pattern is automatically generated from aarch64_ad.m4
11191 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11192 // val & (-1 ^ (val >>> shift)) ==> bic
11193 instruct AndL_reg_URShift_not_reg(iRegLNoSp dst,
11194                          iRegL src1, iRegL src2,
11195                          immI src3, immL_M1 src4) %{
11196   match(Set dst (AndL src1 (XorL(URShiftL src2 src3) src4)));
11197   ins_cost(1.9 * INSN_COST);
11198   format %{ "bic  $dst, $src1, $src2, LSR $src3" %}
11199 
11200   ins_encode %{
11201     __ bic(as_Register($dst$$reg),
11202               as_Register($src1$$reg),
11203               as_Register($src2$$reg),
11204               Assembler::LSR,
11205               $src3$$constant & 0x3f);
11206   %}
11207 
11208   ins_pipe(ialu_reg_reg_shift);
11209 %}
11210 
11211 // This pattern is automatically generated from aarch64_ad.m4
11212 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11213 // val & (-1 ^ (val >> shift)) ==> bicw
11214 instruct AndI_reg_RShift_not_reg(iRegINoSp dst,
11215                          iRegIorL2I src1, iRegIorL2I src2,
11216                          immI src3, immI_M1 src4) %{
11217   match(Set dst (AndI src1 (XorI(RShiftI src2 src3) src4)));
11218   ins_cost(1.9 * INSN_COST);
11219   format %{ "bicw  $dst, $src1, $src2, ASR $src3" %}
11220 
11221   ins_encode %{
11222     __ bicw(as_Register($dst$$reg),
11223               as_Register($src1$$reg),
11224               as_Register($src2$$reg),
11225               Assembler::ASR,
11226               $src3$$constant & 0x1f);
11227   %}
11228 
11229   ins_pipe(ialu_reg_reg_shift);
11230 %}
11231 
11232 // This pattern is automatically generated from aarch64_ad.m4
11233 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11234 // val & (-1 ^ (val >> shift)) ==> bic
11235 instruct AndL_reg_RShift_not_reg(iRegLNoSp dst,
11236                          iRegL src1, iRegL src2,
11237                          immI src3, immL_M1 src4) %{
11238   match(Set dst (AndL src1 (XorL(RShiftL src2 src3) src4)));
11239   ins_cost(1.9 * INSN_COST);
11240   format %{ "bic  $dst, $src1, $src2, ASR $src3" %}
11241 
11242   ins_encode %{
11243     __ bic(as_Register($dst$$reg),
11244               as_Register($src1$$reg),
11245               as_Register($src2$$reg),
11246               Assembler::ASR,
11247               $src3$$constant & 0x3f);
11248   %}
11249 
11250   ins_pipe(ialu_reg_reg_shift);
11251 %}
11252 
11253 // This pattern is automatically generated from aarch64_ad.m4
11254 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11255 // val & (-1 ^ (val ror shift)) ==> bicw
11256 instruct AndI_reg_RotateRight_not_reg(iRegINoSp dst,
11257                          iRegIorL2I src1, iRegIorL2I src2,
11258                          immI src3, immI_M1 src4) %{
11259   match(Set dst (AndI src1 (XorI(RotateRight src2 src3) src4)));
11260   ins_cost(1.9 * INSN_COST);
11261   format %{ "bicw  $dst, $src1, $src2, ROR $src3" %}
11262 
11263   ins_encode %{
11264     __ bicw(as_Register($dst$$reg),
11265               as_Register($src1$$reg),
11266               as_Register($src2$$reg),
11267               Assembler::ROR,
11268               $src3$$constant & 0x1f);
11269   %}
11270 
11271   ins_pipe(ialu_reg_reg_shift);
11272 %}
11273 
11274 // This pattern is automatically generated from aarch64_ad.m4
11275 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11276 // val & (-1 ^ (val ror shift)) ==> bic
11277 instruct AndL_reg_RotateRight_not_reg(iRegLNoSp dst,
11278                          iRegL src1, iRegL src2,
11279                          immI src3, immL_M1 src4) %{
11280   match(Set dst (AndL src1 (XorL(RotateRight src2 src3) src4)));
11281   ins_cost(1.9 * INSN_COST);
11282   format %{ "bic  $dst, $src1, $src2, ROR $src3" %}
11283 
11284   ins_encode %{
11285     __ bic(as_Register($dst$$reg),
11286               as_Register($src1$$reg),
11287               as_Register($src2$$reg),
11288               Assembler::ROR,
11289               $src3$$constant & 0x3f);
11290   %}
11291 
11292   ins_pipe(ialu_reg_reg_shift);
11293 %}
11294 
11295 // This pattern is automatically generated from aarch64_ad.m4
11296 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11297 // val & (-1 ^ (val << shift)) ==> bicw
11298 instruct AndI_reg_LShift_not_reg(iRegINoSp dst,
11299                          iRegIorL2I src1, iRegIorL2I src2,
11300                          immI src3, immI_M1 src4) %{
11301   match(Set dst (AndI src1 (XorI(LShiftI src2 src3) src4)));
11302   ins_cost(1.9 * INSN_COST);
11303   format %{ "bicw  $dst, $src1, $src2, LSL $src3" %}
11304 
11305   ins_encode %{
11306     __ bicw(as_Register($dst$$reg),
11307               as_Register($src1$$reg),
11308               as_Register($src2$$reg),
11309               Assembler::LSL,
11310               $src3$$constant & 0x1f);
11311   %}
11312 
11313   ins_pipe(ialu_reg_reg_shift);
11314 %}
11315 
11316 // This pattern is automatically generated from aarch64_ad.m4
11317 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11318 // val & (-1 ^ (val << shift)) ==> bic
11319 instruct AndL_reg_LShift_not_reg(iRegLNoSp dst,
11320                          iRegL src1, iRegL src2,
11321                          immI src3, immL_M1 src4) %{
11322   match(Set dst (AndL src1 (XorL(LShiftL src2 src3) src4)));
11323   ins_cost(1.9 * INSN_COST);
11324   format %{ "bic  $dst, $src1, $src2, LSL $src3" %}
11325 
11326   ins_encode %{
11327     __ bic(as_Register($dst$$reg),
11328               as_Register($src1$$reg),
11329               as_Register($src2$$reg),
11330               Assembler::LSL,
11331               $src3$$constant & 0x3f);
11332   %}
11333 
11334   ins_pipe(ialu_reg_reg_shift);
11335 %}
11336 
11337 // This pattern is automatically generated from aarch64_ad.m4
11338 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11339 // val ^ (-1 ^ (val >>> shift)) ==> eonw
11340 instruct XorI_reg_URShift_not_reg(iRegINoSp dst,
11341                          iRegIorL2I src1, iRegIorL2I src2,
11342                          immI src3, immI_M1 src4) %{
11343   match(Set dst (XorI src4 (XorI(URShiftI src2 src3) src1)));
11344   ins_cost(1.9 * INSN_COST);
11345   format %{ "eonw  $dst, $src1, $src2, LSR $src3" %}
11346 
11347   ins_encode %{
11348     __ eonw(as_Register($dst$$reg),
11349               as_Register($src1$$reg),
11350               as_Register($src2$$reg),
11351               Assembler::LSR,
11352               $src3$$constant & 0x1f);
11353   %}
11354 
11355   ins_pipe(ialu_reg_reg_shift);
11356 %}
11357 
11358 // This pattern is automatically generated from aarch64_ad.m4
11359 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11360 // val ^ (-1 ^ (val >>> shift)) ==> eon
11361 instruct XorL_reg_URShift_not_reg(iRegLNoSp dst,
11362                          iRegL src1, iRegL src2,
11363                          immI src3, immL_M1 src4) %{
11364   match(Set dst (XorL src4 (XorL(URShiftL src2 src3) src1)));
11365   ins_cost(1.9 * INSN_COST);
11366   format %{ "eon  $dst, $src1, $src2, LSR $src3" %}
11367 
11368   ins_encode %{
11369     __ eon(as_Register($dst$$reg),
11370               as_Register($src1$$reg),
11371               as_Register($src2$$reg),
11372               Assembler::LSR,
11373               $src3$$constant & 0x3f);
11374   %}
11375 
11376   ins_pipe(ialu_reg_reg_shift);
11377 %}
11378 
11379 // This pattern is automatically generated from aarch64_ad.m4
11380 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11381 // val ^ (-1 ^ (val >> shift)) ==> eonw
11382 instruct XorI_reg_RShift_not_reg(iRegINoSp dst,
11383                          iRegIorL2I src1, iRegIorL2I src2,
11384                          immI src3, immI_M1 src4) %{
11385   match(Set dst (XorI src4 (XorI(RShiftI src2 src3) src1)));
11386   ins_cost(1.9 * INSN_COST);
11387   format %{ "eonw  $dst, $src1, $src2, ASR $src3" %}
11388 
11389   ins_encode %{
11390     __ eonw(as_Register($dst$$reg),
11391               as_Register($src1$$reg),
11392               as_Register($src2$$reg),
11393               Assembler::ASR,
11394               $src3$$constant & 0x1f);
11395   %}
11396 
11397   ins_pipe(ialu_reg_reg_shift);
11398 %}
11399 
11400 // This pattern is automatically generated from aarch64_ad.m4
11401 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11402 // val ^ (-1 ^ (val >> shift)) ==> eon
11403 instruct XorL_reg_RShift_not_reg(iRegLNoSp dst,
11404                          iRegL src1, iRegL src2,
11405                          immI src3, immL_M1 src4) %{
11406   match(Set dst (XorL src4 (XorL(RShiftL src2 src3) src1)));
11407   ins_cost(1.9 * INSN_COST);
11408   format %{ "eon  $dst, $src1, $src2, ASR $src3" %}
11409 
11410   ins_encode %{
11411     __ eon(as_Register($dst$$reg),
11412               as_Register($src1$$reg),
11413               as_Register($src2$$reg),
11414               Assembler::ASR,
11415               $src3$$constant & 0x3f);
11416   %}
11417 
11418   ins_pipe(ialu_reg_reg_shift);
11419 %}
11420 
11421 // This pattern is automatically generated from aarch64_ad.m4
11422 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11423 // val ^ (-1 ^ (val ror shift)) ==> eonw
11424 instruct XorI_reg_RotateRight_not_reg(iRegINoSp dst,
11425                          iRegIorL2I src1, iRegIorL2I src2,
11426                          immI src3, immI_M1 src4) %{
11427   match(Set dst (XorI src4 (XorI(RotateRight src2 src3) src1)));
11428   ins_cost(1.9 * INSN_COST);
11429   format %{ "eonw  $dst, $src1, $src2, ROR $src3" %}
11430 
11431   ins_encode %{
11432     __ eonw(as_Register($dst$$reg),
11433               as_Register($src1$$reg),
11434               as_Register($src2$$reg),
11435               Assembler::ROR,
11436               $src3$$constant & 0x1f);
11437   %}
11438 
11439   ins_pipe(ialu_reg_reg_shift);
11440 %}
11441 
11442 // This pattern is automatically generated from aarch64_ad.m4
11443 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11444 // val ^ (-1 ^ (val ror shift)) ==> eon
11445 instruct XorL_reg_RotateRight_not_reg(iRegLNoSp dst,
11446                          iRegL src1, iRegL src2,
11447                          immI src3, immL_M1 src4) %{
11448   match(Set dst (XorL src4 (XorL(RotateRight src2 src3) src1)));
11449   ins_cost(1.9 * INSN_COST);
11450   format %{ "eon  $dst, $src1, $src2, ROR $src3" %}
11451 
11452   ins_encode %{
11453     __ eon(as_Register($dst$$reg),
11454               as_Register($src1$$reg),
11455               as_Register($src2$$reg),
11456               Assembler::ROR,
11457               $src3$$constant & 0x3f);
11458   %}
11459 
11460   ins_pipe(ialu_reg_reg_shift);
11461 %}
11462 
11463 // This pattern is automatically generated from aarch64_ad.m4
11464 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11465 // val ^ (-1 ^ (val << shift)) ==> eonw
11466 instruct XorI_reg_LShift_not_reg(iRegINoSp dst,
11467                          iRegIorL2I src1, iRegIorL2I src2,
11468                          immI src3, immI_M1 src4) %{
11469   match(Set dst (XorI src4 (XorI(LShiftI src2 src3) src1)));
11470   ins_cost(1.9 * INSN_COST);
11471   format %{ "eonw  $dst, $src1, $src2, LSL $src3" %}
11472 
11473   ins_encode %{
11474     __ eonw(as_Register($dst$$reg),
11475               as_Register($src1$$reg),
11476               as_Register($src2$$reg),
11477               Assembler::LSL,
11478               $src3$$constant & 0x1f);
11479   %}
11480 
11481   ins_pipe(ialu_reg_reg_shift);
11482 %}
11483 
11484 // This pattern is automatically generated from aarch64_ad.m4
11485 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11486 // val ^ (-1 ^ (val << shift)) ==> eon
11487 instruct XorL_reg_LShift_not_reg(iRegLNoSp dst,
11488                          iRegL src1, iRegL src2,
11489                          immI src3, immL_M1 src4) %{
11490   match(Set dst (XorL src4 (XorL(LShiftL src2 src3) src1)));
11491   ins_cost(1.9 * INSN_COST);
11492   format %{ "eon  $dst, $src1, $src2, LSL $src3" %}
11493 
11494   ins_encode %{
11495     __ eon(as_Register($dst$$reg),
11496               as_Register($src1$$reg),
11497               as_Register($src2$$reg),
11498               Assembler::LSL,
11499               $src3$$constant & 0x3f);
11500   %}
11501 
11502   ins_pipe(ialu_reg_reg_shift);
11503 %}
11504 
11505 // This pattern is automatically generated from aarch64_ad.m4
11506 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11507 // val | (-1 ^ (val >>> shift)) ==> ornw
11508 instruct OrI_reg_URShift_not_reg(iRegINoSp dst,
11509                          iRegIorL2I src1, iRegIorL2I src2,
11510                          immI src3, immI_M1 src4) %{
11511   match(Set dst (OrI src1 (XorI(URShiftI src2 src3) src4)));
11512   ins_cost(1.9 * INSN_COST);
11513   format %{ "ornw  $dst, $src1, $src2, LSR $src3" %}
11514 
11515   ins_encode %{
11516     __ ornw(as_Register($dst$$reg),
11517               as_Register($src1$$reg),
11518               as_Register($src2$$reg),
11519               Assembler::LSR,
11520               $src3$$constant & 0x1f);
11521   %}
11522 
11523   ins_pipe(ialu_reg_reg_shift);
11524 %}
11525 
11526 // This pattern is automatically generated from aarch64_ad.m4
11527 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11528 // val | (-1 ^ (val >>> shift)) ==> orn
11529 instruct OrL_reg_URShift_not_reg(iRegLNoSp dst,
11530                          iRegL src1, iRegL src2,
11531                          immI src3, immL_M1 src4) %{
11532   match(Set dst (OrL src1 (XorL(URShiftL src2 src3) src4)));
11533   ins_cost(1.9 * INSN_COST);
11534   format %{ "orn  $dst, $src1, $src2, LSR $src3" %}
11535 
11536   ins_encode %{
11537     __ orn(as_Register($dst$$reg),
11538               as_Register($src1$$reg),
11539               as_Register($src2$$reg),
11540               Assembler::LSR,
11541               $src3$$constant & 0x3f);
11542   %}
11543 
11544   ins_pipe(ialu_reg_reg_shift);
11545 %}
11546 
11547 // This pattern is automatically generated from aarch64_ad.m4
11548 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11549 // val | (-1 ^ (val >> shift)) ==> ornw
11550 instruct OrI_reg_RShift_not_reg(iRegINoSp dst,
11551                          iRegIorL2I src1, iRegIorL2I src2,
11552                          immI src3, immI_M1 src4) %{
11553   match(Set dst (OrI src1 (XorI(RShiftI src2 src3) src4)));
11554   ins_cost(1.9 * INSN_COST);
11555   format %{ "ornw  $dst, $src1, $src2, ASR $src3" %}
11556 
11557   ins_encode %{
11558     __ ornw(as_Register($dst$$reg),
11559               as_Register($src1$$reg),
11560               as_Register($src2$$reg),
11561               Assembler::ASR,
11562               $src3$$constant & 0x1f);
11563   %}
11564 
11565   ins_pipe(ialu_reg_reg_shift);
11566 %}
11567 
11568 // This pattern is automatically generated from aarch64_ad.m4
11569 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11570 // val | (-1 ^ (val >> shift)) ==> orn
11571 instruct OrL_reg_RShift_not_reg(iRegLNoSp dst,
11572                          iRegL src1, iRegL src2,
11573                          immI src3, immL_M1 src4) %{
11574   match(Set dst (OrL src1 (XorL(RShiftL src2 src3) src4)));
11575   ins_cost(1.9 * INSN_COST);
11576   format %{ "orn  $dst, $src1, $src2, ASR $src3" %}
11577 
11578   ins_encode %{
11579     __ orn(as_Register($dst$$reg),
11580               as_Register($src1$$reg),
11581               as_Register($src2$$reg),
11582               Assembler::ASR,
11583               $src3$$constant & 0x3f);
11584   %}
11585 
11586   ins_pipe(ialu_reg_reg_shift);
11587 %}
11588 
11589 // This pattern is automatically generated from aarch64_ad.m4
11590 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11591 // val | (-1 ^ (val ror shift)) ==> ornw
11592 instruct OrI_reg_RotateRight_not_reg(iRegINoSp dst,
11593                          iRegIorL2I src1, iRegIorL2I src2,
11594                          immI src3, immI_M1 src4) %{
11595   match(Set dst (OrI src1 (XorI(RotateRight src2 src3) src4)));
11596   ins_cost(1.9 * INSN_COST);
11597   format %{ "ornw  $dst, $src1, $src2, ROR $src3" %}
11598 
11599   ins_encode %{
11600     __ ornw(as_Register($dst$$reg),
11601               as_Register($src1$$reg),
11602               as_Register($src2$$reg),
11603               Assembler::ROR,
11604               $src3$$constant & 0x1f);
11605   %}
11606 
11607   ins_pipe(ialu_reg_reg_shift);
11608 %}
11609 
11610 // This pattern is automatically generated from aarch64_ad.m4
11611 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11612 // val | (-1 ^ (val ror shift)) ==> orn
11613 instruct OrL_reg_RotateRight_not_reg(iRegLNoSp dst,
11614                          iRegL src1, iRegL src2,
11615                          immI src3, immL_M1 src4) %{
11616   match(Set dst (OrL src1 (XorL(RotateRight src2 src3) src4)));
11617   ins_cost(1.9 * INSN_COST);
11618   format %{ "orn  $dst, $src1, $src2, ROR $src3" %}
11619 
11620   ins_encode %{
11621     __ orn(as_Register($dst$$reg),
11622               as_Register($src1$$reg),
11623               as_Register($src2$$reg),
11624               Assembler::ROR,
11625               $src3$$constant & 0x3f);
11626   %}
11627 
11628   ins_pipe(ialu_reg_reg_shift);
11629 %}
11630 
11631 // This pattern is automatically generated from aarch64_ad.m4
11632 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11633 // val | (-1 ^ (val << shift)) ==> ornw
11634 instruct OrI_reg_LShift_not_reg(iRegINoSp dst,
11635                          iRegIorL2I src1, iRegIorL2I src2,
11636                          immI src3, immI_M1 src4) %{
11637   match(Set dst (OrI src1 (XorI(LShiftI src2 src3) src4)));
11638   ins_cost(1.9 * INSN_COST);
11639   format %{ "ornw  $dst, $src1, $src2, LSL $src3" %}
11640 
11641   ins_encode %{
11642     __ ornw(as_Register($dst$$reg),
11643               as_Register($src1$$reg),
11644               as_Register($src2$$reg),
11645               Assembler::LSL,
11646               $src3$$constant & 0x1f);
11647   %}
11648 
11649   ins_pipe(ialu_reg_reg_shift);
11650 %}
11651 
11652 // This pattern is automatically generated from aarch64_ad.m4
11653 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11654 // val | (-1 ^ (val << shift)) ==> orn
11655 instruct OrL_reg_LShift_not_reg(iRegLNoSp dst,
11656                          iRegL src1, iRegL src2,
11657                          immI src3, immL_M1 src4) %{
11658   match(Set dst (OrL src1 (XorL(LShiftL src2 src3) src4)));
11659   ins_cost(1.9 * INSN_COST);
11660   format %{ "orn  $dst, $src1, $src2, LSL $src3" %}
11661 
11662   ins_encode %{
11663     __ orn(as_Register($dst$$reg),
11664               as_Register($src1$$reg),
11665               as_Register($src2$$reg),
11666               Assembler::LSL,
11667               $src3$$constant & 0x3f);
11668   %}
11669 
11670   ins_pipe(ialu_reg_reg_shift);
11671 %}
11672 
11673 // This pattern is automatically generated from aarch64_ad.m4
11674 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11675 instruct AndI_reg_URShift_reg(iRegINoSp dst,
11676                          iRegIorL2I src1, iRegIorL2I src2,
11677                          immI src3) %{
11678   match(Set dst (AndI src1 (URShiftI src2 src3)));
11679 
11680   ins_cost(1.9 * INSN_COST);
11681   format %{ "andw  $dst, $src1, $src2, LSR $src3" %}
11682 
11683   ins_encode %{
11684     __ andw(as_Register($dst$$reg),
11685               as_Register($src1$$reg),
11686               as_Register($src2$$reg),
11687               Assembler::LSR,
11688               $src3$$constant & 0x1f);
11689   %}
11690 
11691   ins_pipe(ialu_reg_reg_shift);
11692 %}
11693 
11694 // This pattern is automatically generated from aarch64_ad.m4
11695 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11696 instruct AndL_reg_URShift_reg(iRegLNoSp dst,
11697                          iRegL src1, iRegL src2,
11698                          immI src3) %{
11699   match(Set dst (AndL src1 (URShiftL src2 src3)));
11700 
11701   ins_cost(1.9 * INSN_COST);
11702   format %{ "andr  $dst, $src1, $src2, LSR $src3" %}
11703 
11704   ins_encode %{
11705     __ andr(as_Register($dst$$reg),
11706               as_Register($src1$$reg),
11707               as_Register($src2$$reg),
11708               Assembler::LSR,
11709               $src3$$constant & 0x3f);
11710   %}
11711 
11712   ins_pipe(ialu_reg_reg_shift);
11713 %}
11714 
11715 // This pattern is automatically generated from aarch64_ad.m4
11716 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11717 instruct AndI_reg_RShift_reg(iRegINoSp dst,
11718                          iRegIorL2I src1, iRegIorL2I src2,
11719                          immI src3) %{
11720   match(Set dst (AndI src1 (RShiftI src2 src3)));
11721 
11722   ins_cost(1.9 * INSN_COST);
11723   format %{ "andw  $dst, $src1, $src2, ASR $src3" %}
11724 
11725   ins_encode %{
11726     __ andw(as_Register($dst$$reg),
11727               as_Register($src1$$reg),
11728               as_Register($src2$$reg),
11729               Assembler::ASR,
11730               $src3$$constant & 0x1f);
11731   %}
11732 
11733   ins_pipe(ialu_reg_reg_shift);
11734 %}
11735 
11736 // This pattern is automatically generated from aarch64_ad.m4
11737 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11738 instruct AndL_reg_RShift_reg(iRegLNoSp dst,
11739                          iRegL src1, iRegL src2,
11740                          immI src3) %{
11741   match(Set dst (AndL src1 (RShiftL src2 src3)));
11742 
11743   ins_cost(1.9 * INSN_COST);
11744   format %{ "andr  $dst, $src1, $src2, ASR $src3" %}
11745 
11746   ins_encode %{
11747     __ andr(as_Register($dst$$reg),
11748               as_Register($src1$$reg),
11749               as_Register($src2$$reg),
11750               Assembler::ASR,
11751               $src3$$constant & 0x3f);
11752   %}
11753 
11754   ins_pipe(ialu_reg_reg_shift);
11755 %}
11756 
11757 // This pattern is automatically generated from aarch64_ad.m4
11758 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11759 instruct AndI_reg_LShift_reg(iRegINoSp dst,
11760                          iRegIorL2I src1, iRegIorL2I src2,
11761                          immI src3) %{
11762   match(Set dst (AndI src1 (LShiftI src2 src3)));
11763 
11764   ins_cost(1.9 * INSN_COST);
11765   format %{ "andw  $dst, $src1, $src2, LSL $src3" %}
11766 
11767   ins_encode %{
11768     __ andw(as_Register($dst$$reg),
11769               as_Register($src1$$reg),
11770               as_Register($src2$$reg),
11771               Assembler::LSL,
11772               $src3$$constant & 0x1f);
11773   %}
11774 
11775   ins_pipe(ialu_reg_reg_shift);
11776 %}
11777 
11778 // This pattern is automatically generated from aarch64_ad.m4
11779 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11780 instruct AndL_reg_LShift_reg(iRegLNoSp dst,
11781                          iRegL src1, iRegL src2,
11782                          immI src3) %{
11783   match(Set dst (AndL src1 (LShiftL src2 src3)));
11784 
11785   ins_cost(1.9 * INSN_COST);
11786   format %{ "andr  $dst, $src1, $src2, LSL $src3" %}
11787 
11788   ins_encode %{
11789     __ andr(as_Register($dst$$reg),
11790               as_Register($src1$$reg),
11791               as_Register($src2$$reg),
11792               Assembler::LSL,
11793               $src3$$constant & 0x3f);
11794   %}
11795 
11796   ins_pipe(ialu_reg_reg_shift);
11797 %}
11798 
11799 // This pattern is automatically generated from aarch64_ad.m4
11800 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11801 instruct AndI_reg_RotateRight_reg(iRegINoSp dst,
11802                          iRegIorL2I src1, iRegIorL2I src2,
11803                          immI src3) %{
11804   match(Set dst (AndI src1 (RotateRight src2 src3)));
11805 
11806   ins_cost(1.9 * INSN_COST);
11807   format %{ "andw  $dst, $src1, $src2, ROR $src3" %}
11808 
11809   ins_encode %{
11810     __ andw(as_Register($dst$$reg),
11811               as_Register($src1$$reg),
11812               as_Register($src2$$reg),
11813               Assembler::ROR,
11814               $src3$$constant & 0x1f);
11815   %}
11816 
11817   ins_pipe(ialu_reg_reg_shift);
11818 %}
11819 
11820 // This pattern is automatically generated from aarch64_ad.m4
11821 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11822 instruct AndL_reg_RotateRight_reg(iRegLNoSp dst,
11823                          iRegL src1, iRegL src2,
11824                          immI src3) %{
11825   match(Set dst (AndL src1 (RotateRight src2 src3)));
11826 
11827   ins_cost(1.9 * INSN_COST);
11828   format %{ "andr  $dst, $src1, $src2, ROR $src3" %}
11829 
11830   ins_encode %{
11831     __ andr(as_Register($dst$$reg),
11832               as_Register($src1$$reg),
11833               as_Register($src2$$reg),
11834               Assembler::ROR,
11835               $src3$$constant & 0x3f);
11836   %}
11837 
11838   ins_pipe(ialu_reg_reg_shift);
11839 %}
11840 
11841 // This pattern is automatically generated from aarch64_ad.m4
11842 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11843 instruct XorI_reg_URShift_reg(iRegINoSp dst,
11844                          iRegIorL2I src1, iRegIorL2I src2,
11845                          immI src3) %{
11846   match(Set dst (XorI src1 (URShiftI src2 src3)));
11847 
11848   ins_cost(1.9 * INSN_COST);
11849   format %{ "eorw  $dst, $src1, $src2, LSR $src3" %}
11850 
11851   ins_encode %{
11852     __ eorw(as_Register($dst$$reg),
11853               as_Register($src1$$reg),
11854               as_Register($src2$$reg),
11855               Assembler::LSR,
11856               $src3$$constant & 0x1f);
11857   %}
11858 
11859   ins_pipe(ialu_reg_reg_shift);
11860 %}
11861 
11862 // This pattern is automatically generated from aarch64_ad.m4
11863 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11864 instruct XorL_reg_URShift_reg(iRegLNoSp dst,
11865                          iRegL src1, iRegL src2,
11866                          immI src3) %{
11867   match(Set dst (XorL src1 (URShiftL src2 src3)));
11868 
11869   ins_cost(1.9 * INSN_COST);
11870   format %{ "eor  $dst, $src1, $src2, LSR $src3" %}
11871 
11872   ins_encode %{
11873     __ eor(as_Register($dst$$reg),
11874               as_Register($src1$$reg),
11875               as_Register($src2$$reg),
11876               Assembler::LSR,
11877               $src3$$constant & 0x3f);
11878   %}
11879 
11880   ins_pipe(ialu_reg_reg_shift);
11881 %}
11882 
11883 // This pattern is automatically generated from aarch64_ad.m4
11884 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11885 instruct XorI_reg_RShift_reg(iRegINoSp dst,
11886                          iRegIorL2I src1, iRegIorL2I src2,
11887                          immI src3) %{
11888   match(Set dst (XorI src1 (RShiftI src2 src3)));
11889 
11890   ins_cost(1.9 * INSN_COST);
11891   format %{ "eorw  $dst, $src1, $src2, ASR $src3" %}
11892 
11893   ins_encode %{
11894     __ eorw(as_Register($dst$$reg),
11895               as_Register($src1$$reg),
11896               as_Register($src2$$reg),
11897               Assembler::ASR,
11898               $src3$$constant & 0x1f);
11899   %}
11900 
11901   ins_pipe(ialu_reg_reg_shift);
11902 %}
11903 
11904 // This pattern is automatically generated from aarch64_ad.m4
11905 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11906 instruct XorL_reg_RShift_reg(iRegLNoSp dst,
11907                          iRegL src1, iRegL src2,
11908                          immI src3) %{
11909   match(Set dst (XorL src1 (RShiftL src2 src3)));
11910 
11911   ins_cost(1.9 * INSN_COST);
11912   format %{ "eor  $dst, $src1, $src2, ASR $src3" %}
11913 
11914   ins_encode %{
11915     __ eor(as_Register($dst$$reg),
11916               as_Register($src1$$reg),
11917               as_Register($src2$$reg),
11918               Assembler::ASR,
11919               $src3$$constant & 0x3f);
11920   %}
11921 
11922   ins_pipe(ialu_reg_reg_shift);
11923 %}
11924 
11925 // This pattern is automatically generated from aarch64_ad.m4
11926 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11927 instruct XorI_reg_LShift_reg(iRegINoSp dst,
11928                          iRegIorL2I src1, iRegIorL2I src2,
11929                          immI src3) %{
11930   match(Set dst (XorI src1 (LShiftI src2 src3)));
11931 
11932   ins_cost(1.9 * INSN_COST);
11933   format %{ "eorw  $dst, $src1, $src2, LSL $src3" %}
11934 
11935   ins_encode %{
11936     __ eorw(as_Register($dst$$reg),
11937               as_Register($src1$$reg),
11938               as_Register($src2$$reg),
11939               Assembler::LSL,
11940               $src3$$constant & 0x1f);
11941   %}
11942 
11943   ins_pipe(ialu_reg_reg_shift);
11944 %}
11945 
11946 // This pattern is automatically generated from aarch64_ad.m4
11947 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11948 instruct XorL_reg_LShift_reg(iRegLNoSp dst,
11949                          iRegL src1, iRegL src2,
11950                          immI src3) %{
11951   match(Set dst (XorL src1 (LShiftL src2 src3)));
11952 
11953   ins_cost(1.9 * INSN_COST);
11954   format %{ "eor  $dst, $src1, $src2, LSL $src3" %}
11955 
11956   ins_encode %{
11957     __ eor(as_Register($dst$$reg),
11958               as_Register($src1$$reg),
11959               as_Register($src2$$reg),
11960               Assembler::LSL,
11961               $src3$$constant & 0x3f);
11962   %}
11963 
11964   ins_pipe(ialu_reg_reg_shift);
11965 %}
11966 
11967 // This pattern is automatically generated from aarch64_ad.m4
11968 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11969 instruct XorI_reg_RotateRight_reg(iRegINoSp dst,
11970                          iRegIorL2I src1, iRegIorL2I src2,
11971                          immI src3) %{
11972   match(Set dst (XorI src1 (RotateRight src2 src3)));
11973 
11974   ins_cost(1.9 * INSN_COST);
11975   format %{ "eorw  $dst, $src1, $src2, ROR $src3" %}
11976 
11977   ins_encode %{
11978     __ eorw(as_Register($dst$$reg),
11979               as_Register($src1$$reg),
11980               as_Register($src2$$reg),
11981               Assembler::ROR,
11982               $src3$$constant & 0x1f);
11983   %}
11984 
11985   ins_pipe(ialu_reg_reg_shift);
11986 %}
11987 
11988 // This pattern is automatically generated from aarch64_ad.m4
11989 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11990 instruct XorL_reg_RotateRight_reg(iRegLNoSp dst,
11991                          iRegL src1, iRegL src2,
11992                          immI src3) %{
11993   match(Set dst (XorL src1 (RotateRight src2 src3)));
11994 
11995   ins_cost(1.9 * INSN_COST);
11996   format %{ "eor  $dst, $src1, $src2, ROR $src3" %}
11997 
11998   ins_encode %{
11999     __ eor(as_Register($dst$$reg),
12000               as_Register($src1$$reg),
12001               as_Register($src2$$reg),
12002               Assembler::ROR,
12003               $src3$$constant & 0x3f);
12004   %}
12005 
12006   ins_pipe(ialu_reg_reg_shift);
12007 %}
12008 
12009 // This pattern is automatically generated from aarch64_ad.m4
12010 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12011 instruct OrI_reg_URShift_reg(iRegINoSp dst,
12012                          iRegIorL2I src1, iRegIorL2I src2,
12013                          immI src3) %{
12014   match(Set dst (OrI src1 (URShiftI src2 src3)));
12015 
12016   ins_cost(1.9 * INSN_COST);
12017   format %{ "orrw  $dst, $src1, $src2, LSR $src3" %}
12018 
12019   ins_encode %{
12020     __ orrw(as_Register($dst$$reg),
12021               as_Register($src1$$reg),
12022               as_Register($src2$$reg),
12023               Assembler::LSR,
12024               $src3$$constant & 0x1f);
12025   %}
12026 
12027   ins_pipe(ialu_reg_reg_shift);
12028 %}
12029 
12030 // This pattern is automatically generated from aarch64_ad.m4
12031 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12032 instruct OrL_reg_URShift_reg(iRegLNoSp dst,
12033                          iRegL src1, iRegL src2,
12034                          immI src3) %{
12035   match(Set dst (OrL src1 (URShiftL src2 src3)));
12036 
12037   ins_cost(1.9 * INSN_COST);
12038   format %{ "orr  $dst, $src1, $src2, LSR $src3" %}
12039 
12040   ins_encode %{
12041     __ orr(as_Register($dst$$reg),
12042               as_Register($src1$$reg),
12043               as_Register($src2$$reg),
12044               Assembler::LSR,
12045               $src3$$constant & 0x3f);
12046   %}
12047 
12048   ins_pipe(ialu_reg_reg_shift);
12049 %}
12050 
12051 // This pattern is automatically generated from aarch64_ad.m4
12052 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12053 instruct OrI_reg_RShift_reg(iRegINoSp dst,
12054                          iRegIorL2I src1, iRegIorL2I src2,
12055                          immI src3) %{
12056   match(Set dst (OrI src1 (RShiftI src2 src3)));
12057 
12058   ins_cost(1.9 * INSN_COST);
12059   format %{ "orrw  $dst, $src1, $src2, ASR $src3" %}
12060 
12061   ins_encode %{
12062     __ orrw(as_Register($dst$$reg),
12063               as_Register($src1$$reg),
12064               as_Register($src2$$reg),
12065               Assembler::ASR,
12066               $src3$$constant & 0x1f);
12067   %}
12068 
12069   ins_pipe(ialu_reg_reg_shift);
12070 %}
12071 
12072 // This pattern is automatically generated from aarch64_ad.m4
12073 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12074 instruct OrL_reg_RShift_reg(iRegLNoSp dst,
12075                          iRegL src1, iRegL src2,
12076                          immI src3) %{
12077   match(Set dst (OrL src1 (RShiftL src2 src3)));
12078 
12079   ins_cost(1.9 * INSN_COST);
12080   format %{ "orr  $dst, $src1, $src2, ASR $src3" %}
12081 
12082   ins_encode %{
12083     __ orr(as_Register($dst$$reg),
12084               as_Register($src1$$reg),
12085               as_Register($src2$$reg),
12086               Assembler::ASR,
12087               $src3$$constant & 0x3f);
12088   %}
12089 
12090   ins_pipe(ialu_reg_reg_shift);
12091 %}
12092 
12093 // This pattern is automatically generated from aarch64_ad.m4
12094 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12095 instruct OrI_reg_LShift_reg(iRegINoSp dst,
12096                          iRegIorL2I src1, iRegIorL2I src2,
12097                          immI src3) %{
12098   match(Set dst (OrI src1 (LShiftI src2 src3)));
12099 
12100   ins_cost(1.9 * INSN_COST);
12101   format %{ "orrw  $dst, $src1, $src2, LSL $src3" %}
12102 
12103   ins_encode %{
12104     __ orrw(as_Register($dst$$reg),
12105               as_Register($src1$$reg),
12106               as_Register($src2$$reg),
12107               Assembler::LSL,
12108               $src3$$constant & 0x1f);
12109   %}
12110 
12111   ins_pipe(ialu_reg_reg_shift);
12112 %}
12113 
12114 // This pattern is automatically generated from aarch64_ad.m4
12115 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12116 instruct OrL_reg_LShift_reg(iRegLNoSp dst,
12117                          iRegL src1, iRegL src2,
12118                          immI src3) %{
12119   match(Set dst (OrL src1 (LShiftL src2 src3)));
12120 
12121   ins_cost(1.9 * INSN_COST);
12122   format %{ "orr  $dst, $src1, $src2, LSL $src3" %}
12123 
12124   ins_encode %{
12125     __ orr(as_Register($dst$$reg),
12126               as_Register($src1$$reg),
12127               as_Register($src2$$reg),
12128               Assembler::LSL,
12129               $src3$$constant & 0x3f);
12130   %}
12131 
12132   ins_pipe(ialu_reg_reg_shift);
12133 %}
12134 
12135 // This pattern is automatically generated from aarch64_ad.m4
12136 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12137 instruct OrI_reg_RotateRight_reg(iRegINoSp dst,
12138                          iRegIorL2I src1, iRegIorL2I src2,
12139                          immI src3) %{
12140   match(Set dst (OrI src1 (RotateRight src2 src3)));
12141 
12142   ins_cost(1.9 * INSN_COST);
12143   format %{ "orrw  $dst, $src1, $src2, ROR $src3" %}
12144 
12145   ins_encode %{
12146     __ orrw(as_Register($dst$$reg),
12147               as_Register($src1$$reg),
12148               as_Register($src2$$reg),
12149               Assembler::ROR,
12150               $src3$$constant & 0x1f);
12151   %}
12152 
12153   ins_pipe(ialu_reg_reg_shift);
12154 %}
12155 
12156 // This pattern is automatically generated from aarch64_ad.m4
12157 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12158 instruct OrL_reg_RotateRight_reg(iRegLNoSp dst,
12159                          iRegL src1, iRegL src2,
12160                          immI src3) %{
12161   match(Set dst (OrL src1 (RotateRight src2 src3)));
12162 
12163   ins_cost(1.9 * INSN_COST);
12164   format %{ "orr  $dst, $src1, $src2, ROR $src3" %}
12165 
12166   ins_encode %{
12167     __ orr(as_Register($dst$$reg),
12168               as_Register($src1$$reg),
12169               as_Register($src2$$reg),
12170               Assembler::ROR,
12171               $src3$$constant & 0x3f);
12172   %}
12173 
12174   ins_pipe(ialu_reg_reg_shift);
12175 %}
12176 
12177 // This pattern is automatically generated from aarch64_ad.m4
12178 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12179 instruct AddI_reg_URShift_reg(iRegINoSp dst,
12180                          iRegIorL2I src1, iRegIorL2I src2,
12181                          immI src3) %{
12182   match(Set dst (AddI src1 (URShiftI src2 src3)));
12183 
12184   ins_cost(1.9 * INSN_COST);
12185   format %{ "addw  $dst, $src1, $src2, LSR $src3" %}
12186 
12187   ins_encode %{
12188     __ addw(as_Register($dst$$reg),
12189               as_Register($src1$$reg),
12190               as_Register($src2$$reg),
12191               Assembler::LSR,
12192               $src3$$constant & 0x1f);
12193   %}
12194 
12195   ins_pipe(ialu_reg_reg_shift);
12196 %}
12197 
12198 // This pattern is automatically generated from aarch64_ad.m4
12199 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12200 instruct AddL_reg_URShift_reg(iRegLNoSp dst,
12201                          iRegL src1, iRegL src2,
12202                          immI src3) %{
12203   match(Set dst (AddL src1 (URShiftL src2 src3)));
12204 
12205   ins_cost(1.9 * INSN_COST);
12206   format %{ "add  $dst, $src1, $src2, LSR $src3" %}
12207 
12208   ins_encode %{
12209     __ add(as_Register($dst$$reg),
12210               as_Register($src1$$reg),
12211               as_Register($src2$$reg),
12212               Assembler::LSR,
12213               $src3$$constant & 0x3f);
12214   %}
12215 
12216   ins_pipe(ialu_reg_reg_shift);
12217 %}
12218 
12219 // This pattern is automatically generated from aarch64_ad.m4
12220 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12221 instruct AddI_reg_RShift_reg(iRegINoSp dst,
12222                          iRegIorL2I src1, iRegIorL2I src2,
12223                          immI src3) %{
12224   match(Set dst (AddI src1 (RShiftI src2 src3)));
12225 
12226   ins_cost(1.9 * INSN_COST);
12227   format %{ "addw  $dst, $src1, $src2, ASR $src3" %}
12228 
12229   ins_encode %{
12230     __ addw(as_Register($dst$$reg),
12231               as_Register($src1$$reg),
12232               as_Register($src2$$reg),
12233               Assembler::ASR,
12234               $src3$$constant & 0x1f);
12235   %}
12236 
12237   ins_pipe(ialu_reg_reg_shift);
12238 %}
12239 
12240 // This pattern is automatically generated from aarch64_ad.m4
12241 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12242 instruct AddL_reg_RShift_reg(iRegLNoSp dst,
12243                          iRegL src1, iRegL src2,
12244                          immI src3) %{
12245   match(Set dst (AddL src1 (RShiftL src2 src3)));
12246 
12247   ins_cost(1.9 * INSN_COST);
12248   format %{ "add  $dst, $src1, $src2, ASR $src3" %}
12249 
12250   ins_encode %{
12251     __ add(as_Register($dst$$reg),
12252               as_Register($src1$$reg),
12253               as_Register($src2$$reg),
12254               Assembler::ASR,
12255               $src3$$constant & 0x3f);
12256   %}
12257 
12258   ins_pipe(ialu_reg_reg_shift);
12259 %}
12260 
12261 // This pattern is automatically generated from aarch64_ad.m4
12262 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12263 instruct AddI_reg_LShift_reg(iRegINoSp dst,
12264                          iRegIorL2I src1, iRegIorL2I src2,
12265                          immI src3) %{
12266   match(Set dst (AddI src1 (LShiftI src2 src3)));
12267 
12268   ins_cost(1.9 * INSN_COST);
12269   format %{ "addw  $dst, $src1, $src2, LSL $src3" %}
12270 
12271   ins_encode %{
12272     __ addw(as_Register($dst$$reg),
12273               as_Register($src1$$reg),
12274               as_Register($src2$$reg),
12275               Assembler::LSL,
12276               $src3$$constant & 0x1f);
12277   %}
12278 
12279   ins_pipe(ialu_reg_reg_shift);
12280 %}
12281 
12282 // This pattern is automatically generated from aarch64_ad.m4
12283 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12284 instruct AddL_reg_LShift_reg(iRegLNoSp dst,
12285                          iRegL src1, iRegL src2,
12286                          immI src3) %{
12287   match(Set dst (AddL src1 (LShiftL src2 src3)));
12288 
12289   ins_cost(1.9 * INSN_COST);
12290   format %{ "add  $dst, $src1, $src2, LSL $src3" %}
12291 
12292   ins_encode %{
12293     __ add(as_Register($dst$$reg),
12294               as_Register($src1$$reg),
12295               as_Register($src2$$reg),
12296               Assembler::LSL,
12297               $src3$$constant & 0x3f);
12298   %}
12299 
12300   ins_pipe(ialu_reg_reg_shift);
12301 %}
12302 
12303 // This pattern is automatically generated from aarch64_ad.m4
12304 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12305 instruct SubI_reg_URShift_reg(iRegINoSp dst,
12306                          iRegIorL2I src1, iRegIorL2I src2,
12307                          immI src3) %{
12308   match(Set dst (SubI src1 (URShiftI src2 src3)));
12309 
12310   ins_cost(1.9 * INSN_COST);
12311   format %{ "subw  $dst, $src1, $src2, LSR $src3" %}
12312 
12313   ins_encode %{
12314     __ subw(as_Register($dst$$reg),
12315               as_Register($src1$$reg),
12316               as_Register($src2$$reg),
12317               Assembler::LSR,
12318               $src3$$constant & 0x1f);
12319   %}
12320 
12321   ins_pipe(ialu_reg_reg_shift);
12322 %}
12323 
12324 // This pattern is automatically generated from aarch64_ad.m4
12325 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12326 instruct SubL_reg_URShift_reg(iRegLNoSp dst,
12327                          iRegL src1, iRegL src2,
12328                          immI src3) %{
12329   match(Set dst (SubL src1 (URShiftL src2 src3)));
12330 
12331   ins_cost(1.9 * INSN_COST);
12332   format %{ "sub  $dst, $src1, $src2, LSR $src3" %}
12333 
12334   ins_encode %{
12335     __ sub(as_Register($dst$$reg),
12336               as_Register($src1$$reg),
12337               as_Register($src2$$reg),
12338               Assembler::LSR,
12339               $src3$$constant & 0x3f);
12340   %}
12341 
12342   ins_pipe(ialu_reg_reg_shift);
12343 %}
12344 
12345 // This pattern is automatically generated from aarch64_ad.m4
12346 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12347 instruct SubI_reg_RShift_reg(iRegINoSp dst,
12348                          iRegIorL2I src1, iRegIorL2I src2,
12349                          immI src3) %{
12350   match(Set dst (SubI src1 (RShiftI src2 src3)));
12351 
12352   ins_cost(1.9 * INSN_COST);
12353   format %{ "subw  $dst, $src1, $src2, ASR $src3" %}
12354 
12355   ins_encode %{
12356     __ subw(as_Register($dst$$reg),
12357               as_Register($src1$$reg),
12358               as_Register($src2$$reg),
12359               Assembler::ASR,
12360               $src3$$constant & 0x1f);
12361   %}
12362 
12363   ins_pipe(ialu_reg_reg_shift);
12364 %}
12365 
12366 // This pattern is automatically generated from aarch64_ad.m4
12367 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12368 instruct SubL_reg_RShift_reg(iRegLNoSp dst,
12369                          iRegL src1, iRegL src2,
12370                          immI src3) %{
12371   match(Set dst (SubL src1 (RShiftL src2 src3)));
12372 
12373   ins_cost(1.9 * INSN_COST);
12374   format %{ "sub  $dst, $src1, $src2, ASR $src3" %}
12375 
12376   ins_encode %{
12377     __ sub(as_Register($dst$$reg),
12378               as_Register($src1$$reg),
12379               as_Register($src2$$reg),
12380               Assembler::ASR,
12381               $src3$$constant & 0x3f);
12382   %}
12383 
12384   ins_pipe(ialu_reg_reg_shift);
12385 %}
12386 
12387 // This pattern is automatically generated from aarch64_ad.m4
12388 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12389 instruct SubI_reg_LShift_reg(iRegINoSp dst,
12390                          iRegIorL2I src1, iRegIorL2I src2,
12391                          immI src3) %{
12392   match(Set dst (SubI src1 (LShiftI src2 src3)));
12393 
12394   ins_cost(1.9 * INSN_COST);
12395   format %{ "subw  $dst, $src1, $src2, LSL $src3" %}
12396 
12397   ins_encode %{
12398     __ subw(as_Register($dst$$reg),
12399               as_Register($src1$$reg),
12400               as_Register($src2$$reg),
12401               Assembler::LSL,
12402               $src3$$constant & 0x1f);
12403   %}
12404 
12405   ins_pipe(ialu_reg_reg_shift);
12406 %}
12407 
12408 // This pattern is automatically generated from aarch64_ad.m4
12409 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12410 instruct SubL_reg_LShift_reg(iRegLNoSp dst,
12411                          iRegL src1, iRegL src2,
12412                          immI src3) %{
12413   match(Set dst (SubL src1 (LShiftL src2 src3)));
12414 
12415   ins_cost(1.9 * INSN_COST);
12416   format %{ "sub  $dst, $src1, $src2, LSL $src3" %}
12417 
12418   ins_encode %{
12419     __ sub(as_Register($dst$$reg),
12420               as_Register($src1$$reg),
12421               as_Register($src2$$reg),
12422               Assembler::LSL,
12423               $src3$$constant & 0x3f);
12424   %}
12425 
12426   ins_pipe(ialu_reg_reg_shift);
12427 %}
12428 
12429 // This pattern is automatically generated from aarch64_ad.m4
12430 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12431 
12432 // Shift Left followed by Shift Right.
12433 // This idiom is used by the compiler for the i2b bytecode etc.
12434 instruct sbfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12435 %{
12436   match(Set dst (RShiftL (LShiftL src lshift_count) rshift_count));
12437   ins_cost(INSN_COST * 2);
12438   format %{ "sbfm  $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12439   ins_encode %{
12440     int lshift = $lshift_count$$constant & 63;
12441     int rshift = $rshift_count$$constant & 63;
12442     int s = 63 - lshift;
12443     int r = (rshift - lshift) & 63;
12444     __ sbfm(as_Register($dst$$reg),
12445             as_Register($src$$reg),
12446             r, s);
12447   %}
12448 
12449   ins_pipe(ialu_reg_shift);
12450 %}
12451 
12452 // This pattern is automatically generated from aarch64_ad.m4
12453 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12454 
12455 // Shift Left followed by Shift Right.
12456 // This idiom is used by the compiler for the i2b bytecode etc.
12457 instruct sbfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12458 %{
12459   match(Set dst (RShiftI (LShiftI src lshift_count) rshift_count));
12460   ins_cost(INSN_COST * 2);
12461   format %{ "sbfmw  $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12462   ins_encode %{
12463     int lshift = $lshift_count$$constant & 31;
12464     int rshift = $rshift_count$$constant & 31;
12465     int s = 31 - lshift;
12466     int r = (rshift - lshift) & 31;
12467     __ sbfmw(as_Register($dst$$reg),
12468             as_Register($src$$reg),
12469             r, s);
12470   %}
12471 
12472   ins_pipe(ialu_reg_shift);
12473 %}
12474 
12475 // This pattern is automatically generated from aarch64_ad.m4
12476 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12477 
12478 // Shift Left followed by Shift Right.
12479 // This idiom is used by the compiler for the i2b bytecode etc.
12480 instruct ubfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12481 %{
12482   match(Set dst (URShiftL (LShiftL src lshift_count) rshift_count));
12483   ins_cost(INSN_COST * 2);
12484   format %{ "ubfm  $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12485   ins_encode %{
12486     int lshift = $lshift_count$$constant & 63;
12487     int rshift = $rshift_count$$constant & 63;
12488     int s = 63 - lshift;
12489     int r = (rshift - lshift) & 63;
12490     __ ubfm(as_Register($dst$$reg),
12491             as_Register($src$$reg),
12492             r, s);
12493   %}
12494 
12495   ins_pipe(ialu_reg_shift);
12496 %}
12497 
12498 // This pattern is automatically generated from aarch64_ad.m4
12499 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12500 
12501 // Shift Left followed by Shift Right.
12502 // This idiom is used by the compiler for the i2b bytecode etc.
12503 instruct ubfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12504 %{
12505   match(Set dst (URShiftI (LShiftI src lshift_count) rshift_count));
12506   ins_cost(INSN_COST * 2);
12507   format %{ "ubfmw  $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12508   ins_encode %{
12509     int lshift = $lshift_count$$constant & 31;
12510     int rshift = $rshift_count$$constant & 31;
12511     int s = 31 - lshift;
12512     int r = (rshift - lshift) & 31;
12513     __ ubfmw(as_Register($dst$$reg),
12514             as_Register($src$$reg),
12515             r, s);
12516   %}
12517 
12518   ins_pipe(ialu_reg_shift);
12519 %}
12520 
12521 // Bitfield extract with shift & mask
12522 
12523 // This pattern is automatically generated from aarch64_ad.m4
12524 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12525 instruct ubfxwI(iRegINoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12526 %{
12527   match(Set dst (AndI (URShiftI src rshift) mask));
12528   // Make sure we are not going to exceed what ubfxw can do.
12529   predicate((exact_log2(n->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12530 
12531   ins_cost(INSN_COST);
12532   format %{ "ubfxw $dst, $src, $rshift, $mask" %}
12533   ins_encode %{
12534     int rshift = $rshift$$constant & 31;
12535     intptr_t mask = $mask$$constant;
12536     int width = exact_log2(mask+1);
12537     __ ubfxw(as_Register($dst$$reg),
12538             as_Register($src$$reg), rshift, width);
12539   %}
12540   ins_pipe(ialu_reg_shift);
12541 %}
12542 
12543 // This pattern is automatically generated from aarch64_ad.m4
12544 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12545 instruct ubfxL(iRegLNoSp dst, iRegL src, immI rshift, immL_bitmask mask)
12546 %{
12547   match(Set dst (AndL (URShiftL src rshift) mask));
12548   // Make sure we are not going to exceed what ubfx can do.
12549   predicate((exact_log2_long(n->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= (63 + 1));
12550 
12551   ins_cost(INSN_COST);
12552   format %{ "ubfx $dst, $src, $rshift, $mask" %}
12553   ins_encode %{
12554     int rshift = $rshift$$constant & 63;
12555     intptr_t mask = $mask$$constant;
12556     int width = exact_log2_long(mask+1);
12557     __ ubfx(as_Register($dst$$reg),
12558             as_Register($src$$reg), rshift, width);
12559   %}
12560   ins_pipe(ialu_reg_shift);
12561 %}
12562 
12563 
12564 // This pattern is automatically generated from aarch64_ad.m4
12565 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12566 
12567 // We can use ubfx when extending an And with a mask when we know mask
12568 // is positive.  We know that because immI_bitmask guarantees it.
12569 instruct ubfxIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12570 %{
12571   match(Set dst (ConvI2L (AndI (URShiftI src rshift) mask)));
12572   // Make sure we are not going to exceed what ubfxw can do.
12573   predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12574 
12575   ins_cost(INSN_COST * 2);
12576   format %{ "ubfx $dst, $src, $rshift, $mask" %}
12577   ins_encode %{
12578     int rshift = $rshift$$constant & 31;
12579     intptr_t mask = $mask$$constant;
12580     int width = exact_log2(mask+1);
12581     __ ubfx(as_Register($dst$$reg),
12582             as_Register($src$$reg), rshift, width);
12583   %}
12584   ins_pipe(ialu_reg_shift);
12585 %}
12586 
12587 
12588 // This pattern is automatically generated from aarch64_ad.m4
12589 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12590 
12591 // We can use ubfiz when masking by a positive number and then left shifting the result.
12592 // We know that the mask is positive because immI_bitmask guarantees it.
12593 instruct ubfizwI(iRegINoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12594 %{
12595   match(Set dst (LShiftI (AndI src mask) lshift));
12596   predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 31)) <= (31 + 1));
12597 
12598   ins_cost(INSN_COST);
12599   format %{ "ubfizw $dst, $src, $lshift, $mask" %}
12600   ins_encode %{
12601     int lshift = $lshift$$constant & 31;
12602     intptr_t mask = $mask$$constant;
12603     int width = exact_log2(mask+1);
12604     __ ubfizw(as_Register($dst$$reg),
12605           as_Register($src$$reg), lshift, width);
12606   %}
12607   ins_pipe(ialu_reg_shift);
12608 %}
12609 
12610 // This pattern is automatically generated from aarch64_ad.m4
12611 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12612 
12613 // We can use ubfiz when masking by a positive number and then left shifting the result.
12614 // We know that the mask is positive because immL_bitmask guarantees it.
12615 instruct ubfizL(iRegLNoSp dst, iRegL src, immI lshift, immL_bitmask mask)
12616 %{
12617   match(Set dst (LShiftL (AndL src mask) lshift));
12618   predicate((exact_log2_long(n->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
12619 
12620   ins_cost(INSN_COST);
12621   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12622   ins_encode %{
12623     int lshift = $lshift$$constant & 63;
12624     intptr_t mask = $mask$$constant;
12625     int width = exact_log2_long(mask+1);
12626     __ ubfiz(as_Register($dst$$reg),
12627           as_Register($src$$reg), lshift, width);
12628   %}
12629   ins_pipe(ialu_reg_shift);
12630 %}
12631 
12632 // This pattern is automatically generated from aarch64_ad.m4
12633 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12634 
12635 // We can use ubfiz when masking by a positive number and then left shifting the result.
12636 // We know that the mask is positive because immI_bitmask guarantees it.
12637 instruct ubfizwIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12638 %{
12639   match(Set dst (ConvI2L (LShiftI (AndI src mask) lshift)));
12640   predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= 31);
12641 
12642   ins_cost(INSN_COST);
12643   format %{ "ubfizw $dst, $src, $lshift, $mask" %}
12644   ins_encode %{
12645     int lshift = $lshift$$constant & 31;
12646     intptr_t mask = $mask$$constant;
12647     int width = exact_log2(mask+1);
12648     __ ubfizw(as_Register($dst$$reg),
12649           as_Register($src$$reg), lshift, width);
12650   %}
12651   ins_pipe(ialu_reg_shift);
12652 %}
12653 
12654 // This pattern is automatically generated from aarch64_ad.m4
12655 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12656 
12657 // We can use ubfiz when masking by a positive number and then left shifting the result.
12658 // We know that the mask is positive because immL_bitmask guarantees it.
12659 instruct ubfizLConvL2I(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
12660 %{
12661   match(Set dst (ConvL2I (LShiftL (AndL src mask) lshift)));
12662   predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= 31);
12663 
12664   ins_cost(INSN_COST);
12665   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12666   ins_encode %{
12667     int lshift = $lshift$$constant & 63;
12668     intptr_t mask = $mask$$constant;
12669     int width = exact_log2_long(mask+1);
12670     __ ubfiz(as_Register($dst$$reg),
12671           as_Register($src$$reg), lshift, width);
12672   %}
12673   ins_pipe(ialu_reg_shift);
12674 %}
12675 
12676 
12677 // This pattern is automatically generated from aarch64_ad.m4
12678 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12679 
12680 // If there is a convert I to L block between and AndI and a LShiftL, we can also match ubfiz
12681 instruct ubfizIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12682 %{
12683   match(Set dst (LShiftL (ConvI2L (AndI src mask)) lshift));
12684   predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
12685 
12686   ins_cost(INSN_COST);
12687   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12688   ins_encode %{
12689     int lshift = $lshift$$constant & 63;
12690     intptr_t mask = $mask$$constant;
12691     int width = exact_log2(mask+1);
12692     __ ubfiz(as_Register($dst$$reg),
12693              as_Register($src$$reg), lshift, width);
12694   %}
12695   ins_pipe(ialu_reg_shift);
12696 %}
12697 
12698 // This pattern is automatically generated from aarch64_ad.m4
12699 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12700 
12701 // If there is a convert L to I block between and AndL and a LShiftI, we can also match ubfiz
12702 instruct ubfizLConvL2Ix(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
12703 %{
12704   match(Set dst (LShiftI (ConvL2I (AndL src mask)) lshift));
12705   predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 31)) <= 31);
12706 
12707   ins_cost(INSN_COST);
12708   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12709   ins_encode %{
12710     int lshift = $lshift$$constant & 31;
12711     intptr_t mask = $mask$$constant;
12712     int width = exact_log2(mask+1);
12713     __ ubfiz(as_Register($dst$$reg),
12714              as_Register($src$$reg), lshift, width);
12715   %}
12716   ins_pipe(ialu_reg_shift);
12717 %}
12718 
12719 // This pattern is automatically generated from aarch64_ad.m4
12720 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12721 
12722 // Can skip int2long conversions after AND with small bitmask
12723 instruct ubfizIConvI2LAndI(iRegLNoSp dst, iRegI src, immI_bitmask msk)
12724 %{
12725   match(Set dst (ConvI2L (AndI src msk)));
12726   ins_cost(INSN_COST);
12727   format %{ "ubfiz $dst, $src, 0, exact_log2($msk + 1) " %}
12728   ins_encode %{
12729     __ ubfiz(as_Register($dst$$reg), as_Register($src$$reg), 0, exact_log2($msk$$constant + 1));
12730   %}
12731   ins_pipe(ialu_reg_shift);
12732 %}
12733 
12734 
12735 // Rotations
12736 
12737 // This pattern is automatically generated from aarch64_ad.m4
12738 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12739 instruct extrOrL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
12740 %{
12741   match(Set dst (OrL (LShiftL src1 lshift) (URShiftL src2 rshift)));
12742   predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
12743 
12744   ins_cost(INSN_COST);
12745   format %{ "extr $dst, $src1, $src2, #$rshift" %}
12746 
12747   ins_encode %{
12748     __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12749             $rshift$$constant & 63);
12750   %}
12751   ins_pipe(ialu_reg_reg_extr);
12752 %}
12753 
12754 
12755 // This pattern is automatically generated from aarch64_ad.m4
12756 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12757 instruct extrOrI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
12758 %{
12759   match(Set dst (OrI (LShiftI src1 lshift) (URShiftI src2 rshift)));
12760   predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
12761 
12762   ins_cost(INSN_COST);
12763   format %{ "extr $dst, $src1, $src2, #$rshift" %}
12764 
12765   ins_encode %{
12766     __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12767             $rshift$$constant & 31);
12768   %}
12769   ins_pipe(ialu_reg_reg_extr);
12770 %}
12771 
12772 
12773 // This pattern is automatically generated from aarch64_ad.m4
12774 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12775 instruct extrAddL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
12776 %{
12777   match(Set dst (AddL (LShiftL src1 lshift) (URShiftL src2 rshift)));
12778   predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
12779 
12780   ins_cost(INSN_COST);
12781   format %{ "extr $dst, $src1, $src2, #$rshift" %}
12782 
12783   ins_encode %{
12784     __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12785             $rshift$$constant & 63);
12786   %}
12787   ins_pipe(ialu_reg_reg_extr);
12788 %}
12789 
12790 
12791 // This pattern is automatically generated from aarch64_ad.m4
12792 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12793 instruct extrAddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
12794 %{
12795   match(Set dst (AddI (LShiftI src1 lshift) (URShiftI src2 rshift)));
12796   predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
12797 
12798   ins_cost(INSN_COST);
12799   format %{ "extr $dst, $src1, $src2, #$rshift" %}
12800 
12801   ins_encode %{
12802     __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12803             $rshift$$constant & 31);
12804   %}
12805   ins_pipe(ialu_reg_reg_extr);
12806 %}
12807 
12808 // This pattern is automatically generated from aarch64_ad.m4
12809 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12810 instruct rorI_imm(iRegINoSp dst, iRegI src, immI shift)
12811 %{
12812   match(Set dst (RotateRight src shift));
12813 
12814   ins_cost(INSN_COST);
12815   format %{ "ror    $dst, $src, $shift" %}
12816 
12817   ins_encode %{
12818      __ extrw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
12819                $shift$$constant & 0x1f);
12820   %}
12821   ins_pipe(ialu_reg_reg_vshift);
12822 %}
12823 
12824 // This pattern is automatically generated from aarch64_ad.m4
12825 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12826 instruct rorL_imm(iRegLNoSp dst, iRegL src, immI shift)
12827 %{
12828   match(Set dst (RotateRight src shift));
12829 
12830   ins_cost(INSN_COST);
12831   format %{ "ror    $dst, $src, $shift" %}
12832 
12833   ins_encode %{
12834      __ extr(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
12835                $shift$$constant & 0x3f);
12836   %}
12837   ins_pipe(ialu_reg_reg_vshift);
12838 %}
12839 
12840 // This pattern is automatically generated from aarch64_ad.m4
12841 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12842 instruct rorI_reg(iRegINoSp dst, iRegI src, iRegI shift)
12843 %{
12844   match(Set dst (RotateRight src shift));
12845 
12846   ins_cost(INSN_COST);
12847   format %{ "ror    $dst, $src, $shift" %}
12848 
12849   ins_encode %{
12850      __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
12851   %}
12852   ins_pipe(ialu_reg_reg_vshift);
12853 %}
12854 
12855 // This pattern is automatically generated from aarch64_ad.m4
12856 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12857 instruct rorL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
12858 %{
12859   match(Set dst (RotateRight src shift));
12860 
12861   ins_cost(INSN_COST);
12862   format %{ "ror    $dst, $src, $shift" %}
12863 
12864   ins_encode %{
12865      __ rorv(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
12866   %}
12867   ins_pipe(ialu_reg_reg_vshift);
12868 %}
12869 
12870 // This pattern is automatically generated from aarch64_ad.m4
12871 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12872 instruct rolI_reg(iRegINoSp dst, iRegI src, iRegI shift)
12873 %{
12874   match(Set dst (RotateLeft src shift));
12875 
12876   ins_cost(INSN_COST);
12877   format %{ "rol    $dst, $src, $shift" %}
12878 
12879   ins_encode %{
12880      __ subw(rscratch1, zr, as_Register($shift$$reg));
12881      __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
12882   %}
12883   ins_pipe(ialu_reg_reg_vshift);
12884 %}
12885 
12886 // This pattern is automatically generated from aarch64_ad.m4
12887 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12888 instruct rolL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
12889 %{
12890   match(Set dst (RotateLeft src shift));
12891 
12892   ins_cost(INSN_COST);
12893   format %{ "rol    $dst, $src, $shift" %}
12894 
12895   ins_encode %{
12896      __ subw(rscratch1, zr, as_Register($shift$$reg));
12897      __ rorv(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
12898   %}
12899   ins_pipe(ialu_reg_reg_vshift);
12900 %}
12901 
12902 
12903 // Add/subtract (extended)
12904 
12905 // This pattern is automatically generated from aarch64_ad.m4
12906 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12907 instruct AddExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
12908 %{
12909   match(Set dst (AddL src1 (ConvI2L src2)));
12910   ins_cost(INSN_COST);
12911   format %{ "add  $dst, $src1, $src2, sxtw" %}
12912 
12913    ins_encode %{
12914      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12915             as_Register($src2$$reg), ext::sxtw);
12916    %}
12917   ins_pipe(ialu_reg_reg);
12918 %}
12919 
12920 // This pattern is automatically generated from aarch64_ad.m4
12921 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12922 instruct SubExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
12923 %{
12924   match(Set dst (SubL src1 (ConvI2L src2)));
12925   ins_cost(INSN_COST);
12926   format %{ "sub  $dst, $src1, $src2, sxtw" %}
12927 
12928    ins_encode %{
12929      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12930             as_Register($src2$$reg), ext::sxtw);
12931    %}
12932   ins_pipe(ialu_reg_reg);
12933 %}
12934 
12935 // This pattern is automatically generated from aarch64_ad.m4
12936 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12937 instruct AddExtI_sxth(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_16 lshift, immI_16 rshift, rFlagsReg cr)
12938 %{
12939   match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
12940   ins_cost(INSN_COST);
12941   format %{ "add  $dst, $src1, $src2, sxth" %}
12942 
12943    ins_encode %{
12944      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12945             as_Register($src2$$reg), ext::sxth);
12946    %}
12947   ins_pipe(ialu_reg_reg);
12948 %}
12949 
12950 // This pattern is automatically generated from aarch64_ad.m4
12951 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12952 instruct AddExtI_sxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
12953 %{
12954   match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
12955   ins_cost(INSN_COST);
12956   format %{ "add  $dst, $src1, $src2, sxtb" %}
12957 
12958    ins_encode %{
12959      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12960             as_Register($src2$$reg), ext::sxtb);
12961    %}
12962   ins_pipe(ialu_reg_reg);
12963 %}
12964 
12965 // This pattern is automatically generated from aarch64_ad.m4
12966 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12967 instruct AddExtI_uxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
12968 %{
12969   match(Set dst (AddI src1 (URShiftI (LShiftI src2 lshift) rshift)));
12970   ins_cost(INSN_COST);
12971   format %{ "add  $dst, $src1, $src2, uxtb" %}
12972 
12973    ins_encode %{
12974      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12975             as_Register($src2$$reg), ext::uxtb);
12976    %}
12977   ins_pipe(ialu_reg_reg);
12978 %}
12979 
12980 // This pattern is automatically generated from aarch64_ad.m4
12981 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12982 instruct AddExtL_sxth(iRegLNoSp dst, iRegL src1, iRegL src2, immI_48 lshift, immI_48 rshift, rFlagsReg cr)
12983 %{
12984   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12985   ins_cost(INSN_COST);
12986   format %{ "add  $dst, $src1, $src2, sxth" %}
12987 
12988    ins_encode %{
12989      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12990             as_Register($src2$$reg), ext::sxth);
12991    %}
12992   ins_pipe(ialu_reg_reg);
12993 %}
12994 
12995 // This pattern is automatically generated from aarch64_ad.m4
12996 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12997 instruct AddExtL_sxtw(iRegLNoSp dst, iRegL src1, iRegL src2, immI_32 lshift, immI_32 rshift, rFlagsReg cr)
12998 %{
12999   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13000   ins_cost(INSN_COST);
13001   format %{ "add  $dst, $src1, $src2, sxtw" %}
13002 
13003    ins_encode %{
13004      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13005             as_Register($src2$$reg), ext::sxtw);
13006    %}
13007   ins_pipe(ialu_reg_reg);
13008 %}
13009 
13010 // This pattern is automatically generated from aarch64_ad.m4
13011 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13012 instruct AddExtL_sxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
13013 %{
13014   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13015   ins_cost(INSN_COST);
13016   format %{ "add  $dst, $src1, $src2, sxtb" %}
13017 
13018    ins_encode %{
13019      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13020             as_Register($src2$$reg), ext::sxtb);
13021    %}
13022   ins_pipe(ialu_reg_reg);
13023 %}
13024 
13025 // This pattern is automatically generated from aarch64_ad.m4
13026 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13027 instruct AddExtL_uxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
13028 %{
13029   match(Set dst (AddL src1 (URShiftL (LShiftL src2 lshift) rshift)));
13030   ins_cost(INSN_COST);
13031   format %{ "add  $dst, $src1, $src2, uxtb" %}
13032 
13033    ins_encode %{
13034      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13035             as_Register($src2$$reg), ext::uxtb);
13036    %}
13037   ins_pipe(ialu_reg_reg);
13038 %}
13039 
13040 // This pattern is automatically generated from aarch64_ad.m4
13041 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13042 instruct AddExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
13043 %{
13044   match(Set dst (AddI src1 (AndI src2 mask)));
13045   ins_cost(INSN_COST);
13046   format %{ "addw  $dst, $src1, $src2, uxtb" %}
13047 
13048    ins_encode %{
13049      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13050             as_Register($src2$$reg), ext::uxtb);
13051    %}
13052   ins_pipe(ialu_reg_reg);
13053 %}
13054 
13055 // This pattern is automatically generated from aarch64_ad.m4
13056 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13057 instruct AddExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
13058 %{
13059   match(Set dst (AddI src1 (AndI src2 mask)));
13060   ins_cost(INSN_COST);
13061   format %{ "addw  $dst, $src1, $src2, uxth" %}
13062 
13063    ins_encode %{
13064      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13065             as_Register($src2$$reg), ext::uxth);
13066    %}
13067   ins_pipe(ialu_reg_reg);
13068 %}
13069 
13070 // This pattern is automatically generated from aarch64_ad.m4
13071 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13072 instruct AddExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
13073 %{
13074   match(Set dst (AddL src1 (AndL src2 mask)));
13075   ins_cost(INSN_COST);
13076   format %{ "add  $dst, $src1, $src2, uxtb" %}
13077 
13078    ins_encode %{
13079      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13080             as_Register($src2$$reg), ext::uxtb);
13081    %}
13082   ins_pipe(ialu_reg_reg);
13083 %}
13084 
13085 // This pattern is automatically generated from aarch64_ad.m4
13086 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13087 instruct AddExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
13088 %{
13089   match(Set dst (AddL src1 (AndL src2 mask)));
13090   ins_cost(INSN_COST);
13091   format %{ "add  $dst, $src1, $src2, uxth" %}
13092 
13093    ins_encode %{
13094      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13095             as_Register($src2$$reg), ext::uxth);
13096    %}
13097   ins_pipe(ialu_reg_reg);
13098 %}
13099 
13100 // This pattern is automatically generated from aarch64_ad.m4
13101 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13102 instruct AddExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
13103 %{
13104   match(Set dst (AddL src1 (AndL src2 mask)));
13105   ins_cost(INSN_COST);
13106   format %{ "add  $dst, $src1, $src2, uxtw" %}
13107 
13108    ins_encode %{
13109      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13110             as_Register($src2$$reg), ext::uxtw);
13111    %}
13112   ins_pipe(ialu_reg_reg);
13113 %}
13114 
13115 // This pattern is automatically generated from aarch64_ad.m4
13116 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13117 instruct SubExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
13118 %{
13119   match(Set dst (SubI src1 (AndI src2 mask)));
13120   ins_cost(INSN_COST);
13121   format %{ "subw  $dst, $src1, $src2, uxtb" %}
13122 
13123    ins_encode %{
13124      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13125             as_Register($src2$$reg), ext::uxtb);
13126    %}
13127   ins_pipe(ialu_reg_reg);
13128 %}
13129 
13130 // This pattern is automatically generated from aarch64_ad.m4
13131 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13132 instruct SubExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
13133 %{
13134   match(Set dst (SubI src1 (AndI src2 mask)));
13135   ins_cost(INSN_COST);
13136   format %{ "subw  $dst, $src1, $src2, uxth" %}
13137 
13138    ins_encode %{
13139      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13140             as_Register($src2$$reg), ext::uxth);
13141    %}
13142   ins_pipe(ialu_reg_reg);
13143 %}
13144 
13145 // This pattern is automatically generated from aarch64_ad.m4
13146 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13147 instruct SubExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
13148 %{
13149   match(Set dst (SubL src1 (AndL src2 mask)));
13150   ins_cost(INSN_COST);
13151   format %{ "sub  $dst, $src1, $src2, uxtb" %}
13152 
13153    ins_encode %{
13154      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13155             as_Register($src2$$reg), ext::uxtb);
13156    %}
13157   ins_pipe(ialu_reg_reg);
13158 %}
13159 
13160 // This pattern is automatically generated from aarch64_ad.m4
13161 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13162 instruct SubExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
13163 %{
13164   match(Set dst (SubL src1 (AndL src2 mask)));
13165   ins_cost(INSN_COST);
13166   format %{ "sub  $dst, $src1, $src2, uxth" %}
13167 
13168    ins_encode %{
13169      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13170             as_Register($src2$$reg), ext::uxth);
13171    %}
13172   ins_pipe(ialu_reg_reg);
13173 %}
13174 
13175 // This pattern is automatically generated from aarch64_ad.m4
13176 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13177 instruct SubExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
13178 %{
13179   match(Set dst (SubL src1 (AndL src2 mask)));
13180   ins_cost(INSN_COST);
13181   format %{ "sub  $dst, $src1, $src2, uxtw" %}
13182 
13183    ins_encode %{
13184      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13185             as_Register($src2$$reg), ext::uxtw);
13186    %}
13187   ins_pipe(ialu_reg_reg);
13188 %}
13189 
13190 
13191 // This pattern is automatically generated from aarch64_ad.m4
13192 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13193 instruct AddExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13194 %{
13195   match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13196   ins_cost(1.9 * INSN_COST);
13197   format %{ "add  $dst, $src1, $src2, sxtb #lshift2" %}
13198 
13199    ins_encode %{
13200      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13201             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13202    %}
13203   ins_pipe(ialu_reg_reg_shift);
13204 %}
13205 
13206 // This pattern is automatically generated from aarch64_ad.m4
13207 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13208 instruct AddExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13209 %{
13210   match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13211   ins_cost(1.9 * INSN_COST);
13212   format %{ "add  $dst, $src1, $src2, sxth #lshift2" %}
13213 
13214    ins_encode %{
13215      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13216             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13217    %}
13218   ins_pipe(ialu_reg_reg_shift);
13219 %}
13220 
13221 // This pattern is automatically generated from aarch64_ad.m4
13222 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13223 instruct AddExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13224 %{
13225   match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13226   ins_cost(1.9 * INSN_COST);
13227   format %{ "add  $dst, $src1, $src2, sxtw #lshift2" %}
13228 
13229    ins_encode %{
13230      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13231             as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13232    %}
13233   ins_pipe(ialu_reg_reg_shift);
13234 %}
13235 
13236 // This pattern is automatically generated from aarch64_ad.m4
13237 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13238 instruct SubExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13239 %{
13240   match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13241   ins_cost(1.9 * INSN_COST);
13242   format %{ "sub  $dst, $src1, $src2, sxtb #lshift2" %}
13243 
13244    ins_encode %{
13245      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13246             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13247    %}
13248   ins_pipe(ialu_reg_reg_shift);
13249 %}
13250 
13251 // This pattern is automatically generated from aarch64_ad.m4
13252 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13253 instruct SubExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13254 %{
13255   match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13256   ins_cost(1.9 * INSN_COST);
13257   format %{ "sub  $dst, $src1, $src2, sxth #lshift2" %}
13258 
13259    ins_encode %{
13260      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13261             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13262    %}
13263   ins_pipe(ialu_reg_reg_shift);
13264 %}
13265 
13266 // This pattern is automatically generated from aarch64_ad.m4
13267 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13268 instruct SubExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13269 %{
13270   match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13271   ins_cost(1.9 * INSN_COST);
13272   format %{ "sub  $dst, $src1, $src2, sxtw #lshift2" %}
13273 
13274    ins_encode %{
13275      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13276             as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13277    %}
13278   ins_pipe(ialu_reg_reg_shift);
13279 %}
13280 
13281 // This pattern is automatically generated from aarch64_ad.m4
13282 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13283 instruct AddExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13284 %{
13285   match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13286   ins_cost(1.9 * INSN_COST);
13287   format %{ "addw  $dst, $src1, $src2, sxtb #lshift2" %}
13288 
13289    ins_encode %{
13290      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13291             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13292    %}
13293   ins_pipe(ialu_reg_reg_shift);
13294 %}
13295 
13296 // This pattern is automatically generated from aarch64_ad.m4
13297 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13298 instruct AddExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13299 %{
13300   match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13301   ins_cost(1.9 * INSN_COST);
13302   format %{ "addw  $dst, $src1, $src2, sxth #lshift2" %}
13303 
13304    ins_encode %{
13305      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13306             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13307    %}
13308   ins_pipe(ialu_reg_reg_shift);
13309 %}
13310 
13311 // This pattern is automatically generated from aarch64_ad.m4
13312 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13313 instruct SubExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13314 %{
13315   match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13316   ins_cost(1.9 * INSN_COST);
13317   format %{ "subw  $dst, $src1, $src2, sxtb #lshift2" %}
13318 
13319    ins_encode %{
13320      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13321             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13322    %}
13323   ins_pipe(ialu_reg_reg_shift);
13324 %}
13325 
13326 // This pattern is automatically generated from aarch64_ad.m4
13327 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13328 instruct SubExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13329 %{
13330   match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13331   ins_cost(1.9 * INSN_COST);
13332   format %{ "subw  $dst, $src1, $src2, sxth #lshift2" %}
13333 
13334    ins_encode %{
13335      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13336             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13337    %}
13338   ins_pipe(ialu_reg_reg_shift);
13339 %}
13340 
13341 // This pattern is automatically generated from aarch64_ad.m4
13342 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13343 instruct AddExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13344 %{
13345   match(Set dst (AddL src1 (LShiftL (ConvI2L src2) lshift)));
13346   ins_cost(1.9 * INSN_COST);
13347   format %{ "add  $dst, $src1, $src2, sxtw #lshift" %}
13348 
13349    ins_encode %{
13350      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13351             as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13352    %}
13353   ins_pipe(ialu_reg_reg_shift);
13354 %}
13355 
13356 // This pattern is automatically generated from aarch64_ad.m4
13357 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13358 instruct SubExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13359 %{
13360   match(Set dst (SubL src1 (LShiftL (ConvI2L src2) lshift)));
13361   ins_cost(1.9 * INSN_COST);
13362   format %{ "sub  $dst, $src1, $src2, sxtw #lshift" %}
13363 
13364    ins_encode %{
13365      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13366             as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13367    %}
13368   ins_pipe(ialu_reg_reg_shift);
13369 %}
13370 
13371 // This pattern is automatically generated from aarch64_ad.m4
13372 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13373 instruct AddExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13374 %{
13375   match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13376   ins_cost(1.9 * INSN_COST);
13377   format %{ "add  $dst, $src1, $src2, uxtb #lshift" %}
13378 
13379    ins_encode %{
13380      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13381             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13382    %}
13383   ins_pipe(ialu_reg_reg_shift);
13384 %}
13385 
13386 // This pattern is automatically generated from aarch64_ad.m4
13387 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13388 instruct AddExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13389 %{
13390   match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13391   ins_cost(1.9 * INSN_COST);
13392   format %{ "add  $dst, $src1, $src2, uxth #lshift" %}
13393 
13394    ins_encode %{
13395      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13396             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13397    %}
13398   ins_pipe(ialu_reg_reg_shift);
13399 %}
13400 
13401 // This pattern is automatically generated from aarch64_ad.m4
13402 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13403 instruct AddExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13404 %{
13405   match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13406   ins_cost(1.9 * INSN_COST);
13407   format %{ "add  $dst, $src1, $src2, uxtw #lshift" %}
13408 
13409    ins_encode %{
13410      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13411             as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13412    %}
13413   ins_pipe(ialu_reg_reg_shift);
13414 %}
13415 
13416 // This pattern is automatically generated from aarch64_ad.m4
13417 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13418 instruct SubExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13419 %{
13420   match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13421   ins_cost(1.9 * INSN_COST);
13422   format %{ "sub  $dst, $src1, $src2, uxtb #lshift" %}
13423 
13424    ins_encode %{
13425      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13426             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13427    %}
13428   ins_pipe(ialu_reg_reg_shift);
13429 %}
13430 
13431 // This pattern is automatically generated from aarch64_ad.m4
13432 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13433 instruct SubExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13434 %{
13435   match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13436   ins_cost(1.9 * INSN_COST);
13437   format %{ "sub  $dst, $src1, $src2, uxth #lshift" %}
13438 
13439    ins_encode %{
13440      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13441             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13442    %}
13443   ins_pipe(ialu_reg_reg_shift);
13444 %}
13445 
13446 // This pattern is automatically generated from aarch64_ad.m4
13447 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13448 instruct SubExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13449 %{
13450   match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13451   ins_cost(1.9 * INSN_COST);
13452   format %{ "sub  $dst, $src1, $src2, uxtw #lshift" %}
13453 
13454    ins_encode %{
13455      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13456             as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13457    %}
13458   ins_pipe(ialu_reg_reg_shift);
13459 %}
13460 
13461 // This pattern is automatically generated from aarch64_ad.m4
13462 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13463 instruct AddExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13464 %{
13465   match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13466   ins_cost(1.9 * INSN_COST);
13467   format %{ "addw  $dst, $src1, $src2, uxtb #lshift" %}
13468 
13469    ins_encode %{
13470      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13471             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13472    %}
13473   ins_pipe(ialu_reg_reg_shift);
13474 %}
13475 
13476 // This pattern is automatically generated from aarch64_ad.m4
13477 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13478 instruct AddExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13479 %{
13480   match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13481   ins_cost(1.9 * INSN_COST);
13482   format %{ "addw  $dst, $src1, $src2, uxth #lshift" %}
13483 
13484    ins_encode %{
13485      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13486             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13487    %}
13488   ins_pipe(ialu_reg_reg_shift);
13489 %}
13490 
13491 // This pattern is automatically generated from aarch64_ad.m4
13492 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13493 instruct SubExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13494 %{
13495   match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13496   ins_cost(1.9 * INSN_COST);
13497   format %{ "subw  $dst, $src1, $src2, uxtb #lshift" %}
13498 
13499    ins_encode %{
13500      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13501             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13502    %}
13503   ins_pipe(ialu_reg_reg_shift);
13504 %}
13505 
13506 // This pattern is automatically generated from aarch64_ad.m4
13507 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13508 instruct SubExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13509 %{
13510   match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13511   ins_cost(1.9 * INSN_COST);
13512   format %{ "subw  $dst, $src1, $src2, uxth #lshift" %}
13513 
13514    ins_encode %{
13515      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13516             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13517    %}
13518   ins_pipe(ialu_reg_reg_shift);
13519 %}
13520 
13521 // This pattern is automatically generated from aarch64_ad.m4
13522 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13523 instruct cmovI_reg_reg_lt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
13524 %{
13525   effect(DEF dst, USE src1, USE src2, USE cr);
13526   ins_cost(INSN_COST * 2);
13527   format %{ "cselw $dst, $src1, $src2 lt\t"  %}
13528 
13529   ins_encode %{
13530     __ cselw($dst$$Register,
13531              $src1$$Register,
13532              $src2$$Register,
13533              Assembler::LT);
13534   %}
13535   ins_pipe(icond_reg_reg);
13536 %}
13537 
13538 // This pattern is automatically generated from aarch64_ad.m4
13539 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13540 instruct cmovI_reg_reg_gt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
13541 %{
13542   effect(DEF dst, USE src1, USE src2, USE cr);
13543   ins_cost(INSN_COST * 2);
13544   format %{ "cselw $dst, $src1, $src2 gt\t"  %}
13545 
13546   ins_encode %{
13547     __ cselw($dst$$Register,
13548              $src1$$Register,
13549              $src2$$Register,
13550              Assembler::GT);
13551   %}
13552   ins_pipe(icond_reg_reg);
13553 %}
13554 
13555 // This pattern is automatically generated from aarch64_ad.m4
13556 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13557 instruct cmovI_reg_imm0_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13558 %{
13559   effect(DEF dst, USE src1, USE cr);
13560   ins_cost(INSN_COST * 2);
13561   format %{ "cselw $dst, $src1, zr lt\t"  %}
13562 
13563   ins_encode %{
13564     __ cselw($dst$$Register,
13565              $src1$$Register,
13566              zr,
13567              Assembler::LT);
13568   %}
13569   ins_pipe(icond_reg);
13570 %}
13571 
13572 // This pattern is automatically generated from aarch64_ad.m4
13573 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13574 instruct cmovI_reg_imm0_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13575 %{
13576   effect(DEF dst, USE src1, USE cr);
13577   ins_cost(INSN_COST * 2);
13578   format %{ "cselw $dst, $src1, zr gt\t"  %}
13579 
13580   ins_encode %{
13581     __ cselw($dst$$Register,
13582              $src1$$Register,
13583              zr,
13584              Assembler::GT);
13585   %}
13586   ins_pipe(icond_reg);
13587 %}
13588 
13589 // This pattern is automatically generated from aarch64_ad.m4
13590 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13591 instruct cmovI_reg_imm1_le(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13592 %{
13593   effect(DEF dst, USE src1, USE cr);
13594   ins_cost(INSN_COST * 2);
13595   format %{ "csincw $dst, $src1, zr le\t"  %}
13596 
13597   ins_encode %{
13598     __ csincw($dst$$Register,
13599              $src1$$Register,
13600              zr,
13601              Assembler::LE);
13602   %}
13603   ins_pipe(icond_reg);
13604 %}
13605 
13606 // This pattern is automatically generated from aarch64_ad.m4
13607 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13608 instruct cmovI_reg_imm1_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13609 %{
13610   effect(DEF dst, USE src1, USE cr);
13611   ins_cost(INSN_COST * 2);
13612   format %{ "csincw $dst, $src1, zr gt\t"  %}
13613 
13614   ins_encode %{
13615     __ csincw($dst$$Register,
13616              $src1$$Register,
13617              zr,
13618              Assembler::GT);
13619   %}
13620   ins_pipe(icond_reg);
13621 %}
13622 
13623 // This pattern is automatically generated from aarch64_ad.m4
13624 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13625 instruct cmovI_reg_immM1_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13626 %{
13627   effect(DEF dst, USE src1, USE cr);
13628   ins_cost(INSN_COST * 2);
13629   format %{ "csinvw $dst, $src1, zr lt\t"  %}
13630 
13631   ins_encode %{
13632     __ csinvw($dst$$Register,
13633              $src1$$Register,
13634              zr,
13635              Assembler::LT);
13636   %}
13637   ins_pipe(icond_reg);
13638 %}
13639 
13640 // This pattern is automatically generated from aarch64_ad.m4
13641 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13642 instruct cmovI_reg_immM1_ge(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13643 %{
13644   effect(DEF dst, USE src1, USE cr);
13645   ins_cost(INSN_COST * 2);
13646   format %{ "csinvw $dst, $src1, zr ge\t"  %}
13647 
13648   ins_encode %{
13649     __ csinvw($dst$$Register,
13650              $src1$$Register,
13651              zr,
13652              Assembler::GE);
13653   %}
13654   ins_pipe(icond_reg);
13655 %}
13656 
13657 // This pattern is automatically generated from aarch64_ad.m4
13658 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13659 instruct minI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
13660 %{
13661   match(Set dst (MinI src imm));
13662   ins_cost(INSN_COST * 3);
13663   expand %{
13664     rFlagsReg cr;
13665     compI_reg_imm0(cr, src);
13666     cmovI_reg_imm0_lt(dst, src, cr);
13667   %}
13668 %}
13669 
13670 // This pattern is automatically generated from aarch64_ad.m4
13671 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13672 instruct minI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
13673 %{
13674   match(Set dst (MinI imm src));
13675   ins_cost(INSN_COST * 3);
13676   expand %{
13677     rFlagsReg cr;
13678     compI_reg_imm0(cr, src);
13679     cmovI_reg_imm0_lt(dst, src, cr);
13680   %}
13681 %}
13682 
13683 // This pattern is automatically generated from aarch64_ad.m4
13684 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13685 instruct minI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
13686 %{
13687   match(Set dst (MinI src imm));
13688   ins_cost(INSN_COST * 3);
13689   expand %{
13690     rFlagsReg cr;
13691     compI_reg_imm0(cr, src);
13692     cmovI_reg_imm1_le(dst, src, cr);
13693   %}
13694 %}
13695 
13696 // This pattern is automatically generated from aarch64_ad.m4
13697 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13698 instruct minI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
13699 %{
13700   match(Set dst (MinI imm src));
13701   ins_cost(INSN_COST * 3);
13702   expand %{
13703     rFlagsReg cr;
13704     compI_reg_imm0(cr, src);
13705     cmovI_reg_imm1_le(dst, src, cr);
13706   %}
13707 %}
13708 
13709 // This pattern is automatically generated from aarch64_ad.m4
13710 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13711 instruct minI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
13712 %{
13713   match(Set dst (MinI src imm));
13714   ins_cost(INSN_COST * 3);
13715   expand %{
13716     rFlagsReg cr;
13717     compI_reg_imm0(cr, src);
13718     cmovI_reg_immM1_lt(dst, src, cr);
13719   %}
13720 %}
13721 
13722 // This pattern is automatically generated from aarch64_ad.m4
13723 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13724 instruct minI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
13725 %{
13726   match(Set dst (MinI imm src));
13727   ins_cost(INSN_COST * 3);
13728   expand %{
13729     rFlagsReg cr;
13730     compI_reg_imm0(cr, src);
13731     cmovI_reg_immM1_lt(dst, src, cr);
13732   %}
13733 %}
13734 
13735 // This pattern is automatically generated from aarch64_ad.m4
13736 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13737 instruct maxI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
13738 %{
13739   match(Set dst (MaxI src imm));
13740   ins_cost(INSN_COST * 3);
13741   expand %{
13742     rFlagsReg cr;
13743     compI_reg_imm0(cr, src);
13744     cmovI_reg_imm0_gt(dst, src, cr);
13745   %}
13746 %}
13747 
13748 // This pattern is automatically generated from aarch64_ad.m4
13749 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13750 instruct maxI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
13751 %{
13752   match(Set dst (MaxI imm src));
13753   ins_cost(INSN_COST * 3);
13754   expand %{
13755     rFlagsReg cr;
13756     compI_reg_imm0(cr, src);
13757     cmovI_reg_imm0_gt(dst, src, cr);
13758   %}
13759 %}
13760 
13761 // This pattern is automatically generated from aarch64_ad.m4
13762 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13763 instruct maxI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
13764 %{
13765   match(Set dst (MaxI src imm));
13766   ins_cost(INSN_COST * 3);
13767   expand %{
13768     rFlagsReg cr;
13769     compI_reg_imm0(cr, src);
13770     cmovI_reg_imm1_gt(dst, src, cr);
13771   %}
13772 %}
13773 
13774 // This pattern is automatically generated from aarch64_ad.m4
13775 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13776 instruct maxI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
13777 %{
13778   match(Set dst (MaxI imm src));
13779   ins_cost(INSN_COST * 3);
13780   expand %{
13781     rFlagsReg cr;
13782     compI_reg_imm0(cr, src);
13783     cmovI_reg_imm1_gt(dst, src, cr);
13784   %}
13785 %}
13786 
13787 // This pattern is automatically generated from aarch64_ad.m4
13788 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13789 instruct maxI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
13790 %{
13791   match(Set dst (MaxI src imm));
13792   ins_cost(INSN_COST * 3);
13793   expand %{
13794     rFlagsReg cr;
13795     compI_reg_imm0(cr, src);
13796     cmovI_reg_immM1_ge(dst, src, cr);
13797   %}
13798 %}
13799 
13800 // This pattern is automatically generated from aarch64_ad.m4
13801 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13802 instruct maxI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
13803 %{
13804   match(Set dst (MaxI imm src));
13805   ins_cost(INSN_COST * 3);
13806   expand %{
13807     rFlagsReg cr;
13808     compI_reg_imm0(cr, src);
13809     cmovI_reg_immM1_ge(dst, src, cr);
13810   %}
13811 %}
13812 
13813 // This pattern is automatically generated from aarch64_ad.m4
13814 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13815 instruct bits_reverse_I(iRegINoSp dst, iRegIorL2I src)
13816 %{
13817   match(Set dst (ReverseI src));
13818   ins_cost(INSN_COST);
13819   format %{ "rbitw  $dst, $src" %}
13820   ins_encode %{
13821     __ rbitw($dst$$Register, $src$$Register);
13822   %}
13823   ins_pipe(ialu_reg);
13824 %}
13825 
13826 // This pattern is automatically generated from aarch64_ad.m4
13827 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13828 instruct bits_reverse_L(iRegLNoSp dst, iRegL src)
13829 %{
13830   match(Set dst (ReverseL src));
13831   ins_cost(INSN_COST);
13832   format %{ "rbit  $dst, $src" %}
13833   ins_encode %{
13834     __ rbit($dst$$Register, $src$$Register);
13835   %}
13836   ins_pipe(ialu_reg);
13837 %}
13838 
13839 
13840 // END This section of the file is automatically generated. Do not edit --------------
13841 
13842 
13843 // ============================================================================
13844 // Floating Point Arithmetic Instructions
13845 
13846 instruct addHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13847   match(Set dst (AddHF src1 src2));
13848   format %{ "faddh $dst, $src1, $src2" %}
13849   ins_encode %{
13850     __ faddh($dst$$FloatRegister,
13851              $src1$$FloatRegister,
13852              $src2$$FloatRegister);
13853   %}
13854   ins_pipe(fp_dop_reg_reg_s);
13855 %}
13856 
13857 instruct addF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13858   match(Set dst (AddF src1 src2));
13859 
13860   ins_cost(INSN_COST * 5);
13861   format %{ "fadds   $dst, $src1, $src2" %}
13862 
13863   ins_encode %{
13864     __ fadds(as_FloatRegister($dst$$reg),
13865              as_FloatRegister($src1$$reg),
13866              as_FloatRegister($src2$$reg));
13867   %}
13868 
13869   ins_pipe(fp_dop_reg_reg_s);
13870 %}
13871 
13872 instruct addD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13873   match(Set dst (AddD src1 src2));
13874 
13875   ins_cost(INSN_COST * 5);
13876   format %{ "faddd   $dst, $src1, $src2" %}
13877 
13878   ins_encode %{
13879     __ faddd(as_FloatRegister($dst$$reg),
13880              as_FloatRegister($src1$$reg),
13881              as_FloatRegister($src2$$reg));
13882   %}
13883 
13884   ins_pipe(fp_dop_reg_reg_d);
13885 %}
13886 
13887 instruct subHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13888   match(Set dst (SubHF src1 src2));
13889   format %{ "fsubh $dst, $src1, $src2" %}
13890   ins_encode %{
13891     __ fsubh($dst$$FloatRegister,
13892              $src1$$FloatRegister,
13893              $src2$$FloatRegister);
13894   %}
13895   ins_pipe(fp_dop_reg_reg_s);
13896 %}
13897 
13898 instruct subF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13899   match(Set dst (SubF src1 src2));
13900 
13901   ins_cost(INSN_COST * 5);
13902   format %{ "fsubs   $dst, $src1, $src2" %}
13903 
13904   ins_encode %{
13905     __ fsubs(as_FloatRegister($dst$$reg),
13906              as_FloatRegister($src1$$reg),
13907              as_FloatRegister($src2$$reg));
13908   %}
13909 
13910   ins_pipe(fp_dop_reg_reg_s);
13911 %}
13912 
13913 instruct subD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13914   match(Set dst (SubD src1 src2));
13915 
13916   ins_cost(INSN_COST * 5);
13917   format %{ "fsubd   $dst, $src1, $src2" %}
13918 
13919   ins_encode %{
13920     __ fsubd(as_FloatRegister($dst$$reg),
13921              as_FloatRegister($src1$$reg),
13922              as_FloatRegister($src2$$reg));
13923   %}
13924 
13925   ins_pipe(fp_dop_reg_reg_d);
13926 %}
13927 
13928 instruct mulHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13929   match(Set dst (MulHF src1 src2));
13930   format %{ "fmulh $dst, $src1, $src2" %}
13931   ins_encode %{
13932     __ fmulh($dst$$FloatRegister,
13933              $src1$$FloatRegister,
13934              $src2$$FloatRegister);
13935   %}
13936   ins_pipe(fp_dop_reg_reg_s);
13937 %}
13938 
13939 instruct mulF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13940   match(Set dst (MulF src1 src2));
13941 
13942   ins_cost(INSN_COST * 6);
13943   format %{ "fmuls   $dst, $src1, $src2" %}
13944 
13945   ins_encode %{
13946     __ fmuls(as_FloatRegister($dst$$reg),
13947              as_FloatRegister($src1$$reg),
13948              as_FloatRegister($src2$$reg));
13949   %}
13950 
13951   ins_pipe(fp_dop_reg_reg_s);
13952 %}
13953 
13954 instruct mulD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13955   match(Set dst (MulD src1 src2));
13956 
13957   ins_cost(INSN_COST * 6);
13958   format %{ "fmuld   $dst, $src1, $src2" %}
13959 
13960   ins_encode %{
13961     __ fmuld(as_FloatRegister($dst$$reg),
13962              as_FloatRegister($src1$$reg),
13963              as_FloatRegister($src2$$reg));
13964   %}
13965 
13966   ins_pipe(fp_dop_reg_reg_d);
13967 %}
13968 
13969 // src1 * src2 + src3 (half-precision float)
13970 instruct maddHF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13971   match(Set dst (FmaHF src3 (Binary src1 src2)));
13972   format %{ "fmaddh $dst, $src1, $src2, $src3" %}
13973   ins_encode %{
13974     assert(UseFMA, "Needs FMA instructions support.");
13975     __ fmaddh($dst$$FloatRegister,
13976               $src1$$FloatRegister,
13977               $src2$$FloatRegister,
13978               $src3$$FloatRegister);
13979   %}
13980   ins_pipe(pipe_class_default);
13981 %}
13982 
13983 // src1 * src2 + src3
13984 instruct maddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13985   match(Set dst (FmaF src3 (Binary src1 src2)));
13986 
13987   format %{ "fmadds   $dst, $src1, $src2, $src3" %}
13988 
13989   ins_encode %{
13990     assert(UseFMA, "Needs FMA instructions support.");
13991     __ fmadds(as_FloatRegister($dst$$reg),
13992              as_FloatRegister($src1$$reg),
13993              as_FloatRegister($src2$$reg),
13994              as_FloatRegister($src3$$reg));
13995   %}
13996 
13997   ins_pipe(pipe_class_default);
13998 %}
13999 
14000 // src1 * src2 + src3
14001 instruct maddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
14002   match(Set dst (FmaD src3 (Binary src1 src2)));
14003 
14004   format %{ "fmaddd   $dst, $src1, $src2, $src3" %}
14005 
14006   ins_encode %{
14007     assert(UseFMA, "Needs FMA instructions support.");
14008     __ fmaddd(as_FloatRegister($dst$$reg),
14009              as_FloatRegister($src1$$reg),
14010              as_FloatRegister($src2$$reg),
14011              as_FloatRegister($src3$$reg));
14012   %}
14013 
14014   ins_pipe(pipe_class_default);
14015 %}
14016 
14017 // src1 * (-src2) + src3
14018 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
14019 instruct msubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
14020   match(Set dst (FmaF src3 (Binary src1 (NegF src2))));
14021 
14022   format %{ "fmsubs   $dst, $src1, $src2, $src3" %}
14023 
14024   ins_encode %{
14025     assert(UseFMA, "Needs FMA instructions support.");
14026     __ fmsubs(as_FloatRegister($dst$$reg),
14027               as_FloatRegister($src1$$reg),
14028               as_FloatRegister($src2$$reg),
14029               as_FloatRegister($src3$$reg));
14030   %}
14031 
14032   ins_pipe(pipe_class_default);
14033 %}
14034 
14035 // src1 * (-src2) + src3
14036 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
14037 instruct msubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
14038   match(Set dst (FmaD src3 (Binary src1 (NegD src2))));
14039 
14040   format %{ "fmsubd   $dst, $src1, $src2, $src3" %}
14041 
14042   ins_encode %{
14043     assert(UseFMA, "Needs FMA instructions support.");
14044     __ fmsubd(as_FloatRegister($dst$$reg),
14045               as_FloatRegister($src1$$reg),
14046               as_FloatRegister($src2$$reg),
14047               as_FloatRegister($src3$$reg));
14048   %}
14049 
14050   ins_pipe(pipe_class_default);
14051 %}
14052 
14053 // src1 * (-src2) - src3
14054 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
14055 instruct mnaddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
14056   match(Set dst (FmaF (NegF src3) (Binary src1 (NegF src2))));
14057 
14058   format %{ "fnmadds  $dst, $src1, $src2, $src3" %}
14059 
14060   ins_encode %{
14061     assert(UseFMA, "Needs FMA instructions support.");
14062     __ fnmadds(as_FloatRegister($dst$$reg),
14063                as_FloatRegister($src1$$reg),
14064                as_FloatRegister($src2$$reg),
14065                as_FloatRegister($src3$$reg));
14066   %}
14067 
14068   ins_pipe(pipe_class_default);
14069 %}
14070 
14071 // src1 * (-src2) - src3
14072 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
14073 instruct mnaddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
14074   match(Set dst (FmaD (NegD src3) (Binary src1 (NegD src2))));
14075 
14076   format %{ "fnmaddd   $dst, $src1, $src2, $src3" %}
14077 
14078   ins_encode %{
14079     assert(UseFMA, "Needs FMA instructions support.");
14080     __ fnmaddd(as_FloatRegister($dst$$reg),
14081                as_FloatRegister($src1$$reg),
14082                as_FloatRegister($src2$$reg),
14083                as_FloatRegister($src3$$reg));
14084   %}
14085 
14086   ins_pipe(pipe_class_default);
14087 %}
14088 
14089 // src1 * src2 - src3
14090 instruct mnsubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3, immF0 zero) %{
14091   match(Set dst (FmaF (NegF src3) (Binary src1 src2)));
14092 
14093   format %{ "fnmsubs  $dst, $src1, $src2, $src3" %}
14094 
14095   ins_encode %{
14096     assert(UseFMA, "Needs FMA instructions support.");
14097     __ fnmsubs(as_FloatRegister($dst$$reg),
14098                as_FloatRegister($src1$$reg),
14099                as_FloatRegister($src2$$reg),
14100                as_FloatRegister($src3$$reg));
14101   %}
14102 
14103   ins_pipe(pipe_class_default);
14104 %}
14105 
14106 // src1 * src2 - src3
14107 instruct mnsubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3, immD0 zero) %{
14108   match(Set dst (FmaD (NegD src3) (Binary src1 src2)));
14109 
14110   format %{ "fnmsubd   $dst, $src1, $src2, $src3" %}
14111 
14112   ins_encode %{
14113     assert(UseFMA, "Needs FMA instructions support.");
14114     // n.b. insn name should be fnmsubd
14115     __ fnmsub(as_FloatRegister($dst$$reg),
14116               as_FloatRegister($src1$$reg),
14117               as_FloatRegister($src2$$reg),
14118               as_FloatRegister($src3$$reg));
14119   %}
14120 
14121   ins_pipe(pipe_class_default);
14122 %}
14123 
14124 // Math.max(HH)H (half-precision float)
14125 instruct maxHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14126   match(Set dst (MaxHF src1 src2));
14127   format %{ "fmaxh $dst, $src1, $src2" %}
14128   ins_encode %{
14129     __ fmaxh($dst$$FloatRegister,
14130              $src1$$FloatRegister,
14131              $src2$$FloatRegister);
14132   %}
14133   ins_pipe(fp_dop_reg_reg_s);
14134 %}
14135 
14136 // Math.min(HH)H (half-precision float)
14137 instruct minHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14138   match(Set dst (MinHF src1 src2));
14139   format %{ "fminh $dst, $src1, $src2" %}
14140   ins_encode %{
14141     __ fminh($dst$$FloatRegister,
14142              $src1$$FloatRegister,
14143              $src2$$FloatRegister);
14144   %}
14145   ins_pipe(fp_dop_reg_reg_s);
14146 %}
14147 
14148 // Math.max(FF)F
14149 instruct maxF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14150   match(Set dst (MaxF src1 src2));
14151 
14152   format %{ "fmaxs   $dst, $src1, $src2" %}
14153   ins_encode %{
14154     __ fmaxs(as_FloatRegister($dst$$reg),
14155              as_FloatRegister($src1$$reg),
14156              as_FloatRegister($src2$$reg));
14157   %}
14158 
14159   ins_pipe(fp_dop_reg_reg_s);
14160 %}
14161 
14162 // Math.min(FF)F
14163 instruct minF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14164   match(Set dst (MinF src1 src2));
14165 
14166   format %{ "fmins   $dst, $src1, $src2" %}
14167   ins_encode %{
14168     __ fmins(as_FloatRegister($dst$$reg),
14169              as_FloatRegister($src1$$reg),
14170              as_FloatRegister($src2$$reg));
14171   %}
14172 
14173   ins_pipe(fp_dop_reg_reg_s);
14174 %}
14175 
14176 // Math.max(DD)D
14177 instruct maxD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14178   match(Set dst (MaxD src1 src2));
14179 
14180   format %{ "fmaxd   $dst, $src1, $src2" %}
14181   ins_encode %{
14182     __ fmaxd(as_FloatRegister($dst$$reg),
14183              as_FloatRegister($src1$$reg),
14184              as_FloatRegister($src2$$reg));
14185   %}
14186 
14187   ins_pipe(fp_dop_reg_reg_d);
14188 %}
14189 
14190 // Math.min(DD)D
14191 instruct minD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14192   match(Set dst (MinD src1 src2));
14193 
14194   format %{ "fmind   $dst, $src1, $src2" %}
14195   ins_encode %{
14196     __ fmind(as_FloatRegister($dst$$reg),
14197              as_FloatRegister($src1$$reg),
14198              as_FloatRegister($src2$$reg));
14199   %}
14200 
14201   ins_pipe(fp_dop_reg_reg_d);
14202 %}
14203 
14204 instruct divHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14205   match(Set dst (DivHF src1  src2));
14206   format %{ "fdivh $dst, $src1, $src2" %}
14207   ins_encode %{
14208     __ fdivh($dst$$FloatRegister,
14209              $src1$$FloatRegister,
14210              $src2$$FloatRegister);
14211   %}
14212   ins_pipe(fp_div_s);
14213 %}
14214 
14215 instruct divF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14216   match(Set dst (DivF src1  src2));
14217 
14218   ins_cost(INSN_COST * 18);
14219   format %{ "fdivs   $dst, $src1, $src2" %}
14220 
14221   ins_encode %{
14222     __ fdivs(as_FloatRegister($dst$$reg),
14223              as_FloatRegister($src1$$reg),
14224              as_FloatRegister($src2$$reg));
14225   %}
14226 
14227   ins_pipe(fp_div_s);
14228 %}
14229 
14230 instruct divD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14231   match(Set dst (DivD src1  src2));
14232 
14233   ins_cost(INSN_COST * 32);
14234   format %{ "fdivd   $dst, $src1, $src2" %}
14235 
14236   ins_encode %{
14237     __ fdivd(as_FloatRegister($dst$$reg),
14238              as_FloatRegister($src1$$reg),
14239              as_FloatRegister($src2$$reg));
14240   %}
14241 
14242   ins_pipe(fp_div_d);
14243 %}
14244 
14245 instruct negF_reg_reg(vRegF dst, vRegF src) %{
14246   match(Set dst (NegF src));
14247 
14248   ins_cost(INSN_COST * 3);
14249   format %{ "fneg   $dst, $src" %}
14250 
14251   ins_encode %{
14252     __ fnegs(as_FloatRegister($dst$$reg),
14253              as_FloatRegister($src$$reg));
14254   %}
14255 
14256   ins_pipe(fp_uop_s);
14257 %}
14258 
14259 instruct negD_reg_reg(vRegD dst, vRegD src) %{
14260   match(Set dst (NegD src));
14261 
14262   ins_cost(INSN_COST * 3);
14263   format %{ "fnegd   $dst, $src" %}
14264 
14265   ins_encode %{
14266     __ fnegd(as_FloatRegister($dst$$reg),
14267              as_FloatRegister($src$$reg));
14268   %}
14269 
14270   ins_pipe(fp_uop_d);
14271 %}
14272 
14273 instruct absI_reg(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
14274 %{
14275   match(Set dst (AbsI src));
14276 
14277   effect(KILL cr);
14278   ins_cost(INSN_COST * 2);
14279   format %{ "cmpw  $src, zr\n\t"
14280             "cnegw $dst, $src, Assembler::LT\t# int abs"
14281   %}
14282 
14283   ins_encode %{
14284     __ cmpw(as_Register($src$$reg), zr);
14285     __ cnegw(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14286   %}
14287   ins_pipe(pipe_class_default);
14288 %}
14289 
14290 instruct absL_reg(iRegLNoSp dst, iRegL src, rFlagsReg cr)
14291 %{
14292   match(Set dst (AbsL src));
14293 
14294   effect(KILL cr);
14295   ins_cost(INSN_COST * 2);
14296   format %{ "cmp  $src, zr\n\t"
14297             "cneg $dst, $src, Assembler::LT\t# long abs"
14298   %}
14299 
14300   ins_encode %{
14301     __ cmp(as_Register($src$$reg), zr);
14302     __ cneg(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14303   %}
14304   ins_pipe(pipe_class_default);
14305 %}
14306 
14307 instruct absF_reg(vRegF dst, vRegF src) %{
14308   match(Set dst (AbsF src));
14309 
14310   ins_cost(INSN_COST * 3);
14311   format %{ "fabss   $dst, $src" %}
14312   ins_encode %{
14313     __ fabss(as_FloatRegister($dst$$reg),
14314              as_FloatRegister($src$$reg));
14315   %}
14316 
14317   ins_pipe(fp_uop_s);
14318 %}
14319 
14320 instruct absD_reg(vRegD dst, vRegD src) %{
14321   match(Set dst (AbsD src));
14322 
14323   ins_cost(INSN_COST * 3);
14324   format %{ "fabsd   $dst, $src" %}
14325   ins_encode %{
14326     __ fabsd(as_FloatRegister($dst$$reg),
14327              as_FloatRegister($src$$reg));
14328   %}
14329 
14330   ins_pipe(fp_uop_d);
14331 %}
14332 
14333 instruct absdF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14334   match(Set dst (AbsF (SubF src1 src2)));
14335 
14336   ins_cost(INSN_COST * 3);
14337   format %{ "fabds   $dst, $src1, $src2" %}
14338   ins_encode %{
14339     __ fabds(as_FloatRegister($dst$$reg),
14340              as_FloatRegister($src1$$reg),
14341              as_FloatRegister($src2$$reg));
14342   %}
14343 
14344   ins_pipe(fp_uop_s);
14345 %}
14346 
14347 instruct absdD_reg(vRegD dst, vRegD src1, vRegD src2) %{
14348   match(Set dst (AbsD (SubD src1 src2)));
14349 
14350   ins_cost(INSN_COST * 3);
14351   format %{ "fabdd   $dst, $src1, $src2" %}
14352   ins_encode %{
14353     __ fabdd(as_FloatRegister($dst$$reg),
14354              as_FloatRegister($src1$$reg),
14355              as_FloatRegister($src2$$reg));
14356   %}
14357 
14358   ins_pipe(fp_uop_d);
14359 %}
14360 
14361 instruct sqrtD_reg(vRegD dst, vRegD src) %{
14362   match(Set dst (SqrtD src));
14363 
14364   ins_cost(INSN_COST * 50);
14365   format %{ "fsqrtd  $dst, $src" %}
14366   ins_encode %{
14367     __ fsqrtd(as_FloatRegister($dst$$reg),
14368              as_FloatRegister($src$$reg));
14369   %}
14370 
14371   ins_pipe(fp_div_s);
14372 %}
14373 
14374 instruct sqrtF_reg(vRegF dst, vRegF src) %{
14375   match(Set dst (SqrtF src));
14376 
14377   ins_cost(INSN_COST * 50);
14378   format %{ "fsqrts  $dst, $src" %}
14379   ins_encode %{
14380     __ fsqrts(as_FloatRegister($dst$$reg),
14381              as_FloatRegister($src$$reg));
14382   %}
14383 
14384   ins_pipe(fp_div_d);
14385 %}
14386 
14387 instruct sqrtHF_reg(vRegF dst, vRegF src) %{
14388   match(Set dst (SqrtHF src));
14389   format %{ "fsqrth $dst, $src" %}
14390   ins_encode %{
14391     __ fsqrth($dst$$FloatRegister,
14392               $src$$FloatRegister);
14393   %}
14394   ins_pipe(fp_div_s);
14395 %}
14396 
14397 // Math.rint, floor, ceil
14398 instruct roundD_reg(vRegD dst, vRegD src, immI rmode) %{
14399   match(Set dst (RoundDoubleMode src rmode));
14400   format %{ "frint  $dst, $src, $rmode" %}
14401   ins_encode %{
14402     switch ($rmode$$constant) {
14403       case RoundDoubleModeNode::rmode_rint:
14404         __ frintnd(as_FloatRegister($dst$$reg),
14405                    as_FloatRegister($src$$reg));
14406         break;
14407       case RoundDoubleModeNode::rmode_floor:
14408         __ frintmd(as_FloatRegister($dst$$reg),
14409                    as_FloatRegister($src$$reg));
14410         break;
14411       case RoundDoubleModeNode::rmode_ceil:
14412         __ frintpd(as_FloatRegister($dst$$reg),
14413                    as_FloatRegister($src$$reg));
14414         break;
14415     }
14416   %}
14417   ins_pipe(fp_uop_d);
14418 %}
14419 
14420 instruct copySignD_reg(vRegD dst, vRegD src1, vRegD src2, vRegD zero) %{
14421   match(Set dst (CopySignD src1 (Binary src2 zero)));
14422   effect(TEMP_DEF dst, USE src1, USE src2, USE zero);
14423   format %{ "CopySignD  $dst $src1 $src2" %}
14424   ins_encode %{
14425     FloatRegister dst = as_FloatRegister($dst$$reg),
14426                   src1 = as_FloatRegister($src1$$reg),
14427                   src2 = as_FloatRegister($src2$$reg),
14428                   zero = as_FloatRegister($zero$$reg);
14429     __ fnegd(dst, zero);
14430     __ bsl(dst, __ T8B, src2, src1);
14431   %}
14432   ins_pipe(fp_uop_d);
14433 %}
14434 
14435 instruct copySignF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14436   match(Set dst (CopySignF src1 src2));
14437   effect(TEMP_DEF dst, USE src1, USE src2);
14438   format %{ "CopySignF  $dst $src1 $src2" %}
14439   ins_encode %{
14440     FloatRegister dst = as_FloatRegister($dst$$reg),
14441                   src1 = as_FloatRegister($src1$$reg),
14442                   src2 = as_FloatRegister($src2$$reg);
14443     __ movi(dst, __ T2S, 0x80, 24);
14444     __ bsl(dst, __ T8B, src2, src1);
14445   %}
14446   ins_pipe(fp_uop_d);
14447 %}
14448 
14449 instruct signumD_reg(vRegD dst, vRegD src, vRegD zero, vRegD one) %{
14450   match(Set dst (SignumD src (Binary zero one)));
14451   effect(TEMP_DEF dst, USE src, USE zero, USE one);
14452   format %{ "signumD  $dst, $src" %}
14453   ins_encode %{
14454     FloatRegister src = as_FloatRegister($src$$reg),
14455                   dst = as_FloatRegister($dst$$reg),
14456                   zero = as_FloatRegister($zero$$reg),
14457                   one = as_FloatRegister($one$$reg);
14458     __ facgtd(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14459     __ ushrd(dst, dst, 1);     // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14460     // Bit selection instruction gets bit from "one" for each enabled bit in
14461     // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14462     // NaN the whole "src" will be copied because "dst" is zero. For all other
14463     // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14464     // from "src", and all other bits are copied from 1.0.
14465     __ bsl(dst, __ T8B, one, src);
14466   %}
14467   ins_pipe(fp_uop_d);
14468 %}
14469 
14470 instruct signumF_reg(vRegF dst, vRegF src, vRegF zero, vRegF one) %{
14471   match(Set dst (SignumF src (Binary zero one)));
14472   effect(TEMP_DEF dst, USE src, USE zero, USE one);
14473   format %{ "signumF  $dst, $src" %}
14474   ins_encode %{
14475     FloatRegister src = as_FloatRegister($src$$reg),
14476                   dst = as_FloatRegister($dst$$reg),
14477                   zero = as_FloatRegister($zero$$reg),
14478                   one = as_FloatRegister($one$$reg);
14479     __ facgts(dst, src, zero);    // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14480     __ ushr(dst, __ T2S, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14481     // Bit selection instruction gets bit from "one" for each enabled bit in
14482     // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14483     // NaN the whole "src" will be copied because "dst" is zero. For all other
14484     // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14485     // from "src", and all other bits are copied from 1.0.
14486     __ bsl(dst, __ T8B, one, src);
14487   %}
14488   ins_pipe(fp_uop_d);
14489 %}
14490 
14491 instruct onspinwait() %{
14492   match(OnSpinWait);
14493   ins_cost(INSN_COST);
14494 
14495   format %{ "onspinwait" %}
14496 
14497   ins_encode %{
14498     __ spin_wait();
14499   %}
14500   ins_pipe(pipe_class_empty);
14501 %}
14502 
14503 // ============================================================================
14504 // Logical Instructions
14505 
14506 // Integer Logical Instructions
14507 
14508 // And Instructions
14509 
14510 
14511 instruct andI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, rFlagsReg cr) %{
14512   match(Set dst (AndI src1 src2));
14513 
14514   format %{ "andw  $dst, $src1, $src2\t# int" %}
14515 
14516   ins_cost(INSN_COST);
14517   ins_encode %{
14518     __ andw(as_Register($dst$$reg),
14519             as_Register($src1$$reg),
14520             as_Register($src2$$reg));
14521   %}
14522 
14523   ins_pipe(ialu_reg_reg);
14524 %}
14525 
14526 instruct andI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2, rFlagsReg cr) %{
14527   match(Set dst (AndI src1 src2));
14528 
14529   format %{ "andsw  $dst, $src1, $src2\t# int" %}
14530 
14531   ins_cost(INSN_COST);
14532   ins_encode %{
14533     __ andw(as_Register($dst$$reg),
14534             as_Register($src1$$reg),
14535             (uint64_t)($src2$$constant));
14536   %}
14537 
14538   ins_pipe(ialu_reg_imm);
14539 %}
14540 
14541 // Or Instructions
14542 
14543 instruct orI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14544   match(Set dst (OrI src1 src2));
14545 
14546   format %{ "orrw  $dst, $src1, $src2\t# int" %}
14547 
14548   ins_cost(INSN_COST);
14549   ins_encode %{
14550     __ orrw(as_Register($dst$$reg),
14551             as_Register($src1$$reg),
14552             as_Register($src2$$reg));
14553   %}
14554 
14555   ins_pipe(ialu_reg_reg);
14556 %}
14557 
14558 instruct orI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14559   match(Set dst (OrI src1 src2));
14560 
14561   format %{ "orrw  $dst, $src1, $src2\t# int" %}
14562 
14563   ins_cost(INSN_COST);
14564   ins_encode %{
14565     __ orrw(as_Register($dst$$reg),
14566             as_Register($src1$$reg),
14567             (uint64_t)($src2$$constant));
14568   %}
14569 
14570   ins_pipe(ialu_reg_imm);
14571 %}
14572 
14573 // Xor Instructions
14574 
14575 instruct xorI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14576   match(Set dst (XorI src1 src2));
14577 
14578   format %{ "eorw  $dst, $src1, $src2\t# int" %}
14579 
14580   ins_cost(INSN_COST);
14581   ins_encode %{
14582     __ eorw(as_Register($dst$$reg),
14583             as_Register($src1$$reg),
14584             as_Register($src2$$reg));
14585   %}
14586 
14587   ins_pipe(ialu_reg_reg);
14588 %}
14589 
14590 instruct xorI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14591   match(Set dst (XorI src1 src2));
14592 
14593   format %{ "eorw  $dst, $src1, $src2\t# int" %}
14594 
14595   ins_cost(INSN_COST);
14596   ins_encode %{
14597     __ eorw(as_Register($dst$$reg),
14598             as_Register($src1$$reg),
14599             (uint64_t)($src2$$constant));
14600   %}
14601 
14602   ins_pipe(ialu_reg_imm);
14603 %}
14604 
14605 // Long Logical Instructions
14606 // TODO
14607 
14608 instruct andL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr) %{
14609   match(Set dst (AndL src1 src2));
14610 
14611   format %{ "and  $dst, $src1, $src2\t# int" %}
14612 
14613   ins_cost(INSN_COST);
14614   ins_encode %{
14615     __ andr(as_Register($dst$$reg),
14616             as_Register($src1$$reg),
14617             as_Register($src2$$reg));
14618   %}
14619 
14620   ins_pipe(ialu_reg_reg);
14621 %}
14622 
14623 instruct andL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2, rFlagsReg cr) %{
14624   match(Set dst (AndL src1 src2));
14625 
14626   format %{ "and  $dst, $src1, $src2\t# int" %}
14627 
14628   ins_cost(INSN_COST);
14629   ins_encode %{
14630     __ andr(as_Register($dst$$reg),
14631             as_Register($src1$$reg),
14632             (uint64_t)($src2$$constant));
14633   %}
14634 
14635   ins_pipe(ialu_reg_imm);
14636 %}
14637 
14638 // Or Instructions
14639 
14640 instruct orL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14641   match(Set dst (OrL src1 src2));
14642 
14643   format %{ "orr  $dst, $src1, $src2\t# int" %}
14644 
14645   ins_cost(INSN_COST);
14646   ins_encode %{
14647     __ orr(as_Register($dst$$reg),
14648            as_Register($src1$$reg),
14649            as_Register($src2$$reg));
14650   %}
14651 
14652   ins_pipe(ialu_reg_reg);
14653 %}
14654 
14655 instruct orL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14656   match(Set dst (OrL src1 src2));
14657 
14658   format %{ "orr  $dst, $src1, $src2\t# int" %}
14659 
14660   ins_cost(INSN_COST);
14661   ins_encode %{
14662     __ orr(as_Register($dst$$reg),
14663            as_Register($src1$$reg),
14664            (uint64_t)($src2$$constant));
14665   %}
14666 
14667   ins_pipe(ialu_reg_imm);
14668 %}
14669 
14670 // Xor Instructions
14671 
14672 instruct xorL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14673   match(Set dst (XorL src1 src2));
14674 
14675   format %{ "eor  $dst, $src1, $src2\t# int" %}
14676 
14677   ins_cost(INSN_COST);
14678   ins_encode %{
14679     __ eor(as_Register($dst$$reg),
14680            as_Register($src1$$reg),
14681            as_Register($src2$$reg));
14682   %}
14683 
14684   ins_pipe(ialu_reg_reg);
14685 %}
14686 
14687 instruct xorL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14688   match(Set dst (XorL src1 src2));
14689 
14690   ins_cost(INSN_COST);
14691   format %{ "eor  $dst, $src1, $src2\t# int" %}
14692 
14693   ins_encode %{
14694     __ eor(as_Register($dst$$reg),
14695            as_Register($src1$$reg),
14696            (uint64_t)($src2$$constant));
14697   %}
14698 
14699   ins_pipe(ialu_reg_imm);
14700 %}
14701 
14702 instruct convI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src)
14703 %{
14704   match(Set dst (ConvI2L src));
14705 
14706   ins_cost(INSN_COST);
14707   format %{ "sxtw  $dst, $src\t# i2l" %}
14708   ins_encode %{
14709     __ sbfm($dst$$Register, $src$$Register, 0, 31);
14710   %}
14711   ins_pipe(ialu_reg_shift);
14712 %}
14713 
14714 // this pattern occurs in bigmath arithmetic
14715 instruct convUI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src, immL_32bits mask)
14716 %{
14717   match(Set dst (AndL (ConvI2L src) mask));
14718 
14719   ins_cost(INSN_COST);
14720   format %{ "ubfm  $dst, $src, 0, 31\t# ui2l" %}
14721   ins_encode %{
14722     __ ubfm($dst$$Register, $src$$Register, 0, 31);
14723   %}
14724 
14725   ins_pipe(ialu_reg_shift);
14726 %}
14727 
14728 instruct convL2I_reg(iRegINoSp dst, iRegL src) %{
14729   match(Set dst (ConvL2I src));
14730 
14731   ins_cost(INSN_COST);
14732   format %{ "movw  $dst, $src \t// l2i" %}
14733 
14734   ins_encode %{
14735     __ movw(as_Register($dst$$reg), as_Register($src$$reg));
14736   %}
14737 
14738   ins_pipe(ialu_reg);
14739 %}
14740 
14741 instruct convD2F_reg(vRegF dst, vRegD src) %{
14742   match(Set dst (ConvD2F src));
14743 
14744   ins_cost(INSN_COST * 5);
14745   format %{ "fcvtd  $dst, $src \t// d2f" %}
14746 
14747   ins_encode %{
14748     __ fcvtd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14749   %}
14750 
14751   ins_pipe(fp_d2f);
14752 %}
14753 
14754 instruct convF2D_reg(vRegD dst, vRegF src) %{
14755   match(Set dst (ConvF2D src));
14756 
14757   ins_cost(INSN_COST * 5);
14758   format %{ "fcvts  $dst, $src \t// f2d" %}
14759 
14760   ins_encode %{
14761     __ fcvts(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14762   %}
14763 
14764   ins_pipe(fp_f2d);
14765 %}
14766 
14767 instruct convF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14768   match(Set dst (ConvF2I src));
14769 
14770   ins_cost(INSN_COST * 5);
14771   format %{ "fcvtzsw  $dst, $src \t// f2i" %}
14772 
14773   ins_encode %{
14774     __ fcvtzsw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14775   %}
14776 
14777   ins_pipe(fp_f2i);
14778 %}
14779 
14780 instruct convF2L_reg_reg(iRegLNoSp dst, vRegF src) %{
14781   match(Set dst (ConvF2L src));
14782 
14783   ins_cost(INSN_COST * 5);
14784   format %{ "fcvtzs  $dst, $src \t// f2l" %}
14785 
14786   ins_encode %{
14787     __ fcvtzs(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14788   %}
14789 
14790   ins_pipe(fp_f2l);
14791 %}
14792 
14793 instruct convF2HF_reg_reg(iRegINoSp dst, vRegF src, vRegF tmp) %{
14794   match(Set dst (ConvF2HF src));
14795   format %{ "fcvt $tmp, $src\t# convert single to half precision\n\t"
14796             "smov $dst, $tmp\t# move result from $tmp to $dst"
14797   %}
14798   effect(TEMP tmp);
14799   ins_encode %{
14800       __ flt_to_flt16($dst$$Register, $src$$FloatRegister, $tmp$$FloatRegister);
14801   %}
14802   ins_pipe(pipe_slow);
14803 %}
14804 
14805 instruct convHF2F_reg_reg(vRegF dst, iRegINoSp src, vRegF tmp) %{
14806   match(Set dst (ConvHF2F src));
14807   format %{ "mov $tmp, $src\t# move source from $src to $tmp\n\t"
14808             "fcvt $dst, $tmp\t# convert half to single precision"
14809   %}
14810   effect(TEMP tmp);
14811   ins_encode %{
14812       __ flt16_to_flt($dst$$FloatRegister, $src$$Register, $tmp$$FloatRegister);
14813   %}
14814   ins_pipe(pipe_slow);
14815 %}
14816 
14817 instruct convI2F_reg_reg(vRegF dst, iRegIorL2I src) %{
14818   match(Set dst (ConvI2F src));
14819 
14820   ins_cost(INSN_COST * 5);
14821   format %{ "scvtfws  $dst, $src \t// i2f" %}
14822 
14823   ins_encode %{
14824     __ scvtfws(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14825   %}
14826 
14827   ins_pipe(fp_i2f);
14828 %}
14829 
14830 instruct convL2F_reg_reg(vRegF dst, iRegL src) %{
14831   match(Set dst (ConvL2F src));
14832 
14833   ins_cost(INSN_COST * 5);
14834   format %{ "scvtfs  $dst, $src \t// l2f" %}
14835 
14836   ins_encode %{
14837     __ scvtfs(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14838   %}
14839 
14840   ins_pipe(fp_l2f);
14841 %}
14842 
14843 instruct convD2I_reg_reg(iRegINoSp dst, vRegD src) %{
14844   match(Set dst (ConvD2I src));
14845 
14846   ins_cost(INSN_COST * 5);
14847   format %{ "fcvtzdw  $dst, $src \t// d2i" %}
14848 
14849   ins_encode %{
14850     __ fcvtzdw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14851   %}
14852 
14853   ins_pipe(fp_d2i);
14854 %}
14855 
14856 instruct convD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14857   match(Set dst (ConvD2L src));
14858 
14859   ins_cost(INSN_COST * 5);
14860   format %{ "fcvtzd  $dst, $src \t// d2l" %}
14861 
14862   ins_encode %{
14863     __ fcvtzd(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14864   %}
14865 
14866   ins_pipe(fp_d2l);
14867 %}
14868 
14869 instruct convI2D_reg_reg(vRegD dst, iRegIorL2I src) %{
14870   match(Set dst (ConvI2D src));
14871 
14872   ins_cost(INSN_COST * 5);
14873   format %{ "scvtfwd  $dst, $src \t// i2d" %}
14874 
14875   ins_encode %{
14876     __ scvtfwd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14877   %}
14878 
14879   ins_pipe(fp_i2d);
14880 %}
14881 
14882 instruct convL2D_reg_reg(vRegD dst, iRegL src) %{
14883   match(Set dst (ConvL2D src));
14884 
14885   ins_cost(INSN_COST * 5);
14886   format %{ "scvtfd  $dst, $src \t// l2d" %}
14887 
14888   ins_encode %{
14889     __ scvtfd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14890   %}
14891 
14892   ins_pipe(fp_l2d);
14893 %}
14894 
14895 instruct round_double_reg(iRegLNoSp dst, vRegD src, vRegD ftmp, rFlagsReg cr)
14896 %{
14897   match(Set dst (RoundD src));
14898   effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
14899   format %{ "java_round_double $dst,$src"%}
14900   ins_encode %{
14901     __ java_round_double($dst$$Register, as_FloatRegister($src$$reg),
14902                          as_FloatRegister($ftmp$$reg));
14903   %}
14904   ins_pipe(pipe_slow);
14905 %}
14906 
14907 instruct round_float_reg(iRegINoSp dst, vRegF src, vRegF ftmp, rFlagsReg cr)
14908 %{
14909   match(Set dst (RoundF src));
14910   effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
14911   format %{ "java_round_float $dst,$src"%}
14912   ins_encode %{
14913     __ java_round_float($dst$$Register, as_FloatRegister($src$$reg),
14914                         as_FloatRegister($ftmp$$reg));
14915   %}
14916   ins_pipe(pipe_slow);
14917 %}
14918 
14919 // stack <-> reg and reg <-> reg shuffles with no conversion
14920 
14921 instruct MoveF2I_stack_reg(iRegINoSp dst, stackSlotF src) %{
14922 
14923   match(Set dst (MoveF2I src));
14924 
14925   effect(DEF dst, USE src);
14926 
14927   ins_cost(4 * INSN_COST);
14928 
14929   format %{ "ldrw $dst, $src\t# MoveF2I_stack_reg" %}
14930 
14931   ins_encode %{
14932     __ ldrw($dst$$Register, Address(sp, $src$$disp));
14933   %}
14934 
14935   ins_pipe(iload_reg_reg);
14936 
14937 %}
14938 
14939 instruct MoveI2F_stack_reg(vRegF dst, stackSlotI src) %{
14940 
14941   match(Set dst (MoveI2F src));
14942 
14943   effect(DEF dst, USE src);
14944 
14945   ins_cost(4 * INSN_COST);
14946 
14947   format %{ "ldrs $dst, $src\t# MoveI2F_stack_reg" %}
14948 
14949   ins_encode %{
14950     __ ldrs(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
14951   %}
14952 
14953   ins_pipe(pipe_class_memory);
14954 
14955 %}
14956 
14957 instruct MoveD2L_stack_reg(iRegLNoSp dst, stackSlotD src) %{
14958 
14959   match(Set dst (MoveD2L src));
14960 
14961   effect(DEF dst, USE src);
14962 
14963   ins_cost(4 * INSN_COST);
14964 
14965   format %{ "ldr $dst, $src\t# MoveD2L_stack_reg" %}
14966 
14967   ins_encode %{
14968     __ ldr($dst$$Register, Address(sp, $src$$disp));
14969   %}
14970 
14971   ins_pipe(iload_reg_reg);
14972 
14973 %}
14974 
14975 instruct MoveL2D_stack_reg(vRegD dst, stackSlotL src) %{
14976 
14977   match(Set dst (MoveL2D src));
14978 
14979   effect(DEF dst, USE src);
14980 
14981   ins_cost(4 * INSN_COST);
14982 
14983   format %{ "ldrd $dst, $src\t# MoveL2D_stack_reg" %}
14984 
14985   ins_encode %{
14986     __ ldrd(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
14987   %}
14988 
14989   ins_pipe(pipe_class_memory);
14990 
14991 %}
14992 
14993 instruct MoveF2I_reg_stack(stackSlotI dst, vRegF src) %{
14994 
14995   match(Set dst (MoveF2I src));
14996 
14997   effect(DEF dst, USE src);
14998 
14999   ins_cost(INSN_COST);
15000 
15001   format %{ "strs $src, $dst\t# MoveF2I_reg_stack" %}
15002 
15003   ins_encode %{
15004     __ strs(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
15005   %}
15006 
15007   ins_pipe(pipe_class_memory);
15008 
15009 %}
15010 
15011 instruct MoveI2F_reg_stack(stackSlotF dst, iRegI src) %{
15012 
15013   match(Set dst (MoveI2F src));
15014 
15015   effect(DEF dst, USE src);
15016 
15017   ins_cost(INSN_COST);
15018 
15019   format %{ "strw $src, $dst\t# MoveI2F_reg_stack" %}
15020 
15021   ins_encode %{
15022     __ strw($src$$Register, Address(sp, $dst$$disp));
15023   %}
15024 
15025   ins_pipe(istore_reg_reg);
15026 
15027 %}
15028 
15029 instruct MoveD2L_reg_stack(stackSlotL dst, vRegD src) %{
15030 
15031   match(Set dst (MoveD2L src));
15032 
15033   effect(DEF dst, USE src);
15034 
15035   ins_cost(INSN_COST);
15036 
15037   format %{ "strd $dst, $src\t# MoveD2L_reg_stack" %}
15038 
15039   ins_encode %{
15040     __ strd(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
15041   %}
15042 
15043   ins_pipe(pipe_class_memory);
15044 
15045 %}
15046 
15047 instruct MoveL2D_reg_stack(stackSlotD dst, iRegL src) %{
15048 
15049   match(Set dst (MoveL2D src));
15050 
15051   effect(DEF dst, USE src);
15052 
15053   ins_cost(INSN_COST);
15054 
15055   format %{ "str $src, $dst\t# MoveL2D_reg_stack" %}
15056 
15057   ins_encode %{
15058     __ str($src$$Register, Address(sp, $dst$$disp));
15059   %}
15060 
15061   ins_pipe(istore_reg_reg);
15062 
15063 %}
15064 
15065 instruct MoveF2I_reg_reg(iRegINoSp dst, vRegF src) %{
15066 
15067   match(Set dst (MoveF2I src));
15068 
15069   effect(DEF dst, USE src);
15070 
15071   ins_cost(INSN_COST);
15072 
15073   format %{ "fmovs $dst, $src\t# MoveF2I_reg_reg" %}
15074 
15075   ins_encode %{
15076     __ fmovs($dst$$Register, as_FloatRegister($src$$reg));
15077   %}
15078 
15079   ins_pipe(fp_f2i);
15080 
15081 %}
15082 
15083 instruct MoveI2F_reg_reg(vRegF dst, iRegI src) %{
15084 
15085   match(Set dst (MoveI2F src));
15086 
15087   effect(DEF dst, USE src);
15088 
15089   ins_cost(INSN_COST);
15090 
15091   format %{ "fmovs $dst, $src\t# MoveI2F_reg_reg" %}
15092 
15093   ins_encode %{
15094     __ fmovs(as_FloatRegister($dst$$reg), $src$$Register);
15095   %}
15096 
15097   ins_pipe(fp_i2f);
15098 
15099 %}
15100 
15101 instruct MoveD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
15102 
15103   match(Set dst (MoveD2L src));
15104 
15105   effect(DEF dst, USE src);
15106 
15107   ins_cost(INSN_COST);
15108 
15109   format %{ "fmovd $dst, $src\t# MoveD2L_reg_reg" %}
15110 
15111   ins_encode %{
15112     __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
15113   %}
15114 
15115   ins_pipe(fp_d2l);
15116 
15117 %}
15118 
15119 instruct MoveL2D_reg_reg(vRegD dst, iRegL src) %{
15120 
15121   match(Set dst (MoveL2D src));
15122 
15123   effect(DEF dst, USE src);
15124 
15125   ins_cost(INSN_COST);
15126 
15127   format %{ "fmovd $dst, $src\t# MoveL2D_reg_reg" %}
15128 
15129   ins_encode %{
15130     __ fmovd(as_FloatRegister($dst$$reg), $src$$Register);
15131   %}
15132 
15133   ins_pipe(fp_l2d);
15134 
15135 %}
15136 
15137 // ============================================================================
15138 // clearing of an array
15139 
15140 instruct clearArray_reg_reg_immL0(iRegL_R11 cnt, iRegP_R10 base, immL0 zero, Universe dummy, rFlagsReg cr)
15141 %{
15142   match(Set dummy (ClearArray (Binary cnt base) zero));
15143   effect(USE_KILL cnt, USE_KILL base, KILL cr);
15144 
15145   ins_cost(4 * INSN_COST);
15146   format %{ "ClearArray $cnt, $base" %}
15147 
15148   ins_encode %{
15149     address tpc = __ zero_words($base$$Register, $cnt$$Register);
15150     if (tpc == nullptr) {
15151       ciEnv::current()->record_failure("CodeCache is full");
15152       return;
15153     }
15154   %}
15155 
15156   ins_pipe(pipe_class_memory);
15157 %}
15158 
15159 instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, iRegL val, Universe dummy, rFlagsReg cr)
15160 %{
15161   predicate(((ClearArrayNode*)n)->word_copy_only());
15162   match(Set dummy (ClearArray (Binary cnt base) val));
15163   effect(USE_KILL cnt, USE_KILL base, KILL cr);
15164 
15165   ins_cost(4 * INSN_COST);
15166   format %{ "ClearArray $cnt, $base, $val" %}
15167 
15168   ins_encode %{
15169     __ fill_words($base$$Register, $cnt$$Register, $val$$Register);
15170   %}
15171 
15172   ins_pipe(pipe_class_memory);
15173 %}
15174 
15175 instruct clearArray_imm_reg(immL cnt, iRegP_R10 base, iRegL_R11 temp, Universe dummy, rFlagsReg cr)
15176 %{
15177   predicate((uint64_t)n->in(2)->get_long()
15178             < (uint64_t)(BlockZeroingLowLimit >> LogBytesPerWord)
15179             && !((ClearArrayNode*)n)->word_copy_only());
15180   match(Set dummy (ClearArray cnt base));
15181   effect(TEMP temp, USE_KILL base, KILL cr);
15182 
15183   ins_cost(4 * INSN_COST);
15184   format %{ "ClearArray $cnt, $base" %}
15185 
15186   ins_encode %{
15187     address tpc = __ zero_words($base$$Register, (uint64_t)$cnt$$constant);
15188     if (tpc == nullptr) {
15189       ciEnv::current()->record_failure("CodeCache is full");
15190       return;
15191     }
15192   %}
15193 
15194   ins_pipe(pipe_class_memory);
15195 %}
15196 
15197 // ============================================================================
15198 // Overflow Math Instructions
15199 
15200 instruct overflowAddI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15201 %{
15202   match(Set cr (OverflowAddI op1 op2));
15203 
15204   format %{ "cmnw  $op1, $op2\t# overflow check int" %}
15205   ins_cost(INSN_COST);
15206   ins_encode %{
15207     __ cmnw($op1$$Register, $op2$$Register);
15208   %}
15209 
15210   ins_pipe(icmp_reg_reg);
15211 %}
15212 
15213 instruct overflowAddI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
15214 %{
15215   match(Set cr (OverflowAddI op1 op2));
15216 
15217   format %{ "cmnw  $op1, $op2\t# overflow check int" %}
15218   ins_cost(INSN_COST);
15219   ins_encode %{
15220     __ cmnw($op1$$Register, $op2$$constant);
15221   %}
15222 
15223   ins_pipe(icmp_reg_imm);
15224 %}
15225 
15226 instruct overflowAddL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15227 %{
15228   match(Set cr (OverflowAddL op1 op2));
15229 
15230   format %{ "cmn   $op1, $op2\t# overflow check long" %}
15231   ins_cost(INSN_COST);
15232   ins_encode %{
15233     __ cmn($op1$$Register, $op2$$Register);
15234   %}
15235 
15236   ins_pipe(icmp_reg_reg);
15237 %}
15238 
15239 instruct overflowAddL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
15240 %{
15241   match(Set cr (OverflowAddL op1 op2));
15242 
15243   format %{ "adds  zr, $op1, $op2\t# overflow check long" %}
15244   ins_cost(INSN_COST);
15245   ins_encode %{
15246     __ adds(zr, $op1$$Register, $op2$$constant);
15247   %}
15248 
15249   ins_pipe(icmp_reg_imm);
15250 %}
15251 
15252 instruct overflowSubI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15253 %{
15254   match(Set cr (OverflowSubI op1 op2));
15255 
15256   format %{ "cmpw  $op1, $op2\t# overflow check int" %}
15257   ins_cost(INSN_COST);
15258   ins_encode %{
15259     __ cmpw($op1$$Register, $op2$$Register);
15260   %}
15261 
15262   ins_pipe(icmp_reg_reg);
15263 %}
15264 
15265 instruct overflowSubI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
15266 %{
15267   match(Set cr (OverflowSubI op1 op2));
15268 
15269   format %{ "cmpw  $op1, $op2\t# overflow check int" %}
15270   ins_cost(INSN_COST);
15271   ins_encode %{
15272     __ cmpw($op1$$Register, $op2$$constant);
15273   %}
15274 
15275   ins_pipe(icmp_reg_imm);
15276 %}
15277 
15278 instruct overflowSubL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15279 %{
15280   match(Set cr (OverflowSubL op1 op2));
15281 
15282   format %{ "cmp   $op1, $op2\t# overflow check long" %}
15283   ins_cost(INSN_COST);
15284   ins_encode %{
15285     __ cmp($op1$$Register, $op2$$Register);
15286   %}
15287 
15288   ins_pipe(icmp_reg_reg);
15289 %}
15290 
15291 instruct overflowSubL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
15292 %{
15293   match(Set cr (OverflowSubL op1 op2));
15294 
15295   format %{ "cmp   $op1, $op2\t# overflow check long" %}
15296   ins_cost(INSN_COST);
15297   ins_encode %{
15298     __ subs(zr, $op1$$Register, $op2$$constant);
15299   %}
15300 
15301   ins_pipe(icmp_reg_imm);
15302 %}
15303 
15304 instruct overflowNegI_reg(rFlagsReg cr, immI0 zero, iRegIorL2I op1)
15305 %{
15306   match(Set cr (OverflowSubI zero op1));
15307 
15308   format %{ "cmpw  zr, $op1\t# overflow check int" %}
15309   ins_cost(INSN_COST);
15310   ins_encode %{
15311     __ cmpw(zr, $op1$$Register);
15312   %}
15313 
15314   ins_pipe(icmp_reg_imm);
15315 %}
15316 
15317 instruct overflowNegL_reg(rFlagsReg cr, immI0 zero, iRegL op1)
15318 %{
15319   match(Set cr (OverflowSubL zero op1));
15320 
15321   format %{ "cmp   zr, $op1\t# overflow check long" %}
15322   ins_cost(INSN_COST);
15323   ins_encode %{
15324     __ cmp(zr, $op1$$Register);
15325   %}
15326 
15327   ins_pipe(icmp_reg_imm);
15328 %}
15329 
15330 instruct overflowMulI_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15331 %{
15332   match(Set cr (OverflowMulI op1 op2));
15333 
15334   format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15335             "cmp   rscratch1, rscratch1, sxtw\n\t"
15336             "movw  rscratch1, #0x80000000\n\t"
15337             "cselw rscratch1, rscratch1, zr, NE\n\t"
15338             "cmpw  rscratch1, #1" %}
15339   ins_cost(5 * INSN_COST);
15340   ins_encode %{
15341     __ smull(rscratch1, $op1$$Register, $op2$$Register);
15342     __ subs(zr, rscratch1, rscratch1, ext::sxtw);      // NE => overflow
15343     __ movw(rscratch1, 0x80000000);                    // Develop 0 (EQ),
15344     __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15345     __ cmpw(rscratch1, 1);                             // 0x80000000 - 1 => VS
15346   %}
15347 
15348   ins_pipe(pipe_slow);
15349 %}
15350 
15351 instruct overflowMulI_reg_branch(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, label labl, rFlagsReg cr)
15352 %{
15353   match(If cmp (OverflowMulI op1 op2));
15354   predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15355             || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15356   effect(USE labl, KILL cr);
15357 
15358   format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15359             "cmp   rscratch1, rscratch1, sxtw\n\t"
15360             "b$cmp   $labl" %}
15361   ins_cost(3 * INSN_COST); // Branch is rare so treat as INSN_COST
15362   ins_encode %{
15363     Label* L = $labl$$label;
15364     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15365     __ smull(rscratch1, $op1$$Register, $op2$$Register);
15366     __ subs(zr, rscratch1, rscratch1, ext::sxtw);      // NE => overflow
15367     __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15368   %}
15369 
15370   ins_pipe(pipe_serial);
15371 %}
15372 
15373 instruct overflowMulL_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15374 %{
15375   match(Set cr (OverflowMulL op1 op2));
15376 
15377   format %{ "mul   rscratch1, $op1, $op2\t#overflow check long\n\t"
15378             "smulh rscratch2, $op1, $op2\n\t"
15379             "cmp   rscratch2, rscratch1, ASR #63\n\t"
15380             "movw  rscratch1, #0x80000000\n\t"
15381             "cselw rscratch1, rscratch1, zr, NE\n\t"
15382             "cmpw  rscratch1, #1" %}
15383   ins_cost(6 * INSN_COST);
15384   ins_encode %{
15385     __ mul(rscratch1, $op1$$Register, $op2$$Register);   // Result bits 0..63
15386     __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15387     __ cmp(rscratch2, rscratch1, Assembler::ASR, 63);    // Top is pure sign ext
15388     __ movw(rscratch1, 0x80000000);                    // Develop 0 (EQ),
15389     __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15390     __ cmpw(rscratch1, 1);                             // 0x80000000 - 1 => VS
15391   %}
15392 
15393   ins_pipe(pipe_slow);
15394 %}
15395 
15396 instruct overflowMulL_reg_branch(cmpOp cmp, iRegL op1, iRegL op2, label labl, rFlagsReg cr)
15397 %{
15398   match(If cmp (OverflowMulL op1 op2));
15399   predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15400             || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15401   effect(USE labl, KILL cr);
15402 
15403   format %{ "mul   rscratch1, $op1, $op2\t#overflow check long\n\t"
15404             "smulh rscratch2, $op1, $op2\n\t"
15405             "cmp   rscratch2, rscratch1, ASR #63\n\t"
15406             "b$cmp $labl" %}
15407   ins_cost(4 * INSN_COST); // Branch is rare so treat as INSN_COST
15408   ins_encode %{
15409     Label* L = $labl$$label;
15410     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15411     __ mul(rscratch1, $op1$$Register, $op2$$Register);   // Result bits 0..63
15412     __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15413     __ cmp(rscratch2, rscratch1, Assembler::ASR, 63);    // Top is pure sign ext
15414     __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15415   %}
15416 
15417   ins_pipe(pipe_serial);
15418 %}
15419 
15420 // ============================================================================
15421 // Compare Instructions
15422 
15423 instruct compI_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
15424 %{
15425   match(Set cr (CmpI op1 op2));
15426 
15427   effect(DEF cr, USE op1, USE op2);
15428 
15429   ins_cost(INSN_COST);
15430   format %{ "cmpw  $op1, $op2" %}
15431 
15432   ins_encode(aarch64_enc_cmpw(op1, op2));
15433 
15434   ins_pipe(icmp_reg_reg);
15435 %}
15436 
15437 instruct compI_reg_immI0(rFlagsReg cr, iRegI op1, immI0 zero)
15438 %{
15439   match(Set cr (CmpI op1 zero));
15440 
15441   effect(DEF cr, USE op1);
15442 
15443   ins_cost(INSN_COST);
15444   format %{ "cmpw $op1, 0" %}
15445 
15446   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15447 
15448   ins_pipe(icmp_reg_imm);
15449 %}
15450 
15451 instruct compI_reg_immIAddSub(rFlagsReg cr, iRegI op1, immIAddSub op2)
15452 %{
15453   match(Set cr (CmpI op1 op2));
15454 
15455   effect(DEF cr, USE op1);
15456 
15457   ins_cost(INSN_COST);
15458   format %{ "cmpw  $op1, $op2" %}
15459 
15460   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15461 
15462   ins_pipe(icmp_reg_imm);
15463 %}
15464 
15465 instruct compI_reg_immI(rFlagsReg cr, iRegI op1, immI op2)
15466 %{
15467   match(Set cr (CmpI op1 op2));
15468 
15469   effect(DEF cr, USE op1);
15470 
15471   ins_cost(INSN_COST * 2);
15472   format %{ "cmpw  $op1, $op2" %}
15473 
15474   ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15475 
15476   ins_pipe(icmp_reg_imm);
15477 %}
15478 
15479 // Unsigned compare Instructions; really, same as signed compare
15480 // except it should only be used to feed an If or a CMovI which takes a
15481 // cmpOpU.
15482 
15483 instruct compU_reg_reg(rFlagsRegU cr, iRegI op1, iRegI op2)
15484 %{
15485   match(Set cr (CmpU op1 op2));
15486 
15487   effect(DEF cr, USE op1, USE op2);
15488 
15489   ins_cost(INSN_COST);
15490   format %{ "cmpw  $op1, $op2\t# unsigned" %}
15491 
15492   ins_encode(aarch64_enc_cmpw(op1, op2));
15493 
15494   ins_pipe(icmp_reg_reg);
15495 %}
15496 
15497 instruct compU_reg_immI0(rFlagsRegU cr, iRegI op1, immI0 zero)
15498 %{
15499   match(Set cr (CmpU op1 zero));
15500 
15501   effect(DEF cr, USE op1);
15502 
15503   ins_cost(INSN_COST);
15504   format %{ "cmpw $op1, #0\t# unsigned" %}
15505 
15506   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15507 
15508   ins_pipe(icmp_reg_imm);
15509 %}
15510 
15511 instruct compU_reg_immIAddSub(rFlagsRegU cr, iRegI op1, immIAddSub op2)
15512 %{
15513   match(Set cr (CmpU op1 op2));
15514 
15515   effect(DEF cr, USE op1);
15516 
15517   ins_cost(INSN_COST);
15518   format %{ "cmpw  $op1, $op2\t# unsigned" %}
15519 
15520   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15521 
15522   ins_pipe(icmp_reg_imm);
15523 %}
15524 
15525 instruct compU_reg_immI(rFlagsRegU cr, iRegI op1, immI op2)
15526 %{
15527   match(Set cr (CmpU op1 op2));
15528 
15529   effect(DEF cr, USE op1);
15530 
15531   ins_cost(INSN_COST * 2);
15532   format %{ "cmpw  $op1, $op2\t# unsigned" %}
15533 
15534   ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15535 
15536   ins_pipe(icmp_reg_imm);
15537 %}
15538 
15539 instruct compL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15540 %{
15541   match(Set cr (CmpL op1 op2));
15542 
15543   effect(DEF cr, USE op1, USE op2);
15544 
15545   ins_cost(INSN_COST);
15546   format %{ "cmp  $op1, $op2" %}
15547 
15548   ins_encode(aarch64_enc_cmp(op1, op2));
15549 
15550   ins_pipe(icmp_reg_reg);
15551 %}
15552 
15553 instruct compL_reg_immL0(rFlagsReg cr, iRegL op1, immL0 zero)
15554 %{
15555   match(Set cr (CmpL op1 zero));
15556 
15557   effect(DEF cr, USE op1);
15558 
15559   ins_cost(INSN_COST);
15560   format %{ "tst  $op1" %}
15561 
15562   ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15563 
15564   ins_pipe(icmp_reg_imm);
15565 %}
15566 
15567 instruct compL_reg_immLAddSub(rFlagsReg cr, iRegL op1, immLAddSub op2)
15568 %{
15569   match(Set cr (CmpL op1 op2));
15570 
15571   effect(DEF cr, USE op1);
15572 
15573   ins_cost(INSN_COST);
15574   format %{ "cmp  $op1, $op2" %}
15575 
15576   ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15577 
15578   ins_pipe(icmp_reg_imm);
15579 %}
15580 
15581 instruct compL_reg_immL(rFlagsReg cr, iRegL op1, immL op2)
15582 %{
15583   match(Set cr (CmpL op1 op2));
15584 
15585   effect(DEF cr, USE op1);
15586 
15587   ins_cost(INSN_COST * 2);
15588   format %{ "cmp  $op1, $op2" %}
15589 
15590   ins_encode(aarch64_enc_cmp_imm(op1, op2));
15591 
15592   ins_pipe(icmp_reg_imm);
15593 %}
15594 
15595 instruct compUL_reg_reg(rFlagsRegU cr, iRegL op1, iRegL op2)
15596 %{
15597   match(Set cr (CmpUL op1 op2));
15598 
15599   effect(DEF cr, USE op1, USE op2);
15600 
15601   ins_cost(INSN_COST);
15602   format %{ "cmp  $op1, $op2" %}
15603 
15604   ins_encode(aarch64_enc_cmp(op1, op2));
15605 
15606   ins_pipe(icmp_reg_reg);
15607 %}
15608 
15609 instruct compUL_reg_immL0(rFlagsRegU cr, iRegL op1, immL0 zero)
15610 %{
15611   match(Set cr (CmpUL op1 zero));
15612 
15613   effect(DEF cr, USE op1);
15614 
15615   ins_cost(INSN_COST);
15616   format %{ "tst  $op1" %}
15617 
15618   ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15619 
15620   ins_pipe(icmp_reg_imm);
15621 %}
15622 
15623 instruct compUL_reg_immLAddSub(rFlagsRegU cr, iRegL op1, immLAddSub op2)
15624 %{
15625   match(Set cr (CmpUL op1 op2));
15626 
15627   effect(DEF cr, USE op1);
15628 
15629   ins_cost(INSN_COST);
15630   format %{ "cmp  $op1, $op2" %}
15631 
15632   ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15633 
15634   ins_pipe(icmp_reg_imm);
15635 %}
15636 
15637 instruct compUL_reg_immL(rFlagsRegU cr, iRegL op1, immL op2)
15638 %{
15639   match(Set cr (CmpUL op1 op2));
15640 
15641   effect(DEF cr, USE op1);
15642 
15643   ins_cost(INSN_COST * 2);
15644   format %{ "cmp  $op1, $op2" %}
15645 
15646   ins_encode(aarch64_enc_cmp_imm(op1, op2));
15647 
15648   ins_pipe(icmp_reg_imm);
15649 %}
15650 
15651 instruct compP_reg_reg(rFlagsRegU cr, iRegP op1, iRegP op2)
15652 %{
15653   match(Set cr (CmpP op1 op2));
15654 
15655   effect(DEF cr, USE op1, USE op2);
15656 
15657   ins_cost(INSN_COST);
15658   format %{ "cmp  $op1, $op2\t // ptr" %}
15659 
15660   ins_encode(aarch64_enc_cmpp(op1, op2));
15661 
15662   ins_pipe(icmp_reg_reg);
15663 %}
15664 
15665 instruct compN_reg_reg(rFlagsRegU cr, iRegN op1, iRegN op2)
15666 %{
15667   match(Set cr (CmpN op1 op2));
15668 
15669   effect(DEF cr, USE op1, USE op2);
15670 
15671   ins_cost(INSN_COST);
15672   format %{ "cmp  $op1, $op2\t // compressed ptr" %}
15673 
15674   ins_encode(aarch64_enc_cmpn(op1, op2));
15675 
15676   ins_pipe(icmp_reg_reg);
15677 %}
15678 
15679 instruct testP_reg(rFlagsRegU cr, iRegP op1, immP0 zero)
15680 %{
15681   match(Set cr (CmpP op1 zero));
15682 
15683   effect(DEF cr, USE op1, USE zero);
15684 
15685   ins_cost(INSN_COST);
15686   format %{ "cmp  $op1, 0\t // ptr" %}
15687 
15688   ins_encode(aarch64_enc_testp(op1));
15689 
15690   ins_pipe(icmp_reg_imm);
15691 %}
15692 
15693 instruct testN_reg(rFlagsRegU cr, iRegN op1, immN0 zero)
15694 %{
15695   match(Set cr (CmpN op1 zero));
15696 
15697   effect(DEF cr, USE op1, USE zero);
15698 
15699   ins_cost(INSN_COST);
15700   format %{ "cmp  $op1, 0\t // compressed ptr" %}
15701 
15702   ins_encode(aarch64_enc_testn(op1));
15703 
15704   ins_pipe(icmp_reg_imm);
15705 %}
15706 
15707 // FP comparisons
15708 //
15709 // n.b. CmpF/CmpD set a normal flags reg which then gets compared
15710 // using normal cmpOp. See declaration of rFlagsReg for details.
15711 
15712 instruct compF_reg_reg(rFlagsReg cr, vRegF src1, vRegF src2)
15713 %{
15714   match(Set cr (CmpF src1 src2));
15715 
15716   ins_cost(3 * INSN_COST);
15717   format %{ "fcmps $src1, $src2" %}
15718 
15719   ins_encode %{
15720     __ fcmps(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15721   %}
15722 
15723   ins_pipe(pipe_class_compare);
15724 %}
15725 
15726 instruct compF_reg_zero(rFlagsReg cr, vRegF src1, immF0 src2)
15727 %{
15728   match(Set cr (CmpF src1 src2));
15729 
15730   ins_cost(3 * INSN_COST);
15731   format %{ "fcmps $src1, 0.0" %}
15732 
15733   ins_encode %{
15734     __ fcmps(as_FloatRegister($src1$$reg), 0.0);
15735   %}
15736 
15737   ins_pipe(pipe_class_compare);
15738 %}
15739 // FROM HERE
15740 
15741 instruct compD_reg_reg(rFlagsReg cr, vRegD src1, vRegD src2)
15742 %{
15743   match(Set cr (CmpD src1 src2));
15744 
15745   ins_cost(3 * INSN_COST);
15746   format %{ "fcmpd $src1, $src2" %}
15747 
15748   ins_encode %{
15749     __ fcmpd(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15750   %}
15751 
15752   ins_pipe(pipe_class_compare);
15753 %}
15754 
15755 instruct compD_reg_zero(rFlagsReg cr, vRegD src1, immD0 src2)
15756 %{
15757   match(Set cr (CmpD src1 src2));
15758 
15759   ins_cost(3 * INSN_COST);
15760   format %{ "fcmpd $src1, 0.0" %}
15761 
15762   ins_encode %{
15763     __ fcmpd(as_FloatRegister($src1$$reg), 0.0);
15764   %}
15765 
15766   ins_pipe(pipe_class_compare);
15767 %}
15768 
15769 instruct compF3_reg_reg(iRegINoSp dst, vRegF src1, vRegF src2, rFlagsReg cr)
15770 %{
15771   match(Set dst (CmpF3 src1 src2));
15772   effect(KILL cr);
15773 
15774   ins_cost(5 * INSN_COST);
15775   format %{ "fcmps $src1, $src2\n\t"
15776             "csinvw($dst, zr, zr, eq\n\t"
15777             "csnegw($dst, $dst, $dst, lt)"
15778   %}
15779 
15780   ins_encode %{
15781     Label done;
15782     FloatRegister s1 = as_FloatRegister($src1$$reg);
15783     FloatRegister s2 = as_FloatRegister($src2$$reg);
15784     Register d = as_Register($dst$$reg);
15785     __ fcmps(s1, s2);
15786     // installs 0 if EQ else -1
15787     __ csinvw(d, zr, zr, Assembler::EQ);
15788     // keeps -1 if less or unordered else installs 1
15789     __ csnegw(d, d, d, Assembler::LT);
15790     __ bind(done);
15791   %}
15792 
15793   ins_pipe(pipe_class_default);
15794 
15795 %}
15796 
15797 instruct compD3_reg_reg(iRegINoSp dst, vRegD src1, vRegD src2, rFlagsReg cr)
15798 %{
15799   match(Set dst (CmpD3 src1 src2));
15800   effect(KILL cr);
15801 
15802   ins_cost(5 * INSN_COST);
15803   format %{ "fcmpd $src1, $src2\n\t"
15804             "csinvw($dst, zr, zr, eq\n\t"
15805             "csnegw($dst, $dst, $dst, lt)"
15806   %}
15807 
15808   ins_encode %{
15809     Label done;
15810     FloatRegister s1 = as_FloatRegister($src1$$reg);
15811     FloatRegister s2 = as_FloatRegister($src2$$reg);
15812     Register d = as_Register($dst$$reg);
15813     __ fcmpd(s1, s2);
15814     // installs 0 if EQ else -1
15815     __ csinvw(d, zr, zr, Assembler::EQ);
15816     // keeps -1 if less or unordered else installs 1
15817     __ csnegw(d, d, d, Assembler::LT);
15818     __ bind(done);
15819   %}
15820   ins_pipe(pipe_class_default);
15821 
15822 %}
15823 
15824 instruct compF3_reg_immF0(iRegINoSp dst, vRegF src1, immF0 zero, rFlagsReg cr)
15825 %{
15826   match(Set dst (CmpF3 src1 zero));
15827   effect(KILL cr);
15828 
15829   ins_cost(5 * INSN_COST);
15830   format %{ "fcmps $src1, 0.0\n\t"
15831             "csinvw($dst, zr, zr, eq\n\t"
15832             "csnegw($dst, $dst, $dst, lt)"
15833   %}
15834 
15835   ins_encode %{
15836     Label done;
15837     FloatRegister s1 = as_FloatRegister($src1$$reg);
15838     Register d = as_Register($dst$$reg);
15839     __ fcmps(s1, 0.0);
15840     // installs 0 if EQ else -1
15841     __ csinvw(d, zr, zr, Assembler::EQ);
15842     // keeps -1 if less or unordered else installs 1
15843     __ csnegw(d, d, d, Assembler::LT);
15844     __ bind(done);
15845   %}
15846 
15847   ins_pipe(pipe_class_default);
15848 
15849 %}
15850 
15851 instruct compD3_reg_immD0(iRegINoSp dst, vRegD src1, immD0 zero, rFlagsReg cr)
15852 %{
15853   match(Set dst (CmpD3 src1 zero));
15854   effect(KILL cr);
15855 
15856   ins_cost(5 * INSN_COST);
15857   format %{ "fcmpd $src1, 0.0\n\t"
15858             "csinvw($dst, zr, zr, eq\n\t"
15859             "csnegw($dst, $dst, $dst, lt)"
15860   %}
15861 
15862   ins_encode %{
15863     Label done;
15864     FloatRegister s1 = as_FloatRegister($src1$$reg);
15865     Register d = as_Register($dst$$reg);
15866     __ fcmpd(s1, 0.0);
15867     // installs 0 if EQ else -1
15868     __ csinvw(d, zr, zr, Assembler::EQ);
15869     // keeps -1 if less or unordered else installs 1
15870     __ csnegw(d, d, d, Assembler::LT);
15871     __ bind(done);
15872   %}
15873   ins_pipe(pipe_class_default);
15874 
15875 %}
15876 
15877 instruct cmpLTMask_reg_reg(iRegINoSp dst, iRegIorL2I p, iRegIorL2I q, rFlagsReg cr)
15878 %{
15879   match(Set dst (CmpLTMask p q));
15880   effect(KILL cr);
15881 
15882   ins_cost(3 * INSN_COST);
15883 
15884   format %{ "cmpw $p, $q\t# cmpLTMask\n\t"
15885             "csetw $dst, lt\n\t"
15886             "subw $dst, zr, $dst"
15887   %}
15888 
15889   ins_encode %{
15890     __ cmpw(as_Register($p$$reg), as_Register($q$$reg));
15891     __ csetw(as_Register($dst$$reg), Assembler::LT);
15892     __ subw(as_Register($dst$$reg), zr, as_Register($dst$$reg));
15893   %}
15894 
15895   ins_pipe(ialu_reg_reg);
15896 %}
15897 
15898 instruct cmpLTMask_reg_zero(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr)
15899 %{
15900   match(Set dst (CmpLTMask src zero));
15901   effect(KILL cr);
15902 
15903   ins_cost(INSN_COST);
15904 
15905   format %{ "asrw $dst, $src, #31\t# cmpLTMask0" %}
15906 
15907   ins_encode %{
15908     __ asrw(as_Register($dst$$reg), as_Register($src$$reg), 31);
15909   %}
15910 
15911   ins_pipe(ialu_reg_shift);
15912 %}
15913 
15914 // ============================================================================
15915 // Max and Min
15916 
15917 // Like compI_reg_reg or compI_reg_immI0 but without match rule and second zero parameter.
15918 
15919 instruct compI_reg_imm0(rFlagsReg cr, iRegI src)
15920 %{
15921   effect(DEF cr, USE src);
15922   ins_cost(INSN_COST);
15923   format %{ "cmpw $src, 0" %}
15924 
15925   ins_encode %{
15926     __ cmpw($src$$Register, 0);
15927   %}
15928   ins_pipe(icmp_reg_imm);
15929 %}
15930 
15931 instruct minI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
15932 %{
15933   match(Set dst (MinI src1 src2));
15934   ins_cost(INSN_COST * 3);
15935 
15936   expand %{
15937     rFlagsReg cr;
15938     compI_reg_reg(cr, src1, src2);
15939     cmovI_reg_reg_lt(dst, src1, src2, cr);
15940   %}
15941 %}
15942 
15943 instruct maxI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
15944 %{
15945   match(Set dst (MaxI src1 src2));
15946   ins_cost(INSN_COST * 3);
15947 
15948   expand %{
15949     rFlagsReg cr;
15950     compI_reg_reg(cr, src1, src2);
15951     cmovI_reg_reg_gt(dst, src1, src2, cr);
15952   %}
15953 %}
15954 
15955 
15956 // ============================================================================
15957 // Branch Instructions
15958 
15959 // Direct Branch.
15960 instruct branch(label lbl)
15961 %{
15962   match(Goto);
15963 
15964   effect(USE lbl);
15965 
15966   ins_cost(BRANCH_COST);
15967   format %{ "b  $lbl" %}
15968 
15969   ins_encode(aarch64_enc_b(lbl));
15970 
15971   ins_pipe(pipe_branch);
15972 %}
15973 
15974 // Conditional Near Branch
15975 instruct branchCon(cmpOp cmp, rFlagsReg cr, label lbl)
15976 %{
15977   // Same match rule as `branchConFar'.
15978   match(If cmp cr);
15979 
15980   effect(USE lbl);
15981 
15982   ins_cost(BRANCH_COST);
15983   // If set to 1 this indicates that the current instruction is a
15984   // short variant of a long branch. This avoids using this
15985   // instruction in first-pass matching. It will then only be used in
15986   // the `Shorten_branches' pass.
15987   // ins_short_branch(1);
15988   format %{ "b$cmp  $lbl" %}
15989 
15990   ins_encode(aarch64_enc_br_con(cmp, lbl));
15991 
15992   ins_pipe(pipe_branch_cond);
15993 %}
15994 
15995 // Conditional Near Branch Unsigned
15996 instruct branchConU(cmpOpU cmp, rFlagsRegU cr, label lbl)
15997 %{
15998   // Same match rule as `branchConFar'.
15999   match(If cmp cr);
16000 
16001   effect(USE lbl);
16002 
16003   ins_cost(BRANCH_COST);
16004   // If set to 1 this indicates that the current instruction is a
16005   // short variant of a long branch. This avoids using this
16006   // instruction in first-pass matching. It will then only be used in
16007   // the `Shorten_branches' pass.
16008   // ins_short_branch(1);
16009   format %{ "b$cmp  $lbl\t# unsigned" %}
16010 
16011   ins_encode(aarch64_enc_br_conU(cmp, lbl));
16012 
16013   ins_pipe(pipe_branch_cond);
16014 %}
16015 
16016 // Make use of CBZ and CBNZ.  These instructions, as well as being
16017 // shorter than (cmp; branch), have the additional benefit of not
16018 // killing the flags.
16019 
16020 instruct cmpI_imm0_branch(cmpOpEqNe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsReg cr) %{
16021   match(If cmp (CmpI op1 op2));
16022   effect(USE labl);
16023 
16024   ins_cost(BRANCH_COST);
16025   format %{ "cbw$cmp   $op1, $labl" %}
16026   ins_encode %{
16027     Label* L = $labl$$label;
16028     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16029     if (cond == Assembler::EQ)
16030       __ cbzw($op1$$Register, *L);
16031     else
16032       __ cbnzw($op1$$Register, *L);
16033   %}
16034   ins_pipe(pipe_cmp_branch);
16035 %}
16036 
16037 instruct cmpL_imm0_branch(cmpOpEqNe cmp, iRegL op1, immL0 op2, label labl, rFlagsReg cr) %{
16038   match(If cmp (CmpL op1 op2));
16039   effect(USE labl);
16040 
16041   ins_cost(BRANCH_COST);
16042   format %{ "cb$cmp   $op1, $labl" %}
16043   ins_encode %{
16044     Label* L = $labl$$label;
16045     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16046     if (cond == Assembler::EQ)
16047       __ cbz($op1$$Register, *L);
16048     else
16049       __ cbnz($op1$$Register, *L);
16050   %}
16051   ins_pipe(pipe_cmp_branch);
16052 %}
16053 
16054 instruct cmpP_imm0_branch(cmpOpEqNe cmp, iRegP op1, immP0 op2, label labl, rFlagsReg cr) %{
16055   match(If cmp (CmpP op1 op2));
16056   effect(USE labl);
16057 
16058   ins_cost(BRANCH_COST);
16059   format %{ "cb$cmp   $op1, $labl" %}
16060   ins_encode %{
16061     Label* L = $labl$$label;
16062     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16063     if (cond == Assembler::EQ)
16064       __ cbz($op1$$Register, *L);
16065     else
16066       __ cbnz($op1$$Register, *L);
16067   %}
16068   ins_pipe(pipe_cmp_branch);
16069 %}
16070 
16071 instruct cmpN_imm0_branch(cmpOpEqNe cmp, iRegN op1, immN0 op2, label labl, rFlagsReg cr) %{
16072   match(If cmp (CmpN op1 op2));
16073   effect(USE labl);
16074 
16075   ins_cost(BRANCH_COST);
16076   format %{ "cbw$cmp   $op1, $labl" %}
16077   ins_encode %{
16078     Label* L = $labl$$label;
16079     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16080     if (cond == Assembler::EQ)
16081       __ cbzw($op1$$Register, *L);
16082     else
16083       __ cbnzw($op1$$Register, *L);
16084   %}
16085   ins_pipe(pipe_cmp_branch);
16086 %}
16087 
16088 instruct cmpP_narrowOop_imm0_branch(cmpOpEqNe cmp, iRegN oop, immP0 zero, label labl, rFlagsReg cr) %{
16089   match(If cmp (CmpP (DecodeN oop) zero));
16090   effect(USE labl);
16091 
16092   ins_cost(BRANCH_COST);
16093   format %{ "cb$cmp   $oop, $labl" %}
16094   ins_encode %{
16095     Label* L = $labl$$label;
16096     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16097     if (cond == Assembler::EQ)
16098       __ cbzw($oop$$Register, *L);
16099     else
16100       __ cbnzw($oop$$Register, *L);
16101   %}
16102   ins_pipe(pipe_cmp_branch);
16103 %}
16104 
16105 instruct cmpUI_imm0_branch(cmpOpUEqNeLeGt cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16106   match(If cmp (CmpU op1 op2));
16107   effect(USE labl);
16108 
16109   ins_cost(BRANCH_COST);
16110   format %{ "cbw$cmp   $op1, $labl" %}
16111   ins_encode %{
16112     Label* L = $labl$$label;
16113     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16114     if (cond == Assembler::EQ || cond == Assembler::LS) {
16115       __ cbzw($op1$$Register, *L);
16116     } else {
16117       assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
16118       __ cbnzw($op1$$Register, *L);
16119     }
16120   %}
16121   ins_pipe(pipe_cmp_branch);
16122 %}
16123 
16124 instruct cmpUL_imm0_branch(cmpOpUEqNeLeGt cmp, iRegL op1, immL0 op2, label labl) %{
16125   match(If cmp (CmpUL op1 op2));
16126   effect(USE labl);
16127 
16128   ins_cost(BRANCH_COST);
16129   format %{ "cb$cmp   $op1, $labl" %}
16130   ins_encode %{
16131     Label* L = $labl$$label;
16132     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16133     if (cond == Assembler::EQ || cond == Assembler::LS) {
16134       __ cbz($op1$$Register, *L);
16135     } else {
16136       assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
16137       __ cbnz($op1$$Register, *L);
16138     }
16139   %}
16140   ins_pipe(pipe_cmp_branch);
16141 %}
16142 
16143 // Test bit and Branch
16144 
16145 // Patterns for short (< 32KiB) variants
16146 instruct cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
16147   match(If cmp (CmpL op1 op2));
16148   effect(USE labl);
16149 
16150   ins_cost(BRANCH_COST);
16151   format %{ "cb$cmp   $op1, $labl # long" %}
16152   ins_encode %{
16153     Label* L = $labl$$label;
16154     Assembler::Condition cond =
16155       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16156     __ tbr(cond, $op1$$Register, 63, *L);
16157   %}
16158   ins_pipe(pipe_cmp_branch);
16159   ins_short_branch(1);
16160 %}
16161 
16162 instruct cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16163   match(If cmp (CmpI op1 op2));
16164   effect(USE labl);
16165 
16166   ins_cost(BRANCH_COST);
16167   format %{ "cb$cmp   $op1, $labl # int" %}
16168   ins_encode %{
16169     Label* L = $labl$$label;
16170     Assembler::Condition cond =
16171       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16172     __ tbr(cond, $op1$$Register, 31, *L);
16173   %}
16174   ins_pipe(pipe_cmp_branch);
16175   ins_short_branch(1);
16176 %}
16177 
16178 instruct cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
16179   match(If cmp (CmpL (AndL op1 op2) op3));
16180   predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
16181   effect(USE labl);
16182 
16183   ins_cost(BRANCH_COST);
16184   format %{ "tb$cmp   $op1, $op2, $labl" %}
16185   ins_encode %{
16186     Label* L = $labl$$label;
16187     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16188     int bit = exact_log2_long($op2$$constant);
16189     __ tbr(cond, $op1$$Register, bit, *L);
16190   %}
16191   ins_pipe(pipe_cmp_branch);
16192   ins_short_branch(1);
16193 %}
16194 
16195 instruct cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
16196   match(If cmp (CmpI (AndI op1 op2) op3));
16197   predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
16198   effect(USE labl);
16199 
16200   ins_cost(BRANCH_COST);
16201   format %{ "tb$cmp   $op1, $op2, $labl" %}
16202   ins_encode %{
16203     Label* L = $labl$$label;
16204     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16205     int bit = exact_log2((juint)$op2$$constant);
16206     __ tbr(cond, $op1$$Register, bit, *L);
16207   %}
16208   ins_pipe(pipe_cmp_branch);
16209   ins_short_branch(1);
16210 %}
16211 
16212 // And far variants
16213 instruct far_cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
16214   match(If cmp (CmpL op1 op2));
16215   effect(USE labl);
16216 
16217   ins_cost(BRANCH_COST);
16218   format %{ "cb$cmp   $op1, $labl # long" %}
16219   ins_encode %{
16220     Label* L = $labl$$label;
16221     Assembler::Condition cond =
16222       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16223     __ tbr(cond, $op1$$Register, 63, *L, /*far*/true);
16224   %}
16225   ins_pipe(pipe_cmp_branch);
16226 %}
16227 
16228 instruct far_cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16229   match(If cmp (CmpI op1 op2));
16230   effect(USE labl);
16231 
16232   ins_cost(BRANCH_COST);
16233   format %{ "cb$cmp   $op1, $labl # int" %}
16234   ins_encode %{
16235     Label* L = $labl$$label;
16236     Assembler::Condition cond =
16237       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16238     __ tbr(cond, $op1$$Register, 31, *L, /*far*/true);
16239   %}
16240   ins_pipe(pipe_cmp_branch);
16241 %}
16242 
16243 instruct far_cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
16244   match(If cmp (CmpL (AndL op1 op2) op3));
16245   predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
16246   effect(USE labl);
16247 
16248   ins_cost(BRANCH_COST);
16249   format %{ "tb$cmp   $op1, $op2, $labl" %}
16250   ins_encode %{
16251     Label* L = $labl$$label;
16252     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16253     int bit = exact_log2_long($op2$$constant);
16254     __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
16255   %}
16256   ins_pipe(pipe_cmp_branch);
16257 %}
16258 
16259 instruct far_cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
16260   match(If cmp (CmpI (AndI op1 op2) op3));
16261   predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
16262   effect(USE labl);
16263 
16264   ins_cost(BRANCH_COST);
16265   format %{ "tb$cmp   $op1, $op2, $labl" %}
16266   ins_encode %{
16267     Label* L = $labl$$label;
16268     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16269     int bit = exact_log2((juint)$op2$$constant);
16270     __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
16271   %}
16272   ins_pipe(pipe_cmp_branch);
16273 %}
16274 
16275 // Test bits
16276 
16277 instruct cmpL_and(cmpOp cmp, iRegL op1, immL op2, immL0 op3, rFlagsReg cr) %{
16278   match(Set cr (CmpL (AndL op1 op2) op3));
16279   predicate(Assembler::operand_valid_for_logical_immediate
16280             (/*is_32*/false, n->in(1)->in(2)->get_long()));
16281 
16282   ins_cost(INSN_COST);
16283   format %{ "tst $op1, $op2 # long" %}
16284   ins_encode %{
16285     __ tst($op1$$Register, $op2$$constant);
16286   %}
16287   ins_pipe(ialu_reg_reg);
16288 %}
16289 
16290 instruct cmpI_and(cmpOp cmp, iRegIorL2I op1, immI op2, immI0 op3, rFlagsReg cr) %{
16291   match(Set cr (CmpI (AndI op1 op2) op3));
16292   predicate(Assembler::operand_valid_for_logical_immediate
16293             (/*is_32*/true, n->in(1)->in(2)->get_int()));
16294 
16295   ins_cost(INSN_COST);
16296   format %{ "tst $op1, $op2 # int" %}
16297   ins_encode %{
16298     __ tstw($op1$$Register, $op2$$constant);
16299   %}
16300   ins_pipe(ialu_reg_reg);
16301 %}
16302 
16303 instruct cmpL_and_reg(cmpOp cmp, iRegL op1, iRegL op2, immL0 op3, rFlagsReg cr) %{
16304   match(Set cr (CmpL (AndL op1 op2) op3));
16305 
16306   ins_cost(INSN_COST);
16307   format %{ "tst $op1, $op2 # long" %}
16308   ins_encode %{
16309     __ tst($op1$$Register, $op2$$Register);
16310   %}
16311   ins_pipe(ialu_reg_reg);
16312 %}
16313 
16314 instruct cmpI_and_reg(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, immI0 op3, rFlagsReg cr) %{
16315   match(Set cr (CmpI (AndI op1 op2) op3));
16316 
16317   ins_cost(INSN_COST);
16318   format %{ "tstw $op1, $op2 # int" %}
16319   ins_encode %{
16320     __ tstw($op1$$Register, $op2$$Register);
16321   %}
16322   ins_pipe(ialu_reg_reg);
16323 %}
16324 
16325 
16326 // Conditional Far Branch
16327 // Conditional Far Branch Unsigned
16328 // TODO: fixme
16329 
16330 // counted loop end branch near
16331 instruct branchLoopEnd(cmpOp cmp, rFlagsReg cr, label lbl)
16332 %{
16333   match(CountedLoopEnd cmp cr);
16334 
16335   effect(USE lbl);
16336 
16337   ins_cost(BRANCH_COST);
16338   // short variant.
16339   // ins_short_branch(1);
16340   format %{ "b$cmp $lbl \t// counted loop end" %}
16341 
16342   ins_encode(aarch64_enc_br_con(cmp, lbl));
16343 
16344   ins_pipe(pipe_branch);
16345 %}
16346 
16347 // counted loop end branch far
16348 // TODO: fixme
16349 
16350 // ============================================================================
16351 // inlined locking and unlocking
16352 
16353 instruct cmpFastLockLightweight(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
16354 %{
16355   match(Set cr (FastLock object box));
16356   effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
16357 
16358   ins_cost(5 * INSN_COST);
16359   format %{ "fastlock $object,$box\t! kills $tmp,$tmp2,$tmp3" %}
16360 
16361   ins_encode %{
16362     __ fast_lock_lightweight($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
16363   %}
16364 
16365   ins_pipe(pipe_serial);
16366 %}
16367 
16368 instruct cmpFastUnlockLightweight(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
16369 %{
16370   match(Set cr (FastUnlock object box));
16371   effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
16372 
16373   ins_cost(5 * INSN_COST);
16374   format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2, $tmp3" %}
16375 
16376   ins_encode %{
16377     __ fast_unlock_lightweight($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
16378   %}
16379 
16380   ins_pipe(pipe_serial);
16381 %}
16382 
16383 // ============================================================================
16384 // Safepoint Instructions
16385 
16386 // TODO
16387 // provide a near and far version of this code
16388 
16389 instruct safePoint(rFlagsReg cr, iRegP poll)
16390 %{
16391   match(SafePoint poll);
16392   effect(KILL cr);
16393 
16394   format %{
16395     "ldrw zr, [$poll]\t# Safepoint: poll for GC"
16396   %}
16397   ins_encode %{
16398     __ read_polling_page(as_Register($poll$$reg), relocInfo::poll_type);
16399   %}
16400   ins_pipe(pipe_serial); // ins_pipe(iload_reg_mem);
16401 %}
16402 
16403 
16404 // ============================================================================
16405 // Procedure Call/Return Instructions
16406 
16407 // Call Java Static Instruction
16408 
16409 instruct CallStaticJavaDirect(method meth)
16410 %{
16411   match(CallStaticJava);
16412 
16413   effect(USE meth);
16414 
16415   ins_cost(CALL_COST);
16416 
16417   format %{ "call,static $meth \t// ==> " %}
16418 
16419   ins_encode(aarch64_enc_java_static_call(meth),
16420              aarch64_enc_call_epilog);
16421 
16422   ins_pipe(pipe_class_call);
16423 %}
16424 
16425 // TO HERE
16426 
16427 // Call Java Dynamic Instruction
16428 instruct CallDynamicJavaDirect(method meth)
16429 %{
16430   match(CallDynamicJava);
16431 
16432   effect(USE meth);
16433 
16434   ins_cost(CALL_COST);
16435 
16436   format %{ "CALL,dynamic $meth \t// ==> " %}
16437 
16438   ins_encode(aarch64_enc_java_dynamic_call(meth),
16439              aarch64_enc_call_epilog);
16440 
16441   ins_pipe(pipe_class_call);
16442 %}
16443 
16444 // Call Runtime Instruction
16445 
16446 instruct CallRuntimeDirect(method meth)
16447 %{
16448   match(CallRuntime);
16449 
16450   effect(USE meth);
16451 
16452   ins_cost(CALL_COST);
16453 
16454   format %{ "CALL, runtime $meth" %}
16455 
16456   ins_encode( aarch64_enc_java_to_runtime(meth) );
16457 
16458   ins_pipe(pipe_class_call);
16459 %}
16460 
16461 // Call Runtime Instruction
16462 
16463 instruct CallLeafDirect(method meth)
16464 %{
16465   match(CallLeaf);
16466 
16467   effect(USE meth);
16468 
16469   ins_cost(CALL_COST);
16470 
16471   format %{ "CALL, runtime leaf $meth" %}
16472 
16473   ins_encode( aarch64_enc_java_to_runtime(meth) );
16474 
16475   ins_pipe(pipe_class_call);
16476 %}
16477 
16478 // Call Runtime Instruction without safepoint and with vector arguments
16479 instruct CallLeafDirectVector(method meth)
16480 %{
16481   match(CallLeafVector);
16482 
16483   effect(USE meth);
16484 
16485   ins_cost(CALL_COST);
16486 
16487   format %{ "CALL, runtime leaf vector $meth" %}
16488 
16489   ins_encode(aarch64_enc_java_to_runtime(meth));
16490 
16491   ins_pipe(pipe_class_call);
16492 %}
16493 
16494 // Call Runtime Instruction
16495 
16496 // entry point is null, target holds the address to call
16497 instruct CallLeafNoFPIndirect(iRegP target)
16498 %{
16499   predicate(n->as_Call()->entry_point() == nullptr);
16500 
16501   match(CallLeafNoFP target);
16502 
16503   ins_cost(CALL_COST);
16504 
16505   format %{ "CALL, runtime leaf nofp indirect $target" %}
16506 
16507   ins_encode %{
16508     __ blr($target$$Register);
16509   %}
16510 
16511   ins_pipe(pipe_class_call);
16512 %}
16513 
16514 instruct CallLeafNoFPDirect(method meth)
16515 %{
16516   predicate(n->as_Call()->entry_point() != nullptr);
16517 
16518   match(CallLeafNoFP);
16519 
16520   effect(USE meth);
16521 
16522   ins_cost(CALL_COST);
16523 
16524   format %{ "CALL, runtime leaf nofp $meth" %}
16525 
16526   ins_encode( aarch64_enc_java_to_runtime(meth) );
16527 
16528   ins_pipe(pipe_class_call);
16529 %}
16530 
16531 // Tail Call; Jump from runtime stub to Java code.
16532 // Also known as an 'interprocedural jump'.
16533 // Target of jump will eventually return to caller.
16534 // TailJump below removes the return address.
16535 // Don't use rfp for 'jump_target' because a MachEpilogNode has already been
16536 // emitted just above the TailCall which has reset rfp to the caller state.
16537 instruct TailCalljmpInd(iRegPNoSpNoRfp jump_target, inline_cache_RegP method_ptr)
16538 %{
16539   match(TailCall jump_target method_ptr);
16540 
16541   ins_cost(CALL_COST);
16542 
16543   format %{ "br $jump_target\t# $method_ptr holds method" %}
16544 
16545   ins_encode(aarch64_enc_tail_call(jump_target));
16546 
16547   ins_pipe(pipe_class_call);
16548 %}
16549 
16550 instruct TailjmpInd(iRegPNoSpNoRfp jump_target, iRegP_R0 ex_oop)
16551 %{
16552   match(TailJump jump_target ex_oop);
16553 
16554   ins_cost(CALL_COST);
16555 
16556   format %{ "br $jump_target\t# $ex_oop holds exception oop" %}
16557 
16558   ins_encode(aarch64_enc_tail_jmp(jump_target));
16559 
16560   ins_pipe(pipe_class_call);
16561 %}
16562 
16563 // Forward exception.
16564 instruct ForwardExceptionjmp()
16565 %{
16566   match(ForwardException);
16567   ins_cost(CALL_COST);
16568 
16569   format %{ "b forward_exception_stub" %}
16570   ins_encode %{
16571     __ far_jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
16572   %}
16573   ins_pipe(pipe_class_call);
16574 %}
16575 
16576 // Create exception oop: created by stack-crawling runtime code.
16577 // Created exception is now available to this handler, and is setup
16578 // just prior to jumping to this handler. No code emitted.
16579 // TODO check
16580 // should ex_oop be in r0? intel uses rax, ppc cannot use r0 so uses rarg1
16581 instruct CreateException(iRegP_R0 ex_oop)
16582 %{
16583   match(Set ex_oop (CreateEx));
16584 
16585   format %{ " -- \t// exception oop; no code emitted" %}
16586 
16587   size(0);
16588 
16589   ins_encode( /*empty*/ );
16590 
16591   ins_pipe(pipe_class_empty);
16592 %}
16593 
16594 // Rethrow exception: The exception oop will come in the first
16595 // argument position. Then JUMP (not call) to the rethrow stub code.
16596 instruct RethrowException() %{
16597   match(Rethrow);
16598   ins_cost(CALL_COST);
16599 
16600   format %{ "b rethrow_stub" %}
16601 
16602   ins_encode( aarch64_enc_rethrow() );
16603 
16604   ins_pipe(pipe_class_call);
16605 %}
16606 
16607 
16608 // Return Instruction
16609 // epilog node loads ret address into lr as part of frame pop
16610 instruct Ret()
16611 %{
16612   match(Return);
16613 
16614   format %{ "ret\t// return register" %}
16615 
16616   ins_encode( aarch64_enc_ret() );
16617 
16618   ins_pipe(pipe_branch);
16619 %}
16620 
16621 // Die now.
16622 instruct ShouldNotReachHere() %{
16623   match(Halt);
16624 
16625   ins_cost(CALL_COST);
16626   format %{ "ShouldNotReachHere" %}
16627 
16628   ins_encode %{
16629     if (is_reachable()) {
16630       const char* str = __ code_string(_halt_reason);
16631       __ stop(str);
16632     }
16633   %}
16634 
16635   ins_pipe(pipe_class_default);
16636 %}
16637 
16638 // ============================================================================
16639 // Partial Subtype Check
16640 //
16641 // superklass array for an instance of the superklass.  Set a hidden
16642 // internal cache on a hit (cache is checked with exposed code in
16643 // gen_subtype_check()).  Return NZ for a miss or zero for a hit.  The
16644 // encoding ALSO sets flags.
16645 
16646 instruct partialSubtypeCheck(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, rFlagsReg cr)
16647 %{
16648   match(Set result (PartialSubtypeCheck sub super));
16649   predicate(!UseSecondarySupersTable);
16650   effect(KILL cr, KILL temp);
16651 
16652   ins_cost(20 * INSN_COST);  // slightly larger than the next version
16653   format %{ "partialSubtypeCheck $result, $sub, $super" %}
16654 
16655   ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
16656 
16657   opcode(0x1); // Force zero of result reg on hit
16658 
16659   ins_pipe(pipe_class_memory);
16660 %}
16661 
16662 // Two versions of partialSubtypeCheck, both used when we need to
16663 // search for a super class in the secondary supers array. The first
16664 // is used when we don't know _a priori_ the class being searched
16665 // for. The second, far more common, is used when we do know: this is
16666 // used for instanceof, checkcast, and any case where C2 can determine
16667 // it by constant propagation.
16668 
16669 instruct partialSubtypeCheckVarSuper(iRegP_R4 sub, iRegP_R0 super, vRegD_V0 vtemp, iRegP_R5 result,
16670                                      iRegP_R1 tempR1, iRegP_R2 tempR2, iRegP_R3 tempR3,
16671                                      rFlagsReg cr)
16672 %{
16673   match(Set result (PartialSubtypeCheck sub super));
16674   predicate(UseSecondarySupersTable);
16675   effect(KILL cr, TEMP tempR1, TEMP tempR2, TEMP tempR3, TEMP vtemp);
16676 
16677   ins_cost(10 * INSN_COST);  // slightly larger than the next version
16678   format %{ "partialSubtypeCheck $result, $sub, $super" %}
16679 
16680   ins_encode %{
16681     __ lookup_secondary_supers_table_var($sub$$Register, $super$$Register,
16682                                          $tempR1$$Register, $tempR2$$Register, $tempR3$$Register,
16683                                          $vtemp$$FloatRegister,
16684                                          $result$$Register, /*L_success*/nullptr);
16685   %}
16686 
16687   ins_pipe(pipe_class_memory);
16688 %}
16689 
16690 instruct partialSubtypeCheckConstSuper(iRegP_R4 sub, iRegP_R0 super_reg, immP super_con, vRegD_V0 vtemp, iRegP_R5 result,
16691                                        iRegP_R1 tempR1, iRegP_R2 tempR2, iRegP_R3 tempR3,
16692                                        rFlagsReg cr)
16693 %{
16694   match(Set result (PartialSubtypeCheck sub (Binary super_reg super_con)));
16695   predicate(UseSecondarySupersTable);
16696   effect(KILL cr, TEMP tempR1, TEMP tempR2, TEMP tempR3, TEMP vtemp);
16697 
16698   ins_cost(5 * INSN_COST);  // smaller than the next version
16699   format %{ "partialSubtypeCheck $result, $sub, $super_reg, $super_con" %}
16700 
16701   ins_encode %{
16702     bool success = false;
16703     u1 super_klass_slot = ((Klass*)$super_con$$constant)->hash_slot();
16704     if (InlineSecondarySupersTest) {
16705       success =
16706         __ lookup_secondary_supers_table_const($sub$$Register, $super_reg$$Register,
16707                                                $tempR1$$Register, $tempR2$$Register, $tempR3$$Register,
16708                                                $vtemp$$FloatRegister,
16709                                                $result$$Register,
16710                                                super_klass_slot);
16711     } else {
16712       address call = __ trampoline_call(RuntimeAddress(StubRoutines::lookup_secondary_supers_table_stub(super_klass_slot)));
16713       success = (call != nullptr);
16714     }
16715     if (!success) {
16716       ciEnv::current()->record_failure("CodeCache is full");
16717       return;
16718     }
16719   %}
16720 
16721   ins_pipe(pipe_class_memory);
16722 %}
16723 
16724 // Intrisics for String.compareTo()
16725 
16726 instruct string_compareU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16727                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16728 %{
16729   predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
16730   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16731   effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16732 
16733   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1" %}
16734   ins_encode %{
16735     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16736     __ string_compare($str1$$Register, $str2$$Register,
16737                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16738                       $tmp1$$Register, $tmp2$$Register,
16739                       fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::UU);
16740   %}
16741   ins_pipe(pipe_class_memory);
16742 %}
16743 
16744 instruct string_compareL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16745                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16746 %{
16747   predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
16748   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16749   effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16750 
16751   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1" %}
16752   ins_encode %{
16753     __ string_compare($str1$$Register, $str2$$Register,
16754                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16755                       $tmp1$$Register, $tmp2$$Register,
16756                       fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::LL);
16757   %}
16758   ins_pipe(pipe_class_memory);
16759 %}
16760 
16761 instruct string_compareUL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16762                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16763                         vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16764 %{
16765   predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
16766   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16767   effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16768          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16769 
16770   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16771   ins_encode %{
16772     __ string_compare($str1$$Register, $str2$$Register,
16773                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16774                       $tmp1$$Register, $tmp2$$Register,
16775                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16776                       $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::UL);
16777   %}
16778   ins_pipe(pipe_class_memory);
16779 %}
16780 
16781 instruct string_compareLU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16782                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16783                         vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16784 %{
16785   predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
16786   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16787   effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16788          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16789 
16790   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16791   ins_encode %{
16792     __ string_compare($str1$$Register, $str2$$Register,
16793                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16794                       $tmp1$$Register, $tmp2$$Register,
16795                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16796                       $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::LU);
16797   %}
16798   ins_pipe(pipe_class_memory);
16799 %}
16800 
16801 // Note that Z registers alias the corresponding NEON registers, we declare the vector operands of
16802 // these string_compare variants as NEON register type for convenience so that the prototype of
16803 // string_compare can be shared with all variants.
16804 
16805 instruct string_compareLL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16806                               iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16807                               vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16808                               pRegGov_P1 pgtmp2, rFlagsReg cr)
16809 %{
16810   predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
16811   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16812   effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16813          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16814 
16815   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # USE sve" %}
16816   ins_encode %{
16817     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16818     __ string_compare($str1$$Register, $str2$$Register,
16819                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16820                       $tmp1$$Register, $tmp2$$Register,
16821                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16822                       as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16823                       StrIntrinsicNode::LL);
16824   %}
16825   ins_pipe(pipe_class_memory);
16826 %}
16827 
16828 instruct string_compareLU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16829                               iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16830                               vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16831                               pRegGov_P1 pgtmp2, rFlagsReg cr)
16832 %{
16833   predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
16834   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16835   effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16836          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16837 
16838   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # USE sve" %}
16839   ins_encode %{
16840     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16841     __ string_compare($str1$$Register, $str2$$Register,
16842                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16843                       $tmp1$$Register, $tmp2$$Register,
16844                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16845                       as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16846                       StrIntrinsicNode::LU);
16847   %}
16848   ins_pipe(pipe_class_memory);
16849 %}
16850 
16851 instruct string_compareUL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16852                               iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16853                               vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16854                               pRegGov_P1 pgtmp2, rFlagsReg cr)
16855 %{
16856   predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
16857   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16858   effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16859          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16860 
16861   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # USE sve" %}
16862   ins_encode %{
16863     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16864     __ string_compare($str1$$Register, $str2$$Register,
16865                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16866                       $tmp1$$Register, $tmp2$$Register,
16867                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16868                       as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16869                       StrIntrinsicNode::UL);
16870   %}
16871   ins_pipe(pipe_class_memory);
16872 %}
16873 
16874 instruct string_compareUU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16875                               iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16876                               vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16877                               pRegGov_P1 pgtmp2, rFlagsReg cr)
16878 %{
16879   predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
16880   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16881   effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16882          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16883 
16884   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # USE sve" %}
16885   ins_encode %{
16886     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16887     __ string_compare($str1$$Register, $str2$$Register,
16888                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16889                       $tmp1$$Register, $tmp2$$Register,
16890                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16891                       as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16892                       StrIntrinsicNode::UU);
16893   %}
16894   ins_pipe(pipe_class_memory);
16895 %}
16896 
16897 instruct string_indexofUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16898                           iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16899                           iRegINoSp tmp3, iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
16900                           vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
16901 %{
16902   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16903   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16904   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16905          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
16906          TEMP vtmp0, TEMP vtmp1, KILL cr);
16907   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UU) "
16908             "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
16909 
16910   ins_encode %{
16911     __ string_indexof($str1$$Register, $str2$$Register,
16912                       $cnt1$$Register, $cnt2$$Register,
16913                       $tmp1$$Register, $tmp2$$Register,
16914                       $tmp3$$Register, $tmp4$$Register,
16915                       $tmp5$$Register, $tmp6$$Register,
16916                       -1, $result$$Register, StrIntrinsicNode::UU);
16917   %}
16918   ins_pipe(pipe_class_memory);
16919 %}
16920 
16921 instruct string_indexofLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16922                           iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
16923                           iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
16924                           vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
16925 %{
16926   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16927   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16928   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16929          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
16930          TEMP vtmp0, TEMP vtmp1, KILL cr);
16931   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (LL) "
16932             "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
16933 
16934   ins_encode %{
16935     __ string_indexof($str1$$Register, $str2$$Register,
16936                       $cnt1$$Register, $cnt2$$Register,
16937                       $tmp1$$Register, $tmp2$$Register,
16938                       $tmp3$$Register, $tmp4$$Register,
16939                       $tmp5$$Register, $tmp6$$Register,
16940                       -1, $result$$Register, StrIntrinsicNode::LL);
16941   %}
16942   ins_pipe(pipe_class_memory);
16943 %}
16944 
16945 instruct string_indexofUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16946                           iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,iRegINoSp tmp3,
16947                           iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
16948                           vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
16949 %{
16950   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16951   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16952   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16953          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5,
16954          TEMP tmp6, TEMP vtmp0, TEMP vtmp1, KILL cr);
16955   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UL) "
16956             "# KILL $str1 cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
16957 
16958   ins_encode %{
16959     __ string_indexof($str1$$Register, $str2$$Register,
16960                       $cnt1$$Register, $cnt2$$Register,
16961                       $tmp1$$Register, $tmp2$$Register,
16962                       $tmp3$$Register, $tmp4$$Register,
16963                       $tmp5$$Register, $tmp6$$Register,
16964                       -1, $result$$Register, StrIntrinsicNode::UL);
16965   %}
16966   ins_pipe(pipe_class_memory);
16967 %}
16968 
16969 instruct string_indexof_conUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16970                               immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16971                               iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16972 %{
16973   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16974   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16975   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16976          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16977   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UU) "
16978             "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16979 
16980   ins_encode %{
16981     int icnt2 = (int)$int_cnt2$$constant;
16982     __ string_indexof($str1$$Register, $str2$$Register,
16983                       $cnt1$$Register, zr,
16984                       $tmp1$$Register, $tmp2$$Register,
16985                       $tmp3$$Register, $tmp4$$Register, zr, zr,
16986                       icnt2, $result$$Register, StrIntrinsicNode::UU);
16987   %}
16988   ins_pipe(pipe_class_memory);
16989 %}
16990 
16991 instruct string_indexof_conLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16992                               immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16993                               iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16994 %{
16995   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16996   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16997   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16998          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16999   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (LL) "
17000             "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
17001 
17002   ins_encode %{
17003     int icnt2 = (int)$int_cnt2$$constant;
17004     __ string_indexof($str1$$Register, $str2$$Register,
17005                       $cnt1$$Register, zr,
17006                       $tmp1$$Register, $tmp2$$Register,
17007                       $tmp3$$Register, $tmp4$$Register, zr, zr,
17008                       icnt2, $result$$Register, StrIntrinsicNode::LL);
17009   %}
17010   ins_pipe(pipe_class_memory);
17011 %}
17012 
17013 instruct string_indexof_conUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
17014                               immI_1 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
17015                               iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
17016 %{
17017   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
17018   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
17019   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
17020          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
17021   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UL) "
17022             "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
17023 
17024   ins_encode %{
17025     int icnt2 = (int)$int_cnt2$$constant;
17026     __ string_indexof($str1$$Register, $str2$$Register,
17027                       $cnt1$$Register, zr,
17028                       $tmp1$$Register, $tmp2$$Register,
17029                       $tmp3$$Register, $tmp4$$Register, zr, zr,
17030                       icnt2, $result$$Register, StrIntrinsicNode::UL);
17031   %}
17032   ins_pipe(pipe_class_memory);
17033 %}
17034 
17035 instruct string_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
17036                              iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
17037                              iRegINoSp tmp3, rFlagsReg cr)
17038 %{
17039   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
17040   predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
17041   effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
17042          TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
17043 
17044   format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
17045 
17046   ins_encode %{
17047     __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
17048                            $result$$Register, $tmp1$$Register, $tmp2$$Register,
17049                            $tmp3$$Register);
17050   %}
17051   ins_pipe(pipe_class_memory);
17052 %}
17053 
17054 instruct stringL_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
17055                               iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
17056                               iRegINoSp tmp3, rFlagsReg cr)
17057 %{
17058   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
17059   predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
17060   effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
17061          TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
17062 
17063   format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
17064 
17065   ins_encode %{
17066     __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
17067                             $result$$Register, $tmp1$$Register, $tmp2$$Register,
17068                             $tmp3$$Register);
17069   %}
17070   ins_pipe(pipe_class_memory);
17071 %}
17072 
17073 instruct stringL_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
17074                                   iRegI_R0 result, vecA ztmp1, vecA ztmp2,
17075                                   pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
17076   predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L);
17077   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
17078   effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
17079   format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
17080   ins_encode %{
17081     __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
17082                                $result$$Register, $ztmp1$$FloatRegister,
17083                                $ztmp2$$FloatRegister, $pgtmp$$PRegister,
17084                                $ptmp$$PRegister, true /* isL */);
17085   %}
17086   ins_pipe(pipe_class_memory);
17087 %}
17088 
17089 instruct stringU_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
17090                                   iRegI_R0 result, vecA ztmp1, vecA ztmp2,
17091                                   pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
17092   predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U);
17093   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
17094   effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
17095   format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
17096   ins_encode %{
17097     __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
17098                                $result$$Register, $ztmp1$$FloatRegister,
17099                                $ztmp2$$FloatRegister, $pgtmp$$PRegister,
17100                                $ptmp$$PRegister, false /* isL */);
17101   %}
17102   ins_pipe(pipe_class_memory);
17103 %}
17104 
17105 instruct string_equalsL(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
17106                         iRegI_R0 result, rFlagsReg cr)
17107 %{
17108   predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::LL);
17109   match(Set result (StrEquals (Binary str1 str2) cnt));
17110   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
17111 
17112   format %{ "String Equals $str1,$str2,$cnt -> $result" %}
17113   ins_encode %{
17114     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
17115     __ string_equals($str1$$Register, $str2$$Register,
17116                      $result$$Register, $cnt$$Register);
17117   %}
17118   ins_pipe(pipe_class_memory);
17119 %}
17120 
17121 instruct array_equalsB(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
17122                        iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
17123                        vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17124                        vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
17125                        iRegP_R10 tmp, rFlagsReg cr)
17126 %{
17127   predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
17128   match(Set result (AryEq ary1 ary2));
17129   effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
17130          TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
17131          TEMP vtmp6, TEMP vtmp7, KILL cr);
17132 
17133   format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
17134   ins_encode %{
17135     address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
17136                                    $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
17137                                    $result$$Register, $tmp$$Register, 1);
17138     if (tpc == nullptr) {
17139       ciEnv::current()->record_failure("CodeCache is full");
17140       return;
17141     }
17142   %}
17143   ins_pipe(pipe_class_memory);
17144 %}
17145 
17146 instruct array_equalsC(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
17147                        iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
17148                        vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17149                        vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
17150                        iRegP_R10 tmp, rFlagsReg cr)
17151 %{
17152   predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
17153   match(Set result (AryEq ary1 ary2));
17154   effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
17155          TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
17156          TEMP vtmp6, TEMP vtmp7, KILL cr);
17157 
17158   format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
17159   ins_encode %{
17160     address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
17161                                    $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
17162                                    $result$$Register, $tmp$$Register, 2);
17163     if (tpc == nullptr) {
17164       ciEnv::current()->record_failure("CodeCache is full");
17165       return;
17166     }
17167   %}
17168   ins_pipe(pipe_class_memory);
17169 %}
17170 
17171 instruct arrays_hashcode(iRegP_R1 ary, iRegI_R2 cnt, iRegI_R0 result, immI basic_type,
17172                          vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17173                          vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
17174                          vRegD_V12 vtmp8, vRegD_V13 vtmp9, rFlagsReg cr)
17175 %{
17176   match(Set result (VectorizedHashCode (Binary ary cnt) (Binary result basic_type)));
17177   effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5, TEMP vtmp6,
17178          TEMP vtmp7, TEMP vtmp8, TEMP vtmp9, USE_KILL ary, USE_KILL cnt, USE basic_type, KILL cr);
17179 
17180   format %{ "Array HashCode array[] $ary,$cnt,$result,$basic_type -> $result   // KILL all" %}
17181   ins_encode %{
17182     address tpc = __ arrays_hashcode($ary$$Register, $cnt$$Register, $result$$Register,
17183                                      $vtmp3$$FloatRegister, $vtmp2$$FloatRegister,
17184                                      $vtmp1$$FloatRegister, $vtmp0$$FloatRegister,
17185                                      $vtmp4$$FloatRegister, $vtmp5$$FloatRegister,
17186                                      $vtmp6$$FloatRegister, $vtmp7$$FloatRegister,
17187                                      $vtmp8$$FloatRegister, $vtmp9$$FloatRegister,
17188                                      (BasicType)$basic_type$$constant);
17189     if (tpc == nullptr) {
17190       ciEnv::current()->record_failure("CodeCache is full");
17191       return;
17192     }
17193   %}
17194   ins_pipe(pipe_class_memory);
17195 %}
17196 
17197 instruct count_positives(iRegP_R1 ary1, iRegI_R2 len, iRegI_R0 result, rFlagsReg cr)
17198 %{
17199   match(Set result (CountPositives ary1 len));
17200   effect(USE_KILL ary1, USE_KILL len, KILL cr);
17201   format %{ "count positives byte[] $ary1,$len -> $result" %}
17202   ins_encode %{
17203     address tpc = __ count_positives($ary1$$Register, $len$$Register, $result$$Register);
17204     if (tpc == nullptr) {
17205       ciEnv::current()->record_failure("CodeCache is full");
17206       return;
17207     }
17208   %}
17209   ins_pipe( pipe_slow );
17210 %}
17211 
17212 // fast char[] to byte[] compression
17213 instruct string_compress(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17214                          vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
17215                          vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
17216                          iRegI_R0 result, rFlagsReg cr)
17217 %{
17218   match(Set result (StrCompressedCopy src (Binary dst len)));
17219   effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
17220          USE_KILL src, USE_KILL dst, USE len, KILL cr);
17221 
17222   format %{ "String Compress $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
17223   ins_encode %{
17224     __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
17225                            $result$$Register, $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17226                            $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
17227                            $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
17228   %}
17229   ins_pipe(pipe_slow);
17230 %}
17231 
17232 // fast byte[] to char[] inflation
17233 instruct string_inflate(Universe dummy, iRegP_R0 src, iRegP_R1 dst, iRegI_R2 len, iRegP_R3 tmp,
17234                         vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17235                         vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, rFlagsReg cr)
17236 %{
17237   match(Set dummy (StrInflatedCopy src (Binary dst len)));
17238   effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3,
17239          TEMP vtmp4, TEMP vtmp5, TEMP vtmp6, TEMP tmp,
17240          USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
17241 
17242   format %{ "String Inflate $src,$dst # KILL $tmp $src $dst $len V0-V6 cr" %}
17243   ins_encode %{
17244     address tpc = __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
17245                                         $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17246                                         $vtmp2$$FloatRegister, $tmp$$Register);
17247     if (tpc == nullptr) {
17248       ciEnv::current()->record_failure("CodeCache is full");
17249       return;
17250     }
17251   %}
17252   ins_pipe(pipe_class_memory);
17253 %}
17254 
17255 // encode char[] to byte[] in ISO_8859_1
17256 instruct encode_iso_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17257                           vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
17258                           vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
17259                           iRegI_R0 result, rFlagsReg cr)
17260 %{
17261   predicate(!((EncodeISOArrayNode*)n)->is_ascii());
17262   match(Set result (EncodeISOArray src (Binary dst len)));
17263   effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
17264          KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
17265 
17266   format %{ "Encode ISO array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
17267   ins_encode %{
17268     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
17269                         $result$$Register, false,
17270                         $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17271                         $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
17272                         $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
17273   %}
17274   ins_pipe(pipe_class_memory);
17275 %}
17276 
17277 instruct encode_ascii_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17278                             vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
17279                             vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
17280                             iRegI_R0 result, rFlagsReg cr)
17281 %{
17282   predicate(((EncodeISOArrayNode*)n)->is_ascii());
17283   match(Set result (EncodeISOArray src (Binary dst len)));
17284   effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
17285          KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
17286 
17287   format %{ "Encode ASCII array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
17288   ins_encode %{
17289     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
17290                         $result$$Register, true,
17291                         $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17292                         $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
17293                         $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
17294   %}
17295   ins_pipe(pipe_class_memory);
17296 %}
17297 
17298 //----------------------------- CompressBits/ExpandBits ------------------------
17299 
17300 instruct compressBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
17301                            vRegF tdst, vRegF tsrc, vRegF tmask) %{
17302   match(Set dst (CompressBits src mask));
17303   effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17304   format %{ "mov    $tsrc, $src\n\t"
17305             "mov    $tmask, $mask\n\t"
17306             "bext   $tdst, $tsrc, $tmask\n\t"
17307             "mov    $dst, $tdst"
17308           %}
17309   ins_encode %{
17310     __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
17311     __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
17312     __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17313     __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17314   %}
17315   ins_pipe(pipe_slow);
17316 %}
17317 
17318 instruct compressBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
17319                            vRegF tdst, vRegF tsrc, vRegF tmask) %{
17320   match(Set dst (CompressBits (LoadI mem) mask));
17321   effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17322   format %{ "ldrs   $tsrc, $mem\n\t"
17323             "ldrs   $tmask, $mask\n\t"
17324             "bext   $tdst, $tsrc, $tmask\n\t"
17325             "mov    $dst, $tdst"
17326           %}
17327   ins_encode %{
17328     loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
17329               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
17330     __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
17331     __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17332     __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17333   %}
17334   ins_pipe(pipe_slow);
17335 %}
17336 
17337 instruct compressBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
17338                            vRegD tdst, vRegD tsrc, vRegD tmask) %{
17339   match(Set dst (CompressBits src mask));
17340   effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17341   format %{ "mov    $tsrc, $src\n\t"
17342             "mov    $tmask, $mask\n\t"
17343             "bext   $tdst, $tsrc, $tmask\n\t"
17344             "mov    $dst, $tdst"
17345           %}
17346   ins_encode %{
17347     __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
17348     __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
17349     __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17350     __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17351   %}
17352   ins_pipe(pipe_slow);
17353 %}
17354 
17355 instruct compressBitsL_memcon(iRegLNoSp dst, memory8 mem, immL mask,
17356                            vRegF tdst, vRegF tsrc, vRegF tmask) %{
17357   match(Set dst (CompressBits (LoadL mem) mask));
17358   effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17359   format %{ "ldrd   $tsrc, $mem\n\t"
17360             "ldrd   $tmask, $mask\n\t"
17361             "bext   $tdst, $tsrc, $tmask\n\t"
17362             "mov    $dst, $tdst"
17363           %}
17364   ins_encode %{
17365     loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
17366               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
17367     __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
17368     __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17369     __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17370   %}
17371   ins_pipe(pipe_slow);
17372 %}
17373 
17374 instruct expandBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
17375                          vRegF tdst, vRegF tsrc, vRegF tmask) %{
17376   match(Set dst (ExpandBits src mask));
17377   effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17378   format %{ "mov    $tsrc, $src\n\t"
17379             "mov    $tmask, $mask\n\t"
17380             "bdep   $tdst, $tsrc, $tmask\n\t"
17381             "mov    $dst, $tdst"
17382           %}
17383   ins_encode %{
17384     __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
17385     __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
17386     __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17387     __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17388   %}
17389   ins_pipe(pipe_slow);
17390 %}
17391 
17392 instruct expandBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
17393                          vRegF tdst, vRegF tsrc, vRegF tmask) %{
17394   match(Set dst (ExpandBits (LoadI mem) mask));
17395   effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17396   format %{ "ldrs   $tsrc, $mem\n\t"
17397             "ldrs   $tmask, $mask\n\t"
17398             "bdep   $tdst, $tsrc, $tmask\n\t"
17399             "mov    $dst, $tdst"
17400           %}
17401   ins_encode %{
17402     loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
17403               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
17404     __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
17405     __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17406     __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17407   %}
17408   ins_pipe(pipe_slow);
17409 %}
17410 
17411 instruct expandBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
17412                          vRegD tdst, vRegD tsrc, vRegD tmask) %{
17413   match(Set dst (ExpandBits src mask));
17414   effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17415   format %{ "mov    $tsrc, $src\n\t"
17416             "mov    $tmask, $mask\n\t"
17417             "bdep   $tdst, $tsrc, $tmask\n\t"
17418             "mov    $dst, $tdst"
17419           %}
17420   ins_encode %{
17421     __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
17422     __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
17423     __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17424     __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17425   %}
17426   ins_pipe(pipe_slow);
17427 %}
17428 
17429 
17430 instruct expandBitsL_memcon(iRegINoSp dst, memory8 mem, immL mask,
17431                          vRegF tdst, vRegF tsrc, vRegF tmask) %{
17432   match(Set dst (ExpandBits (LoadL mem) mask));
17433   effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17434   format %{ "ldrd   $tsrc, $mem\n\t"
17435             "ldrd   $tmask, $mask\n\t"
17436             "bdep   $tdst, $tsrc, $tmask\n\t"
17437             "mov    $dst, $tdst"
17438           %}
17439   ins_encode %{
17440     loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
17441               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
17442     __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
17443     __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17444     __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17445   %}
17446   ins_pipe(pipe_slow);
17447 %}
17448 
17449 //----------------------------- Reinterpret ----------------------------------
17450 // Reinterpret a half-precision float value in a floating point register to a general purpose register
17451 instruct reinterpretHF2S(iRegINoSp dst, vRegF src) %{
17452   match(Set dst (ReinterpretHF2S src));
17453   format %{ "reinterpretHF2S $dst, $src" %}
17454   ins_encode %{
17455     __ smov($dst$$Register, $src$$FloatRegister, __ H, 0);
17456   %}
17457   ins_pipe(pipe_slow);
17458 %}
17459 
17460 // Reinterpret a half-precision float value in a general purpose register to a floating point register
17461 instruct reinterpretS2HF(vRegF dst, iRegINoSp src) %{
17462   match(Set dst (ReinterpretS2HF src));
17463   format %{ "reinterpretS2HF $dst, $src" %}
17464   ins_encode %{
17465     __ mov($dst$$FloatRegister, __ H, 0, $src$$Register);
17466   %}
17467   ins_pipe(pipe_slow);
17468 %}
17469 
17470 // Without this optimization, ReinterpretS2HF (ConvF2HF src) would result in the following
17471 // instructions (the first two are for ConvF2HF and the last instruction is for ReinterpretS2HF) -
17472 // fcvt $tmp1_fpr, $src_fpr    // Convert float to half-precision float
17473 // mov  $tmp2_gpr, $tmp1_fpr   // Move half-precision float in FPR to a GPR
17474 // mov  $dst_fpr,  $tmp2_gpr   // Move the result from a GPR to an FPR
17475 // The move from FPR to GPR in ConvF2HF and the move from GPR to FPR in ReinterpretS2HF
17476 // can be omitted in this pattern, resulting in -
17477 // fcvt $dst, $src  // Convert float to half-precision float
17478 instruct convF2HFAndS2HF(vRegF dst, vRegF src)
17479 %{
17480   match(Set dst (ReinterpretS2HF (ConvF2HF src)));
17481   format %{ "convF2HFAndS2HF $dst, $src" %}
17482   ins_encode %{
17483     __ fcvtsh($dst$$FloatRegister, $src$$FloatRegister);
17484   %}
17485   ins_pipe(pipe_slow);
17486 %}
17487 
17488 // Without this optimization, ConvHF2F (ReinterpretHF2S src) would result in the following
17489 // instructions (the first one is for ReinterpretHF2S and the last two are for ConvHF2F) -
17490 // mov  $tmp1_gpr, $src_fpr  // Move the half-precision float from an FPR to a GPR
17491 // mov  $tmp2_fpr, $tmp1_gpr // Move the same value from GPR to an FPR
17492 // fcvt $dst_fpr,  $tmp2_fpr // Convert the half-precision float to 32-bit float
17493 // The move from FPR to GPR in ReinterpretHF2S and the move from GPR to FPR in ConvHF2F
17494 // can be omitted as the input (src) is already in an FPR required for the fcvths instruction
17495 // resulting in -
17496 // fcvt $dst, $src  // Convert half-precision float to a 32-bit float
17497 instruct convHF2SAndHF2F(vRegF dst, vRegF src)
17498 %{
17499   match(Set dst (ConvHF2F (ReinterpretHF2S src)));
17500   format %{ "convHF2SAndHF2F $dst, $src" %}
17501   ins_encode %{
17502     __ fcvths($dst$$FloatRegister, $src$$FloatRegister);
17503   %}
17504   ins_pipe(pipe_slow);
17505 %}
17506 
17507 // ============================================================================
17508 // This name is KNOWN by the ADLC and cannot be changed.
17509 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
17510 // for this guy.
17511 instruct tlsLoadP(thread_RegP dst)
17512 %{
17513   match(Set dst (ThreadLocal));
17514 
17515   ins_cost(0);
17516 
17517   format %{ " -- \t// $dst=Thread::current(), empty" %}
17518 
17519   size(0);
17520 
17521   ins_encode( /*empty*/ );
17522 
17523   ins_pipe(pipe_class_empty);
17524 %}
17525 
17526 //----------PEEPHOLE RULES-----------------------------------------------------
17527 // These must follow all instruction definitions as they use the names
17528 // defined in the instructions definitions.
17529 //
17530 // peepmatch ( root_instr_name [preceding_instruction]* );
17531 //
17532 // peepconstraint %{
17533 // (instruction_number.operand_name relational_op instruction_number.operand_name
17534 //  [, ...] );
17535 // // instruction numbers are zero-based using left to right order in peepmatch
17536 //
17537 // peepreplace ( instr_name  ( [instruction_number.operand_name]* ) );
17538 // // provide an instruction_number.operand_name for each operand that appears
17539 // // in the replacement instruction's match rule
17540 //
17541 // ---------VM FLAGS---------------------------------------------------------
17542 //
17543 // All peephole optimizations can be turned off using -XX:-OptoPeephole
17544 //
17545 // Each peephole rule is given an identifying number starting with zero and
17546 // increasing by one in the order seen by the parser.  An individual peephole
17547 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
17548 // on the command-line.
17549 //
17550 // ---------CURRENT LIMITATIONS----------------------------------------------
17551 //
17552 // Only match adjacent instructions in same basic block
17553 // Only equality constraints
17554 // Only constraints between operands, not (0.dest_reg == RAX_enc)
17555 // Only one replacement instruction
17556 //
17557 // ---------EXAMPLE----------------------------------------------------------
17558 //
17559 // // pertinent parts of existing instructions in architecture description
17560 // instruct movI(iRegINoSp dst, iRegI src)
17561 // %{
17562 //   match(Set dst (CopyI src));
17563 // %}
17564 //
17565 // instruct incI_iReg(iRegINoSp dst, immI1 src, rFlagsReg cr)
17566 // %{
17567 //   match(Set dst (AddI dst src));
17568 //   effect(KILL cr);
17569 // %}
17570 //
17571 // // Change (inc mov) to lea
17572 // peephole %{
17573 //   // increment preceded by register-register move
17574 //   peepmatch ( incI_iReg movI );
17575 //   // require that the destination register of the increment
17576 //   // match the destination register of the move
17577 //   peepconstraint ( 0.dst == 1.dst );
17578 //   // construct a replacement instruction that sets
17579 //   // the destination to ( move's source register + one )
17580 //   peepreplace ( leaI_iReg_immI( 0.dst 1.src 0.src ) );
17581 // %}
17582 //
17583 
17584 // Implementation no longer uses movX instructions since
17585 // machine-independent system no longer uses CopyX nodes.
17586 //
17587 // peephole
17588 // %{
17589 //   peepmatch (incI_iReg movI);
17590 //   peepconstraint (0.dst == 1.dst);
17591 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17592 // %}
17593 
17594 // peephole
17595 // %{
17596 //   peepmatch (decI_iReg movI);
17597 //   peepconstraint (0.dst == 1.dst);
17598 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17599 // %}
17600 
17601 // peephole
17602 // %{
17603 //   peepmatch (addI_iReg_imm movI);
17604 //   peepconstraint (0.dst == 1.dst);
17605 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17606 // %}
17607 
17608 // peephole
17609 // %{
17610 //   peepmatch (incL_iReg movL);
17611 //   peepconstraint (0.dst == 1.dst);
17612 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17613 // %}
17614 
17615 // peephole
17616 // %{
17617 //   peepmatch (decL_iReg movL);
17618 //   peepconstraint (0.dst == 1.dst);
17619 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17620 // %}
17621 
17622 // peephole
17623 // %{
17624 //   peepmatch (addL_iReg_imm movL);
17625 //   peepconstraint (0.dst == 1.dst);
17626 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17627 // %}
17628 
17629 // peephole
17630 // %{
17631 //   peepmatch (addP_iReg_imm movP);
17632 //   peepconstraint (0.dst == 1.dst);
17633 //   peepreplace (leaP_iReg_imm(0.dst 1.src 0.src));
17634 // %}
17635 
17636 // // Change load of spilled value to only a spill
17637 // instruct storeI(memory mem, iRegI src)
17638 // %{
17639 //   match(Set mem (StoreI mem src));
17640 // %}
17641 //
17642 // instruct loadI(iRegINoSp dst, memory mem)
17643 // %{
17644 //   match(Set dst (LoadI mem));
17645 // %}
17646 //
17647 
17648 //----------SMARTSPILL RULES---------------------------------------------------
17649 // These must follow all instruction definitions as they use the names
17650 // defined in the instructions definitions.
17651 
17652 // Local Variables:
17653 // mode: c++
17654 // End: