1 //
    2 // Copyright (c) 2003, 2023, Oracle and/or its affiliates. All rights reserved.
    3 // Copyright (c) 2014, 2021, Red Hat, Inc. All rights reserved.
    4 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
    5 //
    6 // This code is free software; you can redistribute it and/or modify it
    7 // under the terms of the GNU General Public License version 2 only, as
    8 // published by the Free Software Foundation.
    9 //
   10 // This code is distributed in the hope that it will be useful, but WITHOUT
   11 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
   12 // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
   13 // version 2 for more details (a copy is included in the LICENSE file that
   14 // accompanied this code).
   15 //
   16 // You should have received a copy of the GNU General Public License version
   17 // 2 along with this work; if not, write to the Free Software Foundation,
   18 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
   19 //
   20 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
   21 // or visit www.oracle.com if you need additional information or have any
   22 // questions.
   23 //
   24 //
   25 
   26 // AArch64 Architecture Description File
   27 
   28 //----------REGISTER DEFINITION BLOCK------------------------------------------
   29 // This information is used by the matcher and the register allocator to
   30 // describe individual registers and classes of registers within the target
   31 // architecture.
   32 
   33 register %{
   34 //----------Architecture Description Register Definitions----------------------
   35 // General Registers
   36 // "reg_def"  name ( register save type, C convention save type,
   37 //                   ideal register type, encoding );
   38 // Register Save Types:
   39 //
   40 // NS  = No-Save:       The register allocator assumes that these registers
   41 //                      can be used without saving upon entry to the method, &
   42 //                      that they do not need to be saved at call sites.
   43 //
   44 // SOC = Save-On-Call:  The register allocator assumes that these registers
   45 //                      can be used without saving upon entry to the method,
   46 //                      but that they must be saved at call sites.
   47 //
   48 // SOE = Save-On-Entry: The register allocator assumes that these registers
   49 //                      must be saved before using them upon entry to the
   50 //                      method, but they do not need to be saved at call
   51 //                      sites.
   52 //
   53 // AS  = Always-Save:   The register allocator assumes that these registers
   54 //                      must be saved before using them upon entry to the
   55 //                      method, & that they must be saved at call sites.
   56 //
   57 // Ideal Register Type is used to determine how to save & restore a
   58 // register.  Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
   59 // spilled with LoadP/StoreP.  If the register supports both, use Op_RegI.
   60 //
   61 // The encoding number is the actual bit-pattern placed into the opcodes.
   62 
   63 // We must define the 64 bit int registers in two 32 bit halves, the
   64 // real lower register and a virtual upper half register. upper halves
   65 // are used by the register allocator but are not actually supplied as
   66 // operands to memory ops.
   67 //
   68 // follow the C1 compiler in making registers
   69 //
   70 //   r0-r7,r10-r26 volatile (caller save)
   71 //   r27-r32 system (no save, no allocate)
   72 //   r8-r9 non-allocatable (so we can use them as scratch regs)
   73 //
   74 // as regards Java usage. we don't use any callee save registers
   75 // because this makes it difficult to de-optimise a frame (see comment
   76 // in x86 implementation of Deoptimization::unwind_callee_save_values)
   77 //
   78 
   79 // General Registers
   80 
   81 reg_def R0      ( SOC, SOC, Op_RegI,  0, r0->as_VMReg()         );
   82 reg_def R0_H    ( SOC, SOC, Op_RegI,  0, r0->as_VMReg()->next() );
   83 reg_def R1      ( SOC, SOC, Op_RegI,  1, r1->as_VMReg()         );
   84 reg_def R1_H    ( SOC, SOC, Op_RegI,  1, r1->as_VMReg()->next() );
   85 reg_def R2      ( SOC, SOC, Op_RegI,  2, r2->as_VMReg()         );
   86 reg_def R2_H    ( SOC, SOC, Op_RegI,  2, r2->as_VMReg()->next() );
   87 reg_def R3      ( SOC, SOC, Op_RegI,  3, r3->as_VMReg()         );
   88 reg_def R3_H    ( SOC, SOC, Op_RegI,  3, r3->as_VMReg()->next() );
   89 reg_def R4      ( SOC, SOC, Op_RegI,  4, r4->as_VMReg()         );
   90 reg_def R4_H    ( SOC, SOC, Op_RegI,  4, r4->as_VMReg()->next() );
   91 reg_def R5      ( SOC, SOC, Op_RegI,  5, r5->as_VMReg()         );
   92 reg_def R5_H    ( SOC, SOC, Op_RegI,  5, r5->as_VMReg()->next() );
   93 reg_def R6      ( SOC, SOC, Op_RegI,  6, r6->as_VMReg()         );
   94 reg_def R6_H    ( SOC, SOC, Op_RegI,  6, r6->as_VMReg()->next() );
   95 reg_def R7      ( SOC, SOC, Op_RegI,  7, r7->as_VMReg()         );
   96 reg_def R7_H    ( SOC, SOC, Op_RegI,  7, r7->as_VMReg()->next() );
   97 reg_def R8      ( NS,  SOC, Op_RegI,  8, r8->as_VMReg()         ); // rscratch1, non-allocatable
   98 reg_def R8_H    ( NS,  SOC, Op_RegI,  8, r8->as_VMReg()->next() );
   99 reg_def R9      ( NS,  SOC, Op_RegI,  9, r9->as_VMReg()         ); // rscratch2, non-allocatable
  100 reg_def R9_H    ( NS,  SOC, Op_RegI,  9, r9->as_VMReg()->next() );
  101 reg_def R10     ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()        );
  102 reg_def R10_H   ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
  103 reg_def R11     ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()        );
  104 reg_def R11_H   ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
  105 reg_def R12     ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()        );
  106 reg_def R12_H   ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()->next());
  107 reg_def R13     ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()        );
  108 reg_def R13_H   ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()->next());
  109 reg_def R14     ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()        );
  110 reg_def R14_H   ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()->next());
  111 reg_def R15     ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()        );
  112 reg_def R15_H   ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()->next());
  113 reg_def R16     ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()        );
  114 reg_def R16_H   ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
  115 reg_def R17     ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()        );
  116 reg_def R17_H   ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
  117 reg_def R18     ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()        );
  118 reg_def R18_H   ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()->next());
  119 reg_def R19     ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()        );
  120 reg_def R19_H   ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()->next());
  121 reg_def R20     ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()        ); // caller esp
  122 reg_def R20_H   ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()->next());
  123 reg_def R21     ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()        );
  124 reg_def R21_H   ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()->next());
  125 reg_def R22     ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()        );
  126 reg_def R22_H   ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()->next());
  127 reg_def R23     ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()        );
  128 reg_def R23_H   ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()->next());
  129 reg_def R24     ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()        );
  130 reg_def R24_H   ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()->next());
  131 reg_def R25     ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()        );
  132 reg_def R25_H   ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()->next());
  133 reg_def R26     ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()        );
  134 reg_def R26_H   ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()->next());
  135 reg_def R27     ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()        ); // heapbase
  136 reg_def R27_H   ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()->next());
  137 reg_def R28     (  NS, SOE, Op_RegI, 28, r28->as_VMReg()        ); // thread
  138 reg_def R28_H   (  NS, SOE, Op_RegI, 28, r28->as_VMReg()->next());
  139 reg_def R29     (  NS,  NS, Op_RegI, 29, r29->as_VMReg()        ); // fp
  140 reg_def R29_H   (  NS,  NS, Op_RegI, 29, r29->as_VMReg()->next());
  141 reg_def R30     (  NS,  NS, Op_RegI, 30, r30->as_VMReg()        ); // lr
  142 reg_def R30_H   (  NS,  NS, Op_RegI, 30, r30->as_VMReg()->next());
  143 reg_def R31     (  NS,  NS, Op_RegI, 31, r31_sp->as_VMReg()     ); // sp
  144 reg_def R31_H   (  NS,  NS, Op_RegI, 31, r31_sp->as_VMReg()->next());
  145 
  146 // ----------------------------
  147 // Float/Double/Vector Registers
  148 // ----------------------------
  149 
  150 // Double Registers
  151 
  152 // The rules of ADL require that double registers be defined in pairs.
  153 // Each pair must be two 32-bit values, but not necessarily a pair of
  154 // single float registers. In each pair, ADLC-assigned register numbers
  155 // must be adjacent, with the lower number even. Finally, when the
  156 // CPU stores such a register pair to memory, the word associated with
  157 // the lower ADLC-assigned number must be stored to the lower address.
  158 
  159 // AArch64 has 32 floating-point registers. Each can store a vector of
  160 // single or double precision floating-point values up to 8 * 32
  161 // floats, 4 * 64 bit floats or 2 * 128 bit floats.  We currently only
  162 // use the first float or double element of the vector.
  163 
  164 // for Java use float registers v0-v15 are always save on call whereas
  165 // the platform ABI treats v8-v15 as callee save). float registers
  166 // v16-v31 are SOC as per the platform spec
  167 
  168 // For SVE vector registers, we simply extend vector register size to 8
  169 // 'logical' slots. This is nominally 256 bits but it actually covers
  170 // all possible 'physical' SVE vector register lengths from 128 ~ 2048
  171 // bits. The 'physical' SVE vector register length is detected during
  172 // startup, so the register allocator is able to identify the correct
  173 // number of bytes needed for an SVE spill/unspill.
  174 // Note that a vector register with 4 slots denotes a 128-bit NEON
  175 // register allowing it to be distinguished from the corresponding SVE
  176 // vector register when the SVE vector length is 128 bits.
  177 
  178   reg_def V0   ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()          );
  179   reg_def V0_H ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next()  );
  180   reg_def V0_J ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(2) );
  181   reg_def V0_K ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(3) );
  182 
  183   reg_def V1   ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()          );
  184   reg_def V1_H ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next()  );
  185   reg_def V1_J ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(2) );
  186   reg_def V1_K ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(3) );
  187 
  188   reg_def V2   ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()          );
  189   reg_def V2_H ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next()  );
  190   reg_def V2_J ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(2) );
  191   reg_def V2_K ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(3) );
  192 
  193   reg_def V3   ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()          );
  194   reg_def V3_H ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next()  );
  195   reg_def V3_J ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(2) );
  196   reg_def V3_K ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(3) );
  197 
  198   reg_def V4   ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()          );
  199   reg_def V4_H ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next()  );
  200   reg_def V4_J ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(2) );
  201   reg_def V4_K ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(3) );
  202 
  203   reg_def V5   ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()          );
  204   reg_def V5_H ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next()  );
  205   reg_def V5_J ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(2) );
  206   reg_def V5_K ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(3) );
  207 
  208   reg_def V6   ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()          );
  209   reg_def V6_H ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next()  );
  210   reg_def V6_J ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(2) );
  211   reg_def V6_K ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(3) );
  212 
  213   reg_def V7   ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()          );
  214   reg_def V7_H ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next()  );
  215   reg_def V7_J ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(2) );
  216   reg_def V7_K ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(3) );
  217 
  218   reg_def V8   ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()          );
  219   reg_def V8_H ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()->next()  );
  220   reg_def V8_J ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(2) );
  221   reg_def V8_K ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(3) );
  222 
  223   reg_def V9   ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()          );
  224   reg_def V9_H ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()->next()  );
  225   reg_def V9_J ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(2) );
  226   reg_def V9_K ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(3) );
  227 
  228   reg_def V10   ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()          );
  229   reg_def V10_H ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()->next()  );
  230   reg_def V10_J ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(2) );
  231   reg_def V10_K ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(3) );
  232 
  233   reg_def V11   ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()          );
  234   reg_def V11_H ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()->next()  );
  235   reg_def V11_J ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(2) );
  236   reg_def V11_K ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(3) );
  237 
  238   reg_def V12   ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()          );
  239   reg_def V12_H ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()->next()  );
  240   reg_def V12_J ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(2) );
  241   reg_def V12_K ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(3) );
  242 
  243   reg_def V13   ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()          );
  244   reg_def V13_H ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()->next()  );
  245   reg_def V13_J ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(2) );
  246   reg_def V13_K ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(3) );
  247 
  248   reg_def V14   ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()          );
  249   reg_def V14_H ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()->next()  );
  250   reg_def V14_J ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(2) );
  251   reg_def V14_K ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(3) );
  252 
  253   reg_def V15   ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()          );
  254   reg_def V15_H ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()->next()  );
  255   reg_def V15_J ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(2) );
  256   reg_def V15_K ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(3) );
  257 
  258   reg_def V16   ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()          );
  259   reg_def V16_H ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next()  );
  260   reg_def V16_J ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(2) );
  261   reg_def V16_K ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(3) );
  262 
  263   reg_def V17   ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()          );
  264   reg_def V17_H ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next()  );
  265   reg_def V17_J ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(2) );
  266   reg_def V17_K ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(3) );
  267 
  268   reg_def V18   ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()          );
  269   reg_def V18_H ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next()  );
  270   reg_def V18_J ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(2) );
  271   reg_def V18_K ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(3) );
  272 
  273   reg_def V19   ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()          );
  274   reg_def V19_H ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next()  );
  275   reg_def V19_J ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(2) );
  276   reg_def V19_K ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(3) );
  277 
  278   reg_def V20   ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()          );
  279   reg_def V20_H ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next()  );
  280   reg_def V20_J ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(2) );
  281   reg_def V20_K ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(3) );
  282 
  283   reg_def V21   ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()          );
  284   reg_def V21_H ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next()  );
  285   reg_def V21_J ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(2) );
  286   reg_def V21_K ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(3) );
  287 
  288   reg_def V22   ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()          );
  289   reg_def V22_H ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next()  );
  290   reg_def V22_J ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(2) );
  291   reg_def V22_K ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(3) );
  292 
  293   reg_def V23   ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()          );
  294   reg_def V23_H ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next()  );
  295   reg_def V23_J ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(2) );
  296   reg_def V23_K ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(3) );
  297 
  298   reg_def V24   ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()          );
  299   reg_def V24_H ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next()  );
  300   reg_def V24_J ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(2) );
  301   reg_def V24_K ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(3) );
  302 
  303   reg_def V25   ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()          );
  304   reg_def V25_H ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next()  );
  305   reg_def V25_J ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(2) );
  306   reg_def V25_K ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(3) );
  307 
  308   reg_def V26   ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()          );
  309   reg_def V26_H ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next()  );
  310   reg_def V26_J ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(2) );
  311   reg_def V26_K ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(3) );
  312 
  313   reg_def V27   ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()          );
  314   reg_def V27_H ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next()  );
  315   reg_def V27_J ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(2) );
  316   reg_def V27_K ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(3) );
  317 
  318   reg_def V28   ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()          );
  319   reg_def V28_H ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next()  );
  320   reg_def V28_J ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(2) );
  321   reg_def V28_K ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(3) );
  322 
  323   reg_def V29   ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()          );
  324   reg_def V29_H ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next()  );
  325   reg_def V29_J ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(2) );
  326   reg_def V29_K ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(3) );
  327 
  328   reg_def V30   ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()          );
  329   reg_def V30_H ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next()  );
  330   reg_def V30_J ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(2) );
  331   reg_def V30_K ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(3) );
  332 
  333   reg_def V31   ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()          );
  334   reg_def V31_H ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next()  );
  335   reg_def V31_J ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(2) );
  336   reg_def V31_K ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(3) );
  337 
  338 // ----------------------------
  339 // SVE Predicate Registers
  340 // ----------------------------
  341   reg_def P0 (SOC, SOC, Op_RegVectMask, 0, p0->as_VMReg());
  342   reg_def P1 (SOC, SOC, Op_RegVectMask, 1, p1->as_VMReg());
  343   reg_def P2 (SOC, SOC, Op_RegVectMask, 2, p2->as_VMReg());
  344   reg_def P3 (SOC, SOC, Op_RegVectMask, 3, p3->as_VMReg());
  345   reg_def P4 (SOC, SOC, Op_RegVectMask, 4, p4->as_VMReg());
  346   reg_def P5 (SOC, SOC, Op_RegVectMask, 5, p5->as_VMReg());
  347   reg_def P6 (SOC, SOC, Op_RegVectMask, 6, p6->as_VMReg());
  348   reg_def P7 (SOC, SOC, Op_RegVectMask, 7, p7->as_VMReg());
  349   reg_def P8 (SOC, SOC, Op_RegVectMask, 8, p8->as_VMReg());
  350   reg_def P9 (SOC, SOC, Op_RegVectMask, 9, p9->as_VMReg());
  351   reg_def P10 (SOC, SOC, Op_RegVectMask, 10, p10->as_VMReg());
  352   reg_def P11 (SOC, SOC, Op_RegVectMask, 11, p11->as_VMReg());
  353   reg_def P12 (SOC, SOC, Op_RegVectMask, 12, p12->as_VMReg());
  354   reg_def P13 (SOC, SOC, Op_RegVectMask, 13, p13->as_VMReg());
  355   reg_def P14 (SOC, SOC, Op_RegVectMask, 14, p14->as_VMReg());
  356   reg_def P15 (SOC, SOC, Op_RegVectMask, 15, p15->as_VMReg());
  357 
  358 // ----------------------------
  359 // Special Registers
  360 // ----------------------------
  361 
  362 // the AArch64 CSPR status flag register is not directly accessible as
  363 // instruction operand. the FPSR status flag register is a system
  364 // register which can be written/read using MSR/MRS but again does not
  365 // appear as an operand (a code identifying the FSPR occurs as an
  366 // immediate value in the instruction).
  367 
  368 reg_def RFLAGS(SOC, SOC, 0, 32, VMRegImpl::Bad());
  369 
  370 // Specify priority of register selection within phases of register
  371 // allocation.  Highest priority is first.  A useful heuristic is to
  372 // give registers a low priority when they are required by machine
  373 // instructions, like EAX and EDX on I486, and choose no-save registers
  374 // before save-on-call, & save-on-call before save-on-entry.  Registers
  375 // which participate in fixed calling sequences should come last.
  376 // Registers which are used as pairs must fall on an even boundary.
  377 
  378 alloc_class chunk0(
  379     // volatiles
  380     R10, R10_H,
  381     R11, R11_H,
  382     R12, R12_H,
  383     R13, R13_H,
  384     R14, R14_H,
  385     R15, R15_H,
  386     R16, R16_H,
  387     R17, R17_H,
  388     R18, R18_H,
  389 
  390     // arg registers
  391     R0, R0_H,
  392     R1, R1_H,
  393     R2, R2_H,
  394     R3, R3_H,
  395     R4, R4_H,
  396     R5, R5_H,
  397     R6, R6_H,
  398     R7, R7_H,
  399 
  400     // non-volatiles
  401     R19, R19_H,
  402     R20, R20_H,
  403     R21, R21_H,
  404     R22, R22_H,
  405     R23, R23_H,
  406     R24, R24_H,
  407     R25, R25_H,
  408     R26, R26_H,
  409 
  410     // non-allocatable registers
  411 
  412     R27, R27_H, // heapbase
  413     R28, R28_H, // thread
  414     R29, R29_H, // fp
  415     R30, R30_H, // lr
  416     R31, R31_H, // sp
  417     R8, R8_H,   // rscratch1
  418     R9, R9_H,   // rscratch2
  419 );
  420 
  421 alloc_class chunk1(
  422 
  423     // no save
  424     V16, V16_H, V16_J, V16_K,
  425     V17, V17_H, V17_J, V17_K,
  426     V18, V18_H, V18_J, V18_K,
  427     V19, V19_H, V19_J, V19_K,
  428     V20, V20_H, V20_J, V20_K,
  429     V21, V21_H, V21_J, V21_K,
  430     V22, V22_H, V22_J, V22_K,
  431     V23, V23_H, V23_J, V23_K,
  432     V24, V24_H, V24_J, V24_K,
  433     V25, V25_H, V25_J, V25_K,
  434     V26, V26_H, V26_J, V26_K,
  435     V27, V27_H, V27_J, V27_K,
  436     V28, V28_H, V28_J, V28_K,
  437     V29, V29_H, V29_J, V29_K,
  438     V30, V30_H, V30_J, V30_K,
  439     V31, V31_H, V31_J, V31_K,
  440 
  441     // arg registers
  442     V0, V0_H, V0_J, V0_K,
  443     V1, V1_H, V1_J, V1_K,
  444     V2, V2_H, V2_J, V2_K,
  445     V3, V3_H, V3_J, V3_K,
  446     V4, V4_H, V4_J, V4_K,
  447     V5, V5_H, V5_J, V5_K,
  448     V6, V6_H, V6_J, V6_K,
  449     V7, V7_H, V7_J, V7_K,
  450 
  451     // non-volatiles
  452     V8, V8_H, V8_J, V8_K,
  453     V9, V9_H, V9_J, V9_K,
  454     V10, V10_H, V10_J, V10_K,
  455     V11, V11_H, V11_J, V11_K,
  456     V12, V12_H, V12_J, V12_K,
  457     V13, V13_H, V13_J, V13_K,
  458     V14, V14_H, V14_J, V14_K,
  459     V15, V15_H, V15_J, V15_K,
  460 );
  461 
  462 alloc_class chunk2 (
  463     // Governing predicates for load/store and arithmetic
  464     P0,
  465     P1,
  466     P2,
  467     P3,
  468     P4,
  469     P5,
  470     P6,
  471 
  472     // Extra predicates
  473     P8,
  474     P9,
  475     P10,
  476     P11,
  477     P12,
  478     P13,
  479     P14,
  480     P15,
  481 
  482     // Preserved for all-true predicate
  483     P7,
  484 );
  485 
  486 alloc_class chunk3(RFLAGS);
  487 
  488 //----------Architecture Description Register Classes--------------------------
  489 // Several register classes are automatically defined based upon information in
  490 // this architecture description.
  491 // 1) reg_class inline_cache_reg           ( /* as def'd in frame section */ )
  492 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
  493 //
  494 
  495 // Class for all 32 bit general purpose registers
  496 reg_class all_reg32(
  497     R0,
  498     R1,
  499     R2,
  500     R3,
  501     R4,
  502     R5,
  503     R6,
  504     R7,
  505     R10,
  506     R11,
  507     R12,
  508     R13,
  509     R14,
  510     R15,
  511     R16,
  512     R17,
  513     R18,
  514     R19,
  515     R20,
  516     R21,
  517     R22,
  518     R23,
  519     R24,
  520     R25,
  521     R26,
  522     R27,
  523     R28,
  524     R29,
  525     R30,
  526     R31
  527 );
  528 
  529 
  530 // Class for all 32 bit integer registers (excluding SP which
  531 // will never be used as an integer register)
  532 reg_class any_reg32 %{
  533   return _ANY_REG32_mask;
  534 %}
  535 
  536 // Singleton class for R0 int register
  537 reg_class int_r0_reg(R0);
  538 
  539 // Singleton class for R2 int register
  540 reg_class int_r2_reg(R2);
  541 
  542 // Singleton class for R3 int register
  543 reg_class int_r3_reg(R3);
  544 
  545 // Singleton class for R4 int register
  546 reg_class int_r4_reg(R4);
  547 
  548 // Singleton class for R31 int register
  549 reg_class int_r31_reg(R31);
  550 
  551 // Class for all 64 bit general purpose registers
  552 reg_class all_reg(
  553     R0, R0_H,
  554     R1, R1_H,
  555     R2, R2_H,
  556     R3, R3_H,
  557     R4, R4_H,
  558     R5, R5_H,
  559     R6, R6_H,
  560     R7, R7_H,
  561     R10, R10_H,
  562     R11, R11_H,
  563     R12, R12_H,
  564     R13, R13_H,
  565     R14, R14_H,
  566     R15, R15_H,
  567     R16, R16_H,
  568     R17, R17_H,
  569     R18, R18_H,
  570     R19, R19_H,
  571     R20, R20_H,
  572     R21, R21_H,
  573     R22, R22_H,
  574     R23, R23_H,
  575     R24, R24_H,
  576     R25, R25_H,
  577     R26, R26_H,
  578     R27, R27_H,
  579     R28, R28_H,
  580     R29, R29_H,
  581     R30, R30_H,
  582     R31, R31_H
  583 );
  584 
  585 // Class for all long integer registers (including SP)
  586 reg_class any_reg %{
  587   return _ANY_REG_mask;
  588 %}
  589 
  590 // Class for non-allocatable 32 bit registers
  591 reg_class non_allocatable_reg32(
  592 #ifdef R18_RESERVED
  593     // See comment in register_aarch64.hpp
  594     R18,                        // tls on Windows
  595 #endif
  596     R28,                        // thread
  597     R30,                        // lr
  598     R31                         // sp
  599 );
  600 
  601 // Class for non-allocatable 64 bit registers
  602 reg_class non_allocatable_reg(
  603 #ifdef R18_RESERVED
  604     // See comment in register_aarch64.hpp
  605     R18, R18_H,                 // tls on Windows, platform register on macOS
  606 #endif
  607     R28, R28_H,                 // thread
  608     R30, R30_H,                 // lr
  609     R31, R31_H                  // sp
  610 );
  611 
  612 // Class for all non-special integer registers
  613 reg_class no_special_reg32 %{
  614   return _NO_SPECIAL_REG32_mask;
  615 %}
  616 
  617 // Class for all non-special long integer registers
  618 reg_class no_special_reg %{
  619   return _NO_SPECIAL_REG_mask;
  620 %}
  621 
  622 // Class for 64 bit register r0
  623 reg_class r0_reg(
  624     R0, R0_H
  625 );
  626 
  627 // Class for 64 bit register r1
  628 reg_class r1_reg(
  629     R1, R1_H
  630 );
  631 
  632 // Class for 64 bit register r2
  633 reg_class r2_reg(
  634     R2, R2_H
  635 );
  636 
  637 // Class for 64 bit register r3
  638 reg_class r3_reg(
  639     R3, R3_H
  640 );
  641 
  642 // Class for 64 bit register r4
  643 reg_class r4_reg(
  644     R4, R4_H
  645 );
  646 
  647 // Class for 64 bit register r5
  648 reg_class r5_reg(
  649     R5, R5_H
  650 );
  651 
  652 // Class for 64 bit register r10
  653 reg_class r10_reg(
  654     R10, R10_H
  655 );
  656 
  657 // Class for 64 bit register r11
  658 reg_class r11_reg(
  659     R11, R11_H
  660 );
  661 
  662 // Class for method register
  663 reg_class method_reg(
  664     R12, R12_H
  665 );
  666 
  667 // Class for heapbase register
  668 reg_class heapbase_reg(
  669     R27, R27_H
  670 );
  671 
  672 // Class for thread register
  673 reg_class thread_reg(
  674     R28, R28_H
  675 );
  676 
  677 // Class for frame pointer register
  678 reg_class fp_reg(
  679     R29, R29_H
  680 );
  681 
  682 // Class for link register
  683 reg_class lr_reg(
  684     R30, R30_H
  685 );
  686 
  687 // Class for long sp register
  688 reg_class sp_reg(
  689   R31, R31_H
  690 );
  691 
  692 // Class for all pointer registers
  693 reg_class ptr_reg %{
  694   return _PTR_REG_mask;
  695 %}
  696 
  697 // Class for all non_special pointer registers
  698 reg_class no_special_ptr_reg %{
  699   return _NO_SPECIAL_PTR_REG_mask;
  700 %}
  701 
  702 // Class for all float registers
  703 reg_class float_reg(
  704     V0,
  705     V1,
  706     V2,
  707     V3,
  708     V4,
  709     V5,
  710     V6,
  711     V7,
  712     V8,
  713     V9,
  714     V10,
  715     V11,
  716     V12,
  717     V13,
  718     V14,
  719     V15,
  720     V16,
  721     V17,
  722     V18,
  723     V19,
  724     V20,
  725     V21,
  726     V22,
  727     V23,
  728     V24,
  729     V25,
  730     V26,
  731     V27,
  732     V28,
  733     V29,
  734     V30,
  735     V31
  736 );
  737 
  738 // Double precision float registers have virtual `high halves' that
  739 // are needed by the allocator.
  740 // Class for all double registers
  741 reg_class double_reg(
  742     V0, V0_H,
  743     V1, V1_H,
  744     V2, V2_H,
  745     V3, V3_H,
  746     V4, V4_H,
  747     V5, V5_H,
  748     V6, V6_H,
  749     V7, V7_H,
  750     V8, V8_H,
  751     V9, V9_H,
  752     V10, V10_H,
  753     V11, V11_H,
  754     V12, V12_H,
  755     V13, V13_H,
  756     V14, V14_H,
  757     V15, V15_H,
  758     V16, V16_H,
  759     V17, V17_H,
  760     V18, V18_H,
  761     V19, V19_H,
  762     V20, V20_H,
  763     V21, V21_H,
  764     V22, V22_H,
  765     V23, V23_H,
  766     V24, V24_H,
  767     V25, V25_H,
  768     V26, V26_H,
  769     V27, V27_H,
  770     V28, V28_H,
  771     V29, V29_H,
  772     V30, V30_H,
  773     V31, V31_H
  774 );
  775 
  776 // Class for all SVE vector registers.
  777 reg_class vectora_reg (
  778     V0, V0_H, V0_J, V0_K,
  779     V1, V1_H, V1_J, V1_K,
  780     V2, V2_H, V2_J, V2_K,
  781     V3, V3_H, V3_J, V3_K,
  782     V4, V4_H, V4_J, V4_K,
  783     V5, V5_H, V5_J, V5_K,
  784     V6, V6_H, V6_J, V6_K,
  785     V7, V7_H, V7_J, V7_K,
  786     V8, V8_H, V8_J, V8_K,
  787     V9, V9_H, V9_J, V9_K,
  788     V10, V10_H, V10_J, V10_K,
  789     V11, V11_H, V11_J, V11_K,
  790     V12, V12_H, V12_J, V12_K,
  791     V13, V13_H, V13_J, V13_K,
  792     V14, V14_H, V14_J, V14_K,
  793     V15, V15_H, V15_J, V15_K,
  794     V16, V16_H, V16_J, V16_K,
  795     V17, V17_H, V17_J, V17_K,
  796     V18, V18_H, V18_J, V18_K,
  797     V19, V19_H, V19_J, V19_K,
  798     V20, V20_H, V20_J, V20_K,
  799     V21, V21_H, V21_J, V21_K,
  800     V22, V22_H, V22_J, V22_K,
  801     V23, V23_H, V23_J, V23_K,
  802     V24, V24_H, V24_J, V24_K,
  803     V25, V25_H, V25_J, V25_K,
  804     V26, V26_H, V26_J, V26_K,
  805     V27, V27_H, V27_J, V27_K,
  806     V28, V28_H, V28_J, V28_K,
  807     V29, V29_H, V29_J, V29_K,
  808     V30, V30_H, V30_J, V30_K,
  809     V31, V31_H, V31_J, V31_K,
  810 );
  811 
  812 // Class for all 64bit vector registers
  813 reg_class vectord_reg(
  814     V0, V0_H,
  815     V1, V1_H,
  816     V2, V2_H,
  817     V3, V3_H,
  818     V4, V4_H,
  819     V5, V5_H,
  820     V6, V6_H,
  821     V7, V7_H,
  822     V8, V8_H,
  823     V9, V9_H,
  824     V10, V10_H,
  825     V11, V11_H,
  826     V12, V12_H,
  827     V13, V13_H,
  828     V14, V14_H,
  829     V15, V15_H,
  830     V16, V16_H,
  831     V17, V17_H,
  832     V18, V18_H,
  833     V19, V19_H,
  834     V20, V20_H,
  835     V21, V21_H,
  836     V22, V22_H,
  837     V23, V23_H,
  838     V24, V24_H,
  839     V25, V25_H,
  840     V26, V26_H,
  841     V27, V27_H,
  842     V28, V28_H,
  843     V29, V29_H,
  844     V30, V30_H,
  845     V31, V31_H
  846 );
  847 
  848 // Class for all 128bit vector registers
  849 reg_class vectorx_reg(
  850     V0, V0_H, V0_J, V0_K,
  851     V1, V1_H, V1_J, V1_K,
  852     V2, V2_H, V2_J, V2_K,
  853     V3, V3_H, V3_J, V3_K,
  854     V4, V4_H, V4_J, V4_K,
  855     V5, V5_H, V5_J, V5_K,
  856     V6, V6_H, V6_J, V6_K,
  857     V7, V7_H, V7_J, V7_K,
  858     V8, V8_H, V8_J, V8_K,
  859     V9, V9_H, V9_J, V9_K,
  860     V10, V10_H, V10_J, V10_K,
  861     V11, V11_H, V11_J, V11_K,
  862     V12, V12_H, V12_J, V12_K,
  863     V13, V13_H, V13_J, V13_K,
  864     V14, V14_H, V14_J, V14_K,
  865     V15, V15_H, V15_J, V15_K,
  866     V16, V16_H, V16_J, V16_K,
  867     V17, V17_H, V17_J, V17_K,
  868     V18, V18_H, V18_J, V18_K,
  869     V19, V19_H, V19_J, V19_K,
  870     V20, V20_H, V20_J, V20_K,
  871     V21, V21_H, V21_J, V21_K,
  872     V22, V22_H, V22_J, V22_K,
  873     V23, V23_H, V23_J, V23_K,
  874     V24, V24_H, V24_J, V24_K,
  875     V25, V25_H, V25_J, V25_K,
  876     V26, V26_H, V26_J, V26_K,
  877     V27, V27_H, V27_J, V27_K,
  878     V28, V28_H, V28_J, V28_K,
  879     V29, V29_H, V29_J, V29_K,
  880     V30, V30_H, V30_J, V30_K,
  881     V31, V31_H, V31_J, V31_K
  882 );
  883 
  884 // Class for 128 bit register v0
  885 reg_class v0_reg(
  886     V0, V0_H
  887 );
  888 
  889 // Class for 128 bit register v1
  890 reg_class v1_reg(
  891     V1, V1_H
  892 );
  893 
  894 // Class for 128 bit register v2
  895 reg_class v2_reg(
  896     V2, V2_H
  897 );
  898 
  899 // Class for 128 bit register v3
  900 reg_class v3_reg(
  901     V3, V3_H
  902 );
  903 
  904 // Class for 128 bit register v4
  905 reg_class v4_reg(
  906     V4, V4_H
  907 );
  908 
  909 // Class for 128 bit register v5
  910 reg_class v5_reg(
  911     V5, V5_H
  912 );
  913 
  914 // Class for 128 bit register v6
  915 reg_class v6_reg(
  916     V6, V6_H
  917 );
  918 
  919 // Class for 128 bit register v7
  920 reg_class v7_reg(
  921     V7, V7_H
  922 );
  923 
  924 // Class for 128 bit register v8
  925 reg_class v8_reg(
  926     V8, V8_H
  927 );
  928 
  929 // Class for 128 bit register v9
  930 reg_class v9_reg(
  931     V9, V9_H
  932 );
  933 
  934 // Class for 128 bit register v10
  935 reg_class v10_reg(
  936     V10, V10_H
  937 );
  938 
  939 // Class for 128 bit register v11
  940 reg_class v11_reg(
  941     V11, V11_H
  942 );
  943 
  944 // Class for 128 bit register v12
  945 reg_class v12_reg(
  946     V12, V12_H
  947 );
  948 
  949 // Class for 128 bit register v13
  950 reg_class v13_reg(
  951     V13, V13_H
  952 );
  953 
  954 // Class for 128 bit register v14
  955 reg_class v14_reg(
  956     V14, V14_H
  957 );
  958 
  959 // Class for 128 bit register v15
  960 reg_class v15_reg(
  961     V15, V15_H
  962 );
  963 
  964 // Class for 128 bit register v16
  965 reg_class v16_reg(
  966     V16, V16_H
  967 );
  968 
  969 // Class for 128 bit register v17
  970 reg_class v17_reg(
  971     V17, V17_H
  972 );
  973 
  974 // Class for 128 bit register v18
  975 reg_class v18_reg(
  976     V18, V18_H
  977 );
  978 
  979 // Class for 128 bit register v19
  980 reg_class v19_reg(
  981     V19, V19_H
  982 );
  983 
  984 // Class for 128 bit register v20
  985 reg_class v20_reg(
  986     V20, V20_H
  987 );
  988 
  989 // Class for 128 bit register v21
  990 reg_class v21_reg(
  991     V21, V21_H
  992 );
  993 
  994 // Class for 128 bit register v22
  995 reg_class v22_reg(
  996     V22, V22_H
  997 );
  998 
  999 // Class for 128 bit register v23
 1000 reg_class v23_reg(
 1001     V23, V23_H
 1002 );
 1003 
 1004 // Class for 128 bit register v24
 1005 reg_class v24_reg(
 1006     V24, V24_H
 1007 );
 1008 
 1009 // Class for 128 bit register v25
 1010 reg_class v25_reg(
 1011     V25, V25_H
 1012 );
 1013 
 1014 // Class for 128 bit register v26
 1015 reg_class v26_reg(
 1016     V26, V26_H
 1017 );
 1018 
 1019 // Class for 128 bit register v27
 1020 reg_class v27_reg(
 1021     V27, V27_H
 1022 );
 1023 
 1024 // Class for 128 bit register v28
 1025 reg_class v28_reg(
 1026     V28, V28_H
 1027 );
 1028 
 1029 // Class for 128 bit register v29
 1030 reg_class v29_reg(
 1031     V29, V29_H
 1032 );
 1033 
 1034 // Class for 128 bit register v30
 1035 reg_class v30_reg(
 1036     V30, V30_H
 1037 );
 1038 
 1039 // Class for 128 bit register v31
 1040 reg_class v31_reg(
 1041     V31, V31_H
 1042 );
 1043 
 1044 // Class for all SVE predicate registers.
 1045 reg_class pr_reg (
 1046     P0,
 1047     P1,
 1048     P2,
 1049     P3,
 1050     P4,
 1051     P5,
 1052     P6,
 1053     // P7, non-allocatable, preserved with all elements preset to TRUE.
 1054     P8,
 1055     P9,
 1056     P10,
 1057     P11,
 1058     P12,
 1059     P13,
 1060     P14,
 1061     P15
 1062 );
 1063 
 1064 // Class for SVE governing predicate registers, which are used
 1065 // to determine the active elements of a predicated instruction.
 1066 reg_class gov_pr (
 1067     P0,
 1068     P1,
 1069     P2,
 1070     P3,
 1071     P4,
 1072     P5,
 1073     P6,
 1074     // P7, non-allocatable, preserved with all elements preset to TRUE.
 1075 );
 1076 
 1077 reg_class p0_reg(P0);
 1078 reg_class p1_reg(P1);
 1079 
 1080 // Singleton class for condition codes
 1081 reg_class int_flags(RFLAGS);
 1082 
 1083 %}
 1084 
 1085 //----------DEFINITION BLOCK---------------------------------------------------
 1086 // Define name --> value mappings to inform the ADLC of an integer valued name
 1087 // Current support includes integer values in the range [0, 0x7FFFFFFF]
 1088 // Format:
 1089 //        int_def  <name>         ( <int_value>, <expression>);
 1090 // Generated Code in ad_<arch>.hpp
 1091 //        #define  <name>   (<expression>)
 1092 //        // value == <int_value>
 1093 // Generated code in ad_<arch>.cpp adlc_verification()
 1094 //        assert( <name> == <int_value>, "Expect (<expression>) to equal <int_value>");
 1095 //
 1096 
 1097 // we follow the ppc-aix port in using a simple cost model which ranks
 1098 // register operations as cheap, memory ops as more expensive and
 1099 // branches as most expensive. the first two have a low as well as a
 1100 // normal cost. huge cost appears to be a way of saying don't do
 1101 // something
 1102 
 1103 definitions %{
 1104   // The default cost (of a register move instruction).
 1105   int_def INSN_COST            (    100,     100);
 1106   int_def BRANCH_COST          (    200,     2 * INSN_COST);
 1107   int_def CALL_COST            (    200,     2 * INSN_COST);
 1108   int_def VOLATILE_REF_COST    (   1000,     10 * INSN_COST);
 1109 %}
 1110 
 1111 
 1112 //----------SOURCE BLOCK-------------------------------------------------------
 1113 // This is a block of C++ code which provides values, functions, and
 1114 // definitions necessary in the rest of the architecture description
 1115 
 1116 source_hpp %{
 1117 
 1118 #include "asm/macroAssembler.hpp"
 1119 #include "gc/shared/barrierSetAssembler.hpp"
 1120 #include "gc/shared/cardTable.hpp"
 1121 #include "gc/shared/cardTableBarrierSet.hpp"
 1122 #include "gc/shared/collectedHeap.hpp"
 1123 #include "opto/addnode.hpp"
 1124 #include "opto/convertnode.hpp"
 1125 #include "runtime/objectMonitor.hpp"
 1126 
 1127 extern RegMask _ANY_REG32_mask;
 1128 extern RegMask _ANY_REG_mask;
 1129 extern RegMask _PTR_REG_mask;
 1130 extern RegMask _NO_SPECIAL_REG32_mask;
 1131 extern RegMask _NO_SPECIAL_REG_mask;
 1132 extern RegMask _NO_SPECIAL_PTR_REG_mask;
 1133 
 1134 class CallStubImpl {
 1135 
 1136   //--------------------------------------------------------------
 1137   //---<  Used for optimization in Compile::shorten_branches  >---
 1138   //--------------------------------------------------------------
 1139 
 1140  public:
 1141   // Size of call trampoline stub.
 1142   static uint size_call_trampoline() {
 1143     return 0; // no call trampolines on this platform
 1144   }
 1145 
 1146   // number of relocations needed by a call trampoline stub
 1147   static uint reloc_call_trampoline() {
 1148     return 0; // no call trampolines on this platform
 1149   }
 1150 };
 1151 
 1152 class HandlerImpl {
 1153 
 1154  public:
 1155 
 1156   static int emit_exception_handler(CodeBuffer &cbuf);
 1157   static int emit_deopt_handler(CodeBuffer& cbuf);
 1158 
 1159   static uint size_exception_handler() {
 1160     return MacroAssembler::far_codestub_branch_size();
 1161   }
 1162 
 1163   static uint size_deopt_handler() {
 1164     // count one adr and one far branch instruction
 1165     return NativeInstruction::instruction_size + MacroAssembler::far_codestub_branch_size();
 1166   }
 1167 };
 1168 
 1169 class Node::PD {
 1170 public:
 1171   enum NodeFlags {
 1172     _last_flag = Node::_last_flag
 1173   };
 1174 };
 1175 
 1176   bool is_CAS(int opcode, bool maybe_volatile);
 1177 
 1178   // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
 1179 
 1180   bool unnecessary_acquire(const Node *barrier);
 1181   bool needs_acquiring_load(const Node *load);
 1182 
 1183   // predicates controlling emit of str<x>/stlr<x> and associated dmbs
 1184 
 1185   bool unnecessary_release(const Node *barrier);
 1186   bool unnecessary_volatile(const Node *barrier);
 1187   bool needs_releasing_store(const Node *store);
 1188 
 1189   // predicate controlling translation of CompareAndSwapX
 1190   bool needs_acquiring_load_exclusive(const Node *load);
 1191 
 1192   // predicate controlling addressing modes
 1193   bool size_fits_all_mem_uses(AddPNode* addp, int shift);
 1194 %}
 1195 
 1196 source %{
 1197 
 1198   // Derived RegMask with conditionally allocatable registers
 1199 
 1200   void PhaseOutput::pd_perform_mach_node_analysis() {
 1201   }
 1202 
 1203   int MachNode::pd_alignment_required() const {
 1204     return 1;
 1205   }
 1206 
 1207   int MachNode::compute_padding(int current_offset) const {
 1208     return 0;
 1209   }
 1210 
 1211   RegMask _ANY_REG32_mask;
 1212   RegMask _ANY_REG_mask;
 1213   RegMask _PTR_REG_mask;
 1214   RegMask _NO_SPECIAL_REG32_mask;
 1215   RegMask _NO_SPECIAL_REG_mask;
 1216   RegMask _NO_SPECIAL_PTR_REG_mask;
 1217 
 1218   void reg_mask_init() {
 1219     // We derive below RegMask(s) from the ones which are auto-generated from
 1220     // adlc register classes to make AArch64 rheapbase (r27) and rfp (r29)
 1221     // registers conditionally reserved.
 1222 
 1223     _ANY_REG32_mask = _ALL_REG32_mask;
 1224     _ANY_REG32_mask.Remove(OptoReg::as_OptoReg(r31_sp->as_VMReg()));
 1225 
 1226     _ANY_REG_mask = _ALL_REG_mask;
 1227 
 1228     _PTR_REG_mask = _ALL_REG_mask;
 1229 
 1230     _NO_SPECIAL_REG32_mask = _ALL_REG32_mask;
 1231     _NO_SPECIAL_REG32_mask.SUBTRACT(_NON_ALLOCATABLE_REG32_mask);
 1232 
 1233     _NO_SPECIAL_REG_mask = _ALL_REG_mask;
 1234     _NO_SPECIAL_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
 1235 
 1236     _NO_SPECIAL_PTR_REG_mask = _ALL_REG_mask;
 1237     _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
 1238 
 1239     // r27 is not allocatable when compressed oops is on and heapbase is not
 1240     // zero, compressed klass pointers doesn't use r27 after JDK-8234794
 1241     if (UseCompressedOops && (CompressedOops::ptrs_base() != NULL)) {
 1242       _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
 1243       _NO_SPECIAL_REG_mask.SUBTRACT(_HEAPBASE_REG_mask);
 1244       _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_HEAPBASE_REG_mask);
 1245     }
 1246 
 1247     // r29 is not allocatable when PreserveFramePointer is on
 1248     if (PreserveFramePointer) {
 1249       _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
 1250       _NO_SPECIAL_REG_mask.SUBTRACT(_FP_REG_mask);
 1251       _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_FP_REG_mask);
 1252     }
 1253   }
 1254 
 1255   // Optimizaton of volatile gets and puts
 1256   // -------------------------------------
 1257   //
 1258   // AArch64 has ldar<x> and stlr<x> instructions which we can safely
 1259   // use to implement volatile reads and writes. For a volatile read
 1260   // we simply need
 1261   //
 1262   //   ldar<x>
 1263   //
 1264   // and for a volatile write we need
 1265   //
 1266   //   stlr<x>
 1267   //
 1268   // Alternatively, we can implement them by pairing a normal
 1269   // load/store with a memory barrier. For a volatile read we need
 1270   //
 1271   //   ldr<x>
 1272   //   dmb ishld
 1273   //
 1274   // for a volatile write
 1275   //
 1276   //   dmb ish
 1277   //   str<x>
 1278   //   dmb ish
 1279   //
 1280   // We can also use ldaxr and stlxr to implement compare and swap CAS
 1281   // sequences. These are normally translated to an instruction
 1282   // sequence like the following
 1283   //
 1284   //   dmb      ish
 1285   // retry:
 1286   //   ldxr<x>   rval raddr
 1287   //   cmp       rval rold
 1288   //   b.ne done
 1289   //   stlxr<x>  rval, rnew, rold
 1290   //   cbnz      rval retry
 1291   // done:
 1292   //   cset      r0, eq
 1293   //   dmb ishld
 1294   //
 1295   // Note that the exclusive store is already using an stlxr
 1296   // instruction. That is required to ensure visibility to other
 1297   // threads of the exclusive write (assuming it succeeds) before that
 1298   // of any subsequent writes.
 1299   //
 1300   // The following instruction sequence is an improvement on the above
 1301   //
 1302   // retry:
 1303   //   ldaxr<x>  rval raddr
 1304   //   cmp       rval rold
 1305   //   b.ne done
 1306   //   stlxr<x>  rval, rnew, rold
 1307   //   cbnz      rval retry
 1308   // done:
 1309   //   cset      r0, eq
 1310   //
 1311   // We don't need the leading dmb ish since the stlxr guarantees
 1312   // visibility of prior writes in the case that the swap is
 1313   // successful. Crucially we don't have to worry about the case where
 1314   // the swap is not successful since no valid program should be
 1315   // relying on visibility of prior changes by the attempting thread
 1316   // in the case where the CAS fails.
 1317   //
 1318   // Similarly, we don't need the trailing dmb ishld if we substitute
 1319   // an ldaxr instruction since that will provide all the guarantees we
 1320   // require regarding observation of changes made by other threads
 1321   // before any change to the CAS address observed by the load.
 1322   //
 1323   // In order to generate the desired instruction sequence we need to
 1324   // be able to identify specific 'signature' ideal graph node
 1325   // sequences which i) occur as a translation of a volatile reads or
 1326   // writes or CAS operations and ii) do not occur through any other
 1327   // translation or graph transformation. We can then provide
 1328   // alternative aldc matching rules which translate these node
 1329   // sequences to the desired machine code sequences. Selection of the
 1330   // alternative rules can be implemented by predicates which identify
 1331   // the relevant node sequences.
 1332   //
 1333   // The ideal graph generator translates a volatile read to the node
 1334   // sequence
 1335   //
 1336   //   LoadX[mo_acquire]
 1337   //   MemBarAcquire
 1338   //
 1339   // As a special case when using the compressed oops optimization we
 1340   // may also see this variant
 1341   //
 1342   //   LoadN[mo_acquire]
 1343   //   DecodeN
 1344   //   MemBarAcquire
 1345   //
 1346   // A volatile write is translated to the node sequence
 1347   //
 1348   //   MemBarRelease
 1349   //   StoreX[mo_release] {CardMark}-optional
 1350   //   MemBarVolatile
 1351   //
 1352   // n.b. the above node patterns are generated with a strict
 1353   // 'signature' configuration of input and output dependencies (see
 1354   // the predicates below for exact details). The card mark may be as
 1355   // simple as a few extra nodes or, in a few GC configurations, may
 1356   // include more complex control flow between the leading and
 1357   // trailing memory barriers. However, whatever the card mark
 1358   // configuration these signatures are unique to translated volatile
 1359   // reads/stores -- they will not appear as a result of any other
 1360   // bytecode translation or inlining nor as a consequence of
 1361   // optimizing transforms.
 1362   //
 1363   // We also want to catch inlined unsafe volatile gets and puts and
 1364   // be able to implement them using either ldar<x>/stlr<x> or some
 1365   // combination of ldr<x>/stlr<x> and dmb instructions.
 1366   //
 1367   // Inlined unsafe volatiles puts manifest as a minor variant of the
 1368   // normal volatile put node sequence containing an extra cpuorder
 1369   // membar
 1370   //
 1371   //   MemBarRelease
 1372   //   MemBarCPUOrder
 1373   //   StoreX[mo_release] {CardMark}-optional
 1374   //   MemBarCPUOrder
 1375   //   MemBarVolatile
 1376   //
 1377   // n.b. as an aside, a cpuorder membar is not itself subject to
 1378   // matching and translation by adlc rules.  However, the rule
 1379   // predicates need to detect its presence in order to correctly
 1380   // select the desired adlc rules.
 1381   //
 1382   // Inlined unsafe volatile gets manifest as a slightly different
 1383   // node sequence to a normal volatile get because of the
 1384   // introduction of some CPUOrder memory barriers to bracket the
 1385   // Load. However, but the same basic skeleton of a LoadX feeding a
 1386   // MemBarAcquire, possibly through an optional DecodeN, is still
 1387   // present
 1388   //
 1389   //   MemBarCPUOrder
 1390   //        ||       \\
 1391   //   MemBarCPUOrder LoadX[mo_acquire]
 1392   //        ||            |
 1393   //        ||       {DecodeN} optional
 1394   //        ||       /
 1395   //     MemBarAcquire
 1396   //
 1397   // In this case the acquire membar does not directly depend on the
 1398   // load. However, we can be sure that the load is generated from an
 1399   // inlined unsafe volatile get if we see it dependent on this unique
 1400   // sequence of membar nodes. Similarly, given an acquire membar we
 1401   // can know that it was added because of an inlined unsafe volatile
 1402   // get if it is fed and feeds a cpuorder membar and if its feed
 1403   // membar also feeds an acquiring load.
 1404   //
 1405   // Finally an inlined (Unsafe) CAS operation is translated to the
 1406   // following ideal graph
 1407   //
 1408   //   MemBarRelease
 1409   //   MemBarCPUOrder
 1410   //   CompareAndSwapX {CardMark}-optional
 1411   //   MemBarCPUOrder
 1412   //   MemBarAcquire
 1413   //
 1414   // So, where we can identify these volatile read and write
 1415   // signatures we can choose to plant either of the above two code
 1416   // sequences. For a volatile read we can simply plant a normal
 1417   // ldr<x> and translate the MemBarAcquire to a dmb. However, we can
 1418   // also choose to inhibit translation of the MemBarAcquire and
 1419   // inhibit planting of the ldr<x>, instead planting an ldar<x>.
 1420   //
 1421   // When we recognise a volatile store signature we can choose to
 1422   // plant at a dmb ish as a translation for the MemBarRelease, a
 1423   // normal str<x> and then a dmb ish for the MemBarVolatile.
 1424   // Alternatively, we can inhibit translation of the MemBarRelease
 1425   // and MemBarVolatile and instead plant a simple stlr<x>
 1426   // instruction.
 1427   //
 1428   // when we recognise a CAS signature we can choose to plant a dmb
 1429   // ish as a translation for the MemBarRelease, the conventional
 1430   // macro-instruction sequence for the CompareAndSwap node (which
 1431   // uses ldxr<x>) and then a dmb ishld for the MemBarAcquire.
 1432   // Alternatively, we can elide generation of the dmb instructions
 1433   // and plant the alternative CompareAndSwap macro-instruction
 1434   // sequence (which uses ldaxr<x>).
 1435   //
 1436   // Of course, the above only applies when we see these signature
 1437   // configurations. We still want to plant dmb instructions in any
 1438   // other cases where we may see a MemBarAcquire, MemBarRelease or
 1439   // MemBarVolatile. For example, at the end of a constructor which
 1440   // writes final/volatile fields we will see a MemBarRelease
 1441   // instruction and this needs a 'dmb ish' lest we risk the
 1442   // constructed object being visible without making the
 1443   // final/volatile field writes visible.
 1444   //
 1445   // n.b. the translation rules below which rely on detection of the
 1446   // volatile signatures and insert ldar<x> or stlr<x> are failsafe.
 1447   // If we see anything other than the signature configurations we
 1448   // always just translate the loads and stores to ldr<x> and str<x>
 1449   // and translate acquire, release and volatile membars to the
 1450   // relevant dmb instructions.
 1451   //
 1452 
 1453   // is_CAS(int opcode, bool maybe_volatile)
 1454   //
 1455   // return true if opcode is one of the possible CompareAndSwapX
 1456   // values otherwise false.
 1457 
 1458   bool is_CAS(int opcode, bool maybe_volatile)
 1459   {
 1460     switch(opcode) {
 1461       // We handle these
 1462     case Op_CompareAndSwapI:
 1463     case Op_CompareAndSwapL:
 1464     case Op_CompareAndSwapP:
 1465     case Op_CompareAndSwapN:
 1466     case Op_ShenandoahCompareAndSwapP:
 1467     case Op_ShenandoahCompareAndSwapN:
 1468     case Op_CompareAndSwapB:
 1469     case Op_CompareAndSwapS:
 1470     case Op_GetAndSetI:
 1471     case Op_GetAndSetL:
 1472     case Op_GetAndSetP:
 1473     case Op_GetAndSetN:
 1474     case Op_GetAndAddI:
 1475     case Op_GetAndAddL:
 1476       return true;
 1477     case Op_CompareAndExchangeI:
 1478     case Op_CompareAndExchangeN:
 1479     case Op_CompareAndExchangeB:
 1480     case Op_CompareAndExchangeS:
 1481     case Op_CompareAndExchangeL:
 1482     case Op_CompareAndExchangeP:
 1483     case Op_WeakCompareAndSwapB:
 1484     case Op_WeakCompareAndSwapS:
 1485     case Op_WeakCompareAndSwapI:
 1486     case Op_WeakCompareAndSwapL:
 1487     case Op_WeakCompareAndSwapP:
 1488     case Op_WeakCompareAndSwapN:
 1489     case Op_ShenandoahWeakCompareAndSwapP:
 1490     case Op_ShenandoahWeakCompareAndSwapN:
 1491     case Op_ShenandoahCompareAndExchangeP:
 1492     case Op_ShenandoahCompareAndExchangeN:
 1493       return maybe_volatile;
 1494     default:
 1495       return false;
 1496     }
 1497   }
 1498 
 1499   // helper to determine the maximum number of Phi nodes we may need to
 1500   // traverse when searching from a card mark membar for the merge mem
 1501   // feeding a trailing membar or vice versa
 1502 
 1503 // predicates controlling emit of ldr<x>/ldar<x>
 1504 
 1505 bool unnecessary_acquire(const Node *barrier)
 1506 {
 1507   assert(barrier->is_MemBar(), "expecting a membar");
 1508 
 1509   MemBarNode* mb = barrier->as_MemBar();
 1510 
 1511   if (mb->trailing_load()) {
 1512     return true;
 1513   }
 1514 
 1515   if (mb->trailing_load_store()) {
 1516     Node* load_store = mb->in(MemBarNode::Precedent);
 1517     assert(load_store->is_LoadStore(), "unexpected graph shape");
 1518     return is_CAS(load_store->Opcode(), true);
 1519   }
 1520 
 1521   return false;
 1522 }
 1523 
 1524 bool needs_acquiring_load(const Node *n)
 1525 {
 1526   assert(n->is_Load(), "expecting a load");
 1527   LoadNode *ld = n->as_Load();
 1528   return ld->is_acquire();
 1529 }
 1530 
 1531 bool unnecessary_release(const Node *n)
 1532 {
 1533   assert((n->is_MemBar() &&
 1534           n->Opcode() == Op_MemBarRelease),
 1535          "expecting a release membar");
 1536 
 1537   MemBarNode *barrier = n->as_MemBar();
 1538   if (!barrier->leading()) {
 1539     return false;
 1540   } else {
 1541     Node* trailing = barrier->trailing_membar();
 1542     MemBarNode* trailing_mb = trailing->as_MemBar();
 1543     assert(trailing_mb->trailing(), "Not a trailing membar?");
 1544     assert(trailing_mb->leading_membar() == n, "inconsistent leading/trailing membars");
 1545 
 1546     Node* mem = trailing_mb->in(MemBarNode::Precedent);
 1547     if (mem->is_Store()) {
 1548       assert(mem->as_Store()->is_release(), "");
 1549       assert(trailing_mb->Opcode() == Op_MemBarVolatile, "");
 1550       return true;
 1551     } else {
 1552       assert(mem->is_LoadStore(), "");
 1553       assert(trailing_mb->Opcode() == Op_MemBarAcquire, "");
 1554       return is_CAS(mem->Opcode(), true);
 1555     }
 1556   }
 1557   return false;
 1558 }
 1559 
 1560 bool unnecessary_volatile(const Node *n)
 1561 {
 1562   // assert n->is_MemBar();
 1563   MemBarNode *mbvol = n->as_MemBar();
 1564 
 1565   bool release = mbvol->trailing_store();
 1566   assert(!release || (mbvol->in(MemBarNode::Precedent)->is_Store() && mbvol->in(MemBarNode::Precedent)->as_Store()->is_release()), "");
 1567 #ifdef ASSERT
 1568   if (release) {
 1569     Node* leading = mbvol->leading_membar();
 1570     assert(leading->Opcode() == Op_MemBarRelease, "");
 1571     assert(leading->as_MemBar()->leading_store(), "");
 1572     assert(leading->as_MemBar()->trailing_membar() == mbvol, "");
 1573   }
 1574 #endif
 1575 
 1576   return release;
 1577 }
 1578 
 1579 // predicates controlling emit of str<x>/stlr<x>
 1580 
 1581 bool needs_releasing_store(const Node *n)
 1582 {
 1583   // assert n->is_Store();
 1584   StoreNode *st = n->as_Store();
 1585   return st->trailing_membar() != NULL;
 1586 }
 1587 
 1588 // predicate controlling translation of CAS
 1589 //
 1590 // returns true if CAS needs to use an acquiring load otherwise false
 1591 
 1592 bool needs_acquiring_load_exclusive(const Node *n)
 1593 {
 1594   assert(is_CAS(n->Opcode(), true), "expecting a compare and swap");
 1595   LoadStoreNode* ldst = n->as_LoadStore();
 1596   if (is_CAS(n->Opcode(), false)) {
 1597     assert(ldst->trailing_membar() != NULL, "expected trailing membar");
 1598   } else {
 1599     return ldst->trailing_membar() != NULL;
 1600   }
 1601 
 1602   // so we can just return true here
 1603   return true;
 1604 }
 1605 
 1606 #define __ _masm.
 1607 
 1608 // advance declarations for helper functions to convert register
 1609 // indices to register objects
 1610 
 1611 // the ad file has to provide implementations of certain methods
 1612 // expected by the generic code
 1613 //
 1614 // REQUIRED FUNCTIONALITY
 1615 
 1616 //=============================================================================
 1617 
 1618 // !!!!! Special hack to get all types of calls to specify the byte offset
 1619 //       from the start of the call to the point where the return address
 1620 //       will point.
 1621 
 1622 int MachCallStaticJavaNode::ret_addr_offset()
 1623 {
 1624   // call should be a simple bl
 1625   int off = 4;
 1626   return off;
 1627 }
 1628 
 1629 int MachCallDynamicJavaNode::ret_addr_offset()
 1630 {
 1631   return 16; // movz, movk, movk, bl
 1632 }
 1633 
 1634 int MachCallRuntimeNode::ret_addr_offset() {
 1635   // for generated stubs the call will be
 1636   //   bl(addr)
 1637   // or with far branches
 1638   //   bl(trampoline_stub)
 1639   // for real runtime callouts it will be six instructions
 1640   // see aarch64_enc_java_to_runtime
 1641   //   adr(rscratch2, retaddr)
 1642   //   lea(rscratch1, RuntimeAddress(addr)
 1643   //   stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)))
 1644   //   blr(rscratch1)
 1645   CodeBlob *cb = CodeCache::find_blob(_entry_point);
 1646   if (cb) {
 1647     return 1 * NativeInstruction::instruction_size;
 1648   } else {
 1649     return 6 * NativeInstruction::instruction_size;
 1650   }
 1651 }
 1652 
 1653 //=============================================================================
 1654 
 1655 #ifndef PRODUCT
 1656 void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 1657   st->print("BREAKPOINT");
 1658 }
 1659 #endif
 1660 
 1661 void MachBreakpointNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 1662   C2_MacroAssembler _masm(&cbuf);
 1663   __ brk(0);
 1664 }
 1665 
 1666 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
 1667   return MachNode::size(ra_);
 1668 }
 1669 
 1670 //=============================================================================
 1671 
 1672 #ifndef PRODUCT
 1673   void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
 1674     st->print("nop \t# %d bytes pad for loops and calls", _count);
 1675   }
 1676 #endif
 1677 
 1678   void MachNopNode::emit(CodeBuffer &cbuf, PhaseRegAlloc*) const {
 1679     C2_MacroAssembler _masm(&cbuf);
 1680     for (int i = 0; i < _count; i++) {
 1681       __ nop();
 1682     }
 1683   }
 1684 
 1685   uint MachNopNode::size(PhaseRegAlloc*) const {
 1686     return _count * NativeInstruction::instruction_size;
 1687   }
 1688 
 1689 //=============================================================================
 1690 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
 1691 
 1692 int ConstantTable::calculate_table_base_offset() const {
 1693   return 0;  // absolute addressing, no offset
 1694 }
 1695 
 1696 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
 1697 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
 1698   ShouldNotReachHere();
 1699 }
 1700 
 1701 void MachConstantBaseNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const {
 1702   // Empty encoding
 1703 }
 1704 
 1705 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
 1706   return 0;
 1707 }
 1708 
 1709 #ifndef PRODUCT
 1710 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
 1711   st->print("-- \t// MachConstantBaseNode (empty encoding)");
 1712 }
 1713 #endif
 1714 
 1715 #ifndef PRODUCT
 1716 void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 1717   Compile* C = ra_->C;
 1718 
 1719   int framesize = C->output()->frame_slots() << LogBytesPerInt;
 1720 
 1721   if (C->output()->need_stack_bang(framesize))
 1722     st->print("# stack bang size=%d\n\t", framesize);
 1723 
 1724   if (VM_Version::use_rop_protection()) {
 1725     st->print("ldr zr, [lr]\n\t");
 1726     st->print("pacia  lr, rfp\n\t");
 1727   }
 1728   if (framesize < ((1 << 9) + 2 * wordSize)) {
 1729     st->print("sub  sp, sp, #%d\n\t", framesize);
 1730     st->print("stp  rfp, lr, [sp, #%d]", framesize - 2 * wordSize);
 1731     if (PreserveFramePointer) st->print("\n\tadd  rfp, sp, #%d", framesize - 2 * wordSize);
 1732   } else {
 1733     st->print("stp  lr, rfp, [sp, #%d]!\n\t", -(2 * wordSize));
 1734     if (PreserveFramePointer) st->print("mov  rfp, sp\n\t");
 1735     st->print("mov  rscratch1, #%d\n\t", framesize - 2 * wordSize);
 1736     st->print("sub  sp, sp, rscratch1");
 1737   }
 1738   if (C->stub_function() == NULL && BarrierSet::barrier_set()->barrier_set_nmethod() != NULL) {
 1739     st->print("\n\t");
 1740     st->print("ldr  rscratch1, [guard]\n\t");
 1741     st->print("dmb ishld\n\t");
 1742     st->print("ldr  rscratch2, [rthread, #thread_disarmed_guard_value_offset]\n\t");
 1743     st->print("cmp  rscratch1, rscratch2\n\t");
 1744     st->print("b.eq skip");
 1745     st->print("\n\t");
 1746     st->print("blr #nmethod_entry_barrier_stub\n\t");
 1747     st->print("b skip\n\t");
 1748     st->print("guard: int\n\t");
 1749     st->print("\n\t");
 1750     st->print("skip:\n\t");
 1751   }
 1752 }
 1753 #endif
 1754 
 1755 void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 1756   Compile* C = ra_->C;
 1757   C2_MacroAssembler _masm(&cbuf);
 1758 
 1759   // n.b. frame size includes space for return pc and rfp
 1760   const int framesize = C->output()->frame_size_in_bytes();
 1761 
 1762   // insert a nop at the start of the prolog so we can patch in a
 1763   // branch if we need to invalidate the method later
 1764   __ nop();
 1765 
 1766   if (C->clinit_barrier_on_entry()) {
 1767     assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
 1768 
 1769     Label L_skip_barrier;
 1770 
 1771     __ mov_metadata(rscratch2, C->method()->holder()->constant_encoding());
 1772     __ clinit_barrier(rscratch2, rscratch1, &L_skip_barrier);
 1773     __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
 1774     __ bind(L_skip_barrier);
 1775   }
 1776 
 1777   if (C->max_vector_size() > 0) {
 1778     __ reinitialize_ptrue();
 1779   }
 1780 
 1781   int bangsize = C->output()->bang_size_in_bytes();
 1782   if (C->output()->need_stack_bang(bangsize))
 1783     __ generate_stack_overflow_check(bangsize);
 1784 
 1785   __ build_frame(framesize);
 1786 












 1787   if (C->stub_function() == NULL) {
 1788     BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
 1789     if (BarrierSet::barrier_set()->barrier_set_nmethod() != NULL) {
 1790       // Dummy labels for just measuring the code size
 1791       Label dummy_slow_path;
 1792       Label dummy_continuation;
 1793       Label dummy_guard;
 1794       Label* slow_path = &dummy_slow_path;
 1795       Label* continuation = &dummy_continuation;
 1796       Label* guard = &dummy_guard;
 1797       if (!Compile::current()->output()->in_scratch_emit_size()) {
 1798         // Use real labels from actual stub when not emitting code for the purpose of measuring its size
 1799         C2EntryBarrierStub* stub = new (Compile::current()->comp_arena()) C2EntryBarrierStub();
 1800         Compile::current()->output()->add_stub(stub);
 1801         slow_path = &stub->entry();
 1802         continuation = &stub->continuation();
 1803         guard = &stub->guard();
 1804       }
 1805       // In the C2 code, we move the non-hot part of nmethod entry barriers out-of-line to a stub.
 1806       bs->nmethod_entry_barrier(&_masm, slow_path, continuation, guard);
 1807     }
 1808   }
 1809 
 1810   if (VerifyStackAtCalls) {
 1811     Unimplemented();
 1812   }
 1813 
 1814   C->output()->set_frame_complete(cbuf.insts_size());
 1815 
 1816   if (C->has_mach_constant_base_node()) {
 1817     // NOTE: We set the table base offset here because users might be
 1818     // emitted before MachConstantBaseNode.
 1819     ConstantTable& constant_table = C->output()->constant_table();
 1820     constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
 1821   }
 1822 }
 1823 
 1824 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
 1825 {
 1826   return MachNode::size(ra_); // too many variables; just compute it
 1827                               // the hard way
 1828 }
 1829 
 1830 int MachPrologNode::reloc() const
 1831 {
 1832   return 0;
 1833 }
 1834 
 1835 //=============================================================================
 1836 
 1837 #ifndef PRODUCT
 1838 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 1839   Compile* C = ra_->C;
 1840   int framesize = C->output()->frame_slots() << LogBytesPerInt;
 1841 
 1842   st->print("# pop frame %d\n\t",framesize);
 1843 
 1844   if (framesize == 0) {
 1845     st->print("ldp  lr, rfp, [sp],#%d\n\t", (2 * wordSize));
 1846   } else if (framesize < ((1 << 9) + 2 * wordSize)) {
 1847     st->print("ldp  lr, rfp, [sp,#%d]\n\t", framesize - 2 * wordSize);
 1848     st->print("add  sp, sp, #%d\n\t", framesize);
 1849   } else {
 1850     st->print("mov  rscratch1, #%d\n\t", framesize - 2 * wordSize);
 1851     st->print("add  sp, sp, rscratch1\n\t");
 1852     st->print("ldp  lr, rfp, [sp],#%d\n\t", (2 * wordSize));
 1853   }
 1854   if (VM_Version::use_rop_protection()) {
 1855     st->print("autia lr, rfp\n\t");
 1856     st->print("ldr zr, [lr]\n\t");
 1857   }
 1858 
 1859   if (do_polling() && C->is_method_compilation()) {
 1860     st->print("# test polling word\n\t");
 1861     st->print("ldr  rscratch1, [rthread],#%d\n\t", in_bytes(JavaThread::polling_word_offset()));
 1862     st->print("cmp  sp, rscratch1\n\t");
 1863     st->print("bhi #slow_path");
 1864   }
 1865 }
 1866 #endif
 1867 
 1868 void MachEpilogNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 1869   Compile* C = ra_->C;
 1870   C2_MacroAssembler _masm(&cbuf);
 1871   int framesize = C->output()->frame_slots() << LogBytesPerInt;
 1872 
 1873   __ remove_frame(framesize);
 1874 
 1875   if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
 1876     __ reserved_stack_check();
 1877   }
 1878 
 1879   if (do_polling() && C->is_method_compilation()) {
 1880     Label dummy_label;
 1881     Label* code_stub = &dummy_label;
 1882     if (!C->output()->in_scratch_emit_size()) {
 1883       C2SafepointPollStub* stub = new (C->comp_arena()) C2SafepointPollStub(__ offset());
 1884       C->output()->add_stub(stub);
 1885       code_stub = &stub->entry();
 1886     }
 1887     __ relocate(relocInfo::poll_return_type);
 1888     __ safepoint_poll(*code_stub, true /* at_return */, false /* acquire */, true /* in_nmethod */);
 1889   }
 1890 }
 1891 
 1892 uint MachEpilogNode::size(PhaseRegAlloc *ra_) const {
 1893   // Variable size. Determine dynamically.
 1894   return MachNode::size(ra_);
 1895 }
 1896 
 1897 int MachEpilogNode::reloc() const {
 1898   // Return number of relocatable values contained in this instruction.
 1899   return 1; // 1 for polling page.
 1900 }
 1901 
 1902 const Pipeline * MachEpilogNode::pipeline() const {
 1903   return MachNode::pipeline_class();
 1904 }
 1905 
 1906 //=============================================================================
 1907 
 1908 // Figure out which register class each belongs in: rc_int, rc_float or
 1909 // rc_stack.
 1910 enum RC { rc_bad, rc_int, rc_float, rc_predicate, rc_stack };
 1911 
 1912 static enum RC rc_class(OptoReg::Name reg) {
 1913 
 1914   if (reg == OptoReg::Bad) {
 1915     return rc_bad;
 1916   }
 1917 
 1918   // we have 32 int registers * 2 halves
 1919   int slots_of_int_registers = Register::number_of_registers * Register::max_slots_per_register;
 1920 
 1921   if (reg < slots_of_int_registers) {
 1922     return rc_int;
 1923   }
 1924 
 1925   // we have 32 float register * 8 halves
 1926   int slots_of_float_registers = FloatRegister::number_of_registers * FloatRegister::max_slots_per_register;
 1927   if (reg < slots_of_int_registers + slots_of_float_registers) {
 1928     return rc_float;
 1929   }
 1930 
 1931   int slots_of_predicate_registers = PRegister::number_of_registers * PRegister::max_slots_per_register;
 1932   if (reg < slots_of_int_registers + slots_of_float_registers + slots_of_predicate_registers) {
 1933     return rc_predicate;
 1934   }
 1935 
 1936   // Between predicate regs & stack is the flags.
 1937   assert(OptoReg::is_stack(reg), "blow up if spilling flags");
 1938 
 1939   return rc_stack;
 1940 }
 1941 
 1942 uint MachSpillCopyNode::implementation(CodeBuffer *cbuf, PhaseRegAlloc *ra_, bool do_size, outputStream *st) const {
 1943   Compile* C = ra_->C;
 1944 
 1945   // Get registers to move.
 1946   OptoReg::Name src_hi = ra_->get_reg_second(in(1));
 1947   OptoReg::Name src_lo = ra_->get_reg_first(in(1));
 1948   OptoReg::Name dst_hi = ra_->get_reg_second(this);
 1949   OptoReg::Name dst_lo = ra_->get_reg_first(this);
 1950 
 1951   enum RC src_hi_rc = rc_class(src_hi);
 1952   enum RC src_lo_rc = rc_class(src_lo);
 1953   enum RC dst_hi_rc = rc_class(dst_hi);
 1954   enum RC dst_lo_rc = rc_class(dst_lo);
 1955 
 1956   assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
 1957 
 1958   if (src_hi != OptoReg::Bad && !bottom_type()->isa_vectmask()) {
 1959     assert((src_lo&1)==0 && src_lo+1==src_hi &&
 1960            (dst_lo&1)==0 && dst_lo+1==dst_hi,
 1961            "expected aligned-adjacent pairs");
 1962   }
 1963 
 1964   if (src_lo == dst_lo && src_hi == dst_hi) {
 1965     return 0;            // Self copy, no move.
 1966   }
 1967 
 1968   bool is64 = (src_lo & 1) == 0 && src_lo + 1 == src_hi &&
 1969               (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi;
 1970   int src_offset = ra_->reg2offset(src_lo);
 1971   int dst_offset = ra_->reg2offset(dst_lo);
 1972 
 1973   if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
 1974     uint ireg = ideal_reg();
 1975     if (ireg == Op_VecA && cbuf) {
 1976       C2_MacroAssembler _masm(cbuf);
 1977       int sve_vector_reg_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
 1978       if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
 1979         // stack->stack
 1980         __ spill_copy_sve_vector_stack_to_stack(src_offset, dst_offset,
 1981                                                 sve_vector_reg_size_in_bytes);
 1982       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
 1983         __ spill_sve_vector(as_FloatRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
 1984                             sve_vector_reg_size_in_bytes);
 1985       } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
 1986         __ unspill_sve_vector(as_FloatRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
 1987                               sve_vector_reg_size_in_bytes);
 1988       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
 1989         __ sve_orr(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 1990                    as_FloatRegister(Matcher::_regEncode[src_lo]),
 1991                    as_FloatRegister(Matcher::_regEncode[src_lo]));
 1992       } else {
 1993         ShouldNotReachHere();
 1994       }
 1995     } else if (cbuf) {
 1996       assert(ireg == Op_VecD || ireg == Op_VecX, "must be 64 bit or 128 bit vector");
 1997       C2_MacroAssembler _masm(cbuf);
 1998       assert((src_lo_rc != rc_int && dst_lo_rc != rc_int), "sanity");
 1999       if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
 2000         // stack->stack
 2001         assert((src_offset & 7) == 0 && (dst_offset & 7) == 0, "unaligned stack offset");
 2002         if (ireg == Op_VecD) {
 2003           __ unspill(rscratch1, true, src_offset);
 2004           __ spill(rscratch1, true, dst_offset);
 2005         } else {
 2006           __ spill_copy128(src_offset, dst_offset);
 2007         }
 2008       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
 2009         __ mov(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2010                ireg == Op_VecD ? __ T8B : __ T16B,
 2011                as_FloatRegister(Matcher::_regEncode[src_lo]));
 2012       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
 2013         __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
 2014                  ireg == Op_VecD ? __ D : __ Q,
 2015                  ra_->reg2offset(dst_lo));
 2016       } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
 2017         __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2018                    ireg == Op_VecD ? __ D : __ Q,
 2019                    ra_->reg2offset(src_lo));
 2020       } else {
 2021         ShouldNotReachHere();
 2022       }
 2023     }
 2024   } else if (cbuf) {
 2025     C2_MacroAssembler _masm(cbuf);
 2026     switch (src_lo_rc) {
 2027     case rc_int:
 2028       if (dst_lo_rc == rc_int) {  // gpr --> gpr copy
 2029         if (is64) {
 2030             __ mov(as_Register(Matcher::_regEncode[dst_lo]),
 2031                    as_Register(Matcher::_regEncode[src_lo]));
 2032         } else {
 2033             C2_MacroAssembler _masm(cbuf);
 2034             __ movw(as_Register(Matcher::_regEncode[dst_lo]),
 2035                     as_Register(Matcher::_regEncode[src_lo]));
 2036         }
 2037       } else if (dst_lo_rc == rc_float) { // gpr --> fpr copy
 2038         if (is64) {
 2039             __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2040                      as_Register(Matcher::_regEncode[src_lo]));
 2041         } else {
 2042             __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2043                      as_Register(Matcher::_regEncode[src_lo]));
 2044         }
 2045       } else {                    // gpr --> stack spill
 2046         assert(dst_lo_rc == rc_stack, "spill to bad register class");
 2047         __ spill(as_Register(Matcher::_regEncode[src_lo]), is64, dst_offset);
 2048       }
 2049       break;
 2050     case rc_float:
 2051       if (dst_lo_rc == rc_int) {  // fpr --> gpr copy
 2052         if (is64) {
 2053             __ fmovd(as_Register(Matcher::_regEncode[dst_lo]),
 2054                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2055         } else {
 2056             __ fmovs(as_Register(Matcher::_regEncode[dst_lo]),
 2057                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2058         }
 2059       } else if (dst_lo_rc == rc_float) { // fpr --> fpr copy
 2060         if (is64) {
 2061             __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2062                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2063         } else {
 2064             __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2065                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2066         }
 2067       } else {                    // fpr --> stack spill
 2068         assert(dst_lo_rc == rc_stack, "spill to bad register class");
 2069         __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
 2070                  is64 ? __ D : __ S, dst_offset);
 2071       }
 2072       break;
 2073     case rc_stack:
 2074       if (dst_lo_rc == rc_int) {  // stack --> gpr load
 2075         __ unspill(as_Register(Matcher::_regEncode[dst_lo]), is64, src_offset);
 2076       } else if (dst_lo_rc == rc_float) { // stack --> fpr load
 2077         __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2078                    is64 ? __ D : __ S, src_offset);
 2079       } else if (dst_lo_rc == rc_predicate) {
 2080         __ unspill_sve_predicate(as_PRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
 2081                                  Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
 2082       } else {                    // stack --> stack copy
 2083         assert(dst_lo_rc == rc_stack, "spill to bad register class");
 2084         if (ideal_reg() == Op_RegVectMask) {
 2085           __ spill_copy_sve_predicate_stack_to_stack(src_offset, dst_offset,
 2086                                                      Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
 2087         } else {
 2088           __ unspill(rscratch1, is64, src_offset);
 2089           __ spill(rscratch1, is64, dst_offset);
 2090         }
 2091       }
 2092       break;
 2093     case rc_predicate:
 2094       if (dst_lo_rc == rc_predicate) {
 2095         __ sve_mov(as_PRegister(Matcher::_regEncode[dst_lo]), as_PRegister(Matcher::_regEncode[src_lo]));
 2096       } else if (dst_lo_rc == rc_stack) {
 2097         __ spill_sve_predicate(as_PRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
 2098                                Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
 2099       } else {
 2100         assert(false, "bad src and dst rc_class combination.");
 2101         ShouldNotReachHere();
 2102       }
 2103       break;
 2104     default:
 2105       assert(false, "bad rc_class for spill");
 2106       ShouldNotReachHere();
 2107     }
 2108   }
 2109 
 2110   if (st) {
 2111     st->print("spill ");
 2112     if (src_lo_rc == rc_stack) {
 2113       st->print("[sp, #%d] -> ", ra_->reg2offset(src_lo));
 2114     } else {
 2115       st->print("%s -> ", Matcher::regName[src_lo]);
 2116     }
 2117     if (dst_lo_rc == rc_stack) {
 2118       st->print("[sp, #%d]", ra_->reg2offset(dst_lo));
 2119     } else {
 2120       st->print("%s", Matcher::regName[dst_lo]);
 2121     }
 2122     if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
 2123       int vsize = 0;
 2124       switch (ideal_reg()) {
 2125       case Op_VecD:
 2126         vsize = 64;
 2127         break;
 2128       case Op_VecX:
 2129         vsize = 128;
 2130         break;
 2131       case Op_VecA:
 2132         vsize = Matcher::scalable_vector_reg_size(T_BYTE) * 8;
 2133         break;
 2134       default:
 2135         assert(false, "bad register type for spill");
 2136         ShouldNotReachHere();
 2137       }
 2138       st->print("\t# vector spill size = %d", vsize);
 2139     } else if (ideal_reg() == Op_RegVectMask) {
 2140       assert(Matcher::supports_scalable_vector(), "bad register type for spill");
 2141       int vsize = Matcher::scalable_predicate_reg_slots() * 32;
 2142       st->print("\t# predicate spill size = %d", vsize);
 2143     } else {
 2144       st->print("\t# spill size = %d", is64 ? 64 : 32);
 2145     }
 2146   }
 2147 
 2148   return 0;
 2149 
 2150 }
 2151 
 2152 #ifndef PRODUCT
 2153 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 2154   if (!ra_)
 2155     st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
 2156   else
 2157     implementation(NULL, ra_, false, st);
 2158 }
 2159 #endif
 2160 
 2161 void MachSpillCopyNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 2162   implementation(&cbuf, ra_, false, NULL);
 2163 }
 2164 
 2165 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
 2166   return MachNode::size(ra_);
 2167 }
 2168 
 2169 //=============================================================================
 2170 
 2171 #ifndef PRODUCT
 2172 void BoxLockNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 2173   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 2174   int reg = ra_->get_reg_first(this);
 2175   st->print("add %s, rsp, #%d]\t# box lock",
 2176             Matcher::regName[reg], offset);
 2177 }
 2178 #endif
 2179 
 2180 void BoxLockNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 2181   C2_MacroAssembler _masm(&cbuf);
 2182 
 2183   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 2184   int reg    = ra_->get_encode(this);
 2185 
 2186   // This add will handle any 24-bit signed offset. 24 bits allows an
 2187   // 8 megabyte stack frame.
 2188   __ add(as_Register(reg), sp, offset);
 2189 }
 2190 
 2191 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
 2192   // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_).
 2193   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 2194 
 2195   if (Assembler::operand_valid_for_add_sub_immediate(offset)) {
 2196     return NativeInstruction::instruction_size;
 2197   } else {
 2198     return 2 * NativeInstruction::instruction_size;
 2199   }
 2200 }
 2201 
 2202 //=============================================================================
 2203 
 2204 #ifndef PRODUCT
 2205 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 2206 {
 2207   st->print_cr("# MachUEPNode");
 2208   if (UseCompressedClassPointers) {
 2209     st->print_cr("\tldrw rscratch1, j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
 2210     if (CompressedKlassPointers::shift() != 0) {
 2211       st->print_cr("\tdecode_klass_not_null rscratch1, rscratch1");
 2212     }
 2213   } else {
 2214    st->print_cr("\tldr rscratch1, j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
 2215   }
 2216   st->print_cr("\tcmp r0, rscratch1\t # Inline cache check");
 2217   st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
 2218 }
 2219 #endif
 2220 
 2221 void MachUEPNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
 2222 {
 2223   // This is the unverified entry point.
 2224   C2_MacroAssembler _masm(&cbuf);
 2225 
 2226   __ cmp_klass(j_rarg0, rscratch2, rscratch1);
 2227   Label skip;
 2228   // TODO
 2229   // can we avoid this skip and still use a reloc?
 2230   __ br(Assembler::EQ, skip);
 2231   __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
 2232   __ bind(skip);
 2233 }
 2234 
 2235 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
 2236 {
 2237   return MachNode::size(ra_);
 2238 }
 2239 
 2240 // REQUIRED EMIT CODE
 2241 
 2242 //=============================================================================
 2243 
 2244 // Emit exception handler code.
 2245 int HandlerImpl::emit_exception_handler(CodeBuffer& cbuf)
 2246 {
 2247   // mov rscratch1 #exception_blob_entry_point
 2248   // br rscratch1
 2249   // Note that the code buffer's insts_mark is always relative to insts.
 2250   // That's why we must use the macroassembler to generate a handler.
 2251   C2_MacroAssembler _masm(&cbuf);
 2252   address base = __ start_a_stub(size_exception_handler());
 2253   if (base == NULL) {
 2254     ciEnv::current()->record_failure("CodeCache is full");
 2255     return 0;  // CodeBuffer::expand failed
 2256   }
 2257   int offset = __ offset();
 2258   __ far_jump(RuntimeAddress(OptoRuntime::exception_blob()->entry_point()));
 2259   assert(__ offset() - offset <= (int) size_exception_handler(), "overflow");
 2260   __ end_a_stub();
 2261   return offset;
 2262 }
 2263 
 2264 // Emit deopt handler code.
 2265 int HandlerImpl::emit_deopt_handler(CodeBuffer& cbuf)
 2266 {
 2267   // Note that the code buffer's insts_mark is always relative to insts.
 2268   // That's why we must use the macroassembler to generate a handler.
 2269   C2_MacroAssembler _masm(&cbuf);
 2270   address base = __ start_a_stub(size_deopt_handler());
 2271   if (base == NULL) {
 2272     ciEnv::current()->record_failure("CodeCache is full");
 2273     return 0;  // CodeBuffer::expand failed
 2274   }
 2275   int offset = __ offset();
 2276 
 2277   __ adr(lr, __ pc());
 2278   __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
 2279 
 2280   assert(__ offset() - offset == (int) size_deopt_handler(), "overflow");
 2281   __ end_a_stub();
 2282   return offset;
 2283 }
 2284 
 2285 // REQUIRED MATCHER CODE
 2286 
 2287 //=============================================================================
 2288 
 2289 const bool Matcher::match_rule_supported(int opcode) {
 2290   if (!has_match_rule(opcode))
 2291     return false;
 2292 
 2293   bool ret_value = true;
 2294   switch (opcode) {
 2295     case Op_OnSpinWait:
 2296       return VM_Version::supports_on_spin_wait();
 2297     case Op_CacheWB:
 2298     case Op_CacheWBPreSync:
 2299     case Op_CacheWBPostSync:
 2300       if (!VM_Version::supports_data_cache_line_flush()) {
 2301         ret_value = false;
 2302       }
 2303       break;
 2304   }
 2305 
 2306   return ret_value; // Per default match rules are supported.
 2307 }
 2308 
 2309 const RegMask* Matcher::predicate_reg_mask(void) {
 2310   return &_PR_REG_mask;
 2311 }
 2312 
 2313 const TypeVectMask* Matcher::predicate_reg_type(const Type* elemTy, int length) {
 2314   return new TypeVectMask(elemTy, length);
 2315 }
 2316 
 2317 // Vector calling convention not yet implemented.
 2318 const bool Matcher::supports_vector_calling_convention(void) {
 2319   return false;
 2320 }
 2321 
 2322 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
 2323   Unimplemented();
 2324   return OptoRegPair(0, 0);
 2325 }
 2326 
 2327 // Is this branch offset short enough that a short branch can be used?
 2328 //
 2329 // NOTE: If the platform does not provide any short branch variants, then
 2330 //       this method should return false for offset 0.
 2331 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
 2332   // The passed offset is relative to address of the branch.
 2333 
 2334   return (-32768 <= offset && offset < 32768);
 2335 }
 2336 
 2337 // Vector width in bytes.
 2338 const int Matcher::vector_width_in_bytes(BasicType bt) {
 2339   // The MaxVectorSize should have been set by detecting SVE max vector register size.
 2340   int size = MIN2((UseSVE > 0) ? 256 : 16, (int)MaxVectorSize);
 2341   // Minimum 2 values in vector
 2342   if (size < 2*type2aelembytes(bt)) size = 0;
 2343   // But never < 4
 2344   if (size < 4) size = 0;
 2345   return size;
 2346 }
 2347 
 2348 // Limits on vector size (number of elements) loaded into vector.
 2349 const int Matcher::max_vector_size(const BasicType bt) {
 2350   return vector_width_in_bytes(bt)/type2aelembytes(bt);
 2351 }
 2352 
 2353 const int Matcher::min_vector_size(const BasicType bt) {
 2354   int max_size = max_vector_size(bt);
 2355   // Limit the min vector size to 8 bytes.
 2356   int size = 8 / type2aelembytes(bt);
 2357   if (bt == T_BYTE) {
 2358     // To support vector api shuffle/rearrange.
 2359     size = 4;
 2360   } else if (bt == T_BOOLEAN) {
 2361     // To support vector api load/store mask.
 2362     size = 2;
 2363   }
 2364   if (size < 2) size = 2;
 2365   return MIN2(size, max_size);
 2366 }
 2367 
 2368 // Actual max scalable vector register length.
 2369 const int Matcher::scalable_vector_reg_size(const BasicType bt) {
 2370   return Matcher::max_vector_size(bt);
 2371 }
 2372 
 2373 // Vector ideal reg.
 2374 const uint Matcher::vector_ideal_reg(int len) {
 2375   if (UseSVE > 0 && 16 < len && len <= 256) {
 2376     return Op_VecA;
 2377   }
 2378   switch(len) {
 2379     // For 16-bit/32-bit mask vector, reuse VecD.
 2380     case  2:
 2381     case  4:
 2382     case  8: return Op_VecD;
 2383     case 16: return Op_VecX;
 2384   }
 2385   ShouldNotReachHere();
 2386   return 0;
 2387 }
 2388 
 2389 MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* generic_opnd, uint ideal_reg, bool is_temp) {
 2390   assert(Matcher::is_generic_vector(generic_opnd), "not generic");
 2391   switch (ideal_reg) {
 2392     case Op_VecA: return new vecAOper();
 2393     case Op_VecD: return new vecDOper();
 2394     case Op_VecX: return new vecXOper();
 2395   }
 2396   ShouldNotReachHere();
 2397   return NULL;
 2398 }
 2399 
 2400 bool Matcher::is_reg2reg_move(MachNode* m) {
 2401   return false;
 2402 }
 2403 
 2404 bool Matcher::is_generic_vector(MachOper* opnd)  {
 2405   return opnd->opcode() == VREG;
 2406 }
 2407 
 2408 // Return whether or not this register is ever used as an argument.
 2409 // This function is used on startup to build the trampoline stubs in
 2410 // generateOptoStub.  Registers not mentioned will be killed by the VM
 2411 // call in the trampoline, and arguments in those registers not be
 2412 // available to the callee.
 2413 bool Matcher::can_be_java_arg(int reg)
 2414 {
 2415   return
 2416     reg ==  R0_num || reg == R0_H_num ||
 2417     reg ==  R1_num || reg == R1_H_num ||
 2418     reg ==  R2_num || reg == R2_H_num ||
 2419     reg ==  R3_num || reg == R3_H_num ||
 2420     reg ==  R4_num || reg == R4_H_num ||
 2421     reg ==  R5_num || reg == R5_H_num ||
 2422     reg ==  R6_num || reg == R6_H_num ||
 2423     reg ==  R7_num || reg == R7_H_num ||
 2424     reg ==  V0_num || reg == V0_H_num ||
 2425     reg ==  V1_num || reg == V1_H_num ||
 2426     reg ==  V2_num || reg == V2_H_num ||
 2427     reg ==  V3_num || reg == V3_H_num ||
 2428     reg ==  V4_num || reg == V4_H_num ||
 2429     reg ==  V5_num || reg == V5_H_num ||
 2430     reg ==  V6_num || reg == V6_H_num ||
 2431     reg ==  V7_num || reg == V7_H_num;
 2432 }
 2433 
 2434 bool Matcher::is_spillable_arg(int reg)
 2435 {
 2436   return can_be_java_arg(reg);
 2437 }
 2438 
 2439 uint Matcher::int_pressure_limit()
 2440 {
 2441   // JDK-8183543: When taking the number of available registers as int
 2442   // register pressure threshold, the jtreg test:
 2443   // test/hotspot/jtreg/compiler/regalloc/TestC2IntPressure.java
 2444   // failed due to C2 compilation failure with
 2445   // "COMPILE SKIPPED: failed spill-split-recycle sanity check".
 2446   //
 2447   // A derived pointer is live at CallNode and then is flagged by RA
 2448   // as a spilled LRG. Spilling heuristics(Spill-USE) explicitly skip
 2449   // derived pointers and lastly fail to spill after reaching maximum
 2450   // number of iterations. Lowering the default pressure threshold to
 2451   // (_NO_SPECIAL_REG32_mask.Size() minus 1) forces CallNode to become
 2452   // a high register pressure area of the code so that split_DEF can
 2453   // generate DefinitionSpillCopy for the derived pointer.
 2454   uint default_int_pressure_threshold = _NO_SPECIAL_REG32_mask.Size() - 1;
 2455   if (!PreserveFramePointer) {
 2456     // When PreserveFramePointer is off, frame pointer is allocatable,
 2457     // but different from other SOC registers, it is excluded from
 2458     // fatproj's mask because its save type is No-Save. Decrease 1 to
 2459     // ensure high pressure at fatproj when PreserveFramePointer is off.
 2460     // See check_pressure_at_fatproj().
 2461     default_int_pressure_threshold--;
 2462   }
 2463   return (INTPRESSURE == -1) ? default_int_pressure_threshold : INTPRESSURE;
 2464 }
 2465 
 2466 uint Matcher::float_pressure_limit()
 2467 {
 2468   // _FLOAT_REG_mask is generated by adlc from the float_reg register class.
 2469   return (FLOATPRESSURE == -1) ? _FLOAT_REG_mask.Size() : FLOATPRESSURE;
 2470 }
 2471 
 2472 bool Matcher::use_asm_for_ldiv_by_con(jlong divisor) {
 2473   return false;
 2474 }
 2475 
 2476 RegMask Matcher::divI_proj_mask() {
 2477   ShouldNotReachHere();
 2478   return RegMask();
 2479 }
 2480 
 2481 // Register for MODI projection of divmodI.
 2482 RegMask Matcher::modI_proj_mask() {
 2483   ShouldNotReachHere();
 2484   return RegMask();
 2485 }
 2486 
 2487 // Register for DIVL projection of divmodL.
 2488 RegMask Matcher::divL_proj_mask() {
 2489   ShouldNotReachHere();
 2490   return RegMask();
 2491 }
 2492 
 2493 // Register for MODL projection of divmodL.
 2494 RegMask Matcher::modL_proj_mask() {
 2495   ShouldNotReachHere();
 2496   return RegMask();
 2497 }
 2498 
 2499 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
 2500   return FP_REG_mask();
 2501 }
 2502 
 2503 bool size_fits_all_mem_uses(AddPNode* addp, int shift) {
 2504   for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) {
 2505     Node* u = addp->fast_out(i);
 2506     if (u->is_LoadStore()) {
 2507       // On AArch64, LoadStoreNodes (i.e. compare and swap
 2508       // instructions) only take register indirect as an operand, so
 2509       // any attempt to use an AddPNode as an input to a LoadStoreNode
 2510       // must fail.
 2511       return false;
 2512     }
 2513     if (u->is_Mem()) {
 2514       int opsize = u->as_Mem()->memory_size();
 2515       assert(opsize > 0, "unexpected memory operand size");
 2516       if (u->as_Mem()->memory_size() != (1<<shift)) {
 2517         return false;
 2518       }
 2519     }
 2520   }
 2521   return true;
 2522 }
 2523 
 2524 // Binary src (Replicate con)
 2525 bool is_valid_sve_arith_imm_pattern(Node* n, Node* m) {
 2526   if (n == NULL || m == NULL) {
 2527     return false;
 2528   }
 2529 
 2530   if (UseSVE == 0 || !VectorNode::is_invariant_vector(m)) {
 2531     return false;
 2532   }
 2533 
 2534   Node* imm_node = m->in(1);
 2535   if (!imm_node->is_Con()) {
 2536     return false;
 2537   }
 2538 
 2539   const Type* t = imm_node->bottom_type();
 2540   if (!(t->isa_int() || t->isa_long())) {
 2541     return false;
 2542   }
 2543 
 2544   switch (n->Opcode()) {
 2545   case Op_AndV:
 2546   case Op_OrV:
 2547   case Op_XorV: {
 2548     Assembler::SIMD_RegVariant T = Assembler::elemType_to_regVariant(Matcher::vector_element_basic_type(n));
 2549     uint64_t value = t->isa_long() ? (uint64_t)imm_node->get_long() : (uint64_t)imm_node->get_int();
 2550     return Assembler::operand_valid_for_sve_logical_immediate(Assembler::regVariant_to_elemBits(T), value);
 2551   }
 2552   case Op_AddVB:
 2553     return (imm_node->get_int() <= 255 && imm_node->get_int() >= -255);
 2554   case Op_AddVS:
 2555   case Op_AddVI:
 2556     return Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)imm_node->get_int());
 2557   case Op_AddVL:
 2558     return Assembler::operand_valid_for_sve_add_sub_immediate(imm_node->get_long());
 2559   default:
 2560     return false;
 2561   }
 2562 }
 2563 
 2564 // (XorV src (Replicate m1))
 2565 // (XorVMask src (MaskAll m1))
 2566 bool is_vector_bitwise_not_pattern(Node* n, Node* m) {
 2567   if (n != NULL && m != NULL) {
 2568     return (n->Opcode() == Op_XorV || n->Opcode() == Op_XorVMask) &&
 2569            VectorNode::is_all_ones_vector(m);
 2570   }
 2571   return false;
 2572 }
 2573 
 2574 // Should the matcher clone input 'm' of node 'n'?
 2575 bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
 2576   if (is_vshift_con_pattern(n, m) ||
 2577       is_vector_bitwise_not_pattern(n, m) ||
 2578       is_valid_sve_arith_imm_pattern(n, m)) {
 2579     mstack.push(m, Visit);
 2580     return true;
 2581   }
 2582   return false;
 2583 }
 2584 
 2585 // Should the Matcher clone shifts on addressing modes, expecting them
 2586 // to be subsumed into complex addressing expressions or compute them
 2587 // into registers?
 2588 bool Matcher::pd_clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
 2589   if (clone_base_plus_offset_address(m, mstack, address_visited)) {
 2590     return true;
 2591   }
 2592 
 2593   Node *off = m->in(AddPNode::Offset);
 2594   if (off->Opcode() == Op_LShiftL && off->in(2)->is_Con() &&
 2595       size_fits_all_mem_uses(m, off->in(2)->get_int()) &&
 2596       // Are there other uses besides address expressions?
 2597       !is_visited(off)) {
 2598     address_visited.set(off->_idx); // Flag as address_visited
 2599     mstack.push(off->in(2), Visit);
 2600     Node *conv = off->in(1);
 2601     if (conv->Opcode() == Op_ConvI2L &&
 2602         // Are there other uses besides address expressions?
 2603         !is_visited(conv)) {
 2604       address_visited.set(conv->_idx); // Flag as address_visited
 2605       mstack.push(conv->in(1), Pre_Visit);
 2606     } else {
 2607       mstack.push(conv, Pre_Visit);
 2608     }
 2609     address_visited.test_set(m->_idx); // Flag as address_visited
 2610     mstack.push(m->in(AddPNode::Address), Pre_Visit);
 2611     mstack.push(m->in(AddPNode::Base), Pre_Visit);
 2612     return true;
 2613   } else if (off->Opcode() == Op_ConvI2L &&
 2614              // Are there other uses besides address expressions?
 2615              !is_visited(off)) {
 2616     address_visited.test_set(m->_idx); // Flag as address_visited
 2617     address_visited.set(off->_idx); // Flag as address_visited
 2618     mstack.push(off->in(1), Pre_Visit);
 2619     mstack.push(m->in(AddPNode::Address), Pre_Visit);
 2620     mstack.push(m->in(AddPNode::Base), Pre_Visit);
 2621     return true;
 2622   }
 2623   return false;
 2624 }
 2625 
 2626 #define MOV_VOLATILE(REG, BASE, INDEX, SCALE, DISP, SCRATCH, INSN)      \
 2627   C2_MacroAssembler _masm(&cbuf);                                       \
 2628   {                                                                     \
 2629     guarantee(INDEX == -1, "mode not permitted for volatile");          \
 2630     guarantee(DISP == 0, "mode not permitted for volatile");            \
 2631     guarantee(SCALE == 0, "mode not permitted for volatile");           \
 2632     __ INSN(REG, as_Register(BASE));                                    \
 2633   }
 2634 
 2635 
 2636 static Address mem2address(int opcode, Register base, int index, int size, int disp)
 2637   {
 2638     Address::extend scale;
 2639 
 2640     // Hooboy, this is fugly.  We need a way to communicate to the
 2641     // encoder that the index needs to be sign extended, so we have to
 2642     // enumerate all the cases.
 2643     switch (opcode) {
 2644     case INDINDEXSCALEDI2L:
 2645     case INDINDEXSCALEDI2LN:
 2646     case INDINDEXI2L:
 2647     case INDINDEXI2LN:
 2648       scale = Address::sxtw(size);
 2649       break;
 2650     default:
 2651       scale = Address::lsl(size);
 2652     }
 2653 
 2654     if (index == -1) {
 2655       return Address(base, disp);
 2656     } else {
 2657       assert(disp == 0, "unsupported address mode: disp = %d", disp);
 2658       return Address(base, as_Register(index), scale);
 2659     }
 2660   }
 2661 
 2662 
 2663 typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr);
 2664 typedef void (MacroAssembler::* mem_insn2)(Register Rt, Register adr);
 2665 typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr);
 2666 typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt,
 2667                                   MacroAssembler::SIMD_RegVariant T, const Address &adr);
 2668 
 2669   // Used for all non-volatile memory accesses.  The use of
 2670   // $mem->opcode() to discover whether this pattern uses sign-extended
 2671   // offsets is something of a kludge.
 2672   static void loadStore(C2_MacroAssembler masm, mem_insn insn,
 2673                         Register reg, int opcode,
 2674                         Register base, int index, int scale, int disp,
 2675                         int size_in_memory)
 2676   {
 2677     Address addr = mem2address(opcode, base, index, scale, disp);
 2678     if (addr.getMode() == Address::base_plus_offset) {
 2679       /* If we get an out-of-range offset it is a bug in the compiler,
 2680          so we assert here. */
 2681       assert(Address::offset_ok_for_immed(addr.offset(), exact_log2(size_in_memory)),
 2682              "c2 compiler bug");
 2683       /* Fix up any out-of-range offsets. */
 2684       assert_different_registers(rscratch1, base);
 2685       assert_different_registers(rscratch1, reg);
 2686       addr = masm.legitimize_address(addr, size_in_memory, rscratch1);
 2687     }
 2688     (masm.*insn)(reg, addr);
 2689   }
 2690 
 2691   static void loadStore(C2_MacroAssembler masm, mem_float_insn insn,
 2692                         FloatRegister reg, int opcode,
 2693                         Register base, int index, int size, int disp,
 2694                         int size_in_memory)
 2695   {
 2696     Address::extend scale;
 2697 
 2698     switch (opcode) {
 2699     case INDINDEXSCALEDI2L:
 2700     case INDINDEXSCALEDI2LN:
 2701       scale = Address::sxtw(size);
 2702       break;
 2703     default:
 2704       scale = Address::lsl(size);
 2705     }
 2706 
 2707     if (index == -1) {
 2708       /* If we get an out-of-range offset it is a bug in the compiler,
 2709          so we assert here. */
 2710       assert(Address::offset_ok_for_immed(disp, exact_log2(size_in_memory)), "c2 compiler bug");
 2711       /* Fix up any out-of-range offsets. */
 2712       assert_different_registers(rscratch1, base);
 2713       Address addr = Address(base, disp);
 2714       addr = masm.legitimize_address(addr, size_in_memory, rscratch1);
 2715       (masm.*insn)(reg, addr);
 2716     } else {
 2717       assert(disp == 0, "unsupported address mode: disp = %d", disp);
 2718       (masm.*insn)(reg, Address(base, as_Register(index), scale));
 2719     }
 2720   }
 2721 
 2722   static void loadStore(C2_MacroAssembler masm, mem_vector_insn insn,
 2723                         FloatRegister reg, MacroAssembler::SIMD_RegVariant T,
 2724                         int opcode, Register base, int index, int size, int disp)
 2725   {
 2726     if (index == -1) {
 2727       (masm.*insn)(reg, T, Address(base, disp));
 2728     } else {
 2729       assert(disp == 0, "unsupported address mode");
 2730       (masm.*insn)(reg, T, Address(base, as_Register(index), Address::lsl(size)));
 2731     }
 2732   }
 2733 
 2734 %}
 2735 
 2736 
 2737 
 2738 //----------ENCODING BLOCK-----------------------------------------------------
 2739 // This block specifies the encoding classes used by the compiler to
 2740 // output byte streams.  Encoding classes are parameterized macros
 2741 // used by Machine Instruction Nodes in order to generate the bit
 2742 // encoding of the instruction.  Operands specify their base encoding
 2743 // interface with the interface keyword.  There are currently
 2744 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
 2745 // COND_INTER.  REG_INTER causes an operand to generate a function
 2746 // which returns its register number when queried.  CONST_INTER causes
 2747 // an operand to generate a function which returns the value of the
 2748 // constant when queried.  MEMORY_INTER causes an operand to generate
 2749 // four functions which return the Base Register, the Index Register,
 2750 // the Scale Value, and the Offset Value of the operand when queried.
 2751 // COND_INTER causes an operand to generate six functions which return
 2752 // the encoding code (ie - encoding bits for the instruction)
 2753 // associated with each basic boolean condition for a conditional
 2754 // instruction.
 2755 //
 2756 // Instructions specify two basic values for encoding.  Again, a
 2757 // function is available to check if the constant displacement is an
 2758 // oop. They use the ins_encode keyword to specify their encoding
 2759 // classes (which must be a sequence of enc_class names, and their
 2760 // parameters, specified in the encoding block), and they use the
 2761 // opcode keyword to specify, in order, their primary, secondary, and
 2762 // tertiary opcode.  Only the opcode sections which a particular
 2763 // instruction needs for encoding need to be specified.
 2764 encode %{
 2765   // Build emit functions for each basic byte or larger field in the
 2766   // intel encoding scheme (opcode, rm, sib, immediate), and call them
 2767   // from C++ code in the enc_class source block.  Emit functions will
 2768   // live in the main source block for now.  In future, we can
 2769   // generalize this by adding a syntax that specifies the sizes of
 2770   // fields in an order, so that the adlc can build the emit functions
 2771   // automagically
 2772 
 2773   // catch all for unimplemented encodings
 2774   enc_class enc_unimplemented %{
 2775     C2_MacroAssembler _masm(&cbuf);
 2776     __ unimplemented("C2 catch all");
 2777   %}
 2778 
 2779   // BEGIN Non-volatile memory access
 2780 
 2781   // This encoding class is generated automatically from ad_encode.m4.
 2782   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2783   enc_class aarch64_enc_ldrsbw(iRegI dst, memory1 mem) %{
 2784     Register dst_reg = as_Register($dst$$reg);
 2785     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsbw, dst_reg, $mem->opcode(),
 2786                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2787   %}
 2788 
 2789   // This encoding class is generated automatically from ad_encode.m4.
 2790   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2791   enc_class aarch64_enc_ldrsb(iRegI dst, memory1 mem) %{
 2792     Register dst_reg = as_Register($dst$$reg);
 2793     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsb, dst_reg, $mem->opcode(),
 2794                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2795   %}
 2796 
 2797   // This encoding class is generated automatically from ad_encode.m4.
 2798   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2799   enc_class aarch64_enc_ldrb(iRegI dst, memory1 mem) %{
 2800     Register dst_reg = as_Register($dst$$reg);
 2801     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
 2802                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2803   %}
 2804 
 2805   // This encoding class is generated automatically from ad_encode.m4.
 2806   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2807   enc_class aarch64_enc_ldrb(iRegL dst, memory1 mem) %{
 2808     Register dst_reg = as_Register($dst$$reg);
 2809     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
 2810                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2811   %}
 2812 
 2813   // This encoding class is generated automatically from ad_encode.m4.
 2814   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2815   enc_class aarch64_enc_ldrshw(iRegI dst, memory2 mem) %{
 2816     Register dst_reg = as_Register($dst$$reg);
 2817     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrshw, dst_reg, $mem->opcode(),
 2818                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2819   %}
 2820 
 2821   // This encoding class is generated automatically from ad_encode.m4.
 2822   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2823   enc_class aarch64_enc_ldrsh(iRegI dst, memory2 mem) %{
 2824     Register dst_reg = as_Register($dst$$reg);
 2825     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsh, dst_reg, $mem->opcode(),
 2826                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2827   %}
 2828 
 2829   // This encoding class is generated automatically from ad_encode.m4.
 2830   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2831   enc_class aarch64_enc_ldrh(iRegI dst, memory2 mem) %{
 2832     Register dst_reg = as_Register($dst$$reg);
 2833     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
 2834                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2835   %}
 2836 
 2837   // This encoding class is generated automatically from ad_encode.m4.
 2838   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2839   enc_class aarch64_enc_ldrh(iRegL dst, memory2 mem) %{
 2840     Register dst_reg = as_Register($dst$$reg);
 2841     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
 2842                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2843   %}
 2844 
 2845   // This encoding class is generated automatically from ad_encode.m4.
 2846   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2847   enc_class aarch64_enc_ldrw(iRegI dst, memory4 mem) %{
 2848     Register dst_reg = as_Register($dst$$reg);
 2849     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
 2850                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2851   %}
 2852 
 2853   // This encoding class is generated automatically from ad_encode.m4.
 2854   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2855   enc_class aarch64_enc_ldrw(iRegL dst, memory4 mem) %{
 2856     Register dst_reg = as_Register($dst$$reg);
 2857     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
 2858                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2859   %}
 2860 
 2861   // This encoding class is generated automatically from ad_encode.m4.
 2862   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2863   enc_class aarch64_enc_ldrsw(iRegL dst, memory4 mem) %{
 2864     Register dst_reg = as_Register($dst$$reg);
 2865     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsw, dst_reg, $mem->opcode(),
 2866                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2867   %}
 2868 
 2869   // This encoding class is generated automatically from ad_encode.m4.
 2870   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2871   enc_class aarch64_enc_ldr(iRegL dst, memory8 mem) %{
 2872     Register dst_reg = as_Register($dst$$reg);
 2873     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, $mem->opcode(),
 2874                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 2875   %}
 2876 
 2877   // This encoding class is generated automatically from ad_encode.m4.
 2878   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2879   enc_class aarch64_enc_ldrs(vRegF dst, memory4 mem) %{
 2880     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 2881     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrs, dst_reg, $mem->opcode(),
 2882                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2883   %}
 2884 
 2885   // This encoding class is generated automatically from ad_encode.m4.
 2886   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2887   enc_class aarch64_enc_ldrd(vRegD dst, memory8 mem) %{
 2888     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 2889     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrd, dst_reg, $mem->opcode(),
 2890                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 2891   %}
 2892 
 2893   // This encoding class is generated automatically from ad_encode.m4.
 2894   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2895   enc_class aarch64_enc_strb(iRegI src, memory1 mem) %{
 2896     Register src_reg = as_Register($src$$reg);
 2897     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strb, src_reg, $mem->opcode(),
 2898                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2899   %}
 2900 
 2901   // This encoding class is generated automatically from ad_encode.m4.
 2902   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2903   enc_class aarch64_enc_strb0(memory1 mem) %{
 2904     C2_MacroAssembler _masm(&cbuf);
 2905     loadStore(_masm, &MacroAssembler::strb, zr, $mem->opcode(),
 2906                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2907   %}
 2908 
 2909   // This encoding class is generated automatically from ad_encode.m4.
 2910   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2911   enc_class aarch64_enc_strh(iRegI src, memory2 mem) %{
 2912     Register src_reg = as_Register($src$$reg);
 2913     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strh, src_reg, $mem->opcode(),
 2914                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2915   %}
 2916 
 2917   // This encoding class is generated automatically from ad_encode.m4.
 2918   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2919   enc_class aarch64_enc_strh0(memory2 mem) %{
 2920     C2_MacroAssembler _masm(&cbuf);
 2921     loadStore(_masm, &MacroAssembler::strh, zr, $mem->opcode(),
 2922                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2923   %}
 2924 
 2925   // This encoding class is generated automatically from ad_encode.m4.
 2926   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2927   enc_class aarch64_enc_strw(iRegI src, memory4 mem) %{
 2928     Register src_reg = as_Register($src$$reg);
 2929     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strw, src_reg, $mem->opcode(),
 2930                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2931   %}
 2932 
 2933   // This encoding class is generated automatically from ad_encode.m4.
 2934   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2935   enc_class aarch64_enc_strw0(memory4 mem) %{
 2936     C2_MacroAssembler _masm(&cbuf);
 2937     loadStore(_masm, &MacroAssembler::strw, zr, $mem->opcode(),
 2938                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2939   %}
 2940 
 2941   // This encoding class is generated automatically from ad_encode.m4.
 2942   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2943   enc_class aarch64_enc_str(iRegL src, memory8 mem) %{
 2944     Register src_reg = as_Register($src$$reg);
 2945     // we sometimes get asked to store the stack pointer into the
 2946     // current thread -- we cannot do that directly on AArch64
 2947     if (src_reg == r31_sp) {
 2948       C2_MacroAssembler _masm(&cbuf);
 2949       assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
 2950       __ mov(rscratch2, sp);
 2951       src_reg = rscratch2;
 2952     }
 2953     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, $mem->opcode(),
 2954                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 2955   %}
 2956 
 2957   // This encoding class is generated automatically from ad_encode.m4.
 2958   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2959   enc_class aarch64_enc_str0(memory8 mem) %{
 2960     C2_MacroAssembler _masm(&cbuf);
 2961     loadStore(_masm, &MacroAssembler::str, zr, $mem->opcode(),
 2962                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 2963   %}
 2964 
 2965   // This encoding class is generated automatically from ad_encode.m4.
 2966   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2967   enc_class aarch64_enc_strs(vRegF src, memory4 mem) %{
 2968     FloatRegister src_reg = as_FloatRegister($src$$reg);
 2969     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strs, src_reg, $mem->opcode(),
 2970                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2971   %}
 2972 
 2973   // This encoding class is generated automatically from ad_encode.m4.
 2974   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2975   enc_class aarch64_enc_strd(vRegD src, memory8 mem) %{
 2976     FloatRegister src_reg = as_FloatRegister($src$$reg);
 2977     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strd, src_reg, $mem->opcode(),
 2978                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 2979   %}
 2980 
 2981   // This encoding class is generated automatically from ad_encode.m4.
 2982   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2983   enc_class aarch64_enc_strb0_ordered(memory4 mem) %{
 2984       C2_MacroAssembler _masm(&cbuf);
 2985       __ membar(Assembler::StoreStore);
 2986       loadStore(_masm, &MacroAssembler::strb, zr, $mem->opcode(),
 2987                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2988   %}
 2989 
 2990   // END Non-volatile memory access
 2991 
 2992   // Vector loads and stores
 2993   enc_class aarch64_enc_ldrvH(vReg dst, memory mem) %{
 2994     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 2995     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::H,
 2996        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 2997   %}
 2998 
 2999   enc_class aarch64_enc_ldrvS(vReg dst, memory mem) %{
 3000     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3001     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::S,
 3002        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3003   %}
 3004 
 3005   enc_class aarch64_enc_ldrvD(vReg dst, memory mem) %{
 3006     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3007     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::D,
 3008        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3009   %}
 3010 
 3011   enc_class aarch64_enc_ldrvQ(vReg dst, memory mem) %{
 3012     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3013     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::Q,
 3014        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3015   %}
 3016 
 3017   enc_class aarch64_enc_strvH(vReg src, memory mem) %{
 3018     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3019     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::H,
 3020        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3021   %}
 3022 
 3023   enc_class aarch64_enc_strvS(vReg src, memory mem) %{
 3024     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3025     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::S,
 3026        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3027   %}
 3028 
 3029   enc_class aarch64_enc_strvD(vReg src, memory mem) %{
 3030     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3031     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::D,
 3032        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3033   %}
 3034 
 3035   enc_class aarch64_enc_strvQ(vReg src, memory mem) %{
 3036     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3037     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::Q,
 3038        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3039   %}
 3040 
 3041   // volatile loads and stores
 3042 
 3043   enc_class aarch64_enc_stlrb(iRegI src, memory mem) %{
 3044     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3045                  rscratch1, stlrb);
 3046   %}
 3047 
 3048   enc_class aarch64_enc_stlrb0(memory mem) %{
 3049     MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3050                  rscratch1, stlrb);
 3051   %}
 3052 
 3053   enc_class aarch64_enc_stlrh(iRegI src, memory mem) %{
 3054     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3055                  rscratch1, stlrh);
 3056   %}
 3057 
 3058   enc_class aarch64_enc_stlrh0(memory mem) %{
 3059     MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3060                  rscratch1, stlrh);
 3061   %}
 3062 
 3063   enc_class aarch64_enc_stlrw(iRegI src, memory mem) %{
 3064     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3065                  rscratch1, stlrw);
 3066   %}
 3067 
 3068   enc_class aarch64_enc_stlrw0(memory mem) %{
 3069     MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3070                  rscratch1, stlrw);
 3071   %}
 3072 
 3073   enc_class aarch64_enc_ldarsbw(iRegI dst, memory mem) %{
 3074     Register dst_reg = as_Register($dst$$reg);
 3075     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3076              rscratch1, ldarb);
 3077     __ sxtbw(dst_reg, dst_reg);
 3078   %}
 3079 
 3080   enc_class aarch64_enc_ldarsb(iRegL dst, memory mem) %{
 3081     Register dst_reg = as_Register($dst$$reg);
 3082     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3083              rscratch1, ldarb);
 3084     __ sxtb(dst_reg, dst_reg);
 3085   %}
 3086 
 3087   enc_class aarch64_enc_ldarbw(iRegI dst, memory mem) %{
 3088     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3089              rscratch1, ldarb);
 3090   %}
 3091 
 3092   enc_class aarch64_enc_ldarb(iRegL dst, memory mem) %{
 3093     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3094              rscratch1, ldarb);
 3095   %}
 3096 
 3097   enc_class aarch64_enc_ldarshw(iRegI dst, memory mem) %{
 3098     Register dst_reg = as_Register($dst$$reg);
 3099     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3100              rscratch1, ldarh);
 3101     __ sxthw(dst_reg, dst_reg);
 3102   %}
 3103 
 3104   enc_class aarch64_enc_ldarsh(iRegL dst, memory mem) %{
 3105     Register dst_reg = as_Register($dst$$reg);
 3106     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3107              rscratch1, ldarh);
 3108     __ sxth(dst_reg, dst_reg);
 3109   %}
 3110 
 3111   enc_class aarch64_enc_ldarhw(iRegI dst, memory mem) %{
 3112     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3113              rscratch1, ldarh);
 3114   %}
 3115 
 3116   enc_class aarch64_enc_ldarh(iRegL dst, memory mem) %{
 3117     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3118              rscratch1, ldarh);
 3119   %}
 3120 
 3121   enc_class aarch64_enc_ldarw(iRegI dst, memory mem) %{
 3122     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3123              rscratch1, ldarw);
 3124   %}
 3125 
 3126   enc_class aarch64_enc_ldarw(iRegL dst, memory mem) %{
 3127     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3128              rscratch1, ldarw);
 3129   %}
 3130 
 3131   enc_class aarch64_enc_ldar(iRegL dst, memory mem) %{
 3132     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3133              rscratch1, ldar);
 3134   %}
 3135 
 3136   enc_class aarch64_enc_fldars(vRegF dst, memory mem) %{
 3137     MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3138              rscratch1, ldarw);
 3139     __ fmovs(as_FloatRegister($dst$$reg), rscratch1);
 3140   %}
 3141 
 3142   enc_class aarch64_enc_fldard(vRegD dst, memory mem) %{
 3143     MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3144              rscratch1, ldar);
 3145     __ fmovd(as_FloatRegister($dst$$reg), rscratch1);
 3146   %}
 3147 
 3148   enc_class aarch64_enc_stlr(iRegL src, memory mem) %{
 3149     Register src_reg = as_Register($src$$reg);
 3150     // we sometimes get asked to store the stack pointer into the
 3151     // current thread -- we cannot do that directly on AArch64
 3152     if (src_reg == r31_sp) {
 3153       C2_MacroAssembler _masm(&cbuf);
 3154       assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
 3155       __ mov(rscratch2, sp);
 3156       src_reg = rscratch2;
 3157     }
 3158     MOV_VOLATILE(src_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3159                  rscratch1, stlr);
 3160   %}
 3161 
 3162   enc_class aarch64_enc_stlr0(memory mem) %{
 3163     MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3164                  rscratch1, stlr);
 3165   %}
 3166 
 3167   enc_class aarch64_enc_fstlrs(vRegF src, memory mem) %{
 3168     {
 3169       C2_MacroAssembler _masm(&cbuf);
 3170       FloatRegister src_reg = as_FloatRegister($src$$reg);
 3171       __ fmovs(rscratch2, src_reg);
 3172     }
 3173     MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3174                  rscratch1, stlrw);
 3175   %}
 3176 
 3177   enc_class aarch64_enc_fstlrd(vRegD src, memory mem) %{
 3178     {
 3179       C2_MacroAssembler _masm(&cbuf);
 3180       FloatRegister src_reg = as_FloatRegister($src$$reg);
 3181       __ fmovd(rscratch2, src_reg);
 3182     }
 3183     MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3184                  rscratch1, stlr);
 3185   %}
 3186 
 3187   // synchronized read/update encodings
 3188 
 3189   enc_class aarch64_enc_ldaxr(iRegL dst, memory8 mem) %{
 3190     C2_MacroAssembler _masm(&cbuf);
 3191     Register dst_reg = as_Register($dst$$reg);
 3192     Register base = as_Register($mem$$base);
 3193     int index = $mem$$index;
 3194     int scale = $mem$$scale;
 3195     int disp = $mem$$disp;
 3196     if (index == -1) {
 3197        if (disp != 0) {
 3198         __ lea(rscratch1, Address(base, disp));
 3199         __ ldaxr(dst_reg, rscratch1);
 3200       } else {
 3201         // TODO
 3202         // should we ever get anything other than this case?
 3203         __ ldaxr(dst_reg, base);
 3204       }
 3205     } else {
 3206       Register index_reg = as_Register(index);
 3207       if (disp == 0) {
 3208         __ lea(rscratch1, Address(base, index_reg, Address::lsl(scale)));
 3209         __ ldaxr(dst_reg, rscratch1);
 3210       } else {
 3211         __ lea(rscratch1, Address(base, disp));
 3212         __ lea(rscratch1, Address(rscratch1, index_reg, Address::lsl(scale)));
 3213         __ ldaxr(dst_reg, rscratch1);
 3214       }
 3215     }
 3216   %}
 3217 
 3218   enc_class aarch64_enc_stlxr(iRegLNoSp src, memory8 mem) %{
 3219     C2_MacroAssembler _masm(&cbuf);
 3220     Register src_reg = as_Register($src$$reg);
 3221     Register base = as_Register($mem$$base);
 3222     int index = $mem$$index;
 3223     int scale = $mem$$scale;
 3224     int disp = $mem$$disp;
 3225     if (index == -1) {
 3226        if (disp != 0) {
 3227         __ lea(rscratch2, Address(base, disp));
 3228         __ stlxr(rscratch1, src_reg, rscratch2);
 3229       } else {
 3230         // TODO
 3231         // should we ever get anything other than this case?
 3232         __ stlxr(rscratch1, src_reg, base);
 3233       }
 3234     } else {
 3235       Register index_reg = as_Register(index);
 3236       if (disp == 0) {
 3237         __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
 3238         __ stlxr(rscratch1, src_reg, rscratch2);
 3239       } else {
 3240         __ lea(rscratch2, Address(base, disp));
 3241         __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
 3242         __ stlxr(rscratch1, src_reg, rscratch2);
 3243       }
 3244     }
 3245     __ cmpw(rscratch1, zr);
 3246   %}
 3247 
 3248   enc_class aarch64_enc_cmpxchg(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
 3249     C2_MacroAssembler _masm(&cbuf);
 3250     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3251     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3252                Assembler::xword, /*acquire*/ false, /*release*/ true,
 3253                /*weak*/ false, noreg);
 3254   %}
 3255 
 3256   enc_class aarch64_enc_cmpxchgw(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3257     C2_MacroAssembler _masm(&cbuf);
 3258     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3259     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3260                Assembler::word, /*acquire*/ false, /*release*/ true,
 3261                /*weak*/ false, noreg);
 3262   %}
 3263 
 3264   enc_class aarch64_enc_cmpxchgs(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3265     C2_MacroAssembler _masm(&cbuf);
 3266     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3267     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3268                Assembler::halfword, /*acquire*/ false, /*release*/ true,
 3269                /*weak*/ false, noreg);
 3270   %}
 3271 
 3272   enc_class aarch64_enc_cmpxchgb(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3273     C2_MacroAssembler _masm(&cbuf);
 3274     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3275     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3276                Assembler::byte, /*acquire*/ false, /*release*/ true,
 3277                /*weak*/ false, noreg);
 3278   %}
 3279 
 3280 
 3281   // The only difference between aarch64_enc_cmpxchg and
 3282   // aarch64_enc_cmpxchg_acq is that we use load-acquire in the
 3283   // CompareAndSwap sequence to serve as a barrier on acquiring a
 3284   // lock.
 3285   enc_class aarch64_enc_cmpxchg_acq(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
 3286     C2_MacroAssembler _masm(&cbuf);
 3287     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3288     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3289                Assembler::xword, /*acquire*/ true, /*release*/ true,
 3290                /*weak*/ false, noreg);
 3291   %}
 3292 
 3293   enc_class aarch64_enc_cmpxchgw_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3294     C2_MacroAssembler _masm(&cbuf);
 3295     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3296     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3297                Assembler::word, /*acquire*/ true, /*release*/ true,
 3298                /*weak*/ false, noreg);
 3299   %}
 3300 
 3301   enc_class aarch64_enc_cmpxchgs_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3302     C2_MacroAssembler _masm(&cbuf);
 3303     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3304     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3305                Assembler::halfword, /*acquire*/ true, /*release*/ true,
 3306                /*weak*/ false, noreg);
 3307   %}
 3308 
 3309   enc_class aarch64_enc_cmpxchgb_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3310     C2_MacroAssembler _masm(&cbuf);
 3311     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3312     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3313                Assembler::byte, /*acquire*/ true, /*release*/ true,
 3314                /*weak*/ false, noreg);
 3315   %}
 3316 
 3317   // auxiliary used for CompareAndSwapX to set result register
 3318   enc_class aarch64_enc_cset_eq(iRegINoSp res) %{
 3319     C2_MacroAssembler _masm(&cbuf);
 3320     Register res_reg = as_Register($res$$reg);
 3321     __ cset(res_reg, Assembler::EQ);
 3322   %}
 3323 
 3324   // prefetch encodings
 3325 
 3326   enc_class aarch64_enc_prefetchw(memory mem) %{
 3327     C2_MacroAssembler _masm(&cbuf);
 3328     Register base = as_Register($mem$$base);
 3329     int index = $mem$$index;
 3330     int scale = $mem$$scale;
 3331     int disp = $mem$$disp;
 3332     if (index == -1) {
 3333       __ prfm(Address(base, disp), PSTL1KEEP);
 3334     } else {
 3335       Register index_reg = as_Register(index);
 3336       if (disp == 0) {
 3337         __ prfm(Address(base, index_reg, Address::lsl(scale)), PSTL1KEEP);
 3338       } else {
 3339         __ lea(rscratch1, Address(base, disp));
 3340 	__ prfm(Address(rscratch1, index_reg, Address::lsl(scale)), PSTL1KEEP);
 3341       }
 3342     }
 3343   %}
 3344 
 3345   /// mov envcodings
 3346 
 3347   enc_class aarch64_enc_movw_imm(iRegI dst, immI src) %{
 3348     C2_MacroAssembler _masm(&cbuf);
 3349     uint32_t con = (uint32_t)$src$$constant;
 3350     Register dst_reg = as_Register($dst$$reg);
 3351     if (con == 0) {
 3352       __ movw(dst_reg, zr);
 3353     } else {
 3354       __ movw(dst_reg, con);
 3355     }
 3356   %}
 3357 
 3358   enc_class aarch64_enc_mov_imm(iRegL dst, immL src) %{
 3359     C2_MacroAssembler _masm(&cbuf);
 3360     Register dst_reg = as_Register($dst$$reg);
 3361     uint64_t con = (uint64_t)$src$$constant;
 3362     if (con == 0) {
 3363       __ mov(dst_reg, zr);
 3364     } else {
 3365       __ mov(dst_reg, con);
 3366     }
 3367   %}
 3368 
 3369   enc_class aarch64_enc_mov_p(iRegP dst, immP src) %{
 3370     C2_MacroAssembler _masm(&cbuf);
 3371     Register dst_reg = as_Register($dst$$reg);
 3372     address con = (address)$src$$constant;
 3373     if (con == NULL || con == (address)1) {
 3374       ShouldNotReachHere();
 3375     } else {
 3376       relocInfo::relocType rtype = $src->constant_reloc();
 3377       if (rtype == relocInfo::oop_type) {
 3378         __ movoop(dst_reg, (jobject)con);
 3379       } else if (rtype == relocInfo::metadata_type) {
 3380         __ mov_metadata(dst_reg, (Metadata*)con);
 3381       } else {
 3382         assert(rtype == relocInfo::none, "unexpected reloc type");
 3383         if (! __ is_valid_AArch64_address(con) ||
 3384             con < (address)(uintptr_t)os::vm_page_size()) {
 3385           __ mov(dst_reg, con);
 3386         } else {
 3387           uint64_t offset;
 3388           __ adrp(dst_reg, con, offset);
 3389           __ add(dst_reg, dst_reg, offset);
 3390         }
 3391       }
 3392     }
 3393   %}
 3394 
 3395   enc_class aarch64_enc_mov_p0(iRegP dst, immP0 src) %{
 3396     C2_MacroAssembler _masm(&cbuf);
 3397     Register dst_reg = as_Register($dst$$reg);
 3398     __ mov(dst_reg, zr);
 3399   %}
 3400 
 3401   enc_class aarch64_enc_mov_p1(iRegP dst, immP_1 src) %{
 3402     C2_MacroAssembler _masm(&cbuf);
 3403     Register dst_reg = as_Register($dst$$reg);
 3404     __ mov(dst_reg, (uint64_t)1);
 3405   %}
 3406 
 3407   enc_class aarch64_enc_mov_byte_map_base(iRegP dst, immByteMapBase src) %{
 3408     C2_MacroAssembler _masm(&cbuf);
 3409     __ load_byte_map_base($dst$$Register);
 3410   %}
 3411 
 3412   enc_class aarch64_enc_mov_n(iRegN dst, immN src) %{
 3413     C2_MacroAssembler _masm(&cbuf);
 3414     Register dst_reg = as_Register($dst$$reg);
 3415     address con = (address)$src$$constant;
 3416     if (con == NULL) {
 3417       ShouldNotReachHere();
 3418     } else {
 3419       relocInfo::relocType rtype = $src->constant_reloc();
 3420       assert(rtype == relocInfo::oop_type, "unexpected reloc type");
 3421       __ set_narrow_oop(dst_reg, (jobject)con);
 3422     }
 3423   %}
 3424 
 3425   enc_class aarch64_enc_mov_n0(iRegN dst, immN0 src) %{
 3426     C2_MacroAssembler _masm(&cbuf);
 3427     Register dst_reg = as_Register($dst$$reg);
 3428     __ mov(dst_reg, zr);
 3429   %}
 3430 
 3431   enc_class aarch64_enc_mov_nk(iRegN dst, immNKlass src) %{
 3432     C2_MacroAssembler _masm(&cbuf);
 3433     Register dst_reg = as_Register($dst$$reg);
 3434     address con = (address)$src$$constant;
 3435     if (con == NULL) {
 3436       ShouldNotReachHere();
 3437     } else {
 3438       relocInfo::relocType rtype = $src->constant_reloc();
 3439       assert(rtype == relocInfo::metadata_type, "unexpected reloc type");
 3440       __ set_narrow_klass(dst_reg, (Klass *)con);
 3441     }
 3442   %}
 3443 
 3444   // arithmetic encodings
 3445 
 3446   enc_class aarch64_enc_addsubw_imm(iRegI dst, iRegI src1, immIAddSub src2) %{
 3447     C2_MacroAssembler _masm(&cbuf);
 3448     Register dst_reg = as_Register($dst$$reg);
 3449     Register src_reg = as_Register($src1$$reg);
 3450     int32_t con = (int32_t)$src2$$constant;
 3451     // add has primary == 0, subtract has primary == 1
 3452     if ($primary) { con = -con; }
 3453     if (con < 0) {
 3454       __ subw(dst_reg, src_reg, -con);
 3455     } else {
 3456       __ addw(dst_reg, src_reg, con);
 3457     }
 3458   %}
 3459 
 3460   enc_class aarch64_enc_addsub_imm(iRegL dst, iRegL src1, immLAddSub src2) %{
 3461     C2_MacroAssembler _masm(&cbuf);
 3462     Register dst_reg = as_Register($dst$$reg);
 3463     Register src_reg = as_Register($src1$$reg);
 3464     int32_t con = (int32_t)$src2$$constant;
 3465     // add has primary == 0, subtract has primary == 1
 3466     if ($primary) { con = -con; }
 3467     if (con < 0) {
 3468       __ sub(dst_reg, src_reg, -con);
 3469     } else {
 3470       __ add(dst_reg, src_reg, con);
 3471     }
 3472   %}
 3473 
 3474   enc_class aarch64_enc_divw(iRegI dst, iRegI src1, iRegI src2) %{
 3475     C2_MacroAssembler _masm(&cbuf);
 3476    Register dst_reg = as_Register($dst$$reg);
 3477    Register src1_reg = as_Register($src1$$reg);
 3478    Register src2_reg = as_Register($src2$$reg);
 3479     __ corrected_idivl(dst_reg, src1_reg, src2_reg, false, rscratch1);
 3480   %}
 3481 
 3482   enc_class aarch64_enc_div(iRegI dst, iRegI src1, iRegI src2) %{
 3483     C2_MacroAssembler _masm(&cbuf);
 3484    Register dst_reg = as_Register($dst$$reg);
 3485    Register src1_reg = as_Register($src1$$reg);
 3486    Register src2_reg = as_Register($src2$$reg);
 3487     __ corrected_idivq(dst_reg, src1_reg, src2_reg, false, rscratch1);
 3488   %}
 3489 
 3490   enc_class aarch64_enc_modw(iRegI dst, iRegI src1, iRegI src2) %{
 3491     C2_MacroAssembler _masm(&cbuf);
 3492    Register dst_reg = as_Register($dst$$reg);
 3493    Register src1_reg = as_Register($src1$$reg);
 3494    Register src2_reg = as_Register($src2$$reg);
 3495     __ corrected_idivl(dst_reg, src1_reg, src2_reg, true, rscratch1);
 3496   %}
 3497 
 3498   enc_class aarch64_enc_mod(iRegI dst, iRegI src1, iRegI src2) %{
 3499     C2_MacroAssembler _masm(&cbuf);
 3500    Register dst_reg = as_Register($dst$$reg);
 3501    Register src1_reg = as_Register($src1$$reg);
 3502    Register src2_reg = as_Register($src2$$reg);
 3503     __ corrected_idivq(dst_reg, src1_reg, src2_reg, true, rscratch1);
 3504   %}
 3505 
 3506   // compare instruction encodings
 3507 
 3508   enc_class aarch64_enc_cmpw(iRegI src1, iRegI src2) %{
 3509     C2_MacroAssembler _masm(&cbuf);
 3510     Register reg1 = as_Register($src1$$reg);
 3511     Register reg2 = as_Register($src2$$reg);
 3512     __ cmpw(reg1, reg2);
 3513   %}
 3514 
 3515   enc_class aarch64_enc_cmpw_imm_addsub(iRegI src1, immIAddSub src2) %{
 3516     C2_MacroAssembler _masm(&cbuf);
 3517     Register reg = as_Register($src1$$reg);
 3518     int32_t val = $src2$$constant;
 3519     if (val >= 0) {
 3520       __ subsw(zr, reg, val);
 3521     } else {
 3522       __ addsw(zr, reg, -val);
 3523     }
 3524   %}
 3525 
 3526   enc_class aarch64_enc_cmpw_imm(iRegI src1, immI src2) %{
 3527     C2_MacroAssembler _masm(&cbuf);
 3528     Register reg1 = as_Register($src1$$reg);
 3529     uint32_t val = (uint32_t)$src2$$constant;
 3530     __ movw(rscratch1, val);
 3531     __ cmpw(reg1, rscratch1);
 3532   %}
 3533 
 3534   enc_class aarch64_enc_cmp(iRegL src1, iRegL src2) %{
 3535     C2_MacroAssembler _masm(&cbuf);
 3536     Register reg1 = as_Register($src1$$reg);
 3537     Register reg2 = as_Register($src2$$reg);
 3538     __ cmp(reg1, reg2);
 3539   %}
 3540 
 3541   enc_class aarch64_enc_cmp_imm_addsub(iRegL src1, immL12 src2) %{
 3542     C2_MacroAssembler _masm(&cbuf);
 3543     Register reg = as_Register($src1$$reg);
 3544     int64_t val = $src2$$constant;
 3545     if (val >= 0) {
 3546       __ subs(zr, reg, val);
 3547     } else if (val != -val) {
 3548       __ adds(zr, reg, -val);
 3549     } else {
 3550     // aargh, Long.MIN_VALUE is a special case
 3551       __ orr(rscratch1, zr, (uint64_t)val);
 3552       __ subs(zr, reg, rscratch1);
 3553     }
 3554   %}
 3555 
 3556   enc_class aarch64_enc_cmp_imm(iRegL src1, immL src2) %{
 3557     C2_MacroAssembler _masm(&cbuf);
 3558     Register reg1 = as_Register($src1$$reg);
 3559     uint64_t val = (uint64_t)$src2$$constant;
 3560     __ mov(rscratch1, val);
 3561     __ cmp(reg1, rscratch1);
 3562   %}
 3563 
 3564   enc_class aarch64_enc_cmpp(iRegP src1, iRegP src2) %{
 3565     C2_MacroAssembler _masm(&cbuf);
 3566     Register reg1 = as_Register($src1$$reg);
 3567     Register reg2 = as_Register($src2$$reg);
 3568     __ cmp(reg1, reg2);
 3569   %}
 3570 
 3571   enc_class aarch64_enc_cmpn(iRegN src1, iRegN src2) %{
 3572     C2_MacroAssembler _masm(&cbuf);
 3573     Register reg1 = as_Register($src1$$reg);
 3574     Register reg2 = as_Register($src2$$reg);
 3575     __ cmpw(reg1, reg2);
 3576   %}
 3577 
 3578   enc_class aarch64_enc_testp(iRegP src) %{
 3579     C2_MacroAssembler _masm(&cbuf);
 3580     Register reg = as_Register($src$$reg);
 3581     __ cmp(reg, zr);
 3582   %}
 3583 
 3584   enc_class aarch64_enc_testn(iRegN src) %{
 3585     C2_MacroAssembler _masm(&cbuf);
 3586     Register reg = as_Register($src$$reg);
 3587     __ cmpw(reg, zr);
 3588   %}
 3589 
 3590   enc_class aarch64_enc_b(label lbl) %{
 3591     C2_MacroAssembler _masm(&cbuf);
 3592     Label *L = $lbl$$label;
 3593     __ b(*L);
 3594   %}
 3595 
 3596   enc_class aarch64_enc_br_con(cmpOp cmp, label lbl) %{
 3597     C2_MacroAssembler _masm(&cbuf);
 3598     Label *L = $lbl$$label;
 3599     __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
 3600   %}
 3601 
 3602   enc_class aarch64_enc_br_conU(cmpOpU cmp, label lbl) %{
 3603     C2_MacroAssembler _masm(&cbuf);
 3604     Label *L = $lbl$$label;
 3605     __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
 3606   %}
 3607 
 3608   enc_class aarch64_enc_partial_subtype_check(iRegP sub, iRegP super, iRegP temp, iRegP result)
 3609   %{
 3610      Register sub_reg = as_Register($sub$$reg);
 3611      Register super_reg = as_Register($super$$reg);
 3612      Register temp_reg = as_Register($temp$$reg);
 3613      Register result_reg = as_Register($result$$reg);
 3614 
 3615      Label miss;
 3616      C2_MacroAssembler _masm(&cbuf);
 3617      __ check_klass_subtype_slow_path(sub_reg, super_reg, temp_reg, result_reg,
 3618                                      NULL, &miss,
 3619                                      /*set_cond_codes:*/ true);
 3620      if ($primary) {
 3621        __ mov(result_reg, zr);
 3622      }
 3623      __ bind(miss);
 3624   %}
 3625 
 3626   enc_class aarch64_enc_java_static_call(method meth) %{
 3627     C2_MacroAssembler _masm(&cbuf);
 3628 
 3629     address addr = (address)$meth$$method;
 3630     address call;
 3631     if (!_method) {
 3632       // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
 3633       call = __ trampoline_call(Address(addr, relocInfo::runtime_call_type));
 3634       if (call == NULL) {
 3635         ciEnv::current()->record_failure("CodeCache is full");
 3636         return;
 3637       }
 3638     } else if (_method->intrinsic_id() == vmIntrinsicID::_ensureMaterializedForStackWalk) {
 3639       // The NOP here is purely to ensure that eliding a call to
 3640       // JVM_EnsureMaterializedForStackWalk doesn't change the code size.
 3641       __ nop();
 3642       __ block_comment("call JVM_EnsureMaterializedForStackWalk (elided)");
 3643     } else {
 3644       int method_index = resolved_method_index(cbuf);
 3645       RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
 3646                                                   : static_call_Relocation::spec(method_index);
 3647       call = __ trampoline_call(Address(addr, rspec));
 3648       if (call == NULL) {
 3649         ciEnv::current()->record_failure("CodeCache is full");
 3650         return;
 3651       }
 3652       if (CodeBuffer::supports_shared_stubs() && _method->can_be_statically_bound()) {
 3653         // Calls of the same statically bound method can share
 3654         // a stub to the interpreter.
 3655         cbuf.shared_stub_to_interp_for(_method, call - cbuf.insts_begin());
 3656       } else {
 3657         // Emit stub for static call
 3658         address stub = CompiledStaticCall::emit_to_interp_stub(cbuf, call);
 3659         if (stub == NULL) {
 3660           ciEnv::current()->record_failure("CodeCache is full");
 3661           return;
 3662         }
 3663       }
 3664     }
 3665 
 3666     __ post_call_nop();
 3667 
 3668     // Only non uncommon_trap calls need to reinitialize ptrue.
 3669     if (Compile::current()->max_vector_size() > 0 && uncommon_trap_request() == 0) {
 3670       __ reinitialize_ptrue();
 3671     }
 3672   %}
 3673 
 3674   enc_class aarch64_enc_java_dynamic_call(method meth) %{
 3675     C2_MacroAssembler _masm(&cbuf);
 3676     int method_index = resolved_method_index(cbuf);
 3677     address call = __ ic_call((address)$meth$$method, method_index);
 3678     if (call == NULL) {
 3679       ciEnv::current()->record_failure("CodeCache is full");
 3680       return;
 3681     }
 3682     __ post_call_nop();
 3683     if (Compile::current()->max_vector_size() > 0) {
 3684       __ reinitialize_ptrue();
 3685     }
 3686   %}
 3687 
 3688   enc_class aarch64_enc_call_epilog() %{
 3689     C2_MacroAssembler _masm(&cbuf);
 3690     if (VerifyStackAtCalls) {
 3691       // Check that stack depth is unchanged: find majik cookie on stack
 3692       __ call_Unimplemented();
 3693     }
 3694   %}
 3695 
 3696   enc_class aarch64_enc_java_to_runtime(method meth) %{
 3697     C2_MacroAssembler _masm(&cbuf);
 3698 
 3699     // some calls to generated routines (arraycopy code) are scheduled
 3700     // by C2 as runtime calls. if so we can call them using a br (they
 3701     // will be in a reachable segment) otherwise we have to use a blr
 3702     // which loads the absolute address into a register.
 3703     address entry = (address)$meth$$method;
 3704     CodeBlob *cb = CodeCache::find_blob(entry);
 3705     if (cb) {
 3706       address call = __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
 3707       if (call == NULL) {
 3708         ciEnv::current()->record_failure("CodeCache is full");
 3709         return;
 3710       }
 3711       __ post_call_nop();
 3712     } else {
 3713       Label retaddr;
 3714       __ adr(rscratch2, retaddr);
 3715       __ lea(rscratch1, RuntimeAddress(entry));
 3716       // Leave a breadcrumb for JavaFrameAnchor::capture_last_Java_pc()
 3717       __ stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)));
 3718       __ blr(rscratch1);
 3719       __ bind(retaddr);
 3720       __ post_call_nop();
 3721       __ add(sp, sp, 2 * wordSize);
 3722     }
 3723     if (Compile::current()->max_vector_size() > 0) {
 3724       __ reinitialize_ptrue();
 3725     }
 3726   %}
 3727 
 3728   enc_class aarch64_enc_rethrow() %{
 3729     C2_MacroAssembler _masm(&cbuf);
 3730     __ far_jump(RuntimeAddress(OptoRuntime::rethrow_stub()));
 3731   %}
 3732 
 3733   enc_class aarch64_enc_ret() %{
 3734     C2_MacroAssembler _masm(&cbuf);
 3735 #ifdef ASSERT
 3736     if (Compile::current()->max_vector_size() > 0) {
 3737       __ verify_ptrue();
 3738     }
 3739 #endif
 3740     __ ret(lr);
 3741   %}
 3742 
 3743   enc_class aarch64_enc_tail_call(iRegP jump_target) %{
 3744     C2_MacroAssembler _masm(&cbuf);
 3745     Register target_reg = as_Register($jump_target$$reg);
 3746     __ br(target_reg);
 3747   %}
 3748 
 3749   enc_class aarch64_enc_tail_jmp(iRegP jump_target) %{
 3750     C2_MacroAssembler _masm(&cbuf);
 3751     Register target_reg = as_Register($jump_target$$reg);
 3752     // exception oop should be in r0
 3753     // ret addr has been popped into lr
 3754     // callee expects it in r3
 3755     __ mov(r3, lr);
 3756     __ br(target_reg);
 3757   %}
 3758 
 3759   enc_class aarch64_enc_fast_lock(iRegP object, iRegP box, iRegP tmp, iRegP tmp2) %{
 3760     C2_MacroAssembler _masm(&cbuf);
 3761     Register oop = as_Register($object$$reg);
 3762     Register box = as_Register($box$$reg);
 3763     Register disp_hdr = as_Register($tmp$$reg);
 3764     Register tmp = as_Register($tmp2$$reg);
 3765     Label cont;
 3766     Label object_has_monitor;
 3767     Label no_count;
 3768 
 3769     assert_different_registers(oop, box, tmp, disp_hdr);
 3770 
 3771     // Load markWord from object into displaced_header.
 3772     __ ldr(disp_hdr, Address(oop, oopDesc::mark_offset_in_bytes()));
 3773 
 3774     if (DiagnoseSyncOnValueBasedClasses != 0) {
 3775       __ load_klass(tmp, oop);
 3776       __ ldrw(tmp, Address(tmp, Klass::access_flags_offset()));
 3777       __ tstw(tmp, JVM_ACC_IS_VALUE_BASED_CLASS);
 3778       __ br(Assembler::NE, cont);
 3779     }
 3780 
 3781     // Check for existing monitor
 3782     __ tbnz(disp_hdr, exact_log2(markWord::monitor_value), object_has_monitor);
 3783 
 3784     if (!UseHeavyMonitors) {
 3785       // Set tmp to be (markWord of object | UNLOCK_VALUE).
 3786       __ orr(tmp, disp_hdr, markWord::unlocked_value);
 3787 
 3788       // Initialize the box. (Must happen before we update the object mark!)
 3789       __ str(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));
 3790 
 3791       // Compare object markWord with an unlocked value (tmp) and if
 3792       // equal exchange the stack address of our box with object markWord.
 3793       // On failure disp_hdr contains the possibly locked markWord.
 3794       __ cmpxchg(oop, tmp, box, Assembler::xword, /*acquire*/ true,
 3795                  /*release*/ true, /*weak*/ false, disp_hdr);
 3796       __ br(Assembler::EQ, cont);
 3797 
 3798       assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
 3799 
 3800       // If the compare-and-exchange succeeded, then we found an unlocked
 3801       // object, will have now locked it will continue at label cont
 3802 
 3803       // Check if the owner is self by comparing the value in the
 3804       // markWord of object (disp_hdr) with the stack pointer.
 3805       __ mov(rscratch1, sp);
 3806       __ sub(disp_hdr, disp_hdr, rscratch1);
 3807       __ mov(tmp, (address) (~(os::vm_page_size()-1) | markWord::lock_mask_in_place));
 3808       // If condition is true we are cont and hence we can store 0 as the
 3809       // displaced header in the box, which indicates that it is a recursive lock.
 3810       __ ands(tmp/*==0?*/, disp_hdr, tmp);   // Sets flags for result
 3811       __ str(tmp/*==0, perhaps*/, Address(box, BasicLock::displaced_header_offset_in_bytes()));





















 3812     } else {
 3813       __ tst(oop, oop); // Set NE to indicate 'failure' -> take slow-path. We know that oop != 0.

 3814     }
 3815     __ b(cont);
 3816 
 3817     // Handle existing monitor.
 3818     __ bind(object_has_monitor);
 3819 
 3820     // The object's monitor m is unlocked iff m->owner == NULL,
 3821     // otherwise m->owner may contain a thread or a stack address.
 3822     //
 3823     // Try to CAS m->owner from NULL to current thread.
 3824     __ add(tmp, disp_hdr, (ObjectMonitor::owner_offset_in_bytes()-markWord::monitor_value));
 3825     __ cmpxchg(tmp, zr, rthread, Assembler::xword, /*acquire*/ true,
 3826                /*release*/ true, /*weak*/ false, rscratch1); // Sets flags for result
 3827 
 3828     // Store a non-null value into the box to avoid looking like a re-entrant
 3829     // lock. The fast-path monitor unlock code checks for
 3830     // markWord::monitor_value so use markWord::unused_mark which has the
 3831     // relevant bit set, and also matches ObjectSynchronizer::enter.
 3832     __ mov(tmp, (address)markWord::unused_mark().value());
 3833     __ str(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));
 3834 

 3835     __ br(Assembler::EQ, cont); // CAS success means locking succeeded
 3836 
 3837     __ cmp(rscratch1, rthread);
 3838     __ br(Assembler::NE, cont); // Check for recursive locking
 3839 
 3840     // Recursive lock case
 3841     __ increment(Address(disp_hdr, ObjectMonitor::recursions_offset_in_bytes() - markWord::monitor_value), 1);
 3842     // flag == EQ still from the cmp above, checking if this is a reentrant lock
 3843 
 3844     __ bind(cont);
 3845     // flag == EQ indicates success
 3846     // flag == NE indicates failure
 3847     __ br(Assembler::NE, no_count);
 3848 

 3849     __ increment(Address(rthread, JavaThread::held_monitor_count_offset()));
 3850 
 3851     __ bind(no_count);
 3852   %}
 3853 
 3854   enc_class aarch64_enc_fast_unlock(iRegP object, iRegP box, iRegP tmp, iRegP tmp2) %{
 3855     C2_MacroAssembler _masm(&cbuf);
 3856     Register oop = as_Register($object$$reg);
 3857     Register box = as_Register($box$$reg);
 3858     Register disp_hdr = as_Register($tmp$$reg);
 3859     Register tmp = as_Register($tmp2$$reg);
 3860     Label cont;
 3861     Label object_has_monitor;
 3862     Label no_count;
 3863 
 3864     assert_different_registers(oop, box, tmp, disp_hdr);
 3865 
 3866     if (!UseHeavyMonitors) {
 3867       // Find the lock address and load the displaced header from the stack.
 3868       __ ldr(disp_hdr, Address(box, BasicLock::displaced_header_offset_in_bytes()));
 3869 
 3870       // If the displaced header is 0, we have a recursive unlock.
 3871       __ cmp(disp_hdr, zr);
 3872       __ br(Assembler::EQ, cont);
 3873     }
 3874 
 3875     // Handle existing monitor.
 3876     __ ldr(tmp, Address(oop, oopDesc::mark_offset_in_bytes()));
 3877     __ tbnz(tmp, exact_log2(markWord::monitor_value), object_has_monitor);
 3878 
 3879     if (!UseHeavyMonitors) {
 3880       // Check if it is still a light weight lock, this is is true if we
 3881       // see the stack address of the basicLock in the markWord of the
 3882       // object.







 3883 
 3884       __ cmpxchg(oop, box, disp_hdr, Assembler::xword, /*acquire*/ false,
 3885                  /*release*/ true, /*weak*/ false, tmp);


 3886     } else {
 3887       __ tst(oop, oop); // Set NE to indicate 'failure' -> take slow-path. We know that oop != 0.

 3888     }
 3889     __ b(cont);
 3890 
 3891     assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
 3892 
 3893     // Handle existing monitor.
 3894     __ bind(object_has_monitor);
 3895     STATIC_ASSERT(markWord::monitor_value <= INT_MAX);
 3896     __ add(tmp, tmp, -(int)markWord::monitor_value); // monitor











 3897     __ ldr(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset_in_bytes()));
 3898 
 3899     Label notRecursive;
 3900     __ cbz(disp_hdr, notRecursive);
 3901 
 3902     // Recursive lock
 3903     __ sub(disp_hdr, disp_hdr, 1u);
 3904     __ str(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset_in_bytes()));
 3905     __ cmp(disp_hdr, disp_hdr); // Sets flags for result
 3906     __ b(cont);
 3907 
 3908     __ bind(notRecursive);
 3909     __ ldr(rscratch1, Address(tmp, ObjectMonitor::EntryList_offset_in_bytes()));
 3910     __ ldr(disp_hdr, Address(tmp, ObjectMonitor::cxq_offset_in_bytes()));
 3911     __ orr(rscratch1, rscratch1, disp_hdr); // Will be 0 if both are 0.
 3912     __ cmp(rscratch1, zr); // Sets flags for result
 3913     __ cbnz(rscratch1, cont);
 3914     // need a release store here
 3915     __ lea(tmp, Address(tmp, ObjectMonitor::owner_offset_in_bytes()));
 3916     __ stlr(zr, tmp); // set unowned
 3917 
 3918     __ bind(cont);
 3919     // flag == EQ indicates success
 3920     // flag == NE indicates failure
 3921     __ br(Assembler::NE, no_count);
 3922 

 3923     __ decrement(Address(rthread, JavaThread::held_monitor_count_offset()));
 3924 
 3925     __ bind(no_count);
 3926   %}
 3927 
 3928 %}
 3929 
 3930 //----------FRAME--------------------------------------------------------------
 3931 // Definition of frame structure and management information.
 3932 //
 3933 //  S T A C K   L A Y O U T    Allocators stack-slot number
 3934 //                             |   (to get allocators register number
 3935 //  G  Owned by    |        |  v    add OptoReg::stack0())
 3936 //  r   CALLER     |        |
 3937 //  o     |        +--------+      pad to even-align allocators stack-slot
 3938 //  w     V        |  pad0  |        numbers; owned by CALLER
 3939 //  t   -----------+--------+----> Matcher::_in_arg_limit, unaligned
 3940 //  h     ^        |   in   |  5
 3941 //        |        |  args  |  4   Holes in incoming args owned by SELF
 3942 //  |     |        |        |  3
 3943 //  |     |        +--------+
 3944 //  V     |        | old out|      Empty on Intel, window on Sparc
 3945 //        |    old |preserve|      Must be even aligned.
 3946 //        |     SP-+--------+----> Matcher::_old_SP, even aligned
 3947 //        |        |   in   |  3   area for Intel ret address
 3948 //     Owned by    |preserve|      Empty on Sparc.
 3949 //       SELF      +--------+
 3950 //        |        |  pad2  |  2   pad to align old SP
 3951 //        |        +--------+  1
 3952 //        |        | locks  |  0
 3953 //        |        +--------+----> OptoReg::stack0(), even aligned
 3954 //        |        |  pad1  | 11   pad to align new SP
 3955 //        |        +--------+
 3956 //        |        |        | 10
 3957 //        |        | spills |  9   spills
 3958 //        V        |        |  8   (pad0 slot for callee)
 3959 //      -----------+--------+----> Matcher::_out_arg_limit, unaligned
 3960 //        ^        |  out   |  7
 3961 //        |        |  args  |  6   Holes in outgoing args owned by CALLEE
 3962 //     Owned by    +--------+
 3963 //      CALLEE     | new out|  6   Empty on Intel, window on Sparc
 3964 //        |    new |preserve|      Must be even-aligned.
 3965 //        |     SP-+--------+----> Matcher::_new_SP, even aligned
 3966 //        |        |        |
 3967 //
 3968 // Note 1: Only region 8-11 is determined by the allocator.  Region 0-5 is
 3969 //         known from SELF's arguments and the Java calling convention.
 3970 //         Region 6-7 is determined per call site.
 3971 // Note 2: If the calling convention leaves holes in the incoming argument
 3972 //         area, those holes are owned by SELF.  Holes in the outgoing area
 3973 //         are owned by the CALLEE.  Holes should not be necessary in the
 3974 //         incoming area, as the Java calling convention is completely under
 3975 //         the control of the AD file.  Doubles can be sorted and packed to
 3976 //         avoid holes.  Holes in the outgoing arguments may be necessary for
 3977 //         varargs C calling conventions.
 3978 // Note 3: Region 0-3 is even aligned, with pad2 as needed.  Region 3-5 is
 3979 //         even aligned with pad0 as needed.
 3980 //         Region 6 is even aligned.  Region 6-7 is NOT even aligned;
 3981 //           (the latter is true on Intel but is it false on AArch64?)
 3982 //         region 6-11 is even aligned; it may be padded out more so that
 3983 //         the region from SP to FP meets the minimum stack alignment.
 3984 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
 3985 //         alignment.  Region 11, pad1, may be dynamically extended so that
 3986 //         SP meets the minimum alignment.
 3987 
 3988 frame %{
 3989   // These three registers define part of the calling convention
 3990   // between compiled code and the interpreter.
 3991 
 3992   // Inline Cache Register or Method for I2C.
 3993   inline_cache_reg(R12);
 3994 
 3995   // Number of stack slots consumed by locking an object
 3996   sync_stack_slots(2);
 3997 
 3998   // Compiled code's Frame Pointer
 3999   frame_pointer(R31);
 4000 
 4001   // Interpreter stores its frame pointer in a register which is
 4002   // stored to the stack by I2CAdaptors.
 4003   // I2CAdaptors convert from interpreted java to compiled java.
 4004   interpreter_frame_pointer(R29);
 4005 
 4006   // Stack alignment requirement
 4007   stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
 4008 
 4009   // Number of outgoing stack slots killed above the out_preserve_stack_slots
 4010   // for calls to C.  Supports the var-args backing area for register parms.
 4011   varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
 4012 
 4013   // The after-PROLOG location of the return address.  Location of
 4014   // return address specifies a type (REG or STACK) and a number
 4015   // representing the register number (i.e. - use a register name) or
 4016   // stack slot.
 4017   // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
 4018   // Otherwise, it is above the locks and verification slot and alignment word
 4019   // TODO this may well be correct but need to check why that - 2 is there
 4020   // ppc port uses 0 but we definitely need to allow for fixed_slots
 4021   // which folds in the space used for monitors
 4022   return_addr(STACK - 2 +
 4023               align_up((Compile::current()->in_preserve_stack_slots() +
 4024                         Compile::current()->fixed_slots()),
 4025                        stack_alignment_in_slots()));
 4026 
 4027   // Location of compiled Java return values.  Same as C for now.
 4028   return_value
 4029   %{
 4030     // TODO do we allow ideal_reg == Op_RegN???
 4031     assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
 4032            "only return normal values");
 4033 
 4034     static const int lo[Op_RegL + 1] = { // enum name
 4035       0,                                 // Op_Node
 4036       0,                                 // Op_Set
 4037       R0_num,                            // Op_RegN
 4038       R0_num,                            // Op_RegI
 4039       R0_num,                            // Op_RegP
 4040       V0_num,                            // Op_RegF
 4041       V0_num,                            // Op_RegD
 4042       R0_num                             // Op_RegL
 4043     };
 4044 
 4045     static const int hi[Op_RegL + 1] = { // enum name
 4046       0,                                 // Op_Node
 4047       0,                                 // Op_Set
 4048       OptoReg::Bad,                      // Op_RegN
 4049       OptoReg::Bad,                      // Op_RegI
 4050       R0_H_num,                          // Op_RegP
 4051       OptoReg::Bad,                      // Op_RegF
 4052       V0_H_num,                          // Op_RegD
 4053       R0_H_num                           // Op_RegL
 4054     };
 4055 
 4056     return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
 4057   %}
 4058 %}
 4059 
 4060 //----------ATTRIBUTES---------------------------------------------------------
 4061 //----------Operand Attributes-------------------------------------------------
 4062 op_attrib op_cost(1);        // Required cost attribute
 4063 
 4064 //----------Instruction Attributes---------------------------------------------
 4065 ins_attrib ins_cost(INSN_COST); // Required cost attribute
 4066 ins_attrib ins_size(32);        // Required size attribute (in bits)
 4067 ins_attrib ins_short_branch(0); // Required flag: is this instruction
 4068                                 // a non-matching short branch variant
 4069                                 // of some long branch?
 4070 ins_attrib ins_alignment(4);    // Required alignment attribute (must
 4071                                 // be a power of 2) specifies the
 4072                                 // alignment that some part of the
 4073                                 // instruction (not necessarily the
 4074                                 // start) requires.  If > 1, a
 4075                                 // compute_padding() function must be
 4076                                 // provided for the instruction
 4077 
 4078 //----------OPERANDS-----------------------------------------------------------
 4079 // Operand definitions must precede instruction definitions for correct parsing
 4080 // in the ADLC because operands constitute user defined types which are used in
 4081 // instruction definitions.
 4082 
 4083 //----------Simple Operands----------------------------------------------------
 4084 
 4085 // Integer operands 32 bit
 4086 // 32 bit immediate
 4087 operand immI()
 4088 %{
 4089   match(ConI);
 4090 
 4091   op_cost(0);
 4092   format %{ %}
 4093   interface(CONST_INTER);
 4094 %}
 4095 
 4096 // 32 bit zero
 4097 operand immI0()
 4098 %{
 4099   predicate(n->get_int() == 0);
 4100   match(ConI);
 4101 
 4102   op_cost(0);
 4103   format %{ %}
 4104   interface(CONST_INTER);
 4105 %}
 4106 
 4107 // 32 bit unit increment
 4108 operand immI_1()
 4109 %{
 4110   predicate(n->get_int() == 1);
 4111   match(ConI);
 4112 
 4113   op_cost(0);
 4114   format %{ %}
 4115   interface(CONST_INTER);
 4116 %}
 4117 
 4118 // 32 bit unit decrement
 4119 operand immI_M1()
 4120 %{
 4121   predicate(n->get_int() == -1);
 4122   match(ConI);
 4123 
 4124   op_cost(0);
 4125   format %{ %}
 4126   interface(CONST_INTER);
 4127 %}
 4128 
 4129 // Shift values for add/sub extension shift
 4130 operand immIExt()
 4131 %{
 4132   predicate(0 <= n->get_int() && (n->get_int() <= 4));
 4133   match(ConI);
 4134 
 4135   op_cost(0);
 4136   format %{ %}
 4137   interface(CONST_INTER);
 4138 %}
 4139 
 4140 operand immI_gt_1()
 4141 %{
 4142   predicate(n->get_int() > 1);
 4143   match(ConI);
 4144 
 4145   op_cost(0);
 4146   format %{ %}
 4147   interface(CONST_INTER);
 4148 %}
 4149 
 4150 operand immI_le_4()
 4151 %{
 4152   predicate(n->get_int() <= 4);
 4153   match(ConI);
 4154 
 4155   op_cost(0);
 4156   format %{ %}
 4157   interface(CONST_INTER);
 4158 %}
 4159 
 4160 operand immI_16()
 4161 %{
 4162   predicate(n->get_int() == 16);
 4163   match(ConI);
 4164 
 4165   op_cost(0);
 4166   format %{ %}
 4167   interface(CONST_INTER);
 4168 %}
 4169 
 4170 operand immI_24()
 4171 %{
 4172   predicate(n->get_int() == 24);
 4173   match(ConI);
 4174 
 4175   op_cost(0);
 4176   format %{ %}
 4177   interface(CONST_INTER);
 4178 %}
 4179 
 4180 operand immI_32()
 4181 %{
 4182   predicate(n->get_int() == 32);
 4183   match(ConI);
 4184 
 4185   op_cost(0);
 4186   format %{ %}
 4187   interface(CONST_INTER);
 4188 %}
 4189 
 4190 operand immI_48()
 4191 %{
 4192   predicate(n->get_int() == 48);
 4193   match(ConI);
 4194 
 4195   op_cost(0);
 4196   format %{ %}
 4197   interface(CONST_INTER);
 4198 %}
 4199 
 4200 operand immI_56()
 4201 %{
 4202   predicate(n->get_int() == 56);
 4203   match(ConI);
 4204 
 4205   op_cost(0);
 4206   format %{ %}
 4207   interface(CONST_INTER);
 4208 %}
 4209 
 4210 operand immI_63()
 4211 %{
 4212   predicate(n->get_int() == 63);
 4213   match(ConI);
 4214 
 4215   op_cost(0);
 4216   format %{ %}
 4217   interface(CONST_INTER);
 4218 %}
 4219 
 4220 operand immI_64()
 4221 %{
 4222   predicate(n->get_int() == 64);
 4223   match(ConI);
 4224 
 4225   op_cost(0);
 4226   format %{ %}
 4227   interface(CONST_INTER);
 4228 %}
 4229 
 4230 operand immI_255()
 4231 %{
 4232   predicate(n->get_int() == 255);
 4233   match(ConI);
 4234 
 4235   op_cost(0);
 4236   format %{ %}
 4237   interface(CONST_INTER);
 4238 %}
 4239 
 4240 operand immI_65535()
 4241 %{
 4242   predicate(n->get_int() == 65535);
 4243   match(ConI);
 4244 
 4245   op_cost(0);
 4246   format %{ %}
 4247   interface(CONST_INTER);
 4248 %}
 4249 
 4250 operand immI_positive()
 4251 %{
 4252   predicate(n->get_int() > 0);
 4253   match(ConI);
 4254 
 4255   op_cost(0);
 4256   format %{ %}
 4257   interface(CONST_INTER);
 4258 %}
 4259 
 4260 operand immL_255()
 4261 %{
 4262   predicate(n->get_long() == 255L);
 4263   match(ConL);
 4264 
 4265   op_cost(0);
 4266   format %{ %}
 4267   interface(CONST_INTER);
 4268 %}
 4269 
 4270 operand immL_65535()
 4271 %{
 4272   predicate(n->get_long() == 65535L);
 4273   match(ConL);
 4274 
 4275   op_cost(0);
 4276   format %{ %}
 4277   interface(CONST_INTER);
 4278 %}
 4279 
 4280 operand immL_4294967295()
 4281 %{
 4282   predicate(n->get_long() == 4294967295L);
 4283   match(ConL);
 4284 
 4285   op_cost(0);
 4286   format %{ %}
 4287   interface(CONST_INTER);
 4288 %}
 4289 
 4290 operand immL_bitmask()
 4291 %{
 4292   predicate((n->get_long() != 0)
 4293             && ((n->get_long() & 0xc000000000000000l) == 0)
 4294             && is_power_of_2(n->get_long() + 1));
 4295   match(ConL);
 4296 
 4297   op_cost(0);
 4298   format %{ %}
 4299   interface(CONST_INTER);
 4300 %}
 4301 
 4302 operand immI_bitmask()
 4303 %{
 4304   predicate((n->get_int() != 0)
 4305             && ((n->get_int() & 0xc0000000) == 0)
 4306             && is_power_of_2(n->get_int() + 1));
 4307   match(ConI);
 4308 
 4309   op_cost(0);
 4310   format %{ %}
 4311   interface(CONST_INTER);
 4312 %}
 4313 
 4314 operand immL_positive_bitmaskI()
 4315 %{
 4316   predicate((n->get_long() != 0)
 4317             && ((julong)n->get_long() < 0x80000000ULL)
 4318             && is_power_of_2(n->get_long() + 1));
 4319   match(ConL);
 4320 
 4321   op_cost(0);
 4322   format %{ %}
 4323   interface(CONST_INTER);
 4324 %}
 4325 
 4326 // Scale values for scaled offset addressing modes (up to long but not quad)
 4327 operand immIScale()
 4328 %{
 4329   predicate(0 <= n->get_int() && (n->get_int() <= 3));
 4330   match(ConI);
 4331 
 4332   op_cost(0);
 4333   format %{ %}
 4334   interface(CONST_INTER);
 4335 %}
 4336 
 4337 // 26 bit signed offset -- for pc-relative branches
 4338 operand immI26()
 4339 %{
 4340   predicate(((-(1 << 25)) <= n->get_int()) && (n->get_int() < (1 << 25)));
 4341   match(ConI);
 4342 
 4343   op_cost(0);
 4344   format %{ %}
 4345   interface(CONST_INTER);
 4346 %}
 4347 
 4348 // 19 bit signed offset -- for pc-relative loads
 4349 operand immI19()
 4350 %{
 4351   predicate(((-(1 << 18)) <= n->get_int()) && (n->get_int() < (1 << 18)));
 4352   match(ConI);
 4353 
 4354   op_cost(0);
 4355   format %{ %}
 4356   interface(CONST_INTER);
 4357 %}
 4358 
 4359 // 12 bit unsigned offset -- for base plus immediate loads
 4360 operand immIU12()
 4361 %{
 4362   predicate((0 <= n->get_int()) && (n->get_int() < (1 << 12)));
 4363   match(ConI);
 4364 
 4365   op_cost(0);
 4366   format %{ %}
 4367   interface(CONST_INTER);
 4368 %}
 4369 
 4370 operand immLU12()
 4371 %{
 4372   predicate((0 <= n->get_long()) && (n->get_long() < (1 << 12)));
 4373   match(ConL);
 4374 
 4375   op_cost(0);
 4376   format %{ %}
 4377   interface(CONST_INTER);
 4378 %}
 4379 
 4380 // Offset for scaled or unscaled immediate loads and stores
 4381 operand immIOffset()
 4382 %{
 4383   predicate(Address::offset_ok_for_immed(n->get_int(), 0));
 4384   match(ConI);
 4385 
 4386   op_cost(0);
 4387   format %{ %}
 4388   interface(CONST_INTER);
 4389 %}
 4390 
 4391 operand immIOffset1()
 4392 %{
 4393   predicate(Address::offset_ok_for_immed(n->get_int(), 0));
 4394   match(ConI);
 4395 
 4396   op_cost(0);
 4397   format %{ %}
 4398   interface(CONST_INTER);
 4399 %}
 4400 
 4401 operand immIOffset2()
 4402 %{
 4403   predicate(Address::offset_ok_for_immed(n->get_int(), 1));
 4404   match(ConI);
 4405 
 4406   op_cost(0);
 4407   format %{ %}
 4408   interface(CONST_INTER);
 4409 %}
 4410 
 4411 operand immIOffset4()
 4412 %{
 4413   predicate(Address::offset_ok_for_immed(n->get_int(), 2));
 4414   match(ConI);
 4415 
 4416   op_cost(0);
 4417   format %{ %}
 4418   interface(CONST_INTER);
 4419 %}
 4420 
 4421 operand immIOffset8()
 4422 %{
 4423   predicate(Address::offset_ok_for_immed(n->get_int(), 3));
 4424   match(ConI);
 4425 
 4426   op_cost(0);
 4427   format %{ %}
 4428   interface(CONST_INTER);
 4429 %}
 4430 
 4431 operand immIOffset16()
 4432 %{
 4433   predicate(Address::offset_ok_for_immed(n->get_int(), 4));
 4434   match(ConI);
 4435 
 4436   op_cost(0);
 4437   format %{ %}
 4438   interface(CONST_INTER);
 4439 %}
 4440 
 4441 operand immLoffset()
 4442 %{
 4443   predicate(Address::offset_ok_for_immed(n->get_long(), 0));
 4444   match(ConL);
 4445 
 4446   op_cost(0);
 4447   format %{ %}
 4448   interface(CONST_INTER);
 4449 %}
 4450 
 4451 operand immLoffset1()
 4452 %{
 4453   predicate(Address::offset_ok_for_immed(n->get_long(), 0));
 4454   match(ConL);
 4455 
 4456   op_cost(0);
 4457   format %{ %}
 4458   interface(CONST_INTER);
 4459 %}
 4460 
 4461 operand immLoffset2()
 4462 %{
 4463   predicate(Address::offset_ok_for_immed(n->get_long(), 1));
 4464   match(ConL);
 4465 
 4466   op_cost(0);
 4467   format %{ %}
 4468   interface(CONST_INTER);
 4469 %}
 4470 
 4471 operand immLoffset4()
 4472 %{
 4473   predicate(Address::offset_ok_for_immed(n->get_long(), 2));
 4474   match(ConL);
 4475 
 4476   op_cost(0);
 4477   format %{ %}
 4478   interface(CONST_INTER);
 4479 %}
 4480 
 4481 operand immLoffset8()
 4482 %{
 4483   predicate(Address::offset_ok_for_immed(n->get_long(), 3));
 4484   match(ConL);
 4485 
 4486   op_cost(0);
 4487   format %{ %}
 4488   interface(CONST_INTER);
 4489 %}
 4490 
 4491 operand immLoffset16()
 4492 %{
 4493   predicate(Address::offset_ok_for_immed(n->get_long(), 4));
 4494   match(ConL);
 4495 
 4496   op_cost(0);
 4497   format %{ %}
 4498   interface(CONST_INTER);
 4499 %}
 4500 
 4501 // 8 bit signed value.
 4502 operand immI8()
 4503 %{
 4504   predicate(n->get_int() <= 127 && n->get_int() >= -128);
 4505   match(ConI);
 4506 
 4507   op_cost(0);
 4508   format %{ %}
 4509   interface(CONST_INTER);
 4510 %}
 4511 
 4512 // 8 bit signed value (simm8), or #simm8 LSL 8.
 4513 operand immI8_shift8()
 4514 %{
 4515   predicate((n->get_int() <= 127 && n->get_int() >= -128) ||
 4516             (n->get_int() <= 32512 && n->get_int() >= -32768 && (n->get_int() & 0xff) == 0));
 4517   match(ConI);
 4518 
 4519   op_cost(0);
 4520   format %{ %}
 4521   interface(CONST_INTER);
 4522 %}
 4523 
 4524 // 8 bit signed value (simm8), or #simm8 LSL 8.
 4525 operand immL8_shift8()
 4526 %{
 4527   predicate((n->get_long() <= 127 && n->get_long() >= -128) ||
 4528             (n->get_long() <= 32512 && n->get_long() >= -32768 && (n->get_long() & 0xff) == 0));
 4529   match(ConL);
 4530 
 4531   op_cost(0);
 4532   format %{ %}
 4533   interface(CONST_INTER);
 4534 %}
 4535 
 4536 // 8 bit integer valid for vector add sub immediate
 4537 operand immBAddSubV()
 4538 %{
 4539   predicate(n->get_int() <= 255 && n->get_int() >= -255);
 4540   match(ConI);
 4541 
 4542   op_cost(0);
 4543   format %{ %}
 4544   interface(CONST_INTER);
 4545 %}
 4546 
 4547 // 32 bit integer valid for add sub immediate
 4548 operand immIAddSub()
 4549 %{
 4550   predicate(Assembler::operand_valid_for_add_sub_immediate((int64_t)n->get_int()));
 4551   match(ConI);
 4552   op_cost(0);
 4553   format %{ %}
 4554   interface(CONST_INTER);
 4555 %}
 4556 
 4557 // 32 bit integer valid for vector add sub immediate
 4558 operand immIAddSubV()
 4559 %{
 4560   predicate(Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)n->get_int()));
 4561   match(ConI);
 4562 
 4563   op_cost(0);
 4564   format %{ %}
 4565   interface(CONST_INTER);
 4566 %}
 4567 
 4568 // 32 bit unsigned integer valid for logical immediate
 4569 
 4570 operand immBLog()
 4571 %{
 4572   predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerByte, (uint64_t)n->get_int()));
 4573   match(ConI);
 4574 
 4575   op_cost(0);
 4576   format %{ %}
 4577   interface(CONST_INTER);
 4578 %}
 4579 
 4580 operand immSLog()
 4581 %{
 4582   predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerShort, (uint64_t)n->get_int()));
 4583   match(ConI);
 4584 
 4585   op_cost(0);
 4586   format %{ %}
 4587   interface(CONST_INTER);
 4588 %}
 4589 
 4590 operand immILog()
 4591 %{
 4592   predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/true, (uint64_t)n->get_int()));
 4593   match(ConI);
 4594 
 4595   op_cost(0);
 4596   format %{ %}
 4597   interface(CONST_INTER);
 4598 %}
 4599 
 4600 // Integer operands 64 bit
 4601 // 64 bit immediate
 4602 operand immL()
 4603 %{
 4604   match(ConL);
 4605 
 4606   op_cost(0);
 4607   format %{ %}
 4608   interface(CONST_INTER);
 4609 %}
 4610 
 4611 // 64 bit zero
 4612 operand immL0()
 4613 %{
 4614   predicate(n->get_long() == 0);
 4615   match(ConL);
 4616 
 4617   op_cost(0);
 4618   format %{ %}
 4619   interface(CONST_INTER);
 4620 %}
 4621 
 4622 // 64 bit unit increment
 4623 operand immL_1()
 4624 %{
 4625   predicate(n->get_long() == 1);
 4626   match(ConL);
 4627 
 4628   op_cost(0);
 4629   format %{ %}
 4630   interface(CONST_INTER);
 4631 %}
 4632 
 4633 // 64 bit unit decrement
 4634 operand immL_M1()
 4635 %{
 4636   predicate(n->get_long() == -1);
 4637   match(ConL);
 4638 
 4639   op_cost(0);
 4640   format %{ %}
 4641   interface(CONST_INTER);
 4642 %}
 4643 
 4644 // 32 bit offset of pc in thread anchor
 4645 
 4646 operand immL_pc_off()
 4647 %{
 4648   predicate(n->get_long() == in_bytes(JavaThread::frame_anchor_offset()) +
 4649                              in_bytes(JavaFrameAnchor::last_Java_pc_offset()));
 4650   match(ConL);
 4651 
 4652   op_cost(0);
 4653   format %{ %}
 4654   interface(CONST_INTER);
 4655 %}
 4656 
 4657 // 64 bit integer valid for add sub immediate
 4658 operand immLAddSub()
 4659 %{
 4660   predicate(Assembler::operand_valid_for_add_sub_immediate(n->get_long()));
 4661   match(ConL);
 4662   op_cost(0);
 4663   format %{ %}
 4664   interface(CONST_INTER);
 4665 %}
 4666 
 4667 // 64 bit integer valid for addv subv immediate
 4668 operand immLAddSubV()
 4669 %{
 4670   predicate(Assembler::operand_valid_for_sve_add_sub_immediate(n->get_long()));
 4671   match(ConL);
 4672 
 4673   op_cost(0);
 4674   format %{ %}
 4675   interface(CONST_INTER);
 4676 %}
 4677 
 4678 // 64 bit integer valid for logical immediate
 4679 operand immLLog()
 4680 %{
 4681   predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/false, (uint64_t)n->get_long()));
 4682   match(ConL);
 4683   op_cost(0);
 4684   format %{ %}
 4685   interface(CONST_INTER);
 4686 %}
 4687 
 4688 // Long Immediate: low 32-bit mask
 4689 operand immL_32bits()
 4690 %{
 4691   predicate(n->get_long() == 0xFFFFFFFFL);
 4692   match(ConL);
 4693   op_cost(0);
 4694   format %{ %}
 4695   interface(CONST_INTER);
 4696 %}
 4697 
 4698 // Pointer operands
 4699 // Pointer Immediate
 4700 operand immP()
 4701 %{
 4702   match(ConP);
 4703 
 4704   op_cost(0);
 4705   format %{ %}
 4706   interface(CONST_INTER);
 4707 %}
 4708 
 4709 // NULL Pointer Immediate
 4710 operand immP0()
 4711 %{
 4712   predicate(n->get_ptr() == 0);
 4713   match(ConP);
 4714 
 4715   op_cost(0);
 4716   format %{ %}
 4717   interface(CONST_INTER);
 4718 %}
 4719 
 4720 // Pointer Immediate One
 4721 // this is used in object initialization (initial object header)
 4722 operand immP_1()
 4723 %{
 4724   predicate(n->get_ptr() == 1);
 4725   match(ConP);
 4726 
 4727   op_cost(0);
 4728   format %{ %}
 4729   interface(CONST_INTER);
 4730 %}
 4731 
 4732 // Card Table Byte Map Base
 4733 operand immByteMapBase()
 4734 %{
 4735   // Get base of card map
 4736   predicate(BarrierSet::barrier_set()->is_a(BarrierSet::CardTableBarrierSet) &&
 4737             (CardTable::CardValue*)n->get_ptr() == ((CardTableBarrierSet*)(BarrierSet::barrier_set()))->card_table()->byte_map_base());
 4738   match(ConP);
 4739 
 4740   op_cost(0);
 4741   format %{ %}
 4742   interface(CONST_INTER);
 4743 %}
 4744 
 4745 // Pointer Immediate Minus One
 4746 // this is used when we want to write the current PC to the thread anchor
 4747 operand immP_M1()
 4748 %{
 4749   predicate(n->get_ptr() == -1);
 4750   match(ConP);
 4751 
 4752   op_cost(0);
 4753   format %{ %}
 4754   interface(CONST_INTER);
 4755 %}
 4756 
 4757 // Pointer Immediate Minus Two
 4758 // this is used when we want to write the current PC to the thread anchor
 4759 operand immP_M2()
 4760 %{
 4761   predicate(n->get_ptr() == -2);
 4762   match(ConP);
 4763 
 4764   op_cost(0);
 4765   format %{ %}
 4766   interface(CONST_INTER);
 4767 %}
 4768 
 4769 // Float and Double operands
 4770 // Double Immediate
 4771 operand immD()
 4772 %{
 4773   match(ConD);
 4774   op_cost(0);
 4775   format %{ %}
 4776   interface(CONST_INTER);
 4777 %}
 4778 
 4779 // Double Immediate: +0.0d
 4780 operand immD0()
 4781 %{
 4782   predicate(jlong_cast(n->getd()) == 0);
 4783   match(ConD);
 4784 
 4785   op_cost(0);
 4786   format %{ %}
 4787   interface(CONST_INTER);
 4788 %}
 4789 
 4790 // constant 'double +0.0'.
 4791 operand immDPacked()
 4792 %{
 4793   predicate(Assembler::operand_valid_for_float_immediate(n->getd()));
 4794   match(ConD);
 4795   op_cost(0);
 4796   format %{ %}
 4797   interface(CONST_INTER);
 4798 %}
 4799 
 4800 // Float Immediate
 4801 operand immF()
 4802 %{
 4803   match(ConF);
 4804   op_cost(0);
 4805   format %{ %}
 4806   interface(CONST_INTER);
 4807 %}
 4808 
 4809 // Float Immediate: +0.0f.
 4810 operand immF0()
 4811 %{
 4812   predicate(jint_cast(n->getf()) == 0);
 4813   match(ConF);
 4814 
 4815   op_cost(0);
 4816   format %{ %}
 4817   interface(CONST_INTER);
 4818 %}
 4819 
 4820 //
 4821 operand immFPacked()
 4822 %{
 4823   predicate(Assembler::operand_valid_for_float_immediate((double)n->getf()));
 4824   match(ConF);
 4825   op_cost(0);
 4826   format %{ %}
 4827   interface(CONST_INTER);
 4828 %}
 4829 
 4830 // Narrow pointer operands
 4831 // Narrow Pointer Immediate
 4832 operand immN()
 4833 %{
 4834   match(ConN);
 4835 
 4836   op_cost(0);
 4837   format %{ %}
 4838   interface(CONST_INTER);
 4839 %}
 4840 
 4841 // Narrow NULL Pointer Immediate
 4842 operand immN0()
 4843 %{
 4844   predicate(n->get_narrowcon() == 0);
 4845   match(ConN);
 4846 
 4847   op_cost(0);
 4848   format %{ %}
 4849   interface(CONST_INTER);
 4850 %}
 4851 
 4852 operand immNKlass()
 4853 %{
 4854   match(ConNKlass);
 4855 
 4856   op_cost(0);
 4857   format %{ %}
 4858   interface(CONST_INTER);
 4859 %}
 4860 
 4861 // Integer 32 bit Register Operands
 4862 // Integer 32 bitRegister (excludes SP)
 4863 operand iRegI()
 4864 %{
 4865   constraint(ALLOC_IN_RC(any_reg32));
 4866   match(RegI);
 4867   match(iRegINoSp);
 4868   op_cost(0);
 4869   format %{ %}
 4870   interface(REG_INTER);
 4871 %}
 4872 
 4873 // Integer 32 bit Register not Special
 4874 operand iRegINoSp()
 4875 %{
 4876   constraint(ALLOC_IN_RC(no_special_reg32));
 4877   match(RegI);
 4878   op_cost(0);
 4879   format %{ %}
 4880   interface(REG_INTER);
 4881 %}
 4882 
 4883 // Integer 64 bit Register Operands
 4884 // Integer 64 bit Register (includes SP)
 4885 operand iRegL()
 4886 %{
 4887   constraint(ALLOC_IN_RC(any_reg));
 4888   match(RegL);
 4889   match(iRegLNoSp);
 4890   op_cost(0);
 4891   format %{ %}
 4892   interface(REG_INTER);
 4893 %}
 4894 
 4895 // Integer 64 bit Register not Special
 4896 operand iRegLNoSp()
 4897 %{
 4898   constraint(ALLOC_IN_RC(no_special_reg));
 4899   match(RegL);
 4900   match(iRegL_R0);
 4901   format %{ %}
 4902   interface(REG_INTER);
 4903 %}
 4904 
 4905 // Pointer Register Operands
 4906 // Pointer Register
 4907 operand iRegP()
 4908 %{
 4909   constraint(ALLOC_IN_RC(ptr_reg));
 4910   match(RegP);
 4911   match(iRegPNoSp);
 4912   match(iRegP_R0);
 4913   //match(iRegP_R2);
 4914   //match(iRegP_R4);
 4915   match(iRegP_R5);
 4916   match(thread_RegP);
 4917   op_cost(0);
 4918   format %{ %}
 4919   interface(REG_INTER);
 4920 %}
 4921 
 4922 // Pointer 64 bit Register not Special
 4923 operand iRegPNoSp()
 4924 %{
 4925   constraint(ALLOC_IN_RC(no_special_ptr_reg));
 4926   match(RegP);
 4927   // match(iRegP);
 4928   // match(iRegP_R0);
 4929   // match(iRegP_R2);
 4930   // match(iRegP_R4);
 4931   // match(iRegP_R5);
 4932   // match(thread_RegP);
 4933   op_cost(0);
 4934   format %{ %}
 4935   interface(REG_INTER);
 4936 %}
 4937 
 4938 // Pointer 64 bit Register R0 only
 4939 operand iRegP_R0()
 4940 %{
 4941   constraint(ALLOC_IN_RC(r0_reg));
 4942   match(RegP);
 4943   // match(iRegP);
 4944   match(iRegPNoSp);
 4945   op_cost(0);
 4946   format %{ %}
 4947   interface(REG_INTER);
 4948 %}
 4949 
 4950 // Pointer 64 bit Register R1 only
 4951 operand iRegP_R1()
 4952 %{
 4953   constraint(ALLOC_IN_RC(r1_reg));
 4954   match(RegP);
 4955   // match(iRegP);
 4956   match(iRegPNoSp);
 4957   op_cost(0);
 4958   format %{ %}
 4959   interface(REG_INTER);
 4960 %}
 4961 
 4962 // Pointer 64 bit Register R2 only
 4963 operand iRegP_R2()
 4964 %{
 4965   constraint(ALLOC_IN_RC(r2_reg));
 4966   match(RegP);
 4967   // match(iRegP);
 4968   match(iRegPNoSp);
 4969   op_cost(0);
 4970   format %{ %}
 4971   interface(REG_INTER);
 4972 %}
 4973 
 4974 // Pointer 64 bit Register R3 only
 4975 operand iRegP_R3()
 4976 %{
 4977   constraint(ALLOC_IN_RC(r3_reg));
 4978   match(RegP);
 4979   // match(iRegP);
 4980   match(iRegPNoSp);
 4981   op_cost(0);
 4982   format %{ %}
 4983   interface(REG_INTER);
 4984 %}
 4985 
 4986 // Pointer 64 bit Register R4 only
 4987 operand iRegP_R4()
 4988 %{
 4989   constraint(ALLOC_IN_RC(r4_reg));
 4990   match(RegP);
 4991   // match(iRegP);
 4992   match(iRegPNoSp);
 4993   op_cost(0);
 4994   format %{ %}
 4995   interface(REG_INTER);
 4996 %}
 4997 
 4998 // Pointer 64 bit Register R5 only
 4999 operand iRegP_R5()
 5000 %{
 5001   constraint(ALLOC_IN_RC(r5_reg));
 5002   match(RegP);
 5003   // match(iRegP);
 5004   match(iRegPNoSp);
 5005   op_cost(0);
 5006   format %{ %}
 5007   interface(REG_INTER);
 5008 %}
 5009 
 5010 // Pointer 64 bit Register R10 only
 5011 operand iRegP_R10()
 5012 %{
 5013   constraint(ALLOC_IN_RC(r10_reg));
 5014   match(RegP);
 5015   // match(iRegP);
 5016   match(iRegPNoSp);
 5017   op_cost(0);
 5018   format %{ %}
 5019   interface(REG_INTER);
 5020 %}
 5021 
 5022 // Long 64 bit Register R0 only
 5023 operand iRegL_R0()
 5024 %{
 5025   constraint(ALLOC_IN_RC(r0_reg));
 5026   match(RegL);
 5027   match(iRegLNoSp);
 5028   op_cost(0);
 5029   format %{ %}
 5030   interface(REG_INTER);
 5031 %}
 5032 
 5033 // Long 64 bit Register R2 only
 5034 operand iRegL_R2()
 5035 %{
 5036   constraint(ALLOC_IN_RC(r2_reg));
 5037   match(RegL);
 5038   match(iRegLNoSp);
 5039   op_cost(0);
 5040   format %{ %}
 5041   interface(REG_INTER);
 5042 %}
 5043 
 5044 // Long 64 bit Register R3 only
 5045 operand iRegL_R3()
 5046 %{
 5047   constraint(ALLOC_IN_RC(r3_reg));
 5048   match(RegL);
 5049   match(iRegLNoSp);
 5050   op_cost(0);
 5051   format %{ %}
 5052   interface(REG_INTER);
 5053 %}
 5054 
 5055 // Long 64 bit Register R11 only
 5056 operand iRegL_R11()
 5057 %{
 5058   constraint(ALLOC_IN_RC(r11_reg));
 5059   match(RegL);
 5060   match(iRegLNoSp);
 5061   op_cost(0);
 5062   format %{ %}
 5063   interface(REG_INTER);
 5064 %}
 5065 
 5066 // Pointer 64 bit Register FP only
 5067 operand iRegP_FP()
 5068 %{
 5069   constraint(ALLOC_IN_RC(fp_reg));
 5070   match(RegP);
 5071   // match(iRegP);
 5072   op_cost(0);
 5073   format %{ %}
 5074   interface(REG_INTER);
 5075 %}
 5076 
 5077 // Register R0 only
 5078 operand iRegI_R0()
 5079 %{
 5080   constraint(ALLOC_IN_RC(int_r0_reg));
 5081   match(RegI);
 5082   match(iRegINoSp);
 5083   op_cost(0);
 5084   format %{ %}
 5085   interface(REG_INTER);
 5086 %}
 5087 
 5088 // Register R2 only
 5089 operand iRegI_R2()
 5090 %{
 5091   constraint(ALLOC_IN_RC(int_r2_reg));
 5092   match(RegI);
 5093   match(iRegINoSp);
 5094   op_cost(0);
 5095   format %{ %}
 5096   interface(REG_INTER);
 5097 %}
 5098 
 5099 // Register R3 only
 5100 operand iRegI_R3()
 5101 %{
 5102   constraint(ALLOC_IN_RC(int_r3_reg));
 5103   match(RegI);
 5104   match(iRegINoSp);
 5105   op_cost(0);
 5106   format %{ %}
 5107   interface(REG_INTER);
 5108 %}
 5109 
 5110 
 5111 // Register R4 only
 5112 operand iRegI_R4()
 5113 %{
 5114   constraint(ALLOC_IN_RC(int_r4_reg));
 5115   match(RegI);
 5116   match(iRegINoSp);
 5117   op_cost(0);
 5118   format %{ %}
 5119   interface(REG_INTER);
 5120 %}
 5121 
 5122 
 5123 // Pointer Register Operands
 5124 // Narrow Pointer Register
 5125 operand iRegN()
 5126 %{
 5127   constraint(ALLOC_IN_RC(any_reg32));
 5128   match(RegN);
 5129   match(iRegNNoSp);
 5130   op_cost(0);
 5131   format %{ %}
 5132   interface(REG_INTER);
 5133 %}
 5134 
 5135 operand iRegN_R0()
 5136 %{
 5137   constraint(ALLOC_IN_RC(r0_reg));
 5138   match(iRegN);
 5139   op_cost(0);
 5140   format %{ %}
 5141   interface(REG_INTER);
 5142 %}
 5143 
 5144 operand iRegN_R2()
 5145 %{
 5146   constraint(ALLOC_IN_RC(r2_reg));
 5147   match(iRegN);
 5148   op_cost(0);
 5149   format %{ %}
 5150   interface(REG_INTER);
 5151 %}
 5152 
 5153 operand iRegN_R3()
 5154 %{
 5155   constraint(ALLOC_IN_RC(r3_reg));
 5156   match(iRegN);
 5157   op_cost(0);
 5158   format %{ %}
 5159   interface(REG_INTER);
 5160 %}
 5161 
 5162 // Integer 64 bit Register not Special
 5163 operand iRegNNoSp()
 5164 %{
 5165   constraint(ALLOC_IN_RC(no_special_reg32));
 5166   match(RegN);
 5167   op_cost(0);
 5168   format %{ %}
 5169   interface(REG_INTER);
 5170 %}
 5171 
 5172 // heap base register -- used for encoding immN0
 5173 
 5174 operand iRegIHeapbase()
 5175 %{
 5176   constraint(ALLOC_IN_RC(heapbase_reg));
 5177   match(RegI);
 5178   op_cost(0);
 5179   format %{ %}
 5180   interface(REG_INTER);
 5181 %}
 5182 
 5183 // Float Register
 5184 // Float register operands
 5185 operand vRegF()
 5186 %{
 5187   constraint(ALLOC_IN_RC(float_reg));
 5188   match(RegF);
 5189 
 5190   op_cost(0);
 5191   format %{ %}
 5192   interface(REG_INTER);
 5193 %}
 5194 
 5195 // Double Register
 5196 // Double register operands
 5197 operand vRegD()
 5198 %{
 5199   constraint(ALLOC_IN_RC(double_reg));
 5200   match(RegD);
 5201 
 5202   op_cost(0);
 5203   format %{ %}
 5204   interface(REG_INTER);
 5205 %}
 5206 
 5207 // Generic vector class. This will be used for
 5208 // all vector operands, including NEON and SVE.
 5209 operand vReg()
 5210 %{
 5211   constraint(ALLOC_IN_RC(dynamic));
 5212   match(VecA);
 5213   match(VecD);
 5214   match(VecX);
 5215 
 5216   op_cost(0);
 5217   format %{ %}
 5218   interface(REG_INTER);
 5219 %}
 5220 
 5221 operand vecA()
 5222 %{
 5223   constraint(ALLOC_IN_RC(vectora_reg));
 5224   match(VecA);
 5225 
 5226   op_cost(0);
 5227   format %{ %}
 5228   interface(REG_INTER);
 5229 %}
 5230 
 5231 operand vecD()
 5232 %{
 5233   constraint(ALLOC_IN_RC(vectord_reg));
 5234   match(VecD);
 5235 
 5236   op_cost(0);
 5237   format %{ %}
 5238   interface(REG_INTER);
 5239 %}
 5240 
 5241 operand vecX()
 5242 %{
 5243   constraint(ALLOC_IN_RC(vectorx_reg));
 5244   match(VecX);
 5245 
 5246   op_cost(0);
 5247   format %{ %}
 5248   interface(REG_INTER);
 5249 %}
 5250 
 5251 operand vRegD_V0()
 5252 %{
 5253   constraint(ALLOC_IN_RC(v0_reg));
 5254   match(RegD);
 5255   op_cost(0);
 5256   format %{ %}
 5257   interface(REG_INTER);
 5258 %}
 5259 
 5260 operand vRegD_V1()
 5261 %{
 5262   constraint(ALLOC_IN_RC(v1_reg));
 5263   match(RegD);
 5264   op_cost(0);
 5265   format %{ %}
 5266   interface(REG_INTER);
 5267 %}
 5268 
 5269 operand vRegD_V2()
 5270 %{
 5271   constraint(ALLOC_IN_RC(v2_reg));
 5272   match(RegD);
 5273   op_cost(0);
 5274   format %{ %}
 5275   interface(REG_INTER);
 5276 %}
 5277 
 5278 operand vRegD_V3()
 5279 %{
 5280   constraint(ALLOC_IN_RC(v3_reg));
 5281   match(RegD);
 5282   op_cost(0);
 5283   format %{ %}
 5284   interface(REG_INTER);
 5285 %}
 5286 
 5287 operand vRegD_V4()
 5288 %{
 5289   constraint(ALLOC_IN_RC(v4_reg));
 5290   match(RegD);
 5291   op_cost(0);
 5292   format %{ %}
 5293   interface(REG_INTER);
 5294 %}
 5295 
 5296 operand vRegD_V5()
 5297 %{
 5298   constraint(ALLOC_IN_RC(v5_reg));
 5299   match(RegD);
 5300   op_cost(0);
 5301   format %{ %}
 5302   interface(REG_INTER);
 5303 %}
 5304 
 5305 operand vRegD_V6()
 5306 %{
 5307   constraint(ALLOC_IN_RC(v6_reg));
 5308   match(RegD);
 5309   op_cost(0);
 5310   format %{ %}
 5311   interface(REG_INTER);
 5312 %}
 5313 
 5314 operand vRegD_V7()
 5315 %{
 5316   constraint(ALLOC_IN_RC(v7_reg));
 5317   match(RegD);
 5318   op_cost(0);
 5319   format %{ %}
 5320   interface(REG_INTER);
 5321 %}
 5322 
 5323 operand vRegD_V8()
 5324 %{
 5325   constraint(ALLOC_IN_RC(v8_reg));
 5326   match(RegD);
 5327   op_cost(0);
 5328   format %{ %}
 5329   interface(REG_INTER);
 5330 %}
 5331 
 5332 operand vRegD_V9()
 5333 %{
 5334   constraint(ALLOC_IN_RC(v9_reg));
 5335   match(RegD);
 5336   op_cost(0);
 5337   format %{ %}
 5338   interface(REG_INTER);
 5339 %}
 5340 
 5341 operand vRegD_V10()
 5342 %{
 5343   constraint(ALLOC_IN_RC(v10_reg));
 5344   match(RegD);
 5345   op_cost(0);
 5346   format %{ %}
 5347   interface(REG_INTER);
 5348 %}
 5349 
 5350 operand vRegD_V11()
 5351 %{
 5352   constraint(ALLOC_IN_RC(v11_reg));
 5353   match(RegD);
 5354   op_cost(0);
 5355   format %{ %}
 5356   interface(REG_INTER);
 5357 %}
 5358 
 5359 operand vRegD_V12()
 5360 %{
 5361   constraint(ALLOC_IN_RC(v12_reg));
 5362   match(RegD);
 5363   op_cost(0);
 5364   format %{ %}
 5365   interface(REG_INTER);
 5366 %}
 5367 
 5368 operand vRegD_V13()
 5369 %{
 5370   constraint(ALLOC_IN_RC(v13_reg));
 5371   match(RegD);
 5372   op_cost(0);
 5373   format %{ %}
 5374   interface(REG_INTER);
 5375 %}
 5376 
 5377 operand vRegD_V14()
 5378 %{
 5379   constraint(ALLOC_IN_RC(v14_reg));
 5380   match(RegD);
 5381   op_cost(0);
 5382   format %{ %}
 5383   interface(REG_INTER);
 5384 %}
 5385 
 5386 operand vRegD_V15()
 5387 %{
 5388   constraint(ALLOC_IN_RC(v15_reg));
 5389   match(RegD);
 5390   op_cost(0);
 5391   format %{ %}
 5392   interface(REG_INTER);
 5393 %}
 5394 
 5395 operand vRegD_V16()
 5396 %{
 5397   constraint(ALLOC_IN_RC(v16_reg));
 5398   match(RegD);
 5399   op_cost(0);
 5400   format %{ %}
 5401   interface(REG_INTER);
 5402 %}
 5403 
 5404 operand vRegD_V17()
 5405 %{
 5406   constraint(ALLOC_IN_RC(v17_reg));
 5407   match(RegD);
 5408   op_cost(0);
 5409   format %{ %}
 5410   interface(REG_INTER);
 5411 %}
 5412 
 5413 operand vRegD_V18()
 5414 %{
 5415   constraint(ALLOC_IN_RC(v18_reg));
 5416   match(RegD);
 5417   op_cost(0);
 5418   format %{ %}
 5419   interface(REG_INTER);
 5420 %}
 5421 
 5422 operand vRegD_V19()
 5423 %{
 5424   constraint(ALLOC_IN_RC(v19_reg));
 5425   match(RegD);
 5426   op_cost(0);
 5427   format %{ %}
 5428   interface(REG_INTER);
 5429 %}
 5430 
 5431 operand vRegD_V20()
 5432 %{
 5433   constraint(ALLOC_IN_RC(v20_reg));
 5434   match(RegD);
 5435   op_cost(0);
 5436   format %{ %}
 5437   interface(REG_INTER);
 5438 %}
 5439 
 5440 operand vRegD_V21()
 5441 %{
 5442   constraint(ALLOC_IN_RC(v21_reg));
 5443   match(RegD);
 5444   op_cost(0);
 5445   format %{ %}
 5446   interface(REG_INTER);
 5447 %}
 5448 
 5449 operand vRegD_V22()
 5450 %{
 5451   constraint(ALLOC_IN_RC(v22_reg));
 5452   match(RegD);
 5453   op_cost(0);
 5454   format %{ %}
 5455   interface(REG_INTER);
 5456 %}
 5457 
 5458 operand vRegD_V23()
 5459 %{
 5460   constraint(ALLOC_IN_RC(v23_reg));
 5461   match(RegD);
 5462   op_cost(0);
 5463   format %{ %}
 5464   interface(REG_INTER);
 5465 %}
 5466 
 5467 operand vRegD_V24()
 5468 %{
 5469   constraint(ALLOC_IN_RC(v24_reg));
 5470   match(RegD);
 5471   op_cost(0);
 5472   format %{ %}
 5473   interface(REG_INTER);
 5474 %}
 5475 
 5476 operand vRegD_V25()
 5477 %{
 5478   constraint(ALLOC_IN_RC(v25_reg));
 5479   match(RegD);
 5480   op_cost(0);
 5481   format %{ %}
 5482   interface(REG_INTER);
 5483 %}
 5484 
 5485 operand vRegD_V26()
 5486 %{
 5487   constraint(ALLOC_IN_RC(v26_reg));
 5488   match(RegD);
 5489   op_cost(0);
 5490   format %{ %}
 5491   interface(REG_INTER);
 5492 %}
 5493 
 5494 operand vRegD_V27()
 5495 %{
 5496   constraint(ALLOC_IN_RC(v27_reg));
 5497   match(RegD);
 5498   op_cost(0);
 5499   format %{ %}
 5500   interface(REG_INTER);
 5501 %}
 5502 
 5503 operand vRegD_V28()
 5504 %{
 5505   constraint(ALLOC_IN_RC(v28_reg));
 5506   match(RegD);
 5507   op_cost(0);
 5508   format %{ %}
 5509   interface(REG_INTER);
 5510 %}
 5511 
 5512 operand vRegD_V29()
 5513 %{
 5514   constraint(ALLOC_IN_RC(v29_reg));
 5515   match(RegD);
 5516   op_cost(0);
 5517   format %{ %}
 5518   interface(REG_INTER);
 5519 %}
 5520 
 5521 operand vRegD_V30()
 5522 %{
 5523   constraint(ALLOC_IN_RC(v30_reg));
 5524   match(RegD);
 5525   op_cost(0);
 5526   format %{ %}
 5527   interface(REG_INTER);
 5528 %}
 5529 
 5530 operand vRegD_V31()
 5531 %{
 5532   constraint(ALLOC_IN_RC(v31_reg));
 5533   match(RegD);
 5534   op_cost(0);
 5535   format %{ %}
 5536   interface(REG_INTER);
 5537 %}
 5538 
 5539 operand pReg()
 5540 %{
 5541   constraint(ALLOC_IN_RC(pr_reg));
 5542   match(RegVectMask);
 5543   match(pRegGov);
 5544   op_cost(0);
 5545   format %{ %}
 5546   interface(REG_INTER);
 5547 %}
 5548 
 5549 operand pRegGov()
 5550 %{
 5551   constraint(ALLOC_IN_RC(gov_pr));
 5552   match(RegVectMask);
 5553   match(pReg);
 5554   op_cost(0);
 5555   format %{ %}
 5556   interface(REG_INTER);
 5557 %}
 5558 
 5559 operand pRegGov_P0()
 5560 %{
 5561   constraint(ALLOC_IN_RC(p0_reg));
 5562   match(RegVectMask);
 5563   op_cost(0);
 5564   format %{ %}
 5565   interface(REG_INTER);
 5566 %}
 5567 
 5568 operand pRegGov_P1()
 5569 %{
 5570   constraint(ALLOC_IN_RC(p1_reg));
 5571   match(RegVectMask);
 5572   op_cost(0);
 5573   format %{ %}
 5574   interface(REG_INTER);
 5575 %}
 5576 
 5577 // Flags register, used as output of signed compare instructions
 5578 
 5579 // note that on AArch64 we also use this register as the output for
 5580 // for floating point compare instructions (CmpF CmpD). this ensures
 5581 // that ordered inequality tests use GT, GE, LT or LE none of which
 5582 // pass through cases where the result is unordered i.e. one or both
 5583 // inputs to the compare is a NaN. this means that the ideal code can
 5584 // replace e.g. a GT with an LE and not end up capturing the NaN case
 5585 // (where the comparison should always fail). EQ and NE tests are
 5586 // always generated in ideal code so that unordered folds into the NE
 5587 // case, matching the behaviour of AArch64 NE.
 5588 //
 5589 // This differs from x86 where the outputs of FP compares use a
 5590 // special FP flags registers and where compares based on this
 5591 // register are distinguished into ordered inequalities (cmpOpUCF) and
 5592 // EQ/NEQ tests (cmpOpUCF2). x86 has to special case the latter tests
 5593 // to explicitly handle the unordered case in branches. x86 also has
 5594 // to include extra CMoveX rules to accept a cmpOpUCF input.
 5595 
 5596 operand rFlagsReg()
 5597 %{
 5598   constraint(ALLOC_IN_RC(int_flags));
 5599   match(RegFlags);
 5600 
 5601   op_cost(0);
 5602   format %{ "RFLAGS" %}
 5603   interface(REG_INTER);
 5604 %}
 5605 
 5606 // Flags register, used as output of unsigned compare instructions
 5607 operand rFlagsRegU()
 5608 %{
 5609   constraint(ALLOC_IN_RC(int_flags));
 5610   match(RegFlags);
 5611 
 5612   op_cost(0);
 5613   format %{ "RFLAGSU" %}
 5614   interface(REG_INTER);
 5615 %}
 5616 
 5617 // Special Registers
 5618 
 5619 // Method Register
 5620 operand inline_cache_RegP(iRegP reg)
 5621 %{
 5622   constraint(ALLOC_IN_RC(method_reg)); // inline_cache_reg
 5623   match(reg);
 5624   match(iRegPNoSp);
 5625   op_cost(0);
 5626   format %{ %}
 5627   interface(REG_INTER);
 5628 %}
 5629 
 5630 // Thread Register
 5631 operand thread_RegP(iRegP reg)
 5632 %{
 5633   constraint(ALLOC_IN_RC(thread_reg)); // link_reg
 5634   match(reg);
 5635   op_cost(0);
 5636   format %{ %}
 5637   interface(REG_INTER);
 5638 %}
 5639 
 5640 operand lr_RegP(iRegP reg)
 5641 %{
 5642   constraint(ALLOC_IN_RC(lr_reg)); // link_reg
 5643   match(reg);
 5644   op_cost(0);
 5645   format %{ %}
 5646   interface(REG_INTER);
 5647 %}
 5648 
 5649 //----------Memory Operands----------------------------------------------------
 5650 
 5651 operand indirect(iRegP reg)
 5652 %{
 5653   constraint(ALLOC_IN_RC(ptr_reg));
 5654   match(reg);
 5655   op_cost(0);
 5656   format %{ "[$reg]" %}
 5657   interface(MEMORY_INTER) %{
 5658     base($reg);
 5659     index(0xffffffff);
 5660     scale(0x0);
 5661     disp(0x0);
 5662   %}
 5663 %}
 5664 
 5665 operand indIndexScaledI2L(iRegP reg, iRegI ireg, immIScale scale)
 5666 %{
 5667   constraint(ALLOC_IN_RC(ptr_reg));
 5668   predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 5669   match(AddP reg (LShiftL (ConvI2L ireg) scale));
 5670   op_cost(0);
 5671   format %{ "$reg, $ireg sxtw($scale), 0, I2L" %}
 5672   interface(MEMORY_INTER) %{
 5673     base($reg);
 5674     index($ireg);
 5675     scale($scale);
 5676     disp(0x0);
 5677   %}
 5678 %}
 5679 
 5680 operand indIndexScaled(iRegP reg, iRegL lreg, immIScale scale)
 5681 %{
 5682   constraint(ALLOC_IN_RC(ptr_reg));
 5683   predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 5684   match(AddP reg (LShiftL lreg scale));
 5685   op_cost(0);
 5686   format %{ "$reg, $lreg lsl($scale)" %}
 5687   interface(MEMORY_INTER) %{
 5688     base($reg);
 5689     index($lreg);
 5690     scale($scale);
 5691     disp(0x0);
 5692   %}
 5693 %}
 5694 
 5695 operand indIndexI2L(iRegP reg, iRegI ireg)
 5696 %{
 5697   constraint(ALLOC_IN_RC(ptr_reg));
 5698   match(AddP reg (ConvI2L ireg));
 5699   op_cost(0);
 5700   format %{ "$reg, $ireg, 0, I2L" %}
 5701   interface(MEMORY_INTER) %{
 5702     base($reg);
 5703     index($ireg);
 5704     scale(0x0);
 5705     disp(0x0);
 5706   %}
 5707 %}
 5708 
 5709 operand indIndex(iRegP reg, iRegL lreg)
 5710 %{
 5711   constraint(ALLOC_IN_RC(ptr_reg));
 5712   match(AddP reg lreg);
 5713   op_cost(0);
 5714   format %{ "$reg, $lreg" %}
 5715   interface(MEMORY_INTER) %{
 5716     base($reg);
 5717     index($lreg);
 5718     scale(0x0);
 5719     disp(0x0);
 5720   %}
 5721 %}
 5722 
 5723 operand indOffI(iRegP reg, immIOffset off)
 5724 %{
 5725   constraint(ALLOC_IN_RC(ptr_reg));
 5726   match(AddP reg off);
 5727   op_cost(0);
 5728   format %{ "[$reg, $off]" %}
 5729   interface(MEMORY_INTER) %{
 5730     base($reg);
 5731     index(0xffffffff);
 5732     scale(0x0);
 5733     disp($off);
 5734   %}
 5735 %}
 5736 
 5737 operand indOffI1(iRegP reg, immIOffset1 off)
 5738 %{
 5739   constraint(ALLOC_IN_RC(ptr_reg));
 5740   match(AddP reg off);
 5741   op_cost(0);
 5742   format %{ "[$reg, $off]" %}
 5743   interface(MEMORY_INTER) %{
 5744     base($reg);
 5745     index(0xffffffff);
 5746     scale(0x0);
 5747     disp($off);
 5748   %}
 5749 %}
 5750 
 5751 operand indOffI2(iRegP reg, immIOffset2 off)
 5752 %{
 5753   constraint(ALLOC_IN_RC(ptr_reg));
 5754   match(AddP reg off);
 5755   op_cost(0);
 5756   format %{ "[$reg, $off]" %}
 5757   interface(MEMORY_INTER) %{
 5758     base($reg);
 5759     index(0xffffffff);
 5760     scale(0x0);
 5761     disp($off);
 5762   %}
 5763 %}
 5764 
 5765 operand indOffI4(iRegP reg, immIOffset4 off)
 5766 %{
 5767   constraint(ALLOC_IN_RC(ptr_reg));
 5768   match(AddP reg off);
 5769   op_cost(0);
 5770   format %{ "[$reg, $off]" %}
 5771   interface(MEMORY_INTER) %{
 5772     base($reg);
 5773     index(0xffffffff);
 5774     scale(0x0);
 5775     disp($off);
 5776   %}
 5777 %}
 5778 
 5779 operand indOffI8(iRegP reg, immIOffset8 off)
 5780 %{
 5781   constraint(ALLOC_IN_RC(ptr_reg));
 5782   match(AddP reg off);
 5783   op_cost(0);
 5784   format %{ "[$reg, $off]" %}
 5785   interface(MEMORY_INTER) %{
 5786     base($reg);
 5787     index(0xffffffff);
 5788     scale(0x0);
 5789     disp($off);
 5790   %}
 5791 %}
 5792 
 5793 operand indOffI16(iRegP reg, immIOffset16 off)
 5794 %{
 5795   constraint(ALLOC_IN_RC(ptr_reg));
 5796   match(AddP reg off);
 5797   op_cost(0);
 5798   format %{ "[$reg, $off]" %}
 5799   interface(MEMORY_INTER) %{
 5800     base($reg);
 5801     index(0xffffffff);
 5802     scale(0x0);
 5803     disp($off);
 5804   %}
 5805 %}
 5806 
 5807 operand indOffL(iRegP reg, immLoffset off)
 5808 %{
 5809   constraint(ALLOC_IN_RC(ptr_reg));
 5810   match(AddP reg off);
 5811   op_cost(0);
 5812   format %{ "[$reg, $off]" %}
 5813   interface(MEMORY_INTER) %{
 5814     base($reg);
 5815     index(0xffffffff);
 5816     scale(0x0);
 5817     disp($off);
 5818   %}
 5819 %}
 5820 
 5821 operand indOffL1(iRegP reg, immLoffset1 off)
 5822 %{
 5823   constraint(ALLOC_IN_RC(ptr_reg));
 5824   match(AddP reg off);
 5825   op_cost(0);
 5826   format %{ "[$reg, $off]" %}
 5827   interface(MEMORY_INTER) %{
 5828     base($reg);
 5829     index(0xffffffff);
 5830     scale(0x0);
 5831     disp($off);
 5832   %}
 5833 %}
 5834 
 5835 operand indOffL2(iRegP reg, immLoffset2 off)
 5836 %{
 5837   constraint(ALLOC_IN_RC(ptr_reg));
 5838   match(AddP reg off);
 5839   op_cost(0);
 5840   format %{ "[$reg, $off]" %}
 5841   interface(MEMORY_INTER) %{
 5842     base($reg);
 5843     index(0xffffffff);
 5844     scale(0x0);
 5845     disp($off);
 5846   %}
 5847 %}
 5848 
 5849 operand indOffL4(iRegP reg, immLoffset4 off)
 5850 %{
 5851   constraint(ALLOC_IN_RC(ptr_reg));
 5852   match(AddP reg off);
 5853   op_cost(0);
 5854   format %{ "[$reg, $off]" %}
 5855   interface(MEMORY_INTER) %{
 5856     base($reg);
 5857     index(0xffffffff);
 5858     scale(0x0);
 5859     disp($off);
 5860   %}
 5861 %}
 5862 
 5863 operand indOffL8(iRegP reg, immLoffset8 off)
 5864 %{
 5865   constraint(ALLOC_IN_RC(ptr_reg));
 5866   match(AddP reg off);
 5867   op_cost(0);
 5868   format %{ "[$reg, $off]" %}
 5869   interface(MEMORY_INTER) %{
 5870     base($reg);
 5871     index(0xffffffff);
 5872     scale(0x0);
 5873     disp($off);
 5874   %}
 5875 %}
 5876 
 5877 operand indOffL16(iRegP reg, immLoffset16 off)
 5878 %{
 5879   constraint(ALLOC_IN_RC(ptr_reg));
 5880   match(AddP reg off);
 5881   op_cost(0);
 5882   format %{ "[$reg, $off]" %}
 5883   interface(MEMORY_INTER) %{
 5884     base($reg);
 5885     index(0xffffffff);
 5886     scale(0x0);
 5887     disp($off);
 5888   %}
 5889 %}
 5890 
 5891 operand indirectN(iRegN reg)
 5892 %{
 5893   predicate(CompressedOops::shift() == 0);
 5894   constraint(ALLOC_IN_RC(ptr_reg));
 5895   match(DecodeN reg);
 5896   op_cost(0);
 5897   format %{ "[$reg]\t# narrow" %}
 5898   interface(MEMORY_INTER) %{
 5899     base($reg);
 5900     index(0xffffffff);
 5901     scale(0x0);
 5902     disp(0x0);
 5903   %}
 5904 %}
 5905 
 5906 operand indIndexScaledI2LN(iRegN reg, iRegI ireg, immIScale scale)
 5907 %{
 5908   predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 5909   constraint(ALLOC_IN_RC(ptr_reg));
 5910   match(AddP (DecodeN reg) (LShiftL (ConvI2L ireg) scale));
 5911   op_cost(0);
 5912   format %{ "$reg, $ireg sxtw($scale), 0, I2L\t# narrow" %}
 5913   interface(MEMORY_INTER) %{
 5914     base($reg);
 5915     index($ireg);
 5916     scale($scale);
 5917     disp(0x0);
 5918   %}
 5919 %}
 5920 
 5921 operand indIndexScaledN(iRegN reg, iRegL lreg, immIScale scale)
 5922 %{
 5923   predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 5924   constraint(ALLOC_IN_RC(ptr_reg));
 5925   match(AddP (DecodeN reg) (LShiftL lreg scale));
 5926   op_cost(0);
 5927   format %{ "$reg, $lreg lsl($scale)\t# narrow" %}
 5928   interface(MEMORY_INTER) %{
 5929     base($reg);
 5930     index($lreg);
 5931     scale($scale);
 5932     disp(0x0);
 5933   %}
 5934 %}
 5935 
 5936 operand indIndexI2LN(iRegN reg, iRegI ireg)
 5937 %{
 5938   predicate(CompressedOops::shift() == 0);
 5939   constraint(ALLOC_IN_RC(ptr_reg));
 5940   match(AddP (DecodeN reg) (ConvI2L ireg));
 5941   op_cost(0);
 5942   format %{ "$reg, $ireg, 0, I2L\t# narrow" %}
 5943   interface(MEMORY_INTER) %{
 5944     base($reg);
 5945     index($ireg);
 5946     scale(0x0);
 5947     disp(0x0);
 5948   %}
 5949 %}
 5950 
 5951 operand indIndexN(iRegN reg, iRegL lreg)
 5952 %{
 5953   predicate(CompressedOops::shift() == 0);
 5954   constraint(ALLOC_IN_RC(ptr_reg));
 5955   match(AddP (DecodeN reg) lreg);
 5956   op_cost(0);
 5957   format %{ "$reg, $lreg\t# narrow" %}
 5958   interface(MEMORY_INTER) %{
 5959     base($reg);
 5960     index($lreg);
 5961     scale(0x0);
 5962     disp(0x0);
 5963   %}
 5964 %}
 5965 
 5966 operand indOffIN(iRegN reg, immIOffset off)
 5967 %{
 5968   predicate(CompressedOops::shift() == 0);
 5969   constraint(ALLOC_IN_RC(ptr_reg));
 5970   match(AddP (DecodeN reg) off);
 5971   op_cost(0);
 5972   format %{ "[$reg, $off]\t# narrow" %}
 5973   interface(MEMORY_INTER) %{
 5974     base($reg);
 5975     index(0xffffffff);
 5976     scale(0x0);
 5977     disp($off);
 5978   %}
 5979 %}
 5980 
 5981 operand indOffLN(iRegN reg, immLoffset off)
 5982 %{
 5983   predicate(CompressedOops::shift() == 0);
 5984   constraint(ALLOC_IN_RC(ptr_reg));
 5985   match(AddP (DecodeN reg) off);
 5986   op_cost(0);
 5987   format %{ "[$reg, $off]\t# narrow" %}
 5988   interface(MEMORY_INTER) %{
 5989     base($reg);
 5990     index(0xffffffff);
 5991     scale(0x0);
 5992     disp($off);
 5993   %}
 5994 %}
 5995 
 5996 
 5997 
 5998 // AArch64 opto stubs need to write to the pc slot in the thread anchor
 5999 operand thread_anchor_pc(thread_RegP reg, immL_pc_off off)
 6000 %{
 6001   constraint(ALLOC_IN_RC(ptr_reg));
 6002   match(AddP reg off);
 6003   op_cost(0);
 6004   format %{ "[$reg, $off]" %}
 6005   interface(MEMORY_INTER) %{
 6006     base($reg);
 6007     index(0xffffffff);
 6008     scale(0x0);
 6009     disp($off);
 6010   %}
 6011 %}
 6012 
 6013 //----------Special Memory Operands--------------------------------------------
 6014 // Stack Slot Operand - This operand is used for loading and storing temporary
 6015 //                      values on the stack where a match requires a value to
 6016 //                      flow through memory.
 6017 operand stackSlotP(sRegP reg)
 6018 %{
 6019   constraint(ALLOC_IN_RC(stack_slots));
 6020   op_cost(100);
 6021   // No match rule because this operand is only generated in matching
 6022   // match(RegP);
 6023   format %{ "[$reg]" %}
 6024   interface(MEMORY_INTER) %{
 6025     base(0x1e);  // RSP
 6026     index(0x0);  // No Index
 6027     scale(0x0);  // No Scale
 6028     disp($reg);  // Stack Offset
 6029   %}
 6030 %}
 6031 
 6032 operand stackSlotI(sRegI reg)
 6033 %{
 6034   constraint(ALLOC_IN_RC(stack_slots));
 6035   // No match rule because this operand is only generated in matching
 6036   // match(RegI);
 6037   format %{ "[$reg]" %}
 6038   interface(MEMORY_INTER) %{
 6039     base(0x1e);  // RSP
 6040     index(0x0);  // No Index
 6041     scale(0x0);  // No Scale
 6042     disp($reg);  // Stack Offset
 6043   %}
 6044 %}
 6045 
 6046 operand stackSlotF(sRegF reg)
 6047 %{
 6048   constraint(ALLOC_IN_RC(stack_slots));
 6049   // No match rule because this operand is only generated in matching
 6050   // match(RegF);
 6051   format %{ "[$reg]" %}
 6052   interface(MEMORY_INTER) %{
 6053     base(0x1e);  // RSP
 6054     index(0x0);  // No Index
 6055     scale(0x0);  // No Scale
 6056     disp($reg);  // Stack Offset
 6057   %}
 6058 %}
 6059 
 6060 operand stackSlotD(sRegD reg)
 6061 %{
 6062   constraint(ALLOC_IN_RC(stack_slots));
 6063   // No match rule because this operand is only generated in matching
 6064   // match(RegD);
 6065   format %{ "[$reg]" %}
 6066   interface(MEMORY_INTER) %{
 6067     base(0x1e);  // RSP
 6068     index(0x0);  // No Index
 6069     scale(0x0);  // No Scale
 6070     disp($reg);  // Stack Offset
 6071   %}
 6072 %}
 6073 
 6074 operand stackSlotL(sRegL reg)
 6075 %{
 6076   constraint(ALLOC_IN_RC(stack_slots));
 6077   // No match rule because this operand is only generated in matching
 6078   // match(RegL);
 6079   format %{ "[$reg]" %}
 6080   interface(MEMORY_INTER) %{
 6081     base(0x1e);  // RSP
 6082     index(0x0);  // No Index
 6083     scale(0x0);  // No Scale
 6084     disp($reg);  // Stack Offset
 6085   %}
 6086 %}
 6087 
 6088 // Operands for expressing Control Flow
 6089 // NOTE: Label is a predefined operand which should not be redefined in
 6090 //       the AD file. It is generically handled within the ADLC.
 6091 
 6092 //----------Conditional Branch Operands----------------------------------------
 6093 // Comparison Op  - This is the operation of the comparison, and is limited to
 6094 //                  the following set of codes:
 6095 //                  L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
 6096 //
 6097 // Other attributes of the comparison, such as unsignedness, are specified
 6098 // by the comparison instruction that sets a condition code flags register.
 6099 // That result is represented by a flags operand whose subtype is appropriate
 6100 // to the unsignedness (etc.) of the comparison.
 6101 //
 6102 // Later, the instruction which matches both the Comparison Op (a Bool) and
 6103 // the flags (produced by the Cmp) specifies the coding of the comparison op
 6104 // by matching a specific subtype of Bool operand below, such as cmpOpU.
 6105 
 6106 // used for signed integral comparisons and fp comparisons
 6107 
 6108 operand cmpOp()
 6109 %{
 6110   match(Bool);
 6111 
 6112   format %{ "" %}
 6113   interface(COND_INTER) %{
 6114     equal(0x0, "eq");
 6115     not_equal(0x1, "ne");
 6116     less(0xb, "lt");
 6117     greater_equal(0xa, "ge");
 6118     less_equal(0xd, "le");
 6119     greater(0xc, "gt");
 6120     overflow(0x6, "vs");
 6121     no_overflow(0x7, "vc");
 6122   %}
 6123 %}
 6124 
 6125 // used for unsigned integral comparisons
 6126 
 6127 operand cmpOpU()
 6128 %{
 6129   match(Bool);
 6130 
 6131   format %{ "" %}
 6132   interface(COND_INTER) %{
 6133     equal(0x0, "eq");
 6134     not_equal(0x1, "ne");
 6135     less(0x3, "lo");
 6136     greater_equal(0x2, "hs");
 6137     less_equal(0x9, "ls");
 6138     greater(0x8, "hi");
 6139     overflow(0x6, "vs");
 6140     no_overflow(0x7, "vc");
 6141   %}
 6142 %}
 6143 
 6144 // used for certain integral comparisons which can be
 6145 // converted to cbxx or tbxx instructions
 6146 
 6147 operand cmpOpEqNe()
 6148 %{
 6149   match(Bool);
 6150   op_cost(0);
 6151   predicate(n->as_Bool()->_test._test == BoolTest::ne
 6152             || n->as_Bool()->_test._test == BoolTest::eq);
 6153 
 6154   format %{ "" %}
 6155   interface(COND_INTER) %{
 6156     equal(0x0, "eq");
 6157     not_equal(0x1, "ne");
 6158     less(0xb, "lt");
 6159     greater_equal(0xa, "ge");
 6160     less_equal(0xd, "le");
 6161     greater(0xc, "gt");
 6162     overflow(0x6, "vs");
 6163     no_overflow(0x7, "vc");
 6164   %}
 6165 %}
 6166 
 6167 // used for certain integral comparisons which can be
 6168 // converted to cbxx or tbxx instructions
 6169 
 6170 operand cmpOpLtGe()
 6171 %{
 6172   match(Bool);
 6173   op_cost(0);
 6174 
 6175   predicate(n->as_Bool()->_test._test == BoolTest::lt
 6176             || n->as_Bool()->_test._test == BoolTest::ge);
 6177 
 6178   format %{ "" %}
 6179   interface(COND_INTER) %{
 6180     equal(0x0, "eq");
 6181     not_equal(0x1, "ne");
 6182     less(0xb, "lt");
 6183     greater_equal(0xa, "ge");
 6184     less_equal(0xd, "le");
 6185     greater(0xc, "gt");
 6186     overflow(0x6, "vs");
 6187     no_overflow(0x7, "vc");
 6188   %}
 6189 %}
 6190 
 6191 // used for certain unsigned integral comparisons which can be
 6192 // converted to cbxx or tbxx instructions
 6193 
 6194 operand cmpOpUEqNeLtGe()
 6195 %{
 6196   match(Bool);
 6197   op_cost(0);
 6198 
 6199   predicate(n->as_Bool()->_test._test == BoolTest::eq
 6200             || n->as_Bool()->_test._test == BoolTest::ne
 6201             || n->as_Bool()->_test._test == BoolTest::lt
 6202             || n->as_Bool()->_test._test == BoolTest::ge);
 6203 
 6204   format %{ "" %}
 6205   interface(COND_INTER) %{
 6206     equal(0x0, "eq");
 6207     not_equal(0x1, "ne");
 6208     less(0xb, "lt");
 6209     greater_equal(0xa, "ge");
 6210     less_equal(0xd, "le");
 6211     greater(0xc, "gt");
 6212     overflow(0x6, "vs");
 6213     no_overflow(0x7, "vc");
 6214   %}
 6215 %}
 6216 
 6217 // Special operand allowing long args to int ops to be truncated for free
 6218 
 6219 operand iRegL2I(iRegL reg) %{
 6220 
 6221   op_cost(0);
 6222 
 6223   match(ConvL2I reg);
 6224 
 6225   format %{ "l2i($reg)" %}
 6226 
 6227   interface(REG_INTER)
 6228 %}
 6229 
 6230 opclass vmem2(indirect, indIndex, indOffI2, indOffL2);
 6231 opclass vmem4(indirect, indIndex, indOffI4, indOffL4);
 6232 opclass vmem8(indirect, indIndex, indOffI8, indOffL8);
 6233 opclass vmem16(indirect, indIndex, indOffI16, indOffL16);
 6234 
 6235 //----------OPERAND CLASSES----------------------------------------------------
 6236 // Operand Classes are groups of operands that are used as to simplify
 6237 // instruction definitions by not requiring the AD writer to specify
 6238 // separate instructions for every form of operand when the
 6239 // instruction accepts multiple operand types with the same basic
 6240 // encoding and format. The classic case of this is memory operands.
 6241 
 6242 // memory is used to define read/write location for load/store
 6243 // instruction defs. we can turn a memory op into an Address
 6244 
 6245 opclass memory1(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI1, indOffL1,
 6246                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN);
 6247 
 6248 opclass memory2(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI2, indOffL2,
 6249                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN);
 6250 
 6251 opclass memory4(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI4, indOffL4,
 6252                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
 6253 
 6254 opclass memory8(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI8, indOffL8,
 6255                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
 6256 
 6257 // All of the memory operands. For the pipeline description.
 6258 opclass memory(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex,
 6259                indOffI1, indOffL1, indOffI2, indOffL2, indOffI4, indOffL4, indOffI8, indOffL8,
 6260                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
 6261 
 6262 
 6263 // iRegIorL2I is used for src inputs in rules for 32 bit int (I)
 6264 // operations. it allows the src to be either an iRegI or a (ConvL2I
 6265 // iRegL). in the latter case the l2i normally planted for a ConvL2I
 6266 // can be elided because the 32-bit instruction will just employ the
 6267 // lower 32 bits anyway.
 6268 //
 6269 // n.b. this does not elide all L2I conversions. if the truncated
 6270 // value is consumed by more than one operation then the ConvL2I
 6271 // cannot be bundled into the consuming nodes so an l2i gets planted
 6272 // (actually a movw $dst $src) and the downstream instructions consume
 6273 // the result of the l2i as an iRegI input. That's a shame since the
 6274 // movw is actually redundant but its not too costly.
 6275 
 6276 opclass iRegIorL2I(iRegI, iRegL2I);
 6277 
 6278 //----------PIPELINE-----------------------------------------------------------
 6279 // Rules which define the behavior of the target architectures pipeline.
 6280 
 6281 // For specific pipelines, eg A53, define the stages of that pipeline
 6282 //pipe_desc(ISS, EX1, EX2, WR);
 6283 #define ISS S0
 6284 #define EX1 S1
 6285 #define EX2 S2
 6286 #define WR  S3
 6287 
 6288 // Integer ALU reg operation
 6289 pipeline %{
 6290 
 6291 attributes %{
 6292   // ARM instructions are of fixed length
 6293   fixed_size_instructions;        // Fixed size instructions TODO does
 6294   max_instructions_per_bundle = 4;   // A53 = 2, A57 = 4
 6295   // ARM instructions come in 32-bit word units
 6296   instruction_unit_size = 4;         // An instruction is 4 bytes long
 6297   instruction_fetch_unit_size = 64;  // The processor fetches one line
 6298   instruction_fetch_units = 1;       // of 64 bytes
 6299 
 6300   // List of nop instructions
 6301   nops( MachNop );
 6302 %}
 6303 
 6304 // We don't use an actual pipeline model so don't care about resources
 6305 // or description. we do use pipeline classes to introduce fixed
 6306 // latencies
 6307 
 6308 //----------RESOURCES----------------------------------------------------------
 6309 // Resources are the functional units available to the machine
 6310 
 6311 resources( INS0, INS1, INS01 = INS0 | INS1,
 6312            ALU0, ALU1, ALU = ALU0 | ALU1,
 6313            MAC,
 6314            DIV,
 6315            BRANCH,
 6316            LDST,
 6317            NEON_FP);
 6318 
 6319 //----------PIPELINE DESCRIPTION-----------------------------------------------
 6320 // Pipeline Description specifies the stages in the machine's pipeline
 6321 
 6322 // Define the pipeline as a generic 6 stage pipeline
 6323 pipe_desc(S0, S1, S2, S3, S4, S5);
 6324 
 6325 //----------PIPELINE CLASSES---------------------------------------------------
 6326 // Pipeline Classes describe the stages in which input and output are
 6327 // referenced by the hardware pipeline.
 6328 
 6329 pipe_class fp_dop_reg_reg_s(vRegF dst, vRegF src1, vRegF src2)
 6330 %{
 6331   single_instruction;
 6332   src1   : S1(read);
 6333   src2   : S2(read);
 6334   dst    : S5(write);
 6335   INS01  : ISS;
 6336   NEON_FP : S5;
 6337 %}
 6338 
 6339 pipe_class fp_dop_reg_reg_d(vRegD dst, vRegD src1, vRegD src2)
 6340 %{
 6341   single_instruction;
 6342   src1   : S1(read);
 6343   src2   : S2(read);
 6344   dst    : S5(write);
 6345   INS01  : ISS;
 6346   NEON_FP : S5;
 6347 %}
 6348 
 6349 pipe_class fp_uop_s(vRegF dst, vRegF src)
 6350 %{
 6351   single_instruction;
 6352   src    : S1(read);
 6353   dst    : S5(write);
 6354   INS01  : ISS;
 6355   NEON_FP : S5;
 6356 %}
 6357 
 6358 pipe_class fp_uop_d(vRegD dst, vRegD src)
 6359 %{
 6360   single_instruction;
 6361   src    : S1(read);
 6362   dst    : S5(write);
 6363   INS01  : ISS;
 6364   NEON_FP : S5;
 6365 %}
 6366 
 6367 pipe_class fp_d2f(vRegF dst, vRegD src)
 6368 %{
 6369   single_instruction;
 6370   src    : S1(read);
 6371   dst    : S5(write);
 6372   INS01  : ISS;
 6373   NEON_FP : S5;
 6374 %}
 6375 
 6376 pipe_class fp_f2d(vRegD dst, vRegF src)
 6377 %{
 6378   single_instruction;
 6379   src    : S1(read);
 6380   dst    : S5(write);
 6381   INS01  : ISS;
 6382   NEON_FP : S5;
 6383 %}
 6384 
 6385 pipe_class fp_f2i(iRegINoSp dst, vRegF src)
 6386 %{
 6387   single_instruction;
 6388   src    : S1(read);
 6389   dst    : S5(write);
 6390   INS01  : ISS;
 6391   NEON_FP : S5;
 6392 %}
 6393 
 6394 pipe_class fp_f2l(iRegLNoSp dst, vRegF src)
 6395 %{
 6396   single_instruction;
 6397   src    : S1(read);
 6398   dst    : S5(write);
 6399   INS01  : ISS;
 6400   NEON_FP : S5;
 6401 %}
 6402 
 6403 pipe_class fp_i2f(vRegF dst, iRegIorL2I src)
 6404 %{
 6405   single_instruction;
 6406   src    : S1(read);
 6407   dst    : S5(write);
 6408   INS01  : ISS;
 6409   NEON_FP : S5;
 6410 %}
 6411 
 6412 pipe_class fp_l2f(vRegF dst, iRegL src)
 6413 %{
 6414   single_instruction;
 6415   src    : S1(read);
 6416   dst    : S5(write);
 6417   INS01  : ISS;
 6418   NEON_FP : S5;
 6419 %}
 6420 
 6421 pipe_class fp_d2i(iRegINoSp dst, vRegD src)
 6422 %{
 6423   single_instruction;
 6424   src    : S1(read);
 6425   dst    : S5(write);
 6426   INS01  : ISS;
 6427   NEON_FP : S5;
 6428 %}
 6429 
 6430 pipe_class fp_d2l(iRegLNoSp dst, vRegD src)
 6431 %{
 6432   single_instruction;
 6433   src    : S1(read);
 6434   dst    : S5(write);
 6435   INS01  : ISS;
 6436   NEON_FP : S5;
 6437 %}
 6438 
 6439 pipe_class fp_i2d(vRegD dst, iRegIorL2I src)
 6440 %{
 6441   single_instruction;
 6442   src    : S1(read);
 6443   dst    : S5(write);
 6444   INS01  : ISS;
 6445   NEON_FP : S5;
 6446 %}
 6447 
 6448 pipe_class fp_l2d(vRegD dst, iRegIorL2I src)
 6449 %{
 6450   single_instruction;
 6451   src    : S1(read);
 6452   dst    : S5(write);
 6453   INS01  : ISS;
 6454   NEON_FP : S5;
 6455 %}
 6456 
 6457 pipe_class fp_div_s(vRegF dst, vRegF src1, vRegF src2)
 6458 %{
 6459   single_instruction;
 6460   src1   : S1(read);
 6461   src2   : S2(read);
 6462   dst    : S5(write);
 6463   INS0   : ISS;
 6464   NEON_FP : S5;
 6465 %}
 6466 
 6467 pipe_class fp_div_d(vRegD dst, vRegD src1, vRegD src2)
 6468 %{
 6469   single_instruction;
 6470   src1   : S1(read);
 6471   src2   : S2(read);
 6472   dst    : S5(write);
 6473   INS0   : ISS;
 6474   NEON_FP : S5;
 6475 %}
 6476 
 6477 pipe_class fp_cond_reg_reg_s(vRegF dst, vRegF src1, vRegF src2, rFlagsReg cr)
 6478 %{
 6479   single_instruction;
 6480   cr     : S1(read);
 6481   src1   : S1(read);
 6482   src2   : S1(read);
 6483   dst    : S3(write);
 6484   INS01  : ISS;
 6485   NEON_FP : S3;
 6486 %}
 6487 
 6488 pipe_class fp_cond_reg_reg_d(vRegD dst, vRegD src1, vRegD src2, rFlagsReg cr)
 6489 %{
 6490   single_instruction;
 6491   cr     : S1(read);
 6492   src1   : S1(read);
 6493   src2   : S1(read);
 6494   dst    : S3(write);
 6495   INS01  : ISS;
 6496   NEON_FP : S3;
 6497 %}
 6498 
 6499 pipe_class fp_imm_s(vRegF dst)
 6500 %{
 6501   single_instruction;
 6502   dst    : S3(write);
 6503   INS01  : ISS;
 6504   NEON_FP : S3;
 6505 %}
 6506 
 6507 pipe_class fp_imm_d(vRegD dst)
 6508 %{
 6509   single_instruction;
 6510   dst    : S3(write);
 6511   INS01  : ISS;
 6512   NEON_FP : S3;
 6513 %}
 6514 
 6515 pipe_class fp_load_constant_s(vRegF dst)
 6516 %{
 6517   single_instruction;
 6518   dst    : S4(write);
 6519   INS01  : ISS;
 6520   NEON_FP : S4;
 6521 %}
 6522 
 6523 pipe_class fp_load_constant_d(vRegD dst)
 6524 %{
 6525   single_instruction;
 6526   dst    : S4(write);
 6527   INS01  : ISS;
 6528   NEON_FP : S4;
 6529 %}
 6530 
 6531 //------- Integer ALU operations --------------------------
 6532 
 6533 // Integer ALU reg-reg operation
 6534 // Operands needed in EX1, result generated in EX2
 6535 // Eg.  ADD     x0, x1, x2
 6536 pipe_class ialu_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 6537 %{
 6538   single_instruction;
 6539   dst    : EX2(write);
 6540   src1   : EX1(read);
 6541   src2   : EX1(read);
 6542   INS01  : ISS; // Dual issue as instruction 0 or 1
 6543   ALU    : EX2;
 6544 %}
 6545 
 6546 // Integer ALU reg-reg operation with constant shift
 6547 // Shifted register must be available in LATE_ISS instead of EX1
 6548 // Eg.  ADD     x0, x1, x2, LSL #2
 6549 pipe_class ialu_reg_reg_shift(iRegI dst, iRegI src1, iRegI src2, immI shift)
 6550 %{
 6551   single_instruction;
 6552   dst    : EX2(write);
 6553   src1   : EX1(read);
 6554   src2   : ISS(read);
 6555   INS01  : ISS;
 6556   ALU    : EX2;
 6557 %}
 6558 
 6559 // Integer ALU reg operation with constant shift
 6560 // Eg.  LSL     x0, x1, #shift
 6561 pipe_class ialu_reg_shift(iRegI dst, iRegI src1)
 6562 %{
 6563   single_instruction;
 6564   dst    : EX2(write);
 6565   src1   : ISS(read);
 6566   INS01  : ISS;
 6567   ALU    : EX2;
 6568 %}
 6569 
 6570 // Integer ALU reg-reg operation with variable shift
 6571 // Both operands must be available in LATE_ISS instead of EX1
 6572 // Result is available in EX1 instead of EX2
 6573 // Eg.  LSLV    x0, x1, x2
 6574 pipe_class ialu_reg_reg_vshift(iRegI dst, iRegI src1, iRegI src2)
 6575 %{
 6576   single_instruction;
 6577   dst    : EX1(write);
 6578   src1   : ISS(read);
 6579   src2   : ISS(read);
 6580   INS01  : ISS;
 6581   ALU    : EX1;
 6582 %}
 6583 
 6584 // Integer ALU reg-reg operation with extract
 6585 // As for _vshift above, but result generated in EX2
 6586 // Eg.  EXTR    x0, x1, x2, #N
 6587 pipe_class ialu_reg_reg_extr(iRegI dst, iRegI src1, iRegI src2)
 6588 %{
 6589   single_instruction;
 6590   dst    : EX2(write);
 6591   src1   : ISS(read);
 6592   src2   : ISS(read);
 6593   INS1   : ISS; // Can only dual issue as Instruction 1
 6594   ALU    : EX1;
 6595 %}
 6596 
 6597 // Integer ALU reg operation
 6598 // Eg.  NEG     x0, x1
 6599 pipe_class ialu_reg(iRegI dst, iRegI src)
 6600 %{
 6601   single_instruction;
 6602   dst    : EX2(write);
 6603   src    : EX1(read);
 6604   INS01  : ISS;
 6605   ALU    : EX2;
 6606 %}
 6607 
 6608 // Integer ALU reg mmediate operation
 6609 // Eg.  ADD     x0, x1, #N
 6610 pipe_class ialu_reg_imm(iRegI dst, iRegI src1)
 6611 %{
 6612   single_instruction;
 6613   dst    : EX2(write);
 6614   src1   : EX1(read);
 6615   INS01  : ISS;
 6616   ALU    : EX2;
 6617 %}
 6618 
 6619 // Integer ALU immediate operation (no source operands)
 6620 // Eg.  MOV     x0, #N
 6621 pipe_class ialu_imm(iRegI dst)
 6622 %{
 6623   single_instruction;
 6624   dst    : EX1(write);
 6625   INS01  : ISS;
 6626   ALU    : EX1;
 6627 %}
 6628 
 6629 //------- Compare operation -------------------------------
 6630 
 6631 // Compare reg-reg
 6632 // Eg.  CMP     x0, x1
 6633 pipe_class icmp_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
 6634 %{
 6635   single_instruction;
 6636 //  fixed_latency(16);
 6637   cr     : EX2(write);
 6638   op1    : EX1(read);
 6639   op2    : EX1(read);
 6640   INS01  : ISS;
 6641   ALU    : EX2;
 6642 %}
 6643 
 6644 // Compare reg-reg
 6645 // Eg.  CMP     x0, #N
 6646 pipe_class icmp_reg_imm(rFlagsReg cr, iRegI op1)
 6647 %{
 6648   single_instruction;
 6649 //  fixed_latency(16);
 6650   cr     : EX2(write);
 6651   op1    : EX1(read);
 6652   INS01  : ISS;
 6653   ALU    : EX2;
 6654 %}
 6655 
 6656 //------- Conditional instructions ------------------------
 6657 
 6658 // Conditional no operands
 6659 // Eg.  CSINC   x0, zr, zr, <cond>
 6660 pipe_class icond_none(iRegI dst, rFlagsReg cr)
 6661 %{
 6662   single_instruction;
 6663   cr     : EX1(read);
 6664   dst    : EX2(write);
 6665   INS01  : ISS;
 6666   ALU    : EX2;
 6667 %}
 6668 
 6669 // Conditional 2 operand
 6670 // EG.  CSEL    X0, X1, X2, <cond>
 6671 pipe_class icond_reg_reg(iRegI dst, iRegI src1, iRegI src2, rFlagsReg cr)
 6672 %{
 6673   single_instruction;
 6674   cr     : EX1(read);
 6675   src1   : EX1(read);
 6676   src2   : EX1(read);
 6677   dst    : EX2(write);
 6678   INS01  : ISS;
 6679   ALU    : EX2;
 6680 %}
 6681 
 6682 // Conditional 2 operand
 6683 // EG.  CSEL    X0, X1, X2, <cond>
 6684 pipe_class icond_reg(iRegI dst, iRegI src, rFlagsReg cr)
 6685 %{
 6686   single_instruction;
 6687   cr     : EX1(read);
 6688   src    : EX1(read);
 6689   dst    : EX2(write);
 6690   INS01  : ISS;
 6691   ALU    : EX2;
 6692 %}
 6693 
 6694 //------- Multiply pipeline operations --------------------
 6695 
 6696 // Multiply reg-reg
 6697 // Eg.  MUL     w0, w1, w2
 6698 pipe_class imul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 6699 %{
 6700   single_instruction;
 6701   dst    : WR(write);
 6702   src1   : ISS(read);
 6703   src2   : ISS(read);
 6704   INS01  : ISS;
 6705   MAC    : WR;
 6706 %}
 6707 
 6708 // Multiply accumulate
 6709 // Eg.  MADD    w0, w1, w2, w3
 6710 pipe_class imac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
 6711 %{
 6712   single_instruction;
 6713   dst    : WR(write);
 6714   src1   : ISS(read);
 6715   src2   : ISS(read);
 6716   src3   : ISS(read);
 6717   INS01  : ISS;
 6718   MAC    : WR;
 6719 %}
 6720 
 6721 // Eg.  MUL     w0, w1, w2
 6722 pipe_class lmul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 6723 %{
 6724   single_instruction;
 6725   fixed_latency(3); // Maximum latency for 64 bit mul
 6726   dst    : WR(write);
 6727   src1   : ISS(read);
 6728   src2   : ISS(read);
 6729   INS01  : ISS;
 6730   MAC    : WR;
 6731 %}
 6732 
 6733 // Multiply accumulate
 6734 // Eg.  MADD    w0, w1, w2, w3
 6735 pipe_class lmac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
 6736 %{
 6737   single_instruction;
 6738   fixed_latency(3); // Maximum latency for 64 bit mul
 6739   dst    : WR(write);
 6740   src1   : ISS(read);
 6741   src2   : ISS(read);
 6742   src3   : ISS(read);
 6743   INS01  : ISS;
 6744   MAC    : WR;
 6745 %}
 6746 
 6747 //------- Divide pipeline operations --------------------
 6748 
 6749 // Eg.  SDIV    w0, w1, w2
 6750 pipe_class idiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 6751 %{
 6752   single_instruction;
 6753   fixed_latency(8); // Maximum latency for 32 bit divide
 6754   dst    : WR(write);
 6755   src1   : ISS(read);
 6756   src2   : ISS(read);
 6757   INS0   : ISS; // Can only dual issue as instruction 0
 6758   DIV    : WR;
 6759 %}
 6760 
 6761 // Eg.  SDIV    x0, x1, x2
 6762 pipe_class ldiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 6763 %{
 6764   single_instruction;
 6765   fixed_latency(16); // Maximum latency for 64 bit divide
 6766   dst    : WR(write);
 6767   src1   : ISS(read);
 6768   src2   : ISS(read);
 6769   INS0   : ISS; // Can only dual issue as instruction 0
 6770   DIV    : WR;
 6771 %}
 6772 
 6773 //------- Load pipeline operations ------------------------
 6774 
 6775 // Load - prefetch
 6776 // Eg.  PFRM    <mem>
 6777 pipe_class iload_prefetch(memory mem)
 6778 %{
 6779   single_instruction;
 6780   mem    : ISS(read);
 6781   INS01  : ISS;
 6782   LDST   : WR;
 6783 %}
 6784 
 6785 // Load - reg, mem
 6786 // Eg.  LDR     x0, <mem>
 6787 pipe_class iload_reg_mem(iRegI dst, memory mem)
 6788 %{
 6789   single_instruction;
 6790   dst    : WR(write);
 6791   mem    : ISS(read);
 6792   INS01  : ISS;
 6793   LDST   : WR;
 6794 %}
 6795 
 6796 // Load - reg, reg
 6797 // Eg.  LDR     x0, [sp, x1]
 6798 pipe_class iload_reg_reg(iRegI dst, iRegI src)
 6799 %{
 6800   single_instruction;
 6801   dst    : WR(write);
 6802   src    : ISS(read);
 6803   INS01  : ISS;
 6804   LDST   : WR;
 6805 %}
 6806 
 6807 //------- Store pipeline operations -----------------------
 6808 
 6809 // Store - zr, mem
 6810 // Eg.  STR     zr, <mem>
 6811 pipe_class istore_mem(memory mem)
 6812 %{
 6813   single_instruction;
 6814   mem    : ISS(read);
 6815   INS01  : ISS;
 6816   LDST   : WR;
 6817 %}
 6818 
 6819 // Store - reg, mem
 6820 // Eg.  STR     x0, <mem>
 6821 pipe_class istore_reg_mem(iRegI src, memory mem)
 6822 %{
 6823   single_instruction;
 6824   mem    : ISS(read);
 6825   src    : EX2(read);
 6826   INS01  : ISS;
 6827   LDST   : WR;
 6828 %}
 6829 
 6830 // Store - reg, reg
 6831 // Eg. STR      x0, [sp, x1]
 6832 pipe_class istore_reg_reg(iRegI dst, iRegI src)
 6833 %{
 6834   single_instruction;
 6835   dst    : ISS(read);
 6836   src    : EX2(read);
 6837   INS01  : ISS;
 6838   LDST   : WR;
 6839 %}
 6840 
 6841 //------- Store pipeline operations -----------------------
 6842 
 6843 // Branch
 6844 pipe_class pipe_branch()
 6845 %{
 6846   single_instruction;
 6847   INS01  : ISS;
 6848   BRANCH : EX1;
 6849 %}
 6850 
 6851 // Conditional branch
 6852 pipe_class pipe_branch_cond(rFlagsReg cr)
 6853 %{
 6854   single_instruction;
 6855   cr     : EX1(read);
 6856   INS01  : ISS;
 6857   BRANCH : EX1;
 6858 %}
 6859 
 6860 // Compare & Branch
 6861 // EG.  CBZ/CBNZ
 6862 pipe_class pipe_cmp_branch(iRegI op1)
 6863 %{
 6864   single_instruction;
 6865   op1    : EX1(read);
 6866   INS01  : ISS;
 6867   BRANCH : EX1;
 6868 %}
 6869 
 6870 //------- Synchronisation operations ----------------------
 6871 
 6872 // Any operation requiring serialization.
 6873 // EG.  DMB/Atomic Ops/Load Acquire/Str Release
 6874 pipe_class pipe_serial()
 6875 %{
 6876   single_instruction;
 6877   force_serialization;
 6878   fixed_latency(16);
 6879   INS01  : ISS(2); // Cannot dual issue with any other instruction
 6880   LDST   : WR;
 6881 %}
 6882 
 6883 // Generic big/slow expanded idiom - also serialized
 6884 pipe_class pipe_slow()
 6885 %{
 6886   instruction_count(10);
 6887   multiple_bundles;
 6888   force_serialization;
 6889   fixed_latency(16);
 6890   INS01  : ISS(2); // Cannot dual issue with any other instruction
 6891   LDST   : WR;
 6892 %}
 6893 
 6894 // Empty pipeline class
 6895 pipe_class pipe_class_empty()
 6896 %{
 6897   single_instruction;
 6898   fixed_latency(0);
 6899 %}
 6900 
 6901 // Default pipeline class.
 6902 pipe_class pipe_class_default()
 6903 %{
 6904   single_instruction;
 6905   fixed_latency(2);
 6906 %}
 6907 
 6908 // Pipeline class for compares.
 6909 pipe_class pipe_class_compare()
 6910 %{
 6911   single_instruction;
 6912   fixed_latency(16);
 6913 %}
 6914 
 6915 // Pipeline class for memory operations.
 6916 pipe_class pipe_class_memory()
 6917 %{
 6918   single_instruction;
 6919   fixed_latency(16);
 6920 %}
 6921 
 6922 // Pipeline class for call.
 6923 pipe_class pipe_class_call()
 6924 %{
 6925   single_instruction;
 6926   fixed_latency(100);
 6927 %}
 6928 
 6929 // Define the class for the Nop node.
 6930 define %{
 6931    MachNop = pipe_class_empty;
 6932 %}
 6933 
 6934 %}
 6935 //----------INSTRUCTIONS-------------------------------------------------------
 6936 //
 6937 // match      -- States which machine-independent subtree may be replaced
 6938 //               by this instruction.
 6939 // ins_cost   -- The estimated cost of this instruction is used by instruction
 6940 //               selection to identify a minimum cost tree of machine
 6941 //               instructions that matches a tree of machine-independent
 6942 //               instructions.
 6943 // format     -- A string providing the disassembly for this instruction.
 6944 //               The value of an instruction's operand may be inserted
 6945 //               by referring to it with a '$' prefix.
 6946 // opcode     -- Three instruction opcodes may be provided.  These are referred
 6947 //               to within an encode class as $primary, $secondary, and $tertiary
 6948 //               rrspectively.  The primary opcode is commonly used to
 6949 //               indicate the type of machine instruction, while secondary
 6950 //               and tertiary are often used for prefix options or addressing
 6951 //               modes.
 6952 // ins_encode -- A list of encode classes with parameters. The encode class
 6953 //               name must have been defined in an 'enc_class' specification
 6954 //               in the encode section of the architecture description.
 6955 
 6956 // ============================================================================
 6957 // Memory (Load/Store) Instructions
 6958 
 6959 // Load Instructions
 6960 
 6961 // Load Byte (8 bit signed)
 6962 instruct loadB(iRegINoSp dst, memory1 mem)
 6963 %{
 6964   match(Set dst (LoadB mem));
 6965   predicate(!needs_acquiring_load(n));
 6966 
 6967   ins_cost(4 * INSN_COST);
 6968   format %{ "ldrsbw  $dst, $mem\t# byte" %}
 6969 
 6970   ins_encode(aarch64_enc_ldrsbw(dst, mem));
 6971 
 6972   ins_pipe(iload_reg_mem);
 6973 %}
 6974 
 6975 // Load Byte (8 bit signed) into long
 6976 instruct loadB2L(iRegLNoSp dst, memory1 mem)
 6977 %{
 6978   match(Set dst (ConvI2L (LoadB mem)));
 6979   predicate(!needs_acquiring_load(n->in(1)));
 6980 
 6981   ins_cost(4 * INSN_COST);
 6982   format %{ "ldrsb  $dst, $mem\t# byte" %}
 6983 
 6984   ins_encode(aarch64_enc_ldrsb(dst, mem));
 6985 
 6986   ins_pipe(iload_reg_mem);
 6987 %}
 6988 
 6989 // Load Byte (8 bit unsigned)
 6990 instruct loadUB(iRegINoSp dst, memory1 mem)
 6991 %{
 6992   match(Set dst (LoadUB mem));
 6993   predicate(!needs_acquiring_load(n));
 6994 
 6995   ins_cost(4 * INSN_COST);
 6996   format %{ "ldrbw  $dst, $mem\t# byte" %}
 6997 
 6998   ins_encode(aarch64_enc_ldrb(dst, mem));
 6999 
 7000   ins_pipe(iload_reg_mem);
 7001 %}
 7002 
 7003 // Load Byte (8 bit unsigned) into long
 7004 instruct loadUB2L(iRegLNoSp dst, memory1 mem)
 7005 %{
 7006   match(Set dst (ConvI2L (LoadUB mem)));
 7007   predicate(!needs_acquiring_load(n->in(1)));
 7008 
 7009   ins_cost(4 * INSN_COST);
 7010   format %{ "ldrb  $dst, $mem\t# byte" %}
 7011 
 7012   ins_encode(aarch64_enc_ldrb(dst, mem));
 7013 
 7014   ins_pipe(iload_reg_mem);
 7015 %}
 7016 
 7017 // Load Short (16 bit signed)
 7018 instruct loadS(iRegINoSp dst, memory2 mem)
 7019 %{
 7020   match(Set dst (LoadS mem));
 7021   predicate(!needs_acquiring_load(n));
 7022 
 7023   ins_cost(4 * INSN_COST);
 7024   format %{ "ldrshw  $dst, $mem\t# short" %}
 7025 
 7026   ins_encode(aarch64_enc_ldrshw(dst, mem));
 7027 
 7028   ins_pipe(iload_reg_mem);
 7029 %}
 7030 
 7031 // Load Short (16 bit signed) into long
 7032 instruct loadS2L(iRegLNoSp dst, memory2 mem)
 7033 %{
 7034   match(Set dst (ConvI2L (LoadS mem)));
 7035   predicate(!needs_acquiring_load(n->in(1)));
 7036 
 7037   ins_cost(4 * INSN_COST);
 7038   format %{ "ldrsh  $dst, $mem\t# short" %}
 7039 
 7040   ins_encode(aarch64_enc_ldrsh(dst, mem));
 7041 
 7042   ins_pipe(iload_reg_mem);
 7043 %}
 7044 
 7045 // Load Char (16 bit unsigned)
 7046 instruct loadUS(iRegINoSp dst, memory2 mem)
 7047 %{
 7048   match(Set dst (LoadUS mem));
 7049   predicate(!needs_acquiring_load(n));
 7050 
 7051   ins_cost(4 * INSN_COST);
 7052   format %{ "ldrh  $dst, $mem\t# short" %}
 7053 
 7054   ins_encode(aarch64_enc_ldrh(dst, mem));
 7055 
 7056   ins_pipe(iload_reg_mem);
 7057 %}
 7058 
 7059 // Load Short/Char (16 bit unsigned) into long
 7060 instruct loadUS2L(iRegLNoSp dst, memory2 mem)
 7061 %{
 7062   match(Set dst (ConvI2L (LoadUS mem)));
 7063   predicate(!needs_acquiring_load(n->in(1)));
 7064 
 7065   ins_cost(4 * INSN_COST);
 7066   format %{ "ldrh  $dst, $mem\t# short" %}
 7067 
 7068   ins_encode(aarch64_enc_ldrh(dst, mem));
 7069 
 7070   ins_pipe(iload_reg_mem);
 7071 %}
 7072 
 7073 // Load Integer (32 bit signed)
 7074 instruct loadI(iRegINoSp dst, memory4 mem)
 7075 %{
 7076   match(Set dst (LoadI mem));
 7077   predicate(!needs_acquiring_load(n));
 7078 
 7079   ins_cost(4 * INSN_COST);
 7080   format %{ "ldrw  $dst, $mem\t# int" %}
 7081 
 7082   ins_encode(aarch64_enc_ldrw(dst, mem));
 7083 
 7084   ins_pipe(iload_reg_mem);
 7085 %}
 7086 
 7087 // Load Integer (32 bit signed) into long
 7088 instruct loadI2L(iRegLNoSp dst, memory4 mem)
 7089 %{
 7090   match(Set dst (ConvI2L (LoadI mem)));
 7091   predicate(!needs_acquiring_load(n->in(1)));
 7092 
 7093   ins_cost(4 * INSN_COST);
 7094   format %{ "ldrsw  $dst, $mem\t# int" %}
 7095 
 7096   ins_encode(aarch64_enc_ldrsw(dst, mem));
 7097 
 7098   ins_pipe(iload_reg_mem);
 7099 %}
 7100 
 7101 // Load Integer (32 bit unsigned) into long
 7102 instruct loadUI2L(iRegLNoSp dst, memory4 mem, immL_32bits mask)
 7103 %{
 7104   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
 7105   predicate(!needs_acquiring_load(n->in(1)->in(1)->as_Load()));
 7106 
 7107   ins_cost(4 * INSN_COST);
 7108   format %{ "ldrw  $dst, $mem\t# int" %}
 7109 
 7110   ins_encode(aarch64_enc_ldrw(dst, mem));
 7111 
 7112   ins_pipe(iload_reg_mem);
 7113 %}
 7114 
 7115 // Load Long (64 bit signed)
 7116 instruct loadL(iRegLNoSp dst, memory8 mem)
 7117 %{
 7118   match(Set dst (LoadL mem));
 7119   predicate(!needs_acquiring_load(n));
 7120 
 7121   ins_cost(4 * INSN_COST);
 7122   format %{ "ldr  $dst, $mem\t# int" %}
 7123 
 7124   ins_encode(aarch64_enc_ldr(dst, mem));
 7125 
 7126   ins_pipe(iload_reg_mem);
 7127 %}
 7128 
 7129 // Load Range
 7130 instruct loadRange(iRegINoSp dst, memory4 mem)
 7131 %{
 7132   match(Set dst (LoadRange mem));
 7133 
 7134   ins_cost(4 * INSN_COST);
 7135   format %{ "ldrw  $dst, $mem\t# range" %}
 7136 
 7137   ins_encode(aarch64_enc_ldrw(dst, mem));
 7138 
 7139   ins_pipe(iload_reg_mem);
 7140 %}
 7141 
 7142 // Load Pointer
 7143 instruct loadP(iRegPNoSp dst, memory8 mem)
 7144 %{
 7145   match(Set dst (LoadP mem));
 7146   predicate(!needs_acquiring_load(n) && (n->as_Load()->barrier_data() == 0));
 7147 
 7148   ins_cost(4 * INSN_COST);
 7149   format %{ "ldr  $dst, $mem\t# ptr" %}
 7150 
 7151   ins_encode(aarch64_enc_ldr(dst, mem));
 7152 
 7153   ins_pipe(iload_reg_mem);
 7154 %}
 7155 
 7156 // Load Compressed Pointer
 7157 instruct loadN(iRegNNoSp dst, memory4 mem)
 7158 %{
 7159   match(Set dst (LoadN mem));
 7160   predicate(!needs_acquiring_load(n));
 7161 
 7162   ins_cost(4 * INSN_COST);
 7163   format %{ "ldrw  $dst, $mem\t# compressed ptr" %}
 7164 
 7165   ins_encode(aarch64_enc_ldrw(dst, mem));
 7166 
 7167   ins_pipe(iload_reg_mem);
 7168 %}
 7169 
 7170 // Load Klass Pointer
 7171 instruct loadKlass(iRegPNoSp dst, memory8 mem)
 7172 %{
 7173   match(Set dst (LoadKlass mem));
 7174   predicate(!needs_acquiring_load(n));
 7175 
 7176   ins_cost(4 * INSN_COST);
 7177   format %{ "ldr  $dst, $mem\t# class" %}
 7178 
 7179   ins_encode(aarch64_enc_ldr(dst, mem));
 7180 
 7181   ins_pipe(iload_reg_mem);
 7182 %}
 7183 
 7184 // Load Narrow Klass Pointer
 7185 instruct loadNKlass(iRegNNoSp dst, memory4 mem)
 7186 %{
 7187   match(Set dst (LoadNKlass mem));

 7188   predicate(!needs_acquiring_load(n));
 7189 
 7190   ins_cost(4 * INSN_COST);
 7191   format %{ "ldrw  $dst, $mem\t# compressed class ptr" %}
 7192 
 7193   ins_encode(aarch64_enc_ldrw(dst, mem));
 7194 
 7195   ins_pipe(iload_reg_mem);












 7196 %}
 7197 
 7198 // Load Float
 7199 instruct loadF(vRegF dst, memory4 mem)
 7200 %{
 7201   match(Set dst (LoadF mem));
 7202   predicate(!needs_acquiring_load(n));
 7203 
 7204   ins_cost(4 * INSN_COST);
 7205   format %{ "ldrs  $dst, $mem\t# float" %}
 7206 
 7207   ins_encode( aarch64_enc_ldrs(dst, mem) );
 7208 
 7209   ins_pipe(pipe_class_memory);
 7210 %}
 7211 
 7212 // Load Double
 7213 instruct loadD(vRegD dst, memory8 mem)
 7214 %{
 7215   match(Set dst (LoadD mem));
 7216   predicate(!needs_acquiring_load(n));
 7217 
 7218   ins_cost(4 * INSN_COST);
 7219   format %{ "ldrd  $dst, $mem\t# double" %}
 7220 
 7221   ins_encode( aarch64_enc_ldrd(dst, mem) );
 7222 
 7223   ins_pipe(pipe_class_memory);
 7224 %}
 7225 
 7226 
 7227 // Load Int Constant
 7228 instruct loadConI(iRegINoSp dst, immI src)
 7229 %{
 7230   match(Set dst src);
 7231 
 7232   ins_cost(INSN_COST);
 7233   format %{ "mov $dst, $src\t# int" %}
 7234 
 7235   ins_encode( aarch64_enc_movw_imm(dst, src) );
 7236 
 7237   ins_pipe(ialu_imm);
 7238 %}
 7239 
 7240 // Load Long Constant
 7241 instruct loadConL(iRegLNoSp dst, immL src)
 7242 %{
 7243   match(Set dst src);
 7244 
 7245   ins_cost(INSN_COST);
 7246   format %{ "mov $dst, $src\t# long" %}
 7247 
 7248   ins_encode( aarch64_enc_mov_imm(dst, src) );
 7249 
 7250   ins_pipe(ialu_imm);
 7251 %}
 7252 
 7253 // Load Pointer Constant
 7254 
 7255 instruct loadConP(iRegPNoSp dst, immP con)
 7256 %{
 7257   match(Set dst con);
 7258 
 7259   ins_cost(INSN_COST * 4);
 7260   format %{
 7261     "mov  $dst, $con\t# ptr\n\t"
 7262   %}
 7263 
 7264   ins_encode(aarch64_enc_mov_p(dst, con));
 7265 
 7266   ins_pipe(ialu_imm);
 7267 %}
 7268 
 7269 // Load Null Pointer Constant
 7270 
 7271 instruct loadConP0(iRegPNoSp dst, immP0 con)
 7272 %{
 7273   match(Set dst con);
 7274 
 7275   ins_cost(INSN_COST);
 7276   format %{ "mov  $dst, $con\t# NULL ptr" %}
 7277 
 7278   ins_encode(aarch64_enc_mov_p0(dst, con));
 7279 
 7280   ins_pipe(ialu_imm);
 7281 %}
 7282 
 7283 // Load Pointer Constant One
 7284 
 7285 instruct loadConP1(iRegPNoSp dst, immP_1 con)
 7286 %{
 7287   match(Set dst con);
 7288 
 7289   ins_cost(INSN_COST);
 7290   format %{ "mov  $dst, $con\t# NULL ptr" %}
 7291 
 7292   ins_encode(aarch64_enc_mov_p1(dst, con));
 7293 
 7294   ins_pipe(ialu_imm);
 7295 %}
 7296 
 7297 // Load Byte Map Base Constant
 7298 
 7299 instruct loadByteMapBase(iRegPNoSp dst, immByteMapBase con)
 7300 %{
 7301   match(Set dst con);
 7302 
 7303   ins_cost(INSN_COST);
 7304   format %{ "adr  $dst, $con\t# Byte Map Base" %}
 7305 
 7306   ins_encode(aarch64_enc_mov_byte_map_base(dst, con));
 7307 
 7308   ins_pipe(ialu_imm);
 7309 %}
 7310 
 7311 // Load Narrow Pointer Constant
 7312 
 7313 instruct loadConN(iRegNNoSp dst, immN con)
 7314 %{
 7315   match(Set dst con);
 7316 
 7317   ins_cost(INSN_COST * 4);
 7318   format %{ "mov  $dst, $con\t# compressed ptr" %}
 7319 
 7320   ins_encode(aarch64_enc_mov_n(dst, con));
 7321 
 7322   ins_pipe(ialu_imm);
 7323 %}
 7324 
 7325 // Load Narrow Null Pointer Constant
 7326 
 7327 instruct loadConN0(iRegNNoSp dst, immN0 con)
 7328 %{
 7329   match(Set dst con);
 7330 
 7331   ins_cost(INSN_COST);
 7332   format %{ "mov  $dst, $con\t# compressed NULL ptr" %}
 7333 
 7334   ins_encode(aarch64_enc_mov_n0(dst, con));
 7335 
 7336   ins_pipe(ialu_imm);
 7337 %}
 7338 
 7339 // Load Narrow Klass Constant
 7340 
 7341 instruct loadConNKlass(iRegNNoSp dst, immNKlass con)
 7342 %{
 7343   match(Set dst con);
 7344 
 7345   ins_cost(INSN_COST);
 7346   format %{ "mov  $dst, $con\t# compressed klass ptr" %}
 7347 
 7348   ins_encode(aarch64_enc_mov_nk(dst, con));
 7349 
 7350   ins_pipe(ialu_imm);
 7351 %}
 7352 
 7353 // Load Packed Float Constant
 7354 
 7355 instruct loadConF_packed(vRegF dst, immFPacked con) %{
 7356   match(Set dst con);
 7357   ins_cost(INSN_COST * 4);
 7358   format %{ "fmovs  $dst, $con"%}
 7359   ins_encode %{
 7360     __ fmovs(as_FloatRegister($dst$$reg), (double)$con$$constant);
 7361   %}
 7362 
 7363   ins_pipe(fp_imm_s);
 7364 %}
 7365 
 7366 // Load Float Constant
 7367 
 7368 instruct loadConF(vRegF dst, immF con) %{
 7369   match(Set dst con);
 7370 
 7371   ins_cost(INSN_COST * 4);
 7372 
 7373   format %{
 7374     "ldrs $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
 7375   %}
 7376 
 7377   ins_encode %{
 7378     __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
 7379   %}
 7380 
 7381   ins_pipe(fp_load_constant_s);
 7382 %}
 7383 
 7384 // Load Packed Double Constant
 7385 
 7386 instruct loadConD_packed(vRegD dst, immDPacked con) %{
 7387   match(Set dst con);
 7388   ins_cost(INSN_COST);
 7389   format %{ "fmovd  $dst, $con"%}
 7390   ins_encode %{
 7391     __ fmovd(as_FloatRegister($dst$$reg), $con$$constant);
 7392   %}
 7393 
 7394   ins_pipe(fp_imm_d);
 7395 %}
 7396 
 7397 // Load Double Constant
 7398 
 7399 instruct loadConD(vRegD dst, immD con) %{
 7400   match(Set dst con);
 7401 
 7402   ins_cost(INSN_COST * 5);
 7403   format %{
 7404     "ldrd $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
 7405   %}
 7406 
 7407   ins_encode %{
 7408     __ ldrd(as_FloatRegister($dst$$reg), $constantaddress($con));
 7409   %}
 7410 
 7411   ins_pipe(fp_load_constant_d);
 7412 %}
 7413 
 7414 // Store Instructions
 7415 
 7416 // Store CMS card-mark Immediate
 7417 instruct storeimmCM0(immI0 zero, memory1 mem)
 7418 %{
 7419   match(Set mem (StoreCM mem zero));
 7420 
 7421   ins_cost(INSN_COST);
 7422   format %{ "storestore (elided)\n\t"
 7423             "strb zr, $mem\t# byte" %}
 7424 
 7425   ins_encode(aarch64_enc_strb0(mem));
 7426 
 7427   ins_pipe(istore_mem);
 7428 %}
 7429 
 7430 // Store CMS card-mark Immediate with intervening StoreStore
 7431 // needed when using CMS with no conditional card marking
 7432 instruct storeimmCM0_ordered(immI0 zero, memory1 mem)
 7433 %{
 7434   match(Set mem (StoreCM mem zero));
 7435 
 7436   ins_cost(INSN_COST * 2);
 7437   format %{ "storestore\n\t"
 7438             "dmb ishst"
 7439             "\n\tstrb zr, $mem\t# byte" %}
 7440 
 7441   ins_encode(aarch64_enc_strb0_ordered(mem));
 7442 
 7443   ins_pipe(istore_mem);
 7444 %}
 7445 
 7446 // Store Byte
 7447 instruct storeB(iRegIorL2I src, memory1 mem)
 7448 %{
 7449   match(Set mem (StoreB mem src));
 7450   predicate(!needs_releasing_store(n));
 7451 
 7452   ins_cost(INSN_COST);
 7453   format %{ "strb  $src, $mem\t# byte" %}
 7454 
 7455   ins_encode(aarch64_enc_strb(src, mem));
 7456 
 7457   ins_pipe(istore_reg_mem);
 7458 %}
 7459 
 7460 
 7461 instruct storeimmB0(immI0 zero, memory1 mem)
 7462 %{
 7463   match(Set mem (StoreB mem zero));
 7464   predicate(!needs_releasing_store(n));
 7465 
 7466   ins_cost(INSN_COST);
 7467   format %{ "strb rscractch2, $mem\t# byte" %}
 7468 
 7469   ins_encode(aarch64_enc_strb0(mem));
 7470 
 7471   ins_pipe(istore_mem);
 7472 %}
 7473 
 7474 // Store Char/Short
 7475 instruct storeC(iRegIorL2I src, memory2 mem)
 7476 %{
 7477   match(Set mem (StoreC mem src));
 7478   predicate(!needs_releasing_store(n));
 7479 
 7480   ins_cost(INSN_COST);
 7481   format %{ "strh  $src, $mem\t# short" %}
 7482 
 7483   ins_encode(aarch64_enc_strh(src, mem));
 7484 
 7485   ins_pipe(istore_reg_mem);
 7486 %}
 7487 
 7488 instruct storeimmC0(immI0 zero, memory2 mem)
 7489 %{
 7490   match(Set mem (StoreC mem zero));
 7491   predicate(!needs_releasing_store(n));
 7492 
 7493   ins_cost(INSN_COST);
 7494   format %{ "strh  zr, $mem\t# short" %}
 7495 
 7496   ins_encode(aarch64_enc_strh0(mem));
 7497 
 7498   ins_pipe(istore_mem);
 7499 %}
 7500 
 7501 // Store Integer
 7502 
 7503 instruct storeI(iRegIorL2I src, memory4 mem)
 7504 %{
 7505   match(Set mem(StoreI mem src));
 7506   predicate(!needs_releasing_store(n));
 7507 
 7508   ins_cost(INSN_COST);
 7509   format %{ "strw  $src, $mem\t# int" %}
 7510 
 7511   ins_encode(aarch64_enc_strw(src, mem));
 7512 
 7513   ins_pipe(istore_reg_mem);
 7514 %}
 7515 
 7516 instruct storeimmI0(immI0 zero, memory4 mem)
 7517 %{
 7518   match(Set mem(StoreI mem zero));
 7519   predicate(!needs_releasing_store(n));
 7520 
 7521   ins_cost(INSN_COST);
 7522   format %{ "strw  zr, $mem\t# int" %}
 7523 
 7524   ins_encode(aarch64_enc_strw0(mem));
 7525 
 7526   ins_pipe(istore_mem);
 7527 %}
 7528 
 7529 // Store Long (64 bit signed)
 7530 instruct storeL(iRegL src, memory8 mem)
 7531 %{
 7532   match(Set mem (StoreL mem src));
 7533   predicate(!needs_releasing_store(n));
 7534 
 7535   ins_cost(INSN_COST);
 7536   format %{ "str  $src, $mem\t# int" %}
 7537 
 7538   ins_encode(aarch64_enc_str(src, mem));
 7539 
 7540   ins_pipe(istore_reg_mem);
 7541 %}
 7542 
 7543 // Store Long (64 bit signed)
 7544 instruct storeimmL0(immL0 zero, memory8 mem)
 7545 %{
 7546   match(Set mem (StoreL mem zero));
 7547   predicate(!needs_releasing_store(n));
 7548 
 7549   ins_cost(INSN_COST);
 7550   format %{ "str  zr, $mem\t# int" %}
 7551 
 7552   ins_encode(aarch64_enc_str0(mem));
 7553 
 7554   ins_pipe(istore_mem);
 7555 %}
 7556 
 7557 // Store Pointer
 7558 instruct storeP(iRegP src, memory8 mem)
 7559 %{
 7560   match(Set mem (StoreP mem src));
 7561   predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
 7562 
 7563   ins_cost(INSN_COST);
 7564   format %{ "str  $src, $mem\t# ptr" %}
 7565 
 7566   ins_encode(aarch64_enc_str(src, mem));
 7567 
 7568   ins_pipe(istore_reg_mem);
 7569 %}
 7570 
 7571 // Store Pointer
 7572 instruct storeimmP0(immP0 zero, memory8 mem)
 7573 %{
 7574   match(Set mem (StoreP mem zero));
 7575   predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
 7576 
 7577   ins_cost(INSN_COST);
 7578   format %{ "str zr, $mem\t# ptr" %}
 7579 
 7580   ins_encode(aarch64_enc_str0(mem));
 7581 
 7582   ins_pipe(istore_mem);
 7583 %}
 7584 
 7585 // Store Compressed Pointer
 7586 instruct storeN(iRegN src, memory4 mem)
 7587 %{
 7588   match(Set mem (StoreN mem src));
 7589   predicate(!needs_releasing_store(n));
 7590 
 7591   ins_cost(INSN_COST);
 7592   format %{ "strw  $src, $mem\t# compressed ptr" %}
 7593 
 7594   ins_encode(aarch64_enc_strw(src, mem));
 7595 
 7596   ins_pipe(istore_reg_mem);
 7597 %}
 7598 
 7599 instruct storeImmN0(immN0 zero, memory4 mem)
 7600 %{
 7601   match(Set mem (StoreN mem zero));
 7602   predicate(!needs_releasing_store(n));
 7603 
 7604   ins_cost(INSN_COST);
 7605   format %{ "strw  zr, $mem\t# compressed ptr" %}
 7606 
 7607   ins_encode(aarch64_enc_strw0(mem));
 7608 
 7609   ins_pipe(istore_mem);
 7610 %}
 7611 
 7612 // Store Float
 7613 instruct storeF(vRegF src, memory4 mem)
 7614 %{
 7615   match(Set mem (StoreF mem src));
 7616   predicate(!needs_releasing_store(n));
 7617 
 7618   ins_cost(INSN_COST);
 7619   format %{ "strs  $src, $mem\t# float" %}
 7620 
 7621   ins_encode( aarch64_enc_strs(src, mem) );
 7622 
 7623   ins_pipe(pipe_class_memory);
 7624 %}
 7625 
 7626 // TODO
 7627 // implement storeImmF0 and storeFImmPacked
 7628 
 7629 // Store Double
 7630 instruct storeD(vRegD src, memory8 mem)
 7631 %{
 7632   match(Set mem (StoreD mem src));
 7633   predicate(!needs_releasing_store(n));
 7634 
 7635   ins_cost(INSN_COST);
 7636   format %{ "strd  $src, $mem\t# double" %}
 7637 
 7638   ins_encode( aarch64_enc_strd(src, mem) );
 7639 
 7640   ins_pipe(pipe_class_memory);
 7641 %}
 7642 
 7643 // Store Compressed Klass Pointer
 7644 instruct storeNKlass(iRegN src, memory4 mem)
 7645 %{
 7646   predicate(!needs_releasing_store(n));
 7647   match(Set mem (StoreNKlass mem src));
 7648 
 7649   ins_cost(INSN_COST);
 7650   format %{ "strw  $src, $mem\t# compressed klass ptr" %}
 7651 
 7652   ins_encode(aarch64_enc_strw(src, mem));
 7653 
 7654   ins_pipe(istore_reg_mem);
 7655 %}
 7656 
 7657 // TODO
 7658 // implement storeImmD0 and storeDImmPacked
 7659 
 7660 // prefetch instructions
 7661 // Must be safe to execute with invalid address (cannot fault).
 7662 
 7663 instruct prefetchalloc( memory8 mem ) %{
 7664   match(PrefetchAllocation mem);
 7665 
 7666   ins_cost(INSN_COST);
 7667   format %{ "prfm $mem, PSTL1KEEP\t# Prefetch into level 1 cache write keep" %}
 7668 
 7669   ins_encode( aarch64_enc_prefetchw(mem) );
 7670 
 7671   ins_pipe(iload_prefetch);
 7672 %}
 7673 
 7674 //  ---------------- volatile loads and stores ----------------
 7675 
 7676 // Load Byte (8 bit signed)
 7677 instruct loadB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 7678 %{
 7679   match(Set dst (LoadB mem));
 7680 
 7681   ins_cost(VOLATILE_REF_COST);
 7682   format %{ "ldarsb  $dst, $mem\t# byte" %}
 7683 
 7684   ins_encode(aarch64_enc_ldarsb(dst, mem));
 7685 
 7686   ins_pipe(pipe_serial);
 7687 %}
 7688 
 7689 // Load Byte (8 bit signed) into long
 7690 instruct loadB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 7691 %{
 7692   match(Set dst (ConvI2L (LoadB mem)));
 7693 
 7694   ins_cost(VOLATILE_REF_COST);
 7695   format %{ "ldarsb  $dst, $mem\t# byte" %}
 7696 
 7697   ins_encode(aarch64_enc_ldarsb(dst, mem));
 7698 
 7699   ins_pipe(pipe_serial);
 7700 %}
 7701 
 7702 // Load Byte (8 bit unsigned)
 7703 instruct loadUB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 7704 %{
 7705   match(Set dst (LoadUB mem));
 7706 
 7707   ins_cost(VOLATILE_REF_COST);
 7708   format %{ "ldarb  $dst, $mem\t# byte" %}
 7709 
 7710   ins_encode(aarch64_enc_ldarb(dst, mem));
 7711 
 7712   ins_pipe(pipe_serial);
 7713 %}
 7714 
 7715 // Load Byte (8 bit unsigned) into long
 7716 instruct loadUB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 7717 %{
 7718   match(Set dst (ConvI2L (LoadUB mem)));
 7719 
 7720   ins_cost(VOLATILE_REF_COST);
 7721   format %{ "ldarb  $dst, $mem\t# byte" %}
 7722 
 7723   ins_encode(aarch64_enc_ldarb(dst, mem));
 7724 
 7725   ins_pipe(pipe_serial);
 7726 %}
 7727 
 7728 // Load Short (16 bit signed)
 7729 instruct loadS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 7730 %{
 7731   match(Set dst (LoadS mem));
 7732 
 7733   ins_cost(VOLATILE_REF_COST);
 7734   format %{ "ldarshw  $dst, $mem\t# short" %}
 7735 
 7736   ins_encode(aarch64_enc_ldarshw(dst, mem));
 7737 
 7738   ins_pipe(pipe_serial);
 7739 %}
 7740 
 7741 instruct loadUS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 7742 %{
 7743   match(Set dst (LoadUS mem));
 7744 
 7745   ins_cost(VOLATILE_REF_COST);
 7746   format %{ "ldarhw  $dst, $mem\t# short" %}
 7747 
 7748   ins_encode(aarch64_enc_ldarhw(dst, mem));
 7749 
 7750   ins_pipe(pipe_serial);
 7751 %}
 7752 
 7753 // Load Short/Char (16 bit unsigned) into long
 7754 instruct loadUS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 7755 %{
 7756   match(Set dst (ConvI2L (LoadUS mem)));
 7757 
 7758   ins_cost(VOLATILE_REF_COST);
 7759   format %{ "ldarh  $dst, $mem\t# short" %}
 7760 
 7761   ins_encode(aarch64_enc_ldarh(dst, mem));
 7762 
 7763   ins_pipe(pipe_serial);
 7764 %}
 7765 
 7766 // Load Short/Char (16 bit signed) into long
 7767 instruct loadS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 7768 %{
 7769   match(Set dst (ConvI2L (LoadS mem)));
 7770 
 7771   ins_cost(VOLATILE_REF_COST);
 7772   format %{ "ldarh  $dst, $mem\t# short" %}
 7773 
 7774   ins_encode(aarch64_enc_ldarsh(dst, mem));
 7775 
 7776   ins_pipe(pipe_serial);
 7777 %}
 7778 
 7779 // Load Integer (32 bit signed)
 7780 instruct loadI_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 7781 %{
 7782   match(Set dst (LoadI mem));
 7783 
 7784   ins_cost(VOLATILE_REF_COST);
 7785   format %{ "ldarw  $dst, $mem\t# int" %}
 7786 
 7787   ins_encode(aarch64_enc_ldarw(dst, mem));
 7788 
 7789   ins_pipe(pipe_serial);
 7790 %}
 7791 
 7792 // Load Integer (32 bit unsigned) into long
 7793 instruct loadUI2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem, immL_32bits mask)
 7794 %{
 7795   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
 7796 
 7797   ins_cost(VOLATILE_REF_COST);
 7798   format %{ "ldarw  $dst, $mem\t# int" %}
 7799 
 7800   ins_encode(aarch64_enc_ldarw(dst, mem));
 7801 
 7802   ins_pipe(pipe_serial);
 7803 %}
 7804 
 7805 // Load Long (64 bit signed)
 7806 instruct loadL_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 7807 %{
 7808   match(Set dst (LoadL mem));
 7809 
 7810   ins_cost(VOLATILE_REF_COST);
 7811   format %{ "ldar  $dst, $mem\t# int" %}
 7812 
 7813   ins_encode(aarch64_enc_ldar(dst, mem));
 7814 
 7815   ins_pipe(pipe_serial);
 7816 %}
 7817 
 7818 // Load Pointer
 7819 instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem)
 7820 %{
 7821   match(Set dst (LoadP mem));
 7822   predicate(n->as_Load()->barrier_data() == 0);
 7823 
 7824   ins_cost(VOLATILE_REF_COST);
 7825   format %{ "ldar  $dst, $mem\t# ptr" %}
 7826 
 7827   ins_encode(aarch64_enc_ldar(dst, mem));
 7828 
 7829   ins_pipe(pipe_serial);
 7830 %}
 7831 
 7832 // Load Compressed Pointer
 7833 instruct loadN_volatile(iRegNNoSp dst, /* sync_memory*/indirect mem)
 7834 %{
 7835   match(Set dst (LoadN mem));
 7836 
 7837   ins_cost(VOLATILE_REF_COST);
 7838   format %{ "ldarw  $dst, $mem\t# compressed ptr" %}
 7839 
 7840   ins_encode(aarch64_enc_ldarw(dst, mem));
 7841 
 7842   ins_pipe(pipe_serial);
 7843 %}
 7844 
 7845 // Load Float
 7846 instruct loadF_volatile(vRegF dst, /* sync_memory*/indirect mem)
 7847 %{
 7848   match(Set dst (LoadF mem));
 7849 
 7850   ins_cost(VOLATILE_REF_COST);
 7851   format %{ "ldars  $dst, $mem\t# float" %}
 7852 
 7853   ins_encode( aarch64_enc_fldars(dst, mem) );
 7854 
 7855   ins_pipe(pipe_serial);
 7856 %}
 7857 
 7858 // Load Double
 7859 instruct loadD_volatile(vRegD dst, /* sync_memory*/indirect mem)
 7860 %{
 7861   match(Set dst (LoadD mem));
 7862 
 7863   ins_cost(VOLATILE_REF_COST);
 7864   format %{ "ldard  $dst, $mem\t# double" %}
 7865 
 7866   ins_encode( aarch64_enc_fldard(dst, mem) );
 7867 
 7868   ins_pipe(pipe_serial);
 7869 %}
 7870 
 7871 // Store Byte
 7872 instruct storeB_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
 7873 %{
 7874   match(Set mem (StoreB mem src));
 7875 
 7876   ins_cost(VOLATILE_REF_COST);
 7877   format %{ "stlrb  $src, $mem\t# byte" %}
 7878 
 7879   ins_encode(aarch64_enc_stlrb(src, mem));
 7880 
 7881   ins_pipe(pipe_class_memory);
 7882 %}
 7883 
 7884 instruct storeimmB0_volatile(immI0 zero, /* sync_memory*/indirect mem)
 7885 %{
 7886   match(Set mem (StoreB mem zero));
 7887 
 7888   ins_cost(VOLATILE_REF_COST);
 7889   format %{ "stlrb  zr, $mem\t# byte" %}
 7890 
 7891   ins_encode(aarch64_enc_stlrb0(mem));
 7892 
 7893   ins_pipe(pipe_class_memory);
 7894 %}
 7895 
 7896 // Store Char/Short
 7897 instruct storeC_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
 7898 %{
 7899   match(Set mem (StoreC mem src));
 7900 
 7901   ins_cost(VOLATILE_REF_COST);
 7902   format %{ "stlrh  $src, $mem\t# short" %}
 7903 
 7904   ins_encode(aarch64_enc_stlrh(src, mem));
 7905 
 7906   ins_pipe(pipe_class_memory);
 7907 %}
 7908 
 7909 instruct storeimmC0_volatile(immI0 zero, /* sync_memory*/indirect mem)
 7910 %{
 7911   match(Set mem (StoreC mem zero));
 7912 
 7913   ins_cost(VOLATILE_REF_COST);
 7914   format %{ "stlrh  zr, $mem\t# short" %}
 7915 
 7916   ins_encode(aarch64_enc_stlrh0(mem));
 7917 
 7918   ins_pipe(pipe_class_memory);
 7919 %}
 7920 
 7921 // Store Integer
 7922 
 7923 instruct storeI_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
 7924 %{
 7925   match(Set mem(StoreI mem src));
 7926 
 7927   ins_cost(VOLATILE_REF_COST);
 7928   format %{ "stlrw  $src, $mem\t# int" %}
 7929 
 7930   ins_encode(aarch64_enc_stlrw(src, mem));
 7931 
 7932   ins_pipe(pipe_class_memory);
 7933 %}
 7934 
 7935 instruct storeimmI0_volatile(immI0 zero, /* sync_memory*/indirect mem)
 7936 %{
 7937   match(Set mem(StoreI mem zero));
 7938 
 7939   ins_cost(VOLATILE_REF_COST);
 7940   format %{ "stlrw  zr, $mem\t# int" %}
 7941 
 7942   ins_encode(aarch64_enc_stlrw0(mem));
 7943 
 7944   ins_pipe(pipe_class_memory);
 7945 %}
 7946 
 7947 // Store Long (64 bit signed)
 7948 instruct storeL_volatile(iRegL src, /* sync_memory*/indirect mem)
 7949 %{
 7950   match(Set mem (StoreL mem src));
 7951 
 7952   ins_cost(VOLATILE_REF_COST);
 7953   format %{ "stlr  $src, $mem\t# int" %}
 7954 
 7955   ins_encode(aarch64_enc_stlr(src, mem));
 7956 
 7957   ins_pipe(pipe_class_memory);
 7958 %}
 7959 
 7960 instruct storeimmL0_volatile(immL0 zero, /* sync_memory*/indirect mem)
 7961 %{
 7962   match(Set mem (StoreL mem zero));
 7963 
 7964   ins_cost(VOLATILE_REF_COST);
 7965   format %{ "stlr  zr, $mem\t# int" %}
 7966 
 7967   ins_encode(aarch64_enc_stlr0(mem));
 7968 
 7969   ins_pipe(pipe_class_memory);
 7970 %}
 7971 
 7972 // Store Pointer
 7973 instruct storeP_volatile(iRegP src, /* sync_memory*/indirect mem)
 7974 %{
 7975   match(Set mem (StoreP mem src));
 7976   predicate(n->as_Store()->barrier_data() == 0);
 7977 
 7978   ins_cost(VOLATILE_REF_COST);
 7979   format %{ "stlr  $src, $mem\t# ptr" %}
 7980 
 7981   ins_encode(aarch64_enc_stlr(src, mem));
 7982 
 7983   ins_pipe(pipe_class_memory);
 7984 %}
 7985 
 7986 instruct storeimmP0_volatile(immP0 zero, /* sync_memory*/indirect mem)
 7987 %{
 7988   match(Set mem (StoreP mem zero));
 7989   predicate(n->as_Store()->barrier_data() == 0);
 7990 
 7991   ins_cost(VOLATILE_REF_COST);
 7992   format %{ "stlr  zr, $mem\t# ptr" %}
 7993 
 7994   ins_encode(aarch64_enc_stlr0(mem));
 7995 
 7996   ins_pipe(pipe_class_memory);
 7997 %}
 7998 
 7999 // Store Compressed Pointer
 8000 instruct storeN_volatile(iRegN src, /* sync_memory*/indirect mem)
 8001 %{
 8002   match(Set mem (StoreN mem src));
 8003 
 8004   ins_cost(VOLATILE_REF_COST);
 8005   format %{ "stlrw  $src, $mem\t# compressed ptr" %}
 8006 
 8007   ins_encode(aarch64_enc_stlrw(src, mem));
 8008 
 8009   ins_pipe(pipe_class_memory);
 8010 %}
 8011 
 8012 instruct storeimmN0_volatile(immN0 zero, /* sync_memory*/indirect mem)
 8013 %{
 8014   match(Set mem (StoreN mem zero));
 8015 
 8016   ins_cost(VOLATILE_REF_COST);
 8017   format %{ "stlrw  zr, $mem\t# compressed ptr" %}
 8018 
 8019   ins_encode(aarch64_enc_stlrw0(mem));
 8020 
 8021   ins_pipe(pipe_class_memory);
 8022 %}
 8023 
 8024 // Store Float
 8025 instruct storeF_volatile(vRegF src, /* sync_memory*/indirect mem)
 8026 %{
 8027   match(Set mem (StoreF mem src));
 8028 
 8029   ins_cost(VOLATILE_REF_COST);
 8030   format %{ "stlrs  $src, $mem\t# float" %}
 8031 
 8032   ins_encode( aarch64_enc_fstlrs(src, mem) );
 8033 
 8034   ins_pipe(pipe_class_memory);
 8035 %}
 8036 
 8037 // TODO
 8038 // implement storeImmF0 and storeFImmPacked
 8039 
 8040 // Store Double
 8041 instruct storeD_volatile(vRegD src, /* sync_memory*/indirect mem)
 8042 %{
 8043   match(Set mem (StoreD mem src));
 8044 
 8045   ins_cost(VOLATILE_REF_COST);
 8046   format %{ "stlrd  $src, $mem\t# double" %}
 8047 
 8048   ins_encode( aarch64_enc_fstlrd(src, mem) );
 8049 
 8050   ins_pipe(pipe_class_memory);
 8051 %}
 8052 
 8053 //  ---------------- end of volatile loads and stores ----------------
 8054 
 8055 instruct cacheWB(indirect addr)
 8056 %{
 8057   predicate(VM_Version::supports_data_cache_line_flush());
 8058   match(CacheWB addr);
 8059 
 8060   ins_cost(100);
 8061   format %{"cache wb $addr" %}
 8062   ins_encode %{
 8063     assert($addr->index_position() < 0, "should be");
 8064     assert($addr$$disp == 0, "should be");
 8065     __ cache_wb(Address($addr$$base$$Register, 0));
 8066   %}
 8067   ins_pipe(pipe_slow); // XXX
 8068 %}
 8069 
 8070 instruct cacheWBPreSync()
 8071 %{
 8072   predicate(VM_Version::supports_data_cache_line_flush());
 8073   match(CacheWBPreSync);
 8074 
 8075   ins_cost(100);
 8076   format %{"cache wb presync" %}
 8077   ins_encode %{
 8078     __ cache_wbsync(true);
 8079   %}
 8080   ins_pipe(pipe_slow); // XXX
 8081 %}
 8082 
 8083 instruct cacheWBPostSync()
 8084 %{
 8085   predicate(VM_Version::supports_data_cache_line_flush());
 8086   match(CacheWBPostSync);
 8087 
 8088   ins_cost(100);
 8089   format %{"cache wb postsync" %}
 8090   ins_encode %{
 8091     __ cache_wbsync(false);
 8092   %}
 8093   ins_pipe(pipe_slow); // XXX
 8094 %}
 8095 
 8096 // ============================================================================
 8097 // BSWAP Instructions
 8098 
 8099 instruct bytes_reverse_int(iRegINoSp dst, iRegIorL2I src) %{
 8100   match(Set dst (ReverseBytesI src));
 8101 
 8102   ins_cost(INSN_COST);
 8103   format %{ "revw  $dst, $src" %}
 8104 
 8105   ins_encode %{
 8106     __ revw(as_Register($dst$$reg), as_Register($src$$reg));
 8107   %}
 8108 
 8109   ins_pipe(ialu_reg);
 8110 %}
 8111 
 8112 instruct bytes_reverse_long(iRegLNoSp dst, iRegL src) %{
 8113   match(Set dst (ReverseBytesL src));
 8114 
 8115   ins_cost(INSN_COST);
 8116   format %{ "rev  $dst, $src" %}
 8117 
 8118   ins_encode %{
 8119     __ rev(as_Register($dst$$reg), as_Register($src$$reg));
 8120   %}
 8121 
 8122   ins_pipe(ialu_reg);
 8123 %}
 8124 
 8125 instruct bytes_reverse_unsigned_short(iRegINoSp dst, iRegIorL2I src) %{
 8126   match(Set dst (ReverseBytesUS src));
 8127 
 8128   ins_cost(INSN_COST);
 8129   format %{ "rev16w  $dst, $src" %}
 8130 
 8131   ins_encode %{
 8132     __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
 8133   %}
 8134 
 8135   ins_pipe(ialu_reg);
 8136 %}
 8137 
 8138 instruct bytes_reverse_short(iRegINoSp dst, iRegIorL2I src) %{
 8139   match(Set dst (ReverseBytesS src));
 8140 
 8141   ins_cost(INSN_COST);
 8142   format %{ "rev16w  $dst, $src\n\t"
 8143             "sbfmw $dst, $dst, #0, #15" %}
 8144 
 8145   ins_encode %{
 8146     __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
 8147     __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 15U);
 8148   %}
 8149 
 8150   ins_pipe(ialu_reg);
 8151 %}
 8152 
 8153 // ============================================================================
 8154 // Zero Count Instructions
 8155 
 8156 instruct countLeadingZerosI(iRegINoSp dst, iRegIorL2I src) %{
 8157   match(Set dst (CountLeadingZerosI src));
 8158 
 8159   ins_cost(INSN_COST);
 8160   format %{ "clzw  $dst, $src" %}
 8161   ins_encode %{
 8162     __ clzw(as_Register($dst$$reg), as_Register($src$$reg));
 8163   %}
 8164 
 8165   ins_pipe(ialu_reg);
 8166 %}
 8167 
 8168 instruct countLeadingZerosL(iRegINoSp dst, iRegL src) %{
 8169   match(Set dst (CountLeadingZerosL src));
 8170 
 8171   ins_cost(INSN_COST);
 8172   format %{ "clz   $dst, $src" %}
 8173   ins_encode %{
 8174     __ clz(as_Register($dst$$reg), as_Register($src$$reg));
 8175   %}
 8176 
 8177   ins_pipe(ialu_reg);
 8178 %}
 8179 
 8180 instruct countTrailingZerosI(iRegINoSp dst, iRegIorL2I src) %{
 8181   match(Set dst (CountTrailingZerosI src));
 8182 
 8183   ins_cost(INSN_COST * 2);
 8184   format %{ "rbitw  $dst, $src\n\t"
 8185             "clzw   $dst, $dst" %}
 8186   ins_encode %{
 8187     __ rbitw(as_Register($dst$$reg), as_Register($src$$reg));
 8188     __ clzw(as_Register($dst$$reg), as_Register($dst$$reg));
 8189   %}
 8190 
 8191   ins_pipe(ialu_reg);
 8192 %}
 8193 
 8194 instruct countTrailingZerosL(iRegINoSp dst, iRegL src) %{
 8195   match(Set dst (CountTrailingZerosL src));
 8196 
 8197   ins_cost(INSN_COST * 2);
 8198   format %{ "rbit   $dst, $src\n\t"
 8199             "clz    $dst, $dst" %}
 8200   ins_encode %{
 8201     __ rbit(as_Register($dst$$reg), as_Register($src$$reg));
 8202     __ clz(as_Register($dst$$reg), as_Register($dst$$reg));
 8203   %}
 8204 
 8205   ins_pipe(ialu_reg);
 8206 %}
 8207 
 8208 //---------- Population Count Instructions -------------------------------------
 8209 //
 8210 
 8211 instruct popCountI(iRegINoSp dst, iRegIorL2I src, vRegF tmp) %{
 8212   match(Set dst (PopCountI src));
 8213   effect(TEMP tmp);
 8214   ins_cost(INSN_COST * 13);
 8215 
 8216   format %{ "movw   $src, $src\n\t"
 8217             "mov    $tmp, $src\t# vector (1D)\n\t"
 8218             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 8219             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 8220             "mov    $dst, $tmp\t# vector (1D)" %}
 8221   ins_encode %{
 8222     __ movw($src$$Register, $src$$Register); // ensure top 32 bits 0
 8223     __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
 8224     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8225     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8226     __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
 8227   %}
 8228 
 8229   ins_pipe(pipe_class_default);
 8230 %}
 8231 
 8232 instruct popCountI_mem(iRegINoSp dst, memory4 mem, vRegF tmp) %{
 8233   match(Set dst (PopCountI (LoadI mem)));
 8234   effect(TEMP tmp);
 8235   ins_cost(INSN_COST * 13);
 8236 
 8237   format %{ "ldrs   $tmp, $mem\n\t"
 8238             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 8239             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 8240             "mov    $dst, $tmp\t# vector (1D)" %}
 8241   ins_encode %{
 8242     FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
 8243     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrs, tmp_reg, $mem->opcode(),
 8244               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 8245     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8246     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8247     __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
 8248   %}
 8249 
 8250   ins_pipe(pipe_class_default);
 8251 %}
 8252 
 8253 // Note: Long.bitCount(long) returns an int.
 8254 instruct popCountL(iRegINoSp dst, iRegL src, vRegD tmp) %{
 8255   match(Set dst (PopCountL src));
 8256   effect(TEMP tmp);
 8257   ins_cost(INSN_COST * 13);
 8258 
 8259   format %{ "mov    $tmp, $src\t# vector (1D)\n\t"
 8260             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 8261             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 8262             "mov    $dst, $tmp\t# vector (1D)" %}
 8263   ins_encode %{
 8264     __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
 8265     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8266     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8267     __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
 8268   %}
 8269 
 8270   ins_pipe(pipe_class_default);
 8271 %}
 8272 
 8273 instruct popCountL_mem(iRegINoSp dst, memory8 mem, vRegD tmp) %{
 8274   match(Set dst (PopCountL (LoadL mem)));
 8275   effect(TEMP tmp);
 8276   ins_cost(INSN_COST * 13);
 8277 
 8278   format %{ "ldrd   $tmp, $mem\n\t"
 8279             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 8280             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 8281             "mov    $dst, $tmp\t# vector (1D)" %}
 8282   ins_encode %{
 8283     FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
 8284     loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrd, tmp_reg, $mem->opcode(),
 8285               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 8286     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8287     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 8288     __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
 8289   %}
 8290 
 8291   ins_pipe(pipe_class_default);
 8292 %}
 8293 
 8294 // ============================================================================
 8295 // MemBar Instruction
 8296 
 8297 instruct load_fence() %{
 8298   match(LoadFence);
 8299   ins_cost(VOLATILE_REF_COST);
 8300 
 8301   format %{ "load_fence" %}
 8302 
 8303   ins_encode %{
 8304     __ membar(Assembler::LoadLoad|Assembler::LoadStore);
 8305   %}
 8306   ins_pipe(pipe_serial);
 8307 %}
 8308 
 8309 instruct unnecessary_membar_acquire() %{
 8310   predicate(unnecessary_acquire(n));
 8311   match(MemBarAcquire);
 8312   ins_cost(0);
 8313 
 8314   format %{ "membar_acquire (elided)" %}
 8315 
 8316   ins_encode %{
 8317     __ block_comment("membar_acquire (elided)");
 8318   %}
 8319 
 8320   ins_pipe(pipe_class_empty);
 8321 %}
 8322 
 8323 instruct membar_acquire() %{
 8324   match(MemBarAcquire);
 8325   ins_cost(VOLATILE_REF_COST);
 8326 
 8327   format %{ "membar_acquire\n\t"
 8328             "dmb ish" %}
 8329 
 8330   ins_encode %{
 8331     __ block_comment("membar_acquire");
 8332     __ membar(Assembler::LoadLoad|Assembler::LoadStore);
 8333   %}
 8334 
 8335   ins_pipe(pipe_serial);
 8336 %}
 8337 
 8338 
 8339 instruct membar_acquire_lock() %{
 8340   match(MemBarAcquireLock);
 8341   ins_cost(VOLATILE_REF_COST);
 8342 
 8343   format %{ "membar_acquire_lock (elided)" %}
 8344 
 8345   ins_encode %{
 8346     __ block_comment("membar_acquire_lock (elided)");
 8347   %}
 8348 
 8349   ins_pipe(pipe_serial);
 8350 %}
 8351 
 8352 instruct store_fence() %{
 8353   match(StoreFence);
 8354   ins_cost(VOLATILE_REF_COST);
 8355 
 8356   format %{ "store_fence" %}
 8357 
 8358   ins_encode %{
 8359     __ membar(Assembler::LoadStore|Assembler::StoreStore);
 8360   %}
 8361   ins_pipe(pipe_serial);
 8362 %}
 8363 
 8364 instruct unnecessary_membar_release() %{
 8365   predicate(unnecessary_release(n));
 8366   match(MemBarRelease);
 8367   ins_cost(0);
 8368 
 8369   format %{ "membar_release (elided)" %}
 8370 
 8371   ins_encode %{
 8372     __ block_comment("membar_release (elided)");
 8373   %}
 8374   ins_pipe(pipe_serial);
 8375 %}
 8376 
 8377 instruct membar_release() %{
 8378   match(MemBarRelease);
 8379   ins_cost(VOLATILE_REF_COST);
 8380 
 8381   format %{ "membar_release\n\t"
 8382             "dmb ish" %}
 8383 
 8384   ins_encode %{
 8385     __ block_comment("membar_release");
 8386     __ membar(Assembler::LoadStore|Assembler::StoreStore);
 8387   %}
 8388   ins_pipe(pipe_serial);
 8389 %}
 8390 
 8391 instruct membar_storestore() %{
 8392   match(MemBarStoreStore);
 8393   match(StoreStoreFence);
 8394   ins_cost(VOLATILE_REF_COST);
 8395 
 8396   format %{ "MEMBAR-store-store" %}
 8397 
 8398   ins_encode %{
 8399     __ membar(Assembler::StoreStore);
 8400   %}
 8401   ins_pipe(pipe_serial);
 8402 %}
 8403 
 8404 instruct membar_release_lock() %{
 8405   match(MemBarReleaseLock);
 8406   ins_cost(VOLATILE_REF_COST);
 8407 
 8408   format %{ "membar_release_lock (elided)" %}
 8409 
 8410   ins_encode %{
 8411     __ block_comment("membar_release_lock (elided)");
 8412   %}
 8413 
 8414   ins_pipe(pipe_serial);
 8415 %}
 8416 
 8417 instruct unnecessary_membar_volatile() %{
 8418   predicate(unnecessary_volatile(n));
 8419   match(MemBarVolatile);
 8420   ins_cost(0);
 8421 
 8422   format %{ "membar_volatile (elided)" %}
 8423 
 8424   ins_encode %{
 8425     __ block_comment("membar_volatile (elided)");
 8426   %}
 8427 
 8428   ins_pipe(pipe_serial);
 8429 %}
 8430 
 8431 instruct membar_volatile() %{
 8432   match(MemBarVolatile);
 8433   ins_cost(VOLATILE_REF_COST*100);
 8434 
 8435   format %{ "membar_volatile\n\t"
 8436              "dmb ish"%}
 8437 
 8438   ins_encode %{
 8439     __ block_comment("membar_volatile");
 8440     __ membar(Assembler::StoreLoad);
 8441   %}
 8442 
 8443   ins_pipe(pipe_serial);
 8444 %}
 8445 
 8446 // ============================================================================
 8447 // Cast/Convert Instructions
 8448 
 8449 instruct castX2P(iRegPNoSp dst, iRegL src) %{
 8450   match(Set dst (CastX2P src));
 8451 
 8452   ins_cost(INSN_COST);
 8453   format %{ "mov $dst, $src\t# long -> ptr" %}
 8454 
 8455   ins_encode %{
 8456     if ($dst$$reg != $src$$reg) {
 8457       __ mov(as_Register($dst$$reg), as_Register($src$$reg));
 8458     }
 8459   %}
 8460 
 8461   ins_pipe(ialu_reg);
 8462 %}
 8463 
 8464 instruct castP2X(iRegLNoSp dst, iRegP src) %{
 8465   match(Set dst (CastP2X src));
 8466 
 8467   ins_cost(INSN_COST);
 8468   format %{ "mov $dst, $src\t# ptr -> long" %}
 8469 
 8470   ins_encode %{
 8471     if ($dst$$reg != $src$$reg) {
 8472       __ mov(as_Register($dst$$reg), as_Register($src$$reg));
 8473     }
 8474   %}
 8475 
 8476   ins_pipe(ialu_reg);
 8477 %}
 8478 
 8479 // Convert oop into int for vectors alignment masking
 8480 instruct convP2I(iRegINoSp dst, iRegP src) %{
 8481   match(Set dst (ConvL2I (CastP2X src)));
 8482 
 8483   ins_cost(INSN_COST);
 8484   format %{ "movw $dst, $src\t# ptr -> int" %}
 8485   ins_encode %{
 8486     __ movw($dst$$Register, $src$$Register);
 8487   %}
 8488 
 8489   ins_pipe(ialu_reg);
 8490 %}
 8491 
 8492 // Convert compressed oop into int for vectors alignment masking
 8493 // in case of 32bit oops (heap < 4Gb).
 8494 instruct convN2I(iRegINoSp dst, iRegN src)
 8495 %{
 8496   predicate(CompressedOops::shift() == 0);
 8497   match(Set dst (ConvL2I (CastP2X (DecodeN src))));
 8498 
 8499   ins_cost(INSN_COST);
 8500   format %{ "mov dst, $src\t# compressed ptr -> int" %}
 8501   ins_encode %{
 8502     __ movw($dst$$Register, $src$$Register);
 8503   %}
 8504 
 8505   ins_pipe(ialu_reg);
 8506 %}
 8507 
 8508 
 8509 // Convert oop pointer into compressed form
 8510 instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
 8511   predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
 8512   match(Set dst (EncodeP src));
 8513   effect(KILL cr);
 8514   ins_cost(INSN_COST * 3);
 8515   format %{ "encode_heap_oop $dst, $src" %}
 8516   ins_encode %{
 8517     Register s = $src$$Register;
 8518     Register d = $dst$$Register;
 8519     __ encode_heap_oop(d, s);
 8520   %}
 8521   ins_pipe(ialu_reg);
 8522 %}
 8523 
 8524 instruct encodeHeapOop_not_null(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
 8525   predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
 8526   match(Set dst (EncodeP src));
 8527   ins_cost(INSN_COST * 3);
 8528   format %{ "encode_heap_oop_not_null $dst, $src" %}
 8529   ins_encode %{
 8530     __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
 8531   %}
 8532   ins_pipe(ialu_reg);
 8533 %}
 8534 
 8535 instruct decodeHeapOop(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
 8536   predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
 8537             n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
 8538   match(Set dst (DecodeN src));
 8539   ins_cost(INSN_COST * 3);
 8540   format %{ "decode_heap_oop $dst, $src" %}
 8541   ins_encode %{
 8542     Register s = $src$$Register;
 8543     Register d = $dst$$Register;
 8544     __ decode_heap_oop(d, s);
 8545   %}
 8546   ins_pipe(ialu_reg);
 8547 %}
 8548 
 8549 instruct decodeHeapOop_not_null(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
 8550   predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
 8551             n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
 8552   match(Set dst (DecodeN src));
 8553   ins_cost(INSN_COST * 3);
 8554   format %{ "decode_heap_oop_not_null $dst, $src" %}
 8555   ins_encode %{
 8556     Register s = $src$$Register;
 8557     Register d = $dst$$Register;
 8558     __ decode_heap_oop_not_null(d, s);
 8559   %}
 8560   ins_pipe(ialu_reg);
 8561 %}
 8562 
 8563 // n.b. AArch64 implementations of encode_klass_not_null and
 8564 // decode_klass_not_null do not modify the flags register so, unlike
 8565 // Intel, we don't kill CR as a side effect here
 8566 
 8567 instruct encodeKlass_not_null(iRegNNoSp dst, iRegP src) %{
 8568   match(Set dst (EncodePKlass src));
 8569 
 8570   ins_cost(INSN_COST * 3);
 8571   format %{ "encode_klass_not_null $dst,$src" %}
 8572 
 8573   ins_encode %{
 8574     Register src_reg = as_Register($src$$reg);
 8575     Register dst_reg = as_Register($dst$$reg);
 8576     __ encode_klass_not_null(dst_reg, src_reg);
 8577   %}
 8578 
 8579    ins_pipe(ialu_reg);
 8580 %}
 8581 
 8582 instruct decodeKlass_not_null(iRegPNoSp dst, iRegN src) %{
 8583   match(Set dst (DecodeNKlass src));
 8584 
 8585   ins_cost(INSN_COST * 3);
 8586   format %{ "decode_klass_not_null $dst,$src" %}
 8587 
 8588   ins_encode %{
 8589     Register src_reg = as_Register($src$$reg);
 8590     Register dst_reg = as_Register($dst$$reg);
 8591     if (dst_reg != src_reg) {
 8592       __ decode_klass_not_null(dst_reg, src_reg);
 8593     } else {
 8594       __ decode_klass_not_null(dst_reg);
 8595     }
 8596   %}
 8597 
 8598    ins_pipe(ialu_reg);
 8599 %}
 8600 
 8601 instruct checkCastPP(iRegPNoSp dst)
 8602 %{
 8603   match(Set dst (CheckCastPP dst));
 8604 
 8605   size(0);
 8606   format %{ "# checkcastPP of $dst" %}
 8607   ins_encode(/* empty encoding */);
 8608   ins_pipe(pipe_class_empty);
 8609 %}
 8610 
 8611 instruct castPP(iRegPNoSp dst)
 8612 %{
 8613   match(Set dst (CastPP dst));
 8614 
 8615   size(0);
 8616   format %{ "# castPP of $dst" %}
 8617   ins_encode(/* empty encoding */);
 8618   ins_pipe(pipe_class_empty);
 8619 %}
 8620 
 8621 instruct castII(iRegI dst)
 8622 %{
 8623   match(Set dst (CastII dst));
 8624 
 8625   size(0);
 8626   format %{ "# castII of $dst" %}
 8627   ins_encode(/* empty encoding */);
 8628   ins_cost(0);
 8629   ins_pipe(pipe_class_empty);
 8630 %}
 8631 
 8632 instruct castLL(iRegL dst)
 8633 %{
 8634   match(Set dst (CastLL dst));
 8635 
 8636   size(0);
 8637   format %{ "# castLL of $dst" %}
 8638   ins_encode(/* empty encoding */);
 8639   ins_cost(0);
 8640   ins_pipe(pipe_class_empty);
 8641 %}
 8642 
 8643 instruct castFF(vRegF dst)
 8644 %{
 8645   match(Set dst (CastFF dst));
 8646 
 8647   size(0);
 8648   format %{ "# castFF of $dst" %}
 8649   ins_encode(/* empty encoding */);
 8650   ins_cost(0);
 8651   ins_pipe(pipe_class_empty);
 8652 %}
 8653 
 8654 instruct castDD(vRegD dst)
 8655 %{
 8656   match(Set dst (CastDD dst));
 8657 
 8658   size(0);
 8659   format %{ "# castDD of $dst" %}
 8660   ins_encode(/* empty encoding */);
 8661   ins_cost(0);
 8662   ins_pipe(pipe_class_empty);
 8663 %}
 8664 
 8665 instruct castVV(vReg dst)
 8666 %{
 8667   match(Set dst (CastVV dst));
 8668 
 8669   size(0);
 8670   format %{ "# castVV of $dst" %}
 8671   ins_encode(/* empty encoding */);
 8672   ins_cost(0);
 8673   ins_pipe(pipe_class_empty);
 8674 %}
 8675 
 8676 instruct castVVMask(pRegGov dst)
 8677 %{
 8678   match(Set dst (CastVV dst));
 8679 
 8680   size(0);
 8681   format %{ "# castVV of $dst" %}
 8682   ins_encode(/* empty encoding */);
 8683   ins_cost(0);
 8684   ins_pipe(pipe_class_empty);
 8685 %}
 8686 
 8687 // ============================================================================
 8688 // Atomic operation instructions
 8689 //
 8690 
 8691 // standard CompareAndSwapX when we are using barriers
 8692 // these have higher priority than the rules selected by a predicate
 8693 
 8694 // XXX No flag versions for CompareAndSwap{I,L,P,N} because matcher
 8695 // can't match them
 8696 
 8697 instruct compareAndSwapB(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 8698 
 8699   match(Set res (CompareAndSwapB mem (Binary oldval newval)));
 8700   ins_cost(2 * VOLATILE_REF_COST);
 8701 
 8702   effect(KILL cr);
 8703 
 8704   format %{
 8705     "cmpxchgb $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 8706     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8707   %}
 8708 
 8709   ins_encode(aarch64_enc_cmpxchgb(mem, oldval, newval),
 8710             aarch64_enc_cset_eq(res));
 8711 
 8712   ins_pipe(pipe_slow);
 8713 %}
 8714 
 8715 instruct compareAndSwapS(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 8716 
 8717   match(Set res (CompareAndSwapS mem (Binary oldval newval)));
 8718   ins_cost(2 * VOLATILE_REF_COST);
 8719 
 8720   effect(KILL cr);
 8721 
 8722   format %{
 8723     "cmpxchgs $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 8724     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8725   %}
 8726 
 8727   ins_encode(aarch64_enc_cmpxchgs(mem, oldval, newval),
 8728             aarch64_enc_cset_eq(res));
 8729 
 8730   ins_pipe(pipe_slow);
 8731 %}
 8732 
 8733 instruct compareAndSwapI(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 8734 
 8735   match(Set res (CompareAndSwapI mem (Binary oldval newval)));
 8736   ins_cost(2 * VOLATILE_REF_COST);
 8737 
 8738   effect(KILL cr);
 8739 
 8740  format %{
 8741     "cmpxchgw $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 8742     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8743  %}
 8744 
 8745  ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
 8746             aarch64_enc_cset_eq(res));
 8747 
 8748   ins_pipe(pipe_slow);
 8749 %}
 8750 
 8751 instruct compareAndSwapL(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
 8752 
 8753   match(Set res (CompareAndSwapL mem (Binary oldval newval)));
 8754   ins_cost(2 * VOLATILE_REF_COST);
 8755 
 8756   effect(KILL cr);
 8757 
 8758  format %{
 8759     "cmpxchg $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
 8760     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8761  %}
 8762 
 8763  ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
 8764             aarch64_enc_cset_eq(res));
 8765 
 8766   ins_pipe(pipe_slow);
 8767 %}
 8768 
 8769 instruct compareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 8770 
 8771   match(Set res (CompareAndSwapP mem (Binary oldval newval)));
 8772   predicate(n->as_LoadStore()->barrier_data() == 0);
 8773   ins_cost(2 * VOLATILE_REF_COST);
 8774 
 8775   effect(KILL cr);
 8776 
 8777  format %{
 8778     "cmpxchg $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
 8779     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8780  %}
 8781 
 8782  ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
 8783             aarch64_enc_cset_eq(res));
 8784 
 8785   ins_pipe(pipe_slow);
 8786 %}
 8787 
 8788 instruct compareAndSwapN(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
 8789 
 8790   match(Set res (CompareAndSwapN mem (Binary oldval newval)));
 8791   ins_cost(2 * VOLATILE_REF_COST);
 8792 
 8793   effect(KILL cr);
 8794 
 8795  format %{
 8796     "cmpxchgw $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
 8797     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8798  %}
 8799 
 8800  ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
 8801             aarch64_enc_cset_eq(res));
 8802 
 8803   ins_pipe(pipe_slow);
 8804 %}
 8805 
 8806 // alternative CompareAndSwapX when we are eliding barriers
 8807 
 8808 instruct compareAndSwapBAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 8809 
 8810   predicate(needs_acquiring_load_exclusive(n));
 8811   match(Set res (CompareAndSwapB mem (Binary oldval newval)));
 8812   ins_cost(VOLATILE_REF_COST);
 8813 
 8814   effect(KILL cr);
 8815 
 8816   format %{
 8817     "cmpxchgb_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 8818     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8819   %}
 8820 
 8821   ins_encode(aarch64_enc_cmpxchgb_acq(mem, oldval, newval),
 8822             aarch64_enc_cset_eq(res));
 8823 
 8824   ins_pipe(pipe_slow);
 8825 %}
 8826 
 8827 instruct compareAndSwapSAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 8828 
 8829   predicate(needs_acquiring_load_exclusive(n));
 8830   match(Set res (CompareAndSwapS mem (Binary oldval newval)));
 8831   ins_cost(VOLATILE_REF_COST);
 8832 
 8833   effect(KILL cr);
 8834 
 8835   format %{
 8836     "cmpxchgs_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 8837     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8838   %}
 8839 
 8840   ins_encode(aarch64_enc_cmpxchgs_acq(mem, oldval, newval),
 8841             aarch64_enc_cset_eq(res));
 8842 
 8843   ins_pipe(pipe_slow);
 8844 %}
 8845 
 8846 instruct compareAndSwapIAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 8847 
 8848   predicate(needs_acquiring_load_exclusive(n));
 8849   match(Set res (CompareAndSwapI mem (Binary oldval newval)));
 8850   ins_cost(VOLATILE_REF_COST);
 8851 
 8852   effect(KILL cr);
 8853 
 8854  format %{
 8855     "cmpxchgw_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 8856     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8857  %}
 8858 
 8859  ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
 8860             aarch64_enc_cset_eq(res));
 8861 
 8862   ins_pipe(pipe_slow);
 8863 %}
 8864 
 8865 instruct compareAndSwapLAcq(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
 8866 
 8867   predicate(needs_acquiring_load_exclusive(n));
 8868   match(Set res (CompareAndSwapL mem (Binary oldval newval)));
 8869   ins_cost(VOLATILE_REF_COST);
 8870 
 8871   effect(KILL cr);
 8872 
 8873  format %{
 8874     "cmpxchg_acq $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
 8875     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8876  %}
 8877 
 8878  ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
 8879             aarch64_enc_cset_eq(res));
 8880 
 8881   ins_pipe(pipe_slow);
 8882 %}
 8883 
 8884 instruct compareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 8885 
 8886   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 8887   match(Set res (CompareAndSwapP mem (Binary oldval newval)));
 8888   ins_cost(VOLATILE_REF_COST);
 8889 
 8890   effect(KILL cr);
 8891 
 8892  format %{
 8893     "cmpxchg_acq $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
 8894     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8895  %}
 8896 
 8897  ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
 8898             aarch64_enc_cset_eq(res));
 8899 
 8900   ins_pipe(pipe_slow);
 8901 %}
 8902 
 8903 instruct compareAndSwapNAcq(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
 8904 
 8905   predicate(needs_acquiring_load_exclusive(n));
 8906   match(Set res (CompareAndSwapN mem (Binary oldval newval)));
 8907   ins_cost(VOLATILE_REF_COST);
 8908 
 8909   effect(KILL cr);
 8910 
 8911  format %{
 8912     "cmpxchgw_acq $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
 8913     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8914  %}
 8915 
 8916  ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
 8917             aarch64_enc_cset_eq(res));
 8918 
 8919   ins_pipe(pipe_slow);
 8920 %}
 8921 
 8922 
 8923 // ---------------------------------------------------------------------
 8924 
 8925 // BEGIN This section of the file is automatically generated. Do not edit --------------
 8926 
 8927 // Sundry CAS operations.  Note that release is always true,
 8928 // regardless of the memory ordering of the CAS.  This is because we
 8929 // need the volatile case to be sequentially consistent but there is
 8930 // no trailing StoreLoad barrier emitted by C2.  Unfortunately we
 8931 // can't check the type of memory ordering here, so we always emit a
 8932 // STLXR.
 8933 
 8934 // This section is generated from aarch64_ad_cas.m4
 8935 
 8936 
 8937 
 8938 // This pattern is generated automatically from cas.m4.
 8939 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8940 instruct compareAndExchangeB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 8941 
 8942   match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
 8943   ins_cost(2 * VOLATILE_REF_COST);
 8944   effect(TEMP_DEF res, KILL cr);
 8945   format %{
 8946     "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 8947   %}
 8948   ins_encode %{
 8949     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8950                Assembler::byte, /*acquire*/ false, /*release*/ true,
 8951                /*weak*/ false, $res$$Register);
 8952     __ sxtbw($res$$Register, $res$$Register);
 8953   %}
 8954   ins_pipe(pipe_slow);
 8955 %}
 8956 
 8957 // This pattern is generated automatically from cas.m4.
 8958 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8959 instruct compareAndExchangeS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 8960 
 8961   match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
 8962   ins_cost(2 * VOLATILE_REF_COST);
 8963   effect(TEMP_DEF res, KILL cr);
 8964   format %{
 8965     "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 8966   %}
 8967   ins_encode %{
 8968     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8969                Assembler::halfword, /*acquire*/ false, /*release*/ true,
 8970                /*weak*/ false, $res$$Register);
 8971     __ sxthw($res$$Register, $res$$Register);
 8972   %}
 8973   ins_pipe(pipe_slow);
 8974 %}
 8975 
 8976 // This pattern is generated automatically from cas.m4.
 8977 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8978 instruct compareAndExchangeI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 8979 
 8980   match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
 8981   ins_cost(2 * VOLATILE_REF_COST);
 8982   effect(TEMP_DEF res, KILL cr);
 8983   format %{
 8984     "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 8985   %}
 8986   ins_encode %{
 8987     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8988                Assembler::word, /*acquire*/ false, /*release*/ true,
 8989                /*weak*/ false, $res$$Register);
 8990   %}
 8991   ins_pipe(pipe_slow);
 8992 %}
 8993 
 8994 // This pattern is generated automatically from cas.m4.
 8995 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8996 instruct compareAndExchangeL(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 8997 
 8998   match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
 8999   ins_cost(2 * VOLATILE_REF_COST);
 9000   effect(TEMP_DEF res, KILL cr);
 9001   format %{
 9002     "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 9003   %}
 9004   ins_encode %{
 9005     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9006                Assembler::xword, /*acquire*/ false, /*release*/ true,
 9007                /*weak*/ false, $res$$Register);
 9008   %}
 9009   ins_pipe(pipe_slow);
 9010 %}
 9011 
 9012 // This pattern is generated automatically from cas.m4.
 9013 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9014 instruct compareAndExchangeN(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 9015 
 9016   match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
 9017   ins_cost(2 * VOLATILE_REF_COST);
 9018   effect(TEMP_DEF res, KILL cr);
 9019   format %{
 9020     "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 9021   %}
 9022   ins_encode %{
 9023     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9024                Assembler::word, /*acquire*/ false, /*release*/ true,
 9025                /*weak*/ false, $res$$Register);
 9026   %}
 9027   ins_pipe(pipe_slow);
 9028 %}
 9029 
 9030 // This pattern is generated automatically from cas.m4.
 9031 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9032 instruct compareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9033   predicate(n->as_LoadStore()->barrier_data() == 0);
 9034   match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
 9035   ins_cost(2 * VOLATILE_REF_COST);
 9036   effect(TEMP_DEF res, KILL cr);
 9037   format %{
 9038     "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 9039   %}
 9040   ins_encode %{
 9041     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9042                Assembler::xword, /*acquire*/ false, /*release*/ true,
 9043                /*weak*/ false, $res$$Register);
 9044   %}
 9045   ins_pipe(pipe_slow);
 9046 %}
 9047 
 9048 // This pattern is generated automatically from cas.m4.
 9049 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9050 instruct compareAndExchangeBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9051   predicate(needs_acquiring_load_exclusive(n));
 9052   match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
 9053   ins_cost(VOLATILE_REF_COST);
 9054   effect(TEMP_DEF res, KILL cr);
 9055   format %{
 9056     "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 9057   %}
 9058   ins_encode %{
 9059     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9060                Assembler::byte, /*acquire*/ true, /*release*/ true,
 9061                /*weak*/ false, $res$$Register);
 9062     __ sxtbw($res$$Register, $res$$Register);
 9063   %}
 9064   ins_pipe(pipe_slow);
 9065 %}
 9066 
 9067 // This pattern is generated automatically from cas.m4.
 9068 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9069 instruct compareAndExchangeSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9070   predicate(needs_acquiring_load_exclusive(n));
 9071   match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
 9072   ins_cost(VOLATILE_REF_COST);
 9073   effect(TEMP_DEF res, KILL cr);
 9074   format %{
 9075     "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 9076   %}
 9077   ins_encode %{
 9078     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9079                Assembler::halfword, /*acquire*/ true, /*release*/ true,
 9080                /*weak*/ false, $res$$Register);
 9081     __ sxthw($res$$Register, $res$$Register);
 9082   %}
 9083   ins_pipe(pipe_slow);
 9084 %}
 9085 
 9086 // This pattern is generated automatically from cas.m4.
 9087 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9088 instruct compareAndExchangeIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9089   predicate(needs_acquiring_load_exclusive(n));
 9090   match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
 9091   ins_cost(VOLATILE_REF_COST);
 9092   effect(TEMP_DEF res, KILL cr);
 9093   format %{
 9094     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 9095   %}
 9096   ins_encode %{
 9097     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9098                Assembler::word, /*acquire*/ true, /*release*/ true,
 9099                /*weak*/ false, $res$$Register);
 9100   %}
 9101   ins_pipe(pipe_slow);
 9102 %}
 9103 
 9104 // This pattern is generated automatically from cas.m4.
 9105 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9106 instruct compareAndExchangeLAcq(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 9107   predicate(needs_acquiring_load_exclusive(n));
 9108   match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
 9109   ins_cost(VOLATILE_REF_COST);
 9110   effect(TEMP_DEF res, KILL cr);
 9111   format %{
 9112     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 9113   %}
 9114   ins_encode %{
 9115     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9116                Assembler::xword, /*acquire*/ true, /*release*/ true,
 9117                /*weak*/ false, $res$$Register);
 9118   %}
 9119   ins_pipe(pipe_slow);
 9120 %}
 9121 
 9122 // This pattern is generated automatically from cas.m4.
 9123 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9124 instruct compareAndExchangeNAcq(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 9125   predicate(needs_acquiring_load_exclusive(n));
 9126   match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
 9127   ins_cost(VOLATILE_REF_COST);
 9128   effect(TEMP_DEF res, KILL cr);
 9129   format %{
 9130     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 9131   %}
 9132   ins_encode %{
 9133     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9134                Assembler::word, /*acquire*/ true, /*release*/ true,
 9135                /*weak*/ false, $res$$Register);
 9136   %}
 9137   ins_pipe(pipe_slow);
 9138 %}
 9139 
 9140 // This pattern is generated automatically from cas.m4.
 9141 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9142 instruct compareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9143   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 9144   match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
 9145   ins_cost(VOLATILE_REF_COST);
 9146   effect(TEMP_DEF res, KILL cr);
 9147   format %{
 9148     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 9149   %}
 9150   ins_encode %{
 9151     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9152                Assembler::xword, /*acquire*/ true, /*release*/ true,
 9153                /*weak*/ false, $res$$Register);
 9154   %}
 9155   ins_pipe(pipe_slow);
 9156 %}
 9157 
 9158 // This pattern is generated automatically from cas.m4.
 9159 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9160 instruct weakCompareAndSwapB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9161 
 9162   match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
 9163   ins_cost(2 * VOLATILE_REF_COST);
 9164   effect(KILL cr);
 9165   format %{
 9166     "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 9167     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9168   %}
 9169   ins_encode %{
 9170     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9171                Assembler::byte, /*acquire*/ false, /*release*/ true,
 9172                /*weak*/ true, noreg);
 9173     __ csetw($res$$Register, Assembler::EQ);
 9174   %}
 9175   ins_pipe(pipe_slow);
 9176 %}
 9177 
 9178 // This pattern is generated automatically from cas.m4.
 9179 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9180 instruct weakCompareAndSwapS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9181 
 9182   match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
 9183   ins_cost(2 * VOLATILE_REF_COST);
 9184   effect(KILL cr);
 9185   format %{
 9186     "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 9187     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9188   %}
 9189   ins_encode %{
 9190     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9191                Assembler::halfword, /*acquire*/ false, /*release*/ true,
 9192                /*weak*/ true, noreg);
 9193     __ csetw($res$$Register, Assembler::EQ);
 9194   %}
 9195   ins_pipe(pipe_slow);
 9196 %}
 9197 
 9198 // This pattern is generated automatically from cas.m4.
 9199 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9200 instruct weakCompareAndSwapI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9201 
 9202   match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
 9203   ins_cost(2 * VOLATILE_REF_COST);
 9204   effect(KILL cr);
 9205   format %{
 9206     "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 9207     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9208   %}
 9209   ins_encode %{
 9210     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9211                Assembler::word, /*acquire*/ false, /*release*/ true,
 9212                /*weak*/ true, noreg);
 9213     __ csetw($res$$Register, Assembler::EQ);
 9214   %}
 9215   ins_pipe(pipe_slow);
 9216 %}
 9217 
 9218 // This pattern is generated automatically from cas.m4.
 9219 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9220 instruct weakCompareAndSwapL(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 9221 
 9222   match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
 9223   ins_cost(2 * VOLATILE_REF_COST);
 9224   effect(KILL cr);
 9225   format %{
 9226     "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 9227     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9228   %}
 9229   ins_encode %{
 9230     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9231                Assembler::xword, /*acquire*/ false, /*release*/ true,
 9232                /*weak*/ true, noreg);
 9233     __ csetw($res$$Register, Assembler::EQ);
 9234   %}
 9235   ins_pipe(pipe_slow);
 9236 %}
 9237 
 9238 // This pattern is generated automatically from cas.m4.
 9239 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9240 instruct weakCompareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 9241 
 9242   match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
 9243   ins_cost(2 * VOLATILE_REF_COST);
 9244   effect(KILL cr);
 9245   format %{
 9246     "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 9247     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9248   %}
 9249   ins_encode %{
 9250     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9251                Assembler::word, /*acquire*/ false, /*release*/ true,
 9252                /*weak*/ true, noreg);
 9253     __ csetw($res$$Register, Assembler::EQ);
 9254   %}
 9255   ins_pipe(pipe_slow);
 9256 %}
 9257 
 9258 // This pattern is generated automatically from cas.m4.
 9259 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9260 instruct weakCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9261   predicate(n->as_LoadStore()->barrier_data() == 0);
 9262   match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
 9263   ins_cost(2 * VOLATILE_REF_COST);
 9264   effect(KILL cr);
 9265   format %{
 9266     "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 9267     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9268   %}
 9269   ins_encode %{
 9270     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9271                Assembler::xword, /*acquire*/ false, /*release*/ true,
 9272                /*weak*/ true, noreg);
 9273     __ csetw($res$$Register, Assembler::EQ);
 9274   %}
 9275   ins_pipe(pipe_slow);
 9276 %}
 9277 
 9278 // This pattern is generated automatically from cas.m4.
 9279 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9280 instruct weakCompareAndSwapBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9281   predicate(needs_acquiring_load_exclusive(n));
 9282   match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
 9283   ins_cost(VOLATILE_REF_COST);
 9284   effect(KILL cr);
 9285   format %{
 9286     "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 9287     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9288   %}
 9289   ins_encode %{
 9290     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9291                Assembler::byte, /*acquire*/ true, /*release*/ true,
 9292                /*weak*/ true, noreg);
 9293     __ csetw($res$$Register, Assembler::EQ);
 9294   %}
 9295   ins_pipe(pipe_slow);
 9296 %}
 9297 
 9298 // This pattern is generated automatically from cas.m4.
 9299 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9300 instruct weakCompareAndSwapSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9301   predicate(needs_acquiring_load_exclusive(n));
 9302   match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
 9303   ins_cost(VOLATILE_REF_COST);
 9304   effect(KILL cr);
 9305   format %{
 9306     "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 9307     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9308   %}
 9309   ins_encode %{
 9310     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9311                Assembler::halfword, /*acquire*/ true, /*release*/ true,
 9312                /*weak*/ true, noreg);
 9313     __ csetw($res$$Register, Assembler::EQ);
 9314   %}
 9315   ins_pipe(pipe_slow);
 9316 %}
 9317 
 9318 // This pattern is generated automatically from cas.m4.
 9319 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9320 instruct weakCompareAndSwapIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 9321   predicate(needs_acquiring_load_exclusive(n));
 9322   match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
 9323   ins_cost(VOLATILE_REF_COST);
 9324   effect(KILL cr);
 9325   format %{
 9326     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 9327     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9328   %}
 9329   ins_encode %{
 9330     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9331                Assembler::word, /*acquire*/ true, /*release*/ true,
 9332                /*weak*/ true, noreg);
 9333     __ csetw($res$$Register, Assembler::EQ);
 9334   %}
 9335   ins_pipe(pipe_slow);
 9336 %}
 9337 
 9338 // This pattern is generated automatically from cas.m4.
 9339 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9340 instruct weakCompareAndSwapLAcq(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 9341   predicate(needs_acquiring_load_exclusive(n));
 9342   match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
 9343   ins_cost(VOLATILE_REF_COST);
 9344   effect(KILL cr);
 9345   format %{
 9346     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 9347     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9348   %}
 9349   ins_encode %{
 9350     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9351                Assembler::xword, /*acquire*/ true, /*release*/ true,
 9352                /*weak*/ true, noreg);
 9353     __ csetw($res$$Register, Assembler::EQ);
 9354   %}
 9355   ins_pipe(pipe_slow);
 9356 %}
 9357 
 9358 // This pattern is generated automatically from cas.m4.
 9359 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9360 instruct weakCompareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 9361   predicate(needs_acquiring_load_exclusive(n));
 9362   match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
 9363   ins_cost(VOLATILE_REF_COST);
 9364   effect(KILL cr);
 9365   format %{
 9366     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 9367     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9368   %}
 9369   ins_encode %{
 9370     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9371                Assembler::word, /*acquire*/ true, /*release*/ true,
 9372                /*weak*/ true, noreg);
 9373     __ csetw($res$$Register, Assembler::EQ);
 9374   %}
 9375   ins_pipe(pipe_slow);
 9376 %}
 9377 
 9378 // This pattern is generated automatically from cas.m4.
 9379 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 9380 instruct weakCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 9381   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 9382   match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
 9383   ins_cost(VOLATILE_REF_COST);
 9384   effect(KILL cr);
 9385   format %{
 9386     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 9387     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 9388   %}
 9389   ins_encode %{
 9390     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 9391                Assembler::xword, /*acquire*/ true, /*release*/ true,
 9392                /*weak*/ true, noreg);
 9393     __ csetw($res$$Register, Assembler::EQ);
 9394   %}
 9395   ins_pipe(pipe_slow);
 9396 %}
 9397 
 9398 // END This section of the file is automatically generated. Do not edit --------------
 9399 // ---------------------------------------------------------------------
 9400 
 9401 instruct get_and_setI(indirect mem, iRegI newv, iRegINoSp prev) %{
 9402   match(Set prev (GetAndSetI mem newv));
 9403   ins_cost(2 * VOLATILE_REF_COST);
 9404   format %{ "atomic_xchgw  $prev, $newv, [$mem]" %}
 9405   ins_encode %{
 9406     __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9407   %}
 9408   ins_pipe(pipe_serial);
 9409 %}
 9410 
 9411 instruct get_and_setL(indirect mem, iRegL newv, iRegLNoSp prev) %{
 9412   match(Set prev (GetAndSetL mem newv));
 9413   ins_cost(2 * VOLATILE_REF_COST);
 9414   format %{ "atomic_xchg  $prev, $newv, [$mem]" %}
 9415   ins_encode %{
 9416     __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9417   %}
 9418   ins_pipe(pipe_serial);
 9419 %}
 9420 
 9421 instruct get_and_setN(indirect mem, iRegN newv, iRegINoSp prev) %{
 9422   match(Set prev (GetAndSetN mem newv));
 9423   ins_cost(2 * VOLATILE_REF_COST);
 9424   format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
 9425   ins_encode %{
 9426     __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9427   %}
 9428   ins_pipe(pipe_serial);
 9429 %}
 9430 
 9431 instruct get_and_setP(indirect mem, iRegP newv, iRegPNoSp prev) %{
 9432   predicate(n->as_LoadStore()->barrier_data() == 0);
 9433   match(Set prev (GetAndSetP mem newv));
 9434   ins_cost(2 * VOLATILE_REF_COST);
 9435   format %{ "atomic_xchg  $prev, $newv, [$mem]" %}
 9436   ins_encode %{
 9437     __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9438   %}
 9439   ins_pipe(pipe_serial);
 9440 %}
 9441 
 9442 instruct get_and_setIAcq(indirect mem, iRegI newv, iRegINoSp prev) %{
 9443   predicate(needs_acquiring_load_exclusive(n));
 9444   match(Set prev (GetAndSetI mem newv));
 9445   ins_cost(VOLATILE_REF_COST);
 9446   format %{ "atomic_xchgw_acq  $prev, $newv, [$mem]" %}
 9447   ins_encode %{
 9448     __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9449   %}
 9450   ins_pipe(pipe_serial);
 9451 %}
 9452 
 9453 instruct get_and_setLAcq(indirect mem, iRegL newv, iRegLNoSp prev) %{
 9454   predicate(needs_acquiring_load_exclusive(n));
 9455   match(Set prev (GetAndSetL mem newv));
 9456   ins_cost(VOLATILE_REF_COST);
 9457   format %{ "atomic_xchg_acq  $prev, $newv, [$mem]" %}
 9458   ins_encode %{
 9459     __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9460   %}
 9461   ins_pipe(pipe_serial);
 9462 %}
 9463 
 9464 instruct get_and_setNAcq(indirect mem, iRegN newv, iRegINoSp prev) %{
 9465   predicate(needs_acquiring_load_exclusive(n));
 9466   match(Set prev (GetAndSetN mem newv));
 9467   ins_cost(VOLATILE_REF_COST);
 9468   format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
 9469   ins_encode %{
 9470     __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9471   %}
 9472   ins_pipe(pipe_serial);
 9473 %}
 9474 
 9475 instruct get_and_setPAcq(indirect mem, iRegP newv, iRegPNoSp prev) %{
 9476   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 9477   match(Set prev (GetAndSetP mem newv));
 9478   ins_cost(VOLATILE_REF_COST);
 9479   format %{ "atomic_xchg_acq  $prev, $newv, [$mem]" %}
 9480   ins_encode %{
 9481     __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
 9482   %}
 9483   ins_pipe(pipe_serial);
 9484 %}
 9485 
 9486 
 9487 instruct get_and_addL(indirect mem, iRegLNoSp newval, iRegL incr) %{
 9488   match(Set newval (GetAndAddL mem incr));
 9489   ins_cost(2 * VOLATILE_REF_COST + 1);
 9490   format %{ "get_and_addL $newval, [$mem], $incr" %}
 9491   ins_encode %{
 9492     __ atomic_add($newval$$Register, $incr$$Register, as_Register($mem$$base));
 9493   %}
 9494   ins_pipe(pipe_serial);
 9495 %}
 9496 
 9497 instruct get_and_addL_no_res(indirect mem, Universe dummy, iRegL incr) %{
 9498   predicate(n->as_LoadStore()->result_not_used());
 9499   match(Set dummy (GetAndAddL mem incr));
 9500   ins_cost(2 * VOLATILE_REF_COST);
 9501   format %{ "get_and_addL [$mem], $incr" %}
 9502   ins_encode %{
 9503     __ atomic_add(noreg, $incr$$Register, as_Register($mem$$base));
 9504   %}
 9505   ins_pipe(pipe_serial);
 9506 %}
 9507 
 9508 instruct get_and_addLi(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
 9509   match(Set newval (GetAndAddL mem incr));
 9510   ins_cost(2 * VOLATILE_REF_COST + 1);
 9511   format %{ "get_and_addL $newval, [$mem], $incr" %}
 9512   ins_encode %{
 9513     __ atomic_add($newval$$Register, $incr$$constant, as_Register($mem$$base));
 9514   %}
 9515   ins_pipe(pipe_serial);
 9516 %}
 9517 
 9518 instruct get_and_addLi_no_res(indirect mem, Universe dummy, immLAddSub incr) %{
 9519   predicate(n->as_LoadStore()->result_not_used());
 9520   match(Set dummy (GetAndAddL mem incr));
 9521   ins_cost(2 * VOLATILE_REF_COST);
 9522   format %{ "get_and_addL [$mem], $incr" %}
 9523   ins_encode %{
 9524     __ atomic_add(noreg, $incr$$constant, as_Register($mem$$base));
 9525   %}
 9526   ins_pipe(pipe_serial);
 9527 %}
 9528 
 9529 instruct get_and_addI(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
 9530   match(Set newval (GetAndAddI mem incr));
 9531   ins_cost(2 * VOLATILE_REF_COST + 1);
 9532   format %{ "get_and_addI $newval, [$mem], $incr" %}
 9533   ins_encode %{
 9534     __ atomic_addw($newval$$Register, $incr$$Register, as_Register($mem$$base));
 9535   %}
 9536   ins_pipe(pipe_serial);
 9537 %}
 9538 
 9539 instruct get_and_addI_no_res(indirect mem, Universe dummy, iRegIorL2I incr) %{
 9540   predicate(n->as_LoadStore()->result_not_used());
 9541   match(Set dummy (GetAndAddI mem incr));
 9542   ins_cost(2 * VOLATILE_REF_COST);
 9543   format %{ "get_and_addI [$mem], $incr" %}
 9544   ins_encode %{
 9545     __ atomic_addw(noreg, $incr$$Register, as_Register($mem$$base));
 9546   %}
 9547   ins_pipe(pipe_serial);
 9548 %}
 9549 
 9550 instruct get_and_addIi(indirect mem, iRegINoSp newval, immIAddSub incr) %{
 9551   match(Set newval (GetAndAddI mem incr));
 9552   ins_cost(2 * VOLATILE_REF_COST + 1);
 9553   format %{ "get_and_addI $newval, [$mem], $incr" %}
 9554   ins_encode %{
 9555     __ atomic_addw($newval$$Register, $incr$$constant, as_Register($mem$$base));
 9556   %}
 9557   ins_pipe(pipe_serial);
 9558 %}
 9559 
 9560 instruct get_and_addIi_no_res(indirect mem, Universe dummy, immIAddSub incr) %{
 9561   predicate(n->as_LoadStore()->result_not_used());
 9562   match(Set dummy (GetAndAddI mem incr));
 9563   ins_cost(2 * VOLATILE_REF_COST);
 9564   format %{ "get_and_addI [$mem], $incr" %}
 9565   ins_encode %{
 9566     __ atomic_addw(noreg, $incr$$constant, as_Register($mem$$base));
 9567   %}
 9568   ins_pipe(pipe_serial);
 9569 %}
 9570 
 9571 instruct get_and_addLAcq(indirect mem, iRegLNoSp newval, iRegL incr) %{
 9572   predicate(needs_acquiring_load_exclusive(n));
 9573   match(Set newval (GetAndAddL mem incr));
 9574   ins_cost(VOLATILE_REF_COST + 1);
 9575   format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
 9576   ins_encode %{
 9577     __ atomic_addal($newval$$Register, $incr$$Register, as_Register($mem$$base));
 9578   %}
 9579   ins_pipe(pipe_serial);
 9580 %}
 9581 
 9582 instruct get_and_addL_no_resAcq(indirect mem, Universe dummy, iRegL incr) %{
 9583   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
 9584   match(Set dummy (GetAndAddL mem incr));
 9585   ins_cost(VOLATILE_REF_COST);
 9586   format %{ "get_and_addL_acq [$mem], $incr" %}
 9587   ins_encode %{
 9588     __ atomic_addal(noreg, $incr$$Register, as_Register($mem$$base));
 9589   %}
 9590   ins_pipe(pipe_serial);
 9591 %}
 9592 
 9593 instruct get_and_addLiAcq(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
 9594   predicate(needs_acquiring_load_exclusive(n));
 9595   match(Set newval (GetAndAddL mem incr));
 9596   ins_cost(VOLATILE_REF_COST + 1);
 9597   format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
 9598   ins_encode %{
 9599     __ atomic_addal($newval$$Register, $incr$$constant, as_Register($mem$$base));
 9600   %}
 9601   ins_pipe(pipe_serial);
 9602 %}
 9603 
 9604 instruct get_and_addLi_no_resAcq(indirect mem, Universe dummy, immLAddSub incr) %{
 9605   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
 9606   match(Set dummy (GetAndAddL mem incr));
 9607   ins_cost(VOLATILE_REF_COST);
 9608   format %{ "get_and_addL_acq [$mem], $incr" %}
 9609   ins_encode %{
 9610     __ atomic_addal(noreg, $incr$$constant, as_Register($mem$$base));
 9611   %}
 9612   ins_pipe(pipe_serial);
 9613 %}
 9614 
 9615 instruct get_and_addIAcq(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
 9616   predicate(needs_acquiring_load_exclusive(n));
 9617   match(Set newval (GetAndAddI mem incr));
 9618   ins_cost(VOLATILE_REF_COST + 1);
 9619   format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
 9620   ins_encode %{
 9621     __ atomic_addalw($newval$$Register, $incr$$Register, as_Register($mem$$base));
 9622   %}
 9623   ins_pipe(pipe_serial);
 9624 %}
 9625 
 9626 instruct get_and_addI_no_resAcq(indirect mem, Universe dummy, iRegIorL2I incr) %{
 9627   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
 9628   match(Set dummy (GetAndAddI mem incr));
 9629   ins_cost(VOLATILE_REF_COST);
 9630   format %{ "get_and_addI_acq [$mem], $incr" %}
 9631   ins_encode %{
 9632     __ atomic_addalw(noreg, $incr$$Register, as_Register($mem$$base));
 9633   %}
 9634   ins_pipe(pipe_serial);
 9635 %}
 9636 
 9637 instruct get_and_addIiAcq(indirect mem, iRegINoSp newval, immIAddSub incr) %{
 9638   predicate(needs_acquiring_load_exclusive(n));
 9639   match(Set newval (GetAndAddI mem incr));
 9640   ins_cost(VOLATILE_REF_COST + 1);
 9641   format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
 9642   ins_encode %{
 9643     __ atomic_addalw($newval$$Register, $incr$$constant, as_Register($mem$$base));
 9644   %}
 9645   ins_pipe(pipe_serial);
 9646 %}
 9647 
 9648 instruct get_and_addIi_no_resAcq(indirect mem, Universe dummy, immIAddSub incr) %{
 9649   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
 9650   match(Set dummy (GetAndAddI mem incr));
 9651   ins_cost(VOLATILE_REF_COST);
 9652   format %{ "get_and_addI_acq [$mem], $incr" %}
 9653   ins_encode %{
 9654     __ atomic_addalw(noreg, $incr$$constant, as_Register($mem$$base));
 9655   %}
 9656   ins_pipe(pipe_serial);
 9657 %}
 9658 
 9659 // Manifest a CmpU result in an integer register.
 9660 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
 9661 instruct cmpU3_reg_reg(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg flags)
 9662 %{
 9663   match(Set dst (CmpU3 src1 src2));
 9664   effect(KILL flags);
 9665 
 9666   ins_cost(INSN_COST * 3);
 9667   format %{
 9668       "cmpw $src1, $src2\n\t"
 9669       "csetw $dst, ne\n\t"
 9670       "cnegw $dst, lo\t# CmpU3(reg)"
 9671   %}
 9672   ins_encode %{
 9673     __ cmpw($src1$$Register, $src2$$Register);
 9674     __ csetw($dst$$Register, Assembler::NE);
 9675     __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
 9676   %}
 9677 
 9678   ins_pipe(pipe_class_default);
 9679 %}
 9680 
 9681 instruct cmpU3_reg_imm(iRegINoSp dst, iRegI src1, immIAddSub src2, rFlagsReg flags)
 9682 %{
 9683   match(Set dst (CmpU3 src1 src2));
 9684   effect(KILL flags);
 9685 
 9686   ins_cost(INSN_COST * 3);
 9687   format %{
 9688       "subsw zr, $src1, $src2\n\t"
 9689       "csetw $dst, ne\n\t"
 9690       "cnegw $dst, lo\t# CmpU3(imm)"
 9691   %}
 9692   ins_encode %{
 9693     __ subsw(zr, $src1$$Register, (int32_t)$src2$$constant);
 9694     __ csetw($dst$$Register, Assembler::NE);
 9695     __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
 9696   %}
 9697 
 9698   ins_pipe(pipe_class_default);
 9699 %}
 9700 
 9701 // Manifest a CmpUL result in an integer register.
 9702 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
 9703 instruct cmpUL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
 9704 %{
 9705   match(Set dst (CmpUL3 src1 src2));
 9706   effect(KILL flags);
 9707 
 9708   ins_cost(INSN_COST * 3);
 9709   format %{
 9710       "cmp $src1, $src2\n\t"
 9711       "csetw $dst, ne\n\t"
 9712       "cnegw $dst, lo\t# CmpUL3(reg)"
 9713   %}
 9714   ins_encode %{
 9715     __ cmp($src1$$Register, $src2$$Register);
 9716     __ csetw($dst$$Register, Assembler::NE);
 9717     __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
 9718   %}
 9719 
 9720   ins_pipe(pipe_class_default);
 9721 %}
 9722 
 9723 instruct cmpUL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
 9724 %{
 9725   match(Set dst (CmpUL3 src1 src2));
 9726   effect(KILL flags);
 9727 
 9728   ins_cost(INSN_COST * 3);
 9729   format %{
 9730       "subs zr, $src1, $src2\n\t"
 9731       "csetw $dst, ne\n\t"
 9732       "cnegw $dst, lo\t# CmpUL3(imm)"
 9733   %}
 9734   ins_encode %{
 9735     __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
 9736     __ csetw($dst$$Register, Assembler::NE);
 9737     __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
 9738   %}
 9739 
 9740   ins_pipe(pipe_class_default);
 9741 %}
 9742 
 9743 // Manifest a CmpL result in an integer register.
 9744 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
 9745 instruct cmpL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
 9746 %{
 9747   match(Set dst (CmpL3 src1 src2));
 9748   effect(KILL flags);
 9749 
 9750   ins_cost(INSN_COST * 3);
 9751   format %{
 9752       "cmp $src1, $src2\n\t"
 9753       "csetw $dst, ne\n\t"
 9754       "cnegw $dst, lt\t# CmpL3(reg)"
 9755   %}
 9756   ins_encode %{
 9757     __ cmp($src1$$Register, $src2$$Register);
 9758     __ csetw($dst$$Register, Assembler::NE);
 9759     __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
 9760   %}
 9761 
 9762   ins_pipe(pipe_class_default);
 9763 %}
 9764 
 9765 instruct cmpL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
 9766 %{
 9767   match(Set dst (CmpL3 src1 src2));
 9768   effect(KILL flags);
 9769 
 9770   ins_cost(INSN_COST * 3);
 9771   format %{
 9772       "subs zr, $src1, $src2\n\t"
 9773       "csetw $dst, ne\n\t"
 9774       "cnegw $dst, lt\t# CmpL3(imm)"
 9775   %}
 9776   ins_encode %{
 9777     __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
 9778     __ csetw($dst$$Register, Assembler::NE);
 9779     __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
 9780   %}
 9781 
 9782   ins_pipe(pipe_class_default);
 9783 %}
 9784 
 9785 // ============================================================================
 9786 // Conditional Move Instructions
 9787 
 9788 // n.b. we have identical rules for both a signed compare op (cmpOp)
 9789 // and an unsigned compare op (cmpOpU). it would be nice if we could
 9790 // define an op class which merged both inputs and use it to type the
 9791 // argument to a single rule. unfortunatelyt his fails because the
 9792 // opclass does not live up to the COND_INTER interface of its
 9793 // component operands. When the generic code tries to negate the
 9794 // operand it ends up running the generci Machoper::negate method
 9795 // which throws a ShouldNotHappen. So, we have to provide two flavours
 9796 // of each rule, one for a cmpOp and a second for a cmpOpU (sigh).
 9797 
 9798 instruct cmovI_reg_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
 9799   match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
 9800 
 9801   ins_cost(INSN_COST * 2);
 9802   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, int"  %}
 9803 
 9804   ins_encode %{
 9805     __ cselw(as_Register($dst$$reg),
 9806              as_Register($src2$$reg),
 9807              as_Register($src1$$reg),
 9808              (Assembler::Condition)$cmp$$cmpcode);
 9809   %}
 9810 
 9811   ins_pipe(icond_reg_reg);
 9812 %}
 9813 
 9814 instruct cmovUI_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
 9815   match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
 9816 
 9817   ins_cost(INSN_COST * 2);
 9818   format %{ "cselw $dst, $src2, $src1 $cmp\t# unsigned, int"  %}
 9819 
 9820   ins_encode %{
 9821     __ cselw(as_Register($dst$$reg),
 9822              as_Register($src2$$reg),
 9823              as_Register($src1$$reg),
 9824              (Assembler::Condition)$cmp$$cmpcode);
 9825   %}
 9826 
 9827   ins_pipe(icond_reg_reg);
 9828 %}
 9829 
 9830 // special cases where one arg is zero
 9831 
 9832 // n.b. this is selected in preference to the rule above because it
 9833 // avoids loading constant 0 into a source register
 9834 
 9835 // TODO
 9836 // we ought only to be able to cull one of these variants as the ideal
 9837 // transforms ought always to order the zero consistently (to left/right?)
 9838 
 9839 instruct cmovI_zero_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
 9840   match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
 9841 
 9842   ins_cost(INSN_COST * 2);
 9843   format %{ "cselw $dst, $src, zr $cmp\t# signed, int"  %}
 9844 
 9845   ins_encode %{
 9846     __ cselw(as_Register($dst$$reg),
 9847              as_Register($src$$reg),
 9848              zr,
 9849              (Assembler::Condition)$cmp$$cmpcode);
 9850   %}
 9851 
 9852   ins_pipe(icond_reg);
 9853 %}
 9854 
 9855 instruct cmovUI_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
 9856   match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
 9857 
 9858   ins_cost(INSN_COST * 2);
 9859   format %{ "cselw $dst, $src, zr $cmp\t# unsigned, int"  %}
 9860 
 9861   ins_encode %{
 9862     __ cselw(as_Register($dst$$reg),
 9863              as_Register($src$$reg),
 9864              zr,
 9865              (Assembler::Condition)$cmp$$cmpcode);
 9866   %}
 9867 
 9868   ins_pipe(icond_reg);
 9869 %}
 9870 
 9871 instruct cmovI_reg_zero(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
 9872   match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
 9873 
 9874   ins_cost(INSN_COST * 2);
 9875   format %{ "cselw $dst, zr, $src $cmp\t# signed, int"  %}
 9876 
 9877   ins_encode %{
 9878     __ cselw(as_Register($dst$$reg),
 9879              zr,
 9880              as_Register($src$$reg),
 9881              (Assembler::Condition)$cmp$$cmpcode);
 9882   %}
 9883 
 9884   ins_pipe(icond_reg);
 9885 %}
 9886 
 9887 instruct cmovUI_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
 9888   match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
 9889 
 9890   ins_cost(INSN_COST * 2);
 9891   format %{ "cselw $dst, zr, $src $cmp\t# unsigned, int"  %}
 9892 
 9893   ins_encode %{
 9894     __ cselw(as_Register($dst$$reg),
 9895              zr,
 9896              as_Register($src$$reg),
 9897              (Assembler::Condition)$cmp$$cmpcode);
 9898   %}
 9899 
 9900   ins_pipe(icond_reg);
 9901 %}
 9902 
 9903 // special case for creating a boolean 0 or 1
 9904 
 9905 // n.b. this is selected in preference to the rule above because it
 9906 // avoids loading constants 0 and 1 into a source register
 9907 
 9908 instruct cmovI_reg_zero_one(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
 9909   match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
 9910 
 9911   ins_cost(INSN_COST * 2);
 9912   format %{ "csincw $dst, zr, zr $cmp\t# signed, int"  %}
 9913 
 9914   ins_encode %{
 9915     // equivalently
 9916     // cset(as_Register($dst$$reg),
 9917     //      negate_condition((Assembler::Condition)$cmp$$cmpcode));
 9918     __ csincw(as_Register($dst$$reg),
 9919              zr,
 9920              zr,
 9921              (Assembler::Condition)$cmp$$cmpcode);
 9922   %}
 9923 
 9924   ins_pipe(icond_none);
 9925 %}
 9926 
 9927 instruct cmovUI_reg_zero_one(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
 9928   match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
 9929 
 9930   ins_cost(INSN_COST * 2);
 9931   format %{ "csincw $dst, zr, zr $cmp\t# unsigned, int"  %}
 9932 
 9933   ins_encode %{
 9934     // equivalently
 9935     // cset(as_Register($dst$$reg),
 9936     //      negate_condition((Assembler::Condition)$cmp$$cmpcode));
 9937     __ csincw(as_Register($dst$$reg),
 9938              zr,
 9939              zr,
 9940              (Assembler::Condition)$cmp$$cmpcode);
 9941   %}
 9942 
 9943   ins_pipe(icond_none);
 9944 %}
 9945 
 9946 instruct cmovL_reg_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
 9947   match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
 9948 
 9949   ins_cost(INSN_COST * 2);
 9950   format %{ "csel $dst, $src2, $src1 $cmp\t# signed, long"  %}
 9951 
 9952   ins_encode %{
 9953     __ csel(as_Register($dst$$reg),
 9954             as_Register($src2$$reg),
 9955             as_Register($src1$$reg),
 9956             (Assembler::Condition)$cmp$$cmpcode);
 9957   %}
 9958 
 9959   ins_pipe(icond_reg_reg);
 9960 %}
 9961 
 9962 instruct cmovUL_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
 9963   match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
 9964 
 9965   ins_cost(INSN_COST * 2);
 9966   format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, long"  %}
 9967 
 9968   ins_encode %{
 9969     __ csel(as_Register($dst$$reg),
 9970             as_Register($src2$$reg),
 9971             as_Register($src1$$reg),
 9972             (Assembler::Condition)$cmp$$cmpcode);
 9973   %}
 9974 
 9975   ins_pipe(icond_reg_reg);
 9976 %}
 9977 
 9978 // special cases where one arg is zero
 9979 
 9980 instruct cmovL_reg_zero(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
 9981   match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
 9982 
 9983   ins_cost(INSN_COST * 2);
 9984   format %{ "csel $dst, zr, $src $cmp\t# signed, long"  %}
 9985 
 9986   ins_encode %{
 9987     __ csel(as_Register($dst$$reg),
 9988             zr,
 9989             as_Register($src$$reg),
 9990             (Assembler::Condition)$cmp$$cmpcode);
 9991   %}
 9992 
 9993   ins_pipe(icond_reg);
 9994 %}
 9995 
 9996 instruct cmovUL_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
 9997   match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
 9998 
 9999   ins_cost(INSN_COST * 2);
10000   format %{ "csel $dst, zr, $src $cmp\t# unsigned, long"  %}
10001 
10002   ins_encode %{
10003     __ csel(as_Register($dst$$reg),
10004             zr,
10005             as_Register($src$$reg),
10006             (Assembler::Condition)$cmp$$cmpcode);
10007   %}
10008 
10009   ins_pipe(icond_reg);
10010 %}
10011 
10012 instruct cmovL_zero_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
10013   match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
10014 
10015   ins_cost(INSN_COST * 2);
10016   format %{ "csel $dst, $src, zr $cmp\t# signed, long"  %}
10017 
10018   ins_encode %{
10019     __ csel(as_Register($dst$$reg),
10020             as_Register($src$$reg),
10021             zr,
10022             (Assembler::Condition)$cmp$$cmpcode);
10023   %}
10024 
10025   ins_pipe(icond_reg);
10026 %}
10027 
10028 instruct cmovUL_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
10029   match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
10030 
10031   ins_cost(INSN_COST * 2);
10032   format %{ "csel $dst, $src, zr $cmp\t# unsigned, long"  %}
10033 
10034   ins_encode %{
10035     __ csel(as_Register($dst$$reg),
10036             as_Register($src$$reg),
10037             zr,
10038             (Assembler::Condition)$cmp$$cmpcode);
10039   %}
10040 
10041   ins_pipe(icond_reg);
10042 %}
10043 
10044 instruct cmovP_reg_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
10045   match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
10046 
10047   ins_cost(INSN_COST * 2);
10048   format %{ "csel $dst, $src2, $src1 $cmp\t# signed, ptr"  %}
10049 
10050   ins_encode %{
10051     __ csel(as_Register($dst$$reg),
10052             as_Register($src2$$reg),
10053             as_Register($src1$$reg),
10054             (Assembler::Condition)$cmp$$cmpcode);
10055   %}
10056 
10057   ins_pipe(icond_reg_reg);
10058 %}
10059 
10060 instruct cmovUP_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
10061   match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
10062 
10063   ins_cost(INSN_COST * 2);
10064   format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, ptr"  %}
10065 
10066   ins_encode %{
10067     __ csel(as_Register($dst$$reg),
10068             as_Register($src2$$reg),
10069             as_Register($src1$$reg),
10070             (Assembler::Condition)$cmp$$cmpcode);
10071   %}
10072 
10073   ins_pipe(icond_reg_reg);
10074 %}
10075 
10076 // special cases where one arg is zero
10077 
10078 instruct cmovP_reg_zero(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
10079   match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
10080 
10081   ins_cost(INSN_COST * 2);
10082   format %{ "csel $dst, zr, $src $cmp\t# signed, ptr"  %}
10083 
10084   ins_encode %{
10085     __ csel(as_Register($dst$$reg),
10086             zr,
10087             as_Register($src$$reg),
10088             (Assembler::Condition)$cmp$$cmpcode);
10089   %}
10090 
10091   ins_pipe(icond_reg);
10092 %}
10093 
10094 instruct cmovUP_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
10095   match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
10096 
10097   ins_cost(INSN_COST * 2);
10098   format %{ "csel $dst, zr, $src $cmp\t# unsigned, ptr"  %}
10099 
10100   ins_encode %{
10101     __ csel(as_Register($dst$$reg),
10102             zr,
10103             as_Register($src$$reg),
10104             (Assembler::Condition)$cmp$$cmpcode);
10105   %}
10106 
10107   ins_pipe(icond_reg);
10108 %}
10109 
10110 instruct cmovP_zero_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
10111   match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
10112 
10113   ins_cost(INSN_COST * 2);
10114   format %{ "csel $dst, $src, zr $cmp\t# signed, ptr"  %}
10115 
10116   ins_encode %{
10117     __ csel(as_Register($dst$$reg),
10118             as_Register($src$$reg),
10119             zr,
10120             (Assembler::Condition)$cmp$$cmpcode);
10121   %}
10122 
10123   ins_pipe(icond_reg);
10124 %}
10125 
10126 instruct cmovUP_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
10127   match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
10128 
10129   ins_cost(INSN_COST * 2);
10130   format %{ "csel $dst, $src, zr $cmp\t# unsigned, ptr"  %}
10131 
10132   ins_encode %{
10133     __ csel(as_Register($dst$$reg),
10134             as_Register($src$$reg),
10135             zr,
10136             (Assembler::Condition)$cmp$$cmpcode);
10137   %}
10138 
10139   ins_pipe(icond_reg);
10140 %}
10141 
10142 instruct cmovN_reg_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
10143   match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
10144 
10145   ins_cost(INSN_COST * 2);
10146   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr"  %}
10147 
10148   ins_encode %{
10149     __ cselw(as_Register($dst$$reg),
10150              as_Register($src2$$reg),
10151              as_Register($src1$$reg),
10152              (Assembler::Condition)$cmp$$cmpcode);
10153   %}
10154 
10155   ins_pipe(icond_reg_reg);
10156 %}
10157 
10158 instruct cmovUN_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
10159   match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
10160 
10161   ins_cost(INSN_COST * 2);
10162   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr"  %}
10163 
10164   ins_encode %{
10165     __ cselw(as_Register($dst$$reg),
10166              as_Register($src2$$reg),
10167              as_Register($src1$$reg),
10168              (Assembler::Condition)$cmp$$cmpcode);
10169   %}
10170 
10171   ins_pipe(icond_reg_reg);
10172 %}
10173 
10174 // special cases where one arg is zero
10175 
10176 instruct cmovN_reg_zero(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
10177   match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
10178 
10179   ins_cost(INSN_COST * 2);
10180   format %{ "cselw $dst, zr, $src $cmp\t# signed, compressed ptr"  %}
10181 
10182   ins_encode %{
10183     __ cselw(as_Register($dst$$reg),
10184              zr,
10185              as_Register($src$$reg),
10186              (Assembler::Condition)$cmp$$cmpcode);
10187   %}
10188 
10189   ins_pipe(icond_reg);
10190 %}
10191 
10192 instruct cmovUN_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
10193   match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
10194 
10195   ins_cost(INSN_COST * 2);
10196   format %{ "cselw $dst, zr, $src $cmp\t# unsigned, compressed ptr"  %}
10197 
10198   ins_encode %{
10199     __ cselw(as_Register($dst$$reg),
10200              zr,
10201              as_Register($src$$reg),
10202              (Assembler::Condition)$cmp$$cmpcode);
10203   %}
10204 
10205   ins_pipe(icond_reg);
10206 %}
10207 
10208 instruct cmovN_zero_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
10209   match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
10210 
10211   ins_cost(INSN_COST * 2);
10212   format %{ "cselw $dst, $src, zr $cmp\t# signed, compressed ptr"  %}
10213 
10214   ins_encode %{
10215     __ cselw(as_Register($dst$$reg),
10216              as_Register($src$$reg),
10217              zr,
10218              (Assembler::Condition)$cmp$$cmpcode);
10219   %}
10220 
10221   ins_pipe(icond_reg);
10222 %}
10223 
10224 instruct cmovUN_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
10225   match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
10226 
10227   ins_cost(INSN_COST * 2);
10228   format %{ "cselw $dst, $src, zr $cmp\t# unsigned, compressed ptr"  %}
10229 
10230   ins_encode %{
10231     __ cselw(as_Register($dst$$reg),
10232              as_Register($src$$reg),
10233              zr,
10234              (Assembler::Condition)$cmp$$cmpcode);
10235   %}
10236 
10237   ins_pipe(icond_reg);
10238 %}
10239 
10240 instruct cmovF_reg(cmpOp cmp, rFlagsReg cr, vRegF dst, vRegF src1,  vRegF src2)
10241 %{
10242   match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
10243 
10244   ins_cost(INSN_COST * 3);
10245 
10246   format %{ "fcsels $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
10247   ins_encode %{
10248     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10249     __ fcsels(as_FloatRegister($dst$$reg),
10250               as_FloatRegister($src2$$reg),
10251               as_FloatRegister($src1$$reg),
10252               cond);
10253   %}
10254 
10255   ins_pipe(fp_cond_reg_reg_s);
10256 %}
10257 
10258 instruct cmovUF_reg(cmpOpU cmp, rFlagsRegU cr, vRegF dst, vRegF src1,  vRegF src2)
10259 %{
10260   match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
10261 
10262   ins_cost(INSN_COST * 3);
10263 
10264   format %{ "fcsels $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
10265   ins_encode %{
10266     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10267     __ fcsels(as_FloatRegister($dst$$reg),
10268               as_FloatRegister($src2$$reg),
10269               as_FloatRegister($src1$$reg),
10270               cond);
10271   %}
10272 
10273   ins_pipe(fp_cond_reg_reg_s);
10274 %}
10275 
10276 instruct cmovD_reg(cmpOp cmp, rFlagsReg cr, vRegD dst, vRegD src1,  vRegD src2)
10277 %{
10278   match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
10279 
10280   ins_cost(INSN_COST * 3);
10281 
10282   format %{ "fcseld $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
10283   ins_encode %{
10284     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10285     __ fcseld(as_FloatRegister($dst$$reg),
10286               as_FloatRegister($src2$$reg),
10287               as_FloatRegister($src1$$reg),
10288               cond);
10289   %}
10290 
10291   ins_pipe(fp_cond_reg_reg_d);
10292 %}
10293 
10294 instruct cmovUD_reg(cmpOpU cmp, rFlagsRegU cr, vRegD dst, vRegD src1,  vRegD src2)
10295 %{
10296   match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
10297 
10298   ins_cost(INSN_COST * 3);
10299 
10300   format %{ "fcseld $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
10301   ins_encode %{
10302     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10303     __ fcseld(as_FloatRegister($dst$$reg),
10304               as_FloatRegister($src2$$reg),
10305               as_FloatRegister($src1$$reg),
10306               cond);
10307   %}
10308 
10309   ins_pipe(fp_cond_reg_reg_d);
10310 %}
10311 
10312 // ============================================================================
10313 // Arithmetic Instructions
10314 //
10315 
10316 // Integer Addition
10317 
10318 // TODO
10319 // these currently employ operations which do not set CR and hence are
10320 // not flagged as killing CR but we would like to isolate the cases
10321 // where we want to set flags from those where we don't. need to work
10322 // out how to do that.
10323 
10324 instruct addI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10325   match(Set dst (AddI src1 src2));
10326 
10327   ins_cost(INSN_COST);
10328   format %{ "addw  $dst, $src1, $src2" %}
10329 
10330   ins_encode %{
10331     __ addw(as_Register($dst$$reg),
10332             as_Register($src1$$reg),
10333             as_Register($src2$$reg));
10334   %}
10335 
10336   ins_pipe(ialu_reg_reg);
10337 %}
10338 
10339 instruct addI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10340   match(Set dst (AddI src1 src2));
10341 
10342   ins_cost(INSN_COST);
10343   format %{ "addw $dst, $src1, $src2" %}
10344 
10345   // use opcode to indicate that this is an add not a sub
10346   opcode(0x0);
10347 
10348   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10349 
10350   ins_pipe(ialu_reg_imm);
10351 %}
10352 
10353 instruct addI_reg_imm_i2l(iRegINoSp dst, iRegL src1, immIAddSub src2) %{
10354   match(Set dst (AddI (ConvL2I src1) src2));
10355 
10356   ins_cost(INSN_COST);
10357   format %{ "addw $dst, $src1, $src2" %}
10358 
10359   // use opcode to indicate that this is an add not a sub
10360   opcode(0x0);
10361 
10362   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10363 
10364   ins_pipe(ialu_reg_imm);
10365 %}
10366 
10367 // Pointer Addition
10368 instruct addP_reg_reg(iRegPNoSp dst, iRegP src1, iRegL src2) %{
10369   match(Set dst (AddP src1 src2));
10370 
10371   ins_cost(INSN_COST);
10372   format %{ "add $dst, $src1, $src2\t# ptr" %}
10373 
10374   ins_encode %{
10375     __ add(as_Register($dst$$reg),
10376            as_Register($src1$$reg),
10377            as_Register($src2$$reg));
10378   %}
10379 
10380   ins_pipe(ialu_reg_reg);
10381 %}
10382 
10383 instruct addP_reg_reg_ext(iRegPNoSp dst, iRegP src1, iRegIorL2I src2) %{
10384   match(Set dst (AddP src1 (ConvI2L src2)));
10385 
10386   ins_cost(1.9 * INSN_COST);
10387   format %{ "add $dst, $src1, $src2, sxtw\t# ptr" %}
10388 
10389   ins_encode %{
10390     __ add(as_Register($dst$$reg),
10391            as_Register($src1$$reg),
10392            as_Register($src2$$reg), ext::sxtw);
10393   %}
10394 
10395   ins_pipe(ialu_reg_reg);
10396 %}
10397 
10398 instruct addP_reg_reg_lsl(iRegPNoSp dst, iRegP src1, iRegL src2, immIScale scale) %{
10399   match(Set dst (AddP src1 (LShiftL src2 scale)));
10400 
10401   ins_cost(1.9 * INSN_COST);
10402   format %{ "add $dst, $src1, $src2, LShiftL $scale\t# ptr" %}
10403 
10404   ins_encode %{
10405     __ lea(as_Register($dst$$reg),
10406            Address(as_Register($src1$$reg), as_Register($src2$$reg),
10407                    Address::lsl($scale$$constant)));
10408   %}
10409 
10410   ins_pipe(ialu_reg_reg_shift);
10411 %}
10412 
10413 instruct addP_reg_reg_ext_shift(iRegPNoSp dst, iRegP src1, iRegIorL2I src2, immIScale scale) %{
10414   match(Set dst (AddP src1 (LShiftL (ConvI2L src2) scale)));
10415 
10416   ins_cost(1.9 * INSN_COST);
10417   format %{ "add $dst, $src1, $src2, I2L $scale\t# ptr" %}
10418 
10419   ins_encode %{
10420     __ lea(as_Register($dst$$reg),
10421            Address(as_Register($src1$$reg), as_Register($src2$$reg),
10422                    Address::sxtw($scale$$constant)));
10423   %}
10424 
10425   ins_pipe(ialu_reg_reg_shift);
10426 %}
10427 
10428 instruct lshift_ext(iRegLNoSp dst, iRegIorL2I src, immI scale, rFlagsReg cr) %{
10429   match(Set dst (LShiftL (ConvI2L src) scale));
10430 
10431   ins_cost(INSN_COST);
10432   format %{ "sbfiz $dst, $src, $scale & 63, -$scale & 63\t" %}
10433 
10434   ins_encode %{
10435     __ sbfiz(as_Register($dst$$reg),
10436           as_Register($src$$reg),
10437           $scale$$constant & 63, MIN2(32, (int)((-$scale$$constant) & 63)));
10438   %}
10439 
10440   ins_pipe(ialu_reg_shift);
10441 %}
10442 
10443 // Pointer Immediate Addition
10444 // n.b. this needs to be more expensive than using an indirect memory
10445 // operand
10446 instruct addP_reg_imm(iRegPNoSp dst, iRegP src1, immLAddSub src2) %{
10447   match(Set dst (AddP src1 src2));
10448 
10449   ins_cost(INSN_COST);
10450   format %{ "add $dst, $src1, $src2\t# ptr" %}
10451 
10452   // use opcode to indicate that this is an add not a sub
10453   opcode(0x0);
10454 
10455   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10456 
10457   ins_pipe(ialu_reg_imm);
10458 %}
10459 
10460 // Long Addition
10461 instruct addL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10462 
10463   match(Set dst (AddL src1 src2));
10464 
10465   ins_cost(INSN_COST);
10466   format %{ "add  $dst, $src1, $src2" %}
10467 
10468   ins_encode %{
10469     __ add(as_Register($dst$$reg),
10470            as_Register($src1$$reg),
10471            as_Register($src2$$reg));
10472   %}
10473 
10474   ins_pipe(ialu_reg_reg);
10475 %}
10476 
10477 // No constant pool entries requiredLong Immediate Addition.
10478 instruct addL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10479   match(Set dst (AddL src1 src2));
10480 
10481   ins_cost(INSN_COST);
10482   format %{ "add $dst, $src1, $src2" %}
10483 
10484   // use opcode to indicate that this is an add not a sub
10485   opcode(0x0);
10486 
10487   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10488 
10489   ins_pipe(ialu_reg_imm);
10490 %}
10491 
10492 // Integer Subtraction
10493 instruct subI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10494   match(Set dst (SubI src1 src2));
10495 
10496   ins_cost(INSN_COST);
10497   format %{ "subw  $dst, $src1, $src2" %}
10498 
10499   ins_encode %{
10500     __ subw(as_Register($dst$$reg),
10501             as_Register($src1$$reg),
10502             as_Register($src2$$reg));
10503   %}
10504 
10505   ins_pipe(ialu_reg_reg);
10506 %}
10507 
10508 // Immediate Subtraction
10509 instruct subI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10510   match(Set dst (SubI src1 src2));
10511 
10512   ins_cost(INSN_COST);
10513   format %{ "subw $dst, $src1, $src2" %}
10514 
10515   // use opcode to indicate that this is a sub not an add
10516   opcode(0x1);
10517 
10518   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10519 
10520   ins_pipe(ialu_reg_imm);
10521 %}
10522 
10523 // Long Subtraction
10524 instruct subL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10525 
10526   match(Set dst (SubL src1 src2));
10527 
10528   ins_cost(INSN_COST);
10529   format %{ "sub  $dst, $src1, $src2" %}
10530 
10531   ins_encode %{
10532     __ sub(as_Register($dst$$reg),
10533            as_Register($src1$$reg),
10534            as_Register($src2$$reg));
10535   %}
10536 
10537   ins_pipe(ialu_reg_reg);
10538 %}
10539 
10540 // No constant pool entries requiredLong Immediate Subtraction.
10541 instruct subL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10542   match(Set dst (SubL src1 src2));
10543 
10544   ins_cost(INSN_COST);
10545   format %{ "sub$dst, $src1, $src2" %}
10546 
10547   // use opcode to indicate that this is a sub not an add
10548   opcode(0x1);
10549 
10550   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10551 
10552   ins_pipe(ialu_reg_imm);
10553 %}
10554 
10555 // Integer Negation (special case for sub)
10556 
10557 instruct negI_reg(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr) %{
10558   match(Set dst (SubI zero src));
10559 
10560   ins_cost(INSN_COST);
10561   format %{ "negw $dst, $src\t# int" %}
10562 
10563   ins_encode %{
10564     __ negw(as_Register($dst$$reg),
10565             as_Register($src$$reg));
10566   %}
10567 
10568   ins_pipe(ialu_reg);
10569 %}
10570 
10571 // Long Negation
10572 
10573 instruct negL_reg(iRegLNoSp dst, iRegL src, immL0 zero, rFlagsReg cr) %{
10574   match(Set dst (SubL zero src));
10575 
10576   ins_cost(INSN_COST);
10577   format %{ "neg $dst, $src\t# long" %}
10578 
10579   ins_encode %{
10580     __ neg(as_Register($dst$$reg),
10581            as_Register($src$$reg));
10582   %}
10583 
10584   ins_pipe(ialu_reg);
10585 %}
10586 
10587 // Integer Multiply
10588 
10589 instruct mulI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10590   match(Set dst (MulI src1 src2));
10591 
10592   ins_cost(INSN_COST * 3);
10593   format %{ "mulw  $dst, $src1, $src2" %}
10594 
10595   ins_encode %{
10596     __ mulw(as_Register($dst$$reg),
10597             as_Register($src1$$reg),
10598             as_Register($src2$$reg));
10599   %}
10600 
10601   ins_pipe(imul_reg_reg);
10602 %}
10603 
10604 instruct smulI(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10605   match(Set dst (MulL (ConvI2L src1) (ConvI2L src2)));
10606 
10607   ins_cost(INSN_COST * 3);
10608   format %{ "smull  $dst, $src1, $src2" %}
10609 
10610   ins_encode %{
10611     __ smull(as_Register($dst$$reg),
10612              as_Register($src1$$reg),
10613              as_Register($src2$$reg));
10614   %}
10615 
10616   ins_pipe(imul_reg_reg);
10617 %}
10618 
10619 // Long Multiply
10620 
10621 instruct mulL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10622   match(Set dst (MulL src1 src2));
10623 
10624   ins_cost(INSN_COST * 5);
10625   format %{ "mul  $dst, $src1, $src2" %}
10626 
10627   ins_encode %{
10628     __ mul(as_Register($dst$$reg),
10629            as_Register($src1$$reg),
10630            as_Register($src2$$reg));
10631   %}
10632 
10633   ins_pipe(lmul_reg_reg);
10634 %}
10635 
10636 instruct mulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10637 %{
10638   match(Set dst (MulHiL src1 src2));
10639 
10640   ins_cost(INSN_COST * 7);
10641   format %{ "smulh   $dst, $src1, $src2\t# mulhi" %}
10642 
10643   ins_encode %{
10644     __ smulh(as_Register($dst$$reg),
10645              as_Register($src1$$reg),
10646              as_Register($src2$$reg));
10647   %}
10648 
10649   ins_pipe(lmul_reg_reg);
10650 %}
10651 
10652 instruct umulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10653 %{
10654   match(Set dst (UMulHiL src1 src2));
10655 
10656   ins_cost(INSN_COST * 7);
10657   format %{ "umulh   $dst, $src1, $src2\t# umulhi" %}
10658 
10659   ins_encode %{
10660     __ umulh(as_Register($dst$$reg),
10661              as_Register($src1$$reg),
10662              as_Register($src2$$reg));
10663   %}
10664 
10665   ins_pipe(lmul_reg_reg);
10666 %}
10667 
10668 // Combined Integer Multiply & Add/Sub
10669 
10670 instruct maddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10671   match(Set dst (AddI src3 (MulI src1 src2)));
10672 
10673   ins_cost(INSN_COST * 3);
10674   format %{ "madd  $dst, $src1, $src2, $src3" %}
10675 
10676   ins_encode %{
10677     __ maddw(as_Register($dst$$reg),
10678              as_Register($src1$$reg),
10679              as_Register($src2$$reg),
10680              as_Register($src3$$reg));
10681   %}
10682 
10683   ins_pipe(imac_reg_reg);
10684 %}
10685 
10686 instruct msubI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10687   match(Set dst (SubI src3 (MulI src1 src2)));
10688 
10689   ins_cost(INSN_COST * 3);
10690   format %{ "msub  $dst, $src1, $src2, $src3" %}
10691 
10692   ins_encode %{
10693     __ msubw(as_Register($dst$$reg),
10694              as_Register($src1$$reg),
10695              as_Register($src2$$reg),
10696              as_Register($src3$$reg));
10697   %}
10698 
10699   ins_pipe(imac_reg_reg);
10700 %}
10701 
10702 // Combined Integer Multiply & Neg
10703 
10704 instruct mnegI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI0 zero) %{
10705   match(Set dst (MulI (SubI zero src1) src2));
10706 
10707   ins_cost(INSN_COST * 3);
10708   format %{ "mneg  $dst, $src1, $src2" %}
10709 
10710   ins_encode %{
10711     __ mnegw(as_Register($dst$$reg),
10712              as_Register($src1$$reg),
10713              as_Register($src2$$reg));
10714   %}
10715 
10716   ins_pipe(imac_reg_reg);
10717 %}
10718 
10719 // Combined Long Multiply & Add/Sub
10720 
10721 instruct maddL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10722   match(Set dst (AddL src3 (MulL src1 src2)));
10723 
10724   ins_cost(INSN_COST * 5);
10725   format %{ "madd  $dst, $src1, $src2, $src3" %}
10726 
10727   ins_encode %{
10728     __ madd(as_Register($dst$$reg),
10729             as_Register($src1$$reg),
10730             as_Register($src2$$reg),
10731             as_Register($src3$$reg));
10732   %}
10733 
10734   ins_pipe(lmac_reg_reg);
10735 %}
10736 
10737 instruct msubL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10738   match(Set dst (SubL src3 (MulL src1 src2)));
10739 
10740   ins_cost(INSN_COST * 5);
10741   format %{ "msub  $dst, $src1, $src2, $src3" %}
10742 
10743   ins_encode %{
10744     __ msub(as_Register($dst$$reg),
10745             as_Register($src1$$reg),
10746             as_Register($src2$$reg),
10747             as_Register($src3$$reg));
10748   %}
10749 
10750   ins_pipe(lmac_reg_reg);
10751 %}
10752 
10753 // Combined Long Multiply & Neg
10754 
10755 instruct mnegL(iRegLNoSp dst, iRegL src1, iRegL src2, immL0 zero) %{
10756   match(Set dst (MulL (SubL zero src1) src2));
10757 
10758   ins_cost(INSN_COST * 5);
10759   format %{ "mneg  $dst, $src1, $src2" %}
10760 
10761   ins_encode %{
10762     __ mneg(as_Register($dst$$reg),
10763             as_Register($src1$$reg),
10764             as_Register($src2$$reg));
10765   %}
10766 
10767   ins_pipe(lmac_reg_reg);
10768 %}
10769 
10770 // Combine Integer Signed Multiply & Add/Sub/Neg Long
10771 
10772 instruct smaddL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10773   match(Set dst (AddL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10774 
10775   ins_cost(INSN_COST * 3);
10776   format %{ "smaddl  $dst, $src1, $src2, $src3" %}
10777 
10778   ins_encode %{
10779     __ smaddl(as_Register($dst$$reg),
10780               as_Register($src1$$reg),
10781               as_Register($src2$$reg),
10782               as_Register($src3$$reg));
10783   %}
10784 
10785   ins_pipe(imac_reg_reg);
10786 %}
10787 
10788 instruct smsubL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10789   match(Set dst (SubL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10790 
10791   ins_cost(INSN_COST * 3);
10792   format %{ "smsubl  $dst, $src1, $src2, $src3" %}
10793 
10794   ins_encode %{
10795     __ smsubl(as_Register($dst$$reg),
10796               as_Register($src1$$reg),
10797               as_Register($src2$$reg),
10798               as_Register($src3$$reg));
10799   %}
10800 
10801   ins_pipe(imac_reg_reg);
10802 %}
10803 
10804 instruct smnegL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, immL0 zero) %{
10805   match(Set dst (MulL (SubL zero (ConvI2L src1)) (ConvI2L src2)));
10806 
10807   ins_cost(INSN_COST * 3);
10808   format %{ "smnegl  $dst, $src1, $src2" %}
10809 
10810   ins_encode %{
10811     __ smnegl(as_Register($dst$$reg),
10812               as_Register($src1$$reg),
10813               as_Register($src2$$reg));
10814   %}
10815 
10816   ins_pipe(imac_reg_reg);
10817 %}
10818 
10819 // Combined Multiply-Add Shorts into Integer (dst = src1 * src2 + src3 * src4)
10820 
10821 instruct muladdS2I(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3, iRegIorL2I src4) %{
10822   match(Set dst (MulAddS2I (Binary src1 src2) (Binary src3 src4)));
10823 
10824   ins_cost(INSN_COST * 5);
10825   format %{ "mulw  rscratch1, $src1, $src2\n\t"
10826             "maddw $dst, $src3, $src4, rscratch1" %}
10827 
10828   ins_encode %{
10829     __ mulw(rscratch1, as_Register($src1$$reg), as_Register($src2$$reg));
10830     __ maddw(as_Register($dst$$reg), as_Register($src3$$reg), as_Register($src4$$reg), rscratch1); %}
10831 
10832   ins_pipe(imac_reg_reg);
10833 %}
10834 
10835 // Integer Divide
10836 
10837 instruct divI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10838   match(Set dst (DivI src1 src2));
10839 
10840   ins_cost(INSN_COST * 19);
10841   format %{ "sdivw  $dst, $src1, $src2" %}
10842 
10843   ins_encode(aarch64_enc_divw(dst, src1, src2));
10844   ins_pipe(idiv_reg_reg);
10845 %}
10846 
10847 // Long Divide
10848 
10849 instruct divL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10850   match(Set dst (DivL src1 src2));
10851 
10852   ins_cost(INSN_COST * 35);
10853   format %{ "sdiv   $dst, $src1, $src2" %}
10854 
10855   ins_encode(aarch64_enc_div(dst, src1, src2));
10856   ins_pipe(ldiv_reg_reg);
10857 %}
10858 
10859 // Integer Remainder
10860 
10861 instruct modI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10862   match(Set dst (ModI src1 src2));
10863 
10864   ins_cost(INSN_COST * 22);
10865   format %{ "sdivw  rscratch1, $src1, $src2\n\t"
10866             "msubw  $dst, rscratch1, $src2, $src1" %}
10867 
10868   ins_encode(aarch64_enc_modw(dst, src1, src2));
10869   ins_pipe(idiv_reg_reg);
10870 %}
10871 
10872 // Long Remainder
10873 
10874 instruct modL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10875   match(Set dst (ModL src1 src2));
10876 
10877   ins_cost(INSN_COST * 38);
10878   format %{ "sdiv   rscratch1, $src1, $src2\n"
10879             "msub   $dst, rscratch1, $src2, $src1" %}
10880 
10881   ins_encode(aarch64_enc_mod(dst, src1, src2));
10882   ins_pipe(ldiv_reg_reg);
10883 %}
10884 
10885 // Unsigned Integer Divide
10886 
10887 instruct UdivI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10888   match(Set dst (UDivI src1 src2));
10889 
10890   ins_cost(INSN_COST * 19);
10891   format %{ "udivw  $dst, $src1, $src2" %}
10892 
10893   ins_encode %{
10894     __ udivw($dst$$Register, $src1$$Register, $src2$$Register);
10895   %}
10896 
10897   ins_pipe(idiv_reg_reg);
10898 %}
10899 
10900 //  Unsigned Long Divide
10901 
10902 instruct UdivL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10903   match(Set dst (UDivL src1 src2));
10904 
10905   ins_cost(INSN_COST * 35);
10906   format %{ "udiv   $dst, $src1, $src2" %}
10907 
10908   ins_encode %{
10909     __ udiv($dst$$Register, $src1$$Register, $src2$$Register);
10910   %}
10911 
10912   ins_pipe(ldiv_reg_reg);
10913 %}
10914 
10915 // Unsigned Integer Remainder
10916 
10917 instruct UmodI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10918   match(Set dst (UModI src1 src2));
10919 
10920   ins_cost(INSN_COST * 22);
10921   format %{ "udivw  rscratch1, $src1, $src2\n\t"
10922             "msubw  $dst, rscratch1, $src2, $src1" %}
10923 
10924   ins_encode %{
10925     __ udivw(rscratch1, $src1$$Register, $src2$$Register);
10926     __ msubw($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
10927   %}
10928 
10929   ins_pipe(idiv_reg_reg);
10930 %}
10931 
10932 // Unsigned Long Remainder
10933 
10934 instruct UModL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10935   match(Set dst (UModL src1 src2));
10936 
10937   ins_cost(INSN_COST * 38);
10938   format %{ "udiv   rscratch1, $src1, $src2\n"
10939             "msub   $dst, rscratch1, $src2, $src1" %}
10940 
10941   ins_encode %{
10942     __ udiv(rscratch1, $src1$$Register, $src2$$Register);
10943     __ msub($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
10944   %}
10945 
10946   ins_pipe(ldiv_reg_reg);
10947 %}
10948 
10949 // Integer Shifts
10950 
10951 // Shift Left Register
10952 instruct lShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10953   match(Set dst (LShiftI src1 src2));
10954 
10955   ins_cost(INSN_COST * 2);
10956   format %{ "lslvw  $dst, $src1, $src2" %}
10957 
10958   ins_encode %{
10959     __ lslvw(as_Register($dst$$reg),
10960              as_Register($src1$$reg),
10961              as_Register($src2$$reg));
10962   %}
10963 
10964   ins_pipe(ialu_reg_reg_vshift);
10965 %}
10966 
10967 // Shift Left Immediate
10968 instruct lShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10969   match(Set dst (LShiftI src1 src2));
10970 
10971   ins_cost(INSN_COST);
10972   format %{ "lslw $dst, $src1, ($src2 & 0x1f)" %}
10973 
10974   ins_encode %{
10975     __ lslw(as_Register($dst$$reg),
10976             as_Register($src1$$reg),
10977             $src2$$constant & 0x1f);
10978   %}
10979 
10980   ins_pipe(ialu_reg_shift);
10981 %}
10982 
10983 // Shift Right Logical Register
10984 instruct urShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10985   match(Set dst (URShiftI src1 src2));
10986 
10987   ins_cost(INSN_COST * 2);
10988   format %{ "lsrvw  $dst, $src1, $src2" %}
10989 
10990   ins_encode %{
10991     __ lsrvw(as_Register($dst$$reg),
10992              as_Register($src1$$reg),
10993              as_Register($src2$$reg));
10994   %}
10995 
10996   ins_pipe(ialu_reg_reg_vshift);
10997 %}
10998 
10999 // Shift Right Logical Immediate
11000 instruct urShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
11001   match(Set dst (URShiftI src1 src2));
11002 
11003   ins_cost(INSN_COST);
11004   format %{ "lsrw $dst, $src1, ($src2 & 0x1f)" %}
11005 
11006   ins_encode %{
11007     __ lsrw(as_Register($dst$$reg),
11008             as_Register($src1$$reg),
11009             $src2$$constant & 0x1f);
11010   %}
11011 
11012   ins_pipe(ialu_reg_shift);
11013 %}
11014 
11015 // Shift Right Arithmetic Register
11016 instruct rShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11017   match(Set dst (RShiftI src1 src2));
11018 
11019   ins_cost(INSN_COST * 2);
11020   format %{ "asrvw  $dst, $src1, $src2" %}
11021 
11022   ins_encode %{
11023     __ asrvw(as_Register($dst$$reg),
11024              as_Register($src1$$reg),
11025              as_Register($src2$$reg));
11026   %}
11027 
11028   ins_pipe(ialu_reg_reg_vshift);
11029 %}
11030 
11031 // Shift Right Arithmetic Immediate
11032 instruct rShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
11033   match(Set dst (RShiftI src1 src2));
11034 
11035   ins_cost(INSN_COST);
11036   format %{ "asrw $dst, $src1, ($src2 & 0x1f)" %}
11037 
11038   ins_encode %{
11039     __ asrw(as_Register($dst$$reg),
11040             as_Register($src1$$reg),
11041             $src2$$constant & 0x1f);
11042   %}
11043 
11044   ins_pipe(ialu_reg_shift);
11045 %}
11046 
11047 // Combined Int Mask and Right Shift (using UBFM)
11048 // TODO
11049 
11050 // Long Shifts
11051 
11052 // Shift Left Register
11053 instruct lShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
11054   match(Set dst (LShiftL src1 src2));
11055 
11056   ins_cost(INSN_COST * 2);
11057   format %{ "lslv  $dst, $src1, $src2" %}
11058 
11059   ins_encode %{
11060     __ lslv(as_Register($dst$$reg),
11061             as_Register($src1$$reg),
11062             as_Register($src2$$reg));
11063   %}
11064 
11065   ins_pipe(ialu_reg_reg_vshift);
11066 %}
11067 
11068 // Shift Left Immediate
11069 instruct lShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
11070   match(Set dst (LShiftL src1 src2));
11071 
11072   ins_cost(INSN_COST);
11073   format %{ "lsl $dst, $src1, ($src2 & 0x3f)" %}
11074 
11075   ins_encode %{
11076     __ lsl(as_Register($dst$$reg),
11077             as_Register($src1$$reg),
11078             $src2$$constant & 0x3f);
11079   %}
11080 
11081   ins_pipe(ialu_reg_shift);
11082 %}
11083 
11084 // Shift Right Logical Register
11085 instruct urShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
11086   match(Set dst (URShiftL src1 src2));
11087 
11088   ins_cost(INSN_COST * 2);
11089   format %{ "lsrv  $dst, $src1, $src2" %}
11090 
11091   ins_encode %{
11092     __ lsrv(as_Register($dst$$reg),
11093             as_Register($src1$$reg),
11094             as_Register($src2$$reg));
11095   %}
11096 
11097   ins_pipe(ialu_reg_reg_vshift);
11098 %}
11099 
11100 // Shift Right Logical Immediate
11101 instruct urShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
11102   match(Set dst (URShiftL src1 src2));
11103 
11104   ins_cost(INSN_COST);
11105   format %{ "lsr $dst, $src1, ($src2 & 0x3f)" %}
11106 
11107   ins_encode %{
11108     __ lsr(as_Register($dst$$reg),
11109            as_Register($src1$$reg),
11110            $src2$$constant & 0x3f);
11111   %}
11112 
11113   ins_pipe(ialu_reg_shift);
11114 %}
11115 
11116 // A special-case pattern for card table stores.
11117 instruct urShiftP_reg_imm(iRegLNoSp dst, iRegP src1, immI src2) %{
11118   match(Set dst (URShiftL (CastP2X src1) src2));
11119 
11120   ins_cost(INSN_COST);
11121   format %{ "lsr $dst, p2x($src1), ($src2 & 0x3f)" %}
11122 
11123   ins_encode %{
11124     __ lsr(as_Register($dst$$reg),
11125            as_Register($src1$$reg),
11126            $src2$$constant & 0x3f);
11127   %}
11128 
11129   ins_pipe(ialu_reg_shift);
11130 %}
11131 
11132 // Shift Right Arithmetic Register
11133 instruct rShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
11134   match(Set dst (RShiftL src1 src2));
11135 
11136   ins_cost(INSN_COST * 2);
11137   format %{ "asrv  $dst, $src1, $src2" %}
11138 
11139   ins_encode %{
11140     __ asrv(as_Register($dst$$reg),
11141             as_Register($src1$$reg),
11142             as_Register($src2$$reg));
11143   %}
11144 
11145   ins_pipe(ialu_reg_reg_vshift);
11146 %}
11147 
11148 // Shift Right Arithmetic Immediate
11149 instruct rShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
11150   match(Set dst (RShiftL src1 src2));
11151 
11152   ins_cost(INSN_COST);
11153   format %{ "asr $dst, $src1, ($src2 & 0x3f)" %}
11154 
11155   ins_encode %{
11156     __ asr(as_Register($dst$$reg),
11157            as_Register($src1$$reg),
11158            $src2$$constant & 0x3f);
11159   %}
11160 
11161   ins_pipe(ialu_reg_shift);
11162 %}
11163 
11164 // BEGIN This section of the file is automatically generated. Do not edit --------------
11165 // This section is generated from aarch64_ad.m4
11166 
11167 // This pattern is automatically generated from aarch64_ad.m4
11168 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11169 instruct regL_not_reg(iRegLNoSp dst,
11170                          iRegL src1, immL_M1 m1,
11171                          rFlagsReg cr) %{
11172   match(Set dst (XorL src1 m1));
11173   ins_cost(INSN_COST);
11174   format %{ "eon  $dst, $src1, zr" %}
11175 
11176   ins_encode %{
11177     __ eon(as_Register($dst$$reg),
11178               as_Register($src1$$reg),
11179               zr,
11180               Assembler::LSL, 0);
11181   %}
11182 
11183   ins_pipe(ialu_reg);
11184 %}
11185 
11186 // This pattern is automatically generated from aarch64_ad.m4
11187 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11188 instruct regI_not_reg(iRegINoSp dst,
11189                          iRegIorL2I src1, immI_M1 m1,
11190                          rFlagsReg cr) %{
11191   match(Set dst (XorI src1 m1));
11192   ins_cost(INSN_COST);
11193   format %{ "eonw  $dst, $src1, zr" %}
11194 
11195   ins_encode %{
11196     __ eonw(as_Register($dst$$reg),
11197               as_Register($src1$$reg),
11198               zr,
11199               Assembler::LSL, 0);
11200   %}
11201 
11202   ins_pipe(ialu_reg);
11203 %}
11204 
11205 // This pattern is automatically generated from aarch64_ad.m4
11206 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11207 instruct NegI_reg_URShift_reg(iRegINoSp dst,
11208                               immI0 zero, iRegIorL2I src1, immI src2) %{
11209   match(Set dst (SubI zero (URShiftI src1 src2)));
11210 
11211   ins_cost(1.9 * INSN_COST);
11212   format %{ "negw  $dst, $src1, LSR $src2" %}
11213 
11214   ins_encode %{
11215     __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
11216             Assembler::LSR, $src2$$constant & 0x1f);
11217   %}
11218 
11219   ins_pipe(ialu_reg_shift);
11220 %}
11221 
11222 // This pattern is automatically generated from aarch64_ad.m4
11223 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11224 instruct NegI_reg_RShift_reg(iRegINoSp dst,
11225                               immI0 zero, iRegIorL2I src1, immI src2) %{
11226   match(Set dst (SubI zero (RShiftI src1 src2)));
11227 
11228   ins_cost(1.9 * INSN_COST);
11229   format %{ "negw  $dst, $src1, ASR $src2" %}
11230 
11231   ins_encode %{
11232     __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
11233             Assembler::ASR, $src2$$constant & 0x1f);
11234   %}
11235 
11236   ins_pipe(ialu_reg_shift);
11237 %}
11238 
11239 // This pattern is automatically generated from aarch64_ad.m4
11240 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11241 instruct NegI_reg_LShift_reg(iRegINoSp dst,
11242                               immI0 zero, iRegIorL2I src1, immI src2) %{
11243   match(Set dst (SubI zero (LShiftI src1 src2)));
11244 
11245   ins_cost(1.9 * INSN_COST);
11246   format %{ "negw  $dst, $src1, LSL $src2" %}
11247 
11248   ins_encode %{
11249     __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
11250             Assembler::LSL, $src2$$constant & 0x1f);
11251   %}
11252 
11253   ins_pipe(ialu_reg_shift);
11254 %}
11255 
11256 // This pattern is automatically generated from aarch64_ad.m4
11257 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11258 instruct NegL_reg_URShift_reg(iRegLNoSp dst,
11259                               immL0 zero, iRegL src1, immI src2) %{
11260   match(Set dst (SubL zero (URShiftL src1 src2)));
11261 
11262   ins_cost(1.9 * INSN_COST);
11263   format %{ "neg  $dst, $src1, LSR $src2" %}
11264 
11265   ins_encode %{
11266     __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
11267             Assembler::LSR, $src2$$constant & 0x3f);
11268   %}
11269 
11270   ins_pipe(ialu_reg_shift);
11271 %}
11272 
11273 // This pattern is automatically generated from aarch64_ad.m4
11274 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11275 instruct NegL_reg_RShift_reg(iRegLNoSp dst,
11276                               immL0 zero, iRegL src1, immI src2) %{
11277   match(Set dst (SubL zero (RShiftL src1 src2)));
11278 
11279   ins_cost(1.9 * INSN_COST);
11280   format %{ "neg  $dst, $src1, ASR $src2" %}
11281 
11282   ins_encode %{
11283     __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
11284             Assembler::ASR, $src2$$constant & 0x3f);
11285   %}
11286 
11287   ins_pipe(ialu_reg_shift);
11288 %}
11289 
11290 // This pattern is automatically generated from aarch64_ad.m4
11291 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11292 instruct NegL_reg_LShift_reg(iRegLNoSp dst,
11293                               immL0 zero, iRegL src1, immI src2) %{
11294   match(Set dst (SubL zero (LShiftL src1 src2)));
11295 
11296   ins_cost(1.9 * INSN_COST);
11297   format %{ "neg  $dst, $src1, LSL $src2" %}
11298 
11299   ins_encode %{
11300     __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
11301             Assembler::LSL, $src2$$constant & 0x3f);
11302   %}
11303 
11304   ins_pipe(ialu_reg_shift);
11305 %}
11306 
11307 // This pattern is automatically generated from aarch64_ad.m4
11308 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11309 instruct AndI_reg_not_reg(iRegINoSp dst,
11310                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11311   match(Set dst (AndI src1 (XorI src2 m1)));
11312   ins_cost(INSN_COST);
11313   format %{ "bicw  $dst, $src1, $src2" %}
11314 
11315   ins_encode %{
11316     __ bicw(as_Register($dst$$reg),
11317               as_Register($src1$$reg),
11318               as_Register($src2$$reg),
11319               Assembler::LSL, 0);
11320   %}
11321 
11322   ins_pipe(ialu_reg_reg);
11323 %}
11324 
11325 // This pattern is automatically generated from aarch64_ad.m4
11326 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11327 instruct AndL_reg_not_reg(iRegLNoSp dst,
11328                          iRegL src1, iRegL src2, immL_M1 m1) %{
11329   match(Set dst (AndL src1 (XorL src2 m1)));
11330   ins_cost(INSN_COST);
11331   format %{ "bic  $dst, $src1, $src2" %}
11332 
11333   ins_encode %{
11334     __ bic(as_Register($dst$$reg),
11335               as_Register($src1$$reg),
11336               as_Register($src2$$reg),
11337               Assembler::LSL, 0);
11338   %}
11339 
11340   ins_pipe(ialu_reg_reg);
11341 %}
11342 
11343 // This pattern is automatically generated from aarch64_ad.m4
11344 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11345 instruct OrI_reg_not_reg(iRegINoSp dst,
11346                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11347   match(Set dst (OrI src1 (XorI src2 m1)));
11348   ins_cost(INSN_COST);
11349   format %{ "ornw  $dst, $src1, $src2" %}
11350 
11351   ins_encode %{
11352     __ ornw(as_Register($dst$$reg),
11353               as_Register($src1$$reg),
11354               as_Register($src2$$reg),
11355               Assembler::LSL, 0);
11356   %}
11357 
11358   ins_pipe(ialu_reg_reg);
11359 %}
11360 
11361 // This pattern is automatically generated from aarch64_ad.m4
11362 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11363 instruct OrL_reg_not_reg(iRegLNoSp dst,
11364                          iRegL src1, iRegL src2, immL_M1 m1) %{
11365   match(Set dst (OrL src1 (XorL src2 m1)));
11366   ins_cost(INSN_COST);
11367   format %{ "orn  $dst, $src1, $src2" %}
11368 
11369   ins_encode %{
11370     __ orn(as_Register($dst$$reg),
11371               as_Register($src1$$reg),
11372               as_Register($src2$$reg),
11373               Assembler::LSL, 0);
11374   %}
11375 
11376   ins_pipe(ialu_reg_reg);
11377 %}
11378 
11379 // This pattern is automatically generated from aarch64_ad.m4
11380 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11381 instruct XorI_reg_not_reg(iRegINoSp dst,
11382                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11383   match(Set dst (XorI m1 (XorI src2 src1)));
11384   ins_cost(INSN_COST);
11385   format %{ "eonw  $dst, $src1, $src2" %}
11386 
11387   ins_encode %{
11388     __ eonw(as_Register($dst$$reg),
11389               as_Register($src1$$reg),
11390               as_Register($src2$$reg),
11391               Assembler::LSL, 0);
11392   %}
11393 
11394   ins_pipe(ialu_reg_reg);
11395 %}
11396 
11397 // This pattern is automatically generated from aarch64_ad.m4
11398 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11399 instruct XorL_reg_not_reg(iRegLNoSp dst,
11400                          iRegL src1, iRegL src2, immL_M1 m1) %{
11401   match(Set dst (XorL m1 (XorL src2 src1)));
11402   ins_cost(INSN_COST);
11403   format %{ "eon  $dst, $src1, $src2" %}
11404 
11405   ins_encode %{
11406     __ eon(as_Register($dst$$reg),
11407               as_Register($src1$$reg),
11408               as_Register($src2$$reg),
11409               Assembler::LSL, 0);
11410   %}
11411 
11412   ins_pipe(ialu_reg_reg);
11413 %}
11414 
11415 // This pattern is automatically generated from aarch64_ad.m4
11416 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11417 // val & (-1 ^ (val >>> shift)) ==> bicw
11418 instruct AndI_reg_URShift_not_reg(iRegINoSp dst,
11419                          iRegIorL2I src1, iRegIorL2I src2,
11420                          immI src3, immI_M1 src4) %{
11421   match(Set dst (AndI src1 (XorI(URShiftI src2 src3) src4)));
11422   ins_cost(1.9 * INSN_COST);
11423   format %{ "bicw  $dst, $src1, $src2, LSR $src3" %}
11424 
11425   ins_encode %{
11426     __ bicw(as_Register($dst$$reg),
11427               as_Register($src1$$reg),
11428               as_Register($src2$$reg),
11429               Assembler::LSR,
11430               $src3$$constant & 0x1f);
11431   %}
11432 
11433   ins_pipe(ialu_reg_reg_shift);
11434 %}
11435 
11436 // This pattern is automatically generated from aarch64_ad.m4
11437 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11438 // val & (-1 ^ (val >>> shift)) ==> bic
11439 instruct AndL_reg_URShift_not_reg(iRegLNoSp dst,
11440                          iRegL src1, iRegL src2,
11441                          immI src3, immL_M1 src4) %{
11442   match(Set dst (AndL src1 (XorL(URShiftL src2 src3) src4)));
11443   ins_cost(1.9 * INSN_COST);
11444   format %{ "bic  $dst, $src1, $src2, LSR $src3" %}
11445 
11446   ins_encode %{
11447     __ bic(as_Register($dst$$reg),
11448               as_Register($src1$$reg),
11449               as_Register($src2$$reg),
11450               Assembler::LSR,
11451               $src3$$constant & 0x3f);
11452   %}
11453 
11454   ins_pipe(ialu_reg_reg_shift);
11455 %}
11456 
11457 // This pattern is automatically generated from aarch64_ad.m4
11458 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11459 // val & (-1 ^ (val >> shift)) ==> bicw
11460 instruct AndI_reg_RShift_not_reg(iRegINoSp dst,
11461                          iRegIorL2I src1, iRegIorL2I src2,
11462                          immI src3, immI_M1 src4) %{
11463   match(Set dst (AndI src1 (XorI(RShiftI src2 src3) src4)));
11464   ins_cost(1.9 * INSN_COST);
11465   format %{ "bicw  $dst, $src1, $src2, ASR $src3" %}
11466 
11467   ins_encode %{
11468     __ bicw(as_Register($dst$$reg),
11469               as_Register($src1$$reg),
11470               as_Register($src2$$reg),
11471               Assembler::ASR,
11472               $src3$$constant & 0x1f);
11473   %}
11474 
11475   ins_pipe(ialu_reg_reg_shift);
11476 %}
11477 
11478 // This pattern is automatically generated from aarch64_ad.m4
11479 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11480 // val & (-1 ^ (val >> shift)) ==> bic
11481 instruct AndL_reg_RShift_not_reg(iRegLNoSp dst,
11482                          iRegL src1, iRegL src2,
11483                          immI src3, immL_M1 src4) %{
11484   match(Set dst (AndL src1 (XorL(RShiftL src2 src3) src4)));
11485   ins_cost(1.9 * INSN_COST);
11486   format %{ "bic  $dst, $src1, $src2, ASR $src3" %}
11487 
11488   ins_encode %{
11489     __ bic(as_Register($dst$$reg),
11490               as_Register($src1$$reg),
11491               as_Register($src2$$reg),
11492               Assembler::ASR,
11493               $src3$$constant & 0x3f);
11494   %}
11495 
11496   ins_pipe(ialu_reg_reg_shift);
11497 %}
11498 
11499 // This pattern is automatically generated from aarch64_ad.m4
11500 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11501 // val & (-1 ^ (val ror shift)) ==> bicw
11502 instruct AndI_reg_RotateRight_not_reg(iRegINoSp dst,
11503                          iRegIorL2I src1, iRegIorL2I src2,
11504                          immI src3, immI_M1 src4) %{
11505   match(Set dst (AndI src1 (XorI(RotateRight src2 src3) src4)));
11506   ins_cost(1.9 * INSN_COST);
11507   format %{ "bicw  $dst, $src1, $src2, ROR $src3" %}
11508 
11509   ins_encode %{
11510     __ bicw(as_Register($dst$$reg),
11511               as_Register($src1$$reg),
11512               as_Register($src2$$reg),
11513               Assembler::ROR,
11514               $src3$$constant & 0x1f);
11515   %}
11516 
11517   ins_pipe(ialu_reg_reg_shift);
11518 %}
11519 
11520 // This pattern is automatically generated from aarch64_ad.m4
11521 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11522 // val & (-1 ^ (val ror shift)) ==> bic
11523 instruct AndL_reg_RotateRight_not_reg(iRegLNoSp dst,
11524                          iRegL src1, iRegL src2,
11525                          immI src3, immL_M1 src4) %{
11526   match(Set dst (AndL src1 (XorL(RotateRight src2 src3) src4)));
11527   ins_cost(1.9 * INSN_COST);
11528   format %{ "bic  $dst, $src1, $src2, ROR $src3" %}
11529 
11530   ins_encode %{
11531     __ bic(as_Register($dst$$reg),
11532               as_Register($src1$$reg),
11533               as_Register($src2$$reg),
11534               Assembler::ROR,
11535               $src3$$constant & 0x3f);
11536   %}
11537 
11538   ins_pipe(ialu_reg_reg_shift);
11539 %}
11540 
11541 // This pattern is automatically generated from aarch64_ad.m4
11542 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11543 // val & (-1 ^ (val << shift)) ==> bicw
11544 instruct AndI_reg_LShift_not_reg(iRegINoSp dst,
11545                          iRegIorL2I src1, iRegIorL2I src2,
11546                          immI src3, immI_M1 src4) %{
11547   match(Set dst (AndI src1 (XorI(LShiftI src2 src3) src4)));
11548   ins_cost(1.9 * INSN_COST);
11549   format %{ "bicw  $dst, $src1, $src2, LSL $src3" %}
11550 
11551   ins_encode %{
11552     __ bicw(as_Register($dst$$reg),
11553               as_Register($src1$$reg),
11554               as_Register($src2$$reg),
11555               Assembler::LSL,
11556               $src3$$constant & 0x1f);
11557   %}
11558 
11559   ins_pipe(ialu_reg_reg_shift);
11560 %}
11561 
11562 // This pattern is automatically generated from aarch64_ad.m4
11563 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11564 // val & (-1 ^ (val << shift)) ==> bic
11565 instruct AndL_reg_LShift_not_reg(iRegLNoSp dst,
11566                          iRegL src1, iRegL src2,
11567                          immI src3, immL_M1 src4) %{
11568   match(Set dst (AndL src1 (XorL(LShiftL src2 src3) src4)));
11569   ins_cost(1.9 * INSN_COST);
11570   format %{ "bic  $dst, $src1, $src2, LSL $src3" %}
11571 
11572   ins_encode %{
11573     __ bic(as_Register($dst$$reg),
11574               as_Register($src1$$reg),
11575               as_Register($src2$$reg),
11576               Assembler::LSL,
11577               $src3$$constant & 0x3f);
11578   %}
11579 
11580   ins_pipe(ialu_reg_reg_shift);
11581 %}
11582 
11583 // This pattern is automatically generated from aarch64_ad.m4
11584 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11585 // val ^ (-1 ^ (val >>> shift)) ==> eonw
11586 instruct XorI_reg_URShift_not_reg(iRegINoSp dst,
11587                          iRegIorL2I src1, iRegIorL2I src2,
11588                          immI src3, immI_M1 src4) %{
11589   match(Set dst (XorI src4 (XorI(URShiftI src2 src3) src1)));
11590   ins_cost(1.9 * INSN_COST);
11591   format %{ "eonw  $dst, $src1, $src2, LSR $src3" %}
11592 
11593   ins_encode %{
11594     __ eonw(as_Register($dst$$reg),
11595               as_Register($src1$$reg),
11596               as_Register($src2$$reg),
11597               Assembler::LSR,
11598               $src3$$constant & 0x1f);
11599   %}
11600 
11601   ins_pipe(ialu_reg_reg_shift);
11602 %}
11603 
11604 // This pattern is automatically generated from aarch64_ad.m4
11605 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11606 // val ^ (-1 ^ (val >>> shift)) ==> eon
11607 instruct XorL_reg_URShift_not_reg(iRegLNoSp dst,
11608                          iRegL src1, iRegL src2,
11609                          immI src3, immL_M1 src4) %{
11610   match(Set dst (XorL src4 (XorL(URShiftL src2 src3) src1)));
11611   ins_cost(1.9 * INSN_COST);
11612   format %{ "eon  $dst, $src1, $src2, LSR $src3" %}
11613 
11614   ins_encode %{
11615     __ eon(as_Register($dst$$reg),
11616               as_Register($src1$$reg),
11617               as_Register($src2$$reg),
11618               Assembler::LSR,
11619               $src3$$constant & 0x3f);
11620   %}
11621 
11622   ins_pipe(ialu_reg_reg_shift);
11623 %}
11624 
11625 // This pattern is automatically generated from aarch64_ad.m4
11626 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11627 // val ^ (-1 ^ (val >> shift)) ==> eonw
11628 instruct XorI_reg_RShift_not_reg(iRegINoSp dst,
11629                          iRegIorL2I src1, iRegIorL2I src2,
11630                          immI src3, immI_M1 src4) %{
11631   match(Set dst (XorI src4 (XorI(RShiftI src2 src3) src1)));
11632   ins_cost(1.9 * INSN_COST);
11633   format %{ "eonw  $dst, $src1, $src2, ASR $src3" %}
11634 
11635   ins_encode %{
11636     __ eonw(as_Register($dst$$reg),
11637               as_Register($src1$$reg),
11638               as_Register($src2$$reg),
11639               Assembler::ASR,
11640               $src3$$constant & 0x1f);
11641   %}
11642 
11643   ins_pipe(ialu_reg_reg_shift);
11644 %}
11645 
11646 // This pattern is automatically generated from aarch64_ad.m4
11647 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11648 // val ^ (-1 ^ (val >> shift)) ==> eon
11649 instruct XorL_reg_RShift_not_reg(iRegLNoSp dst,
11650                          iRegL src1, iRegL src2,
11651                          immI src3, immL_M1 src4) %{
11652   match(Set dst (XorL src4 (XorL(RShiftL src2 src3) src1)));
11653   ins_cost(1.9 * INSN_COST);
11654   format %{ "eon  $dst, $src1, $src2, ASR $src3" %}
11655 
11656   ins_encode %{
11657     __ eon(as_Register($dst$$reg),
11658               as_Register($src1$$reg),
11659               as_Register($src2$$reg),
11660               Assembler::ASR,
11661               $src3$$constant & 0x3f);
11662   %}
11663 
11664   ins_pipe(ialu_reg_reg_shift);
11665 %}
11666 
11667 // This pattern is automatically generated from aarch64_ad.m4
11668 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11669 // val ^ (-1 ^ (val ror shift)) ==> eonw
11670 instruct XorI_reg_RotateRight_not_reg(iRegINoSp dst,
11671                          iRegIorL2I src1, iRegIorL2I src2,
11672                          immI src3, immI_M1 src4) %{
11673   match(Set dst (XorI src4 (XorI(RotateRight src2 src3) src1)));
11674   ins_cost(1.9 * INSN_COST);
11675   format %{ "eonw  $dst, $src1, $src2, ROR $src3" %}
11676 
11677   ins_encode %{
11678     __ eonw(as_Register($dst$$reg),
11679               as_Register($src1$$reg),
11680               as_Register($src2$$reg),
11681               Assembler::ROR,
11682               $src3$$constant & 0x1f);
11683   %}
11684 
11685   ins_pipe(ialu_reg_reg_shift);
11686 %}
11687 
11688 // This pattern is automatically generated from aarch64_ad.m4
11689 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11690 // val ^ (-1 ^ (val ror shift)) ==> eon
11691 instruct XorL_reg_RotateRight_not_reg(iRegLNoSp dst,
11692                          iRegL src1, iRegL src2,
11693                          immI src3, immL_M1 src4) %{
11694   match(Set dst (XorL src4 (XorL(RotateRight src2 src3) src1)));
11695   ins_cost(1.9 * INSN_COST);
11696   format %{ "eon  $dst, $src1, $src2, ROR $src3" %}
11697 
11698   ins_encode %{
11699     __ eon(as_Register($dst$$reg),
11700               as_Register($src1$$reg),
11701               as_Register($src2$$reg),
11702               Assembler::ROR,
11703               $src3$$constant & 0x3f);
11704   %}
11705 
11706   ins_pipe(ialu_reg_reg_shift);
11707 %}
11708 
11709 // This pattern is automatically generated from aarch64_ad.m4
11710 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11711 // val ^ (-1 ^ (val << shift)) ==> eonw
11712 instruct XorI_reg_LShift_not_reg(iRegINoSp dst,
11713                          iRegIorL2I src1, iRegIorL2I src2,
11714                          immI src3, immI_M1 src4) %{
11715   match(Set dst (XorI src4 (XorI(LShiftI src2 src3) src1)));
11716   ins_cost(1.9 * INSN_COST);
11717   format %{ "eonw  $dst, $src1, $src2, LSL $src3" %}
11718 
11719   ins_encode %{
11720     __ eonw(as_Register($dst$$reg),
11721               as_Register($src1$$reg),
11722               as_Register($src2$$reg),
11723               Assembler::LSL,
11724               $src3$$constant & 0x1f);
11725   %}
11726 
11727   ins_pipe(ialu_reg_reg_shift);
11728 %}
11729 
11730 // This pattern is automatically generated from aarch64_ad.m4
11731 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11732 // val ^ (-1 ^ (val << shift)) ==> eon
11733 instruct XorL_reg_LShift_not_reg(iRegLNoSp dst,
11734                          iRegL src1, iRegL src2,
11735                          immI src3, immL_M1 src4) %{
11736   match(Set dst (XorL src4 (XorL(LShiftL src2 src3) src1)));
11737   ins_cost(1.9 * INSN_COST);
11738   format %{ "eon  $dst, $src1, $src2, LSL $src3" %}
11739 
11740   ins_encode %{
11741     __ eon(as_Register($dst$$reg),
11742               as_Register($src1$$reg),
11743               as_Register($src2$$reg),
11744               Assembler::LSL,
11745               $src3$$constant & 0x3f);
11746   %}
11747 
11748   ins_pipe(ialu_reg_reg_shift);
11749 %}
11750 
11751 // This pattern is automatically generated from aarch64_ad.m4
11752 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11753 // val | (-1 ^ (val >>> shift)) ==> ornw
11754 instruct OrI_reg_URShift_not_reg(iRegINoSp dst,
11755                          iRegIorL2I src1, iRegIorL2I src2,
11756                          immI src3, immI_M1 src4) %{
11757   match(Set dst (OrI src1 (XorI(URShiftI src2 src3) src4)));
11758   ins_cost(1.9 * INSN_COST);
11759   format %{ "ornw  $dst, $src1, $src2, LSR $src3" %}
11760 
11761   ins_encode %{
11762     __ ornw(as_Register($dst$$reg),
11763               as_Register($src1$$reg),
11764               as_Register($src2$$reg),
11765               Assembler::LSR,
11766               $src3$$constant & 0x1f);
11767   %}
11768 
11769   ins_pipe(ialu_reg_reg_shift);
11770 %}
11771 
11772 // This pattern is automatically generated from aarch64_ad.m4
11773 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11774 // val | (-1 ^ (val >>> shift)) ==> orn
11775 instruct OrL_reg_URShift_not_reg(iRegLNoSp dst,
11776                          iRegL src1, iRegL src2,
11777                          immI src3, immL_M1 src4) %{
11778   match(Set dst (OrL src1 (XorL(URShiftL src2 src3) src4)));
11779   ins_cost(1.9 * INSN_COST);
11780   format %{ "orn  $dst, $src1, $src2, LSR $src3" %}
11781 
11782   ins_encode %{
11783     __ orn(as_Register($dst$$reg),
11784               as_Register($src1$$reg),
11785               as_Register($src2$$reg),
11786               Assembler::LSR,
11787               $src3$$constant & 0x3f);
11788   %}
11789 
11790   ins_pipe(ialu_reg_reg_shift);
11791 %}
11792 
11793 // This pattern is automatically generated from aarch64_ad.m4
11794 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11795 // val | (-1 ^ (val >> shift)) ==> ornw
11796 instruct OrI_reg_RShift_not_reg(iRegINoSp dst,
11797                          iRegIorL2I src1, iRegIorL2I src2,
11798                          immI src3, immI_M1 src4) %{
11799   match(Set dst (OrI src1 (XorI(RShiftI src2 src3) src4)));
11800   ins_cost(1.9 * INSN_COST);
11801   format %{ "ornw  $dst, $src1, $src2, ASR $src3" %}
11802 
11803   ins_encode %{
11804     __ ornw(as_Register($dst$$reg),
11805               as_Register($src1$$reg),
11806               as_Register($src2$$reg),
11807               Assembler::ASR,
11808               $src3$$constant & 0x1f);
11809   %}
11810 
11811   ins_pipe(ialu_reg_reg_shift);
11812 %}
11813 
11814 // This pattern is automatically generated from aarch64_ad.m4
11815 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11816 // val | (-1 ^ (val >> shift)) ==> orn
11817 instruct OrL_reg_RShift_not_reg(iRegLNoSp dst,
11818                          iRegL src1, iRegL src2,
11819                          immI src3, immL_M1 src4) %{
11820   match(Set dst (OrL src1 (XorL(RShiftL src2 src3) src4)));
11821   ins_cost(1.9 * INSN_COST);
11822   format %{ "orn  $dst, $src1, $src2, ASR $src3" %}
11823 
11824   ins_encode %{
11825     __ orn(as_Register($dst$$reg),
11826               as_Register($src1$$reg),
11827               as_Register($src2$$reg),
11828               Assembler::ASR,
11829               $src3$$constant & 0x3f);
11830   %}
11831 
11832   ins_pipe(ialu_reg_reg_shift);
11833 %}
11834 
11835 // This pattern is automatically generated from aarch64_ad.m4
11836 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11837 // val | (-1 ^ (val ror shift)) ==> ornw
11838 instruct OrI_reg_RotateRight_not_reg(iRegINoSp dst,
11839                          iRegIorL2I src1, iRegIorL2I src2,
11840                          immI src3, immI_M1 src4) %{
11841   match(Set dst (OrI src1 (XorI(RotateRight src2 src3) src4)));
11842   ins_cost(1.9 * INSN_COST);
11843   format %{ "ornw  $dst, $src1, $src2, ROR $src3" %}
11844 
11845   ins_encode %{
11846     __ ornw(as_Register($dst$$reg),
11847               as_Register($src1$$reg),
11848               as_Register($src2$$reg),
11849               Assembler::ROR,
11850               $src3$$constant & 0x1f);
11851   %}
11852 
11853   ins_pipe(ialu_reg_reg_shift);
11854 %}
11855 
11856 // This pattern is automatically generated from aarch64_ad.m4
11857 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11858 // val | (-1 ^ (val ror shift)) ==> orn
11859 instruct OrL_reg_RotateRight_not_reg(iRegLNoSp dst,
11860                          iRegL src1, iRegL src2,
11861                          immI src3, immL_M1 src4) %{
11862   match(Set dst (OrL src1 (XorL(RotateRight src2 src3) src4)));
11863   ins_cost(1.9 * INSN_COST);
11864   format %{ "orn  $dst, $src1, $src2, ROR $src3" %}
11865 
11866   ins_encode %{
11867     __ orn(as_Register($dst$$reg),
11868               as_Register($src1$$reg),
11869               as_Register($src2$$reg),
11870               Assembler::ROR,
11871               $src3$$constant & 0x3f);
11872   %}
11873 
11874   ins_pipe(ialu_reg_reg_shift);
11875 %}
11876 
11877 // This pattern is automatically generated from aarch64_ad.m4
11878 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11879 // val | (-1 ^ (val << shift)) ==> ornw
11880 instruct OrI_reg_LShift_not_reg(iRegINoSp dst,
11881                          iRegIorL2I src1, iRegIorL2I src2,
11882                          immI src3, immI_M1 src4) %{
11883   match(Set dst (OrI src1 (XorI(LShiftI src2 src3) src4)));
11884   ins_cost(1.9 * INSN_COST);
11885   format %{ "ornw  $dst, $src1, $src2, LSL $src3" %}
11886 
11887   ins_encode %{
11888     __ ornw(as_Register($dst$$reg),
11889               as_Register($src1$$reg),
11890               as_Register($src2$$reg),
11891               Assembler::LSL,
11892               $src3$$constant & 0x1f);
11893   %}
11894 
11895   ins_pipe(ialu_reg_reg_shift);
11896 %}
11897 
11898 // This pattern is automatically generated from aarch64_ad.m4
11899 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11900 // val | (-1 ^ (val << shift)) ==> orn
11901 instruct OrL_reg_LShift_not_reg(iRegLNoSp dst,
11902                          iRegL src1, iRegL src2,
11903                          immI src3, immL_M1 src4) %{
11904   match(Set dst (OrL src1 (XorL(LShiftL src2 src3) src4)));
11905   ins_cost(1.9 * INSN_COST);
11906   format %{ "orn  $dst, $src1, $src2, LSL $src3" %}
11907 
11908   ins_encode %{
11909     __ orn(as_Register($dst$$reg),
11910               as_Register($src1$$reg),
11911               as_Register($src2$$reg),
11912               Assembler::LSL,
11913               $src3$$constant & 0x3f);
11914   %}
11915 
11916   ins_pipe(ialu_reg_reg_shift);
11917 %}
11918 
11919 // This pattern is automatically generated from aarch64_ad.m4
11920 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11921 instruct AndI_reg_URShift_reg(iRegINoSp dst,
11922                          iRegIorL2I src1, iRegIorL2I src2,
11923                          immI src3) %{
11924   match(Set dst (AndI src1 (URShiftI src2 src3)));
11925 
11926   ins_cost(1.9 * INSN_COST);
11927   format %{ "andw  $dst, $src1, $src2, LSR $src3" %}
11928 
11929   ins_encode %{
11930     __ andw(as_Register($dst$$reg),
11931               as_Register($src1$$reg),
11932               as_Register($src2$$reg),
11933               Assembler::LSR,
11934               $src3$$constant & 0x1f);
11935   %}
11936 
11937   ins_pipe(ialu_reg_reg_shift);
11938 %}
11939 
11940 // This pattern is automatically generated from aarch64_ad.m4
11941 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11942 instruct AndL_reg_URShift_reg(iRegLNoSp dst,
11943                          iRegL src1, iRegL src2,
11944                          immI src3) %{
11945   match(Set dst (AndL src1 (URShiftL src2 src3)));
11946 
11947   ins_cost(1.9 * INSN_COST);
11948   format %{ "andr  $dst, $src1, $src2, LSR $src3" %}
11949 
11950   ins_encode %{
11951     __ andr(as_Register($dst$$reg),
11952               as_Register($src1$$reg),
11953               as_Register($src2$$reg),
11954               Assembler::LSR,
11955               $src3$$constant & 0x3f);
11956   %}
11957 
11958   ins_pipe(ialu_reg_reg_shift);
11959 %}
11960 
11961 // This pattern is automatically generated from aarch64_ad.m4
11962 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11963 instruct AndI_reg_RShift_reg(iRegINoSp dst,
11964                          iRegIorL2I src1, iRegIorL2I src2,
11965                          immI src3) %{
11966   match(Set dst (AndI src1 (RShiftI src2 src3)));
11967 
11968   ins_cost(1.9 * INSN_COST);
11969   format %{ "andw  $dst, $src1, $src2, ASR $src3" %}
11970 
11971   ins_encode %{
11972     __ andw(as_Register($dst$$reg),
11973               as_Register($src1$$reg),
11974               as_Register($src2$$reg),
11975               Assembler::ASR,
11976               $src3$$constant & 0x1f);
11977   %}
11978 
11979   ins_pipe(ialu_reg_reg_shift);
11980 %}
11981 
11982 // This pattern is automatically generated from aarch64_ad.m4
11983 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11984 instruct AndL_reg_RShift_reg(iRegLNoSp dst,
11985                          iRegL src1, iRegL src2,
11986                          immI src3) %{
11987   match(Set dst (AndL src1 (RShiftL src2 src3)));
11988 
11989   ins_cost(1.9 * INSN_COST);
11990   format %{ "andr  $dst, $src1, $src2, ASR $src3" %}
11991 
11992   ins_encode %{
11993     __ andr(as_Register($dst$$reg),
11994               as_Register($src1$$reg),
11995               as_Register($src2$$reg),
11996               Assembler::ASR,
11997               $src3$$constant & 0x3f);
11998   %}
11999 
12000   ins_pipe(ialu_reg_reg_shift);
12001 %}
12002 
12003 // This pattern is automatically generated from aarch64_ad.m4
12004 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12005 instruct AndI_reg_LShift_reg(iRegINoSp dst,
12006                          iRegIorL2I src1, iRegIorL2I src2,
12007                          immI src3) %{
12008   match(Set dst (AndI src1 (LShiftI src2 src3)));
12009 
12010   ins_cost(1.9 * INSN_COST);
12011   format %{ "andw  $dst, $src1, $src2, LSL $src3" %}
12012 
12013   ins_encode %{
12014     __ andw(as_Register($dst$$reg),
12015               as_Register($src1$$reg),
12016               as_Register($src2$$reg),
12017               Assembler::LSL,
12018               $src3$$constant & 0x1f);
12019   %}
12020 
12021   ins_pipe(ialu_reg_reg_shift);
12022 %}
12023 
12024 // This pattern is automatically generated from aarch64_ad.m4
12025 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12026 instruct AndL_reg_LShift_reg(iRegLNoSp dst,
12027                          iRegL src1, iRegL src2,
12028                          immI src3) %{
12029   match(Set dst (AndL src1 (LShiftL src2 src3)));
12030 
12031   ins_cost(1.9 * INSN_COST);
12032   format %{ "andr  $dst, $src1, $src2, LSL $src3" %}
12033 
12034   ins_encode %{
12035     __ andr(as_Register($dst$$reg),
12036               as_Register($src1$$reg),
12037               as_Register($src2$$reg),
12038               Assembler::LSL,
12039               $src3$$constant & 0x3f);
12040   %}
12041 
12042   ins_pipe(ialu_reg_reg_shift);
12043 %}
12044 
12045 // This pattern is automatically generated from aarch64_ad.m4
12046 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12047 instruct AndI_reg_RotateRight_reg(iRegINoSp dst,
12048                          iRegIorL2I src1, iRegIorL2I src2,
12049                          immI src3) %{
12050   match(Set dst (AndI src1 (RotateRight src2 src3)));
12051 
12052   ins_cost(1.9 * INSN_COST);
12053   format %{ "andw  $dst, $src1, $src2, ROR $src3" %}
12054 
12055   ins_encode %{
12056     __ andw(as_Register($dst$$reg),
12057               as_Register($src1$$reg),
12058               as_Register($src2$$reg),
12059               Assembler::ROR,
12060               $src3$$constant & 0x1f);
12061   %}
12062 
12063   ins_pipe(ialu_reg_reg_shift);
12064 %}
12065 
12066 // This pattern is automatically generated from aarch64_ad.m4
12067 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12068 instruct AndL_reg_RotateRight_reg(iRegLNoSp dst,
12069                          iRegL src1, iRegL src2,
12070                          immI src3) %{
12071   match(Set dst (AndL src1 (RotateRight src2 src3)));
12072 
12073   ins_cost(1.9 * INSN_COST);
12074   format %{ "andr  $dst, $src1, $src2, ROR $src3" %}
12075 
12076   ins_encode %{
12077     __ andr(as_Register($dst$$reg),
12078               as_Register($src1$$reg),
12079               as_Register($src2$$reg),
12080               Assembler::ROR,
12081               $src3$$constant & 0x3f);
12082   %}
12083 
12084   ins_pipe(ialu_reg_reg_shift);
12085 %}
12086 
12087 // This pattern is automatically generated from aarch64_ad.m4
12088 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12089 instruct XorI_reg_URShift_reg(iRegINoSp dst,
12090                          iRegIorL2I src1, iRegIorL2I src2,
12091                          immI src3) %{
12092   match(Set dst (XorI src1 (URShiftI src2 src3)));
12093 
12094   ins_cost(1.9 * INSN_COST);
12095   format %{ "eorw  $dst, $src1, $src2, LSR $src3" %}
12096 
12097   ins_encode %{
12098     __ eorw(as_Register($dst$$reg),
12099               as_Register($src1$$reg),
12100               as_Register($src2$$reg),
12101               Assembler::LSR,
12102               $src3$$constant & 0x1f);
12103   %}
12104 
12105   ins_pipe(ialu_reg_reg_shift);
12106 %}
12107 
12108 // This pattern is automatically generated from aarch64_ad.m4
12109 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12110 instruct XorL_reg_URShift_reg(iRegLNoSp dst,
12111                          iRegL src1, iRegL src2,
12112                          immI src3) %{
12113   match(Set dst (XorL src1 (URShiftL src2 src3)));
12114 
12115   ins_cost(1.9 * INSN_COST);
12116   format %{ "eor  $dst, $src1, $src2, LSR $src3" %}
12117 
12118   ins_encode %{
12119     __ eor(as_Register($dst$$reg),
12120               as_Register($src1$$reg),
12121               as_Register($src2$$reg),
12122               Assembler::LSR,
12123               $src3$$constant & 0x3f);
12124   %}
12125 
12126   ins_pipe(ialu_reg_reg_shift);
12127 %}
12128 
12129 // This pattern is automatically generated from aarch64_ad.m4
12130 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12131 instruct XorI_reg_RShift_reg(iRegINoSp dst,
12132                          iRegIorL2I src1, iRegIorL2I src2,
12133                          immI src3) %{
12134   match(Set dst (XorI src1 (RShiftI src2 src3)));
12135 
12136   ins_cost(1.9 * INSN_COST);
12137   format %{ "eorw  $dst, $src1, $src2, ASR $src3" %}
12138 
12139   ins_encode %{
12140     __ eorw(as_Register($dst$$reg),
12141               as_Register($src1$$reg),
12142               as_Register($src2$$reg),
12143               Assembler::ASR,
12144               $src3$$constant & 0x1f);
12145   %}
12146 
12147   ins_pipe(ialu_reg_reg_shift);
12148 %}
12149 
12150 // This pattern is automatically generated from aarch64_ad.m4
12151 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12152 instruct XorL_reg_RShift_reg(iRegLNoSp dst,
12153                          iRegL src1, iRegL src2,
12154                          immI src3) %{
12155   match(Set dst (XorL src1 (RShiftL src2 src3)));
12156 
12157   ins_cost(1.9 * INSN_COST);
12158   format %{ "eor  $dst, $src1, $src2, ASR $src3" %}
12159 
12160   ins_encode %{
12161     __ eor(as_Register($dst$$reg),
12162               as_Register($src1$$reg),
12163               as_Register($src2$$reg),
12164               Assembler::ASR,
12165               $src3$$constant & 0x3f);
12166   %}
12167 
12168   ins_pipe(ialu_reg_reg_shift);
12169 %}
12170 
12171 // This pattern is automatically generated from aarch64_ad.m4
12172 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12173 instruct XorI_reg_LShift_reg(iRegINoSp dst,
12174                          iRegIorL2I src1, iRegIorL2I src2,
12175                          immI src3) %{
12176   match(Set dst (XorI src1 (LShiftI src2 src3)));
12177 
12178   ins_cost(1.9 * INSN_COST);
12179   format %{ "eorw  $dst, $src1, $src2, LSL $src3" %}
12180 
12181   ins_encode %{
12182     __ eorw(as_Register($dst$$reg),
12183               as_Register($src1$$reg),
12184               as_Register($src2$$reg),
12185               Assembler::LSL,
12186               $src3$$constant & 0x1f);
12187   %}
12188 
12189   ins_pipe(ialu_reg_reg_shift);
12190 %}
12191 
12192 // This pattern is automatically generated from aarch64_ad.m4
12193 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12194 instruct XorL_reg_LShift_reg(iRegLNoSp dst,
12195                          iRegL src1, iRegL src2,
12196                          immI src3) %{
12197   match(Set dst (XorL src1 (LShiftL src2 src3)));
12198 
12199   ins_cost(1.9 * INSN_COST);
12200   format %{ "eor  $dst, $src1, $src2, LSL $src3" %}
12201 
12202   ins_encode %{
12203     __ eor(as_Register($dst$$reg),
12204               as_Register($src1$$reg),
12205               as_Register($src2$$reg),
12206               Assembler::LSL,
12207               $src3$$constant & 0x3f);
12208   %}
12209 
12210   ins_pipe(ialu_reg_reg_shift);
12211 %}
12212 
12213 // This pattern is automatically generated from aarch64_ad.m4
12214 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12215 instruct XorI_reg_RotateRight_reg(iRegINoSp dst,
12216                          iRegIorL2I src1, iRegIorL2I src2,
12217                          immI src3) %{
12218   match(Set dst (XorI src1 (RotateRight src2 src3)));
12219 
12220   ins_cost(1.9 * INSN_COST);
12221   format %{ "eorw  $dst, $src1, $src2, ROR $src3" %}
12222 
12223   ins_encode %{
12224     __ eorw(as_Register($dst$$reg),
12225               as_Register($src1$$reg),
12226               as_Register($src2$$reg),
12227               Assembler::ROR,
12228               $src3$$constant & 0x1f);
12229   %}
12230 
12231   ins_pipe(ialu_reg_reg_shift);
12232 %}
12233 
12234 // This pattern is automatically generated from aarch64_ad.m4
12235 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12236 instruct XorL_reg_RotateRight_reg(iRegLNoSp dst,
12237                          iRegL src1, iRegL src2,
12238                          immI src3) %{
12239   match(Set dst (XorL src1 (RotateRight src2 src3)));
12240 
12241   ins_cost(1.9 * INSN_COST);
12242   format %{ "eor  $dst, $src1, $src2, ROR $src3" %}
12243 
12244   ins_encode %{
12245     __ eor(as_Register($dst$$reg),
12246               as_Register($src1$$reg),
12247               as_Register($src2$$reg),
12248               Assembler::ROR,
12249               $src3$$constant & 0x3f);
12250   %}
12251 
12252   ins_pipe(ialu_reg_reg_shift);
12253 %}
12254 
12255 // This pattern is automatically generated from aarch64_ad.m4
12256 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12257 instruct OrI_reg_URShift_reg(iRegINoSp dst,
12258                          iRegIorL2I src1, iRegIorL2I src2,
12259                          immI src3) %{
12260   match(Set dst (OrI src1 (URShiftI src2 src3)));
12261 
12262   ins_cost(1.9 * INSN_COST);
12263   format %{ "orrw  $dst, $src1, $src2, LSR $src3" %}
12264 
12265   ins_encode %{
12266     __ orrw(as_Register($dst$$reg),
12267               as_Register($src1$$reg),
12268               as_Register($src2$$reg),
12269               Assembler::LSR,
12270               $src3$$constant & 0x1f);
12271   %}
12272 
12273   ins_pipe(ialu_reg_reg_shift);
12274 %}
12275 
12276 // This pattern is automatically generated from aarch64_ad.m4
12277 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12278 instruct OrL_reg_URShift_reg(iRegLNoSp dst,
12279                          iRegL src1, iRegL src2,
12280                          immI src3) %{
12281   match(Set dst (OrL src1 (URShiftL src2 src3)));
12282 
12283   ins_cost(1.9 * INSN_COST);
12284   format %{ "orr  $dst, $src1, $src2, LSR $src3" %}
12285 
12286   ins_encode %{
12287     __ orr(as_Register($dst$$reg),
12288               as_Register($src1$$reg),
12289               as_Register($src2$$reg),
12290               Assembler::LSR,
12291               $src3$$constant & 0x3f);
12292   %}
12293 
12294   ins_pipe(ialu_reg_reg_shift);
12295 %}
12296 
12297 // This pattern is automatically generated from aarch64_ad.m4
12298 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12299 instruct OrI_reg_RShift_reg(iRegINoSp dst,
12300                          iRegIorL2I src1, iRegIorL2I src2,
12301                          immI src3) %{
12302   match(Set dst (OrI src1 (RShiftI src2 src3)));
12303 
12304   ins_cost(1.9 * INSN_COST);
12305   format %{ "orrw  $dst, $src1, $src2, ASR $src3" %}
12306 
12307   ins_encode %{
12308     __ orrw(as_Register($dst$$reg),
12309               as_Register($src1$$reg),
12310               as_Register($src2$$reg),
12311               Assembler::ASR,
12312               $src3$$constant & 0x1f);
12313   %}
12314 
12315   ins_pipe(ialu_reg_reg_shift);
12316 %}
12317 
12318 // This pattern is automatically generated from aarch64_ad.m4
12319 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12320 instruct OrL_reg_RShift_reg(iRegLNoSp dst,
12321                          iRegL src1, iRegL src2,
12322                          immI src3) %{
12323   match(Set dst (OrL src1 (RShiftL src2 src3)));
12324 
12325   ins_cost(1.9 * INSN_COST);
12326   format %{ "orr  $dst, $src1, $src2, ASR $src3" %}
12327 
12328   ins_encode %{
12329     __ orr(as_Register($dst$$reg),
12330               as_Register($src1$$reg),
12331               as_Register($src2$$reg),
12332               Assembler::ASR,
12333               $src3$$constant & 0x3f);
12334   %}
12335 
12336   ins_pipe(ialu_reg_reg_shift);
12337 %}
12338 
12339 // This pattern is automatically generated from aarch64_ad.m4
12340 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12341 instruct OrI_reg_LShift_reg(iRegINoSp dst,
12342                          iRegIorL2I src1, iRegIorL2I src2,
12343                          immI src3) %{
12344   match(Set dst (OrI src1 (LShiftI src2 src3)));
12345 
12346   ins_cost(1.9 * INSN_COST);
12347   format %{ "orrw  $dst, $src1, $src2, LSL $src3" %}
12348 
12349   ins_encode %{
12350     __ orrw(as_Register($dst$$reg),
12351               as_Register($src1$$reg),
12352               as_Register($src2$$reg),
12353               Assembler::LSL,
12354               $src3$$constant & 0x1f);
12355   %}
12356 
12357   ins_pipe(ialu_reg_reg_shift);
12358 %}
12359 
12360 // This pattern is automatically generated from aarch64_ad.m4
12361 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12362 instruct OrL_reg_LShift_reg(iRegLNoSp dst,
12363                          iRegL src1, iRegL src2,
12364                          immI src3) %{
12365   match(Set dst (OrL src1 (LShiftL src2 src3)));
12366 
12367   ins_cost(1.9 * INSN_COST);
12368   format %{ "orr  $dst, $src1, $src2, LSL $src3" %}
12369 
12370   ins_encode %{
12371     __ orr(as_Register($dst$$reg),
12372               as_Register($src1$$reg),
12373               as_Register($src2$$reg),
12374               Assembler::LSL,
12375               $src3$$constant & 0x3f);
12376   %}
12377 
12378   ins_pipe(ialu_reg_reg_shift);
12379 %}
12380 
12381 // This pattern is automatically generated from aarch64_ad.m4
12382 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12383 instruct OrI_reg_RotateRight_reg(iRegINoSp dst,
12384                          iRegIorL2I src1, iRegIorL2I src2,
12385                          immI src3) %{
12386   match(Set dst (OrI src1 (RotateRight src2 src3)));
12387 
12388   ins_cost(1.9 * INSN_COST);
12389   format %{ "orrw  $dst, $src1, $src2, ROR $src3" %}
12390 
12391   ins_encode %{
12392     __ orrw(as_Register($dst$$reg),
12393               as_Register($src1$$reg),
12394               as_Register($src2$$reg),
12395               Assembler::ROR,
12396               $src3$$constant & 0x1f);
12397   %}
12398 
12399   ins_pipe(ialu_reg_reg_shift);
12400 %}
12401 
12402 // This pattern is automatically generated from aarch64_ad.m4
12403 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12404 instruct OrL_reg_RotateRight_reg(iRegLNoSp dst,
12405                          iRegL src1, iRegL src2,
12406                          immI src3) %{
12407   match(Set dst (OrL src1 (RotateRight src2 src3)));
12408 
12409   ins_cost(1.9 * INSN_COST);
12410   format %{ "orr  $dst, $src1, $src2, ROR $src3" %}
12411 
12412   ins_encode %{
12413     __ orr(as_Register($dst$$reg),
12414               as_Register($src1$$reg),
12415               as_Register($src2$$reg),
12416               Assembler::ROR,
12417               $src3$$constant & 0x3f);
12418   %}
12419 
12420   ins_pipe(ialu_reg_reg_shift);
12421 %}
12422 
12423 // This pattern is automatically generated from aarch64_ad.m4
12424 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12425 instruct AddI_reg_URShift_reg(iRegINoSp dst,
12426                          iRegIorL2I src1, iRegIorL2I src2,
12427                          immI src3) %{
12428   match(Set dst (AddI src1 (URShiftI src2 src3)));
12429 
12430   ins_cost(1.9 * INSN_COST);
12431   format %{ "addw  $dst, $src1, $src2, LSR $src3" %}
12432 
12433   ins_encode %{
12434     __ addw(as_Register($dst$$reg),
12435               as_Register($src1$$reg),
12436               as_Register($src2$$reg),
12437               Assembler::LSR,
12438               $src3$$constant & 0x1f);
12439   %}
12440 
12441   ins_pipe(ialu_reg_reg_shift);
12442 %}
12443 
12444 // This pattern is automatically generated from aarch64_ad.m4
12445 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12446 instruct AddL_reg_URShift_reg(iRegLNoSp dst,
12447                          iRegL src1, iRegL src2,
12448                          immI src3) %{
12449   match(Set dst (AddL src1 (URShiftL src2 src3)));
12450 
12451   ins_cost(1.9 * INSN_COST);
12452   format %{ "add  $dst, $src1, $src2, LSR $src3" %}
12453 
12454   ins_encode %{
12455     __ add(as_Register($dst$$reg),
12456               as_Register($src1$$reg),
12457               as_Register($src2$$reg),
12458               Assembler::LSR,
12459               $src3$$constant & 0x3f);
12460   %}
12461 
12462   ins_pipe(ialu_reg_reg_shift);
12463 %}
12464 
12465 // This pattern is automatically generated from aarch64_ad.m4
12466 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12467 instruct AddI_reg_RShift_reg(iRegINoSp dst,
12468                          iRegIorL2I src1, iRegIorL2I src2,
12469                          immI src3) %{
12470   match(Set dst (AddI src1 (RShiftI src2 src3)));
12471 
12472   ins_cost(1.9 * INSN_COST);
12473   format %{ "addw  $dst, $src1, $src2, ASR $src3" %}
12474 
12475   ins_encode %{
12476     __ addw(as_Register($dst$$reg),
12477               as_Register($src1$$reg),
12478               as_Register($src2$$reg),
12479               Assembler::ASR,
12480               $src3$$constant & 0x1f);
12481   %}
12482 
12483   ins_pipe(ialu_reg_reg_shift);
12484 %}
12485 
12486 // This pattern is automatically generated from aarch64_ad.m4
12487 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12488 instruct AddL_reg_RShift_reg(iRegLNoSp dst,
12489                          iRegL src1, iRegL src2,
12490                          immI src3) %{
12491   match(Set dst (AddL src1 (RShiftL src2 src3)));
12492 
12493   ins_cost(1.9 * INSN_COST);
12494   format %{ "add  $dst, $src1, $src2, ASR $src3" %}
12495 
12496   ins_encode %{
12497     __ add(as_Register($dst$$reg),
12498               as_Register($src1$$reg),
12499               as_Register($src2$$reg),
12500               Assembler::ASR,
12501               $src3$$constant & 0x3f);
12502   %}
12503 
12504   ins_pipe(ialu_reg_reg_shift);
12505 %}
12506 
12507 // This pattern is automatically generated from aarch64_ad.m4
12508 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12509 instruct AddI_reg_LShift_reg(iRegINoSp dst,
12510                          iRegIorL2I src1, iRegIorL2I src2,
12511                          immI src3) %{
12512   match(Set dst (AddI src1 (LShiftI src2 src3)));
12513 
12514   ins_cost(1.9 * INSN_COST);
12515   format %{ "addw  $dst, $src1, $src2, LSL $src3" %}
12516 
12517   ins_encode %{
12518     __ addw(as_Register($dst$$reg),
12519               as_Register($src1$$reg),
12520               as_Register($src2$$reg),
12521               Assembler::LSL,
12522               $src3$$constant & 0x1f);
12523   %}
12524 
12525   ins_pipe(ialu_reg_reg_shift);
12526 %}
12527 
12528 // This pattern is automatically generated from aarch64_ad.m4
12529 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12530 instruct AddL_reg_LShift_reg(iRegLNoSp dst,
12531                          iRegL src1, iRegL src2,
12532                          immI src3) %{
12533   match(Set dst (AddL src1 (LShiftL src2 src3)));
12534 
12535   ins_cost(1.9 * INSN_COST);
12536   format %{ "add  $dst, $src1, $src2, LSL $src3" %}
12537 
12538   ins_encode %{
12539     __ add(as_Register($dst$$reg),
12540               as_Register($src1$$reg),
12541               as_Register($src2$$reg),
12542               Assembler::LSL,
12543               $src3$$constant & 0x3f);
12544   %}
12545 
12546   ins_pipe(ialu_reg_reg_shift);
12547 %}
12548 
12549 // This pattern is automatically generated from aarch64_ad.m4
12550 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12551 instruct SubI_reg_URShift_reg(iRegINoSp dst,
12552                          iRegIorL2I src1, iRegIorL2I src2,
12553                          immI src3) %{
12554   match(Set dst (SubI src1 (URShiftI src2 src3)));
12555 
12556   ins_cost(1.9 * INSN_COST);
12557   format %{ "subw  $dst, $src1, $src2, LSR $src3" %}
12558 
12559   ins_encode %{
12560     __ subw(as_Register($dst$$reg),
12561               as_Register($src1$$reg),
12562               as_Register($src2$$reg),
12563               Assembler::LSR,
12564               $src3$$constant & 0x1f);
12565   %}
12566 
12567   ins_pipe(ialu_reg_reg_shift);
12568 %}
12569 
12570 // This pattern is automatically generated from aarch64_ad.m4
12571 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12572 instruct SubL_reg_URShift_reg(iRegLNoSp dst,
12573                          iRegL src1, iRegL src2,
12574                          immI src3) %{
12575   match(Set dst (SubL src1 (URShiftL src2 src3)));
12576 
12577   ins_cost(1.9 * INSN_COST);
12578   format %{ "sub  $dst, $src1, $src2, LSR $src3" %}
12579 
12580   ins_encode %{
12581     __ sub(as_Register($dst$$reg),
12582               as_Register($src1$$reg),
12583               as_Register($src2$$reg),
12584               Assembler::LSR,
12585               $src3$$constant & 0x3f);
12586   %}
12587 
12588   ins_pipe(ialu_reg_reg_shift);
12589 %}
12590 
12591 // This pattern is automatically generated from aarch64_ad.m4
12592 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12593 instruct SubI_reg_RShift_reg(iRegINoSp dst,
12594                          iRegIorL2I src1, iRegIorL2I src2,
12595                          immI src3) %{
12596   match(Set dst (SubI src1 (RShiftI src2 src3)));
12597 
12598   ins_cost(1.9 * INSN_COST);
12599   format %{ "subw  $dst, $src1, $src2, ASR $src3" %}
12600 
12601   ins_encode %{
12602     __ subw(as_Register($dst$$reg),
12603               as_Register($src1$$reg),
12604               as_Register($src2$$reg),
12605               Assembler::ASR,
12606               $src3$$constant & 0x1f);
12607   %}
12608 
12609   ins_pipe(ialu_reg_reg_shift);
12610 %}
12611 
12612 // This pattern is automatically generated from aarch64_ad.m4
12613 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12614 instruct SubL_reg_RShift_reg(iRegLNoSp dst,
12615                          iRegL src1, iRegL src2,
12616                          immI src3) %{
12617   match(Set dst (SubL src1 (RShiftL src2 src3)));
12618 
12619   ins_cost(1.9 * INSN_COST);
12620   format %{ "sub  $dst, $src1, $src2, ASR $src3" %}
12621 
12622   ins_encode %{
12623     __ sub(as_Register($dst$$reg),
12624               as_Register($src1$$reg),
12625               as_Register($src2$$reg),
12626               Assembler::ASR,
12627               $src3$$constant & 0x3f);
12628   %}
12629 
12630   ins_pipe(ialu_reg_reg_shift);
12631 %}
12632 
12633 // This pattern is automatically generated from aarch64_ad.m4
12634 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12635 instruct SubI_reg_LShift_reg(iRegINoSp dst,
12636                          iRegIorL2I src1, iRegIorL2I src2,
12637                          immI src3) %{
12638   match(Set dst (SubI src1 (LShiftI src2 src3)));
12639 
12640   ins_cost(1.9 * INSN_COST);
12641   format %{ "subw  $dst, $src1, $src2, LSL $src3" %}
12642 
12643   ins_encode %{
12644     __ subw(as_Register($dst$$reg),
12645               as_Register($src1$$reg),
12646               as_Register($src2$$reg),
12647               Assembler::LSL,
12648               $src3$$constant & 0x1f);
12649   %}
12650 
12651   ins_pipe(ialu_reg_reg_shift);
12652 %}
12653 
12654 // This pattern is automatically generated from aarch64_ad.m4
12655 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12656 instruct SubL_reg_LShift_reg(iRegLNoSp dst,
12657                          iRegL src1, iRegL src2,
12658                          immI src3) %{
12659   match(Set dst (SubL src1 (LShiftL src2 src3)));
12660 
12661   ins_cost(1.9 * INSN_COST);
12662   format %{ "sub  $dst, $src1, $src2, LSL $src3" %}
12663 
12664   ins_encode %{
12665     __ sub(as_Register($dst$$reg),
12666               as_Register($src1$$reg),
12667               as_Register($src2$$reg),
12668               Assembler::LSL,
12669               $src3$$constant & 0x3f);
12670   %}
12671 
12672   ins_pipe(ialu_reg_reg_shift);
12673 %}
12674 
12675 // This pattern is automatically generated from aarch64_ad.m4
12676 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12677 
12678 // Shift Left followed by Shift Right.
12679 // This idiom is used by the compiler for the i2b bytecode etc.
12680 instruct sbfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12681 %{
12682   match(Set dst (RShiftL (LShiftL src lshift_count) rshift_count));
12683   ins_cost(INSN_COST * 2);
12684   format %{ "sbfm  $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12685   ins_encode %{
12686     int lshift = $lshift_count$$constant & 63;
12687     int rshift = $rshift_count$$constant & 63;
12688     int s = 63 - lshift;
12689     int r = (rshift - lshift) & 63;
12690     __ sbfm(as_Register($dst$$reg),
12691             as_Register($src$$reg),
12692             r, s);
12693   %}
12694 
12695   ins_pipe(ialu_reg_shift);
12696 %}
12697 
12698 // This pattern is automatically generated from aarch64_ad.m4
12699 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12700 
12701 // Shift Left followed by Shift Right.
12702 // This idiom is used by the compiler for the i2b bytecode etc.
12703 instruct sbfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12704 %{
12705   match(Set dst (RShiftI (LShiftI src lshift_count) rshift_count));
12706   ins_cost(INSN_COST * 2);
12707   format %{ "sbfmw  $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12708   ins_encode %{
12709     int lshift = $lshift_count$$constant & 31;
12710     int rshift = $rshift_count$$constant & 31;
12711     int s = 31 - lshift;
12712     int r = (rshift - lshift) & 31;
12713     __ sbfmw(as_Register($dst$$reg),
12714             as_Register($src$$reg),
12715             r, s);
12716   %}
12717 
12718   ins_pipe(ialu_reg_shift);
12719 %}
12720 
12721 // This pattern is automatically generated from aarch64_ad.m4
12722 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12723 
12724 // Shift Left followed by Shift Right.
12725 // This idiom is used by the compiler for the i2b bytecode etc.
12726 instruct ubfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12727 %{
12728   match(Set dst (URShiftL (LShiftL src lshift_count) rshift_count));
12729   ins_cost(INSN_COST * 2);
12730   format %{ "ubfm  $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12731   ins_encode %{
12732     int lshift = $lshift_count$$constant & 63;
12733     int rshift = $rshift_count$$constant & 63;
12734     int s = 63 - lshift;
12735     int r = (rshift - lshift) & 63;
12736     __ ubfm(as_Register($dst$$reg),
12737             as_Register($src$$reg),
12738             r, s);
12739   %}
12740 
12741   ins_pipe(ialu_reg_shift);
12742 %}
12743 
12744 // This pattern is automatically generated from aarch64_ad.m4
12745 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12746 
12747 // Shift Left followed by Shift Right.
12748 // This idiom is used by the compiler for the i2b bytecode etc.
12749 instruct ubfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12750 %{
12751   match(Set dst (URShiftI (LShiftI src lshift_count) rshift_count));
12752   ins_cost(INSN_COST * 2);
12753   format %{ "ubfmw  $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12754   ins_encode %{
12755     int lshift = $lshift_count$$constant & 31;
12756     int rshift = $rshift_count$$constant & 31;
12757     int s = 31 - lshift;
12758     int r = (rshift - lshift) & 31;
12759     __ ubfmw(as_Register($dst$$reg),
12760             as_Register($src$$reg),
12761             r, s);
12762   %}
12763 
12764   ins_pipe(ialu_reg_shift);
12765 %}
12766 
12767 // Bitfield extract with shift & mask
12768 
12769 // This pattern is automatically generated from aarch64_ad.m4
12770 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12771 instruct ubfxwI(iRegINoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12772 %{
12773   match(Set dst (AndI (URShiftI src rshift) mask));
12774   // Make sure we are not going to exceed what ubfxw can do.
12775   predicate((exact_log2(n->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12776 
12777   ins_cost(INSN_COST);
12778   format %{ "ubfxw $dst, $src, $rshift, $mask" %}
12779   ins_encode %{
12780     int rshift = $rshift$$constant & 31;
12781     intptr_t mask = $mask$$constant;
12782     int width = exact_log2(mask+1);
12783     __ ubfxw(as_Register($dst$$reg),
12784             as_Register($src$$reg), rshift, width);
12785   %}
12786   ins_pipe(ialu_reg_shift);
12787 %}
12788 
12789 // This pattern is automatically generated from aarch64_ad.m4
12790 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12791 instruct ubfxL(iRegLNoSp dst, iRegL src, immI rshift, immL_bitmask mask)
12792 %{
12793   match(Set dst (AndL (URShiftL src rshift) mask));
12794   // Make sure we are not going to exceed what ubfx can do.
12795   predicate((exact_log2_long(n->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= (63 + 1));
12796 
12797   ins_cost(INSN_COST);
12798   format %{ "ubfx $dst, $src, $rshift, $mask" %}
12799   ins_encode %{
12800     int rshift = $rshift$$constant & 63;
12801     intptr_t mask = $mask$$constant;
12802     int width = exact_log2_long(mask+1);
12803     __ ubfx(as_Register($dst$$reg),
12804             as_Register($src$$reg), rshift, width);
12805   %}
12806   ins_pipe(ialu_reg_shift);
12807 %}
12808 
12809 
12810 // This pattern is automatically generated from aarch64_ad.m4
12811 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12812 
12813 // We can use ubfx when extending an And with a mask when we know mask
12814 // is positive.  We know that because immI_bitmask guarantees it.
12815 instruct ubfxIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12816 %{
12817   match(Set dst (ConvI2L (AndI (URShiftI src rshift) mask)));
12818   // Make sure we are not going to exceed what ubfxw can do.
12819   predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12820 
12821   ins_cost(INSN_COST * 2);
12822   format %{ "ubfx $dst, $src, $rshift, $mask" %}
12823   ins_encode %{
12824     int rshift = $rshift$$constant & 31;
12825     intptr_t mask = $mask$$constant;
12826     int width = exact_log2(mask+1);
12827     __ ubfx(as_Register($dst$$reg),
12828             as_Register($src$$reg), rshift, width);
12829   %}
12830   ins_pipe(ialu_reg_shift);
12831 %}
12832 
12833 
12834 // This pattern is automatically generated from aarch64_ad.m4
12835 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12836 
12837 // We can use ubfiz when masking by a positive number and then left shifting the result.
12838 // We know that the mask is positive because immI_bitmask guarantees it.
12839 instruct ubfizwI(iRegINoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12840 %{
12841   match(Set dst (LShiftI (AndI src mask) lshift));
12842   predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 31)) <= (31 + 1));
12843 
12844   ins_cost(INSN_COST);
12845   format %{ "ubfizw $dst, $src, $lshift, $mask" %}
12846   ins_encode %{
12847     int lshift = $lshift$$constant & 31;
12848     intptr_t mask = $mask$$constant;
12849     int width = exact_log2(mask+1);
12850     __ ubfizw(as_Register($dst$$reg),
12851           as_Register($src$$reg), lshift, width);
12852   %}
12853   ins_pipe(ialu_reg_shift);
12854 %}
12855 
12856 // This pattern is automatically generated from aarch64_ad.m4
12857 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12858 
12859 // We can use ubfiz when masking by a positive number and then left shifting the result.
12860 // We know that the mask is positive because immL_bitmask guarantees it.
12861 instruct ubfizL(iRegLNoSp dst, iRegL src, immI lshift, immL_bitmask mask)
12862 %{
12863   match(Set dst (LShiftL (AndL src mask) lshift));
12864   predicate((exact_log2_long(n->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
12865 
12866   ins_cost(INSN_COST);
12867   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12868   ins_encode %{
12869     int lshift = $lshift$$constant & 63;
12870     intptr_t mask = $mask$$constant;
12871     int width = exact_log2_long(mask+1);
12872     __ ubfiz(as_Register($dst$$reg),
12873           as_Register($src$$reg), lshift, width);
12874   %}
12875   ins_pipe(ialu_reg_shift);
12876 %}
12877 
12878 // This pattern is automatically generated from aarch64_ad.m4
12879 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12880 
12881 // We can use ubfiz when masking by a positive number and then left shifting the result.
12882 // We know that the mask is positive because immI_bitmask guarantees it.
12883 instruct ubfizwIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12884 %{
12885   match(Set dst (ConvI2L (LShiftI (AndI src mask) lshift)));
12886   predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= 31);
12887 
12888   ins_cost(INSN_COST);
12889   format %{ "ubfizw $dst, $src, $lshift, $mask" %}
12890   ins_encode %{
12891     int lshift = $lshift$$constant & 31;
12892     intptr_t mask = $mask$$constant;
12893     int width = exact_log2(mask+1);
12894     __ ubfizw(as_Register($dst$$reg),
12895           as_Register($src$$reg), lshift, width);
12896   %}
12897   ins_pipe(ialu_reg_shift);
12898 %}
12899 
12900 // This pattern is automatically generated from aarch64_ad.m4
12901 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12902 
12903 // We can use ubfiz when masking by a positive number and then left shifting the result.
12904 // We know that the mask is positive because immL_bitmask guarantees it.
12905 instruct ubfizLConvL2I(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
12906 %{
12907   match(Set dst (ConvL2I (LShiftL (AndL src mask) lshift)));
12908   predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= 31);
12909 
12910   ins_cost(INSN_COST);
12911   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12912   ins_encode %{
12913     int lshift = $lshift$$constant & 63;
12914     intptr_t mask = $mask$$constant;
12915     int width = exact_log2_long(mask+1);
12916     __ ubfiz(as_Register($dst$$reg),
12917           as_Register($src$$reg), lshift, width);
12918   %}
12919   ins_pipe(ialu_reg_shift);
12920 %}
12921 
12922 
12923 // This pattern is automatically generated from aarch64_ad.m4
12924 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12925 
12926 // If there is a convert I to L block between and AndI and a LShiftL, we can also match ubfiz
12927 instruct ubfizIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12928 %{
12929   match(Set dst (LShiftL (ConvI2L (AndI src mask)) lshift));
12930   predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
12931 
12932   ins_cost(INSN_COST);
12933   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12934   ins_encode %{
12935     int lshift = $lshift$$constant & 63;
12936     intptr_t mask = $mask$$constant;
12937     int width = exact_log2(mask+1);
12938     __ ubfiz(as_Register($dst$$reg),
12939              as_Register($src$$reg), lshift, width);
12940   %}
12941   ins_pipe(ialu_reg_shift);
12942 %}
12943 
12944 // This pattern is automatically generated from aarch64_ad.m4
12945 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12946 
12947 // If there is a convert L to I block between and AndL and a LShiftI, we can also match ubfiz
12948 instruct ubfizLConvL2Ix(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
12949 %{
12950   match(Set dst (LShiftI (ConvL2I (AndL src mask)) lshift));
12951   predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 31)) <= 31);
12952 
12953   ins_cost(INSN_COST);
12954   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12955   ins_encode %{
12956     int lshift = $lshift$$constant & 31;
12957     intptr_t mask = $mask$$constant;
12958     int width = exact_log2(mask+1);
12959     __ ubfiz(as_Register($dst$$reg),
12960              as_Register($src$$reg), lshift, width);
12961   %}
12962   ins_pipe(ialu_reg_shift);
12963 %}
12964 
12965 // This pattern is automatically generated from aarch64_ad.m4
12966 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12967 
12968 // Can skip int2long conversions after AND with small bitmask
12969 instruct ubfizIConvI2LAndI(iRegLNoSp dst, iRegI src, immI_bitmask msk)
12970 %{
12971   match(Set dst (ConvI2L (AndI src msk)));
12972   ins_cost(INSN_COST);
12973   format %{ "ubfiz $dst, $src, 0, exact_log2($msk + 1) " %}
12974   ins_encode %{
12975     __ ubfiz(as_Register($dst$$reg), as_Register($src$$reg), 0, exact_log2($msk$$constant + 1));
12976   %}
12977   ins_pipe(ialu_reg_shift);
12978 %}
12979 
12980 
12981 // Rotations
12982 
12983 // This pattern is automatically generated from aarch64_ad.m4
12984 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12985 instruct extrOrL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
12986 %{
12987   match(Set dst (OrL (LShiftL src1 lshift) (URShiftL src2 rshift)));
12988   predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
12989 
12990   ins_cost(INSN_COST);
12991   format %{ "extr $dst, $src1, $src2, #$rshift" %}
12992 
12993   ins_encode %{
12994     __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12995             $rshift$$constant & 63);
12996   %}
12997   ins_pipe(ialu_reg_reg_extr);
12998 %}
12999 
13000 
13001 // This pattern is automatically generated from aarch64_ad.m4
13002 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13003 instruct extrOrI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
13004 %{
13005   match(Set dst (OrI (LShiftI src1 lshift) (URShiftI src2 rshift)));
13006   predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
13007 
13008   ins_cost(INSN_COST);
13009   format %{ "extr $dst, $src1, $src2, #$rshift" %}
13010 
13011   ins_encode %{
13012     __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13013             $rshift$$constant & 31);
13014   %}
13015   ins_pipe(ialu_reg_reg_extr);
13016 %}
13017 
13018 
13019 // This pattern is automatically generated from aarch64_ad.m4
13020 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13021 instruct extrAddL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
13022 %{
13023   match(Set dst (AddL (LShiftL src1 lshift) (URShiftL src2 rshift)));
13024   predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
13025 
13026   ins_cost(INSN_COST);
13027   format %{ "extr $dst, $src1, $src2, #$rshift" %}
13028 
13029   ins_encode %{
13030     __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13031             $rshift$$constant & 63);
13032   %}
13033   ins_pipe(ialu_reg_reg_extr);
13034 %}
13035 
13036 
13037 // This pattern is automatically generated from aarch64_ad.m4
13038 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13039 instruct extrAddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
13040 %{
13041   match(Set dst (AddI (LShiftI src1 lshift) (URShiftI src2 rshift)));
13042   predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
13043 
13044   ins_cost(INSN_COST);
13045   format %{ "extr $dst, $src1, $src2, #$rshift" %}
13046 
13047   ins_encode %{
13048     __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13049             $rshift$$constant & 31);
13050   %}
13051   ins_pipe(ialu_reg_reg_extr);
13052 %}
13053 
13054 // This pattern is automatically generated from aarch64_ad.m4
13055 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13056 instruct rorI_imm(iRegINoSp dst, iRegI src, immI shift)
13057 %{
13058   match(Set dst (RotateRight src shift));
13059 
13060   ins_cost(INSN_COST);
13061   format %{ "ror    $dst, $src, $shift" %}
13062 
13063   ins_encode %{
13064      __ extrw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
13065                $shift$$constant & 0x1f);
13066   %}
13067   ins_pipe(ialu_reg_reg_vshift);
13068 %}
13069 
13070 // This pattern is automatically generated from aarch64_ad.m4
13071 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13072 instruct rorL_imm(iRegLNoSp dst, iRegL src, immI shift)
13073 %{
13074   match(Set dst (RotateRight src shift));
13075 
13076   ins_cost(INSN_COST);
13077   format %{ "ror    $dst, $src, $shift" %}
13078 
13079   ins_encode %{
13080      __ extr(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
13081                $shift$$constant & 0x3f);
13082   %}
13083   ins_pipe(ialu_reg_reg_vshift);
13084 %}
13085 
13086 // This pattern is automatically generated from aarch64_ad.m4
13087 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13088 instruct rorI_reg(iRegINoSp dst, iRegI src, iRegI shift)
13089 %{
13090   match(Set dst (RotateRight src shift));
13091 
13092   ins_cost(INSN_COST);
13093   format %{ "ror    $dst, $src, $shift" %}
13094 
13095   ins_encode %{
13096      __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
13097   %}
13098   ins_pipe(ialu_reg_reg_vshift);
13099 %}
13100 
13101 // This pattern is automatically generated from aarch64_ad.m4
13102 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13103 instruct rorL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
13104 %{
13105   match(Set dst (RotateRight src shift));
13106 
13107   ins_cost(INSN_COST);
13108   format %{ "ror    $dst, $src, $shift" %}
13109 
13110   ins_encode %{
13111      __ rorv(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
13112   %}
13113   ins_pipe(ialu_reg_reg_vshift);
13114 %}
13115 
13116 // This pattern is automatically generated from aarch64_ad.m4
13117 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13118 instruct rolI_reg(iRegINoSp dst, iRegI src, iRegI shift)
13119 %{
13120   match(Set dst (RotateLeft src shift));
13121 
13122   ins_cost(INSN_COST);
13123   format %{ "rol    $dst, $src, $shift" %}
13124 
13125   ins_encode %{
13126      __ subw(rscratch1, zr, as_Register($shift$$reg));
13127      __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
13128   %}
13129   ins_pipe(ialu_reg_reg_vshift);
13130 %}
13131 
13132 // This pattern is automatically generated from aarch64_ad.m4
13133 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13134 instruct rolL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
13135 %{
13136   match(Set dst (RotateLeft src shift));
13137 
13138   ins_cost(INSN_COST);
13139   format %{ "rol    $dst, $src, $shift" %}
13140 
13141   ins_encode %{
13142      __ subw(rscratch1, zr, as_Register($shift$$reg));
13143      __ rorv(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
13144   %}
13145   ins_pipe(ialu_reg_reg_vshift);
13146 %}
13147 
13148 
13149 // Add/subtract (extended)
13150 
13151 // This pattern is automatically generated from aarch64_ad.m4
13152 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13153 instruct AddExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
13154 %{
13155   match(Set dst (AddL src1 (ConvI2L src2)));
13156   ins_cost(INSN_COST);
13157   format %{ "add  $dst, $src1, $src2, sxtw" %}
13158 
13159    ins_encode %{
13160      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13161             as_Register($src2$$reg), ext::sxtw);
13162    %}
13163   ins_pipe(ialu_reg_reg);
13164 %}
13165 
13166 // This pattern is automatically generated from aarch64_ad.m4
13167 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13168 instruct SubExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
13169 %{
13170   match(Set dst (SubL src1 (ConvI2L src2)));
13171   ins_cost(INSN_COST);
13172   format %{ "sub  $dst, $src1, $src2, sxtw" %}
13173 
13174    ins_encode %{
13175      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13176             as_Register($src2$$reg), ext::sxtw);
13177    %}
13178   ins_pipe(ialu_reg_reg);
13179 %}
13180 
13181 // This pattern is automatically generated from aarch64_ad.m4
13182 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13183 instruct AddExtI_sxth(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_16 lshift, immI_16 rshift, rFlagsReg cr)
13184 %{
13185   match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
13186   ins_cost(INSN_COST);
13187   format %{ "add  $dst, $src1, $src2, sxth" %}
13188 
13189    ins_encode %{
13190      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13191             as_Register($src2$$reg), ext::sxth);
13192    %}
13193   ins_pipe(ialu_reg_reg);
13194 %}
13195 
13196 // This pattern is automatically generated from aarch64_ad.m4
13197 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13198 instruct AddExtI_sxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
13199 %{
13200   match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
13201   ins_cost(INSN_COST);
13202   format %{ "add  $dst, $src1, $src2, sxtb" %}
13203 
13204    ins_encode %{
13205      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13206             as_Register($src2$$reg), ext::sxtb);
13207    %}
13208   ins_pipe(ialu_reg_reg);
13209 %}
13210 
13211 // This pattern is automatically generated from aarch64_ad.m4
13212 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13213 instruct AddExtI_uxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
13214 %{
13215   match(Set dst (AddI src1 (URShiftI (LShiftI src2 lshift) rshift)));
13216   ins_cost(INSN_COST);
13217   format %{ "add  $dst, $src1, $src2, uxtb" %}
13218 
13219    ins_encode %{
13220      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13221             as_Register($src2$$reg), ext::uxtb);
13222    %}
13223   ins_pipe(ialu_reg_reg);
13224 %}
13225 
13226 // This pattern is automatically generated from aarch64_ad.m4
13227 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13228 instruct AddExtL_sxth(iRegLNoSp dst, iRegL src1, iRegL src2, immI_48 lshift, immI_48 rshift, rFlagsReg cr)
13229 %{
13230   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13231   ins_cost(INSN_COST);
13232   format %{ "add  $dst, $src1, $src2, sxth" %}
13233 
13234    ins_encode %{
13235      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13236             as_Register($src2$$reg), ext::sxth);
13237    %}
13238   ins_pipe(ialu_reg_reg);
13239 %}
13240 
13241 // This pattern is automatically generated from aarch64_ad.m4
13242 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13243 instruct AddExtL_sxtw(iRegLNoSp dst, iRegL src1, iRegL src2, immI_32 lshift, immI_32 rshift, rFlagsReg cr)
13244 %{
13245   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13246   ins_cost(INSN_COST);
13247   format %{ "add  $dst, $src1, $src2, sxtw" %}
13248 
13249    ins_encode %{
13250      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13251             as_Register($src2$$reg), ext::sxtw);
13252    %}
13253   ins_pipe(ialu_reg_reg);
13254 %}
13255 
13256 // This pattern is automatically generated from aarch64_ad.m4
13257 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13258 instruct AddExtL_sxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
13259 %{
13260   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13261   ins_cost(INSN_COST);
13262   format %{ "add  $dst, $src1, $src2, sxtb" %}
13263 
13264    ins_encode %{
13265      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13266             as_Register($src2$$reg), ext::sxtb);
13267    %}
13268   ins_pipe(ialu_reg_reg);
13269 %}
13270 
13271 // This pattern is automatically generated from aarch64_ad.m4
13272 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13273 instruct AddExtL_uxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
13274 %{
13275   match(Set dst (AddL src1 (URShiftL (LShiftL src2 lshift) rshift)));
13276   ins_cost(INSN_COST);
13277   format %{ "add  $dst, $src1, $src2, uxtb" %}
13278 
13279    ins_encode %{
13280      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13281             as_Register($src2$$reg), ext::uxtb);
13282    %}
13283   ins_pipe(ialu_reg_reg);
13284 %}
13285 
13286 // This pattern is automatically generated from aarch64_ad.m4
13287 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13288 instruct AddExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
13289 %{
13290   match(Set dst (AddI src1 (AndI src2 mask)));
13291   ins_cost(INSN_COST);
13292   format %{ "addw  $dst, $src1, $src2, uxtb" %}
13293 
13294    ins_encode %{
13295      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13296             as_Register($src2$$reg), ext::uxtb);
13297    %}
13298   ins_pipe(ialu_reg_reg);
13299 %}
13300 
13301 // This pattern is automatically generated from aarch64_ad.m4
13302 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13303 instruct AddExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
13304 %{
13305   match(Set dst (AddI src1 (AndI src2 mask)));
13306   ins_cost(INSN_COST);
13307   format %{ "addw  $dst, $src1, $src2, uxth" %}
13308 
13309    ins_encode %{
13310      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13311             as_Register($src2$$reg), ext::uxth);
13312    %}
13313   ins_pipe(ialu_reg_reg);
13314 %}
13315 
13316 // This pattern is automatically generated from aarch64_ad.m4
13317 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13318 instruct AddExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
13319 %{
13320   match(Set dst (AddL src1 (AndL src2 mask)));
13321   ins_cost(INSN_COST);
13322   format %{ "add  $dst, $src1, $src2, uxtb" %}
13323 
13324    ins_encode %{
13325      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13326             as_Register($src2$$reg), ext::uxtb);
13327    %}
13328   ins_pipe(ialu_reg_reg);
13329 %}
13330 
13331 // This pattern is automatically generated from aarch64_ad.m4
13332 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13333 instruct AddExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
13334 %{
13335   match(Set dst (AddL src1 (AndL src2 mask)));
13336   ins_cost(INSN_COST);
13337   format %{ "add  $dst, $src1, $src2, uxth" %}
13338 
13339    ins_encode %{
13340      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13341             as_Register($src2$$reg), ext::uxth);
13342    %}
13343   ins_pipe(ialu_reg_reg);
13344 %}
13345 
13346 // This pattern is automatically generated from aarch64_ad.m4
13347 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13348 instruct AddExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
13349 %{
13350   match(Set dst (AddL src1 (AndL src2 mask)));
13351   ins_cost(INSN_COST);
13352   format %{ "add  $dst, $src1, $src2, uxtw" %}
13353 
13354    ins_encode %{
13355      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13356             as_Register($src2$$reg), ext::uxtw);
13357    %}
13358   ins_pipe(ialu_reg_reg);
13359 %}
13360 
13361 // This pattern is automatically generated from aarch64_ad.m4
13362 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13363 instruct SubExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
13364 %{
13365   match(Set dst (SubI src1 (AndI src2 mask)));
13366   ins_cost(INSN_COST);
13367   format %{ "subw  $dst, $src1, $src2, uxtb" %}
13368 
13369    ins_encode %{
13370      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13371             as_Register($src2$$reg), ext::uxtb);
13372    %}
13373   ins_pipe(ialu_reg_reg);
13374 %}
13375 
13376 // This pattern is automatically generated from aarch64_ad.m4
13377 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13378 instruct SubExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
13379 %{
13380   match(Set dst (SubI src1 (AndI src2 mask)));
13381   ins_cost(INSN_COST);
13382   format %{ "subw  $dst, $src1, $src2, uxth" %}
13383 
13384    ins_encode %{
13385      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13386             as_Register($src2$$reg), ext::uxth);
13387    %}
13388   ins_pipe(ialu_reg_reg);
13389 %}
13390 
13391 // This pattern is automatically generated from aarch64_ad.m4
13392 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13393 instruct SubExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
13394 %{
13395   match(Set dst (SubL src1 (AndL src2 mask)));
13396   ins_cost(INSN_COST);
13397   format %{ "sub  $dst, $src1, $src2, uxtb" %}
13398 
13399    ins_encode %{
13400      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13401             as_Register($src2$$reg), ext::uxtb);
13402    %}
13403   ins_pipe(ialu_reg_reg);
13404 %}
13405 
13406 // This pattern is automatically generated from aarch64_ad.m4
13407 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13408 instruct SubExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
13409 %{
13410   match(Set dst (SubL src1 (AndL src2 mask)));
13411   ins_cost(INSN_COST);
13412   format %{ "sub  $dst, $src1, $src2, uxth" %}
13413 
13414    ins_encode %{
13415      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13416             as_Register($src2$$reg), ext::uxth);
13417    %}
13418   ins_pipe(ialu_reg_reg);
13419 %}
13420 
13421 // This pattern is automatically generated from aarch64_ad.m4
13422 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13423 instruct SubExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
13424 %{
13425   match(Set dst (SubL src1 (AndL src2 mask)));
13426   ins_cost(INSN_COST);
13427   format %{ "sub  $dst, $src1, $src2, uxtw" %}
13428 
13429    ins_encode %{
13430      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13431             as_Register($src2$$reg), ext::uxtw);
13432    %}
13433   ins_pipe(ialu_reg_reg);
13434 %}
13435 
13436 
13437 // This pattern is automatically generated from aarch64_ad.m4
13438 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13439 instruct AddExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13440 %{
13441   match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13442   ins_cost(1.9 * INSN_COST);
13443   format %{ "add  $dst, $src1, $src2, sxtb #lshift2" %}
13444 
13445    ins_encode %{
13446      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13447             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13448    %}
13449   ins_pipe(ialu_reg_reg_shift);
13450 %}
13451 
13452 // This pattern is automatically generated from aarch64_ad.m4
13453 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13454 instruct AddExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13455 %{
13456   match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13457   ins_cost(1.9 * INSN_COST);
13458   format %{ "add  $dst, $src1, $src2, sxth #lshift2" %}
13459 
13460    ins_encode %{
13461      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13462             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13463    %}
13464   ins_pipe(ialu_reg_reg_shift);
13465 %}
13466 
13467 // This pattern is automatically generated from aarch64_ad.m4
13468 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13469 instruct AddExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13470 %{
13471   match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13472   ins_cost(1.9 * INSN_COST);
13473   format %{ "add  $dst, $src1, $src2, sxtw #lshift2" %}
13474 
13475    ins_encode %{
13476      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13477             as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13478    %}
13479   ins_pipe(ialu_reg_reg_shift);
13480 %}
13481 
13482 // This pattern is automatically generated from aarch64_ad.m4
13483 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13484 instruct SubExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13485 %{
13486   match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13487   ins_cost(1.9 * INSN_COST);
13488   format %{ "sub  $dst, $src1, $src2, sxtb #lshift2" %}
13489 
13490    ins_encode %{
13491      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13492             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13493    %}
13494   ins_pipe(ialu_reg_reg_shift);
13495 %}
13496 
13497 // This pattern is automatically generated from aarch64_ad.m4
13498 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13499 instruct SubExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13500 %{
13501   match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13502   ins_cost(1.9 * INSN_COST);
13503   format %{ "sub  $dst, $src1, $src2, sxth #lshift2" %}
13504 
13505    ins_encode %{
13506      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13507             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13508    %}
13509   ins_pipe(ialu_reg_reg_shift);
13510 %}
13511 
13512 // This pattern is automatically generated from aarch64_ad.m4
13513 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13514 instruct SubExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13515 %{
13516   match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13517   ins_cost(1.9 * INSN_COST);
13518   format %{ "sub  $dst, $src1, $src2, sxtw #lshift2" %}
13519 
13520    ins_encode %{
13521      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13522             as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13523    %}
13524   ins_pipe(ialu_reg_reg_shift);
13525 %}
13526 
13527 // This pattern is automatically generated from aarch64_ad.m4
13528 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13529 instruct AddExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13530 %{
13531   match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13532   ins_cost(1.9 * INSN_COST);
13533   format %{ "addw  $dst, $src1, $src2, sxtb #lshift2" %}
13534 
13535    ins_encode %{
13536      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13537             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13538    %}
13539   ins_pipe(ialu_reg_reg_shift);
13540 %}
13541 
13542 // This pattern is automatically generated from aarch64_ad.m4
13543 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13544 instruct AddExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13545 %{
13546   match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13547   ins_cost(1.9 * INSN_COST);
13548   format %{ "addw  $dst, $src1, $src2, sxth #lshift2" %}
13549 
13550    ins_encode %{
13551      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13552             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13553    %}
13554   ins_pipe(ialu_reg_reg_shift);
13555 %}
13556 
13557 // This pattern is automatically generated from aarch64_ad.m4
13558 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13559 instruct SubExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13560 %{
13561   match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13562   ins_cost(1.9 * INSN_COST);
13563   format %{ "subw  $dst, $src1, $src2, sxtb #lshift2" %}
13564 
13565    ins_encode %{
13566      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13567             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13568    %}
13569   ins_pipe(ialu_reg_reg_shift);
13570 %}
13571 
13572 // This pattern is automatically generated from aarch64_ad.m4
13573 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13574 instruct SubExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13575 %{
13576   match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13577   ins_cost(1.9 * INSN_COST);
13578   format %{ "subw  $dst, $src1, $src2, sxth #lshift2" %}
13579 
13580    ins_encode %{
13581      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13582             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13583    %}
13584   ins_pipe(ialu_reg_reg_shift);
13585 %}
13586 
13587 // This pattern is automatically generated from aarch64_ad.m4
13588 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13589 instruct AddExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13590 %{
13591   match(Set dst (AddL src1 (LShiftL (ConvI2L src2) lshift)));
13592   ins_cost(1.9 * INSN_COST);
13593   format %{ "add  $dst, $src1, $src2, sxtw #lshift" %}
13594 
13595    ins_encode %{
13596      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13597             as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13598    %}
13599   ins_pipe(ialu_reg_reg_shift);
13600 %}
13601 
13602 // This pattern is automatically generated from aarch64_ad.m4
13603 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13604 instruct SubExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13605 %{
13606   match(Set dst (SubL src1 (LShiftL (ConvI2L src2) lshift)));
13607   ins_cost(1.9 * INSN_COST);
13608   format %{ "sub  $dst, $src1, $src2, sxtw #lshift" %}
13609 
13610    ins_encode %{
13611      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13612             as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13613    %}
13614   ins_pipe(ialu_reg_reg_shift);
13615 %}
13616 
13617 // This pattern is automatically generated from aarch64_ad.m4
13618 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13619 instruct AddExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13620 %{
13621   match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13622   ins_cost(1.9 * INSN_COST);
13623   format %{ "add  $dst, $src1, $src2, uxtb #lshift" %}
13624 
13625    ins_encode %{
13626      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13627             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13628    %}
13629   ins_pipe(ialu_reg_reg_shift);
13630 %}
13631 
13632 // This pattern is automatically generated from aarch64_ad.m4
13633 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13634 instruct AddExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13635 %{
13636   match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13637   ins_cost(1.9 * INSN_COST);
13638   format %{ "add  $dst, $src1, $src2, uxth #lshift" %}
13639 
13640    ins_encode %{
13641      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13642             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13643    %}
13644   ins_pipe(ialu_reg_reg_shift);
13645 %}
13646 
13647 // This pattern is automatically generated from aarch64_ad.m4
13648 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13649 instruct AddExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13650 %{
13651   match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13652   ins_cost(1.9 * INSN_COST);
13653   format %{ "add  $dst, $src1, $src2, uxtw #lshift" %}
13654 
13655    ins_encode %{
13656      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13657             as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13658    %}
13659   ins_pipe(ialu_reg_reg_shift);
13660 %}
13661 
13662 // This pattern is automatically generated from aarch64_ad.m4
13663 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13664 instruct SubExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13665 %{
13666   match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13667   ins_cost(1.9 * INSN_COST);
13668   format %{ "sub  $dst, $src1, $src2, uxtb #lshift" %}
13669 
13670    ins_encode %{
13671      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13672             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13673    %}
13674   ins_pipe(ialu_reg_reg_shift);
13675 %}
13676 
13677 // This pattern is automatically generated from aarch64_ad.m4
13678 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13679 instruct SubExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13680 %{
13681   match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13682   ins_cost(1.9 * INSN_COST);
13683   format %{ "sub  $dst, $src1, $src2, uxth #lshift" %}
13684 
13685    ins_encode %{
13686      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13687             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13688    %}
13689   ins_pipe(ialu_reg_reg_shift);
13690 %}
13691 
13692 // This pattern is automatically generated from aarch64_ad.m4
13693 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13694 instruct SubExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13695 %{
13696   match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13697   ins_cost(1.9 * INSN_COST);
13698   format %{ "sub  $dst, $src1, $src2, uxtw #lshift" %}
13699 
13700    ins_encode %{
13701      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13702             as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13703    %}
13704   ins_pipe(ialu_reg_reg_shift);
13705 %}
13706 
13707 // This pattern is automatically generated from aarch64_ad.m4
13708 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13709 instruct AddExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13710 %{
13711   match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13712   ins_cost(1.9 * INSN_COST);
13713   format %{ "addw  $dst, $src1, $src2, uxtb #lshift" %}
13714 
13715    ins_encode %{
13716      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13717             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13718    %}
13719   ins_pipe(ialu_reg_reg_shift);
13720 %}
13721 
13722 // This pattern is automatically generated from aarch64_ad.m4
13723 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13724 instruct AddExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13725 %{
13726   match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13727   ins_cost(1.9 * INSN_COST);
13728   format %{ "addw  $dst, $src1, $src2, uxth #lshift" %}
13729 
13730    ins_encode %{
13731      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13732             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13733    %}
13734   ins_pipe(ialu_reg_reg_shift);
13735 %}
13736 
13737 // This pattern is automatically generated from aarch64_ad.m4
13738 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13739 instruct SubExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13740 %{
13741   match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13742   ins_cost(1.9 * INSN_COST);
13743   format %{ "subw  $dst, $src1, $src2, uxtb #lshift" %}
13744 
13745    ins_encode %{
13746      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13747             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13748    %}
13749   ins_pipe(ialu_reg_reg_shift);
13750 %}
13751 
13752 // This pattern is automatically generated from aarch64_ad.m4
13753 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13754 instruct SubExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13755 %{
13756   match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13757   ins_cost(1.9 * INSN_COST);
13758   format %{ "subw  $dst, $src1, $src2, uxth #lshift" %}
13759 
13760    ins_encode %{
13761      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13762             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13763    %}
13764   ins_pipe(ialu_reg_reg_shift);
13765 %}
13766 
13767 // This pattern is automatically generated from aarch64_ad.m4
13768 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13769 instruct cmovI_reg_reg_lt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
13770 %{
13771   effect(DEF dst, USE src1, USE src2, USE cr);
13772   ins_cost(INSN_COST * 2);
13773   format %{ "cselw $dst, $src1, $src2 lt\t"  %}
13774 
13775   ins_encode %{
13776     __ cselw($dst$$Register,
13777              $src1$$Register,
13778              $src2$$Register,
13779              Assembler::LT);
13780   %}
13781   ins_pipe(icond_reg_reg);
13782 %}
13783 
13784 // This pattern is automatically generated from aarch64_ad.m4
13785 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13786 instruct cmovI_reg_reg_gt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
13787 %{
13788   effect(DEF dst, USE src1, USE src2, USE cr);
13789   ins_cost(INSN_COST * 2);
13790   format %{ "cselw $dst, $src1, $src2 gt\t"  %}
13791 
13792   ins_encode %{
13793     __ cselw($dst$$Register,
13794              $src1$$Register,
13795              $src2$$Register,
13796              Assembler::GT);
13797   %}
13798   ins_pipe(icond_reg_reg);
13799 %}
13800 
13801 // This pattern is automatically generated from aarch64_ad.m4
13802 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13803 instruct cmovI_reg_imm0_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13804 %{
13805   effect(DEF dst, USE src1, USE cr);
13806   ins_cost(INSN_COST * 2);
13807   format %{ "cselw $dst, $src1, zr lt\t"  %}
13808 
13809   ins_encode %{
13810     __ cselw($dst$$Register,
13811              $src1$$Register,
13812              zr,
13813              Assembler::LT);
13814   %}
13815   ins_pipe(icond_reg);
13816 %}
13817 
13818 // This pattern is automatically generated from aarch64_ad.m4
13819 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13820 instruct cmovI_reg_imm0_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13821 %{
13822   effect(DEF dst, USE src1, USE cr);
13823   ins_cost(INSN_COST * 2);
13824   format %{ "cselw $dst, $src1, zr gt\t"  %}
13825 
13826   ins_encode %{
13827     __ cselw($dst$$Register,
13828              $src1$$Register,
13829              zr,
13830              Assembler::GT);
13831   %}
13832   ins_pipe(icond_reg);
13833 %}
13834 
13835 // This pattern is automatically generated from aarch64_ad.m4
13836 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13837 instruct cmovI_reg_imm1_le(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13838 %{
13839   effect(DEF dst, USE src1, USE cr);
13840   ins_cost(INSN_COST * 2);
13841   format %{ "csincw $dst, $src1, zr le\t"  %}
13842 
13843   ins_encode %{
13844     __ csincw($dst$$Register,
13845              $src1$$Register,
13846              zr,
13847              Assembler::LE);
13848   %}
13849   ins_pipe(icond_reg);
13850 %}
13851 
13852 // This pattern is automatically generated from aarch64_ad.m4
13853 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13854 instruct cmovI_reg_imm1_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13855 %{
13856   effect(DEF dst, USE src1, USE cr);
13857   ins_cost(INSN_COST * 2);
13858   format %{ "csincw $dst, $src1, zr gt\t"  %}
13859 
13860   ins_encode %{
13861     __ csincw($dst$$Register,
13862              $src1$$Register,
13863              zr,
13864              Assembler::GT);
13865   %}
13866   ins_pipe(icond_reg);
13867 %}
13868 
13869 // This pattern is automatically generated from aarch64_ad.m4
13870 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13871 instruct cmovI_reg_immM1_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13872 %{
13873   effect(DEF dst, USE src1, USE cr);
13874   ins_cost(INSN_COST * 2);
13875   format %{ "csinvw $dst, $src1, zr lt\t"  %}
13876 
13877   ins_encode %{
13878     __ csinvw($dst$$Register,
13879              $src1$$Register,
13880              zr,
13881              Assembler::LT);
13882   %}
13883   ins_pipe(icond_reg);
13884 %}
13885 
13886 // This pattern is automatically generated from aarch64_ad.m4
13887 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13888 instruct cmovI_reg_immM1_ge(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13889 %{
13890   effect(DEF dst, USE src1, USE cr);
13891   ins_cost(INSN_COST * 2);
13892   format %{ "csinvw $dst, $src1, zr ge\t"  %}
13893 
13894   ins_encode %{
13895     __ csinvw($dst$$Register,
13896              $src1$$Register,
13897              zr,
13898              Assembler::GE);
13899   %}
13900   ins_pipe(icond_reg);
13901 %}
13902 
13903 // This pattern is automatically generated from aarch64_ad.m4
13904 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13905 instruct minI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
13906 %{
13907   match(Set dst (MinI src imm));
13908   ins_cost(INSN_COST * 3);
13909   expand %{
13910     rFlagsReg cr;
13911     compI_reg_imm0(cr, src);
13912     cmovI_reg_imm0_lt(dst, src, cr);
13913   %}
13914 %}
13915 
13916 // This pattern is automatically generated from aarch64_ad.m4
13917 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13918 instruct minI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
13919 %{
13920   match(Set dst (MinI imm src));
13921   ins_cost(INSN_COST * 3);
13922   expand %{
13923     rFlagsReg cr;
13924     compI_reg_imm0(cr, src);
13925     cmovI_reg_imm0_lt(dst, src, cr);
13926   %}
13927 %}
13928 
13929 // This pattern is automatically generated from aarch64_ad.m4
13930 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13931 instruct minI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
13932 %{
13933   match(Set dst (MinI src imm));
13934   ins_cost(INSN_COST * 3);
13935   expand %{
13936     rFlagsReg cr;
13937     compI_reg_imm0(cr, src);
13938     cmovI_reg_imm1_le(dst, src, cr);
13939   %}
13940 %}
13941 
13942 // This pattern is automatically generated from aarch64_ad.m4
13943 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13944 instruct minI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
13945 %{
13946   match(Set dst (MinI imm src));
13947   ins_cost(INSN_COST * 3);
13948   expand %{
13949     rFlagsReg cr;
13950     compI_reg_imm0(cr, src);
13951     cmovI_reg_imm1_le(dst, src, cr);
13952   %}
13953 %}
13954 
13955 // This pattern is automatically generated from aarch64_ad.m4
13956 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13957 instruct minI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
13958 %{
13959   match(Set dst (MinI src imm));
13960   ins_cost(INSN_COST * 3);
13961   expand %{
13962     rFlagsReg cr;
13963     compI_reg_imm0(cr, src);
13964     cmovI_reg_immM1_lt(dst, src, cr);
13965   %}
13966 %}
13967 
13968 // This pattern is automatically generated from aarch64_ad.m4
13969 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13970 instruct minI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
13971 %{
13972   match(Set dst (MinI imm src));
13973   ins_cost(INSN_COST * 3);
13974   expand %{
13975     rFlagsReg cr;
13976     compI_reg_imm0(cr, src);
13977     cmovI_reg_immM1_lt(dst, src, cr);
13978   %}
13979 %}
13980 
13981 // This pattern is automatically generated from aarch64_ad.m4
13982 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13983 instruct maxI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
13984 %{
13985   match(Set dst (MaxI src imm));
13986   ins_cost(INSN_COST * 3);
13987   expand %{
13988     rFlagsReg cr;
13989     compI_reg_imm0(cr, src);
13990     cmovI_reg_imm0_gt(dst, src, cr);
13991   %}
13992 %}
13993 
13994 // This pattern is automatically generated from aarch64_ad.m4
13995 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13996 instruct maxI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
13997 %{
13998   match(Set dst (MaxI imm src));
13999   ins_cost(INSN_COST * 3);
14000   expand %{
14001     rFlagsReg cr;
14002     compI_reg_imm0(cr, src);
14003     cmovI_reg_imm0_gt(dst, src, cr);
14004   %}
14005 %}
14006 
14007 // This pattern is automatically generated from aarch64_ad.m4
14008 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14009 instruct maxI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
14010 %{
14011   match(Set dst (MaxI src imm));
14012   ins_cost(INSN_COST * 3);
14013   expand %{
14014     rFlagsReg cr;
14015     compI_reg_imm0(cr, src);
14016     cmovI_reg_imm1_gt(dst, src, cr);
14017   %}
14018 %}
14019 
14020 // This pattern is automatically generated from aarch64_ad.m4
14021 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14022 instruct maxI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
14023 %{
14024   match(Set dst (MaxI imm src));
14025   ins_cost(INSN_COST * 3);
14026   expand %{
14027     rFlagsReg cr;
14028     compI_reg_imm0(cr, src);
14029     cmovI_reg_imm1_gt(dst, src, cr);
14030   %}
14031 %}
14032 
14033 // This pattern is automatically generated from aarch64_ad.m4
14034 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14035 instruct maxI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
14036 %{
14037   match(Set dst (MaxI src imm));
14038   ins_cost(INSN_COST * 3);
14039   expand %{
14040     rFlagsReg cr;
14041     compI_reg_imm0(cr, src);
14042     cmovI_reg_immM1_ge(dst, src, cr);
14043   %}
14044 %}
14045 
14046 // This pattern is automatically generated from aarch64_ad.m4
14047 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14048 instruct maxI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
14049 %{
14050   match(Set dst (MaxI imm src));
14051   ins_cost(INSN_COST * 3);
14052   expand %{
14053     rFlagsReg cr;
14054     compI_reg_imm0(cr, src);
14055     cmovI_reg_immM1_ge(dst, src, cr);
14056   %}
14057 %}
14058 
14059 
14060 
14061 // END This section of the file is automatically generated. Do not edit --------------
14062 
14063 
14064 // ============================================================================
14065 // Floating Point Arithmetic Instructions
14066 
14067 instruct addF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14068   match(Set dst (AddF src1 src2));
14069 
14070   ins_cost(INSN_COST * 5);
14071   format %{ "fadds   $dst, $src1, $src2" %}
14072 
14073   ins_encode %{
14074     __ fadds(as_FloatRegister($dst$$reg),
14075              as_FloatRegister($src1$$reg),
14076              as_FloatRegister($src2$$reg));
14077   %}
14078 
14079   ins_pipe(fp_dop_reg_reg_s);
14080 %}
14081 
14082 instruct addD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14083   match(Set dst (AddD src1 src2));
14084 
14085   ins_cost(INSN_COST * 5);
14086   format %{ "faddd   $dst, $src1, $src2" %}
14087 
14088   ins_encode %{
14089     __ faddd(as_FloatRegister($dst$$reg),
14090              as_FloatRegister($src1$$reg),
14091              as_FloatRegister($src2$$reg));
14092   %}
14093 
14094   ins_pipe(fp_dop_reg_reg_d);
14095 %}
14096 
14097 instruct subF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14098   match(Set dst (SubF src1 src2));
14099 
14100   ins_cost(INSN_COST * 5);
14101   format %{ "fsubs   $dst, $src1, $src2" %}
14102 
14103   ins_encode %{
14104     __ fsubs(as_FloatRegister($dst$$reg),
14105              as_FloatRegister($src1$$reg),
14106              as_FloatRegister($src2$$reg));
14107   %}
14108 
14109   ins_pipe(fp_dop_reg_reg_s);
14110 %}
14111 
14112 instruct subD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14113   match(Set dst (SubD src1 src2));
14114 
14115   ins_cost(INSN_COST * 5);
14116   format %{ "fsubd   $dst, $src1, $src2" %}
14117 
14118   ins_encode %{
14119     __ fsubd(as_FloatRegister($dst$$reg),
14120              as_FloatRegister($src1$$reg),
14121              as_FloatRegister($src2$$reg));
14122   %}
14123 
14124   ins_pipe(fp_dop_reg_reg_d);
14125 %}
14126 
14127 instruct mulF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14128   match(Set dst (MulF src1 src2));
14129 
14130   ins_cost(INSN_COST * 6);
14131   format %{ "fmuls   $dst, $src1, $src2" %}
14132 
14133   ins_encode %{
14134     __ fmuls(as_FloatRegister($dst$$reg),
14135              as_FloatRegister($src1$$reg),
14136              as_FloatRegister($src2$$reg));
14137   %}
14138 
14139   ins_pipe(fp_dop_reg_reg_s);
14140 %}
14141 
14142 instruct mulD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14143   match(Set dst (MulD src1 src2));
14144 
14145   ins_cost(INSN_COST * 6);
14146   format %{ "fmuld   $dst, $src1, $src2" %}
14147 
14148   ins_encode %{
14149     __ fmuld(as_FloatRegister($dst$$reg),
14150              as_FloatRegister($src1$$reg),
14151              as_FloatRegister($src2$$reg));
14152   %}
14153 
14154   ins_pipe(fp_dop_reg_reg_d);
14155 %}
14156 
14157 // src1 * src2 + src3
14158 instruct maddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
14159   predicate(UseFMA);
14160   match(Set dst (FmaF src3 (Binary src1 src2)));
14161 
14162   format %{ "fmadds   $dst, $src1, $src2, $src3" %}
14163 
14164   ins_encode %{
14165     __ fmadds(as_FloatRegister($dst$$reg),
14166              as_FloatRegister($src1$$reg),
14167              as_FloatRegister($src2$$reg),
14168              as_FloatRegister($src3$$reg));
14169   %}
14170 
14171   ins_pipe(pipe_class_default);
14172 %}
14173 
14174 // src1 * src2 + src3
14175 instruct maddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
14176   predicate(UseFMA);
14177   match(Set dst (FmaD src3 (Binary src1 src2)));
14178 
14179   format %{ "fmaddd   $dst, $src1, $src2, $src3" %}
14180 
14181   ins_encode %{
14182     __ fmaddd(as_FloatRegister($dst$$reg),
14183              as_FloatRegister($src1$$reg),
14184              as_FloatRegister($src2$$reg),
14185              as_FloatRegister($src3$$reg));
14186   %}
14187 
14188   ins_pipe(pipe_class_default);
14189 %}
14190 
14191 // -src1 * src2 + src3
14192 instruct msubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
14193   predicate(UseFMA);
14194   match(Set dst (FmaF src3 (Binary (NegF src1) src2)));
14195   match(Set dst (FmaF src3 (Binary src1 (NegF src2))));
14196 
14197   format %{ "fmsubs   $dst, $src1, $src2, $src3" %}
14198 
14199   ins_encode %{
14200     __ fmsubs(as_FloatRegister($dst$$reg),
14201               as_FloatRegister($src1$$reg),
14202               as_FloatRegister($src2$$reg),
14203               as_FloatRegister($src3$$reg));
14204   %}
14205 
14206   ins_pipe(pipe_class_default);
14207 %}
14208 
14209 // -src1 * src2 + src3
14210 instruct msubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
14211   predicate(UseFMA);
14212   match(Set dst (FmaD src3 (Binary (NegD src1) src2)));
14213   match(Set dst (FmaD src3 (Binary src1 (NegD src2))));
14214 
14215   format %{ "fmsubd   $dst, $src1, $src2, $src3" %}
14216 
14217   ins_encode %{
14218     __ fmsubd(as_FloatRegister($dst$$reg),
14219               as_FloatRegister($src1$$reg),
14220               as_FloatRegister($src2$$reg),
14221               as_FloatRegister($src3$$reg));
14222   %}
14223 
14224   ins_pipe(pipe_class_default);
14225 %}
14226 
14227 // -src1 * src2 - src3
14228 instruct mnaddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
14229   predicate(UseFMA);
14230   match(Set dst (FmaF (NegF src3) (Binary (NegF src1) src2)));
14231   match(Set dst (FmaF (NegF src3) (Binary src1 (NegF src2))));
14232 
14233   format %{ "fnmadds  $dst, $src1, $src2, $src3" %}
14234 
14235   ins_encode %{
14236     __ fnmadds(as_FloatRegister($dst$$reg),
14237                as_FloatRegister($src1$$reg),
14238                as_FloatRegister($src2$$reg),
14239                as_FloatRegister($src3$$reg));
14240   %}
14241 
14242   ins_pipe(pipe_class_default);
14243 %}
14244 
14245 // -src1 * src2 - src3
14246 instruct mnaddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
14247   predicate(UseFMA);
14248   match(Set dst (FmaD (NegD src3) (Binary (NegD src1) src2)));
14249   match(Set dst (FmaD (NegD src3) (Binary src1 (NegD src2))));
14250 
14251   format %{ "fnmaddd   $dst, $src1, $src2, $src3" %}
14252 
14253   ins_encode %{
14254     __ fnmaddd(as_FloatRegister($dst$$reg),
14255                as_FloatRegister($src1$$reg),
14256                as_FloatRegister($src2$$reg),
14257                as_FloatRegister($src3$$reg));
14258   %}
14259 
14260   ins_pipe(pipe_class_default);
14261 %}
14262 
14263 // src1 * src2 - src3
14264 instruct mnsubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3, immF0 zero) %{
14265   predicate(UseFMA);
14266   match(Set dst (FmaF (NegF src3) (Binary src1 src2)));
14267 
14268   format %{ "fnmsubs  $dst, $src1, $src2, $src3" %}
14269 
14270   ins_encode %{
14271     __ fnmsubs(as_FloatRegister($dst$$reg),
14272                as_FloatRegister($src1$$reg),
14273                as_FloatRegister($src2$$reg),
14274                as_FloatRegister($src3$$reg));
14275   %}
14276 
14277   ins_pipe(pipe_class_default);
14278 %}
14279 
14280 // src1 * src2 - src3
14281 instruct mnsubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3, immD0 zero) %{
14282   predicate(UseFMA);
14283   match(Set dst (FmaD (NegD src3) (Binary src1 src2)));
14284 
14285   format %{ "fnmsubd   $dst, $src1, $src2, $src3" %}
14286 
14287   ins_encode %{
14288   // n.b. insn name should be fnmsubd
14289     __ fnmsub(as_FloatRegister($dst$$reg),
14290               as_FloatRegister($src1$$reg),
14291               as_FloatRegister($src2$$reg),
14292               as_FloatRegister($src3$$reg));
14293   %}
14294 
14295   ins_pipe(pipe_class_default);
14296 %}
14297 
14298 
14299 // Math.max(FF)F
14300 instruct maxF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14301   match(Set dst (MaxF src1 src2));
14302 
14303   format %{ "fmaxs   $dst, $src1, $src2" %}
14304   ins_encode %{
14305     __ fmaxs(as_FloatRegister($dst$$reg),
14306              as_FloatRegister($src1$$reg),
14307              as_FloatRegister($src2$$reg));
14308   %}
14309 
14310   ins_pipe(fp_dop_reg_reg_s);
14311 %}
14312 
14313 // Math.min(FF)F
14314 instruct minF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14315   match(Set dst (MinF src1 src2));
14316 
14317   format %{ "fmins   $dst, $src1, $src2" %}
14318   ins_encode %{
14319     __ fmins(as_FloatRegister($dst$$reg),
14320              as_FloatRegister($src1$$reg),
14321              as_FloatRegister($src2$$reg));
14322   %}
14323 
14324   ins_pipe(fp_dop_reg_reg_s);
14325 %}
14326 
14327 // Math.max(DD)D
14328 instruct maxD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14329   match(Set dst (MaxD src1 src2));
14330 
14331   format %{ "fmaxd   $dst, $src1, $src2" %}
14332   ins_encode %{
14333     __ fmaxd(as_FloatRegister($dst$$reg),
14334              as_FloatRegister($src1$$reg),
14335              as_FloatRegister($src2$$reg));
14336   %}
14337 
14338   ins_pipe(fp_dop_reg_reg_d);
14339 %}
14340 
14341 // Math.min(DD)D
14342 instruct minD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14343   match(Set dst (MinD src1 src2));
14344 
14345   format %{ "fmind   $dst, $src1, $src2" %}
14346   ins_encode %{
14347     __ fmind(as_FloatRegister($dst$$reg),
14348              as_FloatRegister($src1$$reg),
14349              as_FloatRegister($src2$$reg));
14350   %}
14351 
14352   ins_pipe(fp_dop_reg_reg_d);
14353 %}
14354 
14355 
14356 instruct divF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14357   match(Set dst (DivF src1  src2));
14358 
14359   ins_cost(INSN_COST * 18);
14360   format %{ "fdivs   $dst, $src1, $src2" %}
14361 
14362   ins_encode %{
14363     __ fdivs(as_FloatRegister($dst$$reg),
14364              as_FloatRegister($src1$$reg),
14365              as_FloatRegister($src2$$reg));
14366   %}
14367 
14368   ins_pipe(fp_div_s);
14369 %}
14370 
14371 instruct divD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14372   match(Set dst (DivD src1  src2));
14373 
14374   ins_cost(INSN_COST * 32);
14375   format %{ "fdivd   $dst, $src1, $src2" %}
14376 
14377   ins_encode %{
14378     __ fdivd(as_FloatRegister($dst$$reg),
14379              as_FloatRegister($src1$$reg),
14380              as_FloatRegister($src2$$reg));
14381   %}
14382 
14383   ins_pipe(fp_div_d);
14384 %}
14385 
14386 instruct negF_reg_reg(vRegF dst, vRegF src) %{
14387   match(Set dst (NegF src));
14388 
14389   ins_cost(INSN_COST * 3);
14390   format %{ "fneg   $dst, $src" %}
14391 
14392   ins_encode %{
14393     __ fnegs(as_FloatRegister($dst$$reg),
14394              as_FloatRegister($src$$reg));
14395   %}
14396 
14397   ins_pipe(fp_uop_s);
14398 %}
14399 
14400 instruct negD_reg_reg(vRegD dst, vRegD src) %{
14401   match(Set dst (NegD src));
14402 
14403   ins_cost(INSN_COST * 3);
14404   format %{ "fnegd   $dst, $src" %}
14405 
14406   ins_encode %{
14407     __ fnegd(as_FloatRegister($dst$$reg),
14408              as_FloatRegister($src$$reg));
14409   %}
14410 
14411   ins_pipe(fp_uop_d);
14412 %}
14413 
14414 instruct absI_reg(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
14415 %{
14416   match(Set dst (AbsI src));
14417 
14418   effect(KILL cr);
14419   ins_cost(INSN_COST * 2);
14420   format %{ "cmpw  $src, zr\n\t"
14421             "cnegw $dst, $src, Assembler::LT\t# int abs"
14422   %}
14423 
14424   ins_encode %{
14425     __ cmpw(as_Register($src$$reg), zr);
14426     __ cnegw(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14427   %}
14428   ins_pipe(pipe_class_default);
14429 %}
14430 
14431 instruct absL_reg(iRegLNoSp dst, iRegL src, rFlagsReg cr)
14432 %{
14433   match(Set dst (AbsL src));
14434 
14435   effect(KILL cr);
14436   ins_cost(INSN_COST * 2);
14437   format %{ "cmp  $src, zr\n\t"
14438             "cneg $dst, $src, Assembler::LT\t# long abs"
14439   %}
14440 
14441   ins_encode %{
14442     __ cmp(as_Register($src$$reg), zr);
14443     __ cneg(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14444   %}
14445   ins_pipe(pipe_class_default);
14446 %}
14447 
14448 instruct absF_reg(vRegF dst, vRegF src) %{
14449   match(Set dst (AbsF src));
14450 
14451   ins_cost(INSN_COST * 3);
14452   format %{ "fabss   $dst, $src" %}
14453   ins_encode %{
14454     __ fabss(as_FloatRegister($dst$$reg),
14455              as_FloatRegister($src$$reg));
14456   %}
14457 
14458   ins_pipe(fp_uop_s);
14459 %}
14460 
14461 instruct absD_reg(vRegD dst, vRegD src) %{
14462   match(Set dst (AbsD src));
14463 
14464   ins_cost(INSN_COST * 3);
14465   format %{ "fabsd   $dst, $src" %}
14466   ins_encode %{
14467     __ fabsd(as_FloatRegister($dst$$reg),
14468              as_FloatRegister($src$$reg));
14469   %}
14470 
14471   ins_pipe(fp_uop_d);
14472 %}
14473 
14474 instruct absdF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14475   match(Set dst (AbsF (SubF src1 src2)));
14476 
14477   ins_cost(INSN_COST * 3);
14478   format %{ "fabds   $dst, $src1, $src2" %}
14479   ins_encode %{
14480     __ fabds(as_FloatRegister($dst$$reg),
14481              as_FloatRegister($src1$$reg),
14482              as_FloatRegister($src2$$reg));
14483   %}
14484 
14485   ins_pipe(fp_uop_s);
14486 %}
14487 
14488 instruct absdD_reg(vRegD dst, vRegD src1, vRegD src2) %{
14489   match(Set dst (AbsD (SubD src1 src2)));
14490 
14491   ins_cost(INSN_COST * 3);
14492   format %{ "fabdd   $dst, $src1, $src2" %}
14493   ins_encode %{
14494     __ fabdd(as_FloatRegister($dst$$reg),
14495              as_FloatRegister($src1$$reg),
14496              as_FloatRegister($src2$$reg));
14497   %}
14498 
14499   ins_pipe(fp_uop_d);
14500 %}
14501 
14502 instruct sqrtD_reg(vRegD dst, vRegD src) %{
14503   match(Set dst (SqrtD src));
14504 
14505   ins_cost(INSN_COST * 50);
14506   format %{ "fsqrtd  $dst, $src" %}
14507   ins_encode %{
14508     __ fsqrtd(as_FloatRegister($dst$$reg),
14509              as_FloatRegister($src$$reg));
14510   %}
14511 
14512   ins_pipe(fp_div_s);
14513 %}
14514 
14515 instruct sqrtF_reg(vRegF dst, vRegF src) %{
14516   match(Set dst (SqrtF src));
14517 
14518   ins_cost(INSN_COST * 50);
14519   format %{ "fsqrts  $dst, $src" %}
14520   ins_encode %{
14521     __ fsqrts(as_FloatRegister($dst$$reg),
14522              as_FloatRegister($src$$reg));
14523   %}
14524 
14525   ins_pipe(fp_div_d);
14526 %}
14527 
14528 // Math.rint, floor, ceil
14529 instruct roundD_reg(vRegD dst, vRegD src, immI rmode) %{
14530   match(Set dst (RoundDoubleMode src rmode));
14531   format %{ "frint  $dst, $src, $rmode" %}
14532   ins_encode %{
14533     switch ($rmode$$constant) {
14534       case RoundDoubleModeNode::rmode_rint:
14535         __ frintnd(as_FloatRegister($dst$$reg),
14536                    as_FloatRegister($src$$reg));
14537         break;
14538       case RoundDoubleModeNode::rmode_floor:
14539         __ frintmd(as_FloatRegister($dst$$reg),
14540                    as_FloatRegister($src$$reg));
14541         break;
14542       case RoundDoubleModeNode::rmode_ceil:
14543         __ frintpd(as_FloatRegister($dst$$reg),
14544                    as_FloatRegister($src$$reg));
14545         break;
14546     }
14547   %}
14548   ins_pipe(fp_uop_d);
14549 %}
14550 
14551 instruct copySignD_reg(vRegD dst, vRegD src1, vRegD src2, vRegD zero) %{
14552   match(Set dst (CopySignD src1 (Binary src2 zero)));
14553   effect(TEMP_DEF dst, USE src1, USE src2, USE zero);
14554   format %{ "CopySignD  $dst $src1 $src2" %}
14555   ins_encode %{
14556     FloatRegister dst = as_FloatRegister($dst$$reg),
14557                   src1 = as_FloatRegister($src1$$reg),
14558                   src2 = as_FloatRegister($src2$$reg),
14559                   zero = as_FloatRegister($zero$$reg);
14560     __ fnegd(dst, zero);
14561     __ bsl(dst, __ T8B, src2, src1);
14562   %}
14563   ins_pipe(fp_uop_d);
14564 %}
14565 
14566 instruct copySignF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14567   match(Set dst (CopySignF src1 src2));
14568   effect(TEMP_DEF dst, USE src1, USE src2);
14569   format %{ "CopySignF  $dst $src1 $src2" %}
14570   ins_encode %{
14571     FloatRegister dst = as_FloatRegister($dst$$reg),
14572                   src1 = as_FloatRegister($src1$$reg),
14573                   src2 = as_FloatRegister($src2$$reg);
14574     __ movi(dst, __ T2S, 0x80, 24);
14575     __ bsl(dst, __ T8B, src2, src1);
14576   %}
14577   ins_pipe(fp_uop_d);
14578 %}
14579 
14580 instruct signumD_reg(vRegD dst, vRegD src, vRegD zero, vRegD one) %{
14581   match(Set dst (SignumD src (Binary zero one)));
14582   effect(TEMP_DEF dst, USE src, USE zero, USE one);
14583   format %{ "signumD  $dst, $src" %}
14584   ins_encode %{
14585     FloatRegister src = as_FloatRegister($src$$reg),
14586                   dst = as_FloatRegister($dst$$reg),
14587                   zero = as_FloatRegister($zero$$reg),
14588                   one = as_FloatRegister($one$$reg);
14589     __ facgtd(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14590     __ ushrd(dst, dst, 1);     // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14591     // Bit selection instruction gets bit from "one" for each enabled bit in
14592     // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14593     // NaN the whole "src" will be copied because "dst" is zero. For all other
14594     // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14595     // from "src", and all other bits are copied from 1.0.
14596     __ bsl(dst, __ T8B, one, src);
14597   %}
14598   ins_pipe(fp_uop_d);
14599 %}
14600 
14601 instruct signumF_reg(vRegF dst, vRegF src, vRegF zero, vRegF one) %{
14602   match(Set dst (SignumF src (Binary zero one)));
14603   effect(TEMP_DEF dst, USE src, USE zero, USE one);
14604   format %{ "signumF  $dst, $src" %}
14605   ins_encode %{
14606     FloatRegister src = as_FloatRegister($src$$reg),
14607                   dst = as_FloatRegister($dst$$reg),
14608                   zero = as_FloatRegister($zero$$reg),
14609                   one = as_FloatRegister($one$$reg);
14610     __ facgts(dst, src, zero);    // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14611     __ ushr(dst, __ T2S, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14612     // Bit selection instruction gets bit from "one" for each enabled bit in
14613     // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14614     // NaN the whole "src" will be copied because "dst" is zero. For all other
14615     // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14616     // from "src", and all other bits are copied from 1.0.
14617     __ bsl(dst, __ T8B, one, src);
14618   %}
14619   ins_pipe(fp_uop_d);
14620 %}
14621 
14622 instruct onspinwait() %{
14623   match(OnSpinWait);
14624   ins_cost(INSN_COST);
14625 
14626   format %{ "onspinwait" %}
14627 
14628   ins_encode %{
14629     __ spin_wait();
14630   %}
14631   ins_pipe(pipe_class_empty);
14632 %}
14633 
14634 // ============================================================================
14635 // Logical Instructions
14636 
14637 // Integer Logical Instructions
14638 
14639 // And Instructions
14640 
14641 
14642 instruct andI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, rFlagsReg cr) %{
14643   match(Set dst (AndI src1 src2));
14644 
14645   format %{ "andw  $dst, $src1, $src2\t# int" %}
14646 
14647   ins_cost(INSN_COST);
14648   ins_encode %{
14649     __ andw(as_Register($dst$$reg),
14650             as_Register($src1$$reg),
14651             as_Register($src2$$reg));
14652   %}
14653 
14654   ins_pipe(ialu_reg_reg);
14655 %}
14656 
14657 instruct andI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2, rFlagsReg cr) %{
14658   match(Set dst (AndI src1 src2));
14659 
14660   format %{ "andsw  $dst, $src1, $src2\t# int" %}
14661 
14662   ins_cost(INSN_COST);
14663   ins_encode %{
14664     __ andw(as_Register($dst$$reg),
14665             as_Register($src1$$reg),
14666             (uint64_t)($src2$$constant));
14667   %}
14668 
14669   ins_pipe(ialu_reg_imm);
14670 %}
14671 
14672 // Or Instructions
14673 
14674 instruct orI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14675   match(Set dst (OrI src1 src2));
14676 
14677   format %{ "orrw  $dst, $src1, $src2\t# int" %}
14678 
14679   ins_cost(INSN_COST);
14680   ins_encode %{
14681     __ orrw(as_Register($dst$$reg),
14682             as_Register($src1$$reg),
14683             as_Register($src2$$reg));
14684   %}
14685 
14686   ins_pipe(ialu_reg_reg);
14687 %}
14688 
14689 instruct orI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14690   match(Set dst (OrI src1 src2));
14691 
14692   format %{ "orrw  $dst, $src1, $src2\t# int" %}
14693 
14694   ins_cost(INSN_COST);
14695   ins_encode %{
14696     __ orrw(as_Register($dst$$reg),
14697             as_Register($src1$$reg),
14698             (uint64_t)($src2$$constant));
14699   %}
14700 
14701   ins_pipe(ialu_reg_imm);
14702 %}
14703 
14704 // Xor Instructions
14705 
14706 instruct xorI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14707   match(Set dst (XorI src1 src2));
14708 
14709   format %{ "eorw  $dst, $src1, $src2\t# int" %}
14710 
14711   ins_cost(INSN_COST);
14712   ins_encode %{
14713     __ eorw(as_Register($dst$$reg),
14714             as_Register($src1$$reg),
14715             as_Register($src2$$reg));
14716   %}
14717 
14718   ins_pipe(ialu_reg_reg);
14719 %}
14720 
14721 instruct xorI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14722   match(Set dst (XorI src1 src2));
14723 
14724   format %{ "eorw  $dst, $src1, $src2\t# int" %}
14725 
14726   ins_cost(INSN_COST);
14727   ins_encode %{
14728     __ eorw(as_Register($dst$$reg),
14729             as_Register($src1$$reg),
14730             (uint64_t)($src2$$constant));
14731   %}
14732 
14733   ins_pipe(ialu_reg_imm);
14734 %}
14735 
14736 // Long Logical Instructions
14737 // TODO
14738 
14739 instruct andL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr) %{
14740   match(Set dst (AndL src1 src2));
14741 
14742   format %{ "and  $dst, $src1, $src2\t# int" %}
14743 
14744   ins_cost(INSN_COST);
14745   ins_encode %{
14746     __ andr(as_Register($dst$$reg),
14747             as_Register($src1$$reg),
14748             as_Register($src2$$reg));
14749   %}
14750 
14751   ins_pipe(ialu_reg_reg);
14752 %}
14753 
14754 instruct andL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2, rFlagsReg cr) %{
14755   match(Set dst (AndL src1 src2));
14756 
14757   format %{ "and  $dst, $src1, $src2\t# int" %}
14758 
14759   ins_cost(INSN_COST);
14760   ins_encode %{
14761     __ andr(as_Register($dst$$reg),
14762             as_Register($src1$$reg),
14763             (uint64_t)($src2$$constant));
14764   %}
14765 
14766   ins_pipe(ialu_reg_imm);
14767 %}
14768 
14769 // Or Instructions
14770 
14771 instruct orL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14772   match(Set dst (OrL src1 src2));
14773 
14774   format %{ "orr  $dst, $src1, $src2\t# int" %}
14775 
14776   ins_cost(INSN_COST);
14777   ins_encode %{
14778     __ orr(as_Register($dst$$reg),
14779            as_Register($src1$$reg),
14780            as_Register($src2$$reg));
14781   %}
14782 
14783   ins_pipe(ialu_reg_reg);
14784 %}
14785 
14786 instruct orL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14787   match(Set dst (OrL src1 src2));
14788 
14789   format %{ "orr  $dst, $src1, $src2\t# int" %}
14790 
14791   ins_cost(INSN_COST);
14792   ins_encode %{
14793     __ orr(as_Register($dst$$reg),
14794            as_Register($src1$$reg),
14795            (uint64_t)($src2$$constant));
14796   %}
14797 
14798   ins_pipe(ialu_reg_imm);
14799 %}
14800 
14801 // Xor Instructions
14802 
14803 instruct xorL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14804   match(Set dst (XorL src1 src2));
14805 
14806   format %{ "eor  $dst, $src1, $src2\t# int" %}
14807 
14808   ins_cost(INSN_COST);
14809   ins_encode %{
14810     __ eor(as_Register($dst$$reg),
14811            as_Register($src1$$reg),
14812            as_Register($src2$$reg));
14813   %}
14814 
14815   ins_pipe(ialu_reg_reg);
14816 %}
14817 
14818 instruct xorL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14819   match(Set dst (XorL src1 src2));
14820 
14821   ins_cost(INSN_COST);
14822   format %{ "eor  $dst, $src1, $src2\t# int" %}
14823 
14824   ins_encode %{
14825     __ eor(as_Register($dst$$reg),
14826            as_Register($src1$$reg),
14827            (uint64_t)($src2$$constant));
14828   %}
14829 
14830   ins_pipe(ialu_reg_imm);
14831 %}
14832 
14833 instruct convI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src)
14834 %{
14835   match(Set dst (ConvI2L src));
14836 
14837   ins_cost(INSN_COST);
14838   format %{ "sxtw  $dst, $src\t# i2l" %}
14839   ins_encode %{
14840     __ sbfm($dst$$Register, $src$$Register, 0, 31);
14841   %}
14842   ins_pipe(ialu_reg_shift);
14843 %}
14844 
14845 // this pattern occurs in bigmath arithmetic
14846 instruct convUI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src, immL_32bits mask)
14847 %{
14848   match(Set dst (AndL (ConvI2L src) mask));
14849 
14850   ins_cost(INSN_COST);
14851   format %{ "ubfm  $dst, $src, 0, 31\t# ui2l" %}
14852   ins_encode %{
14853     __ ubfm($dst$$Register, $src$$Register, 0, 31);
14854   %}
14855 
14856   ins_pipe(ialu_reg_shift);
14857 %}
14858 
14859 instruct convL2I_reg(iRegINoSp dst, iRegL src) %{
14860   match(Set dst (ConvL2I src));
14861 
14862   ins_cost(INSN_COST);
14863   format %{ "movw  $dst, $src \t// l2i" %}
14864 
14865   ins_encode %{
14866     __ movw(as_Register($dst$$reg), as_Register($src$$reg));
14867   %}
14868 
14869   ins_pipe(ialu_reg);
14870 %}
14871 
14872 instruct convI2B(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
14873 %{
14874   match(Set dst (Conv2B src));
14875   effect(KILL cr);
14876 
14877   format %{
14878     "cmpw $src, zr\n\t"
14879     "cset $dst, ne"
14880   %}
14881 
14882   ins_encode %{
14883     __ cmpw(as_Register($src$$reg), zr);
14884     __ cset(as_Register($dst$$reg), Assembler::NE);
14885   %}
14886 
14887   ins_pipe(ialu_reg);
14888 %}
14889 
14890 instruct convP2B(iRegINoSp dst, iRegP src, rFlagsReg cr)
14891 %{
14892   match(Set dst (Conv2B src));
14893   effect(KILL cr);
14894 
14895   format %{
14896     "cmp  $src, zr\n\t"
14897     "cset $dst, ne"
14898   %}
14899 
14900   ins_encode %{
14901     __ cmp(as_Register($src$$reg), zr);
14902     __ cset(as_Register($dst$$reg), Assembler::NE);
14903   %}
14904 
14905   ins_pipe(ialu_reg);
14906 %}
14907 
14908 instruct convD2F_reg(vRegF dst, vRegD src) %{
14909   match(Set dst (ConvD2F src));
14910 
14911   ins_cost(INSN_COST * 5);
14912   format %{ "fcvtd  $dst, $src \t// d2f" %}
14913 
14914   ins_encode %{
14915     __ fcvtd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14916   %}
14917 
14918   ins_pipe(fp_d2f);
14919 %}
14920 
14921 instruct convF2D_reg(vRegD dst, vRegF src) %{
14922   match(Set dst (ConvF2D src));
14923 
14924   ins_cost(INSN_COST * 5);
14925   format %{ "fcvts  $dst, $src \t// f2d" %}
14926 
14927   ins_encode %{
14928     __ fcvts(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14929   %}
14930 
14931   ins_pipe(fp_f2d);
14932 %}
14933 
14934 instruct convF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14935   match(Set dst (ConvF2I src));
14936 
14937   ins_cost(INSN_COST * 5);
14938   format %{ "fcvtzsw  $dst, $src \t// f2i" %}
14939 
14940   ins_encode %{
14941     __ fcvtzsw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14942   %}
14943 
14944   ins_pipe(fp_f2i);
14945 %}
14946 
14947 instruct convF2L_reg_reg(iRegLNoSp dst, vRegF src) %{
14948   match(Set dst (ConvF2L src));
14949 
14950   ins_cost(INSN_COST * 5);
14951   format %{ "fcvtzs  $dst, $src \t// f2l" %}
14952 
14953   ins_encode %{
14954     __ fcvtzs(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14955   %}
14956 
14957   ins_pipe(fp_f2l);
14958 %}
14959 
14960 instruct convF2HF_reg_reg(iRegINoSp dst, vRegF src, vRegF tmp) %{
14961   match(Set dst (ConvF2HF src));
14962   format %{ "fcvt $tmp, $src\t# convert single to half precision\n\t"
14963             "smov $dst, $tmp\t# move result from $tmp to $dst"
14964   %}
14965   effect(TEMP tmp);
14966   ins_encode %{
14967       __ fcvtsh($tmp$$FloatRegister, $src$$FloatRegister);
14968       __ smov($dst$$Register, $tmp$$FloatRegister, __ H, 0);
14969   %}
14970   ins_pipe(pipe_slow);
14971 %}
14972 
14973 instruct convHF2F_reg_reg(vRegF dst, iRegINoSp src, vRegF tmp) %{
14974   match(Set dst (ConvHF2F src));
14975   format %{ "mov $tmp, $src\t# move source from $src to $tmp\n\t"
14976             "fcvt $dst, $tmp\t# convert half to single precision"
14977   %}
14978   effect(TEMP tmp);
14979   ins_encode %{
14980       __ mov($tmp$$FloatRegister, __ H, 0, $src$$Register);
14981       __ fcvths($dst$$FloatRegister, $tmp$$FloatRegister);
14982   %}
14983   ins_pipe(pipe_slow);
14984 %}
14985 
14986 instruct convI2F_reg_reg(vRegF dst, iRegIorL2I src) %{
14987   match(Set dst (ConvI2F src));
14988 
14989   ins_cost(INSN_COST * 5);
14990   format %{ "scvtfws  $dst, $src \t// i2f" %}
14991 
14992   ins_encode %{
14993     __ scvtfws(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14994   %}
14995 
14996   ins_pipe(fp_i2f);
14997 %}
14998 
14999 instruct convL2F_reg_reg(vRegF dst, iRegL src) %{
15000   match(Set dst (ConvL2F src));
15001 
15002   ins_cost(INSN_COST * 5);
15003   format %{ "scvtfs  $dst, $src \t// l2f" %}
15004 
15005   ins_encode %{
15006     __ scvtfs(as_FloatRegister($dst$$reg), as_Register($src$$reg));
15007   %}
15008 
15009   ins_pipe(fp_l2f);
15010 %}
15011 
15012 instruct convD2I_reg_reg(iRegINoSp dst, vRegD src) %{
15013   match(Set dst (ConvD2I src));
15014 
15015   ins_cost(INSN_COST * 5);
15016   format %{ "fcvtzdw  $dst, $src \t// d2i" %}
15017 
15018   ins_encode %{
15019     __ fcvtzdw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
15020   %}
15021 
15022   ins_pipe(fp_d2i);
15023 %}
15024 
15025 instruct convD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
15026   match(Set dst (ConvD2L src));
15027 
15028   ins_cost(INSN_COST * 5);
15029   format %{ "fcvtzd  $dst, $src \t// d2l" %}
15030 
15031   ins_encode %{
15032     __ fcvtzd(as_Register($dst$$reg), as_FloatRegister($src$$reg));
15033   %}
15034 
15035   ins_pipe(fp_d2l);
15036 %}
15037 
15038 instruct convI2D_reg_reg(vRegD dst, iRegIorL2I src) %{
15039   match(Set dst (ConvI2D src));
15040 
15041   ins_cost(INSN_COST * 5);
15042   format %{ "scvtfwd  $dst, $src \t// i2d" %}
15043 
15044   ins_encode %{
15045     __ scvtfwd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
15046   %}
15047 
15048   ins_pipe(fp_i2d);
15049 %}
15050 
15051 instruct convL2D_reg_reg(vRegD dst, iRegL src) %{
15052   match(Set dst (ConvL2D src));
15053 
15054   ins_cost(INSN_COST * 5);
15055   format %{ "scvtfd  $dst, $src \t// l2d" %}
15056 
15057   ins_encode %{
15058     __ scvtfd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
15059   %}
15060 
15061   ins_pipe(fp_l2d);
15062 %}
15063 
15064 instruct round_double_reg(iRegLNoSp dst, vRegD src, vRegD ftmp, rFlagsReg cr)
15065 %{
15066   match(Set dst (RoundD src));
15067   effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
15068   format %{ "java_round_double $dst,$src"%}
15069   ins_encode %{
15070     __ java_round_double($dst$$Register, as_FloatRegister($src$$reg),
15071                          as_FloatRegister($ftmp$$reg));
15072   %}
15073   ins_pipe(pipe_slow);
15074 %}
15075 
15076 instruct round_float_reg(iRegINoSp dst, vRegF src, vRegF ftmp, rFlagsReg cr)
15077 %{
15078   match(Set dst (RoundF src));
15079   effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
15080   format %{ "java_round_float $dst,$src"%}
15081   ins_encode %{
15082     __ java_round_float($dst$$Register, as_FloatRegister($src$$reg),
15083                         as_FloatRegister($ftmp$$reg));
15084   %}
15085   ins_pipe(pipe_slow);
15086 %}
15087 
15088 // stack <-> reg and reg <-> reg shuffles with no conversion
15089 
15090 instruct MoveF2I_stack_reg(iRegINoSp dst, stackSlotF src) %{
15091 
15092   match(Set dst (MoveF2I src));
15093 
15094   effect(DEF dst, USE src);
15095 
15096   ins_cost(4 * INSN_COST);
15097 
15098   format %{ "ldrw $dst, $src\t# MoveF2I_stack_reg" %}
15099 
15100   ins_encode %{
15101     __ ldrw($dst$$Register, Address(sp, $src$$disp));
15102   %}
15103 
15104   ins_pipe(iload_reg_reg);
15105 
15106 %}
15107 
15108 instruct MoveI2F_stack_reg(vRegF dst, stackSlotI src) %{
15109 
15110   match(Set dst (MoveI2F src));
15111 
15112   effect(DEF dst, USE src);
15113 
15114   ins_cost(4 * INSN_COST);
15115 
15116   format %{ "ldrs $dst, $src\t# MoveI2F_stack_reg" %}
15117 
15118   ins_encode %{
15119     __ ldrs(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
15120   %}
15121 
15122   ins_pipe(pipe_class_memory);
15123 
15124 %}
15125 
15126 instruct MoveD2L_stack_reg(iRegLNoSp dst, stackSlotD src) %{
15127 
15128   match(Set dst (MoveD2L src));
15129 
15130   effect(DEF dst, USE src);
15131 
15132   ins_cost(4 * INSN_COST);
15133 
15134   format %{ "ldr $dst, $src\t# MoveD2L_stack_reg" %}
15135 
15136   ins_encode %{
15137     __ ldr($dst$$Register, Address(sp, $src$$disp));
15138   %}
15139 
15140   ins_pipe(iload_reg_reg);
15141 
15142 %}
15143 
15144 instruct MoveL2D_stack_reg(vRegD dst, stackSlotL src) %{
15145 
15146   match(Set dst (MoveL2D src));
15147 
15148   effect(DEF dst, USE src);
15149 
15150   ins_cost(4 * INSN_COST);
15151 
15152   format %{ "ldrd $dst, $src\t# MoveL2D_stack_reg" %}
15153 
15154   ins_encode %{
15155     __ ldrd(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
15156   %}
15157 
15158   ins_pipe(pipe_class_memory);
15159 
15160 %}
15161 
15162 instruct MoveF2I_reg_stack(stackSlotI dst, vRegF src) %{
15163 
15164   match(Set dst (MoveF2I src));
15165 
15166   effect(DEF dst, USE src);
15167 
15168   ins_cost(INSN_COST);
15169 
15170   format %{ "strs $src, $dst\t# MoveF2I_reg_stack" %}
15171 
15172   ins_encode %{
15173     __ strs(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
15174   %}
15175 
15176   ins_pipe(pipe_class_memory);
15177 
15178 %}
15179 
15180 instruct MoveI2F_reg_stack(stackSlotF dst, iRegI src) %{
15181 
15182   match(Set dst (MoveI2F src));
15183 
15184   effect(DEF dst, USE src);
15185 
15186   ins_cost(INSN_COST);
15187 
15188   format %{ "strw $src, $dst\t# MoveI2F_reg_stack" %}
15189 
15190   ins_encode %{
15191     __ strw($src$$Register, Address(sp, $dst$$disp));
15192   %}
15193 
15194   ins_pipe(istore_reg_reg);
15195 
15196 %}
15197 
15198 instruct MoveD2L_reg_stack(stackSlotL dst, vRegD src) %{
15199 
15200   match(Set dst (MoveD2L src));
15201 
15202   effect(DEF dst, USE src);
15203 
15204   ins_cost(INSN_COST);
15205 
15206   format %{ "strd $dst, $src\t# MoveD2L_reg_stack" %}
15207 
15208   ins_encode %{
15209     __ strd(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
15210   %}
15211 
15212   ins_pipe(pipe_class_memory);
15213 
15214 %}
15215 
15216 instruct MoveL2D_reg_stack(stackSlotD dst, iRegL src) %{
15217 
15218   match(Set dst (MoveL2D src));
15219 
15220   effect(DEF dst, USE src);
15221 
15222   ins_cost(INSN_COST);
15223 
15224   format %{ "str $src, $dst\t# MoveL2D_reg_stack" %}
15225 
15226   ins_encode %{
15227     __ str($src$$Register, Address(sp, $dst$$disp));
15228   %}
15229 
15230   ins_pipe(istore_reg_reg);
15231 
15232 %}
15233 
15234 instruct MoveF2I_reg_reg(iRegINoSp dst, vRegF src) %{
15235 
15236   match(Set dst (MoveF2I src));
15237 
15238   effect(DEF dst, USE src);
15239 
15240   ins_cost(INSN_COST);
15241 
15242   format %{ "fmovs $dst, $src\t# MoveF2I_reg_reg" %}
15243 
15244   ins_encode %{
15245     __ fmovs($dst$$Register, as_FloatRegister($src$$reg));
15246   %}
15247 
15248   ins_pipe(fp_f2i);
15249 
15250 %}
15251 
15252 instruct MoveI2F_reg_reg(vRegF dst, iRegI src) %{
15253 
15254   match(Set dst (MoveI2F src));
15255 
15256   effect(DEF dst, USE src);
15257 
15258   ins_cost(INSN_COST);
15259 
15260   format %{ "fmovs $dst, $src\t# MoveI2F_reg_reg" %}
15261 
15262   ins_encode %{
15263     __ fmovs(as_FloatRegister($dst$$reg), $src$$Register);
15264   %}
15265 
15266   ins_pipe(fp_i2f);
15267 
15268 %}
15269 
15270 instruct MoveD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
15271 
15272   match(Set dst (MoveD2L src));
15273 
15274   effect(DEF dst, USE src);
15275 
15276   ins_cost(INSN_COST);
15277 
15278   format %{ "fmovd $dst, $src\t# MoveD2L_reg_reg" %}
15279 
15280   ins_encode %{
15281     __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
15282   %}
15283 
15284   ins_pipe(fp_d2l);
15285 
15286 %}
15287 
15288 instruct MoveL2D_reg_reg(vRegD dst, iRegL src) %{
15289 
15290   match(Set dst (MoveL2D src));
15291 
15292   effect(DEF dst, USE src);
15293 
15294   ins_cost(INSN_COST);
15295 
15296   format %{ "fmovd $dst, $src\t# MoveL2D_reg_reg" %}
15297 
15298   ins_encode %{
15299     __ fmovd(as_FloatRegister($dst$$reg), $src$$Register);
15300   %}
15301 
15302   ins_pipe(fp_l2d);
15303 
15304 %}
15305 
15306 // ============================================================================
15307 // clearing of an array
15308 
15309 instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, Universe dummy, rFlagsReg cr)
15310 %{
15311   match(Set dummy (ClearArray cnt base));
15312   effect(USE_KILL cnt, USE_KILL base, KILL cr);
15313 
15314   ins_cost(4 * INSN_COST);
15315   format %{ "ClearArray $cnt, $base" %}
15316 
15317   ins_encode %{
15318     address tpc = __ zero_words($base$$Register, $cnt$$Register);
15319     if (tpc == NULL) {
15320       ciEnv::current()->record_failure("CodeCache is full");
15321       return;
15322     }
15323   %}
15324 
15325   ins_pipe(pipe_class_memory);
15326 %}
15327 
15328 instruct clearArray_imm_reg(immL cnt, iRegP_R10 base, iRegL_R11 temp, Universe dummy, rFlagsReg cr)
15329 %{
15330   predicate((uint64_t)n->in(2)->get_long()
15331             < (uint64_t)(BlockZeroingLowLimit >> LogBytesPerWord));
15332   match(Set dummy (ClearArray cnt base));
15333   effect(TEMP temp, USE_KILL base, KILL cr);
15334 
15335   ins_cost(4 * INSN_COST);
15336   format %{ "ClearArray $cnt, $base" %}
15337 
15338   ins_encode %{
15339     address tpc = __ zero_words($base$$Register, (uint64_t)$cnt$$constant);
15340     if (tpc == NULL) {
15341       ciEnv::current()->record_failure("CodeCache is full");
15342       return;
15343     }
15344   %}
15345 
15346   ins_pipe(pipe_class_memory);
15347 %}
15348 
15349 // ============================================================================
15350 // Overflow Math Instructions
15351 
15352 instruct overflowAddI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15353 %{
15354   match(Set cr (OverflowAddI op1 op2));
15355 
15356   format %{ "cmnw  $op1, $op2\t# overflow check int" %}
15357   ins_cost(INSN_COST);
15358   ins_encode %{
15359     __ cmnw($op1$$Register, $op2$$Register);
15360   %}
15361 
15362   ins_pipe(icmp_reg_reg);
15363 %}
15364 
15365 instruct overflowAddI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
15366 %{
15367   match(Set cr (OverflowAddI op1 op2));
15368 
15369   format %{ "cmnw  $op1, $op2\t# overflow check int" %}
15370   ins_cost(INSN_COST);
15371   ins_encode %{
15372     __ cmnw($op1$$Register, $op2$$constant);
15373   %}
15374 
15375   ins_pipe(icmp_reg_imm);
15376 %}
15377 
15378 instruct overflowAddL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15379 %{
15380   match(Set cr (OverflowAddL op1 op2));
15381 
15382   format %{ "cmn   $op1, $op2\t# overflow check long" %}
15383   ins_cost(INSN_COST);
15384   ins_encode %{
15385     __ cmn($op1$$Register, $op2$$Register);
15386   %}
15387 
15388   ins_pipe(icmp_reg_reg);
15389 %}
15390 
15391 instruct overflowAddL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
15392 %{
15393   match(Set cr (OverflowAddL op1 op2));
15394 
15395   format %{ "adds  zr, $op1, $op2\t# overflow check long" %}
15396   ins_cost(INSN_COST);
15397   ins_encode %{
15398     __ adds(zr, $op1$$Register, $op2$$constant);
15399   %}
15400 
15401   ins_pipe(icmp_reg_imm);
15402 %}
15403 
15404 instruct overflowSubI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15405 %{
15406   match(Set cr (OverflowSubI op1 op2));
15407 
15408   format %{ "cmpw  $op1, $op2\t# overflow check int" %}
15409   ins_cost(INSN_COST);
15410   ins_encode %{
15411     __ cmpw($op1$$Register, $op2$$Register);
15412   %}
15413 
15414   ins_pipe(icmp_reg_reg);
15415 %}
15416 
15417 instruct overflowSubI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
15418 %{
15419   match(Set cr (OverflowSubI op1 op2));
15420 
15421   format %{ "cmpw  $op1, $op2\t# overflow check int" %}
15422   ins_cost(INSN_COST);
15423   ins_encode %{
15424     __ cmpw($op1$$Register, $op2$$constant);
15425   %}
15426 
15427   ins_pipe(icmp_reg_imm);
15428 %}
15429 
15430 instruct overflowSubL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15431 %{
15432   match(Set cr (OverflowSubL op1 op2));
15433 
15434   format %{ "cmp   $op1, $op2\t# overflow check long" %}
15435   ins_cost(INSN_COST);
15436   ins_encode %{
15437     __ cmp($op1$$Register, $op2$$Register);
15438   %}
15439 
15440   ins_pipe(icmp_reg_reg);
15441 %}
15442 
15443 instruct overflowSubL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
15444 %{
15445   match(Set cr (OverflowSubL op1 op2));
15446 
15447   format %{ "cmp   $op1, $op2\t# overflow check long" %}
15448   ins_cost(INSN_COST);
15449   ins_encode %{
15450     __ subs(zr, $op1$$Register, $op2$$constant);
15451   %}
15452 
15453   ins_pipe(icmp_reg_imm);
15454 %}
15455 
15456 instruct overflowNegI_reg(rFlagsReg cr, immI0 zero, iRegIorL2I op1)
15457 %{
15458   match(Set cr (OverflowSubI zero op1));
15459 
15460   format %{ "cmpw  zr, $op1\t# overflow check int" %}
15461   ins_cost(INSN_COST);
15462   ins_encode %{
15463     __ cmpw(zr, $op1$$Register);
15464   %}
15465 
15466   ins_pipe(icmp_reg_imm);
15467 %}
15468 
15469 instruct overflowNegL_reg(rFlagsReg cr, immI0 zero, iRegL op1)
15470 %{
15471   match(Set cr (OverflowSubL zero op1));
15472 
15473   format %{ "cmp   zr, $op1\t# overflow check long" %}
15474   ins_cost(INSN_COST);
15475   ins_encode %{
15476     __ cmp(zr, $op1$$Register);
15477   %}
15478 
15479   ins_pipe(icmp_reg_imm);
15480 %}
15481 
15482 instruct overflowMulI_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15483 %{
15484   match(Set cr (OverflowMulI op1 op2));
15485 
15486   format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15487             "cmp   rscratch1, rscratch1, sxtw\n\t"
15488             "movw  rscratch1, #0x80000000\n\t"
15489             "cselw rscratch1, rscratch1, zr, NE\n\t"
15490             "cmpw  rscratch1, #1" %}
15491   ins_cost(5 * INSN_COST);
15492   ins_encode %{
15493     __ smull(rscratch1, $op1$$Register, $op2$$Register);
15494     __ subs(zr, rscratch1, rscratch1, ext::sxtw);      // NE => overflow
15495     __ movw(rscratch1, 0x80000000);                    // Develop 0 (EQ),
15496     __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15497     __ cmpw(rscratch1, 1);                             // 0x80000000 - 1 => VS
15498   %}
15499 
15500   ins_pipe(pipe_slow);
15501 %}
15502 
15503 instruct overflowMulI_reg_branch(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, label labl, rFlagsReg cr)
15504 %{
15505   match(If cmp (OverflowMulI op1 op2));
15506   predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15507             || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15508   effect(USE labl, KILL cr);
15509 
15510   format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15511             "cmp   rscratch1, rscratch1, sxtw\n\t"
15512             "b$cmp   $labl" %}
15513   ins_cost(3 * INSN_COST); // Branch is rare so treat as INSN_COST
15514   ins_encode %{
15515     Label* L = $labl$$label;
15516     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15517     __ smull(rscratch1, $op1$$Register, $op2$$Register);
15518     __ subs(zr, rscratch1, rscratch1, ext::sxtw);      // NE => overflow
15519     __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15520   %}
15521 
15522   ins_pipe(pipe_serial);
15523 %}
15524 
15525 instruct overflowMulL_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15526 %{
15527   match(Set cr (OverflowMulL op1 op2));
15528 
15529   format %{ "mul   rscratch1, $op1, $op2\t#overflow check long\n\t"
15530             "smulh rscratch2, $op1, $op2\n\t"
15531             "cmp   rscratch2, rscratch1, ASR #63\n\t"
15532             "movw  rscratch1, #0x80000000\n\t"
15533             "cselw rscratch1, rscratch1, zr, NE\n\t"
15534             "cmpw  rscratch1, #1" %}
15535   ins_cost(6 * INSN_COST);
15536   ins_encode %{
15537     __ mul(rscratch1, $op1$$Register, $op2$$Register);   // Result bits 0..63
15538     __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15539     __ cmp(rscratch2, rscratch1, Assembler::ASR, 63);    // Top is pure sign ext
15540     __ movw(rscratch1, 0x80000000);                    // Develop 0 (EQ),
15541     __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15542     __ cmpw(rscratch1, 1);                             // 0x80000000 - 1 => VS
15543   %}
15544 
15545   ins_pipe(pipe_slow);
15546 %}
15547 
15548 instruct overflowMulL_reg_branch(cmpOp cmp, iRegL op1, iRegL op2, label labl, rFlagsReg cr)
15549 %{
15550   match(If cmp (OverflowMulL op1 op2));
15551   predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15552             || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15553   effect(USE labl, KILL cr);
15554 
15555   format %{ "mul   rscratch1, $op1, $op2\t#overflow check long\n\t"
15556             "smulh rscratch2, $op1, $op2\n\t"
15557             "cmp   rscratch2, rscratch1, ASR #63\n\t"
15558             "b$cmp $labl" %}
15559   ins_cost(4 * INSN_COST); // Branch is rare so treat as INSN_COST
15560   ins_encode %{
15561     Label* L = $labl$$label;
15562     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15563     __ mul(rscratch1, $op1$$Register, $op2$$Register);   // Result bits 0..63
15564     __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15565     __ cmp(rscratch2, rscratch1, Assembler::ASR, 63);    // Top is pure sign ext
15566     __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15567   %}
15568 
15569   ins_pipe(pipe_serial);
15570 %}
15571 
15572 // ============================================================================
15573 // Compare Instructions
15574 
15575 instruct compI_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
15576 %{
15577   match(Set cr (CmpI op1 op2));
15578 
15579   effect(DEF cr, USE op1, USE op2);
15580 
15581   ins_cost(INSN_COST);
15582   format %{ "cmpw  $op1, $op2" %}
15583 
15584   ins_encode(aarch64_enc_cmpw(op1, op2));
15585 
15586   ins_pipe(icmp_reg_reg);
15587 %}
15588 
15589 instruct compI_reg_immI0(rFlagsReg cr, iRegI op1, immI0 zero)
15590 %{
15591   match(Set cr (CmpI op1 zero));
15592 
15593   effect(DEF cr, USE op1);
15594 
15595   ins_cost(INSN_COST);
15596   format %{ "cmpw $op1, 0" %}
15597 
15598   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15599 
15600   ins_pipe(icmp_reg_imm);
15601 %}
15602 
15603 instruct compI_reg_immIAddSub(rFlagsReg cr, iRegI op1, immIAddSub op2)
15604 %{
15605   match(Set cr (CmpI op1 op2));
15606 
15607   effect(DEF cr, USE op1);
15608 
15609   ins_cost(INSN_COST);
15610   format %{ "cmpw  $op1, $op2" %}
15611 
15612   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15613 
15614   ins_pipe(icmp_reg_imm);
15615 %}
15616 
15617 instruct compI_reg_immI(rFlagsReg cr, iRegI op1, immI op2)
15618 %{
15619   match(Set cr (CmpI op1 op2));
15620 
15621   effect(DEF cr, USE op1);
15622 
15623   ins_cost(INSN_COST * 2);
15624   format %{ "cmpw  $op1, $op2" %}
15625 
15626   ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15627 
15628   ins_pipe(icmp_reg_imm);
15629 %}
15630 
15631 // Unsigned compare Instructions; really, same as signed compare
15632 // except it should only be used to feed an If or a CMovI which takes a
15633 // cmpOpU.
15634 
15635 instruct compU_reg_reg(rFlagsRegU cr, iRegI op1, iRegI op2)
15636 %{
15637   match(Set cr (CmpU op1 op2));
15638 
15639   effect(DEF cr, USE op1, USE op2);
15640 
15641   ins_cost(INSN_COST);
15642   format %{ "cmpw  $op1, $op2\t# unsigned" %}
15643 
15644   ins_encode(aarch64_enc_cmpw(op1, op2));
15645 
15646   ins_pipe(icmp_reg_reg);
15647 %}
15648 
15649 instruct compU_reg_immI0(rFlagsRegU cr, iRegI op1, immI0 zero)
15650 %{
15651   match(Set cr (CmpU op1 zero));
15652 
15653   effect(DEF cr, USE op1);
15654 
15655   ins_cost(INSN_COST);
15656   format %{ "cmpw $op1, #0\t# unsigned" %}
15657 
15658   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15659 
15660   ins_pipe(icmp_reg_imm);
15661 %}
15662 
15663 instruct compU_reg_immIAddSub(rFlagsRegU cr, iRegI op1, immIAddSub op2)
15664 %{
15665   match(Set cr (CmpU op1 op2));
15666 
15667   effect(DEF cr, USE op1);
15668 
15669   ins_cost(INSN_COST);
15670   format %{ "cmpw  $op1, $op2\t# unsigned" %}
15671 
15672   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15673 
15674   ins_pipe(icmp_reg_imm);
15675 %}
15676 
15677 instruct compU_reg_immI(rFlagsRegU cr, iRegI op1, immI op2)
15678 %{
15679   match(Set cr (CmpU op1 op2));
15680 
15681   effect(DEF cr, USE op1);
15682 
15683   ins_cost(INSN_COST * 2);
15684   format %{ "cmpw  $op1, $op2\t# unsigned" %}
15685 
15686   ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15687 
15688   ins_pipe(icmp_reg_imm);
15689 %}
15690 
15691 instruct compL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15692 %{
15693   match(Set cr (CmpL op1 op2));
15694 
15695   effect(DEF cr, USE op1, USE op2);
15696 
15697   ins_cost(INSN_COST);
15698   format %{ "cmp  $op1, $op2" %}
15699 
15700   ins_encode(aarch64_enc_cmp(op1, op2));
15701 
15702   ins_pipe(icmp_reg_reg);
15703 %}
15704 
15705 instruct compL_reg_immL0(rFlagsReg cr, iRegL op1, immL0 zero)
15706 %{
15707   match(Set cr (CmpL op1 zero));
15708 
15709   effect(DEF cr, USE op1);
15710 
15711   ins_cost(INSN_COST);
15712   format %{ "tst  $op1" %}
15713 
15714   ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15715 
15716   ins_pipe(icmp_reg_imm);
15717 %}
15718 
15719 instruct compL_reg_immLAddSub(rFlagsReg cr, iRegL op1, immLAddSub op2)
15720 %{
15721   match(Set cr (CmpL op1 op2));
15722 
15723   effect(DEF cr, USE op1);
15724 
15725   ins_cost(INSN_COST);
15726   format %{ "cmp  $op1, $op2" %}
15727 
15728   ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15729 
15730   ins_pipe(icmp_reg_imm);
15731 %}
15732 
15733 instruct compL_reg_immL(rFlagsReg cr, iRegL op1, immL op2)
15734 %{
15735   match(Set cr (CmpL op1 op2));
15736 
15737   effect(DEF cr, USE op1);
15738 
15739   ins_cost(INSN_COST * 2);
15740   format %{ "cmp  $op1, $op2" %}
15741 
15742   ins_encode(aarch64_enc_cmp_imm(op1, op2));
15743 
15744   ins_pipe(icmp_reg_imm);
15745 %}
15746 
15747 instruct compUL_reg_reg(rFlagsRegU cr, iRegL op1, iRegL op2)
15748 %{
15749   match(Set cr (CmpUL op1 op2));
15750 
15751   effect(DEF cr, USE op1, USE op2);
15752 
15753   ins_cost(INSN_COST);
15754   format %{ "cmp  $op1, $op2" %}
15755 
15756   ins_encode(aarch64_enc_cmp(op1, op2));
15757 
15758   ins_pipe(icmp_reg_reg);
15759 %}
15760 
15761 instruct compUL_reg_immL0(rFlagsRegU cr, iRegL op1, immL0 zero)
15762 %{
15763   match(Set cr (CmpUL op1 zero));
15764 
15765   effect(DEF cr, USE op1);
15766 
15767   ins_cost(INSN_COST);
15768   format %{ "tst  $op1" %}
15769 
15770   ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15771 
15772   ins_pipe(icmp_reg_imm);
15773 %}
15774 
15775 instruct compUL_reg_immLAddSub(rFlagsRegU cr, iRegL op1, immLAddSub op2)
15776 %{
15777   match(Set cr (CmpUL op1 op2));
15778 
15779   effect(DEF cr, USE op1);
15780 
15781   ins_cost(INSN_COST);
15782   format %{ "cmp  $op1, $op2" %}
15783 
15784   ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15785 
15786   ins_pipe(icmp_reg_imm);
15787 %}
15788 
15789 instruct compUL_reg_immL(rFlagsRegU cr, iRegL op1, immL op2)
15790 %{
15791   match(Set cr (CmpUL op1 op2));
15792 
15793   effect(DEF cr, USE op1);
15794 
15795   ins_cost(INSN_COST * 2);
15796   format %{ "cmp  $op1, $op2" %}
15797 
15798   ins_encode(aarch64_enc_cmp_imm(op1, op2));
15799 
15800   ins_pipe(icmp_reg_imm);
15801 %}
15802 
15803 instruct compP_reg_reg(rFlagsRegU cr, iRegP op1, iRegP op2)
15804 %{
15805   match(Set cr (CmpP op1 op2));
15806 
15807   effect(DEF cr, USE op1, USE op2);
15808 
15809   ins_cost(INSN_COST);
15810   format %{ "cmp  $op1, $op2\t // ptr" %}
15811 
15812   ins_encode(aarch64_enc_cmpp(op1, op2));
15813 
15814   ins_pipe(icmp_reg_reg);
15815 %}
15816 
15817 instruct compN_reg_reg(rFlagsRegU cr, iRegN op1, iRegN op2)
15818 %{
15819   match(Set cr (CmpN op1 op2));
15820 
15821   effect(DEF cr, USE op1, USE op2);
15822 
15823   ins_cost(INSN_COST);
15824   format %{ "cmp  $op1, $op2\t // compressed ptr" %}
15825 
15826   ins_encode(aarch64_enc_cmpn(op1, op2));
15827 
15828   ins_pipe(icmp_reg_reg);
15829 %}
15830 
15831 instruct testP_reg(rFlagsRegU cr, iRegP op1, immP0 zero)
15832 %{
15833   match(Set cr (CmpP op1 zero));
15834 
15835   effect(DEF cr, USE op1, USE zero);
15836 
15837   ins_cost(INSN_COST);
15838   format %{ "cmp  $op1, 0\t // ptr" %}
15839 
15840   ins_encode(aarch64_enc_testp(op1));
15841 
15842   ins_pipe(icmp_reg_imm);
15843 %}
15844 
15845 instruct testN_reg(rFlagsRegU cr, iRegN op1, immN0 zero)
15846 %{
15847   match(Set cr (CmpN op1 zero));
15848 
15849   effect(DEF cr, USE op1, USE zero);
15850 
15851   ins_cost(INSN_COST);
15852   format %{ "cmp  $op1, 0\t // compressed ptr" %}
15853 
15854   ins_encode(aarch64_enc_testn(op1));
15855 
15856   ins_pipe(icmp_reg_imm);
15857 %}
15858 
15859 // FP comparisons
15860 //
15861 // n.b. CmpF/CmpD set a normal flags reg which then gets compared
15862 // using normal cmpOp. See declaration of rFlagsReg for details.
15863 
15864 instruct compF_reg_reg(rFlagsReg cr, vRegF src1, vRegF src2)
15865 %{
15866   match(Set cr (CmpF src1 src2));
15867 
15868   ins_cost(3 * INSN_COST);
15869   format %{ "fcmps $src1, $src2" %}
15870 
15871   ins_encode %{
15872     __ fcmps(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15873   %}
15874 
15875   ins_pipe(pipe_class_compare);
15876 %}
15877 
15878 instruct compF_reg_zero(rFlagsReg cr, vRegF src1, immF0 src2)
15879 %{
15880   match(Set cr (CmpF src1 src2));
15881 
15882   ins_cost(3 * INSN_COST);
15883   format %{ "fcmps $src1, 0.0" %}
15884 
15885   ins_encode %{
15886     __ fcmps(as_FloatRegister($src1$$reg), 0.0);
15887   %}
15888 
15889   ins_pipe(pipe_class_compare);
15890 %}
15891 // FROM HERE
15892 
15893 instruct compD_reg_reg(rFlagsReg cr, vRegD src1, vRegD src2)
15894 %{
15895   match(Set cr (CmpD src1 src2));
15896 
15897   ins_cost(3 * INSN_COST);
15898   format %{ "fcmpd $src1, $src2" %}
15899 
15900   ins_encode %{
15901     __ fcmpd(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15902   %}
15903 
15904   ins_pipe(pipe_class_compare);
15905 %}
15906 
15907 instruct compD_reg_zero(rFlagsReg cr, vRegD src1, immD0 src2)
15908 %{
15909   match(Set cr (CmpD src1 src2));
15910 
15911   ins_cost(3 * INSN_COST);
15912   format %{ "fcmpd $src1, 0.0" %}
15913 
15914   ins_encode %{
15915     __ fcmpd(as_FloatRegister($src1$$reg), 0.0);
15916   %}
15917 
15918   ins_pipe(pipe_class_compare);
15919 %}
15920 
15921 instruct compF3_reg_reg(iRegINoSp dst, vRegF src1, vRegF src2, rFlagsReg cr)
15922 %{
15923   match(Set dst (CmpF3 src1 src2));
15924   effect(KILL cr);
15925 
15926   ins_cost(5 * INSN_COST);
15927   format %{ "fcmps $src1, $src2\n\t"
15928             "csinvw($dst, zr, zr, eq\n\t"
15929             "csnegw($dst, $dst, $dst, lt)"
15930   %}
15931 
15932   ins_encode %{
15933     Label done;
15934     FloatRegister s1 = as_FloatRegister($src1$$reg);
15935     FloatRegister s2 = as_FloatRegister($src2$$reg);
15936     Register d = as_Register($dst$$reg);
15937     __ fcmps(s1, s2);
15938     // installs 0 if EQ else -1
15939     __ csinvw(d, zr, zr, Assembler::EQ);
15940     // keeps -1 if less or unordered else installs 1
15941     __ csnegw(d, d, d, Assembler::LT);
15942     __ bind(done);
15943   %}
15944 
15945   ins_pipe(pipe_class_default);
15946 
15947 %}
15948 
15949 instruct compD3_reg_reg(iRegINoSp dst, vRegD src1, vRegD src2, rFlagsReg cr)
15950 %{
15951   match(Set dst (CmpD3 src1 src2));
15952   effect(KILL cr);
15953 
15954   ins_cost(5 * INSN_COST);
15955   format %{ "fcmpd $src1, $src2\n\t"
15956             "csinvw($dst, zr, zr, eq\n\t"
15957             "csnegw($dst, $dst, $dst, lt)"
15958   %}
15959 
15960   ins_encode %{
15961     Label done;
15962     FloatRegister s1 = as_FloatRegister($src1$$reg);
15963     FloatRegister s2 = as_FloatRegister($src2$$reg);
15964     Register d = as_Register($dst$$reg);
15965     __ fcmpd(s1, s2);
15966     // installs 0 if EQ else -1
15967     __ csinvw(d, zr, zr, Assembler::EQ);
15968     // keeps -1 if less or unordered else installs 1
15969     __ csnegw(d, d, d, Assembler::LT);
15970     __ bind(done);
15971   %}
15972   ins_pipe(pipe_class_default);
15973 
15974 %}
15975 
15976 instruct compF3_reg_immF0(iRegINoSp dst, vRegF src1, immF0 zero, rFlagsReg cr)
15977 %{
15978   match(Set dst (CmpF3 src1 zero));
15979   effect(KILL cr);
15980 
15981   ins_cost(5 * INSN_COST);
15982   format %{ "fcmps $src1, 0.0\n\t"
15983             "csinvw($dst, zr, zr, eq\n\t"
15984             "csnegw($dst, $dst, $dst, lt)"
15985   %}
15986 
15987   ins_encode %{
15988     Label done;
15989     FloatRegister s1 = as_FloatRegister($src1$$reg);
15990     Register d = as_Register($dst$$reg);
15991     __ fcmps(s1, 0.0);
15992     // installs 0 if EQ else -1
15993     __ csinvw(d, zr, zr, Assembler::EQ);
15994     // keeps -1 if less or unordered else installs 1
15995     __ csnegw(d, d, d, Assembler::LT);
15996     __ bind(done);
15997   %}
15998 
15999   ins_pipe(pipe_class_default);
16000 
16001 %}
16002 
16003 instruct compD3_reg_immD0(iRegINoSp dst, vRegD src1, immD0 zero, rFlagsReg cr)
16004 %{
16005   match(Set dst (CmpD3 src1 zero));
16006   effect(KILL cr);
16007 
16008   ins_cost(5 * INSN_COST);
16009   format %{ "fcmpd $src1, 0.0\n\t"
16010             "csinvw($dst, zr, zr, eq\n\t"
16011             "csnegw($dst, $dst, $dst, lt)"
16012   %}
16013 
16014   ins_encode %{
16015     Label done;
16016     FloatRegister s1 = as_FloatRegister($src1$$reg);
16017     Register d = as_Register($dst$$reg);
16018     __ fcmpd(s1, 0.0);
16019     // installs 0 if EQ else -1
16020     __ csinvw(d, zr, zr, Assembler::EQ);
16021     // keeps -1 if less or unordered else installs 1
16022     __ csnegw(d, d, d, Assembler::LT);
16023     __ bind(done);
16024   %}
16025   ins_pipe(pipe_class_default);
16026 
16027 %}
16028 
16029 instruct cmpLTMask_reg_reg(iRegINoSp dst, iRegIorL2I p, iRegIorL2I q, rFlagsReg cr)
16030 %{
16031   match(Set dst (CmpLTMask p q));
16032   effect(KILL cr);
16033 
16034   ins_cost(3 * INSN_COST);
16035 
16036   format %{ "cmpw $p, $q\t# cmpLTMask\n\t"
16037             "csetw $dst, lt\n\t"
16038             "subw $dst, zr, $dst"
16039   %}
16040 
16041   ins_encode %{
16042     __ cmpw(as_Register($p$$reg), as_Register($q$$reg));
16043     __ csetw(as_Register($dst$$reg), Assembler::LT);
16044     __ subw(as_Register($dst$$reg), zr, as_Register($dst$$reg));
16045   %}
16046 
16047   ins_pipe(ialu_reg_reg);
16048 %}
16049 
16050 instruct cmpLTMask_reg_zero(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr)
16051 %{
16052   match(Set dst (CmpLTMask src zero));
16053   effect(KILL cr);
16054 
16055   ins_cost(INSN_COST);
16056 
16057   format %{ "asrw $dst, $src, #31\t# cmpLTMask0" %}
16058 
16059   ins_encode %{
16060     __ asrw(as_Register($dst$$reg), as_Register($src$$reg), 31);
16061   %}
16062 
16063   ins_pipe(ialu_reg_shift);
16064 %}
16065 
16066 // ============================================================================
16067 // Max and Min
16068 
16069 // Like compI_reg_reg or compI_reg_immI0 but without match rule and second zero parameter.
16070 
16071 instruct compI_reg_imm0(rFlagsReg cr, iRegI src)
16072 %{
16073   effect(DEF cr, USE src);
16074   ins_cost(INSN_COST);
16075   format %{ "cmpw $src, 0" %}
16076 
16077   ins_encode %{
16078     __ cmpw($src$$Register, 0);
16079   %}
16080   ins_pipe(icmp_reg_imm);
16081 %}
16082 
16083 instruct minI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
16084 %{
16085   match(Set dst (MinI src1 src2));
16086   ins_cost(INSN_COST * 3);
16087 
16088   expand %{
16089     rFlagsReg cr;
16090     compI_reg_reg(cr, src1, src2);
16091     cmovI_reg_reg_lt(dst, src1, src2, cr);
16092   %}
16093 %}
16094 
16095 instruct maxI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
16096 %{
16097   match(Set dst (MaxI src1 src2));
16098   ins_cost(INSN_COST * 3);
16099 
16100   expand %{
16101     rFlagsReg cr;
16102     compI_reg_reg(cr, src1, src2);
16103     cmovI_reg_reg_gt(dst, src1, src2, cr);
16104   %}
16105 %}
16106 
16107 
16108 // ============================================================================
16109 // Branch Instructions
16110 
16111 // Direct Branch.
16112 instruct branch(label lbl)
16113 %{
16114   match(Goto);
16115 
16116   effect(USE lbl);
16117 
16118   ins_cost(BRANCH_COST);
16119   format %{ "b  $lbl" %}
16120 
16121   ins_encode(aarch64_enc_b(lbl));
16122 
16123   ins_pipe(pipe_branch);
16124 %}
16125 
16126 // Conditional Near Branch
16127 instruct branchCon(cmpOp cmp, rFlagsReg cr, label lbl)
16128 %{
16129   // Same match rule as `branchConFar'.
16130   match(If cmp cr);
16131 
16132   effect(USE lbl);
16133 
16134   ins_cost(BRANCH_COST);
16135   // If set to 1 this indicates that the current instruction is a
16136   // short variant of a long branch. This avoids using this
16137   // instruction in first-pass matching. It will then only be used in
16138   // the `Shorten_branches' pass.
16139   // ins_short_branch(1);
16140   format %{ "b$cmp  $lbl" %}
16141 
16142   ins_encode(aarch64_enc_br_con(cmp, lbl));
16143 
16144   ins_pipe(pipe_branch_cond);
16145 %}
16146 
16147 // Conditional Near Branch Unsigned
16148 instruct branchConU(cmpOpU cmp, rFlagsRegU cr, label lbl)
16149 %{
16150   // Same match rule as `branchConFar'.
16151   match(If cmp cr);
16152 
16153   effect(USE lbl);
16154 
16155   ins_cost(BRANCH_COST);
16156   // If set to 1 this indicates that the current instruction is a
16157   // short variant of a long branch. This avoids using this
16158   // instruction in first-pass matching. It will then only be used in
16159   // the `Shorten_branches' pass.
16160   // ins_short_branch(1);
16161   format %{ "b$cmp  $lbl\t# unsigned" %}
16162 
16163   ins_encode(aarch64_enc_br_conU(cmp, lbl));
16164 
16165   ins_pipe(pipe_branch_cond);
16166 %}
16167 
16168 // Make use of CBZ and CBNZ.  These instructions, as well as being
16169 // shorter than (cmp; branch), have the additional benefit of not
16170 // killing the flags.
16171 
16172 instruct cmpI_imm0_branch(cmpOpEqNe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsReg cr) %{
16173   match(If cmp (CmpI op1 op2));
16174   effect(USE labl);
16175 
16176   ins_cost(BRANCH_COST);
16177   format %{ "cbw$cmp   $op1, $labl" %}
16178   ins_encode %{
16179     Label* L = $labl$$label;
16180     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16181     if (cond == Assembler::EQ)
16182       __ cbzw($op1$$Register, *L);
16183     else
16184       __ cbnzw($op1$$Register, *L);
16185   %}
16186   ins_pipe(pipe_cmp_branch);
16187 %}
16188 
16189 instruct cmpL_imm0_branch(cmpOpEqNe cmp, iRegL op1, immL0 op2, label labl, rFlagsReg cr) %{
16190   match(If cmp (CmpL op1 op2));
16191   effect(USE labl);
16192 
16193   ins_cost(BRANCH_COST);
16194   format %{ "cb$cmp   $op1, $labl" %}
16195   ins_encode %{
16196     Label* L = $labl$$label;
16197     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16198     if (cond == Assembler::EQ)
16199       __ cbz($op1$$Register, *L);
16200     else
16201       __ cbnz($op1$$Register, *L);
16202   %}
16203   ins_pipe(pipe_cmp_branch);
16204 %}
16205 
16206 instruct cmpP_imm0_branch(cmpOpEqNe cmp, iRegP op1, immP0 op2, label labl, rFlagsReg cr) %{
16207   match(If cmp (CmpP op1 op2));
16208   effect(USE labl);
16209 
16210   ins_cost(BRANCH_COST);
16211   format %{ "cb$cmp   $op1, $labl" %}
16212   ins_encode %{
16213     Label* L = $labl$$label;
16214     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16215     if (cond == Assembler::EQ)
16216       __ cbz($op1$$Register, *L);
16217     else
16218       __ cbnz($op1$$Register, *L);
16219   %}
16220   ins_pipe(pipe_cmp_branch);
16221 %}
16222 
16223 instruct cmpN_imm0_branch(cmpOpEqNe cmp, iRegN op1, immN0 op2, label labl, rFlagsReg cr) %{
16224   match(If cmp (CmpN op1 op2));
16225   effect(USE labl);
16226 
16227   ins_cost(BRANCH_COST);
16228   format %{ "cbw$cmp   $op1, $labl" %}
16229   ins_encode %{
16230     Label* L = $labl$$label;
16231     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16232     if (cond == Assembler::EQ)
16233       __ cbzw($op1$$Register, *L);
16234     else
16235       __ cbnzw($op1$$Register, *L);
16236   %}
16237   ins_pipe(pipe_cmp_branch);
16238 %}
16239 
16240 instruct cmpP_narrowOop_imm0_branch(cmpOpEqNe cmp, iRegN oop, immP0 zero, label labl, rFlagsReg cr) %{
16241   match(If cmp (CmpP (DecodeN oop) zero));
16242   effect(USE labl);
16243 
16244   ins_cost(BRANCH_COST);
16245   format %{ "cb$cmp   $oop, $labl" %}
16246   ins_encode %{
16247     Label* L = $labl$$label;
16248     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16249     if (cond == Assembler::EQ)
16250       __ cbzw($oop$$Register, *L);
16251     else
16252       __ cbnzw($oop$$Register, *L);
16253   %}
16254   ins_pipe(pipe_cmp_branch);
16255 %}
16256 
16257 instruct cmpUI_imm0_branch(cmpOpUEqNeLtGe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsRegU cr) %{
16258   match(If cmp (CmpU op1 op2));
16259   effect(USE labl);
16260 
16261   ins_cost(BRANCH_COST);
16262   format %{ "cbw$cmp   $op1, $labl" %}
16263   ins_encode %{
16264     Label* L = $labl$$label;
16265     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16266     if (cond == Assembler::EQ || cond == Assembler::LS)
16267       __ cbzw($op1$$Register, *L);
16268     else
16269       __ cbnzw($op1$$Register, *L);
16270   %}
16271   ins_pipe(pipe_cmp_branch);
16272 %}
16273 
16274 instruct cmpUL_imm0_branch(cmpOpUEqNeLtGe cmp, iRegL op1, immL0 op2, label labl, rFlagsRegU cr) %{
16275   match(If cmp (CmpUL op1 op2));
16276   effect(USE labl);
16277 
16278   ins_cost(BRANCH_COST);
16279   format %{ "cb$cmp   $op1, $labl" %}
16280   ins_encode %{
16281     Label* L = $labl$$label;
16282     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16283     if (cond == Assembler::EQ || cond == Assembler::LS)
16284       __ cbz($op1$$Register, *L);
16285     else
16286       __ cbnz($op1$$Register, *L);
16287   %}
16288   ins_pipe(pipe_cmp_branch);
16289 %}
16290 
16291 // Test bit and Branch
16292 
16293 // Patterns for short (< 32KiB) variants
16294 instruct cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
16295   match(If cmp (CmpL op1 op2));
16296   effect(USE labl);
16297 
16298   ins_cost(BRANCH_COST);
16299   format %{ "cb$cmp   $op1, $labl # long" %}
16300   ins_encode %{
16301     Label* L = $labl$$label;
16302     Assembler::Condition cond =
16303       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16304     __ tbr(cond, $op1$$Register, 63, *L);
16305   %}
16306   ins_pipe(pipe_cmp_branch);
16307   ins_short_branch(1);
16308 %}
16309 
16310 instruct cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16311   match(If cmp (CmpI op1 op2));
16312   effect(USE labl);
16313 
16314   ins_cost(BRANCH_COST);
16315   format %{ "cb$cmp   $op1, $labl # int" %}
16316   ins_encode %{
16317     Label* L = $labl$$label;
16318     Assembler::Condition cond =
16319       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16320     __ tbr(cond, $op1$$Register, 31, *L);
16321   %}
16322   ins_pipe(pipe_cmp_branch);
16323   ins_short_branch(1);
16324 %}
16325 
16326 instruct cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
16327   match(If cmp (CmpL (AndL op1 op2) op3));
16328   predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
16329   effect(USE labl);
16330 
16331   ins_cost(BRANCH_COST);
16332   format %{ "tb$cmp   $op1, $op2, $labl" %}
16333   ins_encode %{
16334     Label* L = $labl$$label;
16335     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16336     int bit = exact_log2_long($op2$$constant);
16337     __ tbr(cond, $op1$$Register, bit, *L);
16338   %}
16339   ins_pipe(pipe_cmp_branch);
16340   ins_short_branch(1);
16341 %}
16342 
16343 instruct cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
16344   match(If cmp (CmpI (AndI op1 op2) op3));
16345   predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
16346   effect(USE labl);
16347 
16348   ins_cost(BRANCH_COST);
16349   format %{ "tb$cmp   $op1, $op2, $labl" %}
16350   ins_encode %{
16351     Label* L = $labl$$label;
16352     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16353     int bit = exact_log2((juint)$op2$$constant);
16354     __ tbr(cond, $op1$$Register, bit, *L);
16355   %}
16356   ins_pipe(pipe_cmp_branch);
16357   ins_short_branch(1);
16358 %}
16359 
16360 // And far variants
16361 instruct far_cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
16362   match(If cmp (CmpL op1 op2));
16363   effect(USE labl);
16364 
16365   ins_cost(BRANCH_COST);
16366   format %{ "cb$cmp   $op1, $labl # long" %}
16367   ins_encode %{
16368     Label* L = $labl$$label;
16369     Assembler::Condition cond =
16370       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16371     __ tbr(cond, $op1$$Register, 63, *L, /*far*/true);
16372   %}
16373   ins_pipe(pipe_cmp_branch);
16374 %}
16375 
16376 instruct far_cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16377   match(If cmp (CmpI op1 op2));
16378   effect(USE labl);
16379 
16380   ins_cost(BRANCH_COST);
16381   format %{ "cb$cmp   $op1, $labl # int" %}
16382   ins_encode %{
16383     Label* L = $labl$$label;
16384     Assembler::Condition cond =
16385       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16386     __ tbr(cond, $op1$$Register, 31, *L, /*far*/true);
16387   %}
16388   ins_pipe(pipe_cmp_branch);
16389 %}
16390 
16391 instruct far_cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
16392   match(If cmp (CmpL (AndL op1 op2) op3));
16393   predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
16394   effect(USE labl);
16395 
16396   ins_cost(BRANCH_COST);
16397   format %{ "tb$cmp   $op1, $op2, $labl" %}
16398   ins_encode %{
16399     Label* L = $labl$$label;
16400     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16401     int bit = exact_log2_long($op2$$constant);
16402     __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
16403   %}
16404   ins_pipe(pipe_cmp_branch);
16405 %}
16406 
16407 instruct far_cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
16408   match(If cmp (CmpI (AndI op1 op2) op3));
16409   predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
16410   effect(USE labl);
16411 
16412   ins_cost(BRANCH_COST);
16413   format %{ "tb$cmp   $op1, $op2, $labl" %}
16414   ins_encode %{
16415     Label* L = $labl$$label;
16416     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16417     int bit = exact_log2((juint)$op2$$constant);
16418     __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
16419   %}
16420   ins_pipe(pipe_cmp_branch);
16421 %}
16422 
16423 // Test bits
16424 
16425 instruct cmpL_and(cmpOp cmp, iRegL op1, immL op2, immL0 op3, rFlagsReg cr) %{
16426   match(Set cr (CmpL (AndL op1 op2) op3));
16427   predicate(Assembler::operand_valid_for_logical_immediate
16428             (/*is_32*/false, n->in(1)->in(2)->get_long()));
16429 
16430   ins_cost(INSN_COST);
16431   format %{ "tst $op1, $op2 # long" %}
16432   ins_encode %{
16433     __ tst($op1$$Register, $op2$$constant);
16434   %}
16435   ins_pipe(ialu_reg_reg);
16436 %}
16437 
16438 instruct cmpI_and(cmpOp cmp, iRegIorL2I op1, immI op2, immI0 op3, rFlagsReg cr) %{
16439   match(Set cr (CmpI (AndI op1 op2) op3));
16440   predicate(Assembler::operand_valid_for_logical_immediate
16441             (/*is_32*/true, n->in(1)->in(2)->get_int()));
16442 
16443   ins_cost(INSN_COST);
16444   format %{ "tst $op1, $op2 # int" %}
16445   ins_encode %{
16446     __ tstw($op1$$Register, $op2$$constant);
16447   %}
16448   ins_pipe(ialu_reg_reg);
16449 %}
16450 
16451 instruct cmpL_and_reg(cmpOp cmp, iRegL op1, iRegL op2, immL0 op3, rFlagsReg cr) %{
16452   match(Set cr (CmpL (AndL op1 op2) op3));
16453 
16454   ins_cost(INSN_COST);
16455   format %{ "tst $op1, $op2 # long" %}
16456   ins_encode %{
16457     __ tst($op1$$Register, $op2$$Register);
16458   %}
16459   ins_pipe(ialu_reg_reg);
16460 %}
16461 
16462 instruct cmpI_and_reg(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, immI0 op3, rFlagsReg cr) %{
16463   match(Set cr (CmpI (AndI op1 op2) op3));
16464 
16465   ins_cost(INSN_COST);
16466   format %{ "tstw $op1, $op2 # int" %}
16467   ins_encode %{
16468     __ tstw($op1$$Register, $op2$$Register);
16469   %}
16470   ins_pipe(ialu_reg_reg);
16471 %}
16472 
16473 
16474 // Conditional Far Branch
16475 // Conditional Far Branch Unsigned
16476 // TODO: fixme
16477 
16478 // counted loop end branch near
16479 instruct branchLoopEnd(cmpOp cmp, rFlagsReg cr, label lbl)
16480 %{
16481   match(CountedLoopEnd cmp cr);
16482 
16483   effect(USE lbl);
16484 
16485   ins_cost(BRANCH_COST);
16486   // short variant.
16487   // ins_short_branch(1);
16488   format %{ "b$cmp $lbl \t// counted loop end" %}
16489 
16490   ins_encode(aarch64_enc_br_con(cmp, lbl));
16491 
16492   ins_pipe(pipe_branch);
16493 %}
16494 
16495 // counted loop end branch near Unsigned
16496 instruct branchLoopEndU(cmpOpU cmp, rFlagsRegU cr, label lbl)
16497 %{
16498   match(CountedLoopEnd cmp cr);
16499 
16500   effect(USE lbl);
16501 
16502   ins_cost(BRANCH_COST);
16503   // short variant.
16504   // ins_short_branch(1);
16505   format %{ "b$cmp $lbl \t// counted loop end unsigned" %}
16506 
16507   ins_encode(aarch64_enc_br_conU(cmp, lbl));
16508 
16509   ins_pipe(pipe_branch);
16510 %}
16511 
16512 // counted loop end branch far
16513 // counted loop end branch far unsigned
16514 // TODO: fixme
16515 
16516 // ============================================================================
16517 // inlined locking and unlocking
16518 
16519 instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
16520 %{
16521   match(Set cr (FastLock object box));
16522   effect(TEMP tmp, TEMP tmp2);
16523 
16524   // TODO
16525   // identify correct cost
16526   ins_cost(5 * INSN_COST);
16527   format %{ "fastlock $object,$box\t! kills $tmp,$tmp2" %}
16528 
16529   ins_encode(aarch64_enc_fast_lock(object, box, tmp, tmp2));
16530 
16531   ins_pipe(pipe_serial);
16532 %}
16533 
16534 instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
16535 %{
16536   match(Set cr (FastUnlock object box));
16537   effect(TEMP tmp, TEMP tmp2);
16538 
16539   ins_cost(5 * INSN_COST);
16540   format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2" %}
16541 
16542   ins_encode(aarch64_enc_fast_unlock(object, box, tmp, tmp2));
16543 
16544   ins_pipe(pipe_serial);
16545 %}
16546 
16547 
16548 // ============================================================================
16549 // Safepoint Instructions
16550 
16551 // TODO
16552 // provide a near and far version of this code
16553 
16554 instruct safePoint(rFlagsReg cr, iRegP poll)
16555 %{
16556   match(SafePoint poll);
16557   effect(KILL cr);
16558 
16559   format %{
16560     "ldrw zr, [$poll]\t# Safepoint: poll for GC"
16561   %}
16562   ins_encode %{
16563     __ read_polling_page(as_Register($poll$$reg), relocInfo::poll_type);
16564   %}
16565   ins_pipe(pipe_serial); // ins_pipe(iload_reg_mem);
16566 %}
16567 
16568 
16569 // ============================================================================
16570 // Procedure Call/Return Instructions
16571 
16572 // Call Java Static Instruction
16573 
16574 instruct CallStaticJavaDirect(method meth)
16575 %{
16576   match(CallStaticJava);
16577 
16578   effect(USE meth);
16579 
16580   ins_cost(CALL_COST);
16581 
16582   format %{ "call,static $meth \t// ==> " %}
16583 
16584   ins_encode(aarch64_enc_java_static_call(meth),
16585              aarch64_enc_call_epilog);
16586 
16587   ins_pipe(pipe_class_call);
16588 %}
16589 
16590 // TO HERE
16591 
16592 // Call Java Dynamic Instruction
16593 instruct CallDynamicJavaDirect(method meth)
16594 %{
16595   match(CallDynamicJava);
16596 
16597   effect(USE meth);
16598 
16599   ins_cost(CALL_COST);
16600 
16601   format %{ "CALL,dynamic $meth \t// ==> " %}
16602 
16603   ins_encode(aarch64_enc_java_dynamic_call(meth),
16604              aarch64_enc_call_epilog);
16605 
16606   ins_pipe(pipe_class_call);
16607 %}
16608 
16609 // Call Runtime Instruction
16610 
16611 instruct CallRuntimeDirect(method meth)
16612 %{
16613   match(CallRuntime);
16614 
16615   effect(USE meth);
16616 
16617   ins_cost(CALL_COST);
16618 
16619   format %{ "CALL, runtime $meth" %}
16620 
16621   ins_encode( aarch64_enc_java_to_runtime(meth) );
16622 
16623   ins_pipe(pipe_class_call);
16624 %}
16625 
16626 // Call Runtime Instruction
16627 
16628 instruct CallLeafDirect(method meth)
16629 %{
16630   match(CallLeaf);
16631 
16632   effect(USE meth);
16633 
16634   ins_cost(CALL_COST);
16635 
16636   format %{ "CALL, runtime leaf $meth" %}
16637 
16638   ins_encode( aarch64_enc_java_to_runtime(meth) );
16639 
16640   ins_pipe(pipe_class_call);
16641 %}
16642 
16643 // Call Runtime Instruction
16644 
16645 instruct CallLeafNoFPDirect(method meth)
16646 %{
16647   match(CallLeafNoFP);
16648 
16649   effect(USE meth);
16650 
16651   ins_cost(CALL_COST);
16652 
16653   format %{ "CALL, runtime leaf nofp $meth" %}
16654 
16655   ins_encode( aarch64_enc_java_to_runtime(meth) );
16656 
16657   ins_pipe(pipe_class_call);
16658 %}
16659 
16660 // Tail Call; Jump from runtime stub to Java code.
16661 // Also known as an 'interprocedural jump'.
16662 // Target of jump will eventually return to caller.
16663 // TailJump below removes the return address.
16664 instruct TailCalljmpInd(iRegPNoSp jump_target, inline_cache_RegP method_ptr)
16665 %{
16666   match(TailCall jump_target method_ptr);
16667 
16668   ins_cost(CALL_COST);
16669 
16670   format %{ "br $jump_target\t# $method_ptr holds method" %}
16671 
16672   ins_encode(aarch64_enc_tail_call(jump_target));
16673 
16674   ins_pipe(pipe_class_call);
16675 %}
16676 
16677 instruct TailjmpInd(iRegPNoSp jump_target, iRegP_R0 ex_oop)
16678 %{
16679   match(TailJump jump_target ex_oop);
16680 
16681   ins_cost(CALL_COST);
16682 
16683   format %{ "br $jump_target\t# $ex_oop holds exception oop" %}
16684 
16685   ins_encode(aarch64_enc_tail_jmp(jump_target));
16686 
16687   ins_pipe(pipe_class_call);
16688 %}
16689 
16690 // Create exception oop: created by stack-crawling runtime code.
16691 // Created exception is now available to this handler, and is setup
16692 // just prior to jumping to this handler. No code emitted.
16693 // TODO check
16694 // should ex_oop be in r0? intel uses rax, ppc cannot use r0 so uses rarg1
16695 instruct CreateException(iRegP_R0 ex_oop)
16696 %{
16697   match(Set ex_oop (CreateEx));
16698 
16699   format %{ " -- \t// exception oop; no code emitted" %}
16700 
16701   size(0);
16702 
16703   ins_encode( /*empty*/ );
16704 
16705   ins_pipe(pipe_class_empty);
16706 %}
16707 
16708 // Rethrow exception: The exception oop will come in the first
16709 // argument position. Then JUMP (not call) to the rethrow stub code.
16710 instruct RethrowException() %{
16711   match(Rethrow);
16712   ins_cost(CALL_COST);
16713 
16714   format %{ "b rethrow_stub" %}
16715 
16716   ins_encode( aarch64_enc_rethrow() );
16717 
16718   ins_pipe(pipe_class_call);
16719 %}
16720 
16721 
16722 // Return Instruction
16723 // epilog node loads ret address into lr as part of frame pop
16724 instruct Ret()
16725 %{
16726   match(Return);
16727 
16728   format %{ "ret\t// return register" %}
16729 
16730   ins_encode( aarch64_enc_ret() );
16731 
16732   ins_pipe(pipe_branch);
16733 %}
16734 
16735 // Die now.
16736 instruct ShouldNotReachHere() %{
16737   match(Halt);
16738 
16739   ins_cost(CALL_COST);
16740   format %{ "ShouldNotReachHere" %}
16741 
16742   ins_encode %{
16743     if (is_reachable()) {
16744       __ stop(_halt_reason);
16745     }
16746   %}
16747 
16748   ins_pipe(pipe_class_default);
16749 %}
16750 
16751 // ============================================================================
16752 // Partial Subtype Check
16753 //
16754 // superklass array for an instance of the superklass.  Set a hidden
16755 // internal cache on a hit (cache is checked with exposed code in
16756 // gen_subtype_check()).  Return NZ for a miss or zero for a hit.  The
16757 // encoding ALSO sets flags.
16758 
16759 instruct partialSubtypeCheck(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, rFlagsReg cr)
16760 %{
16761   match(Set result (PartialSubtypeCheck sub super));
16762   effect(KILL cr, KILL temp);
16763 
16764   ins_cost(1100);  // slightly larger than the next version
16765   format %{ "partialSubtypeCheck $result, $sub, $super" %}
16766 
16767   ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
16768 
16769   opcode(0x1); // Force zero of result reg on hit
16770 
16771   ins_pipe(pipe_class_memory);
16772 %}
16773 
16774 instruct partialSubtypeCheckVsZero(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, immP0 zero, rFlagsReg cr)
16775 %{
16776   match(Set cr (CmpP (PartialSubtypeCheck sub super) zero));
16777   effect(KILL temp, KILL result);
16778 
16779   ins_cost(1100);  // slightly larger than the next version
16780   format %{ "partialSubtypeCheck $result, $sub, $super == 0" %}
16781 
16782   ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
16783 
16784   opcode(0x0); // Don't zero result reg on hit
16785 
16786   ins_pipe(pipe_class_memory);
16787 %}
16788 
16789 // Intrisics for String.compareTo()
16790 
16791 instruct string_compareU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16792                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16793 %{
16794   predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
16795   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16796   effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16797 
16798   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1" %}
16799   ins_encode %{
16800     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16801     __ string_compare($str1$$Register, $str2$$Register,
16802                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16803                       $tmp1$$Register, $tmp2$$Register,
16804                       fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::UU);
16805   %}
16806   ins_pipe(pipe_class_memory);
16807 %}
16808 
16809 instruct string_compareL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16810                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16811 %{
16812   predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
16813   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16814   effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16815 
16816   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1" %}
16817   ins_encode %{
16818     __ string_compare($str1$$Register, $str2$$Register,
16819                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16820                       $tmp1$$Register, $tmp2$$Register,
16821                       fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::LL);
16822   %}
16823   ins_pipe(pipe_class_memory);
16824 %}
16825 
16826 instruct string_compareUL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16827                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16828                         vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16829 %{
16830   predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
16831   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16832   effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16833          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16834 
16835   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16836   ins_encode %{
16837     __ string_compare($str1$$Register, $str2$$Register,
16838                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16839                       $tmp1$$Register, $tmp2$$Register,
16840                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16841                       $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::UL);
16842   %}
16843   ins_pipe(pipe_class_memory);
16844 %}
16845 
16846 instruct string_compareLU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16847                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16848                         vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16849 %{
16850   predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
16851   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16852   effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16853          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16854 
16855   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16856   ins_encode %{
16857     __ string_compare($str1$$Register, $str2$$Register,
16858                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16859                       $tmp1$$Register, $tmp2$$Register,
16860                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16861                       $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::LU);
16862   %}
16863   ins_pipe(pipe_class_memory);
16864 %}
16865 
16866 // Note that Z registers alias the corresponding NEON registers, we declare the vector operands of
16867 // these string_compare variants as NEON register type for convenience so that the prototype of
16868 // string_compare can be shared with all variants.
16869 
16870 instruct string_compareLL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16871                               iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16872                               vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16873                               pRegGov_P1 pgtmp2, rFlagsReg cr)
16874 %{
16875   predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
16876   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16877   effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16878          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16879 
16880   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # USE sve" %}
16881   ins_encode %{
16882     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16883     __ string_compare($str1$$Register, $str2$$Register,
16884                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16885                       $tmp1$$Register, $tmp2$$Register,
16886                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16887                       as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16888                       StrIntrinsicNode::LL);
16889   %}
16890   ins_pipe(pipe_class_memory);
16891 %}
16892 
16893 instruct string_compareLU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16894                               iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16895                               vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16896                               pRegGov_P1 pgtmp2, rFlagsReg cr)
16897 %{
16898   predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
16899   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16900   effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16901          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16902 
16903   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # USE sve" %}
16904   ins_encode %{
16905     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16906     __ string_compare($str1$$Register, $str2$$Register,
16907                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16908                       $tmp1$$Register, $tmp2$$Register,
16909                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16910                       as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16911                       StrIntrinsicNode::LU);
16912   %}
16913   ins_pipe(pipe_class_memory);
16914 %}
16915 
16916 instruct string_compareUL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16917                               iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16918                               vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16919                               pRegGov_P1 pgtmp2, rFlagsReg cr)
16920 %{
16921   predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
16922   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16923   effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16924          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16925 
16926   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # USE sve" %}
16927   ins_encode %{
16928     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16929     __ string_compare($str1$$Register, $str2$$Register,
16930                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16931                       $tmp1$$Register, $tmp2$$Register,
16932                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16933                       as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16934                       StrIntrinsicNode::UL);
16935   %}
16936   ins_pipe(pipe_class_memory);
16937 %}
16938 
16939 instruct string_compareUU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16940                               iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16941                               vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16942                               pRegGov_P1 pgtmp2, rFlagsReg cr)
16943 %{
16944   predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
16945   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16946   effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16947          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16948 
16949   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # USE sve" %}
16950   ins_encode %{
16951     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16952     __ string_compare($str1$$Register, $str2$$Register,
16953                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16954                       $tmp1$$Register, $tmp2$$Register,
16955                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16956                       as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16957                       StrIntrinsicNode::UU);
16958   %}
16959   ins_pipe(pipe_class_memory);
16960 %}
16961 
16962 instruct string_indexofUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16963        iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
16964        iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6, rFlagsReg cr)
16965 %{
16966   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16967   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16968   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16969          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6, KILL cr);
16970   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UU)" %}
16971 
16972   ins_encode %{
16973     __ string_indexof($str1$$Register, $str2$$Register,
16974                       $cnt1$$Register, $cnt2$$Register,
16975                       $tmp1$$Register, $tmp2$$Register,
16976                       $tmp3$$Register, $tmp4$$Register,
16977                       $tmp5$$Register, $tmp6$$Register,
16978                       -1, $result$$Register, StrIntrinsicNode::UU);
16979   %}
16980   ins_pipe(pipe_class_memory);
16981 %}
16982 
16983 instruct string_indexofLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16984        iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
16985        iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6, rFlagsReg cr)
16986 %{
16987   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16988   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16989   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16990          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6, KILL cr);
16991   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (LL)" %}
16992 
16993   ins_encode %{
16994     __ string_indexof($str1$$Register, $str2$$Register,
16995                       $cnt1$$Register, $cnt2$$Register,
16996                       $tmp1$$Register, $tmp2$$Register,
16997                       $tmp3$$Register, $tmp4$$Register,
16998                       $tmp5$$Register, $tmp6$$Register,
16999                       -1, $result$$Register, StrIntrinsicNode::LL);
17000   %}
17001   ins_pipe(pipe_class_memory);
17002 %}
17003 
17004 instruct string_indexofUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
17005        iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
17006        iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6, rFlagsReg cr)
17007 %{
17008   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
17009   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
17010   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
17011          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6, KILL cr);
17012   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UL)" %}
17013 
17014   ins_encode %{
17015     __ string_indexof($str1$$Register, $str2$$Register,
17016                       $cnt1$$Register, $cnt2$$Register,
17017                       $tmp1$$Register, $tmp2$$Register,
17018                       $tmp3$$Register, $tmp4$$Register,
17019                       $tmp5$$Register, $tmp6$$Register,
17020                       -1, $result$$Register, StrIntrinsicNode::UL);
17021   %}
17022   ins_pipe(pipe_class_memory);
17023 %}
17024 
17025 instruct string_indexof_conUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
17026                  immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
17027                  iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
17028 %{
17029   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
17030   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
17031   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
17032          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
17033   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UU)" %}
17034 
17035   ins_encode %{
17036     int icnt2 = (int)$int_cnt2$$constant;
17037     __ string_indexof($str1$$Register, $str2$$Register,
17038                       $cnt1$$Register, zr,
17039                       $tmp1$$Register, $tmp2$$Register,
17040                       $tmp3$$Register, $tmp4$$Register, zr, zr,
17041                       icnt2, $result$$Register, StrIntrinsicNode::UU);
17042   %}
17043   ins_pipe(pipe_class_memory);
17044 %}
17045 
17046 instruct string_indexof_conLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
17047                  immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
17048                  iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
17049 %{
17050   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
17051   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
17052   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
17053          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
17054   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (LL)" %}
17055 
17056   ins_encode %{
17057     int icnt2 = (int)$int_cnt2$$constant;
17058     __ string_indexof($str1$$Register, $str2$$Register,
17059                       $cnt1$$Register, zr,
17060                       $tmp1$$Register, $tmp2$$Register,
17061                       $tmp3$$Register, $tmp4$$Register, zr, zr,
17062                       icnt2, $result$$Register, StrIntrinsicNode::LL);
17063   %}
17064   ins_pipe(pipe_class_memory);
17065 %}
17066 
17067 instruct string_indexof_conUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
17068                  immI_1 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
17069                  iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
17070 %{
17071   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
17072   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
17073   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
17074          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
17075   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UL)" %}
17076 
17077   ins_encode %{
17078     int icnt2 = (int)$int_cnt2$$constant;
17079     __ string_indexof($str1$$Register, $str2$$Register,
17080                       $cnt1$$Register, zr,
17081                       $tmp1$$Register, $tmp2$$Register,
17082                       $tmp3$$Register, $tmp4$$Register, zr, zr,
17083                       icnt2, $result$$Register, StrIntrinsicNode::UL);
17084   %}
17085   ins_pipe(pipe_class_memory);
17086 %}
17087 
17088 instruct string_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
17089                              iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
17090                              iRegINoSp tmp3, rFlagsReg cr)
17091 %{
17092   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
17093   predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
17094   effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
17095          TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
17096 
17097   format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
17098 
17099   ins_encode %{
17100     __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
17101                            $result$$Register, $tmp1$$Register, $tmp2$$Register,
17102                            $tmp3$$Register);
17103   %}
17104   ins_pipe(pipe_class_memory);
17105 %}
17106 
17107 instruct stringL_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
17108                               iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
17109                               iRegINoSp tmp3, rFlagsReg cr)
17110 %{
17111   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
17112   predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
17113   effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
17114          TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
17115 
17116   format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
17117 
17118   ins_encode %{
17119     __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
17120                             $result$$Register, $tmp1$$Register, $tmp2$$Register,
17121                             $tmp3$$Register);
17122   %}
17123   ins_pipe(pipe_class_memory);
17124 %}
17125 
17126 instruct stringL_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
17127                                   iRegI_R0 result, vecA ztmp1, vecA ztmp2,
17128                                   pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
17129   predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L);
17130   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
17131   effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
17132   format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
17133   ins_encode %{
17134     __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
17135                                $result$$Register, $ztmp1$$FloatRegister,
17136                                $ztmp2$$FloatRegister, $pgtmp$$PRegister,
17137                                $ptmp$$PRegister, true /* isL */);
17138   %}
17139   ins_pipe(pipe_class_memory);
17140 %}
17141 
17142 instruct stringU_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
17143                                   iRegI_R0 result, vecA ztmp1, vecA ztmp2,
17144                                   pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
17145   predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U);
17146   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
17147   effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
17148   format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
17149   ins_encode %{
17150     __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
17151                                $result$$Register, $ztmp1$$FloatRegister,
17152                                $ztmp2$$FloatRegister, $pgtmp$$PRegister,
17153                                $ptmp$$PRegister, false /* isL */);
17154   %}
17155   ins_pipe(pipe_class_memory);
17156 %}
17157 
17158 instruct string_equalsL(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
17159                         iRegI_R0 result, rFlagsReg cr)
17160 %{
17161   predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::LL);
17162   match(Set result (StrEquals (Binary str1 str2) cnt));
17163   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
17164 
17165   format %{ "String Equals $str1,$str2,$cnt -> $result" %}
17166   ins_encode %{
17167     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
17168     __ string_equals($str1$$Register, $str2$$Register,
17169                      $result$$Register, $cnt$$Register, 1);
17170   %}
17171   ins_pipe(pipe_class_memory);
17172 %}
17173 
17174 instruct string_equalsU(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
17175                         iRegI_R0 result, rFlagsReg cr)
17176 %{
17177   predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::UU);
17178   match(Set result (StrEquals (Binary str1 str2) cnt));
17179   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
17180 
17181   format %{ "String Equals $str1,$str2,$cnt -> $result" %}
17182   ins_encode %{
17183     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
17184     __ string_equals($str1$$Register, $str2$$Register,
17185                      $result$$Register, $cnt$$Register, 2);
17186   %}
17187   ins_pipe(pipe_class_memory);
17188 %}
17189 
17190 instruct array_equalsB(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
17191                        iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
17192                        iRegP_R10 tmp, rFlagsReg cr)
17193 %{
17194   predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
17195   match(Set result (AryEq ary1 ary2));
17196   effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
17197 
17198   format %{ "Array Equals $ary1,ary2 -> $result    // KILL $tmp" %}
17199   ins_encode %{
17200     address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
17201                                    $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
17202                                    $result$$Register, $tmp$$Register, 1);
17203     if (tpc == NULL) {
17204       ciEnv::current()->record_failure("CodeCache is full");
17205       return;
17206     }
17207   %}
17208   ins_pipe(pipe_class_memory);
17209 %}
17210 
17211 instruct array_equalsC(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
17212                        iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
17213                        iRegP_R10 tmp, rFlagsReg cr)
17214 %{
17215   predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
17216   match(Set result (AryEq ary1 ary2));
17217   effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
17218 
17219   format %{ "Array Equals $ary1,ary2 -> $result    // KILL $tmp" %}
17220   ins_encode %{
17221     address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
17222                                    $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
17223                                    $result$$Register, $tmp$$Register, 2);
17224     if (tpc == NULL) {
17225       ciEnv::current()->record_failure("CodeCache is full");
17226       return;
17227     }
17228   %}
17229   ins_pipe(pipe_class_memory);
17230 %}
17231 
17232 instruct count_positives(iRegP_R1 ary1, iRegI_R2 len, iRegI_R0 result, rFlagsReg cr)
17233 %{
17234   match(Set result (CountPositives ary1 len));
17235   effect(USE_KILL ary1, USE_KILL len, KILL cr);
17236   format %{ "count positives byte[] $ary1,$len -> $result" %}
17237   ins_encode %{
17238     address tpc = __ count_positives($ary1$$Register, $len$$Register, $result$$Register);
17239     if (tpc == NULL) {
17240       ciEnv::current()->record_failure("CodeCache is full");
17241       return;
17242     }
17243   %}
17244   ins_pipe( pipe_slow );
17245 %}
17246 
17247 // fast char[] to byte[] compression
17248 instruct string_compress(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17249                          vRegD_V0 tmp1, vRegD_V1 tmp2,
17250                          vRegD_V2 tmp3, vRegD_V3 tmp4,
17251                          iRegI_R0 result, rFlagsReg cr)
17252 %{
17253   match(Set result (StrCompressedCopy src (Binary dst len)));
17254   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4,
17255          USE_KILL src, USE_KILL dst, USE len, KILL cr);
17256 
17257   format %{ "String Compress $src,$dst,$len -> $result  // KILL $src,$dst" %}
17258   ins_encode %{
17259     __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
17260                            $result$$Register,
17261                            $tmp1$$FloatRegister, $tmp2$$FloatRegister,
17262                            $tmp3$$FloatRegister, $tmp4$$FloatRegister);
17263   %}
17264   ins_pipe(pipe_slow);
17265 %}
17266 
17267 // fast byte[] to char[] inflation
17268 instruct string_inflate(Universe dummy, iRegP_R0 src, iRegP_R1 dst, iRegI_R2 len,
17269                         vRegD_V0 tmp1, vRegD_V1 tmp2, vRegD_V2 tmp3, iRegP_R3 tmp4, rFlagsReg cr)
17270 %{
17271   match(Set dummy (StrInflatedCopy src (Binary dst len)));
17272   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
17273 
17274   format %{ "String Inflate $src,$dst    // KILL $tmp1, $tmp2" %}
17275   ins_encode %{
17276     address tpc = __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
17277                                         $tmp1$$FloatRegister, $tmp2$$FloatRegister,
17278                                         $tmp3$$FloatRegister, $tmp4$$Register);
17279     if (tpc == NULL) {
17280       ciEnv::current()->record_failure("CodeCache is full");
17281       return;
17282     }
17283   %}
17284   ins_pipe(pipe_class_memory);
17285 %}
17286 
17287 // encode char[] to byte[] in ISO_8859_1
17288 instruct encode_iso_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17289                           vRegD_V0 vtmp0, vRegD_V1 vtmp1,
17290                           vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17291                           iRegI_R0 result, rFlagsReg cr)
17292 %{
17293   predicate(!((EncodeISOArrayNode*)n)->is_ascii());
17294   match(Set result (EncodeISOArray src (Binary dst len)));
17295   effect(USE_KILL src, USE_KILL dst, USE len,
17296          KILL vtmp0, KILL vtmp1, KILL vtmp2, KILL vtmp3, KILL cr);
17297 
17298   format %{ "Encode ISO array $src,$dst,$len -> $result" %}
17299   ins_encode %{
17300     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
17301                         $result$$Register, false,
17302                         $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17303                         $vtmp2$$FloatRegister, $vtmp3$$FloatRegister);
17304   %}
17305   ins_pipe(pipe_class_memory);
17306 %}
17307 
17308 instruct encode_ascii_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17309                             vRegD_V0 vtmp0, vRegD_V1 vtmp1,
17310                             vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17311                             iRegI_R0 result, rFlagsReg cr)
17312 %{
17313   predicate(((EncodeISOArrayNode*)n)->is_ascii());
17314   match(Set result (EncodeISOArray src (Binary dst len)));
17315   effect(USE_KILL src, USE_KILL dst, USE len,
17316          KILL vtmp0, KILL vtmp1, KILL vtmp2, KILL vtmp3, KILL cr);
17317 
17318   format %{ "Encode ASCII array $src,$dst,$len -> $result" %}
17319   ins_encode %{
17320     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
17321                         $result$$Register, true,
17322                         $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17323                         $vtmp2$$FloatRegister, $vtmp3$$FloatRegister);
17324   %}
17325   ins_pipe(pipe_class_memory);
17326 %}
17327 
17328 // ============================================================================
17329 // This name is KNOWN by the ADLC and cannot be changed.
17330 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
17331 // for this guy.
17332 instruct tlsLoadP(thread_RegP dst)
17333 %{
17334   match(Set dst (ThreadLocal));
17335 
17336   ins_cost(0);
17337 
17338   format %{ " -- \t// $dst=Thread::current(), empty" %}
17339 
17340   size(0);
17341 
17342   ins_encode( /*empty*/ );
17343 
17344   ins_pipe(pipe_class_empty);
17345 %}
17346 
17347 //----------PEEPHOLE RULES-----------------------------------------------------
17348 // These must follow all instruction definitions as they use the names
17349 // defined in the instructions definitions.
17350 //
17351 // peepmatch ( root_instr_name [preceding_instruction]* );
17352 //
17353 // peepconstraint %{
17354 // (instruction_number.operand_name relational_op instruction_number.operand_name
17355 //  [, ...] );
17356 // // instruction numbers are zero-based using left to right order in peepmatch
17357 //
17358 // peepreplace ( instr_name  ( [instruction_number.operand_name]* ) );
17359 // // provide an instruction_number.operand_name for each operand that appears
17360 // // in the replacement instruction's match rule
17361 //
17362 // ---------VM FLAGS---------------------------------------------------------
17363 //
17364 // All peephole optimizations can be turned off using -XX:-OptoPeephole
17365 //
17366 // Each peephole rule is given an identifying number starting with zero and
17367 // increasing by one in the order seen by the parser.  An individual peephole
17368 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
17369 // on the command-line.
17370 //
17371 // ---------CURRENT LIMITATIONS----------------------------------------------
17372 //
17373 // Only match adjacent instructions in same basic block
17374 // Only equality constraints
17375 // Only constraints between operands, not (0.dest_reg == RAX_enc)
17376 // Only one replacement instruction
17377 //
17378 // ---------EXAMPLE----------------------------------------------------------
17379 //
17380 // // pertinent parts of existing instructions in architecture description
17381 // instruct movI(iRegINoSp dst, iRegI src)
17382 // %{
17383 //   match(Set dst (CopyI src));
17384 // %}
17385 //
17386 // instruct incI_iReg(iRegINoSp dst, immI1 src, rFlagsReg cr)
17387 // %{
17388 //   match(Set dst (AddI dst src));
17389 //   effect(KILL cr);
17390 // %}
17391 //
17392 // // Change (inc mov) to lea
17393 // peephole %{
17394 //   // increment preceded by register-register move
17395 //   peepmatch ( incI_iReg movI );
17396 //   // require that the destination register of the increment
17397 //   // match the destination register of the move
17398 //   peepconstraint ( 0.dst == 1.dst );
17399 //   // construct a replacement instruction that sets
17400 //   // the destination to ( move's source register + one )
17401 //   peepreplace ( leaI_iReg_immI( 0.dst 1.src 0.src ) );
17402 // %}
17403 //
17404 
17405 // Implementation no longer uses movX instructions since
17406 // machine-independent system no longer uses CopyX nodes.
17407 //
17408 // peephole
17409 // %{
17410 //   peepmatch (incI_iReg movI);
17411 //   peepconstraint (0.dst == 1.dst);
17412 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17413 // %}
17414 
17415 // peephole
17416 // %{
17417 //   peepmatch (decI_iReg movI);
17418 //   peepconstraint (0.dst == 1.dst);
17419 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17420 // %}
17421 
17422 // peephole
17423 // %{
17424 //   peepmatch (addI_iReg_imm movI);
17425 //   peepconstraint (0.dst == 1.dst);
17426 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17427 // %}
17428 
17429 // peephole
17430 // %{
17431 //   peepmatch (incL_iReg movL);
17432 //   peepconstraint (0.dst == 1.dst);
17433 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17434 // %}
17435 
17436 // peephole
17437 // %{
17438 //   peepmatch (decL_iReg movL);
17439 //   peepconstraint (0.dst == 1.dst);
17440 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17441 // %}
17442 
17443 // peephole
17444 // %{
17445 //   peepmatch (addL_iReg_imm movL);
17446 //   peepconstraint (0.dst == 1.dst);
17447 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17448 // %}
17449 
17450 // peephole
17451 // %{
17452 //   peepmatch (addP_iReg_imm movP);
17453 //   peepconstraint (0.dst == 1.dst);
17454 //   peepreplace (leaP_iReg_imm(0.dst 1.src 0.src));
17455 // %}
17456 
17457 // // Change load of spilled value to only a spill
17458 // instruct storeI(memory mem, iRegI src)
17459 // %{
17460 //   match(Set mem (StoreI mem src));
17461 // %}
17462 //
17463 // instruct loadI(iRegINoSp dst, memory mem)
17464 // %{
17465 //   match(Set dst (LoadI mem));
17466 // %}
17467 //
17468 
17469 //----------SMARTSPILL RULES---------------------------------------------------
17470 // These must follow all instruction definitions as they use the names
17471 // defined in the instructions definitions.
17472 
17473 // Local Variables:
17474 // mode: c++
17475 // End:
--- EOF ---