1 //
    2 // Copyright (c) 2003, 2024, Oracle and/or its affiliates. All rights reserved.
    3 // Copyright (c) 2014, 2024, Red Hat, Inc. All rights reserved.
    4 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
    5 //
    6 // This code is free software; you can redistribute it and/or modify it
    7 // under the terms of the GNU General Public License version 2 only, as
    8 // published by the Free Software Foundation.
    9 //
   10 // This code is distributed in the hope that it will be useful, but WITHOUT
   11 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
   12 // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
   13 // version 2 for more details (a copy is included in the LICENSE file that
   14 // accompanied this code).
   15 //
   16 // You should have received a copy of the GNU General Public License version
   17 // 2 along with this work; if not, write to the Free Software Foundation,
   18 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
   19 //
   20 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
   21 // or visit www.oracle.com if you need additional information or have any
   22 // questions.
   23 //
   24 //
   25 
   26 // AArch64 Architecture Description File
   27 
   28 //----------REGISTER DEFINITION BLOCK------------------------------------------
   29 // This information is used by the matcher and the register allocator to
   30 // describe individual registers and classes of registers within the target
   31 // architecture.
   32 
   33 register %{
   34 //----------Architecture Description Register Definitions----------------------
   35 // General Registers
   36 // "reg_def"  name ( register save type, C convention save type,
   37 //                   ideal register type, encoding );
   38 // Register Save Types:
   39 //
   40 // NS  = No-Save:       The register allocator assumes that these registers
   41 //                      can be used without saving upon entry to the method, &
   42 //                      that they do not need to be saved at call sites.
   43 //
   44 // SOC = Save-On-Call:  The register allocator assumes that these registers
   45 //                      can be used without saving upon entry to the method,
   46 //                      but that they must be saved at call sites.
   47 //
   48 // SOE = Save-On-Entry: The register allocator assumes that these registers
   49 //                      must be saved before using them upon entry to the
   50 //                      method, but they do not need to be saved at call
   51 //                      sites.
   52 //
   53 // AS  = Always-Save:   The register allocator assumes that these registers
   54 //                      must be saved before using them upon entry to the
   55 //                      method, & that they must be saved at call sites.
   56 //
   57 // Ideal Register Type is used to determine how to save & restore a
   58 // register.  Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
   59 // spilled with LoadP/StoreP.  If the register supports both, use Op_RegI.
   60 //
   61 // The encoding number is the actual bit-pattern placed into the opcodes.
   62 
   63 // We must define the 64 bit int registers in two 32 bit halves, the
   64 // real lower register and a virtual upper half register. upper halves
   65 // are used by the register allocator but are not actually supplied as
   66 // operands to memory ops.
   67 //
   68 // follow the C1 compiler in making registers
   69 //
   70 //   r0-r7,r10-r26 volatile (caller save)
   71 //   r27-r32 system (no save, no allocate)
   72 //   r8-r9 non-allocatable (so we can use them as scratch regs)
   73 //
   74 // as regards Java usage. we don't use any callee save registers
   75 // because this makes it difficult to de-optimise a frame (see comment
   76 // in x86 implementation of Deoptimization::unwind_callee_save_values)
   77 //
   78 
   79 // General Registers
   80 
   81 reg_def R0      ( SOC, SOC, Op_RegI,  0, r0->as_VMReg()         );
   82 reg_def R0_H    ( SOC, SOC, Op_RegI,  0, r0->as_VMReg()->next() );
   83 reg_def R1      ( SOC, SOC, Op_RegI,  1, r1->as_VMReg()         );
   84 reg_def R1_H    ( SOC, SOC, Op_RegI,  1, r1->as_VMReg()->next() );
   85 reg_def R2      ( SOC, SOC, Op_RegI,  2, r2->as_VMReg()         );
   86 reg_def R2_H    ( SOC, SOC, Op_RegI,  2, r2->as_VMReg()->next() );
   87 reg_def R3      ( SOC, SOC, Op_RegI,  3, r3->as_VMReg()         );
   88 reg_def R3_H    ( SOC, SOC, Op_RegI,  3, r3->as_VMReg()->next() );
   89 reg_def R4      ( SOC, SOC, Op_RegI,  4, r4->as_VMReg()         );
   90 reg_def R4_H    ( SOC, SOC, Op_RegI,  4, r4->as_VMReg()->next() );
   91 reg_def R5      ( SOC, SOC, Op_RegI,  5, r5->as_VMReg()         );
   92 reg_def R5_H    ( SOC, SOC, Op_RegI,  5, r5->as_VMReg()->next() );
   93 reg_def R6      ( SOC, SOC, Op_RegI,  6, r6->as_VMReg()         );
   94 reg_def R6_H    ( SOC, SOC, Op_RegI,  6, r6->as_VMReg()->next() );
   95 reg_def R7      ( SOC, SOC, Op_RegI,  7, r7->as_VMReg()         );
   96 reg_def R7_H    ( SOC, SOC, Op_RegI,  7, r7->as_VMReg()->next() );
   97 reg_def R8      ( NS,  SOC, Op_RegI,  8, r8->as_VMReg()         ); // rscratch1, non-allocatable
   98 reg_def R8_H    ( NS,  SOC, Op_RegI,  8, r8->as_VMReg()->next() );
   99 reg_def R9      ( NS,  SOC, Op_RegI,  9, r9->as_VMReg()         ); // rscratch2, non-allocatable
  100 reg_def R9_H    ( NS,  SOC, Op_RegI,  9, r9->as_VMReg()->next() );
  101 reg_def R10     ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()        );
  102 reg_def R10_H   ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
  103 reg_def R11     ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()        );
  104 reg_def R11_H   ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
  105 reg_def R12     ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()        );
  106 reg_def R12_H   ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()->next());
  107 reg_def R13     ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()        );
  108 reg_def R13_H   ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()->next());
  109 reg_def R14     ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()        );
  110 reg_def R14_H   ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()->next());
  111 reg_def R15     ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()        );
  112 reg_def R15_H   ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()->next());
  113 reg_def R16     ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()        );
  114 reg_def R16_H   ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
  115 reg_def R17     ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()        );
  116 reg_def R17_H   ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
  117 reg_def R18     ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()        );
  118 reg_def R18_H   ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()->next());
  119 reg_def R19     ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()        );
  120 reg_def R19_H   ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()->next());
  121 reg_def R20     ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()        ); // caller esp
  122 reg_def R20_H   ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()->next());
  123 reg_def R21     ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()        );
  124 reg_def R21_H   ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()->next());
  125 reg_def R22     ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()        );
  126 reg_def R22_H   ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()->next());
  127 reg_def R23     ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()        );
  128 reg_def R23_H   ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()->next());
  129 reg_def R24     ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()        );
  130 reg_def R24_H   ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()->next());
  131 reg_def R25     ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()        );
  132 reg_def R25_H   ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()->next());
  133 reg_def R26     ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()        );
  134 reg_def R26_H   ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()->next());
  135 reg_def R27     ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()        ); // heapbase
  136 reg_def R27_H   ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()->next());
  137 reg_def R28     (  NS, SOE, Op_RegI, 28, r28->as_VMReg()        ); // thread
  138 reg_def R28_H   (  NS, SOE, Op_RegI, 28, r28->as_VMReg()->next());
  139 reg_def R29     (  NS,  NS, Op_RegI, 29, r29->as_VMReg()        ); // fp
  140 reg_def R29_H   (  NS,  NS, Op_RegI, 29, r29->as_VMReg()->next());
  141 reg_def R30     (  NS,  NS, Op_RegI, 30, r30->as_VMReg()        ); // lr
  142 reg_def R30_H   (  NS,  NS, Op_RegI, 30, r30->as_VMReg()->next());
  143 reg_def R31     (  NS,  NS, Op_RegI, 31, r31_sp->as_VMReg()     ); // sp
  144 reg_def R31_H   (  NS,  NS, Op_RegI, 31, r31_sp->as_VMReg()->next());
  145 
  146 // ----------------------------
  147 // Float/Double/Vector Registers
  148 // ----------------------------
  149 
  150 // Double Registers
  151 
  152 // The rules of ADL require that double registers be defined in pairs.
  153 // Each pair must be two 32-bit values, but not necessarily a pair of
  154 // single float registers. In each pair, ADLC-assigned register numbers
  155 // must be adjacent, with the lower number even. Finally, when the
  156 // CPU stores such a register pair to memory, the word associated with
  157 // the lower ADLC-assigned number must be stored to the lower address.
  158 
  159 // AArch64 has 32 floating-point registers. Each can store a vector of
  160 // single or double precision floating-point values up to 8 * 32
  161 // floats, 4 * 64 bit floats or 2 * 128 bit floats.  We currently only
  162 // use the first float or double element of the vector.
  163 
  164 // for Java use float registers v0-v15 are always save on call whereas
  165 // the platform ABI treats v8-v15 as callee save). float registers
  166 // v16-v31 are SOC as per the platform spec
  167 
  168 // For SVE vector registers, we simply extend vector register size to 8
  169 // 'logical' slots. This is nominally 256 bits but it actually covers
  170 // all possible 'physical' SVE vector register lengths from 128 ~ 2048
  171 // bits. The 'physical' SVE vector register length is detected during
  172 // startup, so the register allocator is able to identify the correct
  173 // number of bytes needed for an SVE spill/unspill.
  174 // Note that a vector register with 4 slots denotes a 128-bit NEON
  175 // register allowing it to be distinguished from the corresponding SVE
  176 // vector register when the SVE vector length is 128 bits.
  177 
  178   reg_def V0   ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()          );
  179   reg_def V0_H ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next()  );
  180   reg_def V0_J ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(2) );
  181   reg_def V0_K ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(3) );
  182 
  183   reg_def V1   ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()          );
  184   reg_def V1_H ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next()  );
  185   reg_def V1_J ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(2) );
  186   reg_def V1_K ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(3) );
  187 
  188   reg_def V2   ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()          );
  189   reg_def V2_H ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next()  );
  190   reg_def V2_J ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(2) );
  191   reg_def V2_K ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(3) );
  192 
  193   reg_def V3   ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()          );
  194   reg_def V3_H ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next()  );
  195   reg_def V3_J ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(2) );
  196   reg_def V3_K ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(3) );
  197 
  198   reg_def V4   ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()          );
  199   reg_def V4_H ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next()  );
  200   reg_def V4_J ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(2) );
  201   reg_def V4_K ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(3) );
  202 
  203   reg_def V5   ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()          );
  204   reg_def V5_H ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next()  );
  205   reg_def V5_J ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(2) );
  206   reg_def V5_K ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(3) );
  207 
  208   reg_def V6   ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()          );
  209   reg_def V6_H ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next()  );
  210   reg_def V6_J ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(2) );
  211   reg_def V6_K ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(3) );
  212 
  213   reg_def V7   ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()          );
  214   reg_def V7_H ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next()  );
  215   reg_def V7_J ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(2) );
  216   reg_def V7_K ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(3) );
  217 
  218   reg_def V8   ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()          );
  219   reg_def V8_H ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()->next()  );
  220   reg_def V8_J ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(2) );
  221   reg_def V8_K ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(3) );
  222 
  223   reg_def V9   ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()          );
  224   reg_def V9_H ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()->next()  );
  225   reg_def V9_J ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(2) );
  226   reg_def V9_K ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(3) );
  227 
  228   reg_def V10   ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()          );
  229   reg_def V10_H ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()->next()  );
  230   reg_def V10_J ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(2) );
  231   reg_def V10_K ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(3) );
  232 
  233   reg_def V11   ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()          );
  234   reg_def V11_H ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()->next()  );
  235   reg_def V11_J ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(2) );
  236   reg_def V11_K ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(3) );
  237 
  238   reg_def V12   ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()          );
  239   reg_def V12_H ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()->next()  );
  240   reg_def V12_J ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(2) );
  241   reg_def V12_K ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(3) );
  242 
  243   reg_def V13   ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()          );
  244   reg_def V13_H ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()->next()  );
  245   reg_def V13_J ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(2) );
  246   reg_def V13_K ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(3) );
  247 
  248   reg_def V14   ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()          );
  249   reg_def V14_H ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()->next()  );
  250   reg_def V14_J ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(2) );
  251   reg_def V14_K ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(3) );
  252 
  253   reg_def V15   ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()          );
  254   reg_def V15_H ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()->next()  );
  255   reg_def V15_J ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(2) );
  256   reg_def V15_K ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(3) );
  257 
  258   reg_def V16   ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()          );
  259   reg_def V16_H ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next()  );
  260   reg_def V16_J ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(2) );
  261   reg_def V16_K ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(3) );
  262 
  263   reg_def V17   ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()          );
  264   reg_def V17_H ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next()  );
  265   reg_def V17_J ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(2) );
  266   reg_def V17_K ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(3) );
  267 
  268   reg_def V18   ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()          );
  269   reg_def V18_H ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next()  );
  270   reg_def V18_J ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(2) );
  271   reg_def V18_K ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(3) );
  272 
  273   reg_def V19   ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()          );
  274   reg_def V19_H ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next()  );
  275   reg_def V19_J ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(2) );
  276   reg_def V19_K ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(3) );
  277 
  278   reg_def V20   ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()          );
  279   reg_def V20_H ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next()  );
  280   reg_def V20_J ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(2) );
  281   reg_def V20_K ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(3) );
  282 
  283   reg_def V21   ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()          );
  284   reg_def V21_H ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next()  );
  285   reg_def V21_J ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(2) );
  286   reg_def V21_K ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(3) );
  287 
  288   reg_def V22   ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()          );
  289   reg_def V22_H ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next()  );
  290   reg_def V22_J ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(2) );
  291   reg_def V22_K ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(3) );
  292 
  293   reg_def V23   ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()          );
  294   reg_def V23_H ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next()  );
  295   reg_def V23_J ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(2) );
  296   reg_def V23_K ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(3) );
  297 
  298   reg_def V24   ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()          );
  299   reg_def V24_H ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next()  );
  300   reg_def V24_J ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(2) );
  301   reg_def V24_K ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(3) );
  302 
  303   reg_def V25   ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()          );
  304   reg_def V25_H ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next()  );
  305   reg_def V25_J ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(2) );
  306   reg_def V25_K ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(3) );
  307 
  308   reg_def V26   ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()          );
  309   reg_def V26_H ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next()  );
  310   reg_def V26_J ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(2) );
  311   reg_def V26_K ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(3) );
  312 
  313   reg_def V27   ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()          );
  314   reg_def V27_H ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next()  );
  315   reg_def V27_J ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(2) );
  316   reg_def V27_K ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(3) );
  317 
  318   reg_def V28   ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()          );
  319   reg_def V28_H ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next()  );
  320   reg_def V28_J ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(2) );
  321   reg_def V28_K ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(3) );
  322 
  323   reg_def V29   ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()          );
  324   reg_def V29_H ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next()  );
  325   reg_def V29_J ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(2) );
  326   reg_def V29_K ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(3) );
  327 
  328   reg_def V30   ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()          );
  329   reg_def V30_H ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next()  );
  330   reg_def V30_J ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(2) );
  331   reg_def V30_K ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(3) );
  332 
  333   reg_def V31   ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()          );
  334   reg_def V31_H ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next()  );
  335   reg_def V31_J ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(2) );
  336   reg_def V31_K ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(3) );
  337 
  338 // ----------------------------
  339 // SVE Predicate Registers
  340 // ----------------------------
  341   reg_def P0 (SOC, SOC, Op_RegVectMask, 0, p0->as_VMReg());
  342   reg_def P1 (SOC, SOC, Op_RegVectMask, 1, p1->as_VMReg());
  343   reg_def P2 (SOC, SOC, Op_RegVectMask, 2, p2->as_VMReg());
  344   reg_def P3 (SOC, SOC, Op_RegVectMask, 3, p3->as_VMReg());
  345   reg_def P4 (SOC, SOC, Op_RegVectMask, 4, p4->as_VMReg());
  346   reg_def P5 (SOC, SOC, Op_RegVectMask, 5, p5->as_VMReg());
  347   reg_def P6 (SOC, SOC, Op_RegVectMask, 6, p6->as_VMReg());
  348   reg_def P7 (SOC, SOC, Op_RegVectMask, 7, p7->as_VMReg());
  349   reg_def P8 (SOC, SOC, Op_RegVectMask, 8, p8->as_VMReg());
  350   reg_def P9 (SOC, SOC, Op_RegVectMask, 9, p9->as_VMReg());
  351   reg_def P10 (SOC, SOC, Op_RegVectMask, 10, p10->as_VMReg());
  352   reg_def P11 (SOC, SOC, Op_RegVectMask, 11, p11->as_VMReg());
  353   reg_def P12 (SOC, SOC, Op_RegVectMask, 12, p12->as_VMReg());
  354   reg_def P13 (SOC, SOC, Op_RegVectMask, 13, p13->as_VMReg());
  355   reg_def P14 (SOC, SOC, Op_RegVectMask, 14, p14->as_VMReg());
  356   reg_def P15 (SOC, SOC, Op_RegVectMask, 15, p15->as_VMReg());
  357 
  358 // ----------------------------
  359 // Special Registers
  360 // ----------------------------
  361 
  362 // the AArch64 CSPR status flag register is not directly accessible as
  363 // instruction operand. the FPSR status flag register is a system
  364 // register which can be written/read using MSR/MRS but again does not
  365 // appear as an operand (a code identifying the FSPR occurs as an
  366 // immediate value in the instruction).
  367 
  368 reg_def RFLAGS(SOC, SOC, 0, 32, VMRegImpl::Bad());
  369 
  370 // Specify priority of register selection within phases of register
  371 // allocation.  Highest priority is first.  A useful heuristic is to
  372 // give registers a low priority when they are required by machine
  373 // instructions, like EAX and EDX on I486, and choose no-save registers
  374 // before save-on-call, & save-on-call before save-on-entry.  Registers
  375 // which participate in fixed calling sequences should come last.
  376 // Registers which are used as pairs must fall on an even boundary.
  377 
  378 alloc_class chunk0(
  379     // volatiles
  380     R10, R10_H,
  381     R11, R11_H,
  382     R12, R12_H,
  383     R13, R13_H,
  384     R14, R14_H,
  385     R15, R15_H,
  386     R16, R16_H,
  387     R17, R17_H,
  388     R18, R18_H,
  389 
  390     // arg registers
  391     R0, R0_H,
  392     R1, R1_H,
  393     R2, R2_H,
  394     R3, R3_H,
  395     R4, R4_H,
  396     R5, R5_H,
  397     R6, R6_H,
  398     R7, R7_H,
  399 
  400     // non-volatiles
  401     R19, R19_H,
  402     R20, R20_H,
  403     R21, R21_H,
  404     R22, R22_H,
  405     R23, R23_H,
  406     R24, R24_H,
  407     R25, R25_H,
  408     R26, R26_H,
  409 
  410     // non-allocatable registers
  411 
  412     R27, R27_H, // heapbase
  413     R28, R28_H, // thread
  414     R29, R29_H, // fp
  415     R30, R30_H, // lr
  416     R31, R31_H, // sp
  417     R8, R8_H,   // rscratch1
  418     R9, R9_H,   // rscratch2
  419 );
  420 
  421 alloc_class chunk1(
  422 
  423     // no save
  424     V16, V16_H, V16_J, V16_K,
  425     V17, V17_H, V17_J, V17_K,
  426     V18, V18_H, V18_J, V18_K,
  427     V19, V19_H, V19_J, V19_K,
  428     V20, V20_H, V20_J, V20_K,
  429     V21, V21_H, V21_J, V21_K,
  430     V22, V22_H, V22_J, V22_K,
  431     V23, V23_H, V23_J, V23_K,
  432     V24, V24_H, V24_J, V24_K,
  433     V25, V25_H, V25_J, V25_K,
  434     V26, V26_H, V26_J, V26_K,
  435     V27, V27_H, V27_J, V27_K,
  436     V28, V28_H, V28_J, V28_K,
  437     V29, V29_H, V29_J, V29_K,
  438     V30, V30_H, V30_J, V30_K,
  439     V31, V31_H, V31_J, V31_K,
  440 
  441     // arg registers
  442     V0, V0_H, V0_J, V0_K,
  443     V1, V1_H, V1_J, V1_K,
  444     V2, V2_H, V2_J, V2_K,
  445     V3, V3_H, V3_J, V3_K,
  446     V4, V4_H, V4_J, V4_K,
  447     V5, V5_H, V5_J, V5_K,
  448     V6, V6_H, V6_J, V6_K,
  449     V7, V7_H, V7_J, V7_K,
  450 
  451     // non-volatiles
  452     V8, V8_H, V8_J, V8_K,
  453     V9, V9_H, V9_J, V9_K,
  454     V10, V10_H, V10_J, V10_K,
  455     V11, V11_H, V11_J, V11_K,
  456     V12, V12_H, V12_J, V12_K,
  457     V13, V13_H, V13_J, V13_K,
  458     V14, V14_H, V14_J, V14_K,
  459     V15, V15_H, V15_J, V15_K,
  460 );
  461 
  462 alloc_class chunk2 (
  463     // Governing predicates for load/store and arithmetic
  464     P0,
  465     P1,
  466     P2,
  467     P3,
  468     P4,
  469     P5,
  470     P6,
  471 
  472     // Extra predicates
  473     P8,
  474     P9,
  475     P10,
  476     P11,
  477     P12,
  478     P13,
  479     P14,
  480     P15,
  481 
  482     // Preserved for all-true predicate
  483     P7,
  484 );
  485 
  486 alloc_class chunk3(RFLAGS);
  487 
  488 //----------Architecture Description Register Classes--------------------------
  489 // Several register classes are automatically defined based upon information in
  490 // this architecture description.
  491 // 1) reg_class inline_cache_reg           ( /* as def'd in frame section */ )
  492 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
  493 //
  494 
  495 // Class for all 32 bit general purpose registers
  496 reg_class all_reg32(
  497     R0,
  498     R1,
  499     R2,
  500     R3,
  501     R4,
  502     R5,
  503     R6,
  504     R7,
  505     R10,
  506     R11,
  507     R12,
  508     R13,
  509     R14,
  510     R15,
  511     R16,
  512     R17,
  513     R18,
  514     R19,
  515     R20,
  516     R21,
  517     R22,
  518     R23,
  519     R24,
  520     R25,
  521     R26,
  522     R27,
  523     R28,
  524     R29,
  525     R30,
  526     R31
  527 );
  528 
  529 
  530 // Class for all 32 bit integer registers (excluding SP which
  531 // will never be used as an integer register)
  532 reg_class any_reg32 %{
  533   return _ANY_REG32_mask;
  534 %}
  535 
  536 // Singleton class for R0 int register
  537 reg_class int_r0_reg(R0);
  538 
  539 // Singleton class for R2 int register
  540 reg_class int_r2_reg(R2);
  541 
  542 // Singleton class for R3 int register
  543 reg_class int_r3_reg(R3);
  544 
  545 // Singleton class for R4 int register
  546 reg_class int_r4_reg(R4);
  547 
  548 // Singleton class for R31 int register
  549 reg_class int_r31_reg(R31);
  550 
  551 // Class for all 64 bit general purpose registers
  552 reg_class all_reg(
  553     R0, R0_H,
  554     R1, R1_H,
  555     R2, R2_H,
  556     R3, R3_H,
  557     R4, R4_H,
  558     R5, R5_H,
  559     R6, R6_H,
  560     R7, R7_H,
  561     R10, R10_H,
  562     R11, R11_H,
  563     R12, R12_H,
  564     R13, R13_H,
  565     R14, R14_H,
  566     R15, R15_H,
  567     R16, R16_H,
  568     R17, R17_H,
  569     R18, R18_H,
  570     R19, R19_H,
  571     R20, R20_H,
  572     R21, R21_H,
  573     R22, R22_H,
  574     R23, R23_H,
  575     R24, R24_H,
  576     R25, R25_H,
  577     R26, R26_H,
  578     R27, R27_H,
  579     R28, R28_H,
  580     R29, R29_H,
  581     R30, R30_H,
  582     R31, R31_H
  583 );
  584 
  585 // Class for all long integer registers (including SP)
  586 reg_class any_reg %{
  587   return _ANY_REG_mask;
  588 %}
  589 
  590 // Class for non-allocatable 32 bit registers
  591 reg_class non_allocatable_reg32(
  592 #ifdef R18_RESERVED
  593     // See comment in register_aarch64.hpp
  594     R18,                        // tls on Windows
  595 #endif
  596     R28,                        // thread
  597     R30,                        // lr
  598     R31                         // sp
  599 );
  600 
  601 // Class for non-allocatable 64 bit registers
  602 reg_class non_allocatable_reg(
  603 #ifdef R18_RESERVED
  604     // See comment in register_aarch64.hpp
  605     R18, R18_H,                 // tls on Windows, platform register on macOS
  606 #endif
  607     R28, R28_H,                 // thread
  608     R30, R30_H,                 // lr
  609     R31, R31_H                  // sp
  610 );
  611 
  612 // Class for all non-special integer registers
  613 reg_class no_special_reg32 %{
  614   return _NO_SPECIAL_REG32_mask;
  615 %}
  616 
  617 // Class for all non-special long integer registers
  618 reg_class no_special_reg %{
  619   return _NO_SPECIAL_REG_mask;
  620 %}
  621 
  622 // Class for 64 bit register r0
  623 reg_class r0_reg(
  624     R0, R0_H
  625 );
  626 
  627 // Class for 64 bit register r1
  628 reg_class r1_reg(
  629     R1, R1_H
  630 );
  631 
  632 // Class for 64 bit register r2
  633 reg_class r2_reg(
  634     R2, R2_H
  635 );
  636 
  637 // Class for 64 bit register r3
  638 reg_class r3_reg(
  639     R3, R3_H
  640 );
  641 
  642 // Class for 64 bit register r4
  643 reg_class r4_reg(
  644     R4, R4_H
  645 );
  646 
  647 // Class for 64 bit register r5
  648 reg_class r5_reg(
  649     R5, R5_H
  650 );
  651 
  652 // Class for 64 bit register r10
  653 reg_class r10_reg(
  654     R10, R10_H
  655 );
  656 
  657 // Class for 64 bit register r11
  658 reg_class r11_reg(
  659     R11, R11_H
  660 );
  661 
  662 // Class for method register
  663 reg_class method_reg(
  664     R12, R12_H
  665 );
  666 
  667 // Class for thread register
  668 reg_class thread_reg(
  669     R28, R28_H
  670 );
  671 
  672 // Class for frame pointer register
  673 reg_class fp_reg(
  674     R29, R29_H
  675 );
  676 
  677 // Class for link register
  678 reg_class lr_reg(
  679     R30, R30_H
  680 );
  681 
  682 // Class for long sp register
  683 reg_class sp_reg(
  684   R31, R31_H
  685 );
  686 
  687 // Class for all pointer registers
  688 reg_class ptr_reg %{
  689   return _PTR_REG_mask;
  690 %}
  691 
  692 // Class for all non_special pointer registers
  693 reg_class no_special_ptr_reg %{
  694   return _NO_SPECIAL_PTR_REG_mask;
  695 %}
  696 
  697 // Class for all non_special pointer registers (excluding rfp)
  698 reg_class no_special_no_rfp_ptr_reg %{
  699   return _NO_SPECIAL_NO_RFP_PTR_REG_mask;
  700 %}
  701 
  702 // Class for all float registers
  703 reg_class float_reg(
  704     V0,
  705     V1,
  706     V2,
  707     V3,
  708     V4,
  709     V5,
  710     V6,
  711     V7,
  712     V8,
  713     V9,
  714     V10,
  715     V11,
  716     V12,
  717     V13,
  718     V14,
  719     V15,
  720     V16,
  721     V17,
  722     V18,
  723     V19,
  724     V20,
  725     V21,
  726     V22,
  727     V23,
  728     V24,
  729     V25,
  730     V26,
  731     V27,
  732     V28,
  733     V29,
  734     V30,
  735     V31
  736 );
  737 
  738 // Double precision float registers have virtual `high halves' that
  739 // are needed by the allocator.
  740 // Class for all double registers
  741 reg_class double_reg(
  742     V0, V0_H,
  743     V1, V1_H,
  744     V2, V2_H,
  745     V3, V3_H,
  746     V4, V4_H,
  747     V5, V5_H,
  748     V6, V6_H,
  749     V7, V7_H,
  750     V8, V8_H,
  751     V9, V9_H,
  752     V10, V10_H,
  753     V11, V11_H,
  754     V12, V12_H,
  755     V13, V13_H,
  756     V14, V14_H,
  757     V15, V15_H,
  758     V16, V16_H,
  759     V17, V17_H,
  760     V18, V18_H,
  761     V19, V19_H,
  762     V20, V20_H,
  763     V21, V21_H,
  764     V22, V22_H,
  765     V23, V23_H,
  766     V24, V24_H,
  767     V25, V25_H,
  768     V26, V26_H,
  769     V27, V27_H,
  770     V28, V28_H,
  771     V29, V29_H,
  772     V30, V30_H,
  773     V31, V31_H
  774 );
  775 
  776 // Class for all SVE vector registers.
  777 reg_class vectora_reg (
  778     V0, V0_H, V0_J, V0_K,
  779     V1, V1_H, V1_J, V1_K,
  780     V2, V2_H, V2_J, V2_K,
  781     V3, V3_H, V3_J, V3_K,
  782     V4, V4_H, V4_J, V4_K,
  783     V5, V5_H, V5_J, V5_K,
  784     V6, V6_H, V6_J, V6_K,
  785     V7, V7_H, V7_J, V7_K,
  786     V8, V8_H, V8_J, V8_K,
  787     V9, V9_H, V9_J, V9_K,
  788     V10, V10_H, V10_J, V10_K,
  789     V11, V11_H, V11_J, V11_K,
  790     V12, V12_H, V12_J, V12_K,
  791     V13, V13_H, V13_J, V13_K,
  792     V14, V14_H, V14_J, V14_K,
  793     V15, V15_H, V15_J, V15_K,
  794     V16, V16_H, V16_J, V16_K,
  795     V17, V17_H, V17_J, V17_K,
  796     V18, V18_H, V18_J, V18_K,
  797     V19, V19_H, V19_J, V19_K,
  798     V20, V20_H, V20_J, V20_K,
  799     V21, V21_H, V21_J, V21_K,
  800     V22, V22_H, V22_J, V22_K,
  801     V23, V23_H, V23_J, V23_K,
  802     V24, V24_H, V24_J, V24_K,
  803     V25, V25_H, V25_J, V25_K,
  804     V26, V26_H, V26_J, V26_K,
  805     V27, V27_H, V27_J, V27_K,
  806     V28, V28_H, V28_J, V28_K,
  807     V29, V29_H, V29_J, V29_K,
  808     V30, V30_H, V30_J, V30_K,
  809     V31, V31_H, V31_J, V31_K,
  810 );
  811 
  812 // Class for all 64bit vector registers
  813 reg_class vectord_reg(
  814     V0, V0_H,
  815     V1, V1_H,
  816     V2, V2_H,
  817     V3, V3_H,
  818     V4, V4_H,
  819     V5, V5_H,
  820     V6, V6_H,
  821     V7, V7_H,
  822     V8, V8_H,
  823     V9, V9_H,
  824     V10, V10_H,
  825     V11, V11_H,
  826     V12, V12_H,
  827     V13, V13_H,
  828     V14, V14_H,
  829     V15, V15_H,
  830     V16, V16_H,
  831     V17, V17_H,
  832     V18, V18_H,
  833     V19, V19_H,
  834     V20, V20_H,
  835     V21, V21_H,
  836     V22, V22_H,
  837     V23, V23_H,
  838     V24, V24_H,
  839     V25, V25_H,
  840     V26, V26_H,
  841     V27, V27_H,
  842     V28, V28_H,
  843     V29, V29_H,
  844     V30, V30_H,
  845     V31, V31_H
  846 );
  847 
  848 // Class for all 128bit vector registers
  849 reg_class vectorx_reg(
  850     V0, V0_H, V0_J, V0_K,
  851     V1, V1_H, V1_J, V1_K,
  852     V2, V2_H, V2_J, V2_K,
  853     V3, V3_H, V3_J, V3_K,
  854     V4, V4_H, V4_J, V4_K,
  855     V5, V5_H, V5_J, V5_K,
  856     V6, V6_H, V6_J, V6_K,
  857     V7, V7_H, V7_J, V7_K,
  858     V8, V8_H, V8_J, V8_K,
  859     V9, V9_H, V9_J, V9_K,
  860     V10, V10_H, V10_J, V10_K,
  861     V11, V11_H, V11_J, V11_K,
  862     V12, V12_H, V12_J, V12_K,
  863     V13, V13_H, V13_J, V13_K,
  864     V14, V14_H, V14_J, V14_K,
  865     V15, V15_H, V15_J, V15_K,
  866     V16, V16_H, V16_J, V16_K,
  867     V17, V17_H, V17_J, V17_K,
  868     V18, V18_H, V18_J, V18_K,
  869     V19, V19_H, V19_J, V19_K,
  870     V20, V20_H, V20_J, V20_K,
  871     V21, V21_H, V21_J, V21_K,
  872     V22, V22_H, V22_J, V22_K,
  873     V23, V23_H, V23_J, V23_K,
  874     V24, V24_H, V24_J, V24_K,
  875     V25, V25_H, V25_J, V25_K,
  876     V26, V26_H, V26_J, V26_K,
  877     V27, V27_H, V27_J, V27_K,
  878     V28, V28_H, V28_J, V28_K,
  879     V29, V29_H, V29_J, V29_K,
  880     V30, V30_H, V30_J, V30_K,
  881     V31, V31_H, V31_J, V31_K
  882 );
  883 
  884 // Class for 128 bit register v0
  885 reg_class v0_reg(
  886     V0, V0_H
  887 );
  888 
  889 // Class for 128 bit register v1
  890 reg_class v1_reg(
  891     V1, V1_H
  892 );
  893 
  894 // Class for 128 bit register v2
  895 reg_class v2_reg(
  896     V2, V2_H
  897 );
  898 
  899 // Class for 128 bit register v3
  900 reg_class v3_reg(
  901     V3, V3_H
  902 );
  903 
  904 // Class for 128 bit register v4
  905 reg_class v4_reg(
  906     V4, V4_H
  907 );
  908 
  909 // Class for 128 bit register v5
  910 reg_class v5_reg(
  911     V5, V5_H
  912 );
  913 
  914 // Class for 128 bit register v6
  915 reg_class v6_reg(
  916     V6, V6_H
  917 );
  918 
  919 // Class for 128 bit register v7
  920 reg_class v7_reg(
  921     V7, V7_H
  922 );
  923 
  924 // Class for 128 bit register v8
  925 reg_class v8_reg(
  926     V8, V8_H
  927 );
  928 
  929 // Class for 128 bit register v9
  930 reg_class v9_reg(
  931     V9, V9_H
  932 );
  933 
  934 // Class for 128 bit register v10
  935 reg_class v10_reg(
  936     V10, V10_H
  937 );
  938 
  939 // Class for 128 bit register v11
  940 reg_class v11_reg(
  941     V11, V11_H
  942 );
  943 
  944 // Class for 128 bit register v12
  945 reg_class v12_reg(
  946     V12, V12_H
  947 );
  948 
  949 // Class for 128 bit register v13
  950 reg_class v13_reg(
  951     V13, V13_H
  952 );
  953 
  954 // Class for 128 bit register v14
  955 reg_class v14_reg(
  956     V14, V14_H
  957 );
  958 
  959 // Class for 128 bit register v15
  960 reg_class v15_reg(
  961     V15, V15_H
  962 );
  963 
  964 // Class for 128 bit register v16
  965 reg_class v16_reg(
  966     V16, V16_H
  967 );
  968 
  969 // Class for 128 bit register v17
  970 reg_class v17_reg(
  971     V17, V17_H
  972 );
  973 
  974 // Class for 128 bit register v18
  975 reg_class v18_reg(
  976     V18, V18_H
  977 );
  978 
  979 // Class for 128 bit register v19
  980 reg_class v19_reg(
  981     V19, V19_H
  982 );
  983 
  984 // Class for 128 bit register v20
  985 reg_class v20_reg(
  986     V20, V20_H
  987 );
  988 
  989 // Class for 128 bit register v21
  990 reg_class v21_reg(
  991     V21, V21_H
  992 );
  993 
  994 // Class for 128 bit register v22
  995 reg_class v22_reg(
  996     V22, V22_H
  997 );
  998 
  999 // Class for 128 bit register v23
 1000 reg_class v23_reg(
 1001     V23, V23_H
 1002 );
 1003 
 1004 // Class for 128 bit register v24
 1005 reg_class v24_reg(
 1006     V24, V24_H
 1007 );
 1008 
 1009 // Class for 128 bit register v25
 1010 reg_class v25_reg(
 1011     V25, V25_H
 1012 );
 1013 
 1014 // Class for 128 bit register v26
 1015 reg_class v26_reg(
 1016     V26, V26_H
 1017 );
 1018 
 1019 // Class for 128 bit register v27
 1020 reg_class v27_reg(
 1021     V27, V27_H
 1022 );
 1023 
 1024 // Class for 128 bit register v28
 1025 reg_class v28_reg(
 1026     V28, V28_H
 1027 );
 1028 
 1029 // Class for 128 bit register v29
 1030 reg_class v29_reg(
 1031     V29, V29_H
 1032 );
 1033 
 1034 // Class for 128 bit register v30
 1035 reg_class v30_reg(
 1036     V30, V30_H
 1037 );
 1038 
 1039 // Class for 128 bit register v31
 1040 reg_class v31_reg(
 1041     V31, V31_H
 1042 );
 1043 
 1044 // Class for all SVE predicate registers.
 1045 reg_class pr_reg (
 1046     P0,
 1047     P1,
 1048     P2,
 1049     P3,
 1050     P4,
 1051     P5,
 1052     P6,
 1053     // P7, non-allocatable, preserved with all elements preset to TRUE.
 1054     P8,
 1055     P9,
 1056     P10,
 1057     P11,
 1058     P12,
 1059     P13,
 1060     P14,
 1061     P15
 1062 );
 1063 
 1064 // Class for SVE governing predicate registers, which are used
 1065 // to determine the active elements of a predicated instruction.
 1066 reg_class gov_pr (
 1067     P0,
 1068     P1,
 1069     P2,
 1070     P3,
 1071     P4,
 1072     P5,
 1073     P6,
 1074     // P7, non-allocatable, preserved with all elements preset to TRUE.
 1075 );
 1076 
 1077 reg_class p0_reg(P0);
 1078 reg_class p1_reg(P1);
 1079 
 1080 // Singleton class for condition codes
 1081 reg_class int_flags(RFLAGS);
 1082 
 1083 %}
 1084 
 1085 //----------DEFINITION BLOCK---------------------------------------------------
 1086 // Define name --> value mappings to inform the ADLC of an integer valued name
 1087 // Current support includes integer values in the range [0, 0x7FFFFFFF]
 1088 // Format:
 1089 //        int_def  <name>         ( <int_value>, <expression>);
 1090 // Generated Code in ad_<arch>.hpp
 1091 //        #define  <name>   (<expression>)
 1092 //        // value == <int_value>
 1093 // Generated code in ad_<arch>.cpp adlc_verification()
 1094 //        assert( <name> == <int_value>, "Expect (<expression>) to equal <int_value>");
 1095 //
 1096 
 1097 // we follow the ppc-aix port in using a simple cost model which ranks
 1098 // register operations as cheap, memory ops as more expensive and
 1099 // branches as most expensive. the first two have a low as well as a
 1100 // normal cost. huge cost appears to be a way of saying don't do
 1101 // something
 1102 
 1103 definitions %{
 1104   // The default cost (of a register move instruction).
 1105   int_def INSN_COST            (    100,     100);
 1106   int_def BRANCH_COST          (    200,     2 * INSN_COST);
 1107   int_def CALL_COST            (    200,     2 * INSN_COST);
 1108   int_def VOLATILE_REF_COST    (   1000,     10 * INSN_COST);
 1109 %}
 1110 
 1111 
 1112 //----------SOURCE BLOCK-------------------------------------------------------
 1113 // This is a block of C++ code which provides values, functions, and
 1114 // definitions necessary in the rest of the architecture description
 1115 
 1116 source_hpp %{
 1117 
 1118 #include "asm/macroAssembler.hpp"
 1119 #include "gc/shared/barrierSetAssembler.hpp"
 1120 #include "gc/shared/cardTable.hpp"
 1121 #include "gc/shared/cardTableBarrierSet.hpp"
 1122 #include "gc/shared/collectedHeap.hpp"
 1123 #include "opto/addnode.hpp"
 1124 #include "opto/convertnode.hpp"
 1125 #include "runtime/objectMonitor.hpp"
 1126 
 1127 extern RegMask _ANY_REG32_mask;
 1128 extern RegMask _ANY_REG_mask;
 1129 extern RegMask _PTR_REG_mask;
 1130 extern RegMask _NO_SPECIAL_REG32_mask;
 1131 extern RegMask _NO_SPECIAL_REG_mask;
 1132 extern RegMask _NO_SPECIAL_PTR_REG_mask;
 1133 extern RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
 1134 
 1135 class CallStubImpl {
 1136 
 1137   //--------------------------------------------------------------
 1138   //---<  Used for optimization in Compile::shorten_branches  >---
 1139   //--------------------------------------------------------------
 1140 
 1141  public:
 1142   // Size of call trampoline stub.
 1143   static uint size_call_trampoline() {
 1144     return 0; // no call trampolines on this platform
 1145   }
 1146 
 1147   // number of relocations needed by a call trampoline stub
 1148   static uint reloc_call_trampoline() {
 1149     return 0; // no call trampolines on this platform
 1150   }
 1151 };
 1152 
 1153 class HandlerImpl {
 1154 
 1155  public:
 1156 
 1157   static int emit_exception_handler(C2_MacroAssembler *masm);
 1158   static int emit_deopt_handler(C2_MacroAssembler* masm);
 1159 
 1160   static uint size_exception_handler() {
 1161     return MacroAssembler::far_codestub_branch_size();
 1162   }
 1163 
 1164   static uint size_deopt_handler() {
 1165     // count one adr and one far branch instruction
 1166     return NativeInstruction::instruction_size + MacroAssembler::far_codestub_branch_size();
 1167   }
 1168 };
 1169 
 1170 class Node::PD {
 1171 public:
 1172   enum NodeFlags {
 1173     _last_flag = Node::_last_flag
 1174   };
 1175 };
 1176 
 1177   bool is_CAS(int opcode, bool maybe_volatile);
 1178 
 1179   // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
 1180 
 1181   bool unnecessary_acquire(const Node *barrier);
 1182   bool needs_acquiring_load(const Node *load);
 1183 
 1184   // predicates controlling emit of str<x>/stlr<x> and associated dmbs
 1185 
 1186   bool unnecessary_release(const Node *barrier);
 1187   bool unnecessary_volatile(const Node *barrier);
 1188   bool needs_releasing_store(const Node *store);
 1189 
 1190   // predicate controlling translation of CompareAndSwapX
 1191   bool needs_acquiring_load_exclusive(const Node *load);
 1192 
 1193   // predicate controlling addressing modes
 1194   bool size_fits_all_mem_uses(AddPNode* addp, int shift);
 1195 
 1196   // Convert BootTest condition to Assembler condition.
 1197   // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
 1198   Assembler::Condition to_assembler_cond(BoolTest::mask cond);
 1199 %}
 1200 
 1201 source %{
 1202 
 1203   // Derived RegMask with conditionally allocatable registers
 1204 
 1205   void PhaseOutput::pd_perform_mach_node_analysis() {
 1206   }
 1207 
 1208   int MachNode::pd_alignment_required() const {
 1209     return 1;
 1210   }
 1211 
 1212   int MachNode::compute_padding(int current_offset) const {
 1213     return 0;
 1214   }
 1215 
 1216   RegMask _ANY_REG32_mask;
 1217   RegMask _ANY_REG_mask;
 1218   RegMask _PTR_REG_mask;
 1219   RegMask _NO_SPECIAL_REG32_mask;
 1220   RegMask _NO_SPECIAL_REG_mask;
 1221   RegMask _NO_SPECIAL_PTR_REG_mask;
 1222   RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
 1223 
 1224   void reg_mask_init() {
 1225     // We derive below RegMask(s) from the ones which are auto-generated from
 1226     // adlc register classes to make AArch64 rheapbase (r27) and rfp (r29)
 1227     // registers conditionally reserved.
 1228 
 1229     _ANY_REG32_mask = _ALL_REG32_mask;
 1230     _ANY_REG32_mask.Remove(OptoReg::as_OptoReg(r31_sp->as_VMReg()));
 1231 
 1232     _ANY_REG_mask = _ALL_REG_mask;
 1233 
 1234     _PTR_REG_mask = _ALL_REG_mask;
 1235 
 1236     _NO_SPECIAL_REG32_mask = _ALL_REG32_mask;
 1237     _NO_SPECIAL_REG32_mask.SUBTRACT(_NON_ALLOCATABLE_REG32_mask);
 1238 
 1239     _NO_SPECIAL_REG_mask = _ALL_REG_mask;
 1240     _NO_SPECIAL_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
 1241 
 1242     _NO_SPECIAL_PTR_REG_mask = _ALL_REG_mask;
 1243     _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
 1244 
 1245     // r27 is not allocatable when compressed oops is on and heapbase is not
 1246     // zero, compressed klass pointers doesn't use r27 after JDK-8234794
 1247     if (UseCompressedOops && (CompressedOops::ptrs_base() != nullptr)) {
 1248       _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
 1249       _NO_SPECIAL_REG_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
 1250       _NO_SPECIAL_PTR_REG_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
 1251     }
 1252 
 1253     // r29 is not allocatable when PreserveFramePointer is on
 1254     if (PreserveFramePointer) {
 1255       _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
 1256       _NO_SPECIAL_REG_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
 1257       _NO_SPECIAL_PTR_REG_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
 1258     }
 1259 
 1260     _NO_SPECIAL_NO_RFP_PTR_REG_mask = _NO_SPECIAL_PTR_REG_mask;
 1261     _NO_SPECIAL_NO_RFP_PTR_REG_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
 1262   }
 1263 
 1264   // Optimizaton of volatile gets and puts
 1265   // -------------------------------------
 1266   //
 1267   // AArch64 has ldar<x> and stlr<x> instructions which we can safely
 1268   // use to implement volatile reads and writes. For a volatile read
 1269   // we simply need
 1270   //
 1271   //   ldar<x>
 1272   //
 1273   // and for a volatile write we need
 1274   //
 1275   //   stlr<x>
 1276   //
 1277   // Alternatively, we can implement them by pairing a normal
 1278   // load/store with a memory barrier. For a volatile read we need
 1279   //
 1280   //   ldr<x>
 1281   //   dmb ishld
 1282   //
 1283   // for a volatile write
 1284   //
 1285   //   dmb ish
 1286   //   str<x>
 1287   //   dmb ish
 1288   //
 1289   // We can also use ldaxr and stlxr to implement compare and swap CAS
 1290   // sequences. These are normally translated to an instruction
 1291   // sequence like the following
 1292   //
 1293   //   dmb      ish
 1294   // retry:
 1295   //   ldxr<x>   rval raddr
 1296   //   cmp       rval rold
 1297   //   b.ne done
 1298   //   stlxr<x>  rval, rnew, rold
 1299   //   cbnz      rval retry
 1300   // done:
 1301   //   cset      r0, eq
 1302   //   dmb ishld
 1303   //
 1304   // Note that the exclusive store is already using an stlxr
 1305   // instruction. That is required to ensure visibility to other
 1306   // threads of the exclusive write (assuming it succeeds) before that
 1307   // of any subsequent writes.
 1308   //
 1309   // The following instruction sequence is an improvement on the above
 1310   //
 1311   // retry:
 1312   //   ldaxr<x>  rval raddr
 1313   //   cmp       rval rold
 1314   //   b.ne done
 1315   //   stlxr<x>  rval, rnew, rold
 1316   //   cbnz      rval retry
 1317   // done:
 1318   //   cset      r0, eq
 1319   //
 1320   // We don't need the leading dmb ish since the stlxr guarantees
 1321   // visibility of prior writes in the case that the swap is
 1322   // successful. Crucially we don't have to worry about the case where
 1323   // the swap is not successful since no valid program should be
 1324   // relying on visibility of prior changes by the attempting thread
 1325   // in the case where the CAS fails.
 1326   //
 1327   // Similarly, we don't need the trailing dmb ishld if we substitute
 1328   // an ldaxr instruction since that will provide all the guarantees we
 1329   // require regarding observation of changes made by other threads
 1330   // before any change to the CAS address observed by the load.
 1331   //
 1332   // In order to generate the desired instruction sequence we need to
 1333   // be able to identify specific 'signature' ideal graph node
 1334   // sequences which i) occur as a translation of a volatile reads or
 1335   // writes or CAS operations and ii) do not occur through any other
 1336   // translation or graph transformation. We can then provide
 1337   // alternative aldc matching rules which translate these node
 1338   // sequences to the desired machine code sequences. Selection of the
 1339   // alternative rules can be implemented by predicates which identify
 1340   // the relevant node sequences.
 1341   //
 1342   // The ideal graph generator translates a volatile read to the node
 1343   // sequence
 1344   //
 1345   //   LoadX[mo_acquire]
 1346   //   MemBarAcquire
 1347   //
 1348   // As a special case when using the compressed oops optimization we
 1349   // may also see this variant
 1350   //
 1351   //   LoadN[mo_acquire]
 1352   //   DecodeN
 1353   //   MemBarAcquire
 1354   //
 1355   // A volatile write is translated to the node sequence
 1356   //
 1357   //   MemBarRelease
 1358   //   StoreX[mo_release] {CardMark}-optional
 1359   //   MemBarVolatile
 1360   //
 1361   // n.b. the above node patterns are generated with a strict
 1362   // 'signature' configuration of input and output dependencies (see
 1363   // the predicates below for exact details). The card mark may be as
 1364   // simple as a few extra nodes or, in a few GC configurations, may
 1365   // include more complex control flow between the leading and
 1366   // trailing memory barriers. However, whatever the card mark
 1367   // configuration these signatures are unique to translated volatile
 1368   // reads/stores -- they will not appear as a result of any other
 1369   // bytecode translation or inlining nor as a consequence of
 1370   // optimizing transforms.
 1371   //
 1372   // We also want to catch inlined unsafe volatile gets and puts and
 1373   // be able to implement them using either ldar<x>/stlr<x> or some
 1374   // combination of ldr<x>/stlr<x> and dmb instructions.
 1375   //
 1376   // Inlined unsafe volatiles puts manifest as a minor variant of the
 1377   // normal volatile put node sequence containing an extra cpuorder
 1378   // membar
 1379   //
 1380   //   MemBarRelease
 1381   //   MemBarCPUOrder
 1382   //   StoreX[mo_release] {CardMark}-optional
 1383   //   MemBarCPUOrder
 1384   //   MemBarVolatile
 1385   //
 1386   // n.b. as an aside, a cpuorder membar is not itself subject to
 1387   // matching and translation by adlc rules.  However, the rule
 1388   // predicates need to detect its presence in order to correctly
 1389   // select the desired adlc rules.
 1390   //
 1391   // Inlined unsafe volatile gets manifest as a slightly different
 1392   // node sequence to a normal volatile get because of the
 1393   // introduction of some CPUOrder memory barriers to bracket the
 1394   // Load. However, but the same basic skeleton of a LoadX feeding a
 1395   // MemBarAcquire, possibly through an optional DecodeN, is still
 1396   // present
 1397   //
 1398   //   MemBarCPUOrder
 1399   //        ||       \\
 1400   //   MemBarCPUOrder LoadX[mo_acquire]
 1401   //        ||            |
 1402   //        ||       {DecodeN} optional
 1403   //        ||       /
 1404   //     MemBarAcquire
 1405   //
 1406   // In this case the acquire membar does not directly depend on the
 1407   // load. However, we can be sure that the load is generated from an
 1408   // inlined unsafe volatile get if we see it dependent on this unique
 1409   // sequence of membar nodes. Similarly, given an acquire membar we
 1410   // can know that it was added because of an inlined unsafe volatile
 1411   // get if it is fed and feeds a cpuorder membar and if its feed
 1412   // membar also feeds an acquiring load.
 1413   //
 1414   // Finally an inlined (Unsafe) CAS operation is translated to the
 1415   // following ideal graph
 1416   //
 1417   //   MemBarRelease
 1418   //   MemBarCPUOrder
 1419   //   CompareAndSwapX {CardMark}-optional
 1420   //   MemBarCPUOrder
 1421   //   MemBarAcquire
 1422   //
 1423   // So, where we can identify these volatile read and write
 1424   // signatures we can choose to plant either of the above two code
 1425   // sequences. For a volatile read we can simply plant a normal
 1426   // ldr<x> and translate the MemBarAcquire to a dmb. However, we can
 1427   // also choose to inhibit translation of the MemBarAcquire and
 1428   // inhibit planting of the ldr<x>, instead planting an ldar<x>.
 1429   //
 1430   // When we recognise a volatile store signature we can choose to
 1431   // plant at a dmb ish as a translation for the MemBarRelease, a
 1432   // normal str<x> and then a dmb ish for the MemBarVolatile.
 1433   // Alternatively, we can inhibit translation of the MemBarRelease
 1434   // and MemBarVolatile and instead plant a simple stlr<x>
 1435   // instruction.
 1436   //
 1437   // when we recognise a CAS signature we can choose to plant a dmb
 1438   // ish as a translation for the MemBarRelease, the conventional
 1439   // macro-instruction sequence for the CompareAndSwap node (which
 1440   // uses ldxr<x>) and then a dmb ishld for the MemBarAcquire.
 1441   // Alternatively, we can elide generation of the dmb instructions
 1442   // and plant the alternative CompareAndSwap macro-instruction
 1443   // sequence (which uses ldaxr<x>).
 1444   //
 1445   // Of course, the above only applies when we see these signature
 1446   // configurations. We still want to plant dmb instructions in any
 1447   // other cases where we may see a MemBarAcquire, MemBarRelease or
 1448   // MemBarVolatile. For example, at the end of a constructor which
 1449   // writes final/volatile fields we will see a MemBarRelease
 1450   // instruction and this needs a 'dmb ish' lest we risk the
 1451   // constructed object being visible without making the
 1452   // final/volatile field writes visible.
 1453   //
 1454   // n.b. the translation rules below which rely on detection of the
 1455   // volatile signatures and insert ldar<x> or stlr<x> are failsafe.
 1456   // If we see anything other than the signature configurations we
 1457   // always just translate the loads and stores to ldr<x> and str<x>
 1458   // and translate acquire, release and volatile membars to the
 1459   // relevant dmb instructions.
 1460   //
 1461 
 1462   // is_CAS(int opcode, bool maybe_volatile)
 1463   //
 1464   // return true if opcode is one of the possible CompareAndSwapX
 1465   // values otherwise false.
 1466 
 1467   bool is_CAS(int opcode, bool maybe_volatile)
 1468   {
 1469     switch(opcode) {
 1470       // We handle these
 1471     case Op_CompareAndSwapI:
 1472     case Op_CompareAndSwapL:
 1473     case Op_CompareAndSwapP:
 1474     case Op_CompareAndSwapN:
 1475     case Op_ShenandoahCompareAndSwapP:
 1476     case Op_ShenandoahCompareAndSwapN:
 1477     case Op_CompareAndSwapB:
 1478     case Op_CompareAndSwapS:
 1479     case Op_GetAndSetI:
 1480     case Op_GetAndSetL:
 1481     case Op_GetAndSetP:
 1482     case Op_GetAndSetN:
 1483     case Op_GetAndAddI:
 1484     case Op_GetAndAddL:
 1485       return true;
 1486     case Op_CompareAndExchangeI:
 1487     case Op_CompareAndExchangeN:
 1488     case Op_CompareAndExchangeB:
 1489     case Op_CompareAndExchangeS:
 1490     case Op_CompareAndExchangeL:
 1491     case Op_CompareAndExchangeP:
 1492     case Op_WeakCompareAndSwapB:
 1493     case Op_WeakCompareAndSwapS:
 1494     case Op_WeakCompareAndSwapI:
 1495     case Op_WeakCompareAndSwapL:
 1496     case Op_WeakCompareAndSwapP:
 1497     case Op_WeakCompareAndSwapN:
 1498     case Op_ShenandoahWeakCompareAndSwapP:
 1499     case Op_ShenandoahWeakCompareAndSwapN:
 1500     case Op_ShenandoahCompareAndExchangeP:
 1501     case Op_ShenandoahCompareAndExchangeN:
 1502       return maybe_volatile;
 1503     default:
 1504       return false;
 1505     }
 1506   }
 1507 
 1508   // helper to determine the maximum number of Phi nodes we may need to
 1509   // traverse when searching from a card mark membar for the merge mem
 1510   // feeding a trailing membar or vice versa
 1511 
 1512 // predicates controlling emit of ldr<x>/ldar<x>
 1513 
 1514 bool unnecessary_acquire(const Node *barrier)
 1515 {
 1516   assert(barrier->is_MemBar(), "expecting a membar");
 1517 
 1518   MemBarNode* mb = barrier->as_MemBar();
 1519 
 1520   if (mb->trailing_load()) {
 1521     return true;
 1522   }
 1523 
 1524   if (mb->trailing_load_store()) {
 1525     Node* load_store = mb->in(MemBarNode::Precedent);
 1526     assert(load_store->is_LoadStore(), "unexpected graph shape");
 1527     return is_CAS(load_store->Opcode(), true);
 1528   }
 1529 
 1530   return false;
 1531 }
 1532 
 1533 bool needs_acquiring_load(const Node *n)
 1534 {
 1535   assert(n->is_Load(), "expecting a load");
 1536   LoadNode *ld = n->as_Load();
 1537   return ld->is_acquire();
 1538 }
 1539 
 1540 bool unnecessary_release(const Node *n)
 1541 {
 1542   assert((n->is_MemBar() &&
 1543           n->Opcode() == Op_MemBarRelease),
 1544          "expecting a release membar");
 1545 
 1546   MemBarNode *barrier = n->as_MemBar();
 1547   if (!barrier->leading()) {
 1548     return false;
 1549   } else {
 1550     Node* trailing = barrier->trailing_membar();
 1551     MemBarNode* trailing_mb = trailing->as_MemBar();
 1552     assert(trailing_mb->trailing(), "Not a trailing membar?");
 1553     assert(trailing_mb->leading_membar() == n, "inconsistent leading/trailing membars");
 1554 
 1555     Node* mem = trailing_mb->in(MemBarNode::Precedent);
 1556     if (mem->is_Store()) {
 1557       assert(mem->as_Store()->is_release(), "");
 1558       assert(trailing_mb->Opcode() == Op_MemBarVolatile, "");
 1559       return true;
 1560     } else {
 1561       assert(mem->is_LoadStore(), "");
 1562       assert(trailing_mb->Opcode() == Op_MemBarAcquire, "");
 1563       return is_CAS(mem->Opcode(), true);
 1564     }
 1565   }
 1566   return false;
 1567 }
 1568 
 1569 bool unnecessary_volatile(const Node *n)
 1570 {
 1571   // assert n->is_MemBar();
 1572   MemBarNode *mbvol = n->as_MemBar();
 1573 
 1574   bool release = mbvol->trailing_store();
 1575   assert(!release || (mbvol->in(MemBarNode::Precedent)->is_Store() && mbvol->in(MemBarNode::Precedent)->as_Store()->is_release()), "");
 1576 #ifdef ASSERT
 1577   if (release) {
 1578     Node* leading = mbvol->leading_membar();
 1579     assert(leading->Opcode() == Op_MemBarRelease, "");
 1580     assert(leading->as_MemBar()->leading_store(), "");
 1581     assert(leading->as_MemBar()->trailing_membar() == mbvol, "");
 1582   }
 1583 #endif
 1584 
 1585   return release;
 1586 }
 1587 
 1588 // predicates controlling emit of str<x>/stlr<x>
 1589 
 1590 bool needs_releasing_store(const Node *n)
 1591 {
 1592   // assert n->is_Store();
 1593   StoreNode *st = n->as_Store();
 1594   return st->trailing_membar() != nullptr;
 1595 }
 1596 
 1597 // predicate controlling translation of CAS
 1598 //
 1599 // returns true if CAS needs to use an acquiring load otherwise false
 1600 
 1601 bool needs_acquiring_load_exclusive(const Node *n)
 1602 {
 1603   assert(is_CAS(n->Opcode(), true), "expecting a compare and swap");
 1604   LoadStoreNode* ldst = n->as_LoadStore();
 1605   if (is_CAS(n->Opcode(), false)) {
 1606     assert(ldst->trailing_membar() != nullptr, "expected trailing membar");
 1607   } else {
 1608     return ldst->trailing_membar() != nullptr;
 1609   }
 1610 
 1611   // so we can just return true here
 1612   return true;
 1613 }
 1614 
 1615 #define __ masm->
 1616 
 1617 // advance declarations for helper functions to convert register
 1618 // indices to register objects
 1619 
 1620 // the ad file has to provide implementations of certain methods
 1621 // expected by the generic code
 1622 //
 1623 // REQUIRED FUNCTIONALITY
 1624 
 1625 //=============================================================================
 1626 
 1627 // !!!!! Special hack to get all types of calls to specify the byte offset
 1628 //       from the start of the call to the point where the return address
 1629 //       will point.
 1630 
 1631 int MachCallStaticJavaNode::ret_addr_offset()
 1632 {
 1633   // call should be a simple bl
 1634   int off = 4;
 1635   return off;
 1636 }
 1637 
 1638 int MachCallDynamicJavaNode::ret_addr_offset()
 1639 {
 1640   return 16; // movz, movk, movk, bl
 1641 }
 1642 
 1643 int MachCallRuntimeNode::ret_addr_offset() {
 1644   // for generated stubs the call will be
 1645   //   bl(addr)
 1646   // or with far branches
 1647   //   bl(trampoline_stub)
 1648   // for real runtime callouts it will be six instructions
 1649   // see aarch64_enc_java_to_runtime
 1650   //   adr(rscratch2, retaddr)
 1651   //   lea(rscratch1, RuntimeAddress(addr)
 1652   //   stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)))
 1653   //   blr(rscratch1)
 1654   CodeBlob *cb = CodeCache::find_blob(_entry_point);
 1655   if (cb) {
 1656     return 1 * NativeInstruction::instruction_size;
 1657   } else if (_entry_point == nullptr) {
 1658     // See CallLeafNoFPIndirect
 1659     return 1 * NativeInstruction::instruction_size;
 1660   } else {
 1661     return 6 * NativeInstruction::instruction_size;
 1662   }
 1663 }
 1664 
 1665 //=============================================================================
 1666 
 1667 #ifndef PRODUCT
 1668 void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 1669   st->print("BREAKPOINT");
 1670 }
 1671 #endif
 1672 
 1673 void MachBreakpointNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
 1674   __ brk(0);
 1675 }
 1676 
 1677 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
 1678   return MachNode::size(ra_);
 1679 }
 1680 
 1681 //=============================================================================
 1682 
 1683 #ifndef PRODUCT
 1684   void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
 1685     st->print("nop \t# %d bytes pad for loops and calls", _count);
 1686   }
 1687 #endif
 1688 
 1689   void MachNopNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc*) const {
 1690     for (int i = 0; i < _count; i++) {
 1691       __ nop();
 1692     }
 1693   }
 1694 
 1695   uint MachNopNode::size(PhaseRegAlloc*) const {
 1696     return _count * NativeInstruction::instruction_size;
 1697   }
 1698 
 1699 //=============================================================================
 1700 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
 1701 
 1702 int ConstantTable::calculate_table_base_offset() const {
 1703   return 0;  // absolute addressing, no offset
 1704 }
 1705 
 1706 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
 1707 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
 1708   ShouldNotReachHere();
 1709 }
 1710 
 1711 void MachConstantBaseNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const {
 1712   // Empty encoding
 1713 }
 1714 
 1715 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
 1716   return 0;
 1717 }
 1718 
 1719 #ifndef PRODUCT
 1720 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
 1721   st->print("-- \t// MachConstantBaseNode (empty encoding)");
 1722 }
 1723 #endif
 1724 
 1725 #ifndef PRODUCT
 1726 void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 1727   Compile* C = ra_->C;
 1728 
 1729   int framesize = C->output()->frame_slots() << LogBytesPerInt;
 1730 
 1731   if (C->output()->need_stack_bang(framesize))
 1732     st->print("# stack bang size=%d\n\t", framesize);
 1733 
 1734   if (VM_Version::use_rop_protection()) {
 1735     st->print("ldr  zr, [lr]\n\t");
 1736     st->print("paciaz\n\t");
 1737   }
 1738   if (framesize < ((1 << 9) + 2 * wordSize)) {
 1739     st->print("sub  sp, sp, #%d\n\t", framesize);
 1740     st->print("stp  rfp, lr, [sp, #%d]", framesize - 2 * wordSize);
 1741     if (PreserveFramePointer) st->print("\n\tadd  rfp, sp, #%d", framesize - 2 * wordSize);
 1742   } else {
 1743     st->print("stp  lr, rfp, [sp, #%d]!\n\t", -(2 * wordSize));
 1744     if (PreserveFramePointer) st->print("mov  rfp, sp\n\t");
 1745     st->print("mov  rscratch1, #%d\n\t", framesize - 2 * wordSize);
 1746     st->print("sub  sp, sp, rscratch1");
 1747   }
 1748   if (C->stub_function() == nullptr && BarrierSet::barrier_set()->barrier_set_nmethod() != nullptr) {
 1749     st->print("\n\t");
 1750     st->print("ldr  rscratch1, [guard]\n\t");
 1751     st->print("dmb ishld\n\t");
 1752     st->print("ldr  rscratch2, [rthread, #thread_disarmed_guard_value_offset]\n\t");
 1753     st->print("cmp  rscratch1, rscratch2\n\t");
 1754     st->print("b.eq skip");
 1755     st->print("\n\t");
 1756     st->print("blr #nmethod_entry_barrier_stub\n\t");
 1757     st->print("b skip\n\t");
 1758     st->print("guard: int\n\t");
 1759     st->print("\n\t");
 1760     st->print("skip:\n\t");
 1761   }
 1762 }
 1763 #endif
 1764 
 1765 void MachPrologNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
 1766   Compile* C = ra_->C;
 1767 
 1768   // insert a nop at the start of the prolog so we can patch in a
 1769   // branch if we need to invalidate the method later
 1770   __ nop();
 1771 
 1772   __ verified_entry(C, 0);
 1773 
 1774   if (C->stub_function() == nullptr) {
 1775     __ entry_barrier();
 1776   }
 1777 
 1778   if (!Compile::current()->output()->in_scratch_emit_size()) {
 1779     __ bind(*_verified_entry);
 1780   }
 1781 
 1782   if (VerifyStackAtCalls) {
 1783     Unimplemented();
 1784   }
 1785 
 1786   C->output()->set_frame_complete(__ offset());
 1787 
 1788   if (C->has_mach_constant_base_node()) {
 1789     // NOTE: We set the table base offset here because users might be
 1790     // emitted before MachConstantBaseNode.
 1791     ConstantTable& constant_table = C->output()->constant_table();
 1792     constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
 1793   }
 1794 }
 1795 
 1796 int MachPrologNode::reloc() const
 1797 {
 1798   return 0;
 1799 }
 1800 
 1801 //=============================================================================
 1802 
 1803 #ifndef PRODUCT
 1804 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 1805   Compile* C = ra_->C;
 1806   int framesize = C->output()->frame_slots() << LogBytesPerInt;
 1807 
 1808   st->print("# pop frame %d\n\t",framesize);
 1809 
 1810   if (framesize == 0) {
 1811     st->print("ldp  lr, rfp, [sp],#%d\n\t", (2 * wordSize));
 1812   } else if (framesize < ((1 << 9) + 2 * wordSize)) {
 1813     st->print("ldp  lr, rfp, [sp,#%d]\n\t", framesize - 2 * wordSize);
 1814     st->print("add  sp, sp, #%d\n\t", framesize);
 1815   } else {
 1816     st->print("mov  rscratch1, #%d\n\t", framesize - 2 * wordSize);
 1817     st->print("add  sp, sp, rscratch1\n\t");
 1818     st->print("ldp  lr, rfp, [sp],#%d\n\t", (2 * wordSize));
 1819   }
 1820   if (VM_Version::use_rop_protection()) {
 1821     st->print("autiaz\n\t");
 1822     st->print("ldr  zr, [lr]\n\t");
 1823   }
 1824 
 1825   if (do_polling() && C->is_method_compilation()) {
 1826     st->print("# test polling word\n\t");
 1827     st->print("ldr  rscratch1, [rthread],#%d\n\t", in_bytes(JavaThread::polling_word_offset()));
 1828     st->print("cmp  sp, rscratch1\n\t");
 1829     st->print("bhi #slow_path");
 1830   }
 1831 }
 1832 #endif
 1833 
 1834 void MachEpilogNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
 1835   Compile* C = ra_->C;
 1836   int framesize = C->output()->frame_slots() << LogBytesPerInt;
 1837 
 1838   __ remove_frame(framesize, C->needs_stack_repair());
 1839 
 1840   if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
 1841     __ reserved_stack_check();
 1842   }
 1843 
 1844   if (do_polling() && C->is_method_compilation()) {
 1845     Label dummy_label;
 1846     Label* code_stub = &dummy_label;
 1847     if (!C->output()->in_scratch_emit_size()) {
 1848       C2SafepointPollStub* stub = new (C->comp_arena()) C2SafepointPollStub(__ offset());
 1849       C->output()->add_stub(stub);
 1850       code_stub = &stub->entry();
 1851     }
 1852     __ relocate(relocInfo::poll_return_type);
 1853     __ safepoint_poll(*code_stub, true /* at_return */, false /* acquire */, true /* in_nmethod */);
 1854   }
 1855 }
 1856 
 1857 int MachEpilogNode::reloc() const {
 1858   // Return number of relocatable values contained in this instruction.
 1859   return 1; // 1 for polling page.
 1860 }
 1861 
 1862 const Pipeline * MachEpilogNode::pipeline() const {
 1863   return MachNode::pipeline_class();
 1864 }
 1865 
 1866 //=============================================================================
 1867 
 1868 static enum RC rc_class(OptoReg::Name reg) {
 1869 
 1870   if (reg == OptoReg::Bad) {
 1871     return rc_bad;
 1872   }
 1873 
 1874   // we have 32 int registers * 2 halves
 1875   int slots_of_int_registers = Register::number_of_registers * Register::max_slots_per_register;
 1876 
 1877   if (reg < slots_of_int_registers) {
 1878     return rc_int;
 1879   }
 1880 
 1881   // we have 32 float register * 8 halves
 1882   int slots_of_float_registers = FloatRegister::number_of_registers * FloatRegister::max_slots_per_register;
 1883   if (reg < slots_of_int_registers + slots_of_float_registers) {
 1884     return rc_float;
 1885   }
 1886 
 1887   int slots_of_predicate_registers = PRegister::number_of_registers * PRegister::max_slots_per_register;
 1888   if (reg < slots_of_int_registers + slots_of_float_registers + slots_of_predicate_registers) {
 1889     return rc_predicate;
 1890   }
 1891 
 1892   // Between predicate regs & stack is the flags.
 1893   assert(OptoReg::is_stack(reg), "blow up if spilling flags");
 1894 
 1895   return rc_stack;
 1896 }
 1897 
 1898 uint MachSpillCopyNode::implementation(C2_MacroAssembler *masm, PhaseRegAlloc *ra_, bool do_size, outputStream *st) const {
 1899   Compile* C = ra_->C;
 1900 
 1901   // Get registers to move.
 1902   OptoReg::Name src_hi = ra_->get_reg_second(in(1));
 1903   OptoReg::Name src_lo = ra_->get_reg_first(in(1));
 1904   OptoReg::Name dst_hi = ra_->get_reg_second(this);
 1905   OptoReg::Name dst_lo = ra_->get_reg_first(this);
 1906 
 1907   enum RC src_hi_rc = rc_class(src_hi);
 1908   enum RC src_lo_rc = rc_class(src_lo);
 1909   enum RC dst_hi_rc = rc_class(dst_hi);
 1910   enum RC dst_lo_rc = rc_class(dst_lo);
 1911 
 1912   assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
 1913 
 1914   if (src_hi != OptoReg::Bad && !bottom_type()->isa_vectmask()) {
 1915     assert((src_lo&1)==0 && src_lo+1==src_hi &&
 1916            (dst_lo&1)==0 && dst_lo+1==dst_hi,
 1917            "expected aligned-adjacent pairs");
 1918   }
 1919 
 1920   if (src_lo == dst_lo && src_hi == dst_hi) {
 1921     return 0;            // Self copy, no move.
 1922   }
 1923 
 1924   bool is64 = (src_lo & 1) == 0 && src_lo + 1 == src_hi &&
 1925               (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi;
 1926   int src_offset = ra_->reg2offset(src_lo);
 1927   int dst_offset = ra_->reg2offset(dst_lo);
 1928 
 1929   if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
 1930     uint ireg = ideal_reg();
 1931     if (ireg == Op_VecA && masm) {
 1932       int sve_vector_reg_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
 1933       if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
 1934         // stack->stack
 1935         __ spill_copy_sve_vector_stack_to_stack(src_offset, dst_offset,
 1936                                                 sve_vector_reg_size_in_bytes);
 1937       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
 1938         __ spill_sve_vector(as_FloatRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
 1939                             sve_vector_reg_size_in_bytes);
 1940       } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
 1941         __ unspill_sve_vector(as_FloatRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
 1942                               sve_vector_reg_size_in_bytes);
 1943       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
 1944         __ sve_orr(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 1945                    as_FloatRegister(Matcher::_regEncode[src_lo]),
 1946                    as_FloatRegister(Matcher::_regEncode[src_lo]));
 1947       } else {
 1948         ShouldNotReachHere();
 1949       }
 1950     } else if (masm) {
 1951       assert(ireg == Op_VecD || ireg == Op_VecX, "must be 64 bit or 128 bit vector");
 1952       assert((src_lo_rc != rc_int && dst_lo_rc != rc_int), "sanity");
 1953       if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
 1954         // stack->stack
 1955         assert((src_offset & 7) == 0 && (dst_offset & 7) == 0, "unaligned stack offset");
 1956         if (ireg == Op_VecD) {
 1957           __ unspill(rscratch1, true, src_offset);
 1958           __ spill(rscratch1, true, dst_offset);
 1959         } else {
 1960           __ spill_copy128(src_offset, dst_offset);
 1961         }
 1962       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
 1963         __ mov(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 1964                ireg == Op_VecD ? __ T8B : __ T16B,
 1965                as_FloatRegister(Matcher::_regEncode[src_lo]));
 1966       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
 1967         __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
 1968                  ireg == Op_VecD ? __ D : __ Q,
 1969                  ra_->reg2offset(dst_lo));
 1970       } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
 1971         __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 1972                    ireg == Op_VecD ? __ D : __ Q,
 1973                    ra_->reg2offset(src_lo));
 1974       } else {
 1975         ShouldNotReachHere();
 1976       }
 1977     }
 1978   } else if (masm) {
 1979     switch (src_lo_rc) {
 1980     case rc_int:
 1981       if (dst_lo_rc == rc_int) {  // gpr --> gpr copy
 1982         if (is64) {
 1983             __ mov(as_Register(Matcher::_regEncode[dst_lo]),
 1984                    as_Register(Matcher::_regEncode[src_lo]));
 1985         } else {
 1986             __ movw(as_Register(Matcher::_regEncode[dst_lo]),
 1987                     as_Register(Matcher::_regEncode[src_lo]));
 1988         }
 1989       } else if (dst_lo_rc == rc_float) { // gpr --> fpr copy
 1990         if (is64) {
 1991             __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 1992                      as_Register(Matcher::_regEncode[src_lo]));
 1993         } else {
 1994             __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 1995                      as_Register(Matcher::_regEncode[src_lo]));
 1996         }
 1997       } else {                    // gpr --> stack spill
 1998         assert(dst_lo_rc == rc_stack, "spill to bad register class");
 1999         __ spill(as_Register(Matcher::_regEncode[src_lo]), is64, dst_offset);
 2000       }
 2001       break;
 2002     case rc_float:
 2003       if (dst_lo_rc == rc_int) {  // fpr --> gpr copy
 2004         if (is64) {
 2005             __ fmovd(as_Register(Matcher::_regEncode[dst_lo]),
 2006                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2007         } else {
 2008             __ fmovs(as_Register(Matcher::_regEncode[dst_lo]),
 2009                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2010         }
 2011       } else if (dst_lo_rc == rc_float) { // fpr --> fpr copy
 2012         if (is64) {
 2013             __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2014                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2015         } else {
 2016             __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2017                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2018         }
 2019       } else {                    // fpr --> stack spill
 2020         assert(dst_lo_rc == rc_stack, "spill to bad register class");
 2021         __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
 2022                  is64 ? __ D : __ S, dst_offset);
 2023       }
 2024       break;
 2025     case rc_stack:
 2026       if (dst_lo_rc == rc_int) {  // stack --> gpr load
 2027         __ unspill(as_Register(Matcher::_regEncode[dst_lo]), is64, src_offset);
 2028       } else if (dst_lo_rc == rc_float) { // stack --> fpr load
 2029         __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2030                    is64 ? __ D : __ S, src_offset);
 2031       } else if (dst_lo_rc == rc_predicate) {
 2032         __ unspill_sve_predicate(as_PRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
 2033                                  Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
 2034       } else {                    // stack --> stack copy
 2035         assert(dst_lo_rc == rc_stack, "spill to bad register class");
 2036         if (ideal_reg() == Op_RegVectMask) {
 2037           __ spill_copy_sve_predicate_stack_to_stack(src_offset, dst_offset,
 2038                                                      Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
 2039         } else {
 2040           __ unspill(rscratch1, is64, src_offset);
 2041           __ spill(rscratch1, is64, dst_offset);
 2042         }
 2043       }
 2044       break;
 2045     case rc_predicate:
 2046       if (dst_lo_rc == rc_predicate) {
 2047         __ sve_mov(as_PRegister(Matcher::_regEncode[dst_lo]), as_PRegister(Matcher::_regEncode[src_lo]));
 2048       } else if (dst_lo_rc == rc_stack) {
 2049         __ spill_sve_predicate(as_PRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
 2050                                Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
 2051       } else {
 2052         assert(false, "bad src and dst rc_class combination.");
 2053         ShouldNotReachHere();
 2054       }
 2055       break;
 2056     default:
 2057       assert(false, "bad rc_class for spill");
 2058       ShouldNotReachHere();
 2059     }
 2060   }
 2061 
 2062   if (st) {
 2063     st->print("spill ");
 2064     if (src_lo_rc == rc_stack) {
 2065       st->print("[sp, #%d] -> ", ra_->reg2offset(src_lo));
 2066     } else {
 2067       st->print("%s -> ", Matcher::regName[src_lo]);
 2068     }
 2069     if (dst_lo_rc == rc_stack) {
 2070       st->print("[sp, #%d]", ra_->reg2offset(dst_lo));
 2071     } else {
 2072       st->print("%s", Matcher::regName[dst_lo]);
 2073     }
 2074     if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
 2075       int vsize = 0;
 2076       switch (ideal_reg()) {
 2077       case Op_VecD:
 2078         vsize = 64;
 2079         break;
 2080       case Op_VecX:
 2081         vsize = 128;
 2082         break;
 2083       case Op_VecA:
 2084         vsize = Matcher::scalable_vector_reg_size(T_BYTE) * 8;
 2085         break;
 2086       default:
 2087         assert(false, "bad register type for spill");
 2088         ShouldNotReachHere();
 2089       }
 2090       st->print("\t# vector spill size = %d", vsize);
 2091     } else if (ideal_reg() == Op_RegVectMask) {
 2092       assert(Matcher::supports_scalable_vector(), "bad register type for spill");
 2093       int vsize = Matcher::scalable_predicate_reg_slots() * 32;
 2094       st->print("\t# predicate spill size = %d", vsize);
 2095     } else {
 2096       st->print("\t# spill size = %d", is64 ? 64 : 32);
 2097     }
 2098   }
 2099 
 2100   return 0;
 2101 
 2102 }
 2103 
 2104 #ifndef PRODUCT
 2105 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 2106   if (!ra_)
 2107     st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
 2108   else
 2109     implementation(nullptr, ra_, false, st);
 2110 }
 2111 #endif
 2112 
 2113 void MachSpillCopyNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
 2114   implementation(masm, ra_, false, nullptr);
 2115 }
 2116 
 2117 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
 2118   return MachNode::size(ra_);
 2119 }
 2120 
 2121 //=============================================================================
 2122 
 2123 #ifndef PRODUCT
 2124 void BoxLockNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 2125   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 2126   int reg = ra_->get_reg_first(this);
 2127   st->print("add %s, rsp, #%d]\t# box lock",
 2128             Matcher::regName[reg], offset);
 2129 }
 2130 #endif
 2131 
 2132 void BoxLockNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
 2133   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 2134   int reg    = ra_->get_encode(this);
 2135 
 2136   // This add will handle any 24-bit signed offset. 24 bits allows an
 2137   // 8 megabyte stack frame.
 2138   __ add(as_Register(reg), sp, offset);
 2139 }
 2140 
 2141 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
 2142   // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_).
 2143   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 2144 
 2145   if (Assembler::operand_valid_for_add_sub_immediate(offset)) {
 2146     return NativeInstruction::instruction_size;
 2147   } else {
 2148     return 2 * NativeInstruction::instruction_size;
 2149   }
 2150 }
 2151 
 2152 ///=============================================================================
 2153 #ifndef PRODUCT
 2154 void MachVEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 2155 {
 2156   st->print_cr("# MachVEPNode");
 2157   if (!_verified) {
 2158     st->print_cr("\t load_class");
 2159   } else {
 2160     st->print_cr("\t unpack_inline_arg");
 2161   }
 2162 }
 2163 #endif
 2164 
 2165 void MachVEPNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc* ra_) const
 2166 {
 2167   if (!_verified) {
 2168     __ ic_check(1);
 2169   } else {
 2170     // insert a nop at the start of the prolog so we can patch in a
 2171     // branch if we need to invalidate the method later
 2172     __ nop();
 2173 
 2174     // TODO 8284443 Avoid creation of temporary frame
 2175     if (ra_->C->stub_function() == nullptr) {
 2176       __ verified_entry(ra_->C, 0);
 2177       __ entry_barrier();
 2178       int framesize = ra_->C->output()->frame_slots() << LogBytesPerInt;
 2179       __ remove_frame(framesize, false);
 2180     }
 2181     // Unpack inline type args passed as oop and then jump to
 2182     // the verified entry point (skipping the unverified entry).
 2183     int sp_inc = __ unpack_inline_args(ra_->C, _receiver_only);
 2184     // Emit code for verified entry and save increment for stack repair on return
 2185     __ verified_entry(ra_->C, sp_inc);
 2186     if (Compile::current()->output()->in_scratch_emit_size()) {
 2187       Label dummy_verified_entry;
 2188       __ b(dummy_verified_entry);
 2189     } else {
 2190       __ b(*_verified_entry);
 2191     }
 2192   }
 2193 }
 2194 
 2195 //=============================================================================
 2196 #ifndef PRODUCT
 2197 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 2198 {
 2199   st->print_cr("# MachUEPNode");
 2200   if (UseCompressedClassPointers) {
 2201     st->print_cr("\tldrw rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
 2202     st->print_cr("\tldrw r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
 2203     st->print_cr("\tcmpw rscratch1, r10");
 2204   } else {
 2205     st->print_cr("\tldr rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
 2206     st->print_cr("\tldr r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
 2207     st->print_cr("\tcmp rscratch1, r10");
 2208   }
 2209   st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
 2210 }
 2211 #endif
 2212 
 2213 void MachUEPNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
 2214 {
 2215   __ ic_check(InteriorEntryAlignment);
 2216 }
 2217 
 2218 // REQUIRED EMIT CODE
 2219 
 2220 //=============================================================================
 2221 
 2222 // Emit exception handler code.
 2223 int HandlerImpl::emit_exception_handler(C2_MacroAssembler* masm)
 2224 {
 2225   // mov rscratch1 #exception_blob_entry_point
 2226   // br rscratch1
 2227   // Note that the code buffer's insts_mark is always relative to insts.
 2228   // That's why we must use the macroassembler to generate a handler.
 2229   address base = __ start_a_stub(size_exception_handler());
 2230   if (base == nullptr) {
 2231     ciEnv::current()->record_failure("CodeCache is full");
 2232     return 0;  // CodeBuffer::expand failed
 2233   }
 2234   int offset = __ offset();
 2235   __ far_jump(RuntimeAddress(OptoRuntime::exception_blob()->entry_point()));
 2236   assert(__ offset() - offset <= (int) size_exception_handler(), "overflow");
 2237   __ end_a_stub();
 2238   return offset;
 2239 }
 2240 
 2241 // Emit deopt handler code.
 2242 int HandlerImpl::emit_deopt_handler(C2_MacroAssembler* masm)
 2243 {
 2244   // Note that the code buffer's insts_mark is always relative to insts.
 2245   // That's why we must use the macroassembler to generate a handler.
 2246   address base = __ start_a_stub(size_deopt_handler());
 2247   if (base == nullptr) {
 2248     ciEnv::current()->record_failure("CodeCache is full");
 2249     return 0;  // CodeBuffer::expand failed
 2250   }
 2251   int offset = __ offset();
 2252 
 2253   __ adr(lr, __ pc());
 2254   __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
 2255 
 2256   assert(__ offset() - offset == (int) size_deopt_handler(), "overflow");
 2257   __ end_a_stub();
 2258   return offset;
 2259 }
 2260 
 2261 // REQUIRED MATCHER CODE
 2262 
 2263 //=============================================================================
 2264 
 2265 bool Matcher::match_rule_supported(int opcode) {
 2266   if (!has_match_rule(opcode))
 2267     return false;
 2268 
 2269   switch (opcode) {
 2270     case Op_OnSpinWait:
 2271       return VM_Version::supports_on_spin_wait();
 2272     case Op_CacheWB:
 2273     case Op_CacheWBPreSync:
 2274     case Op_CacheWBPostSync:
 2275       if (!VM_Version::supports_data_cache_line_flush()) {
 2276         return false;
 2277       }
 2278       break;
 2279     case Op_ExpandBits:
 2280     case Op_CompressBits:
 2281       if (!VM_Version::supports_svebitperm()) {
 2282         return false;
 2283       }
 2284       break;
 2285     case Op_FmaF:
 2286     case Op_FmaD:
 2287     case Op_FmaVF:
 2288     case Op_FmaVD:
 2289       if (!UseFMA) {
 2290         return false;
 2291       }
 2292       break;
 2293   }
 2294 
 2295   return true; // Per default match rules are supported.
 2296 }
 2297 
 2298 const RegMask* Matcher::predicate_reg_mask(void) {
 2299   return &_PR_REG_mask;
 2300 }
 2301 
 2302 const TypeVectMask* Matcher::predicate_reg_type(const Type* elemTy, int length) {
 2303   return new TypeVectMask(elemTy, length);
 2304 }
 2305 
 2306 // Vector calling convention not yet implemented.
 2307 bool Matcher::supports_vector_calling_convention(void) {
 2308   return false;
 2309 }
 2310 
 2311 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
 2312   Unimplemented();
 2313   return OptoRegPair(0, 0);
 2314 }
 2315 
 2316 // Is this branch offset short enough that a short branch can be used?
 2317 //
 2318 // NOTE: If the platform does not provide any short branch variants, then
 2319 //       this method should return false for offset 0.
 2320 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
 2321   // The passed offset is relative to address of the branch.
 2322 
 2323   return (-32768 <= offset && offset < 32768);
 2324 }
 2325 
 2326 // Vector width in bytes.
 2327 int Matcher::vector_width_in_bytes(BasicType bt) {
 2328   // The MaxVectorSize should have been set by detecting SVE max vector register size.
 2329   int size = MIN2((UseSVE > 0) ? (int)FloatRegister::sve_vl_max : (int)FloatRegister::neon_vl, (int)MaxVectorSize);
 2330   // Minimum 2 values in vector
 2331   if (size < 2*type2aelembytes(bt)) size = 0;
 2332   // But never < 4
 2333   if (size < 4) size = 0;
 2334   return size;
 2335 }
 2336 
 2337 // Limits on vector size (number of elements) loaded into vector.
 2338 int Matcher::max_vector_size(const BasicType bt) {
 2339   return vector_width_in_bytes(bt)/type2aelembytes(bt);
 2340 }
 2341 
 2342 int Matcher::min_vector_size(const BasicType bt) {
 2343   int max_size = max_vector_size(bt);
 2344   // Limit the min vector size to 8 bytes.
 2345   int size = 8 / type2aelembytes(bt);
 2346   if (bt == T_BYTE) {
 2347     // To support vector api shuffle/rearrange.
 2348     size = 4;
 2349   } else if (bt == T_BOOLEAN) {
 2350     // To support vector api load/store mask.
 2351     size = 2;
 2352   }
 2353   if (size < 2) size = 2;
 2354   return MIN2(size, max_size);
 2355 }
 2356 
 2357 int Matcher::max_vector_size_auto_vectorization(const BasicType bt) {
 2358   return Matcher::max_vector_size(bt);
 2359 }
 2360 
 2361 // Actual max scalable vector register length.
 2362 int Matcher::scalable_vector_reg_size(const BasicType bt) {
 2363   return Matcher::max_vector_size(bt);
 2364 }
 2365 
 2366 // Vector ideal reg.
 2367 uint Matcher::vector_ideal_reg(int len) {
 2368   if (UseSVE > 0 && FloatRegister::neon_vl < len && len <= FloatRegister::sve_vl_max) {
 2369     return Op_VecA;
 2370   }
 2371   switch(len) {
 2372     // For 16-bit/32-bit mask vector, reuse VecD.
 2373     case  2:
 2374     case  4:
 2375     case  8: return Op_VecD;
 2376     case 16: return Op_VecX;
 2377   }
 2378   ShouldNotReachHere();
 2379   return 0;
 2380 }
 2381 
 2382 MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* generic_opnd, uint ideal_reg, bool is_temp) {
 2383   assert(Matcher::is_generic_vector(generic_opnd), "not generic");
 2384   switch (ideal_reg) {
 2385     case Op_VecA: return new vecAOper();
 2386     case Op_VecD: return new vecDOper();
 2387     case Op_VecX: return new vecXOper();
 2388   }
 2389   ShouldNotReachHere();
 2390   return nullptr;
 2391 }
 2392 
 2393 bool Matcher::is_reg2reg_move(MachNode* m) {
 2394   return false;
 2395 }
 2396 
 2397 bool Matcher::is_generic_vector(MachOper* opnd)  {
 2398   return opnd->opcode() == VREG;
 2399 }
 2400 
 2401 // Return whether or not this register is ever used as an argument.
 2402 // This function is used on startup to build the trampoline stubs in
 2403 // generateOptoStub.  Registers not mentioned will be killed by the VM
 2404 // call in the trampoline, and arguments in those registers not be
 2405 // available to the callee.
 2406 bool Matcher::can_be_java_arg(int reg)
 2407 {
 2408   return
 2409     reg ==  R0_num || reg == R0_H_num ||
 2410     reg ==  R1_num || reg == R1_H_num ||
 2411     reg ==  R2_num || reg == R2_H_num ||
 2412     reg ==  R3_num || reg == R3_H_num ||
 2413     reg ==  R4_num || reg == R4_H_num ||
 2414     reg ==  R5_num || reg == R5_H_num ||
 2415     reg ==  R6_num || reg == R6_H_num ||
 2416     reg ==  R7_num || reg == R7_H_num ||
 2417     reg ==  V0_num || reg == V0_H_num ||
 2418     reg ==  V1_num || reg == V1_H_num ||
 2419     reg ==  V2_num || reg == V2_H_num ||
 2420     reg ==  V3_num || reg == V3_H_num ||
 2421     reg ==  V4_num || reg == V4_H_num ||
 2422     reg ==  V5_num || reg == V5_H_num ||
 2423     reg ==  V6_num || reg == V6_H_num ||
 2424     reg ==  V7_num || reg == V7_H_num;
 2425 }
 2426 
 2427 bool Matcher::is_spillable_arg(int reg)
 2428 {
 2429   return can_be_java_arg(reg);
 2430 }
 2431 
 2432 uint Matcher::int_pressure_limit()
 2433 {
 2434   // JDK-8183543: When taking the number of available registers as int
 2435   // register pressure threshold, the jtreg test:
 2436   // test/hotspot/jtreg/compiler/regalloc/TestC2IntPressure.java
 2437   // failed due to C2 compilation failure with
 2438   // "COMPILE SKIPPED: failed spill-split-recycle sanity check".
 2439   //
 2440   // A derived pointer is live at CallNode and then is flagged by RA
 2441   // as a spilled LRG. Spilling heuristics(Spill-USE) explicitly skip
 2442   // derived pointers and lastly fail to spill after reaching maximum
 2443   // number of iterations. Lowering the default pressure threshold to
 2444   // (_NO_SPECIAL_REG32_mask.Size() minus 1) forces CallNode to become
 2445   // a high register pressure area of the code so that split_DEF can
 2446   // generate DefinitionSpillCopy for the derived pointer.
 2447   uint default_int_pressure_threshold = _NO_SPECIAL_REG32_mask.Size() - 1;
 2448   if (!PreserveFramePointer) {
 2449     // When PreserveFramePointer is off, frame pointer is allocatable,
 2450     // but different from other SOC registers, it is excluded from
 2451     // fatproj's mask because its save type is No-Save. Decrease 1 to
 2452     // ensure high pressure at fatproj when PreserveFramePointer is off.
 2453     // See check_pressure_at_fatproj().
 2454     default_int_pressure_threshold--;
 2455   }
 2456   return (INTPRESSURE == -1) ? default_int_pressure_threshold : INTPRESSURE;
 2457 }
 2458 
 2459 uint Matcher::float_pressure_limit()
 2460 {
 2461   // _FLOAT_REG_mask is generated by adlc from the float_reg register class.
 2462   return (FLOATPRESSURE == -1) ? _FLOAT_REG_mask.Size() : FLOATPRESSURE;
 2463 }
 2464 
 2465 bool Matcher::use_asm_for_ldiv_by_con(jlong divisor) {
 2466   return false;
 2467 }
 2468 
 2469 RegMask Matcher::divI_proj_mask() {
 2470   ShouldNotReachHere();
 2471   return RegMask();
 2472 }
 2473 
 2474 // Register for MODI projection of divmodI.
 2475 RegMask Matcher::modI_proj_mask() {
 2476   ShouldNotReachHere();
 2477   return RegMask();
 2478 }
 2479 
 2480 // Register for DIVL projection of divmodL.
 2481 RegMask Matcher::divL_proj_mask() {
 2482   ShouldNotReachHere();
 2483   return RegMask();
 2484 }
 2485 
 2486 // Register for MODL projection of divmodL.
 2487 RegMask Matcher::modL_proj_mask() {
 2488   ShouldNotReachHere();
 2489   return RegMask();
 2490 }
 2491 
 2492 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
 2493   return FP_REG_mask();
 2494 }
 2495 
 2496 bool size_fits_all_mem_uses(AddPNode* addp, int shift) {
 2497   for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) {
 2498     Node* u = addp->fast_out(i);
 2499     if (u->is_LoadStore()) {
 2500       // On AArch64, LoadStoreNodes (i.e. compare and swap
 2501       // instructions) only take register indirect as an operand, so
 2502       // any attempt to use an AddPNode as an input to a LoadStoreNode
 2503       // must fail.
 2504       return false;
 2505     }
 2506     if (u->is_Mem()) {
 2507       int opsize = u->as_Mem()->memory_size();
 2508       assert(opsize > 0, "unexpected memory operand size");
 2509       if (u->as_Mem()->memory_size() != (1<<shift)) {
 2510         return false;
 2511       }
 2512     }
 2513   }
 2514   return true;
 2515 }
 2516 
 2517 // Convert BootTest condition to Assembler condition.
 2518 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
 2519 Assembler::Condition to_assembler_cond(BoolTest::mask cond) {
 2520   Assembler::Condition result;
 2521   switch(cond) {
 2522     case BoolTest::eq:
 2523       result = Assembler::EQ; break;
 2524     case BoolTest::ne:
 2525       result = Assembler::NE; break;
 2526     case BoolTest::le:
 2527       result = Assembler::LE; break;
 2528     case BoolTest::ge:
 2529       result = Assembler::GE; break;
 2530     case BoolTest::lt:
 2531       result = Assembler::LT; break;
 2532     case BoolTest::gt:
 2533       result = Assembler::GT; break;
 2534     case BoolTest::ule:
 2535       result = Assembler::LS; break;
 2536     case BoolTest::uge:
 2537       result = Assembler::HS; break;
 2538     case BoolTest::ult:
 2539       result = Assembler::LO; break;
 2540     case BoolTest::ugt:
 2541       result = Assembler::HI; break;
 2542     case BoolTest::overflow:
 2543       result = Assembler::VS; break;
 2544     case BoolTest::no_overflow:
 2545       result = Assembler::VC; break;
 2546     default:
 2547       ShouldNotReachHere();
 2548       return Assembler::Condition(-1);
 2549   }
 2550 
 2551   // Check conversion
 2552   if (cond & BoolTest::unsigned_compare) {
 2553     assert(cmpOpUOper((BoolTest::mask)((int)cond & ~(BoolTest::unsigned_compare))).ccode() == result, "Invalid conversion");
 2554   } else {
 2555     assert(cmpOpOper(cond).ccode() == result, "Invalid conversion");
 2556   }
 2557 
 2558   return result;
 2559 }
 2560 
 2561 // Binary src (Replicate con)
 2562 static bool is_valid_sve_arith_imm_pattern(Node* n, Node* m) {
 2563   if (n == nullptr || m == nullptr) {
 2564     return false;
 2565   }
 2566 
 2567   if (UseSVE == 0 || m->Opcode() != Op_Replicate) {
 2568     return false;
 2569   }
 2570 
 2571   Node* imm_node = m->in(1);
 2572   if (!imm_node->is_Con()) {
 2573     return false;
 2574   }
 2575 
 2576   const Type* t = imm_node->bottom_type();
 2577   if (!(t->isa_int() || t->isa_long())) {
 2578     return false;
 2579   }
 2580 
 2581   switch (n->Opcode()) {
 2582   case Op_AndV:
 2583   case Op_OrV:
 2584   case Op_XorV: {
 2585     Assembler::SIMD_RegVariant T = Assembler::elemType_to_regVariant(Matcher::vector_element_basic_type(n));
 2586     uint64_t value = t->isa_long() ? (uint64_t)imm_node->get_long() : (uint64_t)imm_node->get_int();
 2587     return Assembler::operand_valid_for_sve_logical_immediate(Assembler::regVariant_to_elemBits(T), value);
 2588   }
 2589   case Op_AddVB:
 2590     return (imm_node->get_int() <= 255 && imm_node->get_int() >= -255);
 2591   case Op_AddVS:
 2592   case Op_AddVI:
 2593     return Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)imm_node->get_int());
 2594   case Op_AddVL:
 2595     return Assembler::operand_valid_for_sve_add_sub_immediate(imm_node->get_long());
 2596   default:
 2597     return false;
 2598   }
 2599 }
 2600 
 2601 // (XorV src (Replicate m1))
 2602 // (XorVMask src (MaskAll m1))
 2603 static bool is_vector_bitwise_not_pattern(Node* n, Node* m) {
 2604   if (n != nullptr && m != nullptr) {
 2605     return (n->Opcode() == Op_XorV || n->Opcode() == Op_XorVMask) &&
 2606            VectorNode::is_all_ones_vector(m);
 2607   }
 2608   return false;
 2609 }
 2610 
 2611 // Should the matcher clone input 'm' of node 'n'?
 2612 bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
 2613   if (is_vshift_con_pattern(n, m) ||
 2614       is_vector_bitwise_not_pattern(n, m) ||
 2615       is_valid_sve_arith_imm_pattern(n, m)) {
 2616     mstack.push(m, Visit);
 2617     return true;
 2618   }
 2619   return false;
 2620 }
 2621 
 2622 // Should the Matcher clone shifts on addressing modes, expecting them
 2623 // to be subsumed into complex addressing expressions or compute them
 2624 // into registers?
 2625 bool Matcher::pd_clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
 2626   if (clone_base_plus_offset_address(m, mstack, address_visited)) {
 2627     return true;
 2628   }
 2629 
 2630   Node *off = m->in(AddPNode::Offset);
 2631   if (off->Opcode() == Op_LShiftL && off->in(2)->is_Con() &&
 2632       size_fits_all_mem_uses(m, off->in(2)->get_int()) &&
 2633       // Are there other uses besides address expressions?
 2634       !is_visited(off)) {
 2635     address_visited.set(off->_idx); // Flag as address_visited
 2636     mstack.push(off->in(2), Visit);
 2637     Node *conv = off->in(1);
 2638     if (conv->Opcode() == Op_ConvI2L &&
 2639         // Are there other uses besides address expressions?
 2640         !is_visited(conv)) {
 2641       address_visited.set(conv->_idx); // Flag as address_visited
 2642       mstack.push(conv->in(1), Pre_Visit);
 2643     } else {
 2644       mstack.push(conv, Pre_Visit);
 2645     }
 2646     address_visited.test_set(m->_idx); // Flag as address_visited
 2647     mstack.push(m->in(AddPNode::Address), Pre_Visit);
 2648     mstack.push(m->in(AddPNode::Base), Pre_Visit);
 2649     return true;
 2650   } else if (off->Opcode() == Op_ConvI2L &&
 2651              // Are there other uses besides address expressions?
 2652              !is_visited(off)) {
 2653     address_visited.test_set(m->_idx); // Flag as address_visited
 2654     address_visited.set(off->_idx); // Flag as address_visited
 2655     mstack.push(off->in(1), Pre_Visit);
 2656     mstack.push(m->in(AddPNode::Address), Pre_Visit);
 2657     mstack.push(m->in(AddPNode::Base), Pre_Visit);
 2658     return true;
 2659   }
 2660   return false;
 2661 }
 2662 
 2663 #define MOV_VOLATILE(REG, BASE, INDEX, SCALE, DISP, SCRATCH, INSN)      \
 2664   {                                                                     \
 2665     guarantee(INDEX == -1, "mode not permitted for volatile");          \
 2666     guarantee(DISP == 0, "mode not permitted for volatile");            \
 2667     guarantee(SCALE == 0, "mode not permitted for volatile");           \
 2668     __ INSN(REG, as_Register(BASE));                                    \
 2669   }
 2670 
 2671 
 2672 static Address mem2address(int opcode, Register base, int index, int size, int disp)
 2673   {
 2674     Address::extend scale;
 2675 
 2676     // Hooboy, this is fugly.  We need a way to communicate to the
 2677     // encoder that the index needs to be sign extended, so we have to
 2678     // enumerate all the cases.
 2679     switch (opcode) {
 2680     case INDINDEXSCALEDI2L:
 2681     case INDINDEXSCALEDI2LN:
 2682     case INDINDEXI2L:
 2683     case INDINDEXI2LN:
 2684       scale = Address::sxtw(size);
 2685       break;
 2686     default:
 2687       scale = Address::lsl(size);
 2688     }
 2689 
 2690     if (index == -1) {
 2691       return Address(base, disp);
 2692     } else {
 2693       assert(disp == 0, "unsupported address mode: disp = %d", disp);
 2694       return Address(base, as_Register(index), scale);
 2695     }
 2696   }
 2697 
 2698 
 2699 typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr);
 2700 typedef void (MacroAssembler::* mem_insn2)(Register Rt, Register adr);
 2701 typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr);
 2702 typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt,
 2703                                   MacroAssembler::SIMD_RegVariant T, const Address &adr);
 2704 
 2705   // Used for all non-volatile memory accesses.  The use of
 2706   // $mem->opcode() to discover whether this pattern uses sign-extended
 2707   // offsets is something of a kludge.
 2708   static void loadStore(C2_MacroAssembler* masm, mem_insn insn,
 2709                         Register reg, int opcode,
 2710                         Register base, int index, int scale, int disp,
 2711                         int size_in_memory)
 2712   {
 2713     Address addr = mem2address(opcode, base, index, scale, disp);
 2714     if (addr.getMode() == Address::base_plus_offset) {
 2715       // Fix up any out-of-range offsets.
 2716       assert_different_registers(rscratch1, base);
 2717       assert_different_registers(rscratch1, reg);
 2718       addr = __ legitimize_address(addr, size_in_memory, rscratch1);
 2719     }
 2720     (masm->*insn)(reg, addr);
 2721   }
 2722 
 2723   static void loadStore(C2_MacroAssembler* masm, mem_float_insn insn,
 2724                         FloatRegister reg, int opcode,
 2725                         Register base, int index, int size, int disp,
 2726                         int size_in_memory)
 2727   {
 2728     Address::extend scale;
 2729 
 2730     switch (opcode) {
 2731     case INDINDEXSCALEDI2L:
 2732     case INDINDEXSCALEDI2LN:
 2733       scale = Address::sxtw(size);
 2734       break;
 2735     default:
 2736       scale = Address::lsl(size);
 2737     }
 2738 
 2739     if (index == -1) {
 2740       // Fix up any out-of-range offsets.
 2741       assert_different_registers(rscratch1, base);
 2742       Address addr = Address(base, disp);
 2743       addr = __ legitimize_address(addr, size_in_memory, rscratch1);
 2744       (masm->*insn)(reg, addr);
 2745     } else {
 2746       assert(disp == 0, "unsupported address mode: disp = %d", disp);
 2747       (masm->*insn)(reg, Address(base, as_Register(index), scale));
 2748     }
 2749   }
 2750 
 2751   static void loadStore(C2_MacroAssembler* masm, mem_vector_insn insn,
 2752                         FloatRegister reg, MacroAssembler::SIMD_RegVariant T,
 2753                         int opcode, Register base, int index, int size, int disp)
 2754   {
 2755     if (index == -1) {
 2756       // Fix up any out-of-range offsets.
 2757       assert_different_registers(rscratch1, base);
 2758       Address addr = Address(base, disp);
 2759       addr = __ legitimize_address(addr, (1 << T), rscratch1);
 2760       (masm->*insn)(reg, T, addr);
 2761     } else {
 2762       assert(disp == 0, "unsupported address mode");
 2763       (masm->*insn)(reg, T, Address(base, as_Register(index), Address::lsl(size)));
 2764     }
 2765   }
 2766 
 2767 %}
 2768 
 2769 
 2770 
 2771 //----------ENCODING BLOCK-----------------------------------------------------
 2772 // This block specifies the encoding classes used by the compiler to
 2773 // output byte streams.  Encoding classes are parameterized macros
 2774 // used by Machine Instruction Nodes in order to generate the bit
 2775 // encoding of the instruction.  Operands specify their base encoding
 2776 // interface with the interface keyword.  There are currently
 2777 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
 2778 // COND_INTER.  REG_INTER causes an operand to generate a function
 2779 // which returns its register number when queried.  CONST_INTER causes
 2780 // an operand to generate a function which returns the value of the
 2781 // constant when queried.  MEMORY_INTER causes an operand to generate
 2782 // four functions which return the Base Register, the Index Register,
 2783 // the Scale Value, and the Offset Value of the operand when queried.
 2784 // COND_INTER causes an operand to generate six functions which return
 2785 // the encoding code (ie - encoding bits for the instruction)
 2786 // associated with each basic boolean condition for a conditional
 2787 // instruction.
 2788 //
 2789 // Instructions specify two basic values for encoding.  Again, a
 2790 // function is available to check if the constant displacement is an
 2791 // oop. They use the ins_encode keyword to specify their encoding
 2792 // classes (which must be a sequence of enc_class names, and their
 2793 // parameters, specified in the encoding block), and they use the
 2794 // opcode keyword to specify, in order, their primary, secondary, and
 2795 // tertiary opcode.  Only the opcode sections which a particular
 2796 // instruction needs for encoding need to be specified.
 2797 encode %{
 2798   // Build emit functions for each basic byte or larger field in the
 2799   // intel encoding scheme (opcode, rm, sib, immediate), and call them
 2800   // from C++ code in the enc_class source block.  Emit functions will
 2801   // live in the main source block for now.  In future, we can
 2802   // generalize this by adding a syntax that specifies the sizes of
 2803   // fields in an order, so that the adlc can build the emit functions
 2804   // automagically
 2805 
 2806   // catch all for unimplemented encodings
 2807   enc_class enc_unimplemented %{
 2808     __ unimplemented("C2 catch all");
 2809   %}
 2810 
 2811   // BEGIN Non-volatile memory access
 2812 
 2813   // This encoding class is generated automatically from ad_encode.m4.
 2814   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2815   enc_class aarch64_enc_ldrsbw(iRegI dst, memory mem) %{
 2816     Register dst_reg = as_Register($dst$$reg);
 2817     loadStore(masm, &MacroAssembler::ldrsbw, dst_reg, $mem->opcode(),
 2818                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2819   %}
 2820 
 2821   // This encoding class is generated automatically from ad_encode.m4.
 2822   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2823   enc_class aarch64_enc_ldrsb(iRegI dst, memory mem) %{
 2824     Register dst_reg = as_Register($dst$$reg);
 2825     loadStore(masm, &MacroAssembler::ldrsb, dst_reg, $mem->opcode(),
 2826                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2827   %}
 2828 
 2829   // This encoding class is generated automatically from ad_encode.m4.
 2830   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2831   enc_class aarch64_enc_ldrb(iRegI dst, memory mem) %{
 2832     Register dst_reg = as_Register($dst$$reg);
 2833     loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
 2834                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2835   %}
 2836 
 2837   // This encoding class is generated automatically from ad_encode.m4.
 2838   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2839   enc_class aarch64_enc_ldrb(iRegL dst, memory mem) %{
 2840     Register dst_reg = as_Register($dst$$reg);
 2841     loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
 2842                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2843   %}
 2844 
 2845   // This encoding class is generated automatically from ad_encode.m4.
 2846   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2847   enc_class aarch64_enc_ldrshw(iRegI dst, memory mem) %{
 2848     Register dst_reg = as_Register($dst$$reg);
 2849     loadStore(masm, &MacroAssembler::ldrshw, dst_reg, $mem->opcode(),
 2850                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2851   %}
 2852 
 2853   // This encoding class is generated automatically from ad_encode.m4.
 2854   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2855   enc_class aarch64_enc_ldrsh(iRegI dst, memory mem) %{
 2856     Register dst_reg = as_Register($dst$$reg);
 2857     loadStore(masm, &MacroAssembler::ldrsh, dst_reg, $mem->opcode(),
 2858                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2859   %}
 2860 
 2861   // This encoding class is generated automatically from ad_encode.m4.
 2862   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2863   enc_class aarch64_enc_ldrh(iRegI dst, memory mem) %{
 2864     Register dst_reg = as_Register($dst$$reg);
 2865     loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
 2866                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2867   %}
 2868 
 2869   // This encoding class is generated automatically from ad_encode.m4.
 2870   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2871   enc_class aarch64_enc_ldrh(iRegL dst, memory mem) %{
 2872     Register dst_reg = as_Register($dst$$reg);
 2873     loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
 2874                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2875   %}
 2876 
 2877   // This encoding class is generated automatically from ad_encode.m4.
 2878   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2879   enc_class aarch64_enc_ldrw(iRegI dst, memory mem) %{
 2880     Register dst_reg = as_Register($dst$$reg);
 2881     loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
 2882                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2883   %}
 2884 
 2885   // This encoding class is generated automatically from ad_encode.m4.
 2886   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2887   enc_class aarch64_enc_ldrw(iRegL dst, memory mem) %{
 2888     Register dst_reg = as_Register($dst$$reg);
 2889     loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
 2890                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2891   %}
 2892 
 2893   // This encoding class is generated automatically from ad_encode.m4.
 2894   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2895   enc_class aarch64_enc_ldrsw(iRegL dst, memory mem) %{
 2896     Register dst_reg = as_Register($dst$$reg);
 2897     loadStore(masm, &MacroAssembler::ldrsw, dst_reg, $mem->opcode(),
 2898                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2899   %}
 2900 
 2901   // This encoding class is generated automatically from ad_encode.m4.
 2902   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2903   enc_class aarch64_enc_ldr(iRegL dst, memory mem) %{
 2904     Register dst_reg = as_Register($dst$$reg);
 2905     loadStore(masm, &MacroAssembler::ldr, dst_reg, $mem->opcode(),
 2906                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 2907   %}
 2908 
 2909   // This encoding class is generated automatically from ad_encode.m4.
 2910   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2911   enc_class aarch64_enc_ldrs(vRegF dst, memory mem) %{
 2912     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 2913     loadStore(masm, &MacroAssembler::ldrs, dst_reg, $mem->opcode(),
 2914                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2915   %}
 2916 
 2917   // This encoding class is generated automatically from ad_encode.m4.
 2918   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2919   enc_class aarch64_enc_ldrd(vRegD dst, memory mem) %{
 2920     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 2921     loadStore(masm, &MacroAssembler::ldrd, dst_reg, $mem->opcode(),
 2922                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 2923   %}
 2924 
 2925   // This encoding class is generated automatically from ad_encode.m4.
 2926   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2927   enc_class aarch64_enc_strb(iRegI src, memory mem) %{
 2928     Register src_reg = as_Register($src$$reg);
 2929     loadStore(masm, &MacroAssembler::strb, src_reg, $mem->opcode(),
 2930                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2931   %}
 2932 
 2933   // This encoding class is generated automatically from ad_encode.m4.
 2934   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2935   enc_class aarch64_enc_strb0(memory mem) %{
 2936     loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
 2937                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2938   %}
 2939 
 2940   // This encoding class is generated automatically from ad_encode.m4.
 2941   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2942   enc_class aarch64_enc_strh(iRegI src, memory mem) %{
 2943     Register src_reg = as_Register($src$$reg);
 2944     loadStore(masm, &MacroAssembler::strh, src_reg, $mem->opcode(),
 2945                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2946   %}
 2947 
 2948   // This encoding class is generated automatically from ad_encode.m4.
 2949   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2950   enc_class aarch64_enc_strh0(memory mem) %{
 2951     loadStore(masm, &MacroAssembler::strh, zr, $mem->opcode(),
 2952                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2953   %}
 2954 
 2955   // This encoding class is generated automatically from ad_encode.m4.
 2956   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2957   enc_class aarch64_enc_strw(iRegI src, memory mem) %{
 2958     Register src_reg = as_Register($src$$reg);
 2959     loadStore(masm, &MacroAssembler::strw, src_reg, $mem->opcode(),
 2960                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2961   %}
 2962 
 2963   // This encoding class is generated automatically from ad_encode.m4.
 2964   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2965   enc_class aarch64_enc_strw0(memory mem) %{
 2966     loadStore(masm, &MacroAssembler::strw, zr, $mem->opcode(),
 2967                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2968   %}
 2969 
 2970   // This encoding class is generated automatically from ad_encode.m4.
 2971   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2972   enc_class aarch64_enc_str(iRegL src, memory mem) %{
 2973     Register src_reg = as_Register($src$$reg);
 2974     // we sometimes get asked to store the stack pointer into the
 2975     // current thread -- we cannot do that directly on AArch64
 2976     if (src_reg == r31_sp) {
 2977       assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
 2978       __ mov(rscratch2, sp);
 2979       src_reg = rscratch2;
 2980     }
 2981     loadStore(masm, &MacroAssembler::str, src_reg, $mem->opcode(),
 2982                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 2983   %}
 2984 
 2985   // This encoding class is generated automatically from ad_encode.m4.
 2986   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2987   enc_class aarch64_enc_str0(memory mem) %{
 2988     loadStore(masm, &MacroAssembler::str, zr, $mem->opcode(),
 2989                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 2990   %}
 2991 
 2992   // This encoding class is generated automatically from ad_encode.m4.
 2993   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2994   enc_class aarch64_enc_strs(vRegF src, memory mem) %{
 2995     FloatRegister src_reg = as_FloatRegister($src$$reg);
 2996     loadStore(masm, &MacroAssembler::strs, src_reg, $mem->opcode(),
 2997                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2998   %}
 2999 
 3000   // This encoding class is generated automatically from ad_encode.m4.
 3001   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3002   enc_class aarch64_enc_strd(vRegD src, memory mem) %{
 3003     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3004     loadStore(masm, &MacroAssembler::strd, src_reg, $mem->opcode(),
 3005                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 3006   %}
 3007 
 3008   // This encoding class is generated automatically from ad_encode.m4.
 3009   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3010   enc_class aarch64_enc_strb0_ordered(memory mem) %{
 3011       __ membar(Assembler::StoreStore);
 3012       loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
 3013                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 3014   %}
 3015 
 3016   // END Non-volatile memory access
 3017 
 3018   // Vector loads and stores
 3019   enc_class aarch64_enc_ldrvH(vReg dst, memory mem) %{
 3020     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3021     loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::H,
 3022        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3023   %}
 3024 
 3025   enc_class aarch64_enc_ldrvS(vReg dst, memory mem) %{
 3026     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3027     loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::S,
 3028        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3029   %}
 3030 
 3031   enc_class aarch64_enc_ldrvD(vReg dst, memory mem) %{
 3032     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3033     loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::D,
 3034        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3035   %}
 3036 
 3037   enc_class aarch64_enc_ldrvQ(vReg dst, memory mem) %{
 3038     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3039     loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::Q,
 3040        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3041   %}
 3042 
 3043   enc_class aarch64_enc_strvH(vReg src, memory mem) %{
 3044     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3045     loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::H,
 3046        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3047   %}
 3048 
 3049   enc_class aarch64_enc_strvS(vReg src, memory mem) %{
 3050     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3051     loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::S,
 3052        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3053   %}
 3054 
 3055   enc_class aarch64_enc_strvD(vReg src, memory mem) %{
 3056     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3057     loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::D,
 3058        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3059   %}
 3060 
 3061   enc_class aarch64_enc_strvQ(vReg src, memory mem) %{
 3062     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3063     loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::Q,
 3064        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3065   %}
 3066 
 3067   // volatile loads and stores
 3068 
 3069   enc_class aarch64_enc_stlrb(iRegI src, memory mem) %{
 3070     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3071                  rscratch1, stlrb);
 3072   %}
 3073 
 3074   enc_class aarch64_enc_stlrb0(memory mem) %{
 3075     MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3076                  rscratch1, stlrb);
 3077   %}
 3078 
 3079   enc_class aarch64_enc_stlrh(iRegI src, memory mem) %{
 3080     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3081                  rscratch1, stlrh);
 3082   %}
 3083 
 3084   enc_class aarch64_enc_stlrh0(memory mem) %{
 3085     MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3086                  rscratch1, stlrh);
 3087   %}
 3088 
 3089   enc_class aarch64_enc_stlrw(iRegI src, memory mem) %{
 3090     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3091                  rscratch1, stlrw);
 3092   %}
 3093 
 3094   enc_class aarch64_enc_stlrw0(memory mem) %{
 3095     MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3096                  rscratch1, stlrw);
 3097   %}
 3098 
 3099   enc_class aarch64_enc_ldarsbw(iRegI dst, memory mem) %{
 3100     Register dst_reg = as_Register($dst$$reg);
 3101     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3102              rscratch1, ldarb);
 3103     __ sxtbw(dst_reg, dst_reg);
 3104   %}
 3105 
 3106   enc_class aarch64_enc_ldarsb(iRegL dst, memory mem) %{
 3107     Register dst_reg = as_Register($dst$$reg);
 3108     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3109              rscratch1, ldarb);
 3110     __ sxtb(dst_reg, dst_reg);
 3111   %}
 3112 
 3113   enc_class aarch64_enc_ldarbw(iRegI dst, memory mem) %{
 3114     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3115              rscratch1, ldarb);
 3116   %}
 3117 
 3118   enc_class aarch64_enc_ldarb(iRegL dst, memory mem) %{
 3119     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3120              rscratch1, ldarb);
 3121   %}
 3122 
 3123   enc_class aarch64_enc_ldarshw(iRegI dst, memory mem) %{
 3124     Register dst_reg = as_Register($dst$$reg);
 3125     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3126              rscratch1, ldarh);
 3127     __ sxthw(dst_reg, dst_reg);
 3128   %}
 3129 
 3130   enc_class aarch64_enc_ldarsh(iRegL dst, memory mem) %{
 3131     Register dst_reg = as_Register($dst$$reg);
 3132     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3133              rscratch1, ldarh);
 3134     __ sxth(dst_reg, dst_reg);
 3135   %}
 3136 
 3137   enc_class aarch64_enc_ldarhw(iRegI dst, memory mem) %{
 3138     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3139              rscratch1, ldarh);
 3140   %}
 3141 
 3142   enc_class aarch64_enc_ldarh(iRegL dst, memory mem) %{
 3143     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3144              rscratch1, ldarh);
 3145   %}
 3146 
 3147   enc_class aarch64_enc_ldarw(iRegI dst, memory mem) %{
 3148     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3149              rscratch1, ldarw);
 3150   %}
 3151 
 3152   enc_class aarch64_enc_ldarw(iRegL dst, memory mem) %{
 3153     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3154              rscratch1, ldarw);
 3155   %}
 3156 
 3157   enc_class aarch64_enc_ldar(iRegL dst, memory mem) %{
 3158     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3159              rscratch1, ldar);
 3160   %}
 3161 
 3162   enc_class aarch64_enc_fldars(vRegF dst, memory mem) %{
 3163     MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3164              rscratch1, ldarw);
 3165     __ fmovs(as_FloatRegister($dst$$reg), rscratch1);
 3166   %}
 3167 
 3168   enc_class aarch64_enc_fldard(vRegD dst, memory mem) %{
 3169     MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3170              rscratch1, ldar);
 3171     __ fmovd(as_FloatRegister($dst$$reg), rscratch1);
 3172   %}
 3173 
 3174   enc_class aarch64_enc_stlr(iRegL src, memory mem) %{
 3175     Register src_reg = as_Register($src$$reg);
 3176     // we sometimes get asked to store the stack pointer into the
 3177     // current thread -- we cannot do that directly on AArch64
 3178     if (src_reg == r31_sp) {
 3179       assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
 3180       __ mov(rscratch2, sp);
 3181       src_reg = rscratch2;
 3182     }
 3183     MOV_VOLATILE(src_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3184                  rscratch1, stlr);
 3185   %}
 3186 
 3187   enc_class aarch64_enc_stlr0(memory mem) %{
 3188     MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3189                  rscratch1, stlr);
 3190   %}
 3191 
 3192   enc_class aarch64_enc_fstlrs(vRegF src, memory mem) %{
 3193     {
 3194       FloatRegister src_reg = as_FloatRegister($src$$reg);
 3195       __ fmovs(rscratch2, src_reg);
 3196     }
 3197     MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3198                  rscratch1, stlrw);
 3199   %}
 3200 
 3201   enc_class aarch64_enc_fstlrd(vRegD src, memory mem) %{
 3202     {
 3203       FloatRegister src_reg = as_FloatRegister($src$$reg);
 3204       __ fmovd(rscratch2, src_reg);
 3205     }
 3206     MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3207                  rscratch1, stlr);
 3208   %}
 3209 
 3210   // synchronized read/update encodings
 3211 
 3212   enc_class aarch64_enc_ldaxr(iRegL dst, memory mem) %{
 3213     Register dst_reg = as_Register($dst$$reg);
 3214     Register base = as_Register($mem$$base);
 3215     int index = $mem$$index;
 3216     int scale = $mem$$scale;
 3217     int disp = $mem$$disp;
 3218     if (index == -1) {
 3219        if (disp != 0) {
 3220         __ lea(rscratch1, Address(base, disp));
 3221         __ ldaxr(dst_reg, rscratch1);
 3222       } else {
 3223         // TODO
 3224         // should we ever get anything other than this case?
 3225         __ ldaxr(dst_reg, base);
 3226       }
 3227     } else {
 3228       Register index_reg = as_Register(index);
 3229       if (disp == 0) {
 3230         __ lea(rscratch1, Address(base, index_reg, Address::lsl(scale)));
 3231         __ ldaxr(dst_reg, rscratch1);
 3232       } else {
 3233         __ lea(rscratch1, Address(base, disp));
 3234         __ lea(rscratch1, Address(rscratch1, index_reg, Address::lsl(scale)));
 3235         __ ldaxr(dst_reg, rscratch1);
 3236       }
 3237     }
 3238   %}
 3239 
 3240   enc_class aarch64_enc_stlxr(iRegLNoSp src, memory mem) %{
 3241     Register src_reg = as_Register($src$$reg);
 3242     Register base = as_Register($mem$$base);
 3243     int index = $mem$$index;
 3244     int scale = $mem$$scale;
 3245     int disp = $mem$$disp;
 3246     if (index == -1) {
 3247        if (disp != 0) {
 3248         __ lea(rscratch2, Address(base, disp));
 3249         __ stlxr(rscratch1, src_reg, rscratch2);
 3250       } else {
 3251         // TODO
 3252         // should we ever get anything other than this case?
 3253         __ stlxr(rscratch1, src_reg, base);
 3254       }
 3255     } else {
 3256       Register index_reg = as_Register(index);
 3257       if (disp == 0) {
 3258         __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
 3259         __ stlxr(rscratch1, src_reg, rscratch2);
 3260       } else {
 3261         __ lea(rscratch2, Address(base, disp));
 3262         __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
 3263         __ stlxr(rscratch1, src_reg, rscratch2);
 3264       }
 3265     }
 3266     __ cmpw(rscratch1, zr);
 3267   %}
 3268 
 3269   enc_class aarch64_enc_cmpxchg(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
 3270     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3271     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3272                Assembler::xword, /*acquire*/ false, /*release*/ true,
 3273                /*weak*/ false, noreg);
 3274   %}
 3275 
 3276   enc_class aarch64_enc_cmpxchgw(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3277     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3278     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3279                Assembler::word, /*acquire*/ false, /*release*/ true,
 3280                /*weak*/ false, noreg);
 3281   %}
 3282 
 3283   enc_class aarch64_enc_cmpxchgs(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3284     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3285     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3286                Assembler::halfword, /*acquire*/ false, /*release*/ true,
 3287                /*weak*/ false, noreg);
 3288   %}
 3289 
 3290   enc_class aarch64_enc_cmpxchgb(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3291     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3292     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3293                Assembler::byte, /*acquire*/ false, /*release*/ true,
 3294                /*weak*/ false, noreg);
 3295   %}
 3296 
 3297 
 3298   // The only difference between aarch64_enc_cmpxchg and
 3299   // aarch64_enc_cmpxchg_acq is that we use load-acquire in the
 3300   // CompareAndSwap sequence to serve as a barrier on acquiring a
 3301   // lock.
 3302   enc_class aarch64_enc_cmpxchg_acq(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
 3303     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3304     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3305                Assembler::xword, /*acquire*/ true, /*release*/ true,
 3306                /*weak*/ false, noreg);
 3307   %}
 3308 
 3309   enc_class aarch64_enc_cmpxchgw_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3310     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3311     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3312                Assembler::word, /*acquire*/ true, /*release*/ true,
 3313                /*weak*/ false, noreg);
 3314   %}
 3315 
 3316   enc_class aarch64_enc_cmpxchgs_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3317     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3318     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3319                Assembler::halfword, /*acquire*/ true, /*release*/ true,
 3320                /*weak*/ false, noreg);
 3321   %}
 3322 
 3323   enc_class aarch64_enc_cmpxchgb_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3324     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3325     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3326                Assembler::byte, /*acquire*/ true, /*release*/ true,
 3327                /*weak*/ false, noreg);
 3328   %}
 3329 
 3330   // auxiliary used for CompareAndSwapX to set result register
 3331   enc_class aarch64_enc_cset_eq(iRegINoSp res) %{
 3332     Register res_reg = as_Register($res$$reg);
 3333     __ cset(res_reg, Assembler::EQ);
 3334   %}
 3335 
 3336   // prefetch encodings
 3337 
 3338   enc_class aarch64_enc_prefetchw(memory mem) %{
 3339     Register base = as_Register($mem$$base);
 3340     int index = $mem$$index;
 3341     int scale = $mem$$scale;
 3342     int disp = $mem$$disp;
 3343     if (index == -1) {
 3344       // Fix up any out-of-range offsets.
 3345       assert_different_registers(rscratch1, base);
 3346       Address addr = Address(base, disp);
 3347       addr = __ legitimize_address(addr, 8, rscratch1);
 3348       __ prfm(addr, PSTL1KEEP);
 3349     } else {
 3350       Register index_reg = as_Register(index);
 3351       if (disp == 0) {
 3352         __ prfm(Address(base, index_reg, Address::lsl(scale)), PSTL1KEEP);
 3353       } else {
 3354         __ lea(rscratch1, Address(base, disp));
 3355 	__ prfm(Address(rscratch1, index_reg, Address::lsl(scale)), PSTL1KEEP);
 3356       }
 3357     }
 3358   %}
 3359 
 3360   // mov encodings
 3361 
 3362   enc_class aarch64_enc_movw_imm(iRegI dst, immI src) %{
 3363     uint32_t con = (uint32_t)$src$$constant;
 3364     Register dst_reg = as_Register($dst$$reg);
 3365     if (con == 0) {
 3366       __ movw(dst_reg, zr);
 3367     } else {
 3368       __ movw(dst_reg, con);
 3369     }
 3370   %}
 3371 
 3372   enc_class aarch64_enc_mov_imm(iRegL dst, immL src) %{
 3373     Register dst_reg = as_Register($dst$$reg);
 3374     uint64_t con = (uint64_t)$src$$constant;
 3375     if (con == 0) {
 3376       __ mov(dst_reg, zr);
 3377     } else {
 3378       __ mov(dst_reg, con);
 3379     }
 3380   %}
 3381 
 3382   enc_class aarch64_enc_mov_p(iRegP dst, immP src) %{
 3383     Register dst_reg = as_Register($dst$$reg);
 3384     address con = (address)$src$$constant;
 3385     if (con == nullptr || con == (address)1) {
 3386       ShouldNotReachHere();
 3387     } else {
 3388       relocInfo::relocType rtype = $src->constant_reloc();
 3389       if (rtype == relocInfo::oop_type) {
 3390         __ movoop(dst_reg, (jobject)con);
 3391       } else if (rtype == relocInfo::metadata_type) {
 3392         __ mov_metadata(dst_reg, (Metadata*)con);
 3393       } else {
 3394         assert(rtype == relocInfo::none, "unexpected reloc type");
 3395         if (! __ is_valid_AArch64_address(con) ||
 3396             con < (address)(uintptr_t)os::vm_page_size()) {
 3397           __ mov(dst_reg, con);
 3398         } else {
 3399           uint64_t offset;
 3400           __ adrp(dst_reg, con, offset);
 3401           __ add(dst_reg, dst_reg, offset);
 3402         }
 3403       }
 3404     }
 3405   %}
 3406 
 3407   enc_class aarch64_enc_mov_p0(iRegP dst, immP0 src) %{
 3408     Register dst_reg = as_Register($dst$$reg);
 3409     __ mov(dst_reg, zr);
 3410   %}
 3411 
 3412   enc_class aarch64_enc_mov_p1(iRegP dst, immP_1 src) %{
 3413     Register dst_reg = as_Register($dst$$reg);
 3414     __ mov(dst_reg, (uint64_t)1);
 3415   %}
 3416 
 3417   enc_class aarch64_enc_mov_byte_map_base(iRegP dst, immByteMapBase src) %{
 3418     __ load_byte_map_base($dst$$Register);
 3419   %}
 3420 
 3421   enc_class aarch64_enc_mov_n(iRegN dst, immN src) %{
 3422     Register dst_reg = as_Register($dst$$reg);
 3423     address con = (address)$src$$constant;
 3424     if (con == nullptr) {
 3425       ShouldNotReachHere();
 3426     } else {
 3427       relocInfo::relocType rtype = $src->constant_reloc();
 3428       assert(rtype == relocInfo::oop_type, "unexpected reloc type");
 3429       __ set_narrow_oop(dst_reg, (jobject)con);
 3430     }
 3431   %}
 3432 
 3433   enc_class aarch64_enc_mov_n0(iRegN dst, immN0 src) %{
 3434     Register dst_reg = as_Register($dst$$reg);
 3435     __ mov(dst_reg, zr);
 3436   %}
 3437 
 3438   enc_class aarch64_enc_mov_nk(iRegN dst, immNKlass src) %{
 3439     Register dst_reg = as_Register($dst$$reg);
 3440     address con = (address)$src$$constant;
 3441     if (con == nullptr) {
 3442       ShouldNotReachHere();
 3443     } else {
 3444       relocInfo::relocType rtype = $src->constant_reloc();
 3445       assert(rtype == relocInfo::metadata_type, "unexpected reloc type");
 3446       __ set_narrow_klass(dst_reg, (Klass *)con);
 3447     }
 3448   %}
 3449 
 3450   // arithmetic encodings
 3451 
 3452   enc_class aarch64_enc_addsubw_imm(iRegI dst, iRegI src1, immIAddSub src2) %{
 3453     Register dst_reg = as_Register($dst$$reg);
 3454     Register src_reg = as_Register($src1$$reg);
 3455     int32_t con = (int32_t)$src2$$constant;
 3456     // add has primary == 0, subtract has primary == 1
 3457     if ($primary) { con = -con; }
 3458     if (con < 0) {
 3459       __ subw(dst_reg, src_reg, -con);
 3460     } else {
 3461       __ addw(dst_reg, src_reg, con);
 3462     }
 3463   %}
 3464 
 3465   enc_class aarch64_enc_addsub_imm(iRegL dst, iRegL src1, immLAddSub src2) %{
 3466     Register dst_reg = as_Register($dst$$reg);
 3467     Register src_reg = as_Register($src1$$reg);
 3468     int32_t con = (int32_t)$src2$$constant;
 3469     // add has primary == 0, subtract has primary == 1
 3470     if ($primary) { con = -con; }
 3471     if (con < 0) {
 3472       __ sub(dst_reg, src_reg, -con);
 3473     } else {
 3474       __ add(dst_reg, src_reg, con);
 3475     }
 3476   %}
 3477 
 3478   enc_class aarch64_enc_divw(iRegI dst, iRegI src1, iRegI src2) %{
 3479    Register dst_reg = as_Register($dst$$reg);
 3480    Register src1_reg = as_Register($src1$$reg);
 3481    Register src2_reg = as_Register($src2$$reg);
 3482     __ corrected_idivl(dst_reg, src1_reg, src2_reg, false, rscratch1);
 3483   %}
 3484 
 3485   enc_class aarch64_enc_div(iRegI dst, iRegI src1, iRegI src2) %{
 3486    Register dst_reg = as_Register($dst$$reg);
 3487    Register src1_reg = as_Register($src1$$reg);
 3488    Register src2_reg = as_Register($src2$$reg);
 3489     __ corrected_idivq(dst_reg, src1_reg, src2_reg, false, rscratch1);
 3490   %}
 3491 
 3492   enc_class aarch64_enc_modw(iRegI dst, iRegI src1, iRegI src2) %{
 3493    Register dst_reg = as_Register($dst$$reg);
 3494    Register src1_reg = as_Register($src1$$reg);
 3495    Register src2_reg = as_Register($src2$$reg);
 3496     __ corrected_idivl(dst_reg, src1_reg, src2_reg, true, rscratch1);
 3497   %}
 3498 
 3499   enc_class aarch64_enc_mod(iRegI dst, iRegI src1, iRegI src2) %{
 3500    Register dst_reg = as_Register($dst$$reg);
 3501    Register src1_reg = as_Register($src1$$reg);
 3502    Register src2_reg = as_Register($src2$$reg);
 3503     __ corrected_idivq(dst_reg, src1_reg, src2_reg, true, rscratch1);
 3504   %}
 3505 
 3506   // compare instruction encodings
 3507 
 3508   enc_class aarch64_enc_cmpw(iRegI src1, iRegI src2) %{
 3509     Register reg1 = as_Register($src1$$reg);
 3510     Register reg2 = as_Register($src2$$reg);
 3511     __ cmpw(reg1, reg2);
 3512   %}
 3513 
 3514   enc_class aarch64_enc_cmpw_imm_addsub(iRegI src1, immIAddSub src2) %{
 3515     Register reg = as_Register($src1$$reg);
 3516     int32_t val = $src2$$constant;
 3517     if (val >= 0) {
 3518       __ subsw(zr, reg, val);
 3519     } else {
 3520       __ addsw(zr, reg, -val);
 3521     }
 3522   %}
 3523 
 3524   enc_class aarch64_enc_cmpw_imm(iRegI src1, immI src2) %{
 3525     Register reg1 = as_Register($src1$$reg);
 3526     uint32_t val = (uint32_t)$src2$$constant;
 3527     __ movw(rscratch1, val);
 3528     __ cmpw(reg1, rscratch1);
 3529   %}
 3530 
 3531   enc_class aarch64_enc_cmp(iRegL src1, iRegL src2) %{
 3532     Register reg1 = as_Register($src1$$reg);
 3533     Register reg2 = as_Register($src2$$reg);
 3534     __ cmp(reg1, reg2);
 3535   %}
 3536 
 3537   enc_class aarch64_enc_cmp_imm_addsub(iRegL src1, immL12 src2) %{
 3538     Register reg = as_Register($src1$$reg);
 3539     int64_t val = $src2$$constant;
 3540     if (val >= 0) {
 3541       __ subs(zr, reg, val);
 3542     } else if (val != -val) {
 3543       __ adds(zr, reg, -val);
 3544     } else {
 3545     // aargh, Long.MIN_VALUE is a special case
 3546       __ orr(rscratch1, zr, (uint64_t)val);
 3547       __ subs(zr, reg, rscratch1);
 3548     }
 3549   %}
 3550 
 3551   enc_class aarch64_enc_cmp_imm(iRegL src1, immL src2) %{
 3552     Register reg1 = as_Register($src1$$reg);
 3553     uint64_t val = (uint64_t)$src2$$constant;
 3554     __ mov(rscratch1, val);
 3555     __ cmp(reg1, rscratch1);
 3556   %}
 3557 
 3558   enc_class aarch64_enc_cmpp(iRegP src1, iRegP src2) %{
 3559     Register reg1 = as_Register($src1$$reg);
 3560     Register reg2 = as_Register($src2$$reg);
 3561     __ cmp(reg1, reg2);
 3562   %}
 3563 
 3564   enc_class aarch64_enc_cmpn(iRegN src1, iRegN src2) %{
 3565     Register reg1 = as_Register($src1$$reg);
 3566     Register reg2 = as_Register($src2$$reg);
 3567     __ cmpw(reg1, reg2);
 3568   %}
 3569 
 3570   enc_class aarch64_enc_testp(iRegP src) %{
 3571     Register reg = as_Register($src$$reg);
 3572     __ cmp(reg, zr);
 3573   %}
 3574 
 3575   enc_class aarch64_enc_testn(iRegN src) %{
 3576     Register reg = as_Register($src$$reg);
 3577     __ cmpw(reg, zr);
 3578   %}
 3579 
 3580   enc_class aarch64_enc_b(label lbl) %{
 3581     Label *L = $lbl$$label;
 3582     __ b(*L);
 3583   %}
 3584 
 3585   enc_class aarch64_enc_br_con(cmpOp cmp, label lbl) %{
 3586     Label *L = $lbl$$label;
 3587     __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
 3588   %}
 3589 
 3590   enc_class aarch64_enc_br_conU(cmpOpU cmp, label lbl) %{
 3591     Label *L = $lbl$$label;
 3592     __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
 3593   %}
 3594 
 3595   enc_class aarch64_enc_partial_subtype_check(iRegP sub, iRegP super, iRegP temp, iRegP result)
 3596   %{
 3597      Register sub_reg = as_Register($sub$$reg);
 3598      Register super_reg = as_Register($super$$reg);
 3599      Register temp_reg = as_Register($temp$$reg);
 3600      Register result_reg = as_Register($result$$reg);
 3601 
 3602      Label miss;
 3603      __ check_klass_subtype_slow_path(sub_reg, super_reg, temp_reg, result_reg,
 3604                                      nullptr, &miss,
 3605                                      /*set_cond_codes:*/ true);
 3606      if ($primary) {
 3607        __ mov(result_reg, zr);
 3608      }
 3609      __ bind(miss);
 3610   %}
 3611 
 3612   enc_class aarch64_enc_java_static_call(method meth) %{
 3613     address addr = (address)$meth$$method;
 3614     address call;
 3615     if (!_method) {
 3616       // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
 3617       call = __ trampoline_call(Address(addr, relocInfo::runtime_call_type));
 3618       if (call == nullptr) {
 3619         ciEnv::current()->record_failure("CodeCache is full");
 3620         return;
 3621       }
 3622     } else if (_method->intrinsic_id() == vmIntrinsicID::_ensureMaterializedForStackWalk) {
 3623       // The NOP here is purely to ensure that eliding a call to
 3624       // JVM_EnsureMaterializedForStackWalk doesn't change the code size.
 3625       __ nop();
 3626       __ block_comment("call JVM_EnsureMaterializedForStackWalk (elided)");
 3627     } else {
 3628       int method_index = resolved_method_index(masm);
 3629       RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
 3630                                                   : static_call_Relocation::spec(method_index);
 3631       call = __ trampoline_call(Address(addr, rspec));
 3632       if (call == nullptr) {
 3633         ciEnv::current()->record_failure("CodeCache is full");
 3634         return;
 3635       }
 3636       if (CodeBuffer::supports_shared_stubs() && _method->can_be_statically_bound()) {
 3637         // Calls of the same statically bound method can share
 3638         // a stub to the interpreter.
 3639         __ code()->shared_stub_to_interp_for(_method, call - __ begin());
 3640       } else {
 3641         // Emit stub for static call
 3642         address stub = CompiledDirectCall::emit_to_interp_stub(masm, call);
 3643         if (stub == nullptr) {
 3644           ciEnv::current()->record_failure("CodeCache is full");
 3645           return;
 3646         }
 3647       }
 3648     }
 3649 
 3650     __ post_call_nop();
 3651 
 3652     // Only non uncommon_trap calls need to reinitialize ptrue.
 3653     if (Compile::current()->max_vector_size() > 0 && uncommon_trap_request() == 0) {
 3654       __ reinitialize_ptrue();
 3655     }
 3656   %}
 3657 
 3658   enc_class aarch64_enc_java_dynamic_call(method meth) %{
 3659     int method_index = resolved_method_index(masm);
 3660     address call = __ ic_call((address)$meth$$method, method_index);
 3661     if (call == nullptr) {
 3662       ciEnv::current()->record_failure("CodeCache is full");
 3663       return;
 3664     }
 3665     __ post_call_nop();
 3666     if (Compile::current()->max_vector_size() > 0) {
 3667       __ reinitialize_ptrue();
 3668     }
 3669   %}
 3670 
 3671   enc_class aarch64_enc_call_epilog() %{
 3672     if (VerifyStackAtCalls) {
 3673       // Check that stack depth is unchanged: find majik cookie on stack
 3674       __ call_Unimplemented();
 3675     }
 3676     if (tf()->returns_inline_type_as_fields() && !_method->is_method_handle_intrinsic()) {
 3677       // The last return value is not set by the callee but used to pass IsInit information to compiled code.
 3678       // Search for the corresponding projection, get the register and emit code that initialized it.
 3679       uint con = (tf()->range_cc()->cnt() - 1);
 3680       for (DUIterator_Fast imax, i = fast_outs(imax); i < imax; i++) {
 3681         ProjNode* proj = fast_out(i)->as_Proj();
 3682         if (proj->_con == con) {
 3683           // Set IsInit if r0 is non-null (a non-null value is returned buffered or scalarized)
 3684           OptoReg::Name optoReg = ra_->get_reg_first(proj);
 3685           VMReg reg = OptoReg::as_VMReg(optoReg, ra_->_framesize, OptoReg::reg2stack(ra_->_matcher._new_SP));
 3686           Register toReg = reg->is_reg() ? reg->as_Register() : rscratch1;
 3687           __ cmp(r0, zr);
 3688           __ cset(toReg, Assembler::NE);
 3689           if (reg->is_stack()) {
 3690             int st_off = reg->reg2stack() * VMRegImpl::stack_slot_size;
 3691             __ str(toReg, Address(sp, st_off));
 3692           }
 3693           break;
 3694         }
 3695       }
 3696       if (return_value_is_used()) {
 3697         // An inline type is returned as fields in multiple registers.
 3698         // R0 either contains an oop if the inline type is buffered or a pointer
 3699         // to the corresponding InlineKlass with the lowest bit set to 1. Zero r0
 3700         // if the lowest bit is set to allow C2 to use the oop after null checking.
 3701         // r0 &= (r0 & 1) - 1
 3702         __ andr(rscratch1, r0, 0x1);
 3703         __ sub(rscratch1, rscratch1, 0x1);
 3704         __ andr(r0, r0, rscratch1);
 3705       }
 3706     }
 3707   %}
 3708 
 3709   enc_class aarch64_enc_java_to_runtime(method meth) %{
 3710     // some calls to generated routines (arraycopy code) are scheduled
 3711     // by C2 as runtime calls. if so we can call them using a br (they
 3712     // will be in a reachable segment) otherwise we have to use a blr
 3713     // which loads the absolute address into a register.
 3714     address entry = (address)$meth$$method;
 3715     CodeBlob *cb = CodeCache::find_blob(entry);
 3716     if (cb) {
 3717       address call = __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
 3718       if (call == nullptr) {
 3719         ciEnv::current()->record_failure("CodeCache is full");
 3720         return;
 3721       }
 3722       __ post_call_nop();
 3723     } else {
 3724       Label retaddr;
 3725       __ adr(rscratch2, retaddr);
 3726       __ lea(rscratch1, RuntimeAddress(entry));
 3727       // Leave a breadcrumb for JavaFrameAnchor::capture_last_Java_pc()
 3728       __ stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)));
 3729       __ blr(rscratch1);
 3730       __ bind(retaddr);
 3731       __ post_call_nop();
 3732       __ add(sp, sp, 2 * wordSize);
 3733     }
 3734     if (Compile::current()->max_vector_size() > 0) {
 3735       __ reinitialize_ptrue();
 3736     }
 3737   %}
 3738 
 3739   enc_class aarch64_enc_rethrow() %{
 3740     __ far_jump(RuntimeAddress(OptoRuntime::rethrow_stub()));
 3741   %}
 3742 
 3743   enc_class aarch64_enc_ret() %{
 3744 #ifdef ASSERT
 3745     if (Compile::current()->max_vector_size() > 0) {
 3746       __ verify_ptrue();
 3747     }
 3748 #endif
 3749     __ ret(lr);
 3750   %}
 3751 
 3752   enc_class aarch64_enc_tail_call(iRegP jump_target) %{
 3753     Register target_reg = as_Register($jump_target$$reg);
 3754     __ br(target_reg);
 3755   %}
 3756 
 3757   enc_class aarch64_enc_tail_jmp(iRegP jump_target) %{
 3758     Register target_reg = as_Register($jump_target$$reg);
 3759     // exception oop should be in r0
 3760     // ret addr has been popped into lr
 3761     // callee expects it in r3
 3762     __ mov(r3, lr);
 3763     __ br(target_reg);
 3764   %}
 3765 
 3766 %}
 3767 
 3768 //----------FRAME--------------------------------------------------------------
 3769 // Definition of frame structure and management information.
 3770 //
 3771 //  S T A C K   L A Y O U T    Allocators stack-slot number
 3772 //                             |   (to get allocators register number
 3773 //  G  Owned by    |        |  v    add OptoReg::stack0())
 3774 //  r   CALLER     |        |
 3775 //  o     |        +--------+      pad to even-align allocators stack-slot
 3776 //  w     V        |  pad0  |        numbers; owned by CALLER
 3777 //  t   -----------+--------+----> Matcher::_in_arg_limit, unaligned
 3778 //  h     ^        |   in   |  5
 3779 //        |        |  args  |  4   Holes in incoming args owned by SELF
 3780 //  |     |        |        |  3
 3781 //  |     |        +--------+
 3782 //  V     |        | old out|      Empty on Intel, window on Sparc
 3783 //        |    old |preserve|      Must be even aligned.
 3784 //        |     SP-+--------+----> Matcher::_old_SP, even aligned
 3785 //        |        |   in   |  3   area for Intel ret address
 3786 //     Owned by    |preserve|      Empty on Sparc.
 3787 //       SELF      +--------+
 3788 //        |        |  pad2  |  2   pad to align old SP
 3789 //        |        +--------+  1
 3790 //        |        | locks  |  0
 3791 //        |        +--------+----> OptoReg::stack0(), even aligned
 3792 //        |        |  pad1  | 11   pad to align new SP
 3793 //        |        +--------+
 3794 //        |        |        | 10
 3795 //        |        | spills |  9   spills
 3796 //        V        |        |  8   (pad0 slot for callee)
 3797 //      -----------+--------+----> Matcher::_out_arg_limit, unaligned
 3798 //        ^        |  out   |  7
 3799 //        |        |  args  |  6   Holes in outgoing args owned by CALLEE
 3800 //     Owned by    +--------+
 3801 //      CALLEE     | new out|  6   Empty on Intel, window on Sparc
 3802 //        |    new |preserve|      Must be even-aligned.
 3803 //        |     SP-+--------+----> Matcher::_new_SP, even aligned
 3804 //        |        |        |
 3805 //
 3806 // Note 1: Only region 8-11 is determined by the allocator.  Region 0-5 is
 3807 //         known from SELF's arguments and the Java calling convention.
 3808 //         Region 6-7 is determined per call site.
 3809 // Note 2: If the calling convention leaves holes in the incoming argument
 3810 //         area, those holes are owned by SELF.  Holes in the outgoing area
 3811 //         are owned by the CALLEE.  Holes should not be necessary in the
 3812 //         incoming area, as the Java calling convention is completely under
 3813 //         the control of the AD file.  Doubles can be sorted and packed to
 3814 //         avoid holes.  Holes in the outgoing arguments may be necessary for
 3815 //         varargs C calling conventions.
 3816 // Note 3: Region 0-3 is even aligned, with pad2 as needed.  Region 3-5 is
 3817 //         even aligned with pad0 as needed.
 3818 //         Region 6 is even aligned.  Region 6-7 is NOT even aligned;
 3819 //           (the latter is true on Intel but is it false on AArch64?)
 3820 //         region 6-11 is even aligned; it may be padded out more so that
 3821 //         the region from SP to FP meets the minimum stack alignment.
 3822 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
 3823 //         alignment.  Region 11, pad1, may be dynamically extended so that
 3824 //         SP meets the minimum alignment.
 3825 
 3826 frame %{
 3827   // These three registers define part of the calling convention
 3828   // between compiled code and the interpreter.
 3829 
 3830   // Inline Cache Register or Method for I2C.
 3831   inline_cache_reg(R12);
 3832 
 3833   // Number of stack slots consumed by locking an object
 3834   sync_stack_slots(2);
 3835 
 3836   // Compiled code's Frame Pointer
 3837   frame_pointer(R31);
 3838 
 3839   // Interpreter stores its frame pointer in a register which is
 3840   // stored to the stack by I2CAdaptors.
 3841   // I2CAdaptors convert from interpreted java to compiled java.
 3842   interpreter_frame_pointer(R29);
 3843 
 3844   // Stack alignment requirement
 3845   stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
 3846 
 3847   // Number of outgoing stack slots killed above the out_preserve_stack_slots
 3848   // for calls to C.  Supports the var-args backing area for register parms.
 3849   varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
 3850 
 3851   // The after-PROLOG location of the return address.  Location of
 3852   // return address specifies a type (REG or STACK) and a number
 3853   // representing the register number (i.e. - use a register name) or
 3854   // stack slot.
 3855   // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
 3856   // Otherwise, it is above the locks and verification slot and alignment word
 3857   // TODO this may well be correct but need to check why that - 2 is there
 3858   // ppc port uses 0 but we definitely need to allow for fixed_slots
 3859   // which folds in the space used for monitors
 3860   return_addr(STACK - 2 +
 3861               align_up((Compile::current()->in_preserve_stack_slots() +
 3862                         Compile::current()->fixed_slots()),
 3863                        stack_alignment_in_slots()));
 3864 
 3865   // Location of compiled Java return values.  Same as C for now.
 3866   return_value
 3867   %{
 3868     // TODO do we allow ideal_reg == Op_RegN???
 3869     assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
 3870            "only return normal values");
 3871 
 3872     static const int lo[Op_RegL + 1] = { // enum name
 3873       0,                                 // Op_Node
 3874       0,                                 // Op_Set
 3875       R0_num,                            // Op_RegN
 3876       R0_num,                            // Op_RegI
 3877       R0_num,                            // Op_RegP
 3878       V0_num,                            // Op_RegF
 3879       V0_num,                            // Op_RegD
 3880       R0_num                             // Op_RegL
 3881     };
 3882 
 3883     static const int hi[Op_RegL + 1] = { // enum name
 3884       0,                                 // Op_Node
 3885       0,                                 // Op_Set
 3886       OptoReg::Bad,                      // Op_RegN
 3887       OptoReg::Bad,                      // Op_RegI
 3888       R0_H_num,                          // Op_RegP
 3889       OptoReg::Bad,                      // Op_RegF
 3890       V0_H_num,                          // Op_RegD
 3891       R0_H_num                           // Op_RegL
 3892     };
 3893 
 3894     return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
 3895   %}
 3896 %}
 3897 
 3898 //----------ATTRIBUTES---------------------------------------------------------
 3899 //----------Operand Attributes-------------------------------------------------
 3900 op_attrib op_cost(1);        // Required cost attribute
 3901 
 3902 //----------Instruction Attributes---------------------------------------------
 3903 ins_attrib ins_cost(INSN_COST); // Required cost attribute
 3904 ins_attrib ins_size(32);        // Required size attribute (in bits)
 3905 ins_attrib ins_short_branch(0); // Required flag: is this instruction
 3906                                 // a non-matching short branch variant
 3907                                 // of some long branch?
 3908 ins_attrib ins_alignment(4);    // Required alignment attribute (must
 3909                                 // be a power of 2) specifies the
 3910                                 // alignment that some part of the
 3911                                 // instruction (not necessarily the
 3912                                 // start) requires.  If > 1, a
 3913                                 // compute_padding() function must be
 3914                                 // provided for the instruction
 3915 
 3916 //----------OPERANDS-----------------------------------------------------------
 3917 // Operand definitions must precede instruction definitions for correct parsing
 3918 // in the ADLC because operands constitute user defined types which are used in
 3919 // instruction definitions.
 3920 
 3921 //----------Simple Operands----------------------------------------------------
 3922 
 3923 // Integer operands 32 bit
 3924 // 32 bit immediate
 3925 operand immI()
 3926 %{
 3927   match(ConI);
 3928 
 3929   op_cost(0);
 3930   format %{ %}
 3931   interface(CONST_INTER);
 3932 %}
 3933 
 3934 // 32 bit zero
 3935 operand immI0()
 3936 %{
 3937   predicate(n->get_int() == 0);
 3938   match(ConI);
 3939 
 3940   op_cost(0);
 3941   format %{ %}
 3942   interface(CONST_INTER);
 3943 %}
 3944 
 3945 // 32 bit unit increment
 3946 operand immI_1()
 3947 %{
 3948   predicate(n->get_int() == 1);
 3949   match(ConI);
 3950 
 3951   op_cost(0);
 3952   format %{ %}
 3953   interface(CONST_INTER);
 3954 %}
 3955 
 3956 // 32 bit unit decrement
 3957 operand immI_M1()
 3958 %{
 3959   predicate(n->get_int() == -1);
 3960   match(ConI);
 3961 
 3962   op_cost(0);
 3963   format %{ %}
 3964   interface(CONST_INTER);
 3965 %}
 3966 
 3967 // Shift values for add/sub extension shift
 3968 operand immIExt()
 3969 %{
 3970   predicate(0 <= n->get_int() && (n->get_int() <= 4));
 3971   match(ConI);
 3972 
 3973   op_cost(0);
 3974   format %{ %}
 3975   interface(CONST_INTER);
 3976 %}
 3977 
 3978 operand immI_gt_1()
 3979 %{
 3980   predicate(n->get_int() > 1);
 3981   match(ConI);
 3982 
 3983   op_cost(0);
 3984   format %{ %}
 3985   interface(CONST_INTER);
 3986 %}
 3987 
 3988 operand immI_le_4()
 3989 %{
 3990   predicate(n->get_int() <= 4);
 3991   match(ConI);
 3992 
 3993   op_cost(0);
 3994   format %{ %}
 3995   interface(CONST_INTER);
 3996 %}
 3997 
 3998 operand immI_16()
 3999 %{
 4000   predicate(n->get_int() == 16);
 4001   match(ConI);
 4002 
 4003   op_cost(0);
 4004   format %{ %}
 4005   interface(CONST_INTER);
 4006 %}
 4007 
 4008 operand immI_24()
 4009 %{
 4010   predicate(n->get_int() == 24);
 4011   match(ConI);
 4012 
 4013   op_cost(0);
 4014   format %{ %}
 4015   interface(CONST_INTER);
 4016 %}
 4017 
 4018 operand immI_32()
 4019 %{
 4020   predicate(n->get_int() == 32);
 4021   match(ConI);
 4022 
 4023   op_cost(0);
 4024   format %{ %}
 4025   interface(CONST_INTER);
 4026 %}
 4027 
 4028 operand immI_48()
 4029 %{
 4030   predicate(n->get_int() == 48);
 4031   match(ConI);
 4032 
 4033   op_cost(0);
 4034   format %{ %}
 4035   interface(CONST_INTER);
 4036 %}
 4037 
 4038 operand immI_56()
 4039 %{
 4040   predicate(n->get_int() == 56);
 4041   match(ConI);
 4042 
 4043   op_cost(0);
 4044   format %{ %}
 4045   interface(CONST_INTER);
 4046 %}
 4047 
 4048 operand immI_255()
 4049 %{
 4050   predicate(n->get_int() == 255);
 4051   match(ConI);
 4052 
 4053   op_cost(0);
 4054   format %{ %}
 4055   interface(CONST_INTER);
 4056 %}
 4057 
 4058 operand immI_65535()
 4059 %{
 4060   predicate(n->get_int() == 65535);
 4061   match(ConI);
 4062 
 4063   op_cost(0);
 4064   format %{ %}
 4065   interface(CONST_INTER);
 4066 %}
 4067 
 4068 operand immI_positive()
 4069 %{
 4070   predicate(n->get_int() > 0);
 4071   match(ConI);
 4072 
 4073   op_cost(0);
 4074   format %{ %}
 4075   interface(CONST_INTER);
 4076 %}
 4077 
 4078 // BoolTest condition for signed compare
 4079 operand immI_cmp_cond()
 4080 %{
 4081   predicate(!Matcher::is_unsigned_booltest_pred(n->get_int()));
 4082   match(ConI);
 4083 
 4084   op_cost(0);
 4085   format %{ %}
 4086   interface(CONST_INTER);
 4087 %}
 4088 
 4089 // BoolTest condition for unsigned compare
 4090 operand immI_cmpU_cond()
 4091 %{
 4092   predicate(Matcher::is_unsigned_booltest_pred(n->get_int()));
 4093   match(ConI);
 4094 
 4095   op_cost(0);
 4096   format %{ %}
 4097   interface(CONST_INTER);
 4098 %}
 4099 
 4100 operand immL_255()
 4101 %{
 4102   predicate(n->get_long() == 255L);
 4103   match(ConL);
 4104 
 4105   op_cost(0);
 4106   format %{ %}
 4107   interface(CONST_INTER);
 4108 %}
 4109 
 4110 operand immL_65535()
 4111 %{
 4112   predicate(n->get_long() == 65535L);
 4113   match(ConL);
 4114 
 4115   op_cost(0);
 4116   format %{ %}
 4117   interface(CONST_INTER);
 4118 %}
 4119 
 4120 operand immL_4294967295()
 4121 %{
 4122   predicate(n->get_long() == 4294967295L);
 4123   match(ConL);
 4124 
 4125   op_cost(0);
 4126   format %{ %}
 4127   interface(CONST_INTER);
 4128 %}
 4129 
 4130 operand immL_bitmask()
 4131 %{
 4132   predicate((n->get_long() != 0)
 4133             && ((n->get_long() & 0xc000000000000000l) == 0)
 4134             && is_power_of_2(n->get_long() + 1));
 4135   match(ConL);
 4136 
 4137   op_cost(0);
 4138   format %{ %}
 4139   interface(CONST_INTER);
 4140 %}
 4141 
 4142 operand immI_bitmask()
 4143 %{
 4144   predicate((n->get_int() != 0)
 4145             && ((n->get_int() & 0xc0000000) == 0)
 4146             && is_power_of_2(n->get_int() + 1));
 4147   match(ConI);
 4148 
 4149   op_cost(0);
 4150   format %{ %}
 4151   interface(CONST_INTER);
 4152 %}
 4153 
 4154 operand immL_positive_bitmaskI()
 4155 %{
 4156   predicate((n->get_long() != 0)
 4157             && ((julong)n->get_long() < 0x80000000ULL)
 4158             && is_power_of_2(n->get_long() + 1));
 4159   match(ConL);
 4160 
 4161   op_cost(0);
 4162   format %{ %}
 4163   interface(CONST_INTER);
 4164 %}
 4165 
 4166 // Scale values for scaled offset addressing modes (up to long but not quad)
 4167 operand immIScale()
 4168 %{
 4169   predicate(0 <= n->get_int() && (n->get_int() <= 3));
 4170   match(ConI);
 4171 
 4172   op_cost(0);
 4173   format %{ %}
 4174   interface(CONST_INTER);
 4175 %}
 4176 
 4177 // 5 bit signed integer
 4178 operand immI5()
 4179 %{
 4180   predicate(Assembler::is_simm(n->get_int(), 5));
 4181   match(ConI);
 4182 
 4183   op_cost(0);
 4184   format %{ %}
 4185   interface(CONST_INTER);
 4186 %}
 4187 
 4188 // 7 bit unsigned integer
 4189 operand immIU7()
 4190 %{
 4191   predicate(Assembler::is_uimm(n->get_int(), 7));
 4192   match(ConI);
 4193 
 4194   op_cost(0);
 4195   format %{ %}
 4196   interface(CONST_INTER);
 4197 %}
 4198 
 4199 // Offset for immediate loads and stores
 4200 operand immIOffset()
 4201 %{
 4202   predicate(n->get_int() >= -256 && n->get_int() <= 65520);
 4203   match(ConI);
 4204 
 4205   op_cost(0);
 4206   format %{ %}
 4207   interface(CONST_INTER);
 4208 %}
 4209 
 4210 operand immLOffset()
 4211 %{
 4212   predicate(n->get_long() >= -256 && n->get_long() <= 65520);
 4213   match(ConL);
 4214 
 4215   op_cost(0);
 4216   format %{ %}
 4217   interface(CONST_INTER);
 4218 %}
 4219 
 4220 // 5 bit signed long integer
 4221 operand immL5()
 4222 %{
 4223   predicate(Assembler::is_simm(n->get_long(), 5));
 4224   match(ConL);
 4225 
 4226   op_cost(0);
 4227   format %{ %}
 4228   interface(CONST_INTER);
 4229 %}
 4230 
 4231 // 7 bit unsigned long integer
 4232 operand immLU7()
 4233 %{
 4234   predicate(Assembler::is_uimm(n->get_long(), 7));
 4235   match(ConL);
 4236 
 4237   op_cost(0);
 4238   format %{ %}
 4239   interface(CONST_INTER);
 4240 %}
 4241 
 4242 // 8 bit signed value.
 4243 operand immI8()
 4244 %{
 4245   predicate(n->get_int() <= 127 && n->get_int() >= -128);
 4246   match(ConI);
 4247 
 4248   op_cost(0);
 4249   format %{ %}
 4250   interface(CONST_INTER);
 4251 %}
 4252 
 4253 // 8 bit signed value (simm8), or #simm8 LSL 8.
 4254 operand immI8_shift8()
 4255 %{
 4256   predicate((n->get_int() <= 127 && n->get_int() >= -128) ||
 4257             (n->get_int() <= 32512 && n->get_int() >= -32768 && (n->get_int() & 0xff) == 0));
 4258   match(ConI);
 4259 
 4260   op_cost(0);
 4261   format %{ %}
 4262   interface(CONST_INTER);
 4263 %}
 4264 
 4265 // 8 bit signed value (simm8), or #simm8 LSL 8.
 4266 operand immL8_shift8()
 4267 %{
 4268   predicate((n->get_long() <= 127 && n->get_long() >= -128) ||
 4269             (n->get_long() <= 32512 && n->get_long() >= -32768 && (n->get_long() & 0xff) == 0));
 4270   match(ConL);
 4271 
 4272   op_cost(0);
 4273   format %{ %}
 4274   interface(CONST_INTER);
 4275 %}
 4276 
 4277 // 8 bit integer valid for vector add sub immediate
 4278 operand immBAddSubV()
 4279 %{
 4280   predicate(n->get_int() <= 255 && n->get_int() >= -255);
 4281   match(ConI);
 4282 
 4283   op_cost(0);
 4284   format %{ %}
 4285   interface(CONST_INTER);
 4286 %}
 4287 
 4288 // 32 bit integer valid for add sub immediate
 4289 operand immIAddSub()
 4290 %{
 4291   predicate(Assembler::operand_valid_for_add_sub_immediate((int64_t)n->get_int()));
 4292   match(ConI);
 4293   op_cost(0);
 4294   format %{ %}
 4295   interface(CONST_INTER);
 4296 %}
 4297 
 4298 // 32 bit integer valid for vector add sub immediate
 4299 operand immIAddSubV()
 4300 %{
 4301   predicate(Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)n->get_int()));
 4302   match(ConI);
 4303 
 4304   op_cost(0);
 4305   format %{ %}
 4306   interface(CONST_INTER);
 4307 %}
 4308 
 4309 // 32 bit unsigned integer valid for logical immediate
 4310 
 4311 operand immBLog()
 4312 %{
 4313   predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerByte, (uint64_t)n->get_int()));
 4314   match(ConI);
 4315 
 4316   op_cost(0);
 4317   format %{ %}
 4318   interface(CONST_INTER);
 4319 %}
 4320 
 4321 operand immSLog()
 4322 %{
 4323   predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerShort, (uint64_t)n->get_int()));
 4324   match(ConI);
 4325 
 4326   op_cost(0);
 4327   format %{ %}
 4328   interface(CONST_INTER);
 4329 %}
 4330 
 4331 operand immILog()
 4332 %{
 4333   predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/true, (uint64_t)n->get_int()));
 4334   match(ConI);
 4335 
 4336   op_cost(0);
 4337   format %{ %}
 4338   interface(CONST_INTER);
 4339 %}
 4340 
 4341 // Integer operands 64 bit
 4342 // 64 bit immediate
 4343 operand immL()
 4344 %{
 4345   match(ConL);
 4346 
 4347   op_cost(0);
 4348   format %{ %}
 4349   interface(CONST_INTER);
 4350 %}
 4351 
 4352 // 64 bit zero
 4353 operand immL0()
 4354 %{
 4355   predicate(n->get_long() == 0);
 4356   match(ConL);
 4357 
 4358   op_cost(0);
 4359   format %{ %}
 4360   interface(CONST_INTER);
 4361 %}
 4362 
 4363 // 64 bit unit decrement
 4364 operand immL_M1()
 4365 %{
 4366   predicate(n->get_long() == -1);
 4367   match(ConL);
 4368 
 4369   op_cost(0);
 4370   format %{ %}
 4371   interface(CONST_INTER);
 4372 %}
 4373 
 4374 // 64 bit integer valid for add sub immediate
 4375 operand immLAddSub()
 4376 %{
 4377   predicate(Assembler::operand_valid_for_add_sub_immediate(n->get_long()));
 4378   match(ConL);
 4379   op_cost(0);
 4380   format %{ %}
 4381   interface(CONST_INTER);
 4382 %}
 4383 
 4384 // 64 bit integer valid for addv subv immediate
 4385 operand immLAddSubV()
 4386 %{
 4387   predicate(Assembler::operand_valid_for_sve_add_sub_immediate(n->get_long()));
 4388   match(ConL);
 4389 
 4390   op_cost(0);
 4391   format %{ %}
 4392   interface(CONST_INTER);
 4393 %}
 4394 
 4395 // 64 bit integer valid for logical immediate
 4396 operand immLLog()
 4397 %{
 4398   predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/false, (uint64_t)n->get_long()));
 4399   match(ConL);
 4400   op_cost(0);
 4401   format %{ %}
 4402   interface(CONST_INTER);
 4403 %}
 4404 
 4405 // Long Immediate: low 32-bit mask
 4406 operand immL_32bits()
 4407 %{
 4408   predicate(n->get_long() == 0xFFFFFFFFL);
 4409   match(ConL);
 4410   op_cost(0);
 4411   format %{ %}
 4412   interface(CONST_INTER);
 4413 %}
 4414 
 4415 // Pointer operands
 4416 // Pointer Immediate
 4417 operand immP()
 4418 %{
 4419   match(ConP);
 4420 
 4421   op_cost(0);
 4422   format %{ %}
 4423   interface(CONST_INTER);
 4424 %}
 4425 
 4426 // nullptr Pointer Immediate
 4427 operand immP0()
 4428 %{
 4429   predicate(n->get_ptr() == 0);
 4430   match(ConP);
 4431 
 4432   op_cost(0);
 4433   format %{ %}
 4434   interface(CONST_INTER);
 4435 %}
 4436 
 4437 // Pointer Immediate One
 4438 // this is used in object initialization (initial object header)
 4439 operand immP_1()
 4440 %{
 4441   predicate(n->get_ptr() == 1);
 4442   match(ConP);
 4443 
 4444   op_cost(0);
 4445   format %{ %}
 4446   interface(CONST_INTER);
 4447 %}
 4448 
 4449 // Card Table Byte Map Base
 4450 operand immByteMapBase()
 4451 %{
 4452   // Get base of card map
 4453   predicate(BarrierSet::barrier_set()->is_a(BarrierSet::CardTableBarrierSet) &&
 4454             (CardTable::CardValue*)n->get_ptr() == ((CardTableBarrierSet*)(BarrierSet::barrier_set()))->card_table()->byte_map_base());
 4455   match(ConP);
 4456 
 4457   op_cost(0);
 4458   format %{ %}
 4459   interface(CONST_INTER);
 4460 %}
 4461 
 4462 // Float and Double operands
 4463 // Double Immediate
 4464 operand immD()
 4465 %{
 4466   match(ConD);
 4467   op_cost(0);
 4468   format %{ %}
 4469   interface(CONST_INTER);
 4470 %}
 4471 
 4472 // Double Immediate: +0.0d
 4473 operand immD0()
 4474 %{
 4475   predicate(jlong_cast(n->getd()) == 0);
 4476   match(ConD);
 4477 
 4478   op_cost(0);
 4479   format %{ %}
 4480   interface(CONST_INTER);
 4481 %}
 4482 
 4483 // constant 'double +0.0'.
 4484 operand immDPacked()
 4485 %{
 4486   predicate(Assembler::operand_valid_for_float_immediate(n->getd()));
 4487   match(ConD);
 4488   op_cost(0);
 4489   format %{ %}
 4490   interface(CONST_INTER);
 4491 %}
 4492 
 4493 // Float Immediate
 4494 operand immF()
 4495 %{
 4496   match(ConF);
 4497   op_cost(0);
 4498   format %{ %}
 4499   interface(CONST_INTER);
 4500 %}
 4501 
 4502 // Float Immediate: +0.0f.
 4503 operand immF0()
 4504 %{
 4505   predicate(jint_cast(n->getf()) == 0);
 4506   match(ConF);
 4507 
 4508   op_cost(0);
 4509   format %{ %}
 4510   interface(CONST_INTER);
 4511 %}
 4512 
 4513 //
 4514 operand immFPacked()
 4515 %{
 4516   predicate(Assembler::operand_valid_for_float_immediate((double)n->getf()));
 4517   match(ConF);
 4518   op_cost(0);
 4519   format %{ %}
 4520   interface(CONST_INTER);
 4521 %}
 4522 
 4523 // Narrow pointer operands
 4524 // Narrow Pointer Immediate
 4525 operand immN()
 4526 %{
 4527   match(ConN);
 4528 
 4529   op_cost(0);
 4530   format %{ %}
 4531   interface(CONST_INTER);
 4532 %}
 4533 
 4534 // Narrow nullptr Pointer Immediate
 4535 operand immN0()
 4536 %{
 4537   predicate(n->get_narrowcon() == 0);
 4538   match(ConN);
 4539 
 4540   op_cost(0);
 4541   format %{ %}
 4542   interface(CONST_INTER);
 4543 %}
 4544 
 4545 operand immNKlass()
 4546 %{
 4547   match(ConNKlass);
 4548 
 4549   op_cost(0);
 4550   format %{ %}
 4551   interface(CONST_INTER);
 4552 %}
 4553 
 4554 // Integer 32 bit Register Operands
 4555 // Integer 32 bitRegister (excludes SP)
 4556 operand iRegI()
 4557 %{
 4558   constraint(ALLOC_IN_RC(any_reg32));
 4559   match(RegI);
 4560   match(iRegINoSp);
 4561   op_cost(0);
 4562   format %{ %}
 4563   interface(REG_INTER);
 4564 %}
 4565 
 4566 // Integer 32 bit Register not Special
 4567 operand iRegINoSp()
 4568 %{
 4569   constraint(ALLOC_IN_RC(no_special_reg32));
 4570   match(RegI);
 4571   op_cost(0);
 4572   format %{ %}
 4573   interface(REG_INTER);
 4574 %}
 4575 
 4576 // Integer 64 bit Register Operands
 4577 // Integer 64 bit Register (includes SP)
 4578 operand iRegL()
 4579 %{
 4580   constraint(ALLOC_IN_RC(any_reg));
 4581   match(RegL);
 4582   match(iRegLNoSp);
 4583   op_cost(0);
 4584   format %{ %}
 4585   interface(REG_INTER);
 4586 %}
 4587 
 4588 // Integer 64 bit Register not Special
 4589 operand iRegLNoSp()
 4590 %{
 4591   constraint(ALLOC_IN_RC(no_special_reg));
 4592   match(RegL);
 4593   match(iRegL_R0);
 4594   format %{ %}
 4595   interface(REG_INTER);
 4596 %}
 4597 
 4598 // Pointer Register Operands
 4599 // Pointer Register
 4600 operand iRegP()
 4601 %{
 4602   constraint(ALLOC_IN_RC(ptr_reg));
 4603   match(RegP);
 4604   match(iRegPNoSp);
 4605   match(iRegP_R0);
 4606   //match(iRegP_R2);
 4607   //match(iRegP_R4);
 4608   match(iRegP_R5);
 4609   match(thread_RegP);
 4610   op_cost(0);
 4611   format %{ %}
 4612   interface(REG_INTER);
 4613 %}
 4614 
 4615 // Pointer 64 bit Register not Special
 4616 operand iRegPNoSp()
 4617 %{
 4618   constraint(ALLOC_IN_RC(no_special_ptr_reg));
 4619   match(RegP);
 4620   // match(iRegP);
 4621   // match(iRegP_R0);
 4622   // match(iRegP_R2);
 4623   // match(iRegP_R4);
 4624   // match(iRegP_R5);
 4625   // match(thread_RegP);
 4626   op_cost(0);
 4627   format %{ %}
 4628   interface(REG_INTER);
 4629 %}
 4630 
 4631 // This operand is not allowed to use rfp even if
 4632 // rfp is not used to hold the frame pointer.
 4633 operand iRegPNoSpNoRfp()
 4634 %{
 4635   constraint(ALLOC_IN_RC(no_special_no_rfp_ptr_reg));
 4636   match(RegP);
 4637   match(iRegPNoSp);
 4638   op_cost(0);
 4639   format %{ %}
 4640   interface(REG_INTER);
 4641 %}
 4642 
 4643 // Pointer 64 bit Register R0 only
 4644 operand iRegP_R0()
 4645 %{
 4646   constraint(ALLOC_IN_RC(r0_reg));
 4647   match(RegP);
 4648   // match(iRegP);
 4649   match(iRegPNoSp);
 4650   op_cost(0);
 4651   format %{ %}
 4652   interface(REG_INTER);
 4653 %}
 4654 
 4655 // Pointer 64 bit Register R1 only
 4656 operand iRegP_R1()
 4657 %{
 4658   constraint(ALLOC_IN_RC(r1_reg));
 4659   match(RegP);
 4660   // match(iRegP);
 4661   match(iRegPNoSp);
 4662   op_cost(0);
 4663   format %{ %}
 4664   interface(REG_INTER);
 4665 %}
 4666 
 4667 // Pointer 64 bit Register R2 only
 4668 operand iRegP_R2()
 4669 %{
 4670   constraint(ALLOC_IN_RC(r2_reg));
 4671   match(RegP);
 4672   // match(iRegP);
 4673   match(iRegPNoSp);
 4674   op_cost(0);
 4675   format %{ %}
 4676   interface(REG_INTER);
 4677 %}
 4678 
 4679 // Pointer 64 bit Register R3 only
 4680 operand iRegP_R3()
 4681 %{
 4682   constraint(ALLOC_IN_RC(r3_reg));
 4683   match(RegP);
 4684   // match(iRegP);
 4685   match(iRegPNoSp);
 4686   op_cost(0);
 4687   format %{ %}
 4688   interface(REG_INTER);
 4689 %}
 4690 
 4691 // Pointer 64 bit Register R4 only
 4692 operand iRegP_R4()
 4693 %{
 4694   constraint(ALLOC_IN_RC(r4_reg));
 4695   match(RegP);
 4696   // match(iRegP);
 4697   match(iRegPNoSp);
 4698   op_cost(0);
 4699   format %{ %}
 4700   interface(REG_INTER);
 4701 %}
 4702 
 4703 // Pointer 64 bit Register R5 only
 4704 operand iRegP_R5()
 4705 %{
 4706   constraint(ALLOC_IN_RC(r5_reg));
 4707   match(RegP);
 4708   // match(iRegP);
 4709   match(iRegPNoSp);
 4710   op_cost(0);
 4711   format %{ %}
 4712   interface(REG_INTER);
 4713 %}
 4714 
 4715 // Pointer 64 bit Register R10 only
 4716 operand iRegP_R10()
 4717 %{
 4718   constraint(ALLOC_IN_RC(r10_reg));
 4719   match(RegP);
 4720   // match(iRegP);
 4721   match(iRegPNoSp);
 4722   op_cost(0);
 4723   format %{ %}
 4724   interface(REG_INTER);
 4725 %}
 4726 
 4727 // Long 64 bit Register R0 only
 4728 operand iRegL_R0()
 4729 %{
 4730   constraint(ALLOC_IN_RC(r0_reg));
 4731   match(RegL);
 4732   match(iRegLNoSp);
 4733   op_cost(0);
 4734   format %{ %}
 4735   interface(REG_INTER);
 4736 %}
 4737 
 4738 // Long 64 bit Register R11 only
 4739 operand iRegL_R11()
 4740 %{
 4741   constraint(ALLOC_IN_RC(r11_reg));
 4742   match(RegL);
 4743   match(iRegLNoSp);
 4744   op_cost(0);
 4745   format %{ %}
 4746   interface(REG_INTER);
 4747 %}
 4748 
 4749 // Register R0 only
 4750 operand iRegI_R0()
 4751 %{
 4752   constraint(ALLOC_IN_RC(int_r0_reg));
 4753   match(RegI);
 4754   match(iRegINoSp);
 4755   op_cost(0);
 4756   format %{ %}
 4757   interface(REG_INTER);
 4758 %}
 4759 
 4760 // Register R2 only
 4761 operand iRegI_R2()
 4762 %{
 4763   constraint(ALLOC_IN_RC(int_r2_reg));
 4764   match(RegI);
 4765   match(iRegINoSp);
 4766   op_cost(0);
 4767   format %{ %}
 4768   interface(REG_INTER);
 4769 %}
 4770 
 4771 // Register R3 only
 4772 operand iRegI_R3()
 4773 %{
 4774   constraint(ALLOC_IN_RC(int_r3_reg));
 4775   match(RegI);
 4776   match(iRegINoSp);
 4777   op_cost(0);
 4778   format %{ %}
 4779   interface(REG_INTER);
 4780 %}
 4781 
 4782 
 4783 // Register R4 only
 4784 operand iRegI_R4()
 4785 %{
 4786   constraint(ALLOC_IN_RC(int_r4_reg));
 4787   match(RegI);
 4788   match(iRegINoSp);
 4789   op_cost(0);
 4790   format %{ %}
 4791   interface(REG_INTER);
 4792 %}
 4793 
 4794 
 4795 // Pointer Register Operands
 4796 // Narrow Pointer Register
 4797 operand iRegN()
 4798 %{
 4799   constraint(ALLOC_IN_RC(any_reg32));
 4800   match(RegN);
 4801   match(iRegNNoSp);
 4802   op_cost(0);
 4803   format %{ %}
 4804   interface(REG_INTER);
 4805 %}
 4806 
 4807 // Integer 64 bit Register not Special
 4808 operand iRegNNoSp()
 4809 %{
 4810   constraint(ALLOC_IN_RC(no_special_reg32));
 4811   match(RegN);
 4812   op_cost(0);
 4813   format %{ %}
 4814   interface(REG_INTER);
 4815 %}
 4816 
 4817 // Float Register
 4818 // Float register operands
 4819 operand vRegF()
 4820 %{
 4821   constraint(ALLOC_IN_RC(float_reg));
 4822   match(RegF);
 4823 
 4824   op_cost(0);
 4825   format %{ %}
 4826   interface(REG_INTER);
 4827 %}
 4828 
 4829 // Double Register
 4830 // Double register operands
 4831 operand vRegD()
 4832 %{
 4833   constraint(ALLOC_IN_RC(double_reg));
 4834   match(RegD);
 4835 
 4836   op_cost(0);
 4837   format %{ %}
 4838   interface(REG_INTER);
 4839 %}
 4840 
 4841 // Generic vector class. This will be used for
 4842 // all vector operands, including NEON and SVE.
 4843 operand vReg()
 4844 %{
 4845   constraint(ALLOC_IN_RC(dynamic));
 4846   match(VecA);
 4847   match(VecD);
 4848   match(VecX);
 4849 
 4850   op_cost(0);
 4851   format %{ %}
 4852   interface(REG_INTER);
 4853 %}
 4854 
 4855 operand vecA()
 4856 %{
 4857   constraint(ALLOC_IN_RC(vectora_reg));
 4858   match(VecA);
 4859 
 4860   op_cost(0);
 4861   format %{ %}
 4862   interface(REG_INTER);
 4863 %}
 4864 
 4865 operand vecD()
 4866 %{
 4867   constraint(ALLOC_IN_RC(vectord_reg));
 4868   match(VecD);
 4869 
 4870   op_cost(0);
 4871   format %{ %}
 4872   interface(REG_INTER);
 4873 %}
 4874 
 4875 operand vecX()
 4876 %{
 4877   constraint(ALLOC_IN_RC(vectorx_reg));
 4878   match(VecX);
 4879 
 4880   op_cost(0);
 4881   format %{ %}
 4882   interface(REG_INTER);
 4883 %}
 4884 
 4885 operand vRegD_V0()
 4886 %{
 4887   constraint(ALLOC_IN_RC(v0_reg));
 4888   match(RegD);
 4889   op_cost(0);
 4890   format %{ %}
 4891   interface(REG_INTER);
 4892 %}
 4893 
 4894 operand vRegD_V1()
 4895 %{
 4896   constraint(ALLOC_IN_RC(v1_reg));
 4897   match(RegD);
 4898   op_cost(0);
 4899   format %{ %}
 4900   interface(REG_INTER);
 4901 %}
 4902 
 4903 operand vRegD_V2()
 4904 %{
 4905   constraint(ALLOC_IN_RC(v2_reg));
 4906   match(RegD);
 4907   op_cost(0);
 4908   format %{ %}
 4909   interface(REG_INTER);
 4910 %}
 4911 
 4912 operand vRegD_V3()
 4913 %{
 4914   constraint(ALLOC_IN_RC(v3_reg));
 4915   match(RegD);
 4916   op_cost(0);
 4917   format %{ %}
 4918   interface(REG_INTER);
 4919 %}
 4920 
 4921 operand vRegD_V4()
 4922 %{
 4923   constraint(ALLOC_IN_RC(v4_reg));
 4924   match(RegD);
 4925   op_cost(0);
 4926   format %{ %}
 4927   interface(REG_INTER);
 4928 %}
 4929 
 4930 operand vRegD_V5()
 4931 %{
 4932   constraint(ALLOC_IN_RC(v5_reg));
 4933   match(RegD);
 4934   op_cost(0);
 4935   format %{ %}
 4936   interface(REG_INTER);
 4937 %}
 4938 
 4939 operand vRegD_V6()
 4940 %{
 4941   constraint(ALLOC_IN_RC(v6_reg));
 4942   match(RegD);
 4943   op_cost(0);
 4944   format %{ %}
 4945   interface(REG_INTER);
 4946 %}
 4947 
 4948 operand vRegD_V7()
 4949 %{
 4950   constraint(ALLOC_IN_RC(v7_reg));
 4951   match(RegD);
 4952   op_cost(0);
 4953   format %{ %}
 4954   interface(REG_INTER);
 4955 %}
 4956 
 4957 operand pReg()
 4958 %{
 4959   constraint(ALLOC_IN_RC(pr_reg));
 4960   match(RegVectMask);
 4961   match(pRegGov);
 4962   op_cost(0);
 4963   format %{ %}
 4964   interface(REG_INTER);
 4965 %}
 4966 
 4967 operand pRegGov()
 4968 %{
 4969   constraint(ALLOC_IN_RC(gov_pr));
 4970   match(RegVectMask);
 4971   match(pReg);
 4972   op_cost(0);
 4973   format %{ %}
 4974   interface(REG_INTER);
 4975 %}
 4976 
 4977 operand pRegGov_P0()
 4978 %{
 4979   constraint(ALLOC_IN_RC(p0_reg));
 4980   match(RegVectMask);
 4981   op_cost(0);
 4982   format %{ %}
 4983   interface(REG_INTER);
 4984 %}
 4985 
 4986 operand pRegGov_P1()
 4987 %{
 4988   constraint(ALLOC_IN_RC(p1_reg));
 4989   match(RegVectMask);
 4990   op_cost(0);
 4991   format %{ %}
 4992   interface(REG_INTER);
 4993 %}
 4994 
 4995 // Flags register, used as output of signed compare instructions
 4996 
 4997 // note that on AArch64 we also use this register as the output for
 4998 // for floating point compare instructions (CmpF CmpD). this ensures
 4999 // that ordered inequality tests use GT, GE, LT or LE none of which
 5000 // pass through cases where the result is unordered i.e. one or both
 5001 // inputs to the compare is a NaN. this means that the ideal code can
 5002 // replace e.g. a GT with an LE and not end up capturing the NaN case
 5003 // (where the comparison should always fail). EQ and NE tests are
 5004 // always generated in ideal code so that unordered folds into the NE
 5005 // case, matching the behaviour of AArch64 NE.
 5006 //
 5007 // This differs from x86 where the outputs of FP compares use a
 5008 // special FP flags registers and where compares based on this
 5009 // register are distinguished into ordered inequalities (cmpOpUCF) and
 5010 // EQ/NEQ tests (cmpOpUCF2). x86 has to special case the latter tests
 5011 // to explicitly handle the unordered case in branches. x86 also has
 5012 // to include extra CMoveX rules to accept a cmpOpUCF input.
 5013 
 5014 operand rFlagsReg()
 5015 %{
 5016   constraint(ALLOC_IN_RC(int_flags));
 5017   match(RegFlags);
 5018 
 5019   op_cost(0);
 5020   format %{ "RFLAGS" %}
 5021   interface(REG_INTER);
 5022 %}
 5023 
 5024 // Flags register, used as output of unsigned compare instructions
 5025 operand rFlagsRegU()
 5026 %{
 5027   constraint(ALLOC_IN_RC(int_flags));
 5028   match(RegFlags);
 5029 
 5030   op_cost(0);
 5031   format %{ "RFLAGSU" %}
 5032   interface(REG_INTER);
 5033 %}
 5034 
 5035 // Special Registers
 5036 
 5037 // Method Register
 5038 operand inline_cache_RegP(iRegP reg)
 5039 %{
 5040   constraint(ALLOC_IN_RC(method_reg)); // inline_cache_reg
 5041   match(reg);
 5042   match(iRegPNoSp);
 5043   op_cost(0);
 5044   format %{ %}
 5045   interface(REG_INTER);
 5046 %}
 5047 
 5048 // Thread Register
 5049 operand thread_RegP(iRegP reg)
 5050 %{
 5051   constraint(ALLOC_IN_RC(thread_reg)); // link_reg
 5052   match(reg);
 5053   op_cost(0);
 5054   format %{ %}
 5055   interface(REG_INTER);
 5056 %}
 5057 
 5058 //----------Memory Operands----------------------------------------------------
 5059 
 5060 operand indirect(iRegP reg)
 5061 %{
 5062   constraint(ALLOC_IN_RC(ptr_reg));
 5063   match(reg);
 5064   op_cost(0);
 5065   format %{ "[$reg]" %}
 5066   interface(MEMORY_INTER) %{
 5067     base($reg);
 5068     index(0xffffffff);
 5069     scale(0x0);
 5070     disp(0x0);
 5071   %}
 5072 %}
 5073 
 5074 operand indIndexScaledI2L(iRegP reg, iRegI ireg, immIScale scale)
 5075 %{
 5076   constraint(ALLOC_IN_RC(ptr_reg));
 5077   predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 5078   match(AddP reg (LShiftL (ConvI2L ireg) scale));
 5079   op_cost(0);
 5080   format %{ "$reg, $ireg sxtw($scale), 0, I2L" %}
 5081   interface(MEMORY_INTER) %{
 5082     base($reg);
 5083     index($ireg);
 5084     scale($scale);
 5085     disp(0x0);
 5086   %}
 5087 %}
 5088 
 5089 operand indIndexScaled(iRegP reg, iRegL lreg, immIScale scale)
 5090 %{
 5091   constraint(ALLOC_IN_RC(ptr_reg));
 5092   predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 5093   match(AddP reg (LShiftL lreg scale));
 5094   op_cost(0);
 5095   format %{ "$reg, $lreg lsl($scale)" %}
 5096   interface(MEMORY_INTER) %{
 5097     base($reg);
 5098     index($lreg);
 5099     scale($scale);
 5100     disp(0x0);
 5101   %}
 5102 %}
 5103 
 5104 operand indIndexI2L(iRegP reg, iRegI ireg)
 5105 %{
 5106   constraint(ALLOC_IN_RC(ptr_reg));
 5107   match(AddP reg (ConvI2L ireg));
 5108   op_cost(0);
 5109   format %{ "$reg, $ireg, 0, I2L" %}
 5110   interface(MEMORY_INTER) %{
 5111     base($reg);
 5112     index($ireg);
 5113     scale(0x0);
 5114     disp(0x0);
 5115   %}
 5116 %}
 5117 
 5118 operand indIndex(iRegP reg, iRegL lreg)
 5119 %{
 5120   constraint(ALLOC_IN_RC(ptr_reg));
 5121   match(AddP reg lreg);
 5122   op_cost(0);
 5123   format %{ "$reg, $lreg" %}
 5124   interface(MEMORY_INTER) %{
 5125     base($reg);
 5126     index($lreg);
 5127     scale(0x0);
 5128     disp(0x0);
 5129   %}
 5130 %}
 5131 
 5132 operand indOffI(iRegP reg, immIOffset off)
 5133 %{
 5134   constraint(ALLOC_IN_RC(ptr_reg));
 5135   match(AddP reg off);
 5136   op_cost(0);
 5137   format %{ "[$reg, $off]" %}
 5138   interface(MEMORY_INTER) %{
 5139     base($reg);
 5140     index(0xffffffff);
 5141     scale(0x0);
 5142     disp($off);
 5143   %}
 5144 %}
 5145 
 5146 operand indOffL(iRegP reg, immLOffset off)
 5147 %{
 5148   constraint(ALLOC_IN_RC(ptr_reg));
 5149   match(AddP reg off);
 5150   op_cost(0);
 5151   format %{ "[$reg, $off]" %}
 5152   interface(MEMORY_INTER) %{
 5153     base($reg);
 5154     index(0xffffffff);
 5155     scale(0x0);
 5156     disp($off);
 5157   %}
 5158 %}
 5159 
 5160 operand indirectX2P(iRegL reg)
 5161 %{
 5162   constraint(ALLOC_IN_RC(ptr_reg));
 5163   match(CastX2P reg);
 5164   op_cost(0);
 5165   format %{ "[$reg]\t# long -> ptr" %}
 5166   interface(MEMORY_INTER) %{
 5167     base($reg);
 5168     index(0xffffffff);
 5169     scale(0x0);
 5170     disp(0x0);
 5171   %}
 5172 %}
 5173 
 5174 operand indOffX2P(iRegL reg, immLOffset off)
 5175 %{
 5176   constraint(ALLOC_IN_RC(ptr_reg));
 5177   match(AddP (CastX2P reg) off);
 5178   op_cost(0);
 5179   format %{ "[$reg, $off]\t# long -> ptr" %}
 5180   interface(MEMORY_INTER) %{
 5181     base($reg);
 5182     index(0xffffffff);
 5183     scale(0x0);
 5184     disp($off);
 5185   %}
 5186 %}
 5187 
 5188 operand indirectN(iRegN reg)
 5189 %{
 5190   predicate(CompressedOops::shift() == 0);
 5191   constraint(ALLOC_IN_RC(ptr_reg));
 5192   match(DecodeN reg);
 5193   op_cost(0);
 5194   format %{ "[$reg]\t# narrow" %}
 5195   interface(MEMORY_INTER) %{
 5196     base($reg);
 5197     index(0xffffffff);
 5198     scale(0x0);
 5199     disp(0x0);
 5200   %}
 5201 %}
 5202 
 5203 operand indIndexScaledI2LN(iRegN reg, iRegI ireg, immIScale scale)
 5204 %{
 5205   predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 5206   constraint(ALLOC_IN_RC(ptr_reg));
 5207   match(AddP (DecodeN reg) (LShiftL (ConvI2L ireg) scale));
 5208   op_cost(0);
 5209   format %{ "$reg, $ireg sxtw($scale), 0, I2L\t# narrow" %}
 5210   interface(MEMORY_INTER) %{
 5211     base($reg);
 5212     index($ireg);
 5213     scale($scale);
 5214     disp(0x0);
 5215   %}
 5216 %}
 5217 
 5218 operand indIndexScaledN(iRegN reg, iRegL lreg, immIScale scale)
 5219 %{
 5220   predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 5221   constraint(ALLOC_IN_RC(ptr_reg));
 5222   match(AddP (DecodeN reg) (LShiftL lreg scale));
 5223   op_cost(0);
 5224   format %{ "$reg, $lreg lsl($scale)\t# narrow" %}
 5225   interface(MEMORY_INTER) %{
 5226     base($reg);
 5227     index($lreg);
 5228     scale($scale);
 5229     disp(0x0);
 5230   %}
 5231 %}
 5232 
 5233 operand indIndexI2LN(iRegN reg, iRegI ireg)
 5234 %{
 5235   predicate(CompressedOops::shift() == 0);
 5236   constraint(ALLOC_IN_RC(ptr_reg));
 5237   match(AddP (DecodeN reg) (ConvI2L ireg));
 5238   op_cost(0);
 5239   format %{ "$reg, $ireg, 0, I2L\t# narrow" %}
 5240   interface(MEMORY_INTER) %{
 5241     base($reg);
 5242     index($ireg);
 5243     scale(0x0);
 5244     disp(0x0);
 5245   %}
 5246 %}
 5247 
 5248 operand indIndexN(iRegN reg, iRegL lreg)
 5249 %{
 5250   predicate(CompressedOops::shift() == 0);
 5251   constraint(ALLOC_IN_RC(ptr_reg));
 5252   match(AddP (DecodeN reg) lreg);
 5253   op_cost(0);
 5254   format %{ "$reg, $lreg\t# narrow" %}
 5255   interface(MEMORY_INTER) %{
 5256     base($reg);
 5257     index($lreg);
 5258     scale(0x0);
 5259     disp(0x0);
 5260   %}
 5261 %}
 5262 
 5263 operand indOffIN(iRegN reg, immIOffset off)
 5264 %{
 5265   predicate(CompressedOops::shift() == 0);
 5266   constraint(ALLOC_IN_RC(ptr_reg));
 5267   match(AddP (DecodeN reg) off);
 5268   op_cost(0);
 5269   format %{ "[$reg, $off]\t# narrow" %}
 5270   interface(MEMORY_INTER) %{
 5271     base($reg);
 5272     index(0xffffffff);
 5273     scale(0x0);
 5274     disp($off);
 5275   %}
 5276 %}
 5277 
 5278 operand indOffLN(iRegN reg, immLOffset off)
 5279 %{
 5280   predicate(CompressedOops::shift() == 0);
 5281   constraint(ALLOC_IN_RC(ptr_reg));
 5282   match(AddP (DecodeN reg) off);
 5283   op_cost(0);
 5284   format %{ "[$reg, $off]\t# narrow" %}
 5285   interface(MEMORY_INTER) %{
 5286     base($reg);
 5287     index(0xffffffff);
 5288     scale(0x0);
 5289     disp($off);
 5290   %}
 5291 %}
 5292 
 5293 
 5294 //----------Special Memory Operands--------------------------------------------
 5295 // Stack Slot Operand - This operand is used for loading and storing temporary
 5296 //                      values on the stack where a match requires a value to
 5297 //                      flow through memory.
 5298 operand stackSlotP(sRegP reg)
 5299 %{
 5300   constraint(ALLOC_IN_RC(stack_slots));
 5301   op_cost(100);
 5302   // No match rule because this operand is only generated in matching
 5303   // match(RegP);
 5304   format %{ "[$reg]" %}
 5305   interface(MEMORY_INTER) %{
 5306     base(0x1e);  // RSP
 5307     index(0x0);  // No Index
 5308     scale(0x0);  // No Scale
 5309     disp($reg);  // Stack Offset
 5310   %}
 5311 %}
 5312 
 5313 operand stackSlotI(sRegI reg)
 5314 %{
 5315   constraint(ALLOC_IN_RC(stack_slots));
 5316   // No match rule because this operand is only generated in matching
 5317   // match(RegI);
 5318   format %{ "[$reg]" %}
 5319   interface(MEMORY_INTER) %{
 5320     base(0x1e);  // RSP
 5321     index(0x0);  // No Index
 5322     scale(0x0);  // No Scale
 5323     disp($reg);  // Stack Offset
 5324   %}
 5325 %}
 5326 
 5327 operand stackSlotF(sRegF reg)
 5328 %{
 5329   constraint(ALLOC_IN_RC(stack_slots));
 5330   // No match rule because this operand is only generated in matching
 5331   // match(RegF);
 5332   format %{ "[$reg]" %}
 5333   interface(MEMORY_INTER) %{
 5334     base(0x1e);  // RSP
 5335     index(0x0);  // No Index
 5336     scale(0x0);  // No Scale
 5337     disp($reg);  // Stack Offset
 5338   %}
 5339 %}
 5340 
 5341 operand stackSlotD(sRegD reg)
 5342 %{
 5343   constraint(ALLOC_IN_RC(stack_slots));
 5344   // No match rule because this operand is only generated in matching
 5345   // match(RegD);
 5346   format %{ "[$reg]" %}
 5347   interface(MEMORY_INTER) %{
 5348     base(0x1e);  // RSP
 5349     index(0x0);  // No Index
 5350     scale(0x0);  // No Scale
 5351     disp($reg);  // Stack Offset
 5352   %}
 5353 %}
 5354 
 5355 operand stackSlotL(sRegL reg)
 5356 %{
 5357   constraint(ALLOC_IN_RC(stack_slots));
 5358   // No match rule because this operand is only generated in matching
 5359   // match(RegL);
 5360   format %{ "[$reg]" %}
 5361   interface(MEMORY_INTER) %{
 5362     base(0x1e);  // RSP
 5363     index(0x0);  // No Index
 5364     scale(0x0);  // No Scale
 5365     disp($reg);  // Stack Offset
 5366   %}
 5367 %}
 5368 
 5369 // Operands for expressing Control Flow
 5370 // NOTE: Label is a predefined operand which should not be redefined in
 5371 //       the AD file. It is generically handled within the ADLC.
 5372 
 5373 //----------Conditional Branch Operands----------------------------------------
 5374 // Comparison Op  - This is the operation of the comparison, and is limited to
 5375 //                  the following set of codes:
 5376 //                  L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
 5377 //
 5378 // Other attributes of the comparison, such as unsignedness, are specified
 5379 // by the comparison instruction that sets a condition code flags register.
 5380 // That result is represented by a flags operand whose subtype is appropriate
 5381 // to the unsignedness (etc.) of the comparison.
 5382 //
 5383 // Later, the instruction which matches both the Comparison Op (a Bool) and
 5384 // the flags (produced by the Cmp) specifies the coding of the comparison op
 5385 // by matching a specific subtype of Bool operand below, such as cmpOpU.
 5386 
 5387 // used for signed integral comparisons and fp comparisons
 5388 
 5389 operand cmpOp()
 5390 %{
 5391   match(Bool);
 5392 
 5393   format %{ "" %}
 5394   interface(COND_INTER) %{
 5395     equal(0x0, "eq");
 5396     not_equal(0x1, "ne");
 5397     less(0xb, "lt");
 5398     greater_equal(0xa, "ge");
 5399     less_equal(0xd, "le");
 5400     greater(0xc, "gt");
 5401     overflow(0x6, "vs");
 5402     no_overflow(0x7, "vc");
 5403   %}
 5404 %}
 5405 
 5406 // used for unsigned integral comparisons
 5407 
 5408 operand cmpOpU()
 5409 %{
 5410   match(Bool);
 5411 
 5412   format %{ "" %}
 5413   interface(COND_INTER) %{
 5414     equal(0x0, "eq");
 5415     not_equal(0x1, "ne");
 5416     less(0x3, "lo");
 5417     greater_equal(0x2, "hs");
 5418     less_equal(0x9, "ls");
 5419     greater(0x8, "hi");
 5420     overflow(0x6, "vs");
 5421     no_overflow(0x7, "vc");
 5422   %}
 5423 %}
 5424 
 5425 // used for certain integral comparisons which can be
 5426 // converted to cbxx or tbxx instructions
 5427 
 5428 operand cmpOpEqNe()
 5429 %{
 5430   match(Bool);
 5431   op_cost(0);
 5432   predicate(n->as_Bool()->_test._test == BoolTest::ne
 5433             || n->as_Bool()->_test._test == BoolTest::eq);
 5434 
 5435   format %{ "" %}
 5436   interface(COND_INTER) %{
 5437     equal(0x0, "eq");
 5438     not_equal(0x1, "ne");
 5439     less(0xb, "lt");
 5440     greater_equal(0xa, "ge");
 5441     less_equal(0xd, "le");
 5442     greater(0xc, "gt");
 5443     overflow(0x6, "vs");
 5444     no_overflow(0x7, "vc");
 5445   %}
 5446 %}
 5447 
 5448 // used for certain integral comparisons which can be
 5449 // converted to cbxx or tbxx instructions
 5450 
 5451 operand cmpOpLtGe()
 5452 %{
 5453   match(Bool);
 5454   op_cost(0);
 5455 
 5456   predicate(n->as_Bool()->_test._test == BoolTest::lt
 5457             || n->as_Bool()->_test._test == BoolTest::ge);
 5458 
 5459   format %{ "" %}
 5460   interface(COND_INTER) %{
 5461     equal(0x0, "eq");
 5462     not_equal(0x1, "ne");
 5463     less(0xb, "lt");
 5464     greater_equal(0xa, "ge");
 5465     less_equal(0xd, "le");
 5466     greater(0xc, "gt");
 5467     overflow(0x6, "vs");
 5468     no_overflow(0x7, "vc");
 5469   %}
 5470 %}
 5471 
 5472 // used for certain unsigned integral comparisons which can be
 5473 // converted to cbxx or tbxx instructions
 5474 
 5475 operand cmpOpUEqNeLeGt()
 5476 %{
 5477   match(Bool);
 5478   op_cost(0);
 5479 
 5480   predicate(n->as_Bool()->_test._test == BoolTest::eq ||
 5481             n->as_Bool()->_test._test == BoolTest::ne ||
 5482             n->as_Bool()->_test._test == BoolTest::le ||
 5483             n->as_Bool()->_test._test == BoolTest::gt);
 5484 
 5485   format %{ "" %}
 5486   interface(COND_INTER) %{
 5487     equal(0x0, "eq");
 5488     not_equal(0x1, "ne");
 5489     less(0x3, "lo");
 5490     greater_equal(0x2, "hs");
 5491     less_equal(0x9, "ls");
 5492     greater(0x8, "hi");
 5493     overflow(0x6, "vs");
 5494     no_overflow(0x7, "vc");
 5495   %}
 5496 %}
 5497 
 5498 // Special operand allowing long args to int ops to be truncated for free
 5499 
 5500 operand iRegL2I(iRegL reg) %{
 5501 
 5502   op_cost(0);
 5503 
 5504   match(ConvL2I reg);
 5505 
 5506   format %{ "l2i($reg)" %}
 5507 
 5508   interface(REG_INTER)
 5509 %}
 5510 
 5511 operand iRegL2P(iRegL reg) %{
 5512 
 5513   op_cost(0);
 5514 
 5515   match(CastX2P reg);
 5516 
 5517   format %{ "l2p($reg)" %}
 5518 
 5519   interface(REG_INTER)
 5520 %}
 5521 
 5522 opclass vmem(indirect, indIndex, indOffI, indOffL, indOffIN, indOffLN);
 5523 
 5524 //----------OPERAND CLASSES----------------------------------------------------
 5525 // Operand Classes are groups of operands that are used as to simplify
 5526 // instruction definitions by not requiring the AD writer to specify
 5527 // separate instructions for every form of operand when the
 5528 // instruction accepts multiple operand types with the same basic
 5529 // encoding and format. The classic case of this is memory operands.
 5530 
 5531 // memory is used to define read/write location for load/store
 5532 // instruction defs. we can turn a memory op into an Address
 5533 
 5534 opclass memory(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI, indOffL,
 5535                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN,
 5536                indOffLN, indirectX2P, indOffX2P);
 5537 
 5538 // iRegIorL2I is used for src inputs in rules for 32 bit int (I)
 5539 // operations. it allows the src to be either an iRegI or a (ConvL2I
 5540 // iRegL). in the latter case the l2i normally planted for a ConvL2I
 5541 // can be elided because the 32-bit instruction will just employ the
 5542 // lower 32 bits anyway.
 5543 //
 5544 // n.b. this does not elide all L2I conversions. if the truncated
 5545 // value is consumed by more than one operation then the ConvL2I
 5546 // cannot be bundled into the consuming nodes so an l2i gets planted
 5547 // (actually a movw $dst $src) and the downstream instructions consume
 5548 // the result of the l2i as an iRegI input. That's a shame since the
 5549 // movw is actually redundant but its not too costly.
 5550 
 5551 opclass iRegIorL2I(iRegI, iRegL2I);
 5552 opclass iRegPorL2P(iRegP, iRegL2P);
 5553 
 5554 //----------PIPELINE-----------------------------------------------------------
 5555 // Rules which define the behavior of the target architectures pipeline.
 5556 
 5557 // For specific pipelines, eg A53, define the stages of that pipeline
 5558 //pipe_desc(ISS, EX1, EX2, WR);
 5559 #define ISS S0
 5560 #define EX1 S1
 5561 #define EX2 S2
 5562 #define WR  S3
 5563 
 5564 // Integer ALU reg operation
 5565 pipeline %{
 5566 
 5567 attributes %{
 5568   // ARM instructions are of fixed length
 5569   fixed_size_instructions;        // Fixed size instructions TODO does
 5570   max_instructions_per_bundle = 4;   // A53 = 2, A57 = 4
 5571   // ARM instructions come in 32-bit word units
 5572   instruction_unit_size = 4;         // An instruction is 4 bytes long
 5573   instruction_fetch_unit_size = 64;  // The processor fetches one line
 5574   instruction_fetch_units = 1;       // of 64 bytes
 5575 
 5576   // List of nop instructions
 5577   nops( MachNop );
 5578 %}
 5579 
 5580 // We don't use an actual pipeline model so don't care about resources
 5581 // or description. we do use pipeline classes to introduce fixed
 5582 // latencies
 5583 
 5584 //----------RESOURCES----------------------------------------------------------
 5585 // Resources are the functional units available to the machine
 5586 
 5587 resources( INS0, INS1, INS01 = INS0 | INS1,
 5588            ALU0, ALU1, ALU = ALU0 | ALU1,
 5589            MAC,
 5590            DIV,
 5591            BRANCH,
 5592            LDST,
 5593            NEON_FP);
 5594 
 5595 //----------PIPELINE DESCRIPTION-----------------------------------------------
 5596 // Pipeline Description specifies the stages in the machine's pipeline
 5597 
 5598 // Define the pipeline as a generic 6 stage pipeline
 5599 pipe_desc(S0, S1, S2, S3, S4, S5);
 5600 
 5601 //----------PIPELINE CLASSES---------------------------------------------------
 5602 // Pipeline Classes describe the stages in which input and output are
 5603 // referenced by the hardware pipeline.
 5604 
 5605 pipe_class fp_dop_reg_reg_s(vRegF dst, vRegF src1, vRegF src2)
 5606 %{
 5607   single_instruction;
 5608   src1   : S1(read);
 5609   src2   : S2(read);
 5610   dst    : S5(write);
 5611   INS01  : ISS;
 5612   NEON_FP : S5;
 5613 %}
 5614 
 5615 pipe_class fp_dop_reg_reg_d(vRegD dst, vRegD src1, vRegD src2)
 5616 %{
 5617   single_instruction;
 5618   src1   : S1(read);
 5619   src2   : S2(read);
 5620   dst    : S5(write);
 5621   INS01  : ISS;
 5622   NEON_FP : S5;
 5623 %}
 5624 
 5625 pipe_class fp_uop_s(vRegF dst, vRegF src)
 5626 %{
 5627   single_instruction;
 5628   src    : S1(read);
 5629   dst    : S5(write);
 5630   INS01  : ISS;
 5631   NEON_FP : S5;
 5632 %}
 5633 
 5634 pipe_class fp_uop_d(vRegD dst, vRegD src)
 5635 %{
 5636   single_instruction;
 5637   src    : S1(read);
 5638   dst    : S5(write);
 5639   INS01  : ISS;
 5640   NEON_FP : S5;
 5641 %}
 5642 
 5643 pipe_class fp_d2f(vRegF dst, vRegD src)
 5644 %{
 5645   single_instruction;
 5646   src    : S1(read);
 5647   dst    : S5(write);
 5648   INS01  : ISS;
 5649   NEON_FP : S5;
 5650 %}
 5651 
 5652 pipe_class fp_f2d(vRegD dst, vRegF src)
 5653 %{
 5654   single_instruction;
 5655   src    : S1(read);
 5656   dst    : S5(write);
 5657   INS01  : ISS;
 5658   NEON_FP : S5;
 5659 %}
 5660 
 5661 pipe_class fp_f2i(iRegINoSp dst, vRegF src)
 5662 %{
 5663   single_instruction;
 5664   src    : S1(read);
 5665   dst    : S5(write);
 5666   INS01  : ISS;
 5667   NEON_FP : S5;
 5668 %}
 5669 
 5670 pipe_class fp_f2l(iRegLNoSp dst, vRegF src)
 5671 %{
 5672   single_instruction;
 5673   src    : S1(read);
 5674   dst    : S5(write);
 5675   INS01  : ISS;
 5676   NEON_FP : S5;
 5677 %}
 5678 
 5679 pipe_class fp_i2f(vRegF dst, iRegIorL2I src)
 5680 %{
 5681   single_instruction;
 5682   src    : S1(read);
 5683   dst    : S5(write);
 5684   INS01  : ISS;
 5685   NEON_FP : S5;
 5686 %}
 5687 
 5688 pipe_class fp_l2f(vRegF dst, iRegL src)
 5689 %{
 5690   single_instruction;
 5691   src    : S1(read);
 5692   dst    : S5(write);
 5693   INS01  : ISS;
 5694   NEON_FP : S5;
 5695 %}
 5696 
 5697 pipe_class fp_d2i(iRegINoSp dst, vRegD src)
 5698 %{
 5699   single_instruction;
 5700   src    : S1(read);
 5701   dst    : S5(write);
 5702   INS01  : ISS;
 5703   NEON_FP : S5;
 5704 %}
 5705 
 5706 pipe_class fp_d2l(iRegLNoSp dst, vRegD src)
 5707 %{
 5708   single_instruction;
 5709   src    : S1(read);
 5710   dst    : S5(write);
 5711   INS01  : ISS;
 5712   NEON_FP : S5;
 5713 %}
 5714 
 5715 pipe_class fp_i2d(vRegD dst, iRegIorL2I src)
 5716 %{
 5717   single_instruction;
 5718   src    : S1(read);
 5719   dst    : S5(write);
 5720   INS01  : ISS;
 5721   NEON_FP : S5;
 5722 %}
 5723 
 5724 pipe_class fp_l2d(vRegD dst, iRegIorL2I src)
 5725 %{
 5726   single_instruction;
 5727   src    : S1(read);
 5728   dst    : S5(write);
 5729   INS01  : ISS;
 5730   NEON_FP : S5;
 5731 %}
 5732 
 5733 pipe_class fp_div_s(vRegF dst, vRegF src1, vRegF src2)
 5734 %{
 5735   single_instruction;
 5736   src1   : S1(read);
 5737   src2   : S2(read);
 5738   dst    : S5(write);
 5739   INS0   : ISS;
 5740   NEON_FP : S5;
 5741 %}
 5742 
 5743 pipe_class fp_div_d(vRegD dst, vRegD src1, vRegD src2)
 5744 %{
 5745   single_instruction;
 5746   src1   : S1(read);
 5747   src2   : S2(read);
 5748   dst    : S5(write);
 5749   INS0   : ISS;
 5750   NEON_FP : S5;
 5751 %}
 5752 
 5753 pipe_class fp_cond_reg_reg_s(vRegF dst, vRegF src1, vRegF src2, rFlagsReg cr)
 5754 %{
 5755   single_instruction;
 5756   cr     : S1(read);
 5757   src1   : S1(read);
 5758   src2   : S1(read);
 5759   dst    : S3(write);
 5760   INS01  : ISS;
 5761   NEON_FP : S3;
 5762 %}
 5763 
 5764 pipe_class fp_cond_reg_reg_d(vRegD dst, vRegD src1, vRegD src2, rFlagsReg cr)
 5765 %{
 5766   single_instruction;
 5767   cr     : S1(read);
 5768   src1   : S1(read);
 5769   src2   : S1(read);
 5770   dst    : S3(write);
 5771   INS01  : ISS;
 5772   NEON_FP : S3;
 5773 %}
 5774 
 5775 pipe_class fp_imm_s(vRegF dst)
 5776 %{
 5777   single_instruction;
 5778   dst    : S3(write);
 5779   INS01  : ISS;
 5780   NEON_FP : S3;
 5781 %}
 5782 
 5783 pipe_class fp_imm_d(vRegD dst)
 5784 %{
 5785   single_instruction;
 5786   dst    : S3(write);
 5787   INS01  : ISS;
 5788   NEON_FP : S3;
 5789 %}
 5790 
 5791 pipe_class fp_load_constant_s(vRegF dst)
 5792 %{
 5793   single_instruction;
 5794   dst    : S4(write);
 5795   INS01  : ISS;
 5796   NEON_FP : S4;
 5797 %}
 5798 
 5799 pipe_class fp_load_constant_d(vRegD dst)
 5800 %{
 5801   single_instruction;
 5802   dst    : S4(write);
 5803   INS01  : ISS;
 5804   NEON_FP : S4;
 5805 %}
 5806 
 5807 //------- Integer ALU operations --------------------------
 5808 
 5809 // Integer ALU reg-reg operation
 5810 // Operands needed in EX1, result generated in EX2
 5811 // Eg.  ADD     x0, x1, x2
 5812 pipe_class ialu_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 5813 %{
 5814   single_instruction;
 5815   dst    : EX2(write);
 5816   src1   : EX1(read);
 5817   src2   : EX1(read);
 5818   INS01  : ISS; // Dual issue as instruction 0 or 1
 5819   ALU    : EX2;
 5820 %}
 5821 
 5822 // Integer ALU reg-reg operation with constant shift
 5823 // Shifted register must be available in LATE_ISS instead of EX1
 5824 // Eg.  ADD     x0, x1, x2, LSL #2
 5825 pipe_class ialu_reg_reg_shift(iRegI dst, iRegI src1, iRegI src2, immI shift)
 5826 %{
 5827   single_instruction;
 5828   dst    : EX2(write);
 5829   src1   : EX1(read);
 5830   src2   : ISS(read);
 5831   INS01  : ISS;
 5832   ALU    : EX2;
 5833 %}
 5834 
 5835 // Integer ALU reg operation with constant shift
 5836 // Eg.  LSL     x0, x1, #shift
 5837 pipe_class ialu_reg_shift(iRegI dst, iRegI src1)
 5838 %{
 5839   single_instruction;
 5840   dst    : EX2(write);
 5841   src1   : ISS(read);
 5842   INS01  : ISS;
 5843   ALU    : EX2;
 5844 %}
 5845 
 5846 // Integer ALU reg-reg operation with variable shift
 5847 // Both operands must be available in LATE_ISS instead of EX1
 5848 // Result is available in EX1 instead of EX2
 5849 // Eg.  LSLV    x0, x1, x2
 5850 pipe_class ialu_reg_reg_vshift(iRegI dst, iRegI src1, iRegI src2)
 5851 %{
 5852   single_instruction;
 5853   dst    : EX1(write);
 5854   src1   : ISS(read);
 5855   src2   : ISS(read);
 5856   INS01  : ISS;
 5857   ALU    : EX1;
 5858 %}
 5859 
 5860 // Integer ALU reg-reg operation with extract
 5861 // As for _vshift above, but result generated in EX2
 5862 // Eg.  EXTR    x0, x1, x2, #N
 5863 pipe_class ialu_reg_reg_extr(iRegI dst, iRegI src1, iRegI src2)
 5864 %{
 5865   single_instruction;
 5866   dst    : EX2(write);
 5867   src1   : ISS(read);
 5868   src2   : ISS(read);
 5869   INS1   : ISS; // Can only dual issue as Instruction 1
 5870   ALU    : EX1;
 5871 %}
 5872 
 5873 // Integer ALU reg operation
 5874 // Eg.  NEG     x0, x1
 5875 pipe_class ialu_reg(iRegI dst, iRegI src)
 5876 %{
 5877   single_instruction;
 5878   dst    : EX2(write);
 5879   src    : EX1(read);
 5880   INS01  : ISS;
 5881   ALU    : EX2;
 5882 %}
 5883 
 5884 // Integer ALU reg mmediate operation
 5885 // Eg.  ADD     x0, x1, #N
 5886 pipe_class ialu_reg_imm(iRegI dst, iRegI src1)
 5887 %{
 5888   single_instruction;
 5889   dst    : EX2(write);
 5890   src1   : EX1(read);
 5891   INS01  : ISS;
 5892   ALU    : EX2;
 5893 %}
 5894 
 5895 // Integer ALU immediate operation (no source operands)
 5896 // Eg.  MOV     x0, #N
 5897 pipe_class ialu_imm(iRegI dst)
 5898 %{
 5899   single_instruction;
 5900   dst    : EX1(write);
 5901   INS01  : ISS;
 5902   ALU    : EX1;
 5903 %}
 5904 
 5905 //------- Compare operation -------------------------------
 5906 
 5907 // Compare reg-reg
 5908 // Eg.  CMP     x0, x1
 5909 pipe_class icmp_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
 5910 %{
 5911   single_instruction;
 5912 //  fixed_latency(16);
 5913   cr     : EX2(write);
 5914   op1    : EX1(read);
 5915   op2    : EX1(read);
 5916   INS01  : ISS;
 5917   ALU    : EX2;
 5918 %}
 5919 
 5920 // Compare reg-reg
 5921 // Eg.  CMP     x0, #N
 5922 pipe_class icmp_reg_imm(rFlagsReg cr, iRegI op1)
 5923 %{
 5924   single_instruction;
 5925 //  fixed_latency(16);
 5926   cr     : EX2(write);
 5927   op1    : EX1(read);
 5928   INS01  : ISS;
 5929   ALU    : EX2;
 5930 %}
 5931 
 5932 //------- Conditional instructions ------------------------
 5933 
 5934 // Conditional no operands
 5935 // Eg.  CSINC   x0, zr, zr, <cond>
 5936 pipe_class icond_none(iRegI dst, rFlagsReg cr)
 5937 %{
 5938   single_instruction;
 5939   cr     : EX1(read);
 5940   dst    : EX2(write);
 5941   INS01  : ISS;
 5942   ALU    : EX2;
 5943 %}
 5944 
 5945 // Conditional 2 operand
 5946 // EG.  CSEL    X0, X1, X2, <cond>
 5947 pipe_class icond_reg_reg(iRegI dst, iRegI src1, iRegI src2, rFlagsReg cr)
 5948 %{
 5949   single_instruction;
 5950   cr     : EX1(read);
 5951   src1   : EX1(read);
 5952   src2   : EX1(read);
 5953   dst    : EX2(write);
 5954   INS01  : ISS;
 5955   ALU    : EX2;
 5956 %}
 5957 
 5958 // Conditional 2 operand
 5959 // EG.  CSEL    X0, X1, X2, <cond>
 5960 pipe_class icond_reg(iRegI dst, iRegI src, rFlagsReg cr)
 5961 %{
 5962   single_instruction;
 5963   cr     : EX1(read);
 5964   src    : EX1(read);
 5965   dst    : EX2(write);
 5966   INS01  : ISS;
 5967   ALU    : EX2;
 5968 %}
 5969 
 5970 //------- Multiply pipeline operations --------------------
 5971 
 5972 // Multiply reg-reg
 5973 // Eg.  MUL     w0, w1, w2
 5974 pipe_class imul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 5975 %{
 5976   single_instruction;
 5977   dst    : WR(write);
 5978   src1   : ISS(read);
 5979   src2   : ISS(read);
 5980   INS01  : ISS;
 5981   MAC    : WR;
 5982 %}
 5983 
 5984 // Multiply accumulate
 5985 // Eg.  MADD    w0, w1, w2, w3
 5986 pipe_class imac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
 5987 %{
 5988   single_instruction;
 5989   dst    : WR(write);
 5990   src1   : ISS(read);
 5991   src2   : ISS(read);
 5992   src3   : ISS(read);
 5993   INS01  : ISS;
 5994   MAC    : WR;
 5995 %}
 5996 
 5997 // Eg.  MUL     w0, w1, w2
 5998 pipe_class lmul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 5999 %{
 6000   single_instruction;
 6001   fixed_latency(3); // Maximum latency for 64 bit mul
 6002   dst    : WR(write);
 6003   src1   : ISS(read);
 6004   src2   : ISS(read);
 6005   INS01  : ISS;
 6006   MAC    : WR;
 6007 %}
 6008 
 6009 // Multiply accumulate
 6010 // Eg.  MADD    w0, w1, w2, w3
 6011 pipe_class lmac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
 6012 %{
 6013   single_instruction;
 6014   fixed_latency(3); // Maximum latency for 64 bit mul
 6015   dst    : WR(write);
 6016   src1   : ISS(read);
 6017   src2   : ISS(read);
 6018   src3   : ISS(read);
 6019   INS01  : ISS;
 6020   MAC    : WR;
 6021 %}
 6022 
 6023 //------- Divide pipeline operations --------------------
 6024 
 6025 // Eg.  SDIV    w0, w1, w2
 6026 pipe_class idiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 6027 %{
 6028   single_instruction;
 6029   fixed_latency(8); // Maximum latency for 32 bit divide
 6030   dst    : WR(write);
 6031   src1   : ISS(read);
 6032   src2   : ISS(read);
 6033   INS0   : ISS; // Can only dual issue as instruction 0
 6034   DIV    : WR;
 6035 %}
 6036 
 6037 // Eg.  SDIV    x0, x1, x2
 6038 pipe_class ldiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 6039 %{
 6040   single_instruction;
 6041   fixed_latency(16); // Maximum latency for 64 bit divide
 6042   dst    : WR(write);
 6043   src1   : ISS(read);
 6044   src2   : ISS(read);
 6045   INS0   : ISS; // Can only dual issue as instruction 0
 6046   DIV    : WR;
 6047 %}
 6048 
 6049 //------- Load pipeline operations ------------------------
 6050 
 6051 // Load - prefetch
 6052 // Eg.  PFRM    <mem>
 6053 pipe_class iload_prefetch(memory mem)
 6054 %{
 6055   single_instruction;
 6056   mem    : ISS(read);
 6057   INS01  : ISS;
 6058   LDST   : WR;
 6059 %}
 6060 
 6061 // Load - reg, mem
 6062 // Eg.  LDR     x0, <mem>
 6063 pipe_class iload_reg_mem(iRegI dst, memory mem)
 6064 %{
 6065   single_instruction;
 6066   dst    : WR(write);
 6067   mem    : ISS(read);
 6068   INS01  : ISS;
 6069   LDST   : WR;
 6070 %}
 6071 
 6072 // Load - reg, reg
 6073 // Eg.  LDR     x0, [sp, x1]
 6074 pipe_class iload_reg_reg(iRegI dst, iRegI src)
 6075 %{
 6076   single_instruction;
 6077   dst    : WR(write);
 6078   src    : ISS(read);
 6079   INS01  : ISS;
 6080   LDST   : WR;
 6081 %}
 6082 
 6083 //------- Store pipeline operations -----------------------
 6084 
 6085 // Store - zr, mem
 6086 // Eg.  STR     zr, <mem>
 6087 pipe_class istore_mem(memory mem)
 6088 %{
 6089   single_instruction;
 6090   mem    : ISS(read);
 6091   INS01  : ISS;
 6092   LDST   : WR;
 6093 %}
 6094 
 6095 // Store - reg, mem
 6096 // Eg.  STR     x0, <mem>
 6097 pipe_class istore_reg_mem(iRegI src, memory mem)
 6098 %{
 6099   single_instruction;
 6100   mem    : ISS(read);
 6101   src    : EX2(read);
 6102   INS01  : ISS;
 6103   LDST   : WR;
 6104 %}
 6105 
 6106 // Store - reg, reg
 6107 // Eg. STR      x0, [sp, x1]
 6108 pipe_class istore_reg_reg(iRegI dst, iRegI src)
 6109 %{
 6110   single_instruction;
 6111   dst    : ISS(read);
 6112   src    : EX2(read);
 6113   INS01  : ISS;
 6114   LDST   : WR;
 6115 %}
 6116 
 6117 //------- Store pipeline operations -----------------------
 6118 
 6119 // Branch
 6120 pipe_class pipe_branch()
 6121 %{
 6122   single_instruction;
 6123   INS01  : ISS;
 6124   BRANCH : EX1;
 6125 %}
 6126 
 6127 // Conditional branch
 6128 pipe_class pipe_branch_cond(rFlagsReg cr)
 6129 %{
 6130   single_instruction;
 6131   cr     : EX1(read);
 6132   INS01  : ISS;
 6133   BRANCH : EX1;
 6134 %}
 6135 
 6136 // Compare & Branch
 6137 // EG.  CBZ/CBNZ
 6138 pipe_class pipe_cmp_branch(iRegI op1)
 6139 %{
 6140   single_instruction;
 6141   op1    : EX1(read);
 6142   INS01  : ISS;
 6143   BRANCH : EX1;
 6144 %}
 6145 
 6146 //------- Synchronisation operations ----------------------
 6147 
 6148 // Any operation requiring serialization.
 6149 // EG.  DMB/Atomic Ops/Load Acquire/Str Release
 6150 pipe_class pipe_serial()
 6151 %{
 6152   single_instruction;
 6153   force_serialization;
 6154   fixed_latency(16);
 6155   INS01  : ISS(2); // Cannot dual issue with any other instruction
 6156   LDST   : WR;
 6157 %}
 6158 
 6159 // Generic big/slow expanded idiom - also serialized
 6160 pipe_class pipe_slow()
 6161 %{
 6162   instruction_count(10);
 6163   multiple_bundles;
 6164   force_serialization;
 6165   fixed_latency(16);
 6166   INS01  : ISS(2); // Cannot dual issue with any other instruction
 6167   LDST   : WR;
 6168 %}
 6169 
 6170 // Empty pipeline class
 6171 pipe_class pipe_class_empty()
 6172 %{
 6173   single_instruction;
 6174   fixed_latency(0);
 6175 %}
 6176 
 6177 // Default pipeline class.
 6178 pipe_class pipe_class_default()
 6179 %{
 6180   single_instruction;
 6181   fixed_latency(2);
 6182 %}
 6183 
 6184 // Pipeline class for compares.
 6185 pipe_class pipe_class_compare()
 6186 %{
 6187   single_instruction;
 6188   fixed_latency(16);
 6189 %}
 6190 
 6191 // Pipeline class for memory operations.
 6192 pipe_class pipe_class_memory()
 6193 %{
 6194   single_instruction;
 6195   fixed_latency(16);
 6196 %}
 6197 
 6198 // Pipeline class for call.
 6199 pipe_class pipe_class_call()
 6200 %{
 6201   single_instruction;
 6202   fixed_latency(100);
 6203 %}
 6204 
 6205 // Define the class for the Nop node.
 6206 define %{
 6207    MachNop = pipe_class_empty;
 6208 %}
 6209 
 6210 %}
 6211 //----------INSTRUCTIONS-------------------------------------------------------
 6212 //
 6213 // match      -- States which machine-independent subtree may be replaced
 6214 //               by this instruction.
 6215 // ins_cost   -- The estimated cost of this instruction is used by instruction
 6216 //               selection to identify a minimum cost tree of machine
 6217 //               instructions that matches a tree of machine-independent
 6218 //               instructions.
 6219 // format     -- A string providing the disassembly for this instruction.
 6220 //               The value of an instruction's operand may be inserted
 6221 //               by referring to it with a '$' prefix.
 6222 // opcode     -- Three instruction opcodes may be provided.  These are referred
 6223 //               to within an encode class as $primary, $secondary, and $tertiary
 6224 //               rrspectively.  The primary opcode is commonly used to
 6225 //               indicate the type of machine instruction, while secondary
 6226 //               and tertiary are often used for prefix options or addressing
 6227 //               modes.
 6228 // ins_encode -- A list of encode classes with parameters. The encode class
 6229 //               name must have been defined in an 'enc_class' specification
 6230 //               in the encode section of the architecture description.
 6231 
 6232 // ============================================================================
 6233 // Memory (Load/Store) Instructions
 6234 
 6235 // Load Instructions
 6236 
 6237 // Load Byte (8 bit signed)
 6238 instruct loadB(iRegINoSp dst, memory mem)
 6239 %{
 6240   match(Set dst (LoadB mem));
 6241   predicate(!needs_acquiring_load(n));
 6242 
 6243   ins_cost(4 * INSN_COST);
 6244   format %{ "ldrsbw  $dst, $mem\t# byte" %}
 6245 
 6246   ins_encode(aarch64_enc_ldrsbw(dst, mem));
 6247 
 6248   ins_pipe(iload_reg_mem);
 6249 %}
 6250 
 6251 // Load Byte (8 bit signed) into long
 6252 instruct loadB2L(iRegLNoSp dst, memory mem)
 6253 %{
 6254   match(Set dst (ConvI2L (LoadB mem)));
 6255   predicate(!needs_acquiring_load(n->in(1)));
 6256 
 6257   ins_cost(4 * INSN_COST);
 6258   format %{ "ldrsb  $dst, $mem\t# byte" %}
 6259 
 6260   ins_encode(aarch64_enc_ldrsb(dst, mem));
 6261 
 6262   ins_pipe(iload_reg_mem);
 6263 %}
 6264 
 6265 // Load Byte (8 bit unsigned)
 6266 instruct loadUB(iRegINoSp dst, memory mem)
 6267 %{
 6268   match(Set dst (LoadUB mem));
 6269   predicate(!needs_acquiring_load(n));
 6270 
 6271   ins_cost(4 * INSN_COST);
 6272   format %{ "ldrbw  $dst, $mem\t# byte" %}
 6273 
 6274   ins_encode(aarch64_enc_ldrb(dst, mem));
 6275 
 6276   ins_pipe(iload_reg_mem);
 6277 %}
 6278 
 6279 // Load Byte (8 bit unsigned) into long
 6280 instruct loadUB2L(iRegLNoSp dst, memory mem)
 6281 %{
 6282   match(Set dst (ConvI2L (LoadUB mem)));
 6283   predicate(!needs_acquiring_load(n->in(1)));
 6284 
 6285   ins_cost(4 * INSN_COST);
 6286   format %{ "ldrb  $dst, $mem\t# byte" %}
 6287 
 6288   ins_encode(aarch64_enc_ldrb(dst, mem));
 6289 
 6290   ins_pipe(iload_reg_mem);
 6291 %}
 6292 
 6293 // Load Short (16 bit signed)
 6294 instruct loadS(iRegINoSp dst, memory mem)
 6295 %{
 6296   match(Set dst (LoadS mem));
 6297   predicate(!needs_acquiring_load(n));
 6298 
 6299   ins_cost(4 * INSN_COST);
 6300   format %{ "ldrshw  $dst, $mem\t# short" %}
 6301 
 6302   ins_encode(aarch64_enc_ldrshw(dst, mem));
 6303 
 6304   ins_pipe(iload_reg_mem);
 6305 %}
 6306 
 6307 // Load Short (16 bit signed) into long
 6308 instruct loadS2L(iRegLNoSp dst, memory mem)
 6309 %{
 6310   match(Set dst (ConvI2L (LoadS mem)));
 6311   predicate(!needs_acquiring_load(n->in(1)));
 6312 
 6313   ins_cost(4 * INSN_COST);
 6314   format %{ "ldrsh  $dst, $mem\t# short" %}
 6315 
 6316   ins_encode(aarch64_enc_ldrsh(dst, mem));
 6317 
 6318   ins_pipe(iload_reg_mem);
 6319 %}
 6320 
 6321 // Load Char (16 bit unsigned)
 6322 instruct loadUS(iRegINoSp dst, memory mem)
 6323 %{
 6324   match(Set dst (LoadUS mem));
 6325   predicate(!needs_acquiring_load(n));
 6326 
 6327   ins_cost(4 * INSN_COST);
 6328   format %{ "ldrh  $dst, $mem\t# short" %}
 6329 
 6330   ins_encode(aarch64_enc_ldrh(dst, mem));
 6331 
 6332   ins_pipe(iload_reg_mem);
 6333 %}
 6334 
 6335 // Load Short/Char (16 bit unsigned) into long
 6336 instruct loadUS2L(iRegLNoSp dst, memory mem)
 6337 %{
 6338   match(Set dst (ConvI2L (LoadUS mem)));
 6339   predicate(!needs_acquiring_load(n->in(1)));
 6340 
 6341   ins_cost(4 * INSN_COST);
 6342   format %{ "ldrh  $dst, $mem\t# short" %}
 6343 
 6344   ins_encode(aarch64_enc_ldrh(dst, mem));
 6345 
 6346   ins_pipe(iload_reg_mem);
 6347 %}
 6348 
 6349 // Load Integer (32 bit signed)
 6350 instruct loadI(iRegINoSp dst, memory mem)
 6351 %{
 6352   match(Set dst (LoadI mem));
 6353   predicate(!needs_acquiring_load(n));
 6354 
 6355   ins_cost(4 * INSN_COST);
 6356   format %{ "ldrw  $dst, $mem\t# int" %}
 6357 
 6358   ins_encode(aarch64_enc_ldrw(dst, mem));
 6359 
 6360   ins_pipe(iload_reg_mem);
 6361 %}
 6362 
 6363 // Load Integer (32 bit signed) into long
 6364 instruct loadI2L(iRegLNoSp dst, memory mem)
 6365 %{
 6366   match(Set dst (ConvI2L (LoadI mem)));
 6367   predicate(!needs_acquiring_load(n->in(1)));
 6368 
 6369   ins_cost(4 * INSN_COST);
 6370   format %{ "ldrsw  $dst, $mem\t# int" %}
 6371 
 6372   ins_encode(aarch64_enc_ldrsw(dst, mem));
 6373 
 6374   ins_pipe(iload_reg_mem);
 6375 %}
 6376 
 6377 // Load Integer (32 bit unsigned) into long
 6378 instruct loadUI2L(iRegLNoSp dst, memory mem, immL_32bits mask)
 6379 %{
 6380   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
 6381   predicate(!needs_acquiring_load(n->in(1)->in(1)->as_Load()));
 6382 
 6383   ins_cost(4 * INSN_COST);
 6384   format %{ "ldrw  $dst, $mem\t# int" %}
 6385 
 6386   ins_encode(aarch64_enc_ldrw(dst, mem));
 6387 
 6388   ins_pipe(iload_reg_mem);
 6389 %}
 6390 
 6391 // Load Long (64 bit signed)
 6392 instruct loadL(iRegLNoSp dst, memory mem)
 6393 %{
 6394   match(Set dst (LoadL mem));
 6395   predicate(!needs_acquiring_load(n));
 6396 
 6397   ins_cost(4 * INSN_COST);
 6398   format %{ "ldr  $dst, $mem\t# int" %}
 6399 
 6400   ins_encode(aarch64_enc_ldr(dst, mem));
 6401 
 6402   ins_pipe(iload_reg_mem);
 6403 %}
 6404 
 6405 // Load Range
 6406 instruct loadRange(iRegINoSp dst, memory mem)
 6407 %{
 6408   match(Set dst (LoadRange mem));
 6409 
 6410   ins_cost(4 * INSN_COST);
 6411   format %{ "ldrw  $dst, $mem\t# range" %}
 6412 
 6413   ins_encode(aarch64_enc_ldrw(dst, mem));
 6414 
 6415   ins_pipe(iload_reg_mem);
 6416 %}
 6417 
 6418 // Load Pointer
 6419 instruct loadP(iRegPNoSp dst, memory mem)
 6420 %{
 6421   match(Set dst (LoadP mem));
 6422   predicate(!needs_acquiring_load(n) && (n->as_Load()->barrier_data() == 0));
 6423 
 6424   ins_cost(4 * INSN_COST);
 6425   format %{ "ldr  $dst, $mem\t# ptr" %}
 6426 
 6427   ins_encode(aarch64_enc_ldr(dst, mem));
 6428 
 6429   ins_pipe(iload_reg_mem);
 6430 %}
 6431 
 6432 // Load Compressed Pointer
 6433 instruct loadN(iRegNNoSp dst, memory mem)
 6434 %{
 6435   match(Set dst (LoadN mem));
 6436   predicate(!needs_acquiring_load(n));
 6437 
 6438   ins_cost(4 * INSN_COST);
 6439   format %{ "ldrw  $dst, $mem\t# compressed ptr" %}
 6440 
 6441   ins_encode(aarch64_enc_ldrw(dst, mem));
 6442 
 6443   ins_pipe(iload_reg_mem);
 6444 %}
 6445 
 6446 // Load Klass Pointer
 6447 instruct loadKlass(iRegPNoSp dst, memory mem)
 6448 %{
 6449   match(Set dst (LoadKlass mem));
 6450   predicate(!needs_acquiring_load(n));
 6451 
 6452   ins_cost(4 * INSN_COST);
 6453   format %{ "ldr  $dst, $mem\t# class" %}
 6454 
 6455   ins_encode(aarch64_enc_ldr(dst, mem));
 6456 
 6457   ins_pipe(iload_reg_mem);
 6458 %}
 6459 
 6460 // Load Narrow Klass Pointer
 6461 instruct loadNKlass(iRegNNoSp dst, memory mem)
 6462 %{
 6463   match(Set dst (LoadNKlass mem));
 6464   predicate(!needs_acquiring_load(n));
 6465 
 6466   ins_cost(4 * INSN_COST);
 6467   format %{ "ldrw  $dst, $mem\t# compressed class ptr" %}
 6468 
 6469   ins_encode(aarch64_enc_ldrw(dst, mem));
 6470 
 6471   ins_pipe(iload_reg_mem);
 6472 %}
 6473 
 6474 // Load Float
 6475 instruct loadF(vRegF dst, memory mem)
 6476 %{
 6477   match(Set dst (LoadF mem));
 6478   predicate(!needs_acquiring_load(n));
 6479 
 6480   ins_cost(4 * INSN_COST);
 6481   format %{ "ldrs  $dst, $mem\t# float" %}
 6482 
 6483   ins_encode( aarch64_enc_ldrs(dst, mem) );
 6484 
 6485   ins_pipe(pipe_class_memory);
 6486 %}
 6487 
 6488 // Load Double
 6489 instruct loadD(vRegD dst, memory mem)
 6490 %{
 6491   match(Set dst (LoadD mem));
 6492   predicate(!needs_acquiring_load(n));
 6493 
 6494   ins_cost(4 * INSN_COST);
 6495   format %{ "ldrd  $dst, $mem\t# double" %}
 6496 
 6497   ins_encode( aarch64_enc_ldrd(dst, mem) );
 6498 
 6499   ins_pipe(pipe_class_memory);
 6500 %}
 6501 
 6502 
 6503 // Load Int Constant
 6504 instruct loadConI(iRegINoSp dst, immI src)
 6505 %{
 6506   match(Set dst src);
 6507 
 6508   ins_cost(INSN_COST);
 6509   format %{ "mov $dst, $src\t# int" %}
 6510 
 6511   ins_encode( aarch64_enc_movw_imm(dst, src) );
 6512 
 6513   ins_pipe(ialu_imm);
 6514 %}
 6515 
 6516 // Load Long Constant
 6517 instruct loadConL(iRegLNoSp dst, immL src)
 6518 %{
 6519   match(Set dst src);
 6520 
 6521   ins_cost(INSN_COST);
 6522   format %{ "mov $dst, $src\t# long" %}
 6523 
 6524   ins_encode( aarch64_enc_mov_imm(dst, src) );
 6525 
 6526   ins_pipe(ialu_imm);
 6527 %}
 6528 
 6529 // Load Pointer Constant
 6530 
 6531 instruct loadConP(iRegPNoSp dst, immP con)
 6532 %{
 6533   match(Set dst con);
 6534 
 6535   ins_cost(INSN_COST * 4);
 6536   format %{
 6537     "mov  $dst, $con\t# ptr"
 6538   %}
 6539 
 6540   ins_encode(aarch64_enc_mov_p(dst, con));
 6541 
 6542   ins_pipe(ialu_imm);
 6543 %}
 6544 
 6545 // Load Null Pointer Constant
 6546 
 6547 instruct loadConP0(iRegPNoSp dst, immP0 con)
 6548 %{
 6549   match(Set dst con);
 6550 
 6551   ins_cost(INSN_COST);
 6552   format %{ "mov  $dst, $con\t# nullptr ptr" %}
 6553 
 6554   ins_encode(aarch64_enc_mov_p0(dst, con));
 6555 
 6556   ins_pipe(ialu_imm);
 6557 %}
 6558 
 6559 // Load Pointer Constant One
 6560 
 6561 instruct loadConP1(iRegPNoSp dst, immP_1 con)
 6562 %{
 6563   match(Set dst con);
 6564 
 6565   ins_cost(INSN_COST);
 6566   format %{ "mov  $dst, $con\t# nullptr ptr" %}
 6567 
 6568   ins_encode(aarch64_enc_mov_p1(dst, con));
 6569 
 6570   ins_pipe(ialu_imm);
 6571 %}
 6572 
 6573 // Load Byte Map Base Constant
 6574 
 6575 instruct loadByteMapBase(iRegPNoSp dst, immByteMapBase con)
 6576 %{
 6577   match(Set dst con);
 6578 
 6579   ins_cost(INSN_COST);
 6580   format %{ "adr  $dst, $con\t# Byte Map Base" %}
 6581 
 6582   ins_encode(aarch64_enc_mov_byte_map_base(dst, con));
 6583 
 6584   ins_pipe(ialu_imm);
 6585 %}
 6586 
 6587 // Load Narrow Pointer Constant
 6588 
 6589 instruct loadConN(iRegNNoSp dst, immN con)
 6590 %{
 6591   match(Set dst con);
 6592 
 6593   ins_cost(INSN_COST * 4);
 6594   format %{ "mov  $dst, $con\t# compressed ptr" %}
 6595 
 6596   ins_encode(aarch64_enc_mov_n(dst, con));
 6597 
 6598   ins_pipe(ialu_imm);
 6599 %}
 6600 
 6601 // Load Narrow Null Pointer Constant
 6602 
 6603 instruct loadConN0(iRegNNoSp dst, immN0 con)
 6604 %{
 6605   match(Set dst con);
 6606 
 6607   ins_cost(INSN_COST);
 6608   format %{ "mov  $dst, $con\t# compressed nullptr ptr" %}
 6609 
 6610   ins_encode(aarch64_enc_mov_n0(dst, con));
 6611 
 6612   ins_pipe(ialu_imm);
 6613 %}
 6614 
 6615 // Load Narrow Klass Constant
 6616 
 6617 instruct loadConNKlass(iRegNNoSp dst, immNKlass con)
 6618 %{
 6619   match(Set dst con);
 6620 
 6621   ins_cost(INSN_COST);
 6622   format %{ "mov  $dst, $con\t# compressed klass ptr" %}
 6623 
 6624   ins_encode(aarch64_enc_mov_nk(dst, con));
 6625 
 6626   ins_pipe(ialu_imm);
 6627 %}
 6628 
 6629 // Load Packed Float Constant
 6630 
 6631 instruct loadConF_packed(vRegF dst, immFPacked con) %{
 6632   match(Set dst con);
 6633   ins_cost(INSN_COST * 4);
 6634   format %{ "fmovs  $dst, $con"%}
 6635   ins_encode %{
 6636     __ fmovs(as_FloatRegister($dst$$reg), (double)$con$$constant);
 6637   %}
 6638 
 6639   ins_pipe(fp_imm_s);
 6640 %}
 6641 
 6642 // Load Float Constant
 6643 
 6644 instruct loadConF(vRegF dst, immF con) %{
 6645   match(Set dst con);
 6646 
 6647   ins_cost(INSN_COST * 4);
 6648 
 6649   format %{
 6650     "ldrs $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
 6651   %}
 6652 
 6653   ins_encode %{
 6654     __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
 6655   %}
 6656 
 6657   ins_pipe(fp_load_constant_s);
 6658 %}
 6659 
 6660 // Load Packed Double Constant
 6661 
 6662 instruct loadConD_packed(vRegD dst, immDPacked con) %{
 6663   match(Set dst con);
 6664   ins_cost(INSN_COST);
 6665   format %{ "fmovd  $dst, $con"%}
 6666   ins_encode %{
 6667     __ fmovd(as_FloatRegister($dst$$reg), $con$$constant);
 6668   %}
 6669 
 6670   ins_pipe(fp_imm_d);
 6671 %}
 6672 
 6673 // Load Double Constant
 6674 
 6675 instruct loadConD(vRegD dst, immD con) %{
 6676   match(Set dst con);
 6677 
 6678   ins_cost(INSN_COST * 5);
 6679   format %{
 6680     "ldrd $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
 6681   %}
 6682 
 6683   ins_encode %{
 6684     __ ldrd(as_FloatRegister($dst$$reg), $constantaddress($con));
 6685   %}
 6686 
 6687   ins_pipe(fp_load_constant_d);
 6688 %}
 6689 
 6690 // Store Instructions
 6691 
 6692 // Store CMS card-mark Immediate
 6693 instruct storeimmCM0(immI0 zero, memory mem)
 6694 %{
 6695   match(Set mem (StoreCM mem zero));
 6696 
 6697   ins_cost(INSN_COST);
 6698   format %{ "storestore (elided)\n\t"
 6699             "strb zr, $mem\t# byte" %}
 6700 
 6701   ins_encode(aarch64_enc_strb0(mem));
 6702 
 6703   ins_pipe(istore_mem);
 6704 %}
 6705 
 6706 // Store CMS card-mark Immediate with intervening StoreStore
 6707 // needed when using CMS with no conditional card marking
 6708 instruct storeimmCM0_ordered(immI0 zero, memory mem)
 6709 %{
 6710   match(Set mem (StoreCM mem zero));
 6711 
 6712   ins_cost(INSN_COST * 2);
 6713   format %{ "storestore\n\t"
 6714             "dmb ishst"
 6715             "\n\tstrb zr, $mem\t# byte" %}
 6716 
 6717   ins_encode(aarch64_enc_strb0_ordered(mem));
 6718 
 6719   ins_pipe(istore_mem);
 6720 %}
 6721 
 6722 // Store Byte
 6723 instruct storeB(iRegIorL2I src, memory mem)
 6724 %{
 6725   match(Set mem (StoreB mem src));
 6726   predicate(!needs_releasing_store(n));
 6727 
 6728   ins_cost(INSN_COST);
 6729   format %{ "strb  $src, $mem\t# byte" %}
 6730 
 6731   ins_encode(aarch64_enc_strb(src, mem));
 6732 
 6733   ins_pipe(istore_reg_mem);
 6734 %}
 6735 
 6736 
 6737 instruct storeimmB0(immI0 zero, memory mem)
 6738 %{
 6739   match(Set mem (StoreB mem zero));
 6740   predicate(!needs_releasing_store(n));
 6741 
 6742   ins_cost(INSN_COST);
 6743   format %{ "strb rscractch2, $mem\t# byte" %}
 6744 
 6745   ins_encode(aarch64_enc_strb0(mem));
 6746 
 6747   ins_pipe(istore_mem);
 6748 %}
 6749 
 6750 // Store Char/Short
 6751 instruct storeC(iRegIorL2I src, memory mem)
 6752 %{
 6753   match(Set mem (StoreC mem src));
 6754   predicate(!needs_releasing_store(n));
 6755 
 6756   ins_cost(INSN_COST);
 6757   format %{ "strh  $src, $mem\t# short" %}
 6758 
 6759   ins_encode(aarch64_enc_strh(src, mem));
 6760 
 6761   ins_pipe(istore_reg_mem);
 6762 %}
 6763 
 6764 instruct storeimmC0(immI0 zero, memory mem)
 6765 %{
 6766   match(Set mem (StoreC mem zero));
 6767   predicate(!needs_releasing_store(n));
 6768 
 6769   ins_cost(INSN_COST);
 6770   format %{ "strh  zr, $mem\t# short" %}
 6771 
 6772   ins_encode(aarch64_enc_strh0(mem));
 6773 
 6774   ins_pipe(istore_mem);
 6775 %}
 6776 
 6777 // Store Integer
 6778 
 6779 instruct storeI(iRegIorL2I src, memory mem)
 6780 %{
 6781   match(Set mem(StoreI mem src));
 6782   predicate(!needs_releasing_store(n));
 6783 
 6784   ins_cost(INSN_COST);
 6785   format %{ "strw  $src, $mem\t# int" %}
 6786 
 6787   ins_encode(aarch64_enc_strw(src, mem));
 6788 
 6789   ins_pipe(istore_reg_mem);
 6790 %}
 6791 
 6792 instruct storeimmI0(immI0 zero, memory mem)
 6793 %{
 6794   match(Set mem(StoreI mem zero));
 6795   predicate(!needs_releasing_store(n));
 6796 
 6797   ins_cost(INSN_COST);
 6798   format %{ "strw  zr, $mem\t# int" %}
 6799 
 6800   ins_encode(aarch64_enc_strw0(mem));
 6801 
 6802   ins_pipe(istore_mem);
 6803 %}
 6804 
 6805 // Store Long (64 bit signed)
 6806 instruct storeL(iRegL src, memory mem)
 6807 %{
 6808   match(Set mem (StoreL mem src));
 6809   predicate(!needs_releasing_store(n));
 6810 
 6811   ins_cost(INSN_COST);
 6812   format %{ "str  $src, $mem\t# int" %}
 6813 
 6814   ins_encode(aarch64_enc_str(src, mem));
 6815 
 6816   ins_pipe(istore_reg_mem);
 6817 %}
 6818 
 6819 // Store Long (64 bit signed)
 6820 instruct storeimmL0(immL0 zero, memory mem)
 6821 %{
 6822   match(Set mem (StoreL mem zero));
 6823   predicate(!needs_releasing_store(n));
 6824 
 6825   ins_cost(INSN_COST);
 6826   format %{ "str  zr, $mem\t# int" %}
 6827 
 6828   ins_encode(aarch64_enc_str0(mem));
 6829 
 6830   ins_pipe(istore_mem);
 6831 %}
 6832 
 6833 // Store Pointer
 6834 instruct storeP(iRegP src, memory mem)
 6835 %{
 6836   match(Set mem (StoreP mem src));
 6837   predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
 6838 
 6839   ins_cost(INSN_COST);
 6840   format %{ "str  $src, $mem\t# ptr" %}
 6841 
 6842   ins_encode(aarch64_enc_str(src, mem));
 6843 
 6844   ins_pipe(istore_reg_mem);
 6845 %}
 6846 
 6847 // Store Pointer
 6848 instruct storeimmP0(immP0 zero, memory mem)
 6849 %{
 6850   match(Set mem (StoreP mem zero));
 6851   predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
 6852 
 6853   ins_cost(INSN_COST);
 6854   format %{ "str zr, $mem\t# ptr" %}
 6855 
 6856   ins_encode(aarch64_enc_str0(mem));
 6857 
 6858   ins_pipe(istore_mem);
 6859 %}
 6860 
 6861 // Store Compressed Pointer
 6862 instruct storeN(iRegN src, memory mem)
 6863 %{
 6864   match(Set mem (StoreN mem src));
 6865   predicate(!needs_releasing_store(n));
 6866 
 6867   ins_cost(INSN_COST);
 6868   format %{ "strw  $src, $mem\t# compressed ptr" %}
 6869 
 6870   ins_encode(aarch64_enc_strw(src, mem));
 6871 
 6872   ins_pipe(istore_reg_mem);
 6873 %}
 6874 
 6875 instruct storeImmN0(immN0 zero, memory mem)
 6876 %{
 6877   match(Set mem (StoreN mem zero));
 6878   predicate(!needs_releasing_store(n));
 6879 
 6880   ins_cost(INSN_COST);
 6881   format %{ "strw  zr, $mem\t# compressed ptr" %}
 6882 
 6883   ins_encode(aarch64_enc_strw0(mem));
 6884 
 6885   ins_pipe(istore_mem);
 6886 %}
 6887 
 6888 // Store Float
 6889 instruct storeF(vRegF src, memory mem)
 6890 %{
 6891   match(Set mem (StoreF mem src));
 6892   predicate(!needs_releasing_store(n));
 6893 
 6894   ins_cost(INSN_COST);
 6895   format %{ "strs  $src, $mem\t# float" %}
 6896 
 6897   ins_encode( aarch64_enc_strs(src, mem) );
 6898 
 6899   ins_pipe(pipe_class_memory);
 6900 %}
 6901 
 6902 // TODO
 6903 // implement storeImmF0 and storeFImmPacked
 6904 
 6905 // Store Double
 6906 instruct storeD(vRegD src, memory mem)
 6907 %{
 6908   match(Set mem (StoreD mem src));
 6909   predicate(!needs_releasing_store(n));
 6910 
 6911   ins_cost(INSN_COST);
 6912   format %{ "strd  $src, $mem\t# double" %}
 6913 
 6914   ins_encode( aarch64_enc_strd(src, mem) );
 6915 
 6916   ins_pipe(pipe_class_memory);
 6917 %}
 6918 
 6919 // Store Compressed Klass Pointer
 6920 instruct storeNKlass(iRegN src, memory mem)
 6921 %{
 6922   predicate(!needs_releasing_store(n));
 6923   match(Set mem (StoreNKlass mem src));
 6924 
 6925   ins_cost(INSN_COST);
 6926   format %{ "strw  $src, $mem\t# compressed klass ptr" %}
 6927 
 6928   ins_encode(aarch64_enc_strw(src, mem));
 6929 
 6930   ins_pipe(istore_reg_mem);
 6931 %}
 6932 
 6933 // TODO
 6934 // implement storeImmD0 and storeDImmPacked
 6935 
 6936 // prefetch instructions
 6937 // Must be safe to execute with invalid address (cannot fault).
 6938 
 6939 instruct prefetchalloc( memory mem ) %{
 6940   match(PrefetchAllocation mem);
 6941 
 6942   ins_cost(INSN_COST);
 6943   format %{ "prfm $mem, PSTL1KEEP\t# Prefetch into level 1 cache write keep" %}
 6944 
 6945   ins_encode( aarch64_enc_prefetchw(mem) );
 6946 
 6947   ins_pipe(iload_prefetch);
 6948 %}
 6949 
 6950 //  ---------------- volatile loads and stores ----------------
 6951 
 6952 // Load Byte (8 bit signed)
 6953 instruct loadB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 6954 %{
 6955   match(Set dst (LoadB mem));
 6956 
 6957   ins_cost(VOLATILE_REF_COST);
 6958   format %{ "ldarsb  $dst, $mem\t# byte" %}
 6959 
 6960   ins_encode(aarch64_enc_ldarsb(dst, mem));
 6961 
 6962   ins_pipe(pipe_serial);
 6963 %}
 6964 
 6965 // Load Byte (8 bit signed) into long
 6966 instruct loadB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 6967 %{
 6968   match(Set dst (ConvI2L (LoadB mem)));
 6969 
 6970   ins_cost(VOLATILE_REF_COST);
 6971   format %{ "ldarsb  $dst, $mem\t# byte" %}
 6972 
 6973   ins_encode(aarch64_enc_ldarsb(dst, mem));
 6974 
 6975   ins_pipe(pipe_serial);
 6976 %}
 6977 
 6978 // Load Byte (8 bit unsigned)
 6979 instruct loadUB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 6980 %{
 6981   match(Set dst (LoadUB mem));
 6982 
 6983   ins_cost(VOLATILE_REF_COST);
 6984   format %{ "ldarb  $dst, $mem\t# byte" %}
 6985 
 6986   ins_encode(aarch64_enc_ldarb(dst, mem));
 6987 
 6988   ins_pipe(pipe_serial);
 6989 %}
 6990 
 6991 // Load Byte (8 bit unsigned) into long
 6992 instruct loadUB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 6993 %{
 6994   match(Set dst (ConvI2L (LoadUB mem)));
 6995 
 6996   ins_cost(VOLATILE_REF_COST);
 6997   format %{ "ldarb  $dst, $mem\t# byte" %}
 6998 
 6999   ins_encode(aarch64_enc_ldarb(dst, mem));
 7000 
 7001   ins_pipe(pipe_serial);
 7002 %}
 7003 
 7004 // Load Short (16 bit signed)
 7005 instruct loadS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 7006 %{
 7007   match(Set dst (LoadS mem));
 7008 
 7009   ins_cost(VOLATILE_REF_COST);
 7010   format %{ "ldarshw  $dst, $mem\t# short" %}
 7011 
 7012   ins_encode(aarch64_enc_ldarshw(dst, mem));
 7013 
 7014   ins_pipe(pipe_serial);
 7015 %}
 7016 
 7017 instruct loadUS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 7018 %{
 7019   match(Set dst (LoadUS mem));
 7020 
 7021   ins_cost(VOLATILE_REF_COST);
 7022   format %{ "ldarhw  $dst, $mem\t# short" %}
 7023 
 7024   ins_encode(aarch64_enc_ldarhw(dst, mem));
 7025 
 7026   ins_pipe(pipe_serial);
 7027 %}
 7028 
 7029 // Load Short/Char (16 bit unsigned) into long
 7030 instruct loadUS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 7031 %{
 7032   match(Set dst (ConvI2L (LoadUS mem)));
 7033 
 7034   ins_cost(VOLATILE_REF_COST);
 7035   format %{ "ldarh  $dst, $mem\t# short" %}
 7036 
 7037   ins_encode(aarch64_enc_ldarh(dst, mem));
 7038 
 7039   ins_pipe(pipe_serial);
 7040 %}
 7041 
 7042 // Load Short/Char (16 bit signed) into long
 7043 instruct loadS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 7044 %{
 7045   match(Set dst (ConvI2L (LoadS mem)));
 7046 
 7047   ins_cost(VOLATILE_REF_COST);
 7048   format %{ "ldarh  $dst, $mem\t# short" %}
 7049 
 7050   ins_encode(aarch64_enc_ldarsh(dst, mem));
 7051 
 7052   ins_pipe(pipe_serial);
 7053 %}
 7054 
 7055 // Load Integer (32 bit signed)
 7056 instruct loadI_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 7057 %{
 7058   match(Set dst (LoadI mem));
 7059 
 7060   ins_cost(VOLATILE_REF_COST);
 7061   format %{ "ldarw  $dst, $mem\t# int" %}
 7062 
 7063   ins_encode(aarch64_enc_ldarw(dst, mem));
 7064 
 7065   ins_pipe(pipe_serial);
 7066 %}
 7067 
 7068 // Load Integer (32 bit unsigned) into long
 7069 instruct loadUI2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem, immL_32bits mask)
 7070 %{
 7071   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
 7072 
 7073   ins_cost(VOLATILE_REF_COST);
 7074   format %{ "ldarw  $dst, $mem\t# int" %}
 7075 
 7076   ins_encode(aarch64_enc_ldarw(dst, mem));
 7077 
 7078   ins_pipe(pipe_serial);
 7079 %}
 7080 
 7081 // Load Long (64 bit signed)
 7082 instruct loadL_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 7083 %{
 7084   match(Set dst (LoadL mem));
 7085 
 7086   ins_cost(VOLATILE_REF_COST);
 7087   format %{ "ldar  $dst, $mem\t# int" %}
 7088 
 7089   ins_encode(aarch64_enc_ldar(dst, mem));
 7090 
 7091   ins_pipe(pipe_serial);
 7092 %}
 7093 
 7094 // Load Pointer
 7095 instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem)
 7096 %{
 7097   match(Set dst (LoadP mem));
 7098   predicate(n->as_Load()->barrier_data() == 0);
 7099 
 7100   ins_cost(VOLATILE_REF_COST);
 7101   format %{ "ldar  $dst, $mem\t# ptr" %}
 7102 
 7103   ins_encode(aarch64_enc_ldar(dst, mem));
 7104 
 7105   ins_pipe(pipe_serial);
 7106 %}
 7107 
 7108 // Load Compressed Pointer
 7109 instruct loadN_volatile(iRegNNoSp dst, /* sync_memory*/indirect mem)
 7110 %{
 7111   match(Set dst (LoadN mem));
 7112 
 7113   ins_cost(VOLATILE_REF_COST);
 7114   format %{ "ldarw  $dst, $mem\t# compressed ptr" %}
 7115 
 7116   ins_encode(aarch64_enc_ldarw(dst, mem));
 7117 
 7118   ins_pipe(pipe_serial);
 7119 %}
 7120 
 7121 // Load Float
 7122 instruct loadF_volatile(vRegF dst, /* sync_memory*/indirect mem)
 7123 %{
 7124   match(Set dst (LoadF mem));
 7125 
 7126   ins_cost(VOLATILE_REF_COST);
 7127   format %{ "ldars  $dst, $mem\t# float" %}
 7128 
 7129   ins_encode( aarch64_enc_fldars(dst, mem) );
 7130 
 7131   ins_pipe(pipe_serial);
 7132 %}
 7133 
 7134 // Load Double
 7135 instruct loadD_volatile(vRegD dst, /* sync_memory*/indirect mem)
 7136 %{
 7137   match(Set dst (LoadD mem));
 7138 
 7139   ins_cost(VOLATILE_REF_COST);
 7140   format %{ "ldard  $dst, $mem\t# double" %}
 7141 
 7142   ins_encode( aarch64_enc_fldard(dst, mem) );
 7143 
 7144   ins_pipe(pipe_serial);
 7145 %}
 7146 
 7147 // Store Byte
 7148 instruct storeB_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
 7149 %{
 7150   match(Set mem (StoreB mem src));
 7151 
 7152   ins_cost(VOLATILE_REF_COST);
 7153   format %{ "stlrb  $src, $mem\t# byte" %}
 7154 
 7155   ins_encode(aarch64_enc_stlrb(src, mem));
 7156 
 7157   ins_pipe(pipe_class_memory);
 7158 %}
 7159 
 7160 instruct storeimmB0_volatile(immI0 zero, /* sync_memory*/indirect mem)
 7161 %{
 7162   match(Set mem (StoreB mem zero));
 7163 
 7164   ins_cost(VOLATILE_REF_COST);
 7165   format %{ "stlrb  zr, $mem\t# byte" %}
 7166 
 7167   ins_encode(aarch64_enc_stlrb0(mem));
 7168 
 7169   ins_pipe(pipe_class_memory);
 7170 %}
 7171 
 7172 // Store Char/Short
 7173 instruct storeC_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
 7174 %{
 7175   match(Set mem (StoreC mem src));
 7176 
 7177   ins_cost(VOLATILE_REF_COST);
 7178   format %{ "stlrh  $src, $mem\t# short" %}
 7179 
 7180   ins_encode(aarch64_enc_stlrh(src, mem));
 7181 
 7182   ins_pipe(pipe_class_memory);
 7183 %}
 7184 
 7185 instruct storeimmC0_volatile(immI0 zero, /* sync_memory*/indirect mem)
 7186 %{
 7187   match(Set mem (StoreC mem zero));
 7188 
 7189   ins_cost(VOLATILE_REF_COST);
 7190   format %{ "stlrh  zr, $mem\t# short" %}
 7191 
 7192   ins_encode(aarch64_enc_stlrh0(mem));
 7193 
 7194   ins_pipe(pipe_class_memory);
 7195 %}
 7196 
 7197 // Store Integer
 7198 
 7199 instruct storeI_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
 7200 %{
 7201   match(Set mem(StoreI mem src));
 7202 
 7203   ins_cost(VOLATILE_REF_COST);
 7204   format %{ "stlrw  $src, $mem\t# int" %}
 7205 
 7206   ins_encode(aarch64_enc_stlrw(src, mem));
 7207 
 7208   ins_pipe(pipe_class_memory);
 7209 %}
 7210 
 7211 instruct storeimmI0_volatile(immI0 zero, /* sync_memory*/indirect mem)
 7212 %{
 7213   match(Set mem(StoreI mem zero));
 7214 
 7215   ins_cost(VOLATILE_REF_COST);
 7216   format %{ "stlrw  zr, $mem\t# int" %}
 7217 
 7218   ins_encode(aarch64_enc_stlrw0(mem));
 7219 
 7220   ins_pipe(pipe_class_memory);
 7221 %}
 7222 
 7223 // Store Long (64 bit signed)
 7224 instruct storeL_volatile(iRegL src, /* sync_memory*/indirect mem)
 7225 %{
 7226   match(Set mem (StoreL mem src));
 7227 
 7228   ins_cost(VOLATILE_REF_COST);
 7229   format %{ "stlr  $src, $mem\t# int" %}
 7230 
 7231   ins_encode(aarch64_enc_stlr(src, mem));
 7232 
 7233   ins_pipe(pipe_class_memory);
 7234 %}
 7235 
 7236 instruct storeimmL0_volatile(immL0 zero, /* sync_memory*/indirect mem)
 7237 %{
 7238   match(Set mem (StoreL mem zero));
 7239 
 7240   ins_cost(VOLATILE_REF_COST);
 7241   format %{ "stlr  zr, $mem\t# int" %}
 7242 
 7243   ins_encode(aarch64_enc_stlr0(mem));
 7244 
 7245   ins_pipe(pipe_class_memory);
 7246 %}
 7247 
 7248 // Store Pointer
 7249 instruct storeP_volatile(iRegP src, /* sync_memory*/indirect mem)
 7250 %{
 7251   match(Set mem (StoreP mem src));
 7252   predicate(n->as_Store()->barrier_data() == 0);
 7253 
 7254   ins_cost(VOLATILE_REF_COST);
 7255   format %{ "stlr  $src, $mem\t# ptr" %}
 7256 
 7257   ins_encode(aarch64_enc_stlr(src, mem));
 7258 
 7259   ins_pipe(pipe_class_memory);
 7260 %}
 7261 
 7262 instruct storeimmP0_volatile(immP0 zero, /* sync_memory*/indirect mem)
 7263 %{
 7264   match(Set mem (StoreP mem zero));
 7265   predicate(n->as_Store()->barrier_data() == 0);
 7266 
 7267   ins_cost(VOLATILE_REF_COST);
 7268   format %{ "stlr  zr, $mem\t# ptr" %}
 7269 
 7270   ins_encode(aarch64_enc_stlr0(mem));
 7271 
 7272   ins_pipe(pipe_class_memory);
 7273 %}
 7274 
 7275 // Store Compressed Pointer
 7276 instruct storeN_volatile(iRegN src, /* sync_memory*/indirect mem)
 7277 %{
 7278   match(Set mem (StoreN mem src));
 7279 
 7280   ins_cost(VOLATILE_REF_COST);
 7281   format %{ "stlrw  $src, $mem\t# compressed ptr" %}
 7282 
 7283   ins_encode(aarch64_enc_stlrw(src, mem));
 7284 
 7285   ins_pipe(pipe_class_memory);
 7286 %}
 7287 
 7288 instruct storeimmN0_volatile(immN0 zero, /* sync_memory*/indirect mem)
 7289 %{
 7290   match(Set mem (StoreN mem zero));
 7291 
 7292   ins_cost(VOLATILE_REF_COST);
 7293   format %{ "stlrw  zr, $mem\t# compressed ptr" %}
 7294 
 7295   ins_encode(aarch64_enc_stlrw0(mem));
 7296 
 7297   ins_pipe(pipe_class_memory);
 7298 %}
 7299 
 7300 // Store Float
 7301 instruct storeF_volatile(vRegF src, /* sync_memory*/indirect mem)
 7302 %{
 7303   match(Set mem (StoreF mem src));
 7304 
 7305   ins_cost(VOLATILE_REF_COST);
 7306   format %{ "stlrs  $src, $mem\t# float" %}
 7307 
 7308   ins_encode( aarch64_enc_fstlrs(src, mem) );
 7309 
 7310   ins_pipe(pipe_class_memory);
 7311 %}
 7312 
 7313 // TODO
 7314 // implement storeImmF0 and storeFImmPacked
 7315 
 7316 // Store Double
 7317 instruct storeD_volatile(vRegD src, /* sync_memory*/indirect mem)
 7318 %{
 7319   match(Set mem (StoreD mem src));
 7320 
 7321   ins_cost(VOLATILE_REF_COST);
 7322   format %{ "stlrd  $src, $mem\t# double" %}
 7323 
 7324   ins_encode( aarch64_enc_fstlrd(src, mem) );
 7325 
 7326   ins_pipe(pipe_class_memory);
 7327 %}
 7328 
 7329 //  ---------------- end of volatile loads and stores ----------------
 7330 
 7331 instruct cacheWB(indirect addr)
 7332 %{
 7333   predicate(VM_Version::supports_data_cache_line_flush());
 7334   match(CacheWB addr);
 7335 
 7336   ins_cost(100);
 7337   format %{"cache wb $addr" %}
 7338   ins_encode %{
 7339     assert($addr->index_position() < 0, "should be");
 7340     assert($addr$$disp == 0, "should be");
 7341     __ cache_wb(Address($addr$$base$$Register, 0));
 7342   %}
 7343   ins_pipe(pipe_slow); // XXX
 7344 %}
 7345 
 7346 instruct cacheWBPreSync()
 7347 %{
 7348   predicate(VM_Version::supports_data_cache_line_flush());
 7349   match(CacheWBPreSync);
 7350 
 7351   ins_cost(100);
 7352   format %{"cache wb presync" %}
 7353   ins_encode %{
 7354     __ cache_wbsync(true);
 7355   %}
 7356   ins_pipe(pipe_slow); // XXX
 7357 %}
 7358 
 7359 instruct cacheWBPostSync()
 7360 %{
 7361   predicate(VM_Version::supports_data_cache_line_flush());
 7362   match(CacheWBPostSync);
 7363 
 7364   ins_cost(100);
 7365   format %{"cache wb postsync" %}
 7366   ins_encode %{
 7367     __ cache_wbsync(false);
 7368   %}
 7369   ins_pipe(pipe_slow); // XXX
 7370 %}
 7371 
 7372 // ============================================================================
 7373 // BSWAP Instructions
 7374 
 7375 instruct bytes_reverse_int(iRegINoSp dst, iRegIorL2I src) %{
 7376   match(Set dst (ReverseBytesI src));
 7377 
 7378   ins_cost(INSN_COST);
 7379   format %{ "revw  $dst, $src" %}
 7380 
 7381   ins_encode %{
 7382     __ revw(as_Register($dst$$reg), as_Register($src$$reg));
 7383   %}
 7384 
 7385   ins_pipe(ialu_reg);
 7386 %}
 7387 
 7388 instruct bytes_reverse_long(iRegLNoSp dst, iRegL src) %{
 7389   match(Set dst (ReverseBytesL src));
 7390 
 7391   ins_cost(INSN_COST);
 7392   format %{ "rev  $dst, $src" %}
 7393 
 7394   ins_encode %{
 7395     __ rev(as_Register($dst$$reg), as_Register($src$$reg));
 7396   %}
 7397 
 7398   ins_pipe(ialu_reg);
 7399 %}
 7400 
 7401 instruct bytes_reverse_unsigned_short(iRegINoSp dst, iRegIorL2I src) %{
 7402   match(Set dst (ReverseBytesUS src));
 7403 
 7404   ins_cost(INSN_COST);
 7405   format %{ "rev16w  $dst, $src" %}
 7406 
 7407   ins_encode %{
 7408     __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
 7409   %}
 7410 
 7411   ins_pipe(ialu_reg);
 7412 %}
 7413 
 7414 instruct bytes_reverse_short(iRegINoSp dst, iRegIorL2I src) %{
 7415   match(Set dst (ReverseBytesS src));
 7416 
 7417   ins_cost(INSN_COST);
 7418   format %{ "rev16w  $dst, $src\n\t"
 7419             "sbfmw $dst, $dst, #0, #15" %}
 7420 
 7421   ins_encode %{
 7422     __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
 7423     __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 15U);
 7424   %}
 7425 
 7426   ins_pipe(ialu_reg);
 7427 %}
 7428 
 7429 // ============================================================================
 7430 // Zero Count Instructions
 7431 
 7432 instruct countLeadingZerosI(iRegINoSp dst, iRegIorL2I src) %{
 7433   match(Set dst (CountLeadingZerosI src));
 7434 
 7435   ins_cost(INSN_COST);
 7436   format %{ "clzw  $dst, $src" %}
 7437   ins_encode %{
 7438     __ clzw(as_Register($dst$$reg), as_Register($src$$reg));
 7439   %}
 7440 
 7441   ins_pipe(ialu_reg);
 7442 %}
 7443 
 7444 instruct countLeadingZerosL(iRegINoSp dst, iRegL src) %{
 7445   match(Set dst (CountLeadingZerosL src));
 7446 
 7447   ins_cost(INSN_COST);
 7448   format %{ "clz   $dst, $src" %}
 7449   ins_encode %{
 7450     __ clz(as_Register($dst$$reg), as_Register($src$$reg));
 7451   %}
 7452 
 7453   ins_pipe(ialu_reg);
 7454 %}
 7455 
 7456 instruct countTrailingZerosI(iRegINoSp dst, iRegIorL2I src) %{
 7457   match(Set dst (CountTrailingZerosI src));
 7458 
 7459   ins_cost(INSN_COST * 2);
 7460   format %{ "rbitw  $dst, $src\n\t"
 7461             "clzw   $dst, $dst" %}
 7462   ins_encode %{
 7463     __ rbitw(as_Register($dst$$reg), as_Register($src$$reg));
 7464     __ clzw(as_Register($dst$$reg), as_Register($dst$$reg));
 7465   %}
 7466 
 7467   ins_pipe(ialu_reg);
 7468 %}
 7469 
 7470 instruct countTrailingZerosL(iRegINoSp dst, iRegL src) %{
 7471   match(Set dst (CountTrailingZerosL src));
 7472 
 7473   ins_cost(INSN_COST * 2);
 7474   format %{ "rbit   $dst, $src\n\t"
 7475             "clz    $dst, $dst" %}
 7476   ins_encode %{
 7477     __ rbit(as_Register($dst$$reg), as_Register($src$$reg));
 7478     __ clz(as_Register($dst$$reg), as_Register($dst$$reg));
 7479   %}
 7480 
 7481   ins_pipe(ialu_reg);
 7482 %}
 7483 
 7484 //---------- Population Count Instructions -------------------------------------
 7485 //
 7486 
 7487 instruct popCountI(iRegINoSp dst, iRegIorL2I src, vRegF tmp) %{
 7488   match(Set dst (PopCountI src));
 7489   effect(TEMP tmp);
 7490   ins_cost(INSN_COST * 13);
 7491 
 7492   format %{ "movw   $src, $src\n\t"
 7493             "mov    $tmp, $src\t# vector (1D)\n\t"
 7494             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 7495             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 7496             "mov    $dst, $tmp\t# vector (1D)" %}
 7497   ins_encode %{
 7498     __ movw($src$$Register, $src$$Register); // ensure top 32 bits 0
 7499     __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
 7500     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 7501     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 7502     __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
 7503   %}
 7504 
 7505   ins_pipe(pipe_class_default);
 7506 %}
 7507 
 7508 instruct popCountI_mem(iRegINoSp dst, memory mem, vRegF tmp) %{
 7509   match(Set dst (PopCountI (LoadI mem)));
 7510   effect(TEMP tmp);
 7511   ins_cost(INSN_COST * 13);
 7512 
 7513   format %{ "ldrs   $tmp, $mem\n\t"
 7514             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 7515             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 7516             "mov    $dst, $tmp\t# vector (1D)" %}
 7517   ins_encode %{
 7518     FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
 7519     loadStore(masm, &MacroAssembler::ldrs, tmp_reg, $mem->opcode(),
 7520               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 7521     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 7522     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 7523     __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
 7524   %}
 7525 
 7526   ins_pipe(pipe_class_default);
 7527 %}
 7528 
 7529 // Note: Long.bitCount(long) returns an int.
 7530 instruct popCountL(iRegINoSp dst, iRegL src, vRegD tmp) %{
 7531   match(Set dst (PopCountL src));
 7532   effect(TEMP tmp);
 7533   ins_cost(INSN_COST * 13);
 7534 
 7535   format %{ "mov    $tmp, $src\t# vector (1D)\n\t"
 7536             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 7537             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 7538             "mov    $dst, $tmp\t# vector (1D)" %}
 7539   ins_encode %{
 7540     __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
 7541     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 7542     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 7543     __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
 7544   %}
 7545 
 7546   ins_pipe(pipe_class_default);
 7547 %}
 7548 
 7549 instruct popCountL_mem(iRegINoSp dst, memory mem, vRegD tmp) %{
 7550   match(Set dst (PopCountL (LoadL mem)));
 7551   effect(TEMP tmp);
 7552   ins_cost(INSN_COST * 13);
 7553 
 7554   format %{ "ldrd   $tmp, $mem\n\t"
 7555             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 7556             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 7557             "mov    $dst, $tmp\t# vector (1D)" %}
 7558   ins_encode %{
 7559     FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
 7560     loadStore(masm, &MacroAssembler::ldrd, tmp_reg, $mem->opcode(),
 7561               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 7562     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 7563     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 7564     __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
 7565   %}
 7566 
 7567   ins_pipe(pipe_class_default);
 7568 %}
 7569 
 7570 // ============================================================================
 7571 // VerifyVectorAlignment Instruction
 7572 
 7573 instruct verify_vector_alignment(iRegP addr, immL_positive_bitmaskI mask, rFlagsReg cr) %{
 7574   match(Set addr (VerifyVectorAlignment addr mask));
 7575   effect(KILL cr);
 7576   format %{ "verify_vector_alignment $addr $mask \t! verify alignment" %}
 7577   ins_encode %{
 7578     Label Lskip;
 7579     // check if masked bits of addr are zero
 7580     __ tst($addr$$Register, $mask$$constant);
 7581     __ br(Assembler::EQ, Lskip);
 7582     __ stop("verify_vector_alignment found a misaligned vector memory access");
 7583     __ bind(Lskip);
 7584   %}
 7585   ins_pipe(pipe_slow);
 7586 %}
 7587 
 7588 // ============================================================================
 7589 // MemBar Instruction
 7590 
 7591 instruct load_fence() %{
 7592   match(LoadFence);
 7593   ins_cost(VOLATILE_REF_COST);
 7594 
 7595   format %{ "load_fence" %}
 7596 
 7597   ins_encode %{
 7598     __ membar(Assembler::LoadLoad|Assembler::LoadStore);
 7599   %}
 7600   ins_pipe(pipe_serial);
 7601 %}
 7602 
 7603 instruct unnecessary_membar_acquire() %{
 7604   predicate(unnecessary_acquire(n));
 7605   match(MemBarAcquire);
 7606   ins_cost(0);
 7607 
 7608   format %{ "membar_acquire (elided)" %}
 7609 
 7610   ins_encode %{
 7611     __ block_comment("membar_acquire (elided)");
 7612   %}
 7613 
 7614   ins_pipe(pipe_class_empty);
 7615 %}
 7616 
 7617 instruct membar_acquire() %{
 7618   match(MemBarAcquire);
 7619   ins_cost(VOLATILE_REF_COST);
 7620 
 7621   format %{ "membar_acquire\n\t"
 7622             "dmb ishld" %}
 7623 
 7624   ins_encode %{
 7625     __ block_comment("membar_acquire");
 7626     __ membar(Assembler::LoadLoad|Assembler::LoadStore);
 7627   %}
 7628 
 7629   ins_pipe(pipe_serial);
 7630 %}
 7631 
 7632 
 7633 instruct membar_acquire_lock() %{
 7634   match(MemBarAcquireLock);
 7635   ins_cost(VOLATILE_REF_COST);
 7636 
 7637   format %{ "membar_acquire_lock (elided)" %}
 7638 
 7639   ins_encode %{
 7640     __ block_comment("membar_acquire_lock (elided)");
 7641   %}
 7642 
 7643   ins_pipe(pipe_serial);
 7644 %}
 7645 
 7646 instruct store_fence() %{
 7647   match(StoreFence);
 7648   ins_cost(VOLATILE_REF_COST);
 7649 
 7650   format %{ "store_fence" %}
 7651 
 7652   ins_encode %{
 7653     __ membar(Assembler::LoadStore|Assembler::StoreStore);
 7654   %}
 7655   ins_pipe(pipe_serial);
 7656 %}
 7657 
 7658 instruct unnecessary_membar_release() %{
 7659   predicate(unnecessary_release(n));
 7660   match(MemBarRelease);
 7661   ins_cost(0);
 7662 
 7663   format %{ "membar_release (elided)" %}
 7664 
 7665   ins_encode %{
 7666     __ block_comment("membar_release (elided)");
 7667   %}
 7668   ins_pipe(pipe_serial);
 7669 %}
 7670 
 7671 instruct membar_release() %{
 7672   match(MemBarRelease);
 7673   ins_cost(VOLATILE_REF_COST);
 7674 
 7675   format %{ "membar_release\n\t"
 7676             "dmb ishst\n\tdmb ishld" %}
 7677 
 7678   ins_encode %{
 7679     __ block_comment("membar_release");
 7680     // These will be merged if AlwaysMergeDMB is enabled.
 7681     __ membar(Assembler::StoreStore);
 7682     __ membar(Assembler::LoadStore);
 7683   %}
 7684   ins_pipe(pipe_serial);
 7685 %}
 7686 
 7687 instruct membar_storestore() %{
 7688   match(MemBarStoreStore);
 7689   match(StoreStoreFence);
 7690   ins_cost(VOLATILE_REF_COST);
 7691 
 7692   format %{ "MEMBAR-store-store" %}
 7693 
 7694   ins_encode %{
 7695     __ membar(Assembler::StoreStore);
 7696   %}
 7697   ins_pipe(pipe_serial);
 7698 %}
 7699 
 7700 instruct membar_release_lock() %{
 7701   match(MemBarReleaseLock);
 7702   ins_cost(VOLATILE_REF_COST);
 7703 
 7704   format %{ "membar_release_lock (elided)" %}
 7705 
 7706   ins_encode %{
 7707     __ block_comment("membar_release_lock (elided)");
 7708   %}
 7709 
 7710   ins_pipe(pipe_serial);
 7711 %}
 7712 
 7713 instruct unnecessary_membar_volatile() %{
 7714   predicate(unnecessary_volatile(n));
 7715   match(MemBarVolatile);
 7716   ins_cost(0);
 7717 
 7718   format %{ "membar_volatile (elided)" %}
 7719 
 7720   ins_encode %{
 7721     __ block_comment("membar_volatile (elided)");
 7722   %}
 7723 
 7724   ins_pipe(pipe_serial);
 7725 %}
 7726 
 7727 instruct membar_volatile() %{
 7728   match(MemBarVolatile);
 7729   ins_cost(VOLATILE_REF_COST*100);
 7730 
 7731   format %{ "membar_volatile\n\t"
 7732              "dmb ish"%}
 7733 
 7734   ins_encode %{
 7735     __ block_comment("membar_volatile");
 7736     __ membar(Assembler::StoreLoad);
 7737   %}
 7738 
 7739   ins_pipe(pipe_serial);
 7740 %}
 7741 
 7742 // ============================================================================
 7743 // Cast/Convert Instructions
 7744 
 7745 instruct castX2P(iRegPNoSp dst, iRegL src) %{
 7746   match(Set dst (CastX2P src));
 7747 
 7748   ins_cost(INSN_COST);
 7749   format %{ "mov $dst, $src\t# long -> ptr" %}
 7750 
 7751   ins_encode %{
 7752     if ($dst$$reg != $src$$reg) {
 7753       __ mov(as_Register($dst$$reg), as_Register($src$$reg));
 7754     }
 7755   %}
 7756 
 7757   ins_pipe(ialu_reg);
 7758 %}
 7759 
 7760 instruct castN2X(iRegLNoSp dst, iRegN src) %{
 7761   match(Set dst (CastP2X src));
 7762 
 7763   ins_cost(INSN_COST);
 7764   format %{ "mov $dst, $src\t# ptr -> long" %}
 7765 
 7766   ins_encode %{
 7767     if ($dst$$reg != $src$$reg) {
 7768       __ mov(as_Register($dst$$reg), as_Register($src$$reg));
 7769     }
 7770   %}
 7771 
 7772   ins_pipe(ialu_reg);
 7773 %}
 7774 
 7775 instruct castP2X(iRegLNoSp dst, iRegP src) %{
 7776   match(Set dst (CastP2X src));
 7777 
 7778   ins_cost(INSN_COST);
 7779   format %{ "mov $dst, $src\t# ptr -> long" %}
 7780 
 7781   ins_encode %{
 7782     if ($dst$$reg != $src$$reg) {
 7783       __ mov(as_Register($dst$$reg), as_Register($src$$reg));
 7784     }
 7785   %}
 7786 
 7787   ins_pipe(ialu_reg);
 7788 %}
 7789 
 7790 // Convert oop into int for vectors alignment masking
 7791 instruct convP2I(iRegINoSp dst, iRegP src) %{
 7792   match(Set dst (ConvL2I (CastP2X src)));
 7793 
 7794   ins_cost(INSN_COST);
 7795   format %{ "movw $dst, $src\t# ptr -> int" %}
 7796   ins_encode %{
 7797     __ movw($dst$$Register, $src$$Register);
 7798   %}
 7799 
 7800   ins_pipe(ialu_reg);
 7801 %}
 7802 
 7803 // Convert compressed oop into int for vectors alignment masking
 7804 // in case of 32bit oops (heap < 4Gb).
 7805 instruct convN2I(iRegINoSp dst, iRegN src)
 7806 %{
 7807   predicate(CompressedOops::shift() == 0);
 7808   match(Set dst (ConvL2I (CastP2X (DecodeN src))));
 7809 
 7810   ins_cost(INSN_COST);
 7811   format %{ "mov dst, $src\t# compressed ptr -> int" %}
 7812   ins_encode %{
 7813     __ movw($dst$$Register, $src$$Register);
 7814   %}
 7815 
 7816   ins_pipe(ialu_reg);
 7817 %}
 7818 
 7819 
 7820 // Convert oop pointer into compressed form
 7821 instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
 7822   predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
 7823   match(Set dst (EncodeP src));
 7824   effect(KILL cr);
 7825   ins_cost(INSN_COST * 3);
 7826   format %{ "encode_heap_oop $dst, $src" %}
 7827   ins_encode %{
 7828     Register s = $src$$Register;
 7829     Register d = $dst$$Register;
 7830     __ encode_heap_oop(d, s);
 7831   %}
 7832   ins_pipe(ialu_reg);
 7833 %}
 7834 
 7835 instruct encodeHeapOop_not_null(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
 7836   predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
 7837   match(Set dst (EncodeP src));
 7838   ins_cost(INSN_COST * 3);
 7839   format %{ "encode_heap_oop_not_null $dst, $src" %}
 7840   ins_encode %{
 7841     __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
 7842   %}
 7843   ins_pipe(ialu_reg);
 7844 %}
 7845 
 7846 instruct decodeHeapOop(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
 7847   predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
 7848             n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
 7849   match(Set dst (DecodeN src));
 7850   ins_cost(INSN_COST * 3);
 7851   format %{ "decode_heap_oop $dst, $src" %}
 7852   ins_encode %{
 7853     Register s = $src$$Register;
 7854     Register d = $dst$$Register;
 7855     __ decode_heap_oop(d, s);
 7856   %}
 7857   ins_pipe(ialu_reg);
 7858 %}
 7859 
 7860 instruct decodeHeapOop_not_null(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
 7861   predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
 7862             n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
 7863   match(Set dst (DecodeN src));
 7864   ins_cost(INSN_COST * 3);
 7865   format %{ "decode_heap_oop_not_null $dst, $src" %}
 7866   ins_encode %{
 7867     Register s = $src$$Register;
 7868     Register d = $dst$$Register;
 7869     __ decode_heap_oop_not_null(d, s);
 7870   %}
 7871   ins_pipe(ialu_reg);
 7872 %}
 7873 
 7874 // n.b. AArch64 implementations of encode_klass_not_null and
 7875 // decode_klass_not_null do not modify the flags register so, unlike
 7876 // Intel, we don't kill CR as a side effect here
 7877 
 7878 instruct encodeKlass_not_null(iRegNNoSp dst, iRegP src) %{
 7879   match(Set dst (EncodePKlass src));
 7880 
 7881   ins_cost(INSN_COST * 3);
 7882   format %{ "encode_klass_not_null $dst,$src" %}
 7883 
 7884   ins_encode %{
 7885     Register src_reg = as_Register($src$$reg);
 7886     Register dst_reg = as_Register($dst$$reg);
 7887     __ encode_klass_not_null(dst_reg, src_reg);
 7888   %}
 7889 
 7890    ins_pipe(ialu_reg);
 7891 %}
 7892 
 7893 instruct decodeKlass_not_null(iRegPNoSp dst, iRegN src) %{
 7894   match(Set dst (DecodeNKlass src));
 7895 
 7896   ins_cost(INSN_COST * 3);
 7897   format %{ "decode_klass_not_null $dst,$src" %}
 7898 
 7899   ins_encode %{
 7900     Register src_reg = as_Register($src$$reg);
 7901     Register dst_reg = as_Register($dst$$reg);
 7902     if (dst_reg != src_reg) {
 7903       __ decode_klass_not_null(dst_reg, src_reg);
 7904     } else {
 7905       __ decode_klass_not_null(dst_reg);
 7906     }
 7907   %}
 7908 
 7909    ins_pipe(ialu_reg);
 7910 %}
 7911 
 7912 instruct checkCastPP(iRegPNoSp dst)
 7913 %{
 7914   match(Set dst (CheckCastPP dst));
 7915 
 7916   size(0);
 7917   format %{ "# checkcastPP of $dst" %}
 7918   ins_encode(/* empty encoding */);
 7919   ins_pipe(pipe_class_empty);
 7920 %}
 7921 
 7922 instruct castPP(iRegPNoSp dst)
 7923 %{
 7924   match(Set dst (CastPP dst));
 7925 
 7926   size(0);
 7927   format %{ "# castPP of $dst" %}
 7928   ins_encode(/* empty encoding */);
 7929   ins_pipe(pipe_class_empty);
 7930 %}
 7931 
 7932 instruct castII(iRegI dst)
 7933 %{
 7934   match(Set dst (CastII dst));
 7935 
 7936   size(0);
 7937   format %{ "# castII of $dst" %}
 7938   ins_encode(/* empty encoding */);
 7939   ins_cost(0);
 7940   ins_pipe(pipe_class_empty);
 7941 %}
 7942 
 7943 instruct castLL(iRegL dst)
 7944 %{
 7945   match(Set dst (CastLL dst));
 7946 
 7947   size(0);
 7948   format %{ "# castLL of $dst" %}
 7949   ins_encode(/* empty encoding */);
 7950   ins_cost(0);
 7951   ins_pipe(pipe_class_empty);
 7952 %}
 7953 
 7954 instruct castFF(vRegF dst)
 7955 %{
 7956   match(Set dst (CastFF dst));
 7957 
 7958   size(0);
 7959   format %{ "# castFF of $dst" %}
 7960   ins_encode(/* empty encoding */);
 7961   ins_cost(0);
 7962   ins_pipe(pipe_class_empty);
 7963 %}
 7964 
 7965 instruct castDD(vRegD dst)
 7966 %{
 7967   match(Set dst (CastDD dst));
 7968 
 7969   size(0);
 7970   format %{ "# castDD of $dst" %}
 7971   ins_encode(/* empty encoding */);
 7972   ins_cost(0);
 7973   ins_pipe(pipe_class_empty);
 7974 %}
 7975 
 7976 instruct castVV(vReg dst)
 7977 %{
 7978   match(Set dst (CastVV dst));
 7979 
 7980   size(0);
 7981   format %{ "# castVV of $dst" %}
 7982   ins_encode(/* empty encoding */);
 7983   ins_cost(0);
 7984   ins_pipe(pipe_class_empty);
 7985 %}
 7986 
 7987 instruct castVVMask(pRegGov dst)
 7988 %{
 7989   match(Set dst (CastVV dst));
 7990 
 7991   size(0);
 7992   format %{ "# castVV of $dst" %}
 7993   ins_encode(/* empty encoding */);
 7994   ins_cost(0);
 7995   ins_pipe(pipe_class_empty);
 7996 %}
 7997 
 7998 // ============================================================================
 7999 // Atomic operation instructions
 8000 //
 8001 
 8002 // standard CompareAndSwapX when we are using barriers
 8003 // these have higher priority than the rules selected by a predicate
 8004 
 8005 // XXX No flag versions for CompareAndSwap{I,L,P,N} because matcher
 8006 // can't match them
 8007 
 8008 instruct compareAndSwapB(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 8009 
 8010   match(Set res (CompareAndSwapB mem (Binary oldval newval)));
 8011   ins_cost(2 * VOLATILE_REF_COST);
 8012 
 8013   effect(KILL cr);
 8014 
 8015   format %{
 8016     "cmpxchgb $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 8017     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8018   %}
 8019 
 8020   ins_encode(aarch64_enc_cmpxchgb(mem, oldval, newval),
 8021             aarch64_enc_cset_eq(res));
 8022 
 8023   ins_pipe(pipe_slow);
 8024 %}
 8025 
 8026 instruct compareAndSwapS(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 8027 
 8028   match(Set res (CompareAndSwapS mem (Binary oldval newval)));
 8029   ins_cost(2 * VOLATILE_REF_COST);
 8030 
 8031   effect(KILL cr);
 8032 
 8033   format %{
 8034     "cmpxchgs $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 8035     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8036   %}
 8037 
 8038   ins_encode(aarch64_enc_cmpxchgs(mem, oldval, newval),
 8039             aarch64_enc_cset_eq(res));
 8040 
 8041   ins_pipe(pipe_slow);
 8042 %}
 8043 
 8044 instruct compareAndSwapI(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 8045 
 8046   match(Set res (CompareAndSwapI mem (Binary oldval newval)));
 8047   ins_cost(2 * VOLATILE_REF_COST);
 8048 
 8049   effect(KILL cr);
 8050 
 8051  format %{
 8052     "cmpxchgw $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 8053     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8054  %}
 8055 
 8056  ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
 8057             aarch64_enc_cset_eq(res));
 8058 
 8059   ins_pipe(pipe_slow);
 8060 %}
 8061 
 8062 instruct compareAndSwapL(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
 8063 
 8064   match(Set res (CompareAndSwapL mem (Binary oldval newval)));
 8065   ins_cost(2 * VOLATILE_REF_COST);
 8066 
 8067   effect(KILL cr);
 8068 
 8069  format %{
 8070     "cmpxchg $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
 8071     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8072  %}
 8073 
 8074  ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
 8075             aarch64_enc_cset_eq(res));
 8076 
 8077   ins_pipe(pipe_slow);
 8078 %}
 8079 
 8080 instruct compareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 8081 
 8082   match(Set res (CompareAndSwapP mem (Binary oldval newval)));
 8083   predicate(n->as_LoadStore()->barrier_data() == 0);
 8084   ins_cost(2 * VOLATILE_REF_COST);
 8085 
 8086   effect(KILL cr);
 8087 
 8088  format %{
 8089     "cmpxchg $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
 8090     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8091  %}
 8092 
 8093  ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
 8094             aarch64_enc_cset_eq(res));
 8095 
 8096   ins_pipe(pipe_slow);
 8097 %}
 8098 
 8099 instruct compareAndSwapN(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
 8100 
 8101   match(Set res (CompareAndSwapN mem (Binary oldval newval)));
 8102   ins_cost(2 * VOLATILE_REF_COST);
 8103 
 8104   effect(KILL cr);
 8105 
 8106  format %{
 8107     "cmpxchgw $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
 8108     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8109  %}
 8110 
 8111  ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
 8112             aarch64_enc_cset_eq(res));
 8113 
 8114   ins_pipe(pipe_slow);
 8115 %}
 8116 
 8117 // alternative CompareAndSwapX when we are eliding barriers
 8118 
 8119 instruct compareAndSwapBAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 8120 
 8121   predicate(needs_acquiring_load_exclusive(n));
 8122   match(Set res (CompareAndSwapB mem (Binary oldval newval)));
 8123   ins_cost(VOLATILE_REF_COST);
 8124 
 8125   effect(KILL cr);
 8126 
 8127   format %{
 8128     "cmpxchgb_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 8129     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8130   %}
 8131 
 8132   ins_encode(aarch64_enc_cmpxchgb_acq(mem, oldval, newval),
 8133             aarch64_enc_cset_eq(res));
 8134 
 8135   ins_pipe(pipe_slow);
 8136 %}
 8137 
 8138 instruct compareAndSwapSAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 8139 
 8140   predicate(needs_acquiring_load_exclusive(n));
 8141   match(Set res (CompareAndSwapS mem (Binary oldval newval)));
 8142   ins_cost(VOLATILE_REF_COST);
 8143 
 8144   effect(KILL cr);
 8145 
 8146   format %{
 8147     "cmpxchgs_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 8148     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8149   %}
 8150 
 8151   ins_encode(aarch64_enc_cmpxchgs_acq(mem, oldval, newval),
 8152             aarch64_enc_cset_eq(res));
 8153 
 8154   ins_pipe(pipe_slow);
 8155 %}
 8156 
 8157 instruct compareAndSwapIAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 8158 
 8159   predicate(needs_acquiring_load_exclusive(n));
 8160   match(Set res (CompareAndSwapI mem (Binary oldval newval)));
 8161   ins_cost(VOLATILE_REF_COST);
 8162 
 8163   effect(KILL cr);
 8164 
 8165  format %{
 8166     "cmpxchgw_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 8167     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8168  %}
 8169 
 8170  ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
 8171             aarch64_enc_cset_eq(res));
 8172 
 8173   ins_pipe(pipe_slow);
 8174 %}
 8175 
 8176 instruct compareAndSwapLAcq(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
 8177 
 8178   predicate(needs_acquiring_load_exclusive(n));
 8179   match(Set res (CompareAndSwapL mem (Binary oldval newval)));
 8180   ins_cost(VOLATILE_REF_COST);
 8181 
 8182   effect(KILL cr);
 8183 
 8184  format %{
 8185     "cmpxchg_acq $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
 8186     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8187  %}
 8188 
 8189  ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
 8190             aarch64_enc_cset_eq(res));
 8191 
 8192   ins_pipe(pipe_slow);
 8193 %}
 8194 
 8195 instruct compareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 8196 
 8197   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 8198   match(Set res (CompareAndSwapP mem (Binary oldval newval)));
 8199   ins_cost(VOLATILE_REF_COST);
 8200 
 8201   effect(KILL cr);
 8202 
 8203  format %{
 8204     "cmpxchg_acq $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
 8205     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8206  %}
 8207 
 8208  ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
 8209             aarch64_enc_cset_eq(res));
 8210 
 8211   ins_pipe(pipe_slow);
 8212 %}
 8213 
 8214 instruct compareAndSwapNAcq(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
 8215 
 8216   predicate(needs_acquiring_load_exclusive(n));
 8217   match(Set res (CompareAndSwapN mem (Binary oldval newval)));
 8218   ins_cost(VOLATILE_REF_COST);
 8219 
 8220   effect(KILL cr);
 8221 
 8222  format %{
 8223     "cmpxchgw_acq $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
 8224     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8225  %}
 8226 
 8227  ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
 8228             aarch64_enc_cset_eq(res));
 8229 
 8230   ins_pipe(pipe_slow);
 8231 %}
 8232 
 8233 
 8234 // ---------------------------------------------------------------------
 8235 
 8236 // BEGIN This section of the file is automatically generated. Do not edit --------------
 8237 
 8238 // Sundry CAS operations.  Note that release is always true,
 8239 // regardless of the memory ordering of the CAS.  This is because we
 8240 // need the volatile case to be sequentially consistent but there is
 8241 // no trailing StoreLoad barrier emitted by C2.  Unfortunately we
 8242 // can't check the type of memory ordering here, so we always emit a
 8243 // STLXR.
 8244 
 8245 // This section is generated from cas.m4
 8246 
 8247 
 8248 // This pattern is generated automatically from cas.m4.
 8249 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8250 instruct compareAndExchangeB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 8251   match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
 8252   ins_cost(2 * VOLATILE_REF_COST);
 8253   effect(TEMP_DEF res, KILL cr);
 8254   format %{
 8255     "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 8256   %}
 8257   ins_encode %{
 8258     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8259                Assembler::byte, /*acquire*/ false, /*release*/ true,
 8260                /*weak*/ false, $res$$Register);
 8261     __ sxtbw($res$$Register, $res$$Register);
 8262   %}
 8263   ins_pipe(pipe_slow);
 8264 %}
 8265 
 8266 // This pattern is generated automatically from cas.m4.
 8267 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8268 instruct compareAndExchangeS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 8269   match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
 8270   ins_cost(2 * VOLATILE_REF_COST);
 8271   effect(TEMP_DEF res, KILL cr);
 8272   format %{
 8273     "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 8274   %}
 8275   ins_encode %{
 8276     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8277                Assembler::halfword, /*acquire*/ false, /*release*/ true,
 8278                /*weak*/ false, $res$$Register);
 8279     __ sxthw($res$$Register, $res$$Register);
 8280   %}
 8281   ins_pipe(pipe_slow);
 8282 %}
 8283 
 8284 // This pattern is generated automatically from cas.m4.
 8285 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8286 instruct compareAndExchangeI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 8287   match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
 8288   ins_cost(2 * VOLATILE_REF_COST);
 8289   effect(TEMP_DEF res, KILL cr);
 8290   format %{
 8291     "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 8292   %}
 8293   ins_encode %{
 8294     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8295                Assembler::word, /*acquire*/ false, /*release*/ true,
 8296                /*weak*/ false, $res$$Register);
 8297   %}
 8298   ins_pipe(pipe_slow);
 8299 %}
 8300 
 8301 // This pattern is generated automatically from cas.m4.
 8302 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8303 instruct compareAndExchangeL(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 8304   match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
 8305   ins_cost(2 * VOLATILE_REF_COST);
 8306   effect(TEMP_DEF res, KILL cr);
 8307   format %{
 8308     "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 8309   %}
 8310   ins_encode %{
 8311     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8312                Assembler::xword, /*acquire*/ false, /*release*/ true,
 8313                /*weak*/ false, $res$$Register);
 8314   %}
 8315   ins_pipe(pipe_slow);
 8316 %}
 8317 
 8318 // This pattern is generated automatically from cas.m4.
 8319 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8320 instruct compareAndExchangeN(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 8321   match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
 8322   ins_cost(2 * VOLATILE_REF_COST);
 8323   effect(TEMP_DEF res, KILL cr);
 8324   format %{
 8325     "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 8326   %}
 8327   ins_encode %{
 8328     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8329                Assembler::word, /*acquire*/ false, /*release*/ true,
 8330                /*weak*/ false, $res$$Register);
 8331   %}
 8332   ins_pipe(pipe_slow);
 8333 %}
 8334 
 8335 // This pattern is generated automatically from cas.m4.
 8336 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8337 instruct compareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 8338   predicate(n->as_LoadStore()->barrier_data() == 0);
 8339   match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
 8340   ins_cost(2 * VOLATILE_REF_COST);
 8341   effect(TEMP_DEF res, KILL cr);
 8342   format %{
 8343     "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 8344   %}
 8345   ins_encode %{
 8346     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8347                Assembler::xword, /*acquire*/ false, /*release*/ true,
 8348                /*weak*/ false, $res$$Register);
 8349   %}
 8350   ins_pipe(pipe_slow);
 8351 %}
 8352 
 8353 // This pattern is generated automatically from cas.m4.
 8354 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8355 instruct compareAndExchangeBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 8356   predicate(needs_acquiring_load_exclusive(n));
 8357   match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
 8358   ins_cost(VOLATILE_REF_COST);
 8359   effect(TEMP_DEF res, KILL cr);
 8360   format %{
 8361     "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 8362   %}
 8363   ins_encode %{
 8364     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8365                Assembler::byte, /*acquire*/ true, /*release*/ true,
 8366                /*weak*/ false, $res$$Register);
 8367     __ sxtbw($res$$Register, $res$$Register);
 8368   %}
 8369   ins_pipe(pipe_slow);
 8370 %}
 8371 
 8372 // This pattern is generated automatically from cas.m4.
 8373 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8374 instruct compareAndExchangeSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 8375   predicate(needs_acquiring_load_exclusive(n));
 8376   match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
 8377   ins_cost(VOLATILE_REF_COST);
 8378   effect(TEMP_DEF res, KILL cr);
 8379   format %{
 8380     "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 8381   %}
 8382   ins_encode %{
 8383     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8384                Assembler::halfword, /*acquire*/ true, /*release*/ true,
 8385                /*weak*/ false, $res$$Register);
 8386     __ sxthw($res$$Register, $res$$Register);
 8387   %}
 8388   ins_pipe(pipe_slow);
 8389 %}
 8390 
 8391 // This pattern is generated automatically from cas.m4.
 8392 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8393 instruct compareAndExchangeIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 8394   predicate(needs_acquiring_load_exclusive(n));
 8395   match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
 8396   ins_cost(VOLATILE_REF_COST);
 8397   effect(TEMP_DEF res, KILL cr);
 8398   format %{
 8399     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 8400   %}
 8401   ins_encode %{
 8402     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8403                Assembler::word, /*acquire*/ true, /*release*/ true,
 8404                /*weak*/ false, $res$$Register);
 8405   %}
 8406   ins_pipe(pipe_slow);
 8407 %}
 8408 
 8409 // This pattern is generated automatically from cas.m4.
 8410 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8411 instruct compareAndExchangeLAcq(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 8412   predicate(needs_acquiring_load_exclusive(n));
 8413   match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
 8414   ins_cost(VOLATILE_REF_COST);
 8415   effect(TEMP_DEF res, KILL cr);
 8416   format %{
 8417     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 8418   %}
 8419   ins_encode %{
 8420     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8421                Assembler::xword, /*acquire*/ true, /*release*/ true,
 8422                /*weak*/ false, $res$$Register);
 8423   %}
 8424   ins_pipe(pipe_slow);
 8425 %}
 8426 
 8427 // This pattern is generated automatically from cas.m4.
 8428 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8429 instruct compareAndExchangeNAcq(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 8430   predicate(needs_acquiring_load_exclusive(n));
 8431   match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
 8432   ins_cost(VOLATILE_REF_COST);
 8433   effect(TEMP_DEF res, KILL cr);
 8434   format %{
 8435     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 8436   %}
 8437   ins_encode %{
 8438     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8439                Assembler::word, /*acquire*/ true, /*release*/ true,
 8440                /*weak*/ false, $res$$Register);
 8441   %}
 8442   ins_pipe(pipe_slow);
 8443 %}
 8444 
 8445 // This pattern is generated automatically from cas.m4.
 8446 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8447 instruct compareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 8448   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 8449   match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
 8450   ins_cost(VOLATILE_REF_COST);
 8451   effect(TEMP_DEF res, KILL cr);
 8452   format %{
 8453     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 8454   %}
 8455   ins_encode %{
 8456     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8457                Assembler::xword, /*acquire*/ true, /*release*/ true,
 8458                /*weak*/ false, $res$$Register);
 8459   %}
 8460   ins_pipe(pipe_slow);
 8461 %}
 8462 
 8463 // This pattern is generated automatically from cas.m4.
 8464 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8465 instruct weakCompareAndSwapB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 8466   match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
 8467   ins_cost(2 * VOLATILE_REF_COST);
 8468   effect(KILL cr);
 8469   format %{
 8470     "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 8471     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8472   %}
 8473   ins_encode %{
 8474     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8475                Assembler::byte, /*acquire*/ false, /*release*/ true,
 8476                /*weak*/ true, noreg);
 8477     __ csetw($res$$Register, Assembler::EQ);
 8478   %}
 8479   ins_pipe(pipe_slow);
 8480 %}
 8481 
 8482 // This pattern is generated automatically from cas.m4.
 8483 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8484 instruct weakCompareAndSwapS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 8485   match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
 8486   ins_cost(2 * VOLATILE_REF_COST);
 8487   effect(KILL cr);
 8488   format %{
 8489     "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 8490     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8491   %}
 8492   ins_encode %{
 8493     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8494                Assembler::halfword, /*acquire*/ false, /*release*/ true,
 8495                /*weak*/ true, noreg);
 8496     __ csetw($res$$Register, Assembler::EQ);
 8497   %}
 8498   ins_pipe(pipe_slow);
 8499 %}
 8500 
 8501 // This pattern is generated automatically from cas.m4.
 8502 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8503 instruct weakCompareAndSwapI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 8504   match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
 8505   ins_cost(2 * VOLATILE_REF_COST);
 8506   effect(KILL cr);
 8507   format %{
 8508     "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 8509     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8510   %}
 8511   ins_encode %{
 8512     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8513                Assembler::word, /*acquire*/ false, /*release*/ true,
 8514                /*weak*/ true, noreg);
 8515     __ csetw($res$$Register, Assembler::EQ);
 8516   %}
 8517   ins_pipe(pipe_slow);
 8518 %}
 8519 
 8520 // This pattern is generated automatically from cas.m4.
 8521 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8522 instruct weakCompareAndSwapL(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 8523   match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
 8524   ins_cost(2 * VOLATILE_REF_COST);
 8525   effect(KILL cr);
 8526   format %{
 8527     "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 8528     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8529   %}
 8530   ins_encode %{
 8531     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8532                Assembler::xword, /*acquire*/ false, /*release*/ true,
 8533                /*weak*/ true, noreg);
 8534     __ csetw($res$$Register, Assembler::EQ);
 8535   %}
 8536   ins_pipe(pipe_slow);
 8537 %}
 8538 
 8539 // This pattern is generated automatically from cas.m4.
 8540 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8541 instruct weakCompareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 8542   match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
 8543   ins_cost(2 * VOLATILE_REF_COST);
 8544   effect(KILL cr);
 8545   format %{
 8546     "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 8547     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8548   %}
 8549   ins_encode %{
 8550     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8551                Assembler::word, /*acquire*/ false, /*release*/ true,
 8552                /*weak*/ true, noreg);
 8553     __ csetw($res$$Register, Assembler::EQ);
 8554   %}
 8555   ins_pipe(pipe_slow);
 8556 %}
 8557 
 8558 // This pattern is generated automatically from cas.m4.
 8559 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8560 instruct weakCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 8561   predicate(n->as_LoadStore()->barrier_data() == 0);
 8562   match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
 8563   ins_cost(2 * VOLATILE_REF_COST);
 8564   effect(KILL cr);
 8565   format %{
 8566     "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 8567     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8568   %}
 8569   ins_encode %{
 8570     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8571                Assembler::xword, /*acquire*/ false, /*release*/ true,
 8572                /*weak*/ true, noreg);
 8573     __ csetw($res$$Register, Assembler::EQ);
 8574   %}
 8575   ins_pipe(pipe_slow);
 8576 %}
 8577 
 8578 // This pattern is generated automatically from cas.m4.
 8579 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8580 instruct weakCompareAndSwapBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 8581   predicate(needs_acquiring_load_exclusive(n));
 8582   match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
 8583   ins_cost(VOLATILE_REF_COST);
 8584   effect(KILL cr);
 8585   format %{
 8586     "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 8587     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8588   %}
 8589   ins_encode %{
 8590     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8591                Assembler::byte, /*acquire*/ true, /*release*/ true,
 8592                /*weak*/ true, noreg);
 8593     __ csetw($res$$Register, Assembler::EQ);
 8594   %}
 8595   ins_pipe(pipe_slow);
 8596 %}
 8597 
 8598 // This pattern is generated automatically from cas.m4.
 8599 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8600 instruct weakCompareAndSwapSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 8601   predicate(needs_acquiring_load_exclusive(n));
 8602   match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
 8603   ins_cost(VOLATILE_REF_COST);
 8604   effect(KILL cr);
 8605   format %{
 8606     "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 8607     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8608   %}
 8609   ins_encode %{
 8610     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8611                Assembler::halfword, /*acquire*/ true, /*release*/ true,
 8612                /*weak*/ true, noreg);
 8613     __ csetw($res$$Register, Assembler::EQ);
 8614   %}
 8615   ins_pipe(pipe_slow);
 8616 %}
 8617 
 8618 // This pattern is generated automatically from cas.m4.
 8619 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8620 instruct weakCompareAndSwapIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 8621   predicate(needs_acquiring_load_exclusive(n));
 8622   match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
 8623   ins_cost(VOLATILE_REF_COST);
 8624   effect(KILL cr);
 8625   format %{
 8626     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 8627     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8628   %}
 8629   ins_encode %{
 8630     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8631                Assembler::word, /*acquire*/ true, /*release*/ true,
 8632                /*weak*/ true, noreg);
 8633     __ csetw($res$$Register, Assembler::EQ);
 8634   %}
 8635   ins_pipe(pipe_slow);
 8636 %}
 8637 
 8638 // This pattern is generated automatically from cas.m4.
 8639 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8640 instruct weakCompareAndSwapLAcq(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 8641   predicate(needs_acquiring_load_exclusive(n));
 8642   match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
 8643   ins_cost(VOLATILE_REF_COST);
 8644   effect(KILL cr);
 8645   format %{
 8646     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 8647     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8648   %}
 8649   ins_encode %{
 8650     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8651                Assembler::xword, /*acquire*/ true, /*release*/ true,
 8652                /*weak*/ true, noreg);
 8653     __ csetw($res$$Register, Assembler::EQ);
 8654   %}
 8655   ins_pipe(pipe_slow);
 8656 %}
 8657 
 8658 // This pattern is generated automatically from cas.m4.
 8659 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8660 instruct weakCompareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 8661   predicate(needs_acquiring_load_exclusive(n));
 8662   match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
 8663   ins_cost(VOLATILE_REF_COST);
 8664   effect(KILL cr);
 8665   format %{
 8666     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 8667     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8668   %}
 8669   ins_encode %{
 8670     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8671                Assembler::word, /*acquire*/ true, /*release*/ true,
 8672                /*weak*/ true, noreg);
 8673     __ csetw($res$$Register, Assembler::EQ);
 8674   %}
 8675   ins_pipe(pipe_slow);
 8676 %}
 8677 
 8678 // This pattern is generated automatically from cas.m4.
 8679 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8680 instruct weakCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 8681   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 8682   match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
 8683   ins_cost(VOLATILE_REF_COST);
 8684   effect(KILL cr);
 8685   format %{
 8686     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 8687     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8688   %}
 8689   ins_encode %{
 8690     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8691                Assembler::xword, /*acquire*/ true, /*release*/ true,
 8692                /*weak*/ true, noreg);
 8693     __ csetw($res$$Register, Assembler::EQ);
 8694   %}
 8695   ins_pipe(pipe_slow);
 8696 %}
 8697 
 8698 // END This section of the file is automatically generated. Do not edit --------------
 8699 // ---------------------------------------------------------------------
 8700 
 8701 instruct get_and_setI(indirect mem, iRegI newv, iRegINoSp prev) %{
 8702   match(Set prev (GetAndSetI mem newv));
 8703   ins_cost(2 * VOLATILE_REF_COST);
 8704   format %{ "atomic_xchgw  $prev, $newv, [$mem]" %}
 8705   ins_encode %{
 8706     __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 8707   %}
 8708   ins_pipe(pipe_serial);
 8709 %}
 8710 
 8711 instruct get_and_setL(indirect mem, iRegL newv, iRegLNoSp prev) %{
 8712   match(Set prev (GetAndSetL mem newv));
 8713   ins_cost(2 * VOLATILE_REF_COST);
 8714   format %{ "atomic_xchg  $prev, $newv, [$mem]" %}
 8715   ins_encode %{
 8716     __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
 8717   %}
 8718   ins_pipe(pipe_serial);
 8719 %}
 8720 
 8721 instruct get_and_setN(indirect mem, iRegN newv, iRegINoSp prev) %{
 8722   match(Set prev (GetAndSetN mem newv));
 8723   ins_cost(2 * VOLATILE_REF_COST);
 8724   format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
 8725   ins_encode %{
 8726     __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 8727   %}
 8728   ins_pipe(pipe_serial);
 8729 %}
 8730 
 8731 instruct get_and_setP(indirect mem, iRegP newv, iRegPNoSp prev) %{
 8732   predicate(n->as_LoadStore()->barrier_data() == 0);
 8733   match(Set prev (GetAndSetP mem newv));
 8734   ins_cost(2 * VOLATILE_REF_COST);
 8735   format %{ "atomic_xchg  $prev, $newv, [$mem]" %}
 8736   ins_encode %{
 8737     __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
 8738   %}
 8739   ins_pipe(pipe_serial);
 8740 %}
 8741 
 8742 instruct get_and_setIAcq(indirect mem, iRegI newv, iRegINoSp prev) %{
 8743   predicate(needs_acquiring_load_exclusive(n));
 8744   match(Set prev (GetAndSetI mem newv));
 8745   ins_cost(VOLATILE_REF_COST);
 8746   format %{ "atomic_xchgw_acq  $prev, $newv, [$mem]" %}
 8747   ins_encode %{
 8748     __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 8749   %}
 8750   ins_pipe(pipe_serial);
 8751 %}
 8752 
 8753 instruct get_and_setLAcq(indirect mem, iRegL newv, iRegLNoSp prev) %{
 8754   predicate(needs_acquiring_load_exclusive(n));
 8755   match(Set prev (GetAndSetL mem newv));
 8756   ins_cost(VOLATILE_REF_COST);
 8757   format %{ "atomic_xchg_acq  $prev, $newv, [$mem]" %}
 8758   ins_encode %{
 8759     __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
 8760   %}
 8761   ins_pipe(pipe_serial);
 8762 %}
 8763 
 8764 instruct get_and_setNAcq(indirect mem, iRegN newv, iRegINoSp prev) %{
 8765   predicate(needs_acquiring_load_exclusive(n));
 8766   match(Set prev (GetAndSetN mem newv));
 8767   ins_cost(VOLATILE_REF_COST);
 8768   format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
 8769   ins_encode %{
 8770     __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 8771   %}
 8772   ins_pipe(pipe_serial);
 8773 %}
 8774 
 8775 instruct get_and_setPAcq(indirect mem, iRegP newv, iRegPNoSp prev) %{
 8776   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 8777   match(Set prev (GetAndSetP mem newv));
 8778   ins_cost(VOLATILE_REF_COST);
 8779   format %{ "atomic_xchg_acq  $prev, $newv, [$mem]" %}
 8780   ins_encode %{
 8781     __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
 8782   %}
 8783   ins_pipe(pipe_serial);
 8784 %}
 8785 
 8786 
 8787 instruct get_and_addL(indirect mem, iRegLNoSp newval, iRegL incr) %{
 8788   match(Set newval (GetAndAddL mem incr));
 8789   ins_cost(2 * VOLATILE_REF_COST + 1);
 8790   format %{ "get_and_addL $newval, [$mem], $incr" %}
 8791   ins_encode %{
 8792     __ atomic_add($newval$$Register, $incr$$Register, as_Register($mem$$base));
 8793   %}
 8794   ins_pipe(pipe_serial);
 8795 %}
 8796 
 8797 instruct get_and_addL_no_res(indirect mem, Universe dummy, iRegL incr) %{
 8798   predicate(n->as_LoadStore()->result_not_used());
 8799   match(Set dummy (GetAndAddL mem incr));
 8800   ins_cost(2 * VOLATILE_REF_COST);
 8801   format %{ "get_and_addL [$mem], $incr" %}
 8802   ins_encode %{
 8803     __ atomic_add(noreg, $incr$$Register, as_Register($mem$$base));
 8804   %}
 8805   ins_pipe(pipe_serial);
 8806 %}
 8807 
 8808 instruct get_and_addLi(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
 8809   match(Set newval (GetAndAddL mem incr));
 8810   ins_cost(2 * VOLATILE_REF_COST + 1);
 8811   format %{ "get_and_addL $newval, [$mem], $incr" %}
 8812   ins_encode %{
 8813     __ atomic_add($newval$$Register, $incr$$constant, as_Register($mem$$base));
 8814   %}
 8815   ins_pipe(pipe_serial);
 8816 %}
 8817 
 8818 instruct get_and_addLi_no_res(indirect mem, Universe dummy, immLAddSub incr) %{
 8819   predicate(n->as_LoadStore()->result_not_used());
 8820   match(Set dummy (GetAndAddL mem incr));
 8821   ins_cost(2 * VOLATILE_REF_COST);
 8822   format %{ "get_and_addL [$mem], $incr" %}
 8823   ins_encode %{
 8824     __ atomic_add(noreg, $incr$$constant, as_Register($mem$$base));
 8825   %}
 8826   ins_pipe(pipe_serial);
 8827 %}
 8828 
 8829 instruct get_and_addI(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
 8830   match(Set newval (GetAndAddI mem incr));
 8831   ins_cost(2 * VOLATILE_REF_COST + 1);
 8832   format %{ "get_and_addI $newval, [$mem], $incr" %}
 8833   ins_encode %{
 8834     __ atomic_addw($newval$$Register, $incr$$Register, as_Register($mem$$base));
 8835   %}
 8836   ins_pipe(pipe_serial);
 8837 %}
 8838 
 8839 instruct get_and_addI_no_res(indirect mem, Universe dummy, iRegIorL2I incr) %{
 8840   predicate(n->as_LoadStore()->result_not_used());
 8841   match(Set dummy (GetAndAddI mem incr));
 8842   ins_cost(2 * VOLATILE_REF_COST);
 8843   format %{ "get_and_addI [$mem], $incr" %}
 8844   ins_encode %{
 8845     __ atomic_addw(noreg, $incr$$Register, as_Register($mem$$base));
 8846   %}
 8847   ins_pipe(pipe_serial);
 8848 %}
 8849 
 8850 instruct get_and_addIi(indirect mem, iRegINoSp newval, immIAddSub incr) %{
 8851   match(Set newval (GetAndAddI mem incr));
 8852   ins_cost(2 * VOLATILE_REF_COST + 1);
 8853   format %{ "get_and_addI $newval, [$mem], $incr" %}
 8854   ins_encode %{
 8855     __ atomic_addw($newval$$Register, $incr$$constant, as_Register($mem$$base));
 8856   %}
 8857   ins_pipe(pipe_serial);
 8858 %}
 8859 
 8860 instruct get_and_addIi_no_res(indirect mem, Universe dummy, immIAddSub incr) %{
 8861   predicate(n->as_LoadStore()->result_not_used());
 8862   match(Set dummy (GetAndAddI mem incr));
 8863   ins_cost(2 * VOLATILE_REF_COST);
 8864   format %{ "get_and_addI [$mem], $incr" %}
 8865   ins_encode %{
 8866     __ atomic_addw(noreg, $incr$$constant, as_Register($mem$$base));
 8867   %}
 8868   ins_pipe(pipe_serial);
 8869 %}
 8870 
 8871 instruct get_and_addLAcq(indirect mem, iRegLNoSp newval, iRegL incr) %{
 8872   predicate(needs_acquiring_load_exclusive(n));
 8873   match(Set newval (GetAndAddL mem incr));
 8874   ins_cost(VOLATILE_REF_COST + 1);
 8875   format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
 8876   ins_encode %{
 8877     __ atomic_addal($newval$$Register, $incr$$Register, as_Register($mem$$base));
 8878   %}
 8879   ins_pipe(pipe_serial);
 8880 %}
 8881 
 8882 instruct get_and_addL_no_resAcq(indirect mem, Universe dummy, iRegL incr) %{
 8883   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
 8884   match(Set dummy (GetAndAddL mem incr));
 8885   ins_cost(VOLATILE_REF_COST);
 8886   format %{ "get_and_addL_acq [$mem], $incr" %}
 8887   ins_encode %{
 8888     __ atomic_addal(noreg, $incr$$Register, as_Register($mem$$base));
 8889   %}
 8890   ins_pipe(pipe_serial);
 8891 %}
 8892 
 8893 instruct get_and_addLiAcq(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
 8894   predicate(needs_acquiring_load_exclusive(n));
 8895   match(Set newval (GetAndAddL mem incr));
 8896   ins_cost(VOLATILE_REF_COST + 1);
 8897   format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
 8898   ins_encode %{
 8899     __ atomic_addal($newval$$Register, $incr$$constant, as_Register($mem$$base));
 8900   %}
 8901   ins_pipe(pipe_serial);
 8902 %}
 8903 
 8904 instruct get_and_addLi_no_resAcq(indirect mem, Universe dummy, immLAddSub incr) %{
 8905   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
 8906   match(Set dummy (GetAndAddL mem incr));
 8907   ins_cost(VOLATILE_REF_COST);
 8908   format %{ "get_and_addL_acq [$mem], $incr" %}
 8909   ins_encode %{
 8910     __ atomic_addal(noreg, $incr$$constant, as_Register($mem$$base));
 8911   %}
 8912   ins_pipe(pipe_serial);
 8913 %}
 8914 
 8915 instruct get_and_addIAcq(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
 8916   predicate(needs_acquiring_load_exclusive(n));
 8917   match(Set newval (GetAndAddI mem incr));
 8918   ins_cost(VOLATILE_REF_COST + 1);
 8919   format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
 8920   ins_encode %{
 8921     __ atomic_addalw($newval$$Register, $incr$$Register, as_Register($mem$$base));
 8922   %}
 8923   ins_pipe(pipe_serial);
 8924 %}
 8925 
 8926 instruct get_and_addI_no_resAcq(indirect mem, Universe dummy, iRegIorL2I incr) %{
 8927   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
 8928   match(Set dummy (GetAndAddI mem incr));
 8929   ins_cost(VOLATILE_REF_COST);
 8930   format %{ "get_and_addI_acq [$mem], $incr" %}
 8931   ins_encode %{
 8932     __ atomic_addalw(noreg, $incr$$Register, as_Register($mem$$base));
 8933   %}
 8934   ins_pipe(pipe_serial);
 8935 %}
 8936 
 8937 instruct get_and_addIiAcq(indirect mem, iRegINoSp newval, immIAddSub incr) %{
 8938   predicate(needs_acquiring_load_exclusive(n));
 8939   match(Set newval (GetAndAddI mem incr));
 8940   ins_cost(VOLATILE_REF_COST + 1);
 8941   format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
 8942   ins_encode %{
 8943     __ atomic_addalw($newval$$Register, $incr$$constant, as_Register($mem$$base));
 8944   %}
 8945   ins_pipe(pipe_serial);
 8946 %}
 8947 
 8948 instruct get_and_addIi_no_resAcq(indirect mem, Universe dummy, immIAddSub incr) %{
 8949   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
 8950   match(Set dummy (GetAndAddI mem incr));
 8951   ins_cost(VOLATILE_REF_COST);
 8952   format %{ "get_and_addI_acq [$mem], $incr" %}
 8953   ins_encode %{
 8954     __ atomic_addalw(noreg, $incr$$constant, as_Register($mem$$base));
 8955   %}
 8956   ins_pipe(pipe_serial);
 8957 %}
 8958 
 8959 // Manifest a CmpU result in an integer register.
 8960 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
 8961 instruct cmpU3_reg_reg(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg flags)
 8962 %{
 8963   match(Set dst (CmpU3 src1 src2));
 8964   effect(KILL flags);
 8965 
 8966   ins_cost(INSN_COST * 3);
 8967   format %{
 8968       "cmpw $src1, $src2\n\t"
 8969       "csetw $dst, ne\n\t"
 8970       "cnegw $dst, lo\t# CmpU3(reg)"
 8971   %}
 8972   ins_encode %{
 8973     __ cmpw($src1$$Register, $src2$$Register);
 8974     __ csetw($dst$$Register, Assembler::NE);
 8975     __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
 8976   %}
 8977 
 8978   ins_pipe(pipe_class_default);
 8979 %}
 8980 
 8981 instruct cmpU3_reg_imm(iRegINoSp dst, iRegI src1, immIAddSub src2, rFlagsReg flags)
 8982 %{
 8983   match(Set dst (CmpU3 src1 src2));
 8984   effect(KILL flags);
 8985 
 8986   ins_cost(INSN_COST * 3);
 8987   format %{
 8988       "subsw zr, $src1, $src2\n\t"
 8989       "csetw $dst, ne\n\t"
 8990       "cnegw $dst, lo\t# CmpU3(imm)"
 8991   %}
 8992   ins_encode %{
 8993     __ subsw(zr, $src1$$Register, (int32_t)$src2$$constant);
 8994     __ csetw($dst$$Register, Assembler::NE);
 8995     __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
 8996   %}
 8997 
 8998   ins_pipe(pipe_class_default);
 8999 %}
 9000 
 9001 // Manifest a CmpUL result in an integer register.
 9002 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
 9003 instruct cmpUL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
 9004 %{
 9005   match(Set dst (CmpUL3 src1 src2));
 9006   effect(KILL flags);
 9007 
 9008   ins_cost(INSN_COST * 3);
 9009   format %{
 9010       "cmp $src1, $src2\n\t"
 9011       "csetw $dst, ne\n\t"
 9012       "cnegw $dst, lo\t# CmpUL3(reg)"
 9013   %}
 9014   ins_encode %{
 9015     __ cmp($src1$$Register, $src2$$Register);
 9016     __ csetw($dst$$Register, Assembler::NE);
 9017     __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
 9018   %}
 9019 
 9020   ins_pipe(pipe_class_default);
 9021 %}
 9022 
 9023 instruct cmpUL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
 9024 %{
 9025   match(Set dst (CmpUL3 src1 src2));
 9026   effect(KILL flags);
 9027 
 9028   ins_cost(INSN_COST * 3);
 9029   format %{
 9030       "subs zr, $src1, $src2\n\t"
 9031       "csetw $dst, ne\n\t"
 9032       "cnegw $dst, lo\t# CmpUL3(imm)"
 9033   %}
 9034   ins_encode %{
 9035     __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
 9036     __ csetw($dst$$Register, Assembler::NE);
 9037     __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
 9038   %}
 9039 
 9040   ins_pipe(pipe_class_default);
 9041 %}
 9042 
 9043 // Manifest a CmpL result in an integer register.
 9044 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
 9045 instruct cmpL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
 9046 %{
 9047   match(Set dst (CmpL3 src1 src2));
 9048   effect(KILL flags);
 9049 
 9050   ins_cost(INSN_COST * 3);
 9051   format %{
 9052       "cmp $src1, $src2\n\t"
 9053       "csetw $dst, ne\n\t"
 9054       "cnegw $dst, lt\t# CmpL3(reg)"
 9055   %}
 9056   ins_encode %{
 9057     __ cmp($src1$$Register, $src2$$Register);
 9058     __ csetw($dst$$Register, Assembler::NE);
 9059     __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
 9060   %}
 9061 
 9062   ins_pipe(pipe_class_default);
 9063 %}
 9064 
 9065 instruct cmpL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
 9066 %{
 9067   match(Set dst (CmpL3 src1 src2));
 9068   effect(KILL flags);
 9069 
 9070   ins_cost(INSN_COST * 3);
 9071   format %{
 9072       "subs zr, $src1, $src2\n\t"
 9073       "csetw $dst, ne\n\t"
 9074       "cnegw $dst, lt\t# CmpL3(imm)"
 9075   %}
 9076   ins_encode %{
 9077     __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
 9078     __ csetw($dst$$Register, Assembler::NE);
 9079     __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
 9080   %}
 9081 
 9082   ins_pipe(pipe_class_default);
 9083 %}
 9084 
 9085 // ============================================================================
 9086 // Conditional Move Instructions
 9087 
 9088 // n.b. we have identical rules for both a signed compare op (cmpOp)
 9089 // and an unsigned compare op (cmpOpU). it would be nice if we could
 9090 // define an op class which merged both inputs and use it to type the
 9091 // argument to a single rule. unfortunatelyt his fails because the
 9092 // opclass does not live up to the COND_INTER interface of its
 9093 // component operands. When the generic code tries to negate the
 9094 // operand it ends up running the generci Machoper::negate method
 9095 // which throws a ShouldNotHappen. So, we have to provide two flavours
 9096 // of each rule, one for a cmpOp and a second for a cmpOpU (sigh).
 9097 
 9098 instruct cmovI_reg_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
 9099   match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
 9100 
 9101   ins_cost(INSN_COST * 2);
 9102   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, int"  %}
 9103 
 9104   ins_encode %{
 9105     __ cselw(as_Register($dst$$reg),
 9106              as_Register($src2$$reg),
 9107              as_Register($src1$$reg),
 9108              (Assembler::Condition)$cmp$$cmpcode);
 9109   %}
 9110 
 9111   ins_pipe(icond_reg_reg);
 9112 %}
 9113 
 9114 instruct cmovUI_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
 9115   match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
 9116 
 9117   ins_cost(INSN_COST * 2);
 9118   format %{ "cselw $dst, $src2, $src1 $cmp\t# unsigned, int"  %}
 9119 
 9120   ins_encode %{
 9121     __ cselw(as_Register($dst$$reg),
 9122              as_Register($src2$$reg),
 9123              as_Register($src1$$reg),
 9124              (Assembler::Condition)$cmp$$cmpcode);
 9125   %}
 9126 
 9127   ins_pipe(icond_reg_reg);
 9128 %}
 9129 
 9130 // special cases where one arg is zero
 9131 
 9132 // n.b. this is selected in preference to the rule above because it
 9133 // avoids loading constant 0 into a source register
 9134 
 9135 // TODO
 9136 // we ought only to be able to cull one of these variants as the ideal
 9137 // transforms ought always to order the zero consistently (to left/right?)
 9138 
 9139 instruct cmovI_zero_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
 9140   match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
 9141 
 9142   ins_cost(INSN_COST * 2);
 9143   format %{ "cselw $dst, $src, zr $cmp\t# signed, int"  %}
 9144 
 9145   ins_encode %{
 9146     __ cselw(as_Register($dst$$reg),
 9147              as_Register($src$$reg),
 9148              zr,
 9149              (Assembler::Condition)$cmp$$cmpcode);
 9150   %}
 9151 
 9152   ins_pipe(icond_reg);
 9153 %}
 9154 
 9155 instruct cmovUI_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
 9156   match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
 9157 
 9158   ins_cost(INSN_COST * 2);
 9159   format %{ "cselw $dst, $src, zr $cmp\t# unsigned, int"  %}
 9160 
 9161   ins_encode %{
 9162     __ cselw(as_Register($dst$$reg),
 9163              as_Register($src$$reg),
 9164              zr,
 9165              (Assembler::Condition)$cmp$$cmpcode);
 9166   %}
 9167 
 9168   ins_pipe(icond_reg);
 9169 %}
 9170 
 9171 instruct cmovI_reg_zero(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
 9172   match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
 9173 
 9174   ins_cost(INSN_COST * 2);
 9175   format %{ "cselw $dst, zr, $src $cmp\t# signed, int"  %}
 9176 
 9177   ins_encode %{
 9178     __ cselw(as_Register($dst$$reg),
 9179              zr,
 9180              as_Register($src$$reg),
 9181              (Assembler::Condition)$cmp$$cmpcode);
 9182   %}
 9183 
 9184   ins_pipe(icond_reg);
 9185 %}
 9186 
 9187 instruct cmovUI_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
 9188   match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
 9189 
 9190   ins_cost(INSN_COST * 2);
 9191   format %{ "cselw $dst, zr, $src $cmp\t# unsigned, int"  %}
 9192 
 9193   ins_encode %{
 9194     __ cselw(as_Register($dst$$reg),
 9195              zr,
 9196              as_Register($src$$reg),
 9197              (Assembler::Condition)$cmp$$cmpcode);
 9198   %}
 9199 
 9200   ins_pipe(icond_reg);
 9201 %}
 9202 
 9203 // special case for creating a boolean 0 or 1
 9204 
 9205 // n.b. this is selected in preference to the rule above because it
 9206 // avoids loading constants 0 and 1 into a source register
 9207 
 9208 instruct cmovI_reg_zero_one(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
 9209   match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
 9210 
 9211   ins_cost(INSN_COST * 2);
 9212   format %{ "csincw $dst, zr, zr $cmp\t# signed, int"  %}
 9213 
 9214   ins_encode %{
 9215     // equivalently
 9216     // cset(as_Register($dst$$reg),
 9217     //      negate_condition((Assembler::Condition)$cmp$$cmpcode));
 9218     __ csincw(as_Register($dst$$reg),
 9219              zr,
 9220              zr,
 9221              (Assembler::Condition)$cmp$$cmpcode);
 9222   %}
 9223 
 9224   ins_pipe(icond_none);
 9225 %}
 9226 
 9227 instruct cmovUI_reg_zero_one(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
 9228   match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
 9229 
 9230   ins_cost(INSN_COST * 2);
 9231   format %{ "csincw $dst, zr, zr $cmp\t# unsigned, int"  %}
 9232 
 9233   ins_encode %{
 9234     // equivalently
 9235     // cset(as_Register($dst$$reg),
 9236     //      negate_condition((Assembler::Condition)$cmp$$cmpcode));
 9237     __ csincw(as_Register($dst$$reg),
 9238              zr,
 9239              zr,
 9240              (Assembler::Condition)$cmp$$cmpcode);
 9241   %}
 9242 
 9243   ins_pipe(icond_none);
 9244 %}
 9245 
 9246 instruct cmovL_reg_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
 9247   match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
 9248 
 9249   ins_cost(INSN_COST * 2);
 9250   format %{ "csel $dst, $src2, $src1 $cmp\t# signed, long"  %}
 9251 
 9252   ins_encode %{
 9253     __ csel(as_Register($dst$$reg),
 9254             as_Register($src2$$reg),
 9255             as_Register($src1$$reg),
 9256             (Assembler::Condition)$cmp$$cmpcode);
 9257   %}
 9258 
 9259   ins_pipe(icond_reg_reg);
 9260 %}
 9261 
 9262 instruct cmovUL_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
 9263   match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
 9264 
 9265   ins_cost(INSN_COST * 2);
 9266   format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, long"  %}
 9267 
 9268   ins_encode %{
 9269     __ csel(as_Register($dst$$reg),
 9270             as_Register($src2$$reg),
 9271             as_Register($src1$$reg),
 9272             (Assembler::Condition)$cmp$$cmpcode);
 9273   %}
 9274 
 9275   ins_pipe(icond_reg_reg);
 9276 %}
 9277 
 9278 // special cases where one arg is zero
 9279 
 9280 instruct cmovL_reg_zero(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
 9281   match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
 9282 
 9283   ins_cost(INSN_COST * 2);
 9284   format %{ "csel $dst, zr, $src $cmp\t# signed, long"  %}
 9285 
 9286   ins_encode %{
 9287     __ csel(as_Register($dst$$reg),
 9288             zr,
 9289             as_Register($src$$reg),
 9290             (Assembler::Condition)$cmp$$cmpcode);
 9291   %}
 9292 
 9293   ins_pipe(icond_reg);
 9294 %}
 9295 
 9296 instruct cmovUL_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
 9297   match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
 9298 
 9299   ins_cost(INSN_COST * 2);
 9300   format %{ "csel $dst, zr, $src $cmp\t# unsigned, long"  %}
 9301 
 9302   ins_encode %{
 9303     __ csel(as_Register($dst$$reg),
 9304             zr,
 9305             as_Register($src$$reg),
 9306             (Assembler::Condition)$cmp$$cmpcode);
 9307   %}
 9308 
 9309   ins_pipe(icond_reg);
 9310 %}
 9311 
 9312 instruct cmovL_zero_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
 9313   match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
 9314 
 9315   ins_cost(INSN_COST * 2);
 9316   format %{ "csel $dst, $src, zr $cmp\t# signed, long"  %}
 9317 
 9318   ins_encode %{
 9319     __ csel(as_Register($dst$$reg),
 9320             as_Register($src$$reg),
 9321             zr,
 9322             (Assembler::Condition)$cmp$$cmpcode);
 9323   %}
 9324 
 9325   ins_pipe(icond_reg);
 9326 %}
 9327 
 9328 instruct cmovUL_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
 9329   match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
 9330 
 9331   ins_cost(INSN_COST * 2);
 9332   format %{ "csel $dst, $src, zr $cmp\t# unsigned, long"  %}
 9333 
 9334   ins_encode %{
 9335     __ csel(as_Register($dst$$reg),
 9336             as_Register($src$$reg),
 9337             zr,
 9338             (Assembler::Condition)$cmp$$cmpcode);
 9339   %}
 9340 
 9341   ins_pipe(icond_reg);
 9342 %}
 9343 
 9344 instruct cmovP_reg_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
 9345   match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
 9346 
 9347   ins_cost(INSN_COST * 2);
 9348   format %{ "csel $dst, $src2, $src1 $cmp\t# signed, ptr"  %}
 9349 
 9350   ins_encode %{
 9351     __ csel(as_Register($dst$$reg),
 9352             as_Register($src2$$reg),
 9353             as_Register($src1$$reg),
 9354             (Assembler::Condition)$cmp$$cmpcode);
 9355   %}
 9356 
 9357   ins_pipe(icond_reg_reg);
 9358 %}
 9359 
 9360 instruct cmovUP_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
 9361   match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
 9362 
 9363   ins_cost(INSN_COST * 2);
 9364   format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, ptr"  %}
 9365 
 9366   ins_encode %{
 9367     __ csel(as_Register($dst$$reg),
 9368             as_Register($src2$$reg),
 9369             as_Register($src1$$reg),
 9370             (Assembler::Condition)$cmp$$cmpcode);
 9371   %}
 9372 
 9373   ins_pipe(icond_reg_reg);
 9374 %}
 9375 
 9376 // special cases where one arg is zero
 9377 
 9378 instruct cmovP_reg_zero(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
 9379   match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
 9380 
 9381   ins_cost(INSN_COST * 2);
 9382   format %{ "csel $dst, zr, $src $cmp\t# signed, ptr"  %}
 9383 
 9384   ins_encode %{
 9385     __ csel(as_Register($dst$$reg),
 9386             zr,
 9387             as_Register($src$$reg),
 9388             (Assembler::Condition)$cmp$$cmpcode);
 9389   %}
 9390 
 9391   ins_pipe(icond_reg);
 9392 %}
 9393 
 9394 instruct cmovUP_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
 9395   match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
 9396 
 9397   ins_cost(INSN_COST * 2);
 9398   format %{ "csel $dst, zr, $src $cmp\t# unsigned, ptr"  %}
 9399 
 9400   ins_encode %{
 9401     __ csel(as_Register($dst$$reg),
 9402             zr,
 9403             as_Register($src$$reg),
 9404             (Assembler::Condition)$cmp$$cmpcode);
 9405   %}
 9406 
 9407   ins_pipe(icond_reg);
 9408 %}
 9409 
 9410 instruct cmovP_zero_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
 9411   match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
 9412 
 9413   ins_cost(INSN_COST * 2);
 9414   format %{ "csel $dst, $src, zr $cmp\t# signed, ptr"  %}
 9415 
 9416   ins_encode %{
 9417     __ csel(as_Register($dst$$reg),
 9418             as_Register($src$$reg),
 9419             zr,
 9420             (Assembler::Condition)$cmp$$cmpcode);
 9421   %}
 9422 
 9423   ins_pipe(icond_reg);
 9424 %}
 9425 
 9426 instruct cmovUP_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
 9427   match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
 9428 
 9429   ins_cost(INSN_COST * 2);
 9430   format %{ "csel $dst, $src, zr $cmp\t# unsigned, ptr"  %}
 9431 
 9432   ins_encode %{
 9433     __ csel(as_Register($dst$$reg),
 9434             as_Register($src$$reg),
 9435             zr,
 9436             (Assembler::Condition)$cmp$$cmpcode);
 9437   %}
 9438 
 9439   ins_pipe(icond_reg);
 9440 %}
 9441 
 9442 instruct cmovN_reg_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
 9443   match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
 9444 
 9445   ins_cost(INSN_COST * 2);
 9446   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr"  %}
 9447 
 9448   ins_encode %{
 9449     __ cselw(as_Register($dst$$reg),
 9450              as_Register($src2$$reg),
 9451              as_Register($src1$$reg),
 9452              (Assembler::Condition)$cmp$$cmpcode);
 9453   %}
 9454 
 9455   ins_pipe(icond_reg_reg);
 9456 %}
 9457 
 9458 instruct cmovUN_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
 9459   match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
 9460 
 9461   ins_cost(INSN_COST * 2);
 9462   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr"  %}
 9463 
 9464   ins_encode %{
 9465     __ cselw(as_Register($dst$$reg),
 9466              as_Register($src2$$reg),
 9467              as_Register($src1$$reg),
 9468              (Assembler::Condition)$cmp$$cmpcode);
 9469   %}
 9470 
 9471   ins_pipe(icond_reg_reg);
 9472 %}
 9473 
 9474 // special cases where one arg is zero
 9475 
 9476 instruct cmovN_reg_zero(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
 9477   match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
 9478 
 9479   ins_cost(INSN_COST * 2);
 9480   format %{ "cselw $dst, zr, $src $cmp\t# signed, compressed ptr"  %}
 9481 
 9482   ins_encode %{
 9483     __ cselw(as_Register($dst$$reg),
 9484              zr,
 9485              as_Register($src$$reg),
 9486              (Assembler::Condition)$cmp$$cmpcode);
 9487   %}
 9488 
 9489   ins_pipe(icond_reg);
 9490 %}
 9491 
 9492 instruct cmovUN_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
 9493   match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
 9494 
 9495   ins_cost(INSN_COST * 2);
 9496   format %{ "cselw $dst, zr, $src $cmp\t# unsigned, compressed ptr"  %}
 9497 
 9498   ins_encode %{
 9499     __ cselw(as_Register($dst$$reg),
 9500              zr,
 9501              as_Register($src$$reg),
 9502              (Assembler::Condition)$cmp$$cmpcode);
 9503   %}
 9504 
 9505   ins_pipe(icond_reg);
 9506 %}
 9507 
 9508 instruct cmovN_zero_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
 9509   match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
 9510 
 9511   ins_cost(INSN_COST * 2);
 9512   format %{ "cselw $dst, $src, zr $cmp\t# signed, compressed ptr"  %}
 9513 
 9514   ins_encode %{
 9515     __ cselw(as_Register($dst$$reg),
 9516              as_Register($src$$reg),
 9517              zr,
 9518              (Assembler::Condition)$cmp$$cmpcode);
 9519   %}
 9520 
 9521   ins_pipe(icond_reg);
 9522 %}
 9523 
 9524 instruct cmovUN_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
 9525   match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
 9526 
 9527   ins_cost(INSN_COST * 2);
 9528   format %{ "cselw $dst, $src, zr $cmp\t# unsigned, compressed ptr"  %}
 9529 
 9530   ins_encode %{
 9531     __ cselw(as_Register($dst$$reg),
 9532              as_Register($src$$reg),
 9533              zr,
 9534              (Assembler::Condition)$cmp$$cmpcode);
 9535   %}
 9536 
 9537   ins_pipe(icond_reg);
 9538 %}
 9539 
 9540 instruct cmovF_reg(cmpOp cmp, rFlagsReg cr, vRegF dst, vRegF src1,  vRegF src2)
 9541 %{
 9542   match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
 9543 
 9544   ins_cost(INSN_COST * 3);
 9545 
 9546   format %{ "fcsels $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
 9547   ins_encode %{
 9548     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
 9549     __ fcsels(as_FloatRegister($dst$$reg),
 9550               as_FloatRegister($src2$$reg),
 9551               as_FloatRegister($src1$$reg),
 9552               cond);
 9553   %}
 9554 
 9555   ins_pipe(fp_cond_reg_reg_s);
 9556 %}
 9557 
 9558 instruct cmovUF_reg(cmpOpU cmp, rFlagsRegU cr, vRegF dst, vRegF src1,  vRegF src2)
 9559 %{
 9560   match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
 9561 
 9562   ins_cost(INSN_COST * 3);
 9563 
 9564   format %{ "fcsels $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
 9565   ins_encode %{
 9566     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
 9567     __ fcsels(as_FloatRegister($dst$$reg),
 9568               as_FloatRegister($src2$$reg),
 9569               as_FloatRegister($src1$$reg),
 9570               cond);
 9571   %}
 9572 
 9573   ins_pipe(fp_cond_reg_reg_s);
 9574 %}
 9575 
 9576 instruct cmovD_reg(cmpOp cmp, rFlagsReg cr, vRegD dst, vRegD src1,  vRegD src2)
 9577 %{
 9578   match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
 9579 
 9580   ins_cost(INSN_COST * 3);
 9581 
 9582   format %{ "fcseld $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
 9583   ins_encode %{
 9584     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
 9585     __ fcseld(as_FloatRegister($dst$$reg),
 9586               as_FloatRegister($src2$$reg),
 9587               as_FloatRegister($src1$$reg),
 9588               cond);
 9589   %}
 9590 
 9591   ins_pipe(fp_cond_reg_reg_d);
 9592 %}
 9593 
 9594 instruct cmovUD_reg(cmpOpU cmp, rFlagsRegU cr, vRegD dst, vRegD src1,  vRegD src2)
 9595 %{
 9596   match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
 9597 
 9598   ins_cost(INSN_COST * 3);
 9599 
 9600   format %{ "fcseld $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
 9601   ins_encode %{
 9602     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
 9603     __ fcseld(as_FloatRegister($dst$$reg),
 9604               as_FloatRegister($src2$$reg),
 9605               as_FloatRegister($src1$$reg),
 9606               cond);
 9607   %}
 9608 
 9609   ins_pipe(fp_cond_reg_reg_d);
 9610 %}
 9611 
 9612 // ============================================================================
 9613 // Arithmetic Instructions
 9614 //
 9615 
 9616 // Integer Addition
 9617 
 9618 // TODO
 9619 // these currently employ operations which do not set CR and hence are
 9620 // not flagged as killing CR but we would like to isolate the cases
 9621 // where we want to set flags from those where we don't. need to work
 9622 // out how to do that.
 9623 
 9624 instruct addI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
 9625   match(Set dst (AddI src1 src2));
 9626 
 9627   ins_cost(INSN_COST);
 9628   format %{ "addw  $dst, $src1, $src2" %}
 9629 
 9630   ins_encode %{
 9631     __ addw(as_Register($dst$$reg),
 9632             as_Register($src1$$reg),
 9633             as_Register($src2$$reg));
 9634   %}
 9635 
 9636   ins_pipe(ialu_reg_reg);
 9637 %}
 9638 
 9639 instruct addI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
 9640   match(Set dst (AddI src1 src2));
 9641 
 9642   ins_cost(INSN_COST);
 9643   format %{ "addw $dst, $src1, $src2" %}
 9644 
 9645   // use opcode to indicate that this is an add not a sub
 9646   opcode(0x0);
 9647 
 9648   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
 9649 
 9650   ins_pipe(ialu_reg_imm);
 9651 %}
 9652 
 9653 instruct addI_reg_imm_i2l(iRegINoSp dst, iRegL src1, immIAddSub src2) %{
 9654   match(Set dst (AddI (ConvL2I src1) src2));
 9655 
 9656   ins_cost(INSN_COST);
 9657   format %{ "addw $dst, $src1, $src2" %}
 9658 
 9659   // use opcode to indicate that this is an add not a sub
 9660   opcode(0x0);
 9661 
 9662   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
 9663 
 9664   ins_pipe(ialu_reg_imm);
 9665 %}
 9666 
 9667 // Pointer Addition
 9668 instruct addP_reg_reg(iRegPNoSp dst, iRegPorL2P src1, iRegL src2) %{
 9669   match(Set dst (AddP src1 src2));
 9670 
 9671   ins_cost(INSN_COST);
 9672   format %{ "add $dst, $src1, $src2\t# ptr" %}
 9673 
 9674   ins_encode %{
 9675     __ add(as_Register($dst$$reg),
 9676            as_Register($src1$$reg),
 9677            as_Register($src2$$reg));
 9678   %}
 9679 
 9680   ins_pipe(ialu_reg_reg);
 9681 %}
 9682 
 9683 instruct addP_reg_reg_ext(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2) %{
 9684   match(Set dst (AddP src1 (ConvI2L src2)));
 9685 
 9686   ins_cost(1.9 * INSN_COST);
 9687   format %{ "add $dst, $src1, $src2, sxtw\t# ptr" %}
 9688 
 9689   ins_encode %{
 9690     __ add(as_Register($dst$$reg),
 9691            as_Register($src1$$reg),
 9692            as_Register($src2$$reg), ext::sxtw);
 9693   %}
 9694 
 9695   ins_pipe(ialu_reg_reg);
 9696 %}
 9697 
 9698 instruct addP_reg_reg_lsl(iRegPNoSp dst, iRegPorL2P src1, iRegL src2, immIScale scale) %{
 9699   match(Set dst (AddP src1 (LShiftL src2 scale)));
 9700 
 9701   ins_cost(1.9 * INSN_COST);
 9702   format %{ "add $dst, $src1, $src2, LShiftL $scale\t# ptr" %}
 9703 
 9704   ins_encode %{
 9705     __ lea(as_Register($dst$$reg),
 9706            Address(as_Register($src1$$reg), as_Register($src2$$reg),
 9707                    Address::lsl($scale$$constant)));
 9708   %}
 9709 
 9710   ins_pipe(ialu_reg_reg_shift);
 9711 %}
 9712 
 9713 instruct addP_reg_reg_ext_shift(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2, immIScale scale) %{
 9714   match(Set dst (AddP src1 (LShiftL (ConvI2L src2) scale)));
 9715 
 9716   ins_cost(1.9 * INSN_COST);
 9717   format %{ "add $dst, $src1, $src2, I2L $scale\t# ptr" %}
 9718 
 9719   ins_encode %{
 9720     __ lea(as_Register($dst$$reg),
 9721            Address(as_Register($src1$$reg), as_Register($src2$$reg),
 9722                    Address::sxtw($scale$$constant)));
 9723   %}
 9724 
 9725   ins_pipe(ialu_reg_reg_shift);
 9726 %}
 9727 
 9728 instruct lshift_ext(iRegLNoSp dst, iRegIorL2I src, immI scale, rFlagsReg cr) %{
 9729   match(Set dst (LShiftL (ConvI2L src) scale));
 9730 
 9731   ins_cost(INSN_COST);
 9732   format %{ "sbfiz $dst, $src, $scale & 63, -$scale & 63\t" %}
 9733 
 9734   ins_encode %{
 9735     __ sbfiz(as_Register($dst$$reg),
 9736           as_Register($src$$reg),
 9737           $scale$$constant & 63, MIN2(32, (int)((-$scale$$constant) & 63)));
 9738   %}
 9739 
 9740   ins_pipe(ialu_reg_shift);
 9741 %}
 9742 
 9743 // Pointer Immediate Addition
 9744 // n.b. this needs to be more expensive than using an indirect memory
 9745 // operand
 9746 instruct addP_reg_imm(iRegPNoSp dst, iRegPorL2P src1, immLAddSub src2) %{
 9747   match(Set dst (AddP src1 src2));
 9748 
 9749   ins_cost(INSN_COST);
 9750   format %{ "add $dst, $src1, $src2\t# ptr" %}
 9751 
 9752   // use opcode to indicate that this is an add not a sub
 9753   opcode(0x0);
 9754 
 9755   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
 9756 
 9757   ins_pipe(ialu_reg_imm);
 9758 %}
 9759 
 9760 // Long Addition
 9761 instruct addL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
 9762 
 9763   match(Set dst (AddL src1 src2));
 9764 
 9765   ins_cost(INSN_COST);
 9766   format %{ "add  $dst, $src1, $src2" %}
 9767 
 9768   ins_encode %{
 9769     __ add(as_Register($dst$$reg),
 9770            as_Register($src1$$reg),
 9771            as_Register($src2$$reg));
 9772   %}
 9773 
 9774   ins_pipe(ialu_reg_reg);
 9775 %}
 9776 
 9777 // No constant pool entries requiredLong Immediate Addition.
 9778 instruct addL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
 9779   match(Set dst (AddL src1 src2));
 9780 
 9781   ins_cost(INSN_COST);
 9782   format %{ "add $dst, $src1, $src2" %}
 9783 
 9784   // use opcode to indicate that this is an add not a sub
 9785   opcode(0x0);
 9786 
 9787   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
 9788 
 9789   ins_pipe(ialu_reg_imm);
 9790 %}
 9791 
 9792 // Integer Subtraction
 9793 instruct subI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
 9794   match(Set dst (SubI src1 src2));
 9795 
 9796   ins_cost(INSN_COST);
 9797   format %{ "subw  $dst, $src1, $src2" %}
 9798 
 9799   ins_encode %{
 9800     __ subw(as_Register($dst$$reg),
 9801             as_Register($src1$$reg),
 9802             as_Register($src2$$reg));
 9803   %}
 9804 
 9805   ins_pipe(ialu_reg_reg);
 9806 %}
 9807 
 9808 // Immediate Subtraction
 9809 instruct subI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
 9810   match(Set dst (SubI src1 src2));
 9811 
 9812   ins_cost(INSN_COST);
 9813   format %{ "subw $dst, $src1, $src2" %}
 9814 
 9815   // use opcode to indicate that this is a sub not an add
 9816   opcode(0x1);
 9817 
 9818   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
 9819 
 9820   ins_pipe(ialu_reg_imm);
 9821 %}
 9822 
 9823 // Long Subtraction
 9824 instruct subL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
 9825 
 9826   match(Set dst (SubL src1 src2));
 9827 
 9828   ins_cost(INSN_COST);
 9829   format %{ "sub  $dst, $src1, $src2" %}
 9830 
 9831   ins_encode %{
 9832     __ sub(as_Register($dst$$reg),
 9833            as_Register($src1$$reg),
 9834            as_Register($src2$$reg));
 9835   %}
 9836 
 9837   ins_pipe(ialu_reg_reg);
 9838 %}
 9839 
 9840 // No constant pool entries requiredLong Immediate Subtraction.
 9841 instruct subL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
 9842   match(Set dst (SubL src1 src2));
 9843 
 9844   ins_cost(INSN_COST);
 9845   format %{ "sub$dst, $src1, $src2" %}
 9846 
 9847   // use opcode to indicate that this is a sub not an add
 9848   opcode(0x1);
 9849 
 9850   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
 9851 
 9852   ins_pipe(ialu_reg_imm);
 9853 %}
 9854 
 9855 // Integer Negation (special case for sub)
 9856 
 9857 instruct negI_reg(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr) %{
 9858   match(Set dst (SubI zero src));
 9859 
 9860   ins_cost(INSN_COST);
 9861   format %{ "negw $dst, $src\t# int" %}
 9862 
 9863   ins_encode %{
 9864     __ negw(as_Register($dst$$reg),
 9865             as_Register($src$$reg));
 9866   %}
 9867 
 9868   ins_pipe(ialu_reg);
 9869 %}
 9870 
 9871 // Long Negation
 9872 
 9873 instruct negL_reg(iRegLNoSp dst, iRegL src, immL0 zero, rFlagsReg cr) %{
 9874   match(Set dst (SubL zero src));
 9875 
 9876   ins_cost(INSN_COST);
 9877   format %{ "neg $dst, $src\t# long" %}
 9878 
 9879   ins_encode %{
 9880     __ neg(as_Register($dst$$reg),
 9881            as_Register($src$$reg));
 9882   %}
 9883 
 9884   ins_pipe(ialu_reg);
 9885 %}
 9886 
 9887 // Integer Multiply
 9888 
 9889 instruct mulI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
 9890   match(Set dst (MulI src1 src2));
 9891 
 9892   ins_cost(INSN_COST * 3);
 9893   format %{ "mulw  $dst, $src1, $src2" %}
 9894 
 9895   ins_encode %{
 9896     __ mulw(as_Register($dst$$reg),
 9897             as_Register($src1$$reg),
 9898             as_Register($src2$$reg));
 9899   %}
 9900 
 9901   ins_pipe(imul_reg_reg);
 9902 %}
 9903 
 9904 instruct smulI(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
 9905   match(Set dst (MulL (ConvI2L src1) (ConvI2L src2)));
 9906 
 9907   ins_cost(INSN_COST * 3);
 9908   format %{ "smull  $dst, $src1, $src2" %}
 9909 
 9910   ins_encode %{
 9911     __ smull(as_Register($dst$$reg),
 9912              as_Register($src1$$reg),
 9913              as_Register($src2$$reg));
 9914   %}
 9915 
 9916   ins_pipe(imul_reg_reg);
 9917 %}
 9918 
 9919 // Long Multiply
 9920 
 9921 instruct mulL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
 9922   match(Set dst (MulL src1 src2));
 9923 
 9924   ins_cost(INSN_COST * 5);
 9925   format %{ "mul  $dst, $src1, $src2" %}
 9926 
 9927   ins_encode %{
 9928     __ mul(as_Register($dst$$reg),
 9929            as_Register($src1$$reg),
 9930            as_Register($src2$$reg));
 9931   %}
 9932 
 9933   ins_pipe(lmul_reg_reg);
 9934 %}
 9935 
 9936 instruct mulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
 9937 %{
 9938   match(Set dst (MulHiL src1 src2));
 9939 
 9940   ins_cost(INSN_COST * 7);
 9941   format %{ "smulh   $dst, $src1, $src2\t# mulhi" %}
 9942 
 9943   ins_encode %{
 9944     __ smulh(as_Register($dst$$reg),
 9945              as_Register($src1$$reg),
 9946              as_Register($src2$$reg));
 9947   %}
 9948 
 9949   ins_pipe(lmul_reg_reg);
 9950 %}
 9951 
 9952 instruct umulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
 9953 %{
 9954   match(Set dst (UMulHiL src1 src2));
 9955 
 9956   ins_cost(INSN_COST * 7);
 9957   format %{ "umulh   $dst, $src1, $src2\t# umulhi" %}
 9958 
 9959   ins_encode %{
 9960     __ umulh(as_Register($dst$$reg),
 9961              as_Register($src1$$reg),
 9962              as_Register($src2$$reg));
 9963   %}
 9964 
 9965   ins_pipe(lmul_reg_reg);
 9966 %}
 9967 
 9968 // Combined Integer Multiply & Add/Sub
 9969 
 9970 instruct maddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
 9971   match(Set dst (AddI src3 (MulI src1 src2)));
 9972 
 9973   ins_cost(INSN_COST * 3);
 9974   format %{ "madd  $dst, $src1, $src2, $src3" %}
 9975 
 9976   ins_encode %{
 9977     __ maddw(as_Register($dst$$reg),
 9978              as_Register($src1$$reg),
 9979              as_Register($src2$$reg),
 9980              as_Register($src3$$reg));
 9981   %}
 9982 
 9983   ins_pipe(imac_reg_reg);
 9984 %}
 9985 
 9986 instruct msubI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
 9987   match(Set dst (SubI src3 (MulI src1 src2)));
 9988 
 9989   ins_cost(INSN_COST * 3);
 9990   format %{ "msub  $dst, $src1, $src2, $src3" %}
 9991 
 9992   ins_encode %{
 9993     __ msubw(as_Register($dst$$reg),
 9994              as_Register($src1$$reg),
 9995              as_Register($src2$$reg),
 9996              as_Register($src3$$reg));
 9997   %}
 9998 
 9999   ins_pipe(imac_reg_reg);
10000 %}
10001 
10002 // Combined Integer Multiply & Neg
10003 
10004 instruct mnegI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI0 zero) %{
10005   match(Set dst (MulI (SubI zero src1) src2));
10006 
10007   ins_cost(INSN_COST * 3);
10008   format %{ "mneg  $dst, $src1, $src2" %}
10009 
10010   ins_encode %{
10011     __ mnegw(as_Register($dst$$reg),
10012              as_Register($src1$$reg),
10013              as_Register($src2$$reg));
10014   %}
10015 
10016   ins_pipe(imac_reg_reg);
10017 %}
10018 
10019 // Combined Long Multiply & Add/Sub
10020 
10021 instruct maddL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10022   match(Set dst (AddL src3 (MulL src1 src2)));
10023 
10024   ins_cost(INSN_COST * 5);
10025   format %{ "madd  $dst, $src1, $src2, $src3" %}
10026 
10027   ins_encode %{
10028     __ madd(as_Register($dst$$reg),
10029             as_Register($src1$$reg),
10030             as_Register($src2$$reg),
10031             as_Register($src3$$reg));
10032   %}
10033 
10034   ins_pipe(lmac_reg_reg);
10035 %}
10036 
10037 instruct msubL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10038   match(Set dst (SubL src3 (MulL src1 src2)));
10039 
10040   ins_cost(INSN_COST * 5);
10041   format %{ "msub  $dst, $src1, $src2, $src3" %}
10042 
10043   ins_encode %{
10044     __ msub(as_Register($dst$$reg),
10045             as_Register($src1$$reg),
10046             as_Register($src2$$reg),
10047             as_Register($src3$$reg));
10048   %}
10049 
10050   ins_pipe(lmac_reg_reg);
10051 %}
10052 
10053 // Combined Long Multiply & Neg
10054 
10055 instruct mnegL(iRegLNoSp dst, iRegL src1, iRegL src2, immL0 zero) %{
10056   match(Set dst (MulL (SubL zero src1) src2));
10057 
10058   ins_cost(INSN_COST * 5);
10059   format %{ "mneg  $dst, $src1, $src2" %}
10060 
10061   ins_encode %{
10062     __ mneg(as_Register($dst$$reg),
10063             as_Register($src1$$reg),
10064             as_Register($src2$$reg));
10065   %}
10066 
10067   ins_pipe(lmac_reg_reg);
10068 %}
10069 
10070 // Combine Integer Signed Multiply & Add/Sub/Neg Long
10071 
10072 instruct smaddL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10073   match(Set dst (AddL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10074 
10075   ins_cost(INSN_COST * 3);
10076   format %{ "smaddl  $dst, $src1, $src2, $src3" %}
10077 
10078   ins_encode %{
10079     __ smaddl(as_Register($dst$$reg),
10080               as_Register($src1$$reg),
10081               as_Register($src2$$reg),
10082               as_Register($src3$$reg));
10083   %}
10084 
10085   ins_pipe(imac_reg_reg);
10086 %}
10087 
10088 instruct smsubL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10089   match(Set dst (SubL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10090 
10091   ins_cost(INSN_COST * 3);
10092   format %{ "smsubl  $dst, $src1, $src2, $src3" %}
10093 
10094   ins_encode %{
10095     __ smsubl(as_Register($dst$$reg),
10096               as_Register($src1$$reg),
10097               as_Register($src2$$reg),
10098               as_Register($src3$$reg));
10099   %}
10100 
10101   ins_pipe(imac_reg_reg);
10102 %}
10103 
10104 instruct smnegL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, immL0 zero) %{
10105   match(Set dst (MulL (SubL zero (ConvI2L src1)) (ConvI2L src2)));
10106 
10107   ins_cost(INSN_COST * 3);
10108   format %{ "smnegl  $dst, $src1, $src2" %}
10109 
10110   ins_encode %{
10111     __ smnegl(as_Register($dst$$reg),
10112               as_Register($src1$$reg),
10113               as_Register($src2$$reg));
10114   %}
10115 
10116   ins_pipe(imac_reg_reg);
10117 %}
10118 
10119 // Combined Multiply-Add Shorts into Integer (dst = src1 * src2 + src3 * src4)
10120 
10121 instruct muladdS2I(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3, iRegIorL2I src4) %{
10122   match(Set dst (MulAddS2I (Binary src1 src2) (Binary src3 src4)));
10123 
10124   ins_cost(INSN_COST * 5);
10125   format %{ "mulw  rscratch1, $src1, $src2\n\t"
10126             "maddw $dst, $src3, $src4, rscratch1" %}
10127 
10128   ins_encode %{
10129     __ mulw(rscratch1, as_Register($src1$$reg), as_Register($src2$$reg));
10130     __ maddw(as_Register($dst$$reg), as_Register($src3$$reg), as_Register($src4$$reg), rscratch1); %}
10131 
10132   ins_pipe(imac_reg_reg);
10133 %}
10134 
10135 // Integer Divide
10136 
10137 instruct divI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10138   match(Set dst (DivI src1 src2));
10139 
10140   ins_cost(INSN_COST * 19);
10141   format %{ "sdivw  $dst, $src1, $src2" %}
10142 
10143   ins_encode(aarch64_enc_divw(dst, src1, src2));
10144   ins_pipe(idiv_reg_reg);
10145 %}
10146 
10147 // Long Divide
10148 
10149 instruct divL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10150   match(Set dst (DivL src1 src2));
10151 
10152   ins_cost(INSN_COST * 35);
10153   format %{ "sdiv   $dst, $src1, $src2" %}
10154 
10155   ins_encode(aarch64_enc_div(dst, src1, src2));
10156   ins_pipe(ldiv_reg_reg);
10157 %}
10158 
10159 // Integer Remainder
10160 
10161 instruct modI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10162   match(Set dst (ModI src1 src2));
10163 
10164   ins_cost(INSN_COST * 22);
10165   format %{ "sdivw  rscratch1, $src1, $src2\n\t"
10166             "msubw  $dst, rscratch1, $src2, $src1" %}
10167 
10168   ins_encode(aarch64_enc_modw(dst, src1, src2));
10169   ins_pipe(idiv_reg_reg);
10170 %}
10171 
10172 // Long Remainder
10173 
10174 instruct modL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10175   match(Set dst (ModL src1 src2));
10176 
10177   ins_cost(INSN_COST * 38);
10178   format %{ "sdiv   rscratch1, $src1, $src2\n"
10179             "msub   $dst, rscratch1, $src2, $src1" %}
10180 
10181   ins_encode(aarch64_enc_mod(dst, src1, src2));
10182   ins_pipe(ldiv_reg_reg);
10183 %}
10184 
10185 // Unsigned Integer Divide
10186 
10187 instruct UdivI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10188   match(Set dst (UDivI src1 src2));
10189 
10190   ins_cost(INSN_COST * 19);
10191   format %{ "udivw  $dst, $src1, $src2" %}
10192 
10193   ins_encode %{
10194     __ udivw($dst$$Register, $src1$$Register, $src2$$Register);
10195   %}
10196 
10197   ins_pipe(idiv_reg_reg);
10198 %}
10199 
10200 //  Unsigned Long Divide
10201 
10202 instruct UdivL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10203   match(Set dst (UDivL src1 src2));
10204 
10205   ins_cost(INSN_COST * 35);
10206   format %{ "udiv   $dst, $src1, $src2" %}
10207 
10208   ins_encode %{
10209     __ udiv($dst$$Register, $src1$$Register, $src2$$Register);
10210   %}
10211 
10212   ins_pipe(ldiv_reg_reg);
10213 %}
10214 
10215 // Unsigned Integer Remainder
10216 
10217 instruct UmodI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10218   match(Set dst (UModI src1 src2));
10219 
10220   ins_cost(INSN_COST * 22);
10221   format %{ "udivw  rscratch1, $src1, $src2\n\t"
10222             "msubw  $dst, rscratch1, $src2, $src1" %}
10223 
10224   ins_encode %{
10225     __ udivw(rscratch1, $src1$$Register, $src2$$Register);
10226     __ msubw($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
10227   %}
10228 
10229   ins_pipe(idiv_reg_reg);
10230 %}
10231 
10232 // Unsigned Long Remainder
10233 
10234 instruct UModL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10235   match(Set dst (UModL src1 src2));
10236 
10237   ins_cost(INSN_COST * 38);
10238   format %{ "udiv   rscratch1, $src1, $src2\n"
10239             "msub   $dst, rscratch1, $src2, $src1" %}
10240 
10241   ins_encode %{
10242     __ udiv(rscratch1, $src1$$Register, $src2$$Register);
10243     __ msub($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
10244   %}
10245 
10246   ins_pipe(ldiv_reg_reg);
10247 %}
10248 
10249 // Integer Shifts
10250 
10251 // Shift Left Register
10252 instruct lShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10253   match(Set dst (LShiftI src1 src2));
10254 
10255   ins_cost(INSN_COST * 2);
10256   format %{ "lslvw  $dst, $src1, $src2" %}
10257 
10258   ins_encode %{
10259     __ lslvw(as_Register($dst$$reg),
10260              as_Register($src1$$reg),
10261              as_Register($src2$$reg));
10262   %}
10263 
10264   ins_pipe(ialu_reg_reg_vshift);
10265 %}
10266 
10267 // Shift Left Immediate
10268 instruct lShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10269   match(Set dst (LShiftI src1 src2));
10270 
10271   ins_cost(INSN_COST);
10272   format %{ "lslw $dst, $src1, ($src2 & 0x1f)" %}
10273 
10274   ins_encode %{
10275     __ lslw(as_Register($dst$$reg),
10276             as_Register($src1$$reg),
10277             $src2$$constant & 0x1f);
10278   %}
10279 
10280   ins_pipe(ialu_reg_shift);
10281 %}
10282 
10283 // Shift Right Logical Register
10284 instruct urShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10285   match(Set dst (URShiftI src1 src2));
10286 
10287   ins_cost(INSN_COST * 2);
10288   format %{ "lsrvw  $dst, $src1, $src2" %}
10289 
10290   ins_encode %{
10291     __ lsrvw(as_Register($dst$$reg),
10292              as_Register($src1$$reg),
10293              as_Register($src2$$reg));
10294   %}
10295 
10296   ins_pipe(ialu_reg_reg_vshift);
10297 %}
10298 
10299 // Shift Right Logical Immediate
10300 instruct urShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10301   match(Set dst (URShiftI src1 src2));
10302 
10303   ins_cost(INSN_COST);
10304   format %{ "lsrw $dst, $src1, ($src2 & 0x1f)" %}
10305 
10306   ins_encode %{
10307     __ lsrw(as_Register($dst$$reg),
10308             as_Register($src1$$reg),
10309             $src2$$constant & 0x1f);
10310   %}
10311 
10312   ins_pipe(ialu_reg_shift);
10313 %}
10314 
10315 // Shift Right Arithmetic Register
10316 instruct rShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10317   match(Set dst (RShiftI src1 src2));
10318 
10319   ins_cost(INSN_COST * 2);
10320   format %{ "asrvw  $dst, $src1, $src2" %}
10321 
10322   ins_encode %{
10323     __ asrvw(as_Register($dst$$reg),
10324              as_Register($src1$$reg),
10325              as_Register($src2$$reg));
10326   %}
10327 
10328   ins_pipe(ialu_reg_reg_vshift);
10329 %}
10330 
10331 // Shift Right Arithmetic Immediate
10332 instruct rShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10333   match(Set dst (RShiftI src1 src2));
10334 
10335   ins_cost(INSN_COST);
10336   format %{ "asrw $dst, $src1, ($src2 & 0x1f)" %}
10337 
10338   ins_encode %{
10339     __ asrw(as_Register($dst$$reg),
10340             as_Register($src1$$reg),
10341             $src2$$constant & 0x1f);
10342   %}
10343 
10344   ins_pipe(ialu_reg_shift);
10345 %}
10346 
10347 // Combined Int Mask and Right Shift (using UBFM)
10348 // TODO
10349 
10350 // Long Shifts
10351 
10352 // Shift Left Register
10353 instruct lShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10354   match(Set dst (LShiftL src1 src2));
10355 
10356   ins_cost(INSN_COST * 2);
10357   format %{ "lslv  $dst, $src1, $src2" %}
10358 
10359   ins_encode %{
10360     __ lslv(as_Register($dst$$reg),
10361             as_Register($src1$$reg),
10362             as_Register($src2$$reg));
10363   %}
10364 
10365   ins_pipe(ialu_reg_reg_vshift);
10366 %}
10367 
10368 // Shift Left Immediate
10369 instruct lShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10370   match(Set dst (LShiftL src1 src2));
10371 
10372   ins_cost(INSN_COST);
10373   format %{ "lsl $dst, $src1, ($src2 & 0x3f)" %}
10374 
10375   ins_encode %{
10376     __ lsl(as_Register($dst$$reg),
10377             as_Register($src1$$reg),
10378             $src2$$constant & 0x3f);
10379   %}
10380 
10381   ins_pipe(ialu_reg_shift);
10382 %}
10383 
10384 // Shift Right Logical Register
10385 instruct urShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10386   match(Set dst (URShiftL src1 src2));
10387 
10388   ins_cost(INSN_COST * 2);
10389   format %{ "lsrv  $dst, $src1, $src2" %}
10390 
10391   ins_encode %{
10392     __ lsrv(as_Register($dst$$reg),
10393             as_Register($src1$$reg),
10394             as_Register($src2$$reg));
10395   %}
10396 
10397   ins_pipe(ialu_reg_reg_vshift);
10398 %}
10399 
10400 // Shift Right Logical Immediate
10401 instruct urShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10402   match(Set dst (URShiftL src1 src2));
10403 
10404   ins_cost(INSN_COST);
10405   format %{ "lsr $dst, $src1, ($src2 & 0x3f)" %}
10406 
10407   ins_encode %{
10408     __ lsr(as_Register($dst$$reg),
10409            as_Register($src1$$reg),
10410            $src2$$constant & 0x3f);
10411   %}
10412 
10413   ins_pipe(ialu_reg_shift);
10414 %}
10415 
10416 // A special-case pattern for card table stores.
10417 instruct urShiftP_reg_imm(iRegLNoSp dst, iRegP src1, immI src2) %{
10418   match(Set dst (URShiftL (CastP2X src1) src2));
10419 
10420   ins_cost(INSN_COST);
10421   format %{ "lsr $dst, p2x($src1), ($src2 & 0x3f)" %}
10422 
10423   ins_encode %{
10424     __ lsr(as_Register($dst$$reg),
10425            as_Register($src1$$reg),
10426            $src2$$constant & 0x3f);
10427   %}
10428 
10429   ins_pipe(ialu_reg_shift);
10430 %}
10431 
10432 // Shift Right Arithmetic Register
10433 instruct rShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10434   match(Set dst (RShiftL src1 src2));
10435 
10436   ins_cost(INSN_COST * 2);
10437   format %{ "asrv  $dst, $src1, $src2" %}
10438 
10439   ins_encode %{
10440     __ asrv(as_Register($dst$$reg),
10441             as_Register($src1$$reg),
10442             as_Register($src2$$reg));
10443   %}
10444 
10445   ins_pipe(ialu_reg_reg_vshift);
10446 %}
10447 
10448 // Shift Right Arithmetic Immediate
10449 instruct rShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10450   match(Set dst (RShiftL src1 src2));
10451 
10452   ins_cost(INSN_COST);
10453   format %{ "asr $dst, $src1, ($src2 & 0x3f)" %}
10454 
10455   ins_encode %{
10456     __ asr(as_Register($dst$$reg),
10457            as_Register($src1$$reg),
10458            $src2$$constant & 0x3f);
10459   %}
10460 
10461   ins_pipe(ialu_reg_shift);
10462 %}
10463 
10464 // BEGIN This section of the file is automatically generated. Do not edit --------------
10465 // This section is generated from aarch64_ad.m4
10466 
10467 // This pattern is automatically generated from aarch64_ad.m4
10468 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10469 instruct regL_not_reg(iRegLNoSp dst,
10470                          iRegL src1, immL_M1 m1,
10471                          rFlagsReg cr) %{
10472   match(Set dst (XorL src1 m1));
10473   ins_cost(INSN_COST);
10474   format %{ "eon  $dst, $src1, zr" %}
10475 
10476   ins_encode %{
10477     __ eon(as_Register($dst$$reg),
10478               as_Register($src1$$reg),
10479               zr,
10480               Assembler::LSL, 0);
10481   %}
10482 
10483   ins_pipe(ialu_reg);
10484 %}
10485 
10486 // This pattern is automatically generated from aarch64_ad.m4
10487 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10488 instruct regI_not_reg(iRegINoSp dst,
10489                          iRegIorL2I src1, immI_M1 m1,
10490                          rFlagsReg cr) %{
10491   match(Set dst (XorI src1 m1));
10492   ins_cost(INSN_COST);
10493   format %{ "eonw  $dst, $src1, zr" %}
10494 
10495   ins_encode %{
10496     __ eonw(as_Register($dst$$reg),
10497               as_Register($src1$$reg),
10498               zr,
10499               Assembler::LSL, 0);
10500   %}
10501 
10502   ins_pipe(ialu_reg);
10503 %}
10504 
10505 // This pattern is automatically generated from aarch64_ad.m4
10506 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10507 instruct NegI_reg_URShift_reg(iRegINoSp dst,
10508                               immI0 zero, iRegIorL2I src1, immI src2) %{
10509   match(Set dst (SubI zero (URShiftI src1 src2)));
10510 
10511   ins_cost(1.9 * INSN_COST);
10512   format %{ "negw  $dst, $src1, LSR $src2" %}
10513 
10514   ins_encode %{
10515     __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
10516             Assembler::LSR, $src2$$constant & 0x1f);
10517   %}
10518 
10519   ins_pipe(ialu_reg_shift);
10520 %}
10521 
10522 // This pattern is automatically generated from aarch64_ad.m4
10523 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10524 instruct NegI_reg_RShift_reg(iRegINoSp dst,
10525                               immI0 zero, iRegIorL2I src1, immI src2) %{
10526   match(Set dst (SubI zero (RShiftI src1 src2)));
10527 
10528   ins_cost(1.9 * INSN_COST);
10529   format %{ "negw  $dst, $src1, ASR $src2" %}
10530 
10531   ins_encode %{
10532     __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
10533             Assembler::ASR, $src2$$constant & 0x1f);
10534   %}
10535 
10536   ins_pipe(ialu_reg_shift);
10537 %}
10538 
10539 // This pattern is automatically generated from aarch64_ad.m4
10540 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10541 instruct NegI_reg_LShift_reg(iRegINoSp dst,
10542                               immI0 zero, iRegIorL2I src1, immI src2) %{
10543   match(Set dst (SubI zero (LShiftI src1 src2)));
10544 
10545   ins_cost(1.9 * INSN_COST);
10546   format %{ "negw  $dst, $src1, LSL $src2" %}
10547 
10548   ins_encode %{
10549     __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
10550             Assembler::LSL, $src2$$constant & 0x1f);
10551   %}
10552 
10553   ins_pipe(ialu_reg_shift);
10554 %}
10555 
10556 // This pattern is automatically generated from aarch64_ad.m4
10557 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10558 instruct NegL_reg_URShift_reg(iRegLNoSp dst,
10559                               immL0 zero, iRegL src1, immI src2) %{
10560   match(Set dst (SubL zero (URShiftL src1 src2)));
10561 
10562   ins_cost(1.9 * INSN_COST);
10563   format %{ "neg  $dst, $src1, LSR $src2" %}
10564 
10565   ins_encode %{
10566     __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10567             Assembler::LSR, $src2$$constant & 0x3f);
10568   %}
10569 
10570   ins_pipe(ialu_reg_shift);
10571 %}
10572 
10573 // This pattern is automatically generated from aarch64_ad.m4
10574 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10575 instruct NegL_reg_RShift_reg(iRegLNoSp dst,
10576                               immL0 zero, iRegL src1, immI src2) %{
10577   match(Set dst (SubL zero (RShiftL src1 src2)));
10578 
10579   ins_cost(1.9 * INSN_COST);
10580   format %{ "neg  $dst, $src1, ASR $src2" %}
10581 
10582   ins_encode %{
10583     __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10584             Assembler::ASR, $src2$$constant & 0x3f);
10585   %}
10586 
10587   ins_pipe(ialu_reg_shift);
10588 %}
10589 
10590 // This pattern is automatically generated from aarch64_ad.m4
10591 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10592 instruct NegL_reg_LShift_reg(iRegLNoSp dst,
10593                               immL0 zero, iRegL src1, immI src2) %{
10594   match(Set dst (SubL zero (LShiftL src1 src2)));
10595 
10596   ins_cost(1.9 * INSN_COST);
10597   format %{ "neg  $dst, $src1, LSL $src2" %}
10598 
10599   ins_encode %{
10600     __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10601             Assembler::LSL, $src2$$constant & 0x3f);
10602   %}
10603 
10604   ins_pipe(ialu_reg_shift);
10605 %}
10606 
10607 // This pattern is automatically generated from aarch64_ad.m4
10608 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10609 instruct AndI_reg_not_reg(iRegINoSp dst,
10610                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10611   match(Set dst (AndI src1 (XorI src2 m1)));
10612   ins_cost(INSN_COST);
10613   format %{ "bicw  $dst, $src1, $src2" %}
10614 
10615   ins_encode %{
10616     __ bicw(as_Register($dst$$reg),
10617               as_Register($src1$$reg),
10618               as_Register($src2$$reg),
10619               Assembler::LSL, 0);
10620   %}
10621 
10622   ins_pipe(ialu_reg_reg);
10623 %}
10624 
10625 // This pattern is automatically generated from aarch64_ad.m4
10626 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10627 instruct AndL_reg_not_reg(iRegLNoSp dst,
10628                          iRegL src1, iRegL src2, immL_M1 m1) %{
10629   match(Set dst (AndL src1 (XorL src2 m1)));
10630   ins_cost(INSN_COST);
10631   format %{ "bic  $dst, $src1, $src2" %}
10632 
10633   ins_encode %{
10634     __ bic(as_Register($dst$$reg),
10635               as_Register($src1$$reg),
10636               as_Register($src2$$reg),
10637               Assembler::LSL, 0);
10638   %}
10639 
10640   ins_pipe(ialu_reg_reg);
10641 %}
10642 
10643 // This pattern is automatically generated from aarch64_ad.m4
10644 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10645 instruct OrI_reg_not_reg(iRegINoSp dst,
10646                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10647   match(Set dst (OrI src1 (XorI src2 m1)));
10648   ins_cost(INSN_COST);
10649   format %{ "ornw  $dst, $src1, $src2" %}
10650 
10651   ins_encode %{
10652     __ ornw(as_Register($dst$$reg),
10653               as_Register($src1$$reg),
10654               as_Register($src2$$reg),
10655               Assembler::LSL, 0);
10656   %}
10657 
10658   ins_pipe(ialu_reg_reg);
10659 %}
10660 
10661 // This pattern is automatically generated from aarch64_ad.m4
10662 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10663 instruct OrL_reg_not_reg(iRegLNoSp dst,
10664                          iRegL src1, iRegL src2, immL_M1 m1) %{
10665   match(Set dst (OrL src1 (XorL src2 m1)));
10666   ins_cost(INSN_COST);
10667   format %{ "orn  $dst, $src1, $src2" %}
10668 
10669   ins_encode %{
10670     __ orn(as_Register($dst$$reg),
10671               as_Register($src1$$reg),
10672               as_Register($src2$$reg),
10673               Assembler::LSL, 0);
10674   %}
10675 
10676   ins_pipe(ialu_reg_reg);
10677 %}
10678 
10679 // This pattern is automatically generated from aarch64_ad.m4
10680 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10681 instruct XorI_reg_not_reg(iRegINoSp dst,
10682                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10683   match(Set dst (XorI m1 (XorI src2 src1)));
10684   ins_cost(INSN_COST);
10685   format %{ "eonw  $dst, $src1, $src2" %}
10686 
10687   ins_encode %{
10688     __ eonw(as_Register($dst$$reg),
10689               as_Register($src1$$reg),
10690               as_Register($src2$$reg),
10691               Assembler::LSL, 0);
10692   %}
10693 
10694   ins_pipe(ialu_reg_reg);
10695 %}
10696 
10697 // This pattern is automatically generated from aarch64_ad.m4
10698 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10699 instruct XorL_reg_not_reg(iRegLNoSp dst,
10700                          iRegL src1, iRegL src2, immL_M1 m1) %{
10701   match(Set dst (XorL m1 (XorL src2 src1)));
10702   ins_cost(INSN_COST);
10703   format %{ "eon  $dst, $src1, $src2" %}
10704 
10705   ins_encode %{
10706     __ eon(as_Register($dst$$reg),
10707               as_Register($src1$$reg),
10708               as_Register($src2$$reg),
10709               Assembler::LSL, 0);
10710   %}
10711 
10712   ins_pipe(ialu_reg_reg);
10713 %}
10714 
10715 // This pattern is automatically generated from aarch64_ad.m4
10716 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10717 // val & (-1 ^ (val >>> shift)) ==> bicw
10718 instruct AndI_reg_URShift_not_reg(iRegINoSp dst,
10719                          iRegIorL2I src1, iRegIorL2I src2,
10720                          immI src3, immI_M1 src4) %{
10721   match(Set dst (AndI src1 (XorI(URShiftI src2 src3) src4)));
10722   ins_cost(1.9 * INSN_COST);
10723   format %{ "bicw  $dst, $src1, $src2, LSR $src3" %}
10724 
10725   ins_encode %{
10726     __ bicw(as_Register($dst$$reg),
10727               as_Register($src1$$reg),
10728               as_Register($src2$$reg),
10729               Assembler::LSR,
10730               $src3$$constant & 0x1f);
10731   %}
10732 
10733   ins_pipe(ialu_reg_reg_shift);
10734 %}
10735 
10736 // This pattern is automatically generated from aarch64_ad.m4
10737 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10738 // val & (-1 ^ (val >>> shift)) ==> bic
10739 instruct AndL_reg_URShift_not_reg(iRegLNoSp dst,
10740                          iRegL src1, iRegL src2,
10741                          immI src3, immL_M1 src4) %{
10742   match(Set dst (AndL src1 (XorL(URShiftL src2 src3) src4)));
10743   ins_cost(1.9 * INSN_COST);
10744   format %{ "bic  $dst, $src1, $src2, LSR $src3" %}
10745 
10746   ins_encode %{
10747     __ bic(as_Register($dst$$reg),
10748               as_Register($src1$$reg),
10749               as_Register($src2$$reg),
10750               Assembler::LSR,
10751               $src3$$constant & 0x3f);
10752   %}
10753 
10754   ins_pipe(ialu_reg_reg_shift);
10755 %}
10756 
10757 // This pattern is automatically generated from aarch64_ad.m4
10758 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10759 // val & (-1 ^ (val >> shift)) ==> bicw
10760 instruct AndI_reg_RShift_not_reg(iRegINoSp dst,
10761                          iRegIorL2I src1, iRegIorL2I src2,
10762                          immI src3, immI_M1 src4) %{
10763   match(Set dst (AndI src1 (XorI(RShiftI src2 src3) src4)));
10764   ins_cost(1.9 * INSN_COST);
10765   format %{ "bicw  $dst, $src1, $src2, ASR $src3" %}
10766 
10767   ins_encode %{
10768     __ bicw(as_Register($dst$$reg),
10769               as_Register($src1$$reg),
10770               as_Register($src2$$reg),
10771               Assembler::ASR,
10772               $src3$$constant & 0x1f);
10773   %}
10774 
10775   ins_pipe(ialu_reg_reg_shift);
10776 %}
10777 
10778 // This pattern is automatically generated from aarch64_ad.m4
10779 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10780 // val & (-1 ^ (val >> shift)) ==> bic
10781 instruct AndL_reg_RShift_not_reg(iRegLNoSp dst,
10782                          iRegL src1, iRegL src2,
10783                          immI src3, immL_M1 src4) %{
10784   match(Set dst (AndL src1 (XorL(RShiftL src2 src3) src4)));
10785   ins_cost(1.9 * INSN_COST);
10786   format %{ "bic  $dst, $src1, $src2, ASR $src3" %}
10787 
10788   ins_encode %{
10789     __ bic(as_Register($dst$$reg),
10790               as_Register($src1$$reg),
10791               as_Register($src2$$reg),
10792               Assembler::ASR,
10793               $src3$$constant & 0x3f);
10794   %}
10795 
10796   ins_pipe(ialu_reg_reg_shift);
10797 %}
10798 
10799 // This pattern is automatically generated from aarch64_ad.m4
10800 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10801 // val & (-1 ^ (val ror shift)) ==> bicw
10802 instruct AndI_reg_RotateRight_not_reg(iRegINoSp dst,
10803                          iRegIorL2I src1, iRegIorL2I src2,
10804                          immI src3, immI_M1 src4) %{
10805   match(Set dst (AndI src1 (XorI(RotateRight src2 src3) src4)));
10806   ins_cost(1.9 * INSN_COST);
10807   format %{ "bicw  $dst, $src1, $src2, ROR $src3" %}
10808 
10809   ins_encode %{
10810     __ bicw(as_Register($dst$$reg),
10811               as_Register($src1$$reg),
10812               as_Register($src2$$reg),
10813               Assembler::ROR,
10814               $src3$$constant & 0x1f);
10815   %}
10816 
10817   ins_pipe(ialu_reg_reg_shift);
10818 %}
10819 
10820 // This pattern is automatically generated from aarch64_ad.m4
10821 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10822 // val & (-1 ^ (val ror shift)) ==> bic
10823 instruct AndL_reg_RotateRight_not_reg(iRegLNoSp dst,
10824                          iRegL src1, iRegL src2,
10825                          immI src3, immL_M1 src4) %{
10826   match(Set dst (AndL src1 (XorL(RotateRight src2 src3) src4)));
10827   ins_cost(1.9 * INSN_COST);
10828   format %{ "bic  $dst, $src1, $src2, ROR $src3" %}
10829 
10830   ins_encode %{
10831     __ bic(as_Register($dst$$reg),
10832               as_Register($src1$$reg),
10833               as_Register($src2$$reg),
10834               Assembler::ROR,
10835               $src3$$constant & 0x3f);
10836   %}
10837 
10838   ins_pipe(ialu_reg_reg_shift);
10839 %}
10840 
10841 // This pattern is automatically generated from aarch64_ad.m4
10842 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10843 // val & (-1 ^ (val << shift)) ==> bicw
10844 instruct AndI_reg_LShift_not_reg(iRegINoSp dst,
10845                          iRegIorL2I src1, iRegIorL2I src2,
10846                          immI src3, immI_M1 src4) %{
10847   match(Set dst (AndI src1 (XorI(LShiftI src2 src3) src4)));
10848   ins_cost(1.9 * INSN_COST);
10849   format %{ "bicw  $dst, $src1, $src2, LSL $src3" %}
10850 
10851   ins_encode %{
10852     __ bicw(as_Register($dst$$reg),
10853               as_Register($src1$$reg),
10854               as_Register($src2$$reg),
10855               Assembler::LSL,
10856               $src3$$constant & 0x1f);
10857   %}
10858 
10859   ins_pipe(ialu_reg_reg_shift);
10860 %}
10861 
10862 // This pattern is automatically generated from aarch64_ad.m4
10863 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10864 // val & (-1 ^ (val << shift)) ==> bic
10865 instruct AndL_reg_LShift_not_reg(iRegLNoSp dst,
10866                          iRegL src1, iRegL src2,
10867                          immI src3, immL_M1 src4) %{
10868   match(Set dst (AndL src1 (XorL(LShiftL src2 src3) src4)));
10869   ins_cost(1.9 * INSN_COST);
10870   format %{ "bic  $dst, $src1, $src2, LSL $src3" %}
10871 
10872   ins_encode %{
10873     __ bic(as_Register($dst$$reg),
10874               as_Register($src1$$reg),
10875               as_Register($src2$$reg),
10876               Assembler::LSL,
10877               $src3$$constant & 0x3f);
10878   %}
10879 
10880   ins_pipe(ialu_reg_reg_shift);
10881 %}
10882 
10883 // This pattern is automatically generated from aarch64_ad.m4
10884 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10885 // val ^ (-1 ^ (val >>> shift)) ==> eonw
10886 instruct XorI_reg_URShift_not_reg(iRegINoSp dst,
10887                          iRegIorL2I src1, iRegIorL2I src2,
10888                          immI src3, immI_M1 src4) %{
10889   match(Set dst (XorI src4 (XorI(URShiftI src2 src3) src1)));
10890   ins_cost(1.9 * INSN_COST);
10891   format %{ "eonw  $dst, $src1, $src2, LSR $src3" %}
10892 
10893   ins_encode %{
10894     __ eonw(as_Register($dst$$reg),
10895               as_Register($src1$$reg),
10896               as_Register($src2$$reg),
10897               Assembler::LSR,
10898               $src3$$constant & 0x1f);
10899   %}
10900 
10901   ins_pipe(ialu_reg_reg_shift);
10902 %}
10903 
10904 // This pattern is automatically generated from aarch64_ad.m4
10905 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10906 // val ^ (-1 ^ (val >>> shift)) ==> eon
10907 instruct XorL_reg_URShift_not_reg(iRegLNoSp dst,
10908                          iRegL src1, iRegL src2,
10909                          immI src3, immL_M1 src4) %{
10910   match(Set dst (XorL src4 (XorL(URShiftL src2 src3) src1)));
10911   ins_cost(1.9 * INSN_COST);
10912   format %{ "eon  $dst, $src1, $src2, LSR $src3" %}
10913 
10914   ins_encode %{
10915     __ eon(as_Register($dst$$reg),
10916               as_Register($src1$$reg),
10917               as_Register($src2$$reg),
10918               Assembler::LSR,
10919               $src3$$constant & 0x3f);
10920   %}
10921 
10922   ins_pipe(ialu_reg_reg_shift);
10923 %}
10924 
10925 // This pattern is automatically generated from aarch64_ad.m4
10926 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10927 // val ^ (-1 ^ (val >> shift)) ==> eonw
10928 instruct XorI_reg_RShift_not_reg(iRegINoSp dst,
10929                          iRegIorL2I src1, iRegIorL2I src2,
10930                          immI src3, immI_M1 src4) %{
10931   match(Set dst (XorI src4 (XorI(RShiftI src2 src3) src1)));
10932   ins_cost(1.9 * INSN_COST);
10933   format %{ "eonw  $dst, $src1, $src2, ASR $src3" %}
10934 
10935   ins_encode %{
10936     __ eonw(as_Register($dst$$reg),
10937               as_Register($src1$$reg),
10938               as_Register($src2$$reg),
10939               Assembler::ASR,
10940               $src3$$constant & 0x1f);
10941   %}
10942 
10943   ins_pipe(ialu_reg_reg_shift);
10944 %}
10945 
10946 // This pattern is automatically generated from aarch64_ad.m4
10947 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10948 // val ^ (-1 ^ (val >> shift)) ==> eon
10949 instruct XorL_reg_RShift_not_reg(iRegLNoSp dst,
10950                          iRegL src1, iRegL src2,
10951                          immI src3, immL_M1 src4) %{
10952   match(Set dst (XorL src4 (XorL(RShiftL src2 src3) src1)));
10953   ins_cost(1.9 * INSN_COST);
10954   format %{ "eon  $dst, $src1, $src2, ASR $src3" %}
10955 
10956   ins_encode %{
10957     __ eon(as_Register($dst$$reg),
10958               as_Register($src1$$reg),
10959               as_Register($src2$$reg),
10960               Assembler::ASR,
10961               $src3$$constant & 0x3f);
10962   %}
10963 
10964   ins_pipe(ialu_reg_reg_shift);
10965 %}
10966 
10967 // This pattern is automatically generated from aarch64_ad.m4
10968 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10969 // val ^ (-1 ^ (val ror shift)) ==> eonw
10970 instruct XorI_reg_RotateRight_not_reg(iRegINoSp dst,
10971                          iRegIorL2I src1, iRegIorL2I src2,
10972                          immI src3, immI_M1 src4) %{
10973   match(Set dst (XorI src4 (XorI(RotateRight src2 src3) src1)));
10974   ins_cost(1.9 * INSN_COST);
10975   format %{ "eonw  $dst, $src1, $src2, ROR $src3" %}
10976 
10977   ins_encode %{
10978     __ eonw(as_Register($dst$$reg),
10979               as_Register($src1$$reg),
10980               as_Register($src2$$reg),
10981               Assembler::ROR,
10982               $src3$$constant & 0x1f);
10983   %}
10984 
10985   ins_pipe(ialu_reg_reg_shift);
10986 %}
10987 
10988 // This pattern is automatically generated from aarch64_ad.m4
10989 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10990 // val ^ (-1 ^ (val ror shift)) ==> eon
10991 instruct XorL_reg_RotateRight_not_reg(iRegLNoSp dst,
10992                          iRegL src1, iRegL src2,
10993                          immI src3, immL_M1 src4) %{
10994   match(Set dst (XorL src4 (XorL(RotateRight src2 src3) src1)));
10995   ins_cost(1.9 * INSN_COST);
10996   format %{ "eon  $dst, $src1, $src2, ROR $src3" %}
10997 
10998   ins_encode %{
10999     __ eon(as_Register($dst$$reg),
11000               as_Register($src1$$reg),
11001               as_Register($src2$$reg),
11002               Assembler::ROR,
11003               $src3$$constant & 0x3f);
11004   %}
11005 
11006   ins_pipe(ialu_reg_reg_shift);
11007 %}
11008 
11009 // This pattern is automatically generated from aarch64_ad.m4
11010 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11011 // val ^ (-1 ^ (val << shift)) ==> eonw
11012 instruct XorI_reg_LShift_not_reg(iRegINoSp dst,
11013                          iRegIorL2I src1, iRegIorL2I src2,
11014                          immI src3, immI_M1 src4) %{
11015   match(Set dst (XorI src4 (XorI(LShiftI src2 src3) src1)));
11016   ins_cost(1.9 * INSN_COST);
11017   format %{ "eonw  $dst, $src1, $src2, LSL $src3" %}
11018 
11019   ins_encode %{
11020     __ eonw(as_Register($dst$$reg),
11021               as_Register($src1$$reg),
11022               as_Register($src2$$reg),
11023               Assembler::LSL,
11024               $src3$$constant & 0x1f);
11025   %}
11026 
11027   ins_pipe(ialu_reg_reg_shift);
11028 %}
11029 
11030 // This pattern is automatically generated from aarch64_ad.m4
11031 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11032 // val ^ (-1 ^ (val << shift)) ==> eon
11033 instruct XorL_reg_LShift_not_reg(iRegLNoSp dst,
11034                          iRegL src1, iRegL src2,
11035                          immI src3, immL_M1 src4) %{
11036   match(Set dst (XorL src4 (XorL(LShiftL src2 src3) src1)));
11037   ins_cost(1.9 * INSN_COST);
11038   format %{ "eon  $dst, $src1, $src2, LSL $src3" %}
11039 
11040   ins_encode %{
11041     __ eon(as_Register($dst$$reg),
11042               as_Register($src1$$reg),
11043               as_Register($src2$$reg),
11044               Assembler::LSL,
11045               $src3$$constant & 0x3f);
11046   %}
11047 
11048   ins_pipe(ialu_reg_reg_shift);
11049 %}
11050 
11051 // This pattern is automatically generated from aarch64_ad.m4
11052 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11053 // val | (-1 ^ (val >>> shift)) ==> ornw
11054 instruct OrI_reg_URShift_not_reg(iRegINoSp dst,
11055                          iRegIorL2I src1, iRegIorL2I src2,
11056                          immI src3, immI_M1 src4) %{
11057   match(Set dst (OrI src1 (XorI(URShiftI src2 src3) src4)));
11058   ins_cost(1.9 * INSN_COST);
11059   format %{ "ornw  $dst, $src1, $src2, LSR $src3" %}
11060 
11061   ins_encode %{
11062     __ ornw(as_Register($dst$$reg),
11063               as_Register($src1$$reg),
11064               as_Register($src2$$reg),
11065               Assembler::LSR,
11066               $src3$$constant & 0x1f);
11067   %}
11068 
11069   ins_pipe(ialu_reg_reg_shift);
11070 %}
11071 
11072 // This pattern is automatically generated from aarch64_ad.m4
11073 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11074 // val | (-1 ^ (val >>> shift)) ==> orn
11075 instruct OrL_reg_URShift_not_reg(iRegLNoSp dst,
11076                          iRegL src1, iRegL src2,
11077                          immI src3, immL_M1 src4) %{
11078   match(Set dst (OrL src1 (XorL(URShiftL src2 src3) src4)));
11079   ins_cost(1.9 * INSN_COST);
11080   format %{ "orn  $dst, $src1, $src2, LSR $src3" %}
11081 
11082   ins_encode %{
11083     __ orn(as_Register($dst$$reg),
11084               as_Register($src1$$reg),
11085               as_Register($src2$$reg),
11086               Assembler::LSR,
11087               $src3$$constant & 0x3f);
11088   %}
11089 
11090   ins_pipe(ialu_reg_reg_shift);
11091 %}
11092 
11093 // This pattern is automatically generated from aarch64_ad.m4
11094 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11095 // val | (-1 ^ (val >> shift)) ==> ornw
11096 instruct OrI_reg_RShift_not_reg(iRegINoSp dst,
11097                          iRegIorL2I src1, iRegIorL2I src2,
11098                          immI src3, immI_M1 src4) %{
11099   match(Set dst (OrI src1 (XorI(RShiftI src2 src3) src4)));
11100   ins_cost(1.9 * INSN_COST);
11101   format %{ "ornw  $dst, $src1, $src2, ASR $src3" %}
11102 
11103   ins_encode %{
11104     __ ornw(as_Register($dst$$reg),
11105               as_Register($src1$$reg),
11106               as_Register($src2$$reg),
11107               Assembler::ASR,
11108               $src3$$constant & 0x1f);
11109   %}
11110 
11111   ins_pipe(ialu_reg_reg_shift);
11112 %}
11113 
11114 // This pattern is automatically generated from aarch64_ad.m4
11115 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11116 // val | (-1 ^ (val >> shift)) ==> orn
11117 instruct OrL_reg_RShift_not_reg(iRegLNoSp dst,
11118                          iRegL src1, iRegL src2,
11119                          immI src3, immL_M1 src4) %{
11120   match(Set dst (OrL src1 (XorL(RShiftL src2 src3) src4)));
11121   ins_cost(1.9 * INSN_COST);
11122   format %{ "orn  $dst, $src1, $src2, ASR $src3" %}
11123 
11124   ins_encode %{
11125     __ orn(as_Register($dst$$reg),
11126               as_Register($src1$$reg),
11127               as_Register($src2$$reg),
11128               Assembler::ASR,
11129               $src3$$constant & 0x3f);
11130   %}
11131 
11132   ins_pipe(ialu_reg_reg_shift);
11133 %}
11134 
11135 // This pattern is automatically generated from aarch64_ad.m4
11136 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11137 // val | (-1 ^ (val ror shift)) ==> ornw
11138 instruct OrI_reg_RotateRight_not_reg(iRegINoSp dst,
11139                          iRegIorL2I src1, iRegIorL2I src2,
11140                          immI src3, immI_M1 src4) %{
11141   match(Set dst (OrI src1 (XorI(RotateRight src2 src3) src4)));
11142   ins_cost(1.9 * INSN_COST);
11143   format %{ "ornw  $dst, $src1, $src2, ROR $src3" %}
11144 
11145   ins_encode %{
11146     __ ornw(as_Register($dst$$reg),
11147               as_Register($src1$$reg),
11148               as_Register($src2$$reg),
11149               Assembler::ROR,
11150               $src3$$constant & 0x1f);
11151   %}
11152 
11153   ins_pipe(ialu_reg_reg_shift);
11154 %}
11155 
11156 // This pattern is automatically generated from aarch64_ad.m4
11157 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11158 // val | (-1 ^ (val ror shift)) ==> orn
11159 instruct OrL_reg_RotateRight_not_reg(iRegLNoSp dst,
11160                          iRegL src1, iRegL src2,
11161                          immI src3, immL_M1 src4) %{
11162   match(Set dst (OrL src1 (XorL(RotateRight src2 src3) src4)));
11163   ins_cost(1.9 * INSN_COST);
11164   format %{ "orn  $dst, $src1, $src2, ROR $src3" %}
11165 
11166   ins_encode %{
11167     __ orn(as_Register($dst$$reg),
11168               as_Register($src1$$reg),
11169               as_Register($src2$$reg),
11170               Assembler::ROR,
11171               $src3$$constant & 0x3f);
11172   %}
11173 
11174   ins_pipe(ialu_reg_reg_shift);
11175 %}
11176 
11177 // This pattern is automatically generated from aarch64_ad.m4
11178 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11179 // val | (-1 ^ (val << shift)) ==> ornw
11180 instruct OrI_reg_LShift_not_reg(iRegINoSp dst,
11181                          iRegIorL2I src1, iRegIorL2I src2,
11182                          immI src3, immI_M1 src4) %{
11183   match(Set dst (OrI src1 (XorI(LShiftI src2 src3) src4)));
11184   ins_cost(1.9 * INSN_COST);
11185   format %{ "ornw  $dst, $src1, $src2, LSL $src3" %}
11186 
11187   ins_encode %{
11188     __ ornw(as_Register($dst$$reg),
11189               as_Register($src1$$reg),
11190               as_Register($src2$$reg),
11191               Assembler::LSL,
11192               $src3$$constant & 0x1f);
11193   %}
11194 
11195   ins_pipe(ialu_reg_reg_shift);
11196 %}
11197 
11198 // This pattern is automatically generated from aarch64_ad.m4
11199 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11200 // val | (-1 ^ (val << shift)) ==> orn
11201 instruct OrL_reg_LShift_not_reg(iRegLNoSp dst,
11202                          iRegL src1, iRegL src2,
11203                          immI src3, immL_M1 src4) %{
11204   match(Set dst (OrL src1 (XorL(LShiftL src2 src3) src4)));
11205   ins_cost(1.9 * INSN_COST);
11206   format %{ "orn  $dst, $src1, $src2, LSL $src3" %}
11207 
11208   ins_encode %{
11209     __ orn(as_Register($dst$$reg),
11210               as_Register($src1$$reg),
11211               as_Register($src2$$reg),
11212               Assembler::LSL,
11213               $src3$$constant & 0x3f);
11214   %}
11215 
11216   ins_pipe(ialu_reg_reg_shift);
11217 %}
11218 
11219 // This pattern is automatically generated from aarch64_ad.m4
11220 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11221 instruct AndI_reg_URShift_reg(iRegINoSp dst,
11222                          iRegIorL2I src1, iRegIorL2I src2,
11223                          immI src3) %{
11224   match(Set dst (AndI src1 (URShiftI src2 src3)));
11225 
11226   ins_cost(1.9 * INSN_COST);
11227   format %{ "andw  $dst, $src1, $src2, LSR $src3" %}
11228 
11229   ins_encode %{
11230     __ andw(as_Register($dst$$reg),
11231               as_Register($src1$$reg),
11232               as_Register($src2$$reg),
11233               Assembler::LSR,
11234               $src3$$constant & 0x1f);
11235   %}
11236 
11237   ins_pipe(ialu_reg_reg_shift);
11238 %}
11239 
11240 // This pattern is automatically generated from aarch64_ad.m4
11241 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11242 instruct AndL_reg_URShift_reg(iRegLNoSp dst,
11243                          iRegL src1, iRegL src2,
11244                          immI src3) %{
11245   match(Set dst (AndL src1 (URShiftL src2 src3)));
11246 
11247   ins_cost(1.9 * INSN_COST);
11248   format %{ "andr  $dst, $src1, $src2, LSR $src3" %}
11249 
11250   ins_encode %{
11251     __ andr(as_Register($dst$$reg),
11252               as_Register($src1$$reg),
11253               as_Register($src2$$reg),
11254               Assembler::LSR,
11255               $src3$$constant & 0x3f);
11256   %}
11257 
11258   ins_pipe(ialu_reg_reg_shift);
11259 %}
11260 
11261 // This pattern is automatically generated from aarch64_ad.m4
11262 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11263 instruct AndI_reg_RShift_reg(iRegINoSp dst,
11264                          iRegIorL2I src1, iRegIorL2I src2,
11265                          immI src3) %{
11266   match(Set dst (AndI src1 (RShiftI src2 src3)));
11267 
11268   ins_cost(1.9 * INSN_COST);
11269   format %{ "andw  $dst, $src1, $src2, ASR $src3" %}
11270 
11271   ins_encode %{
11272     __ andw(as_Register($dst$$reg),
11273               as_Register($src1$$reg),
11274               as_Register($src2$$reg),
11275               Assembler::ASR,
11276               $src3$$constant & 0x1f);
11277   %}
11278 
11279   ins_pipe(ialu_reg_reg_shift);
11280 %}
11281 
11282 // This pattern is automatically generated from aarch64_ad.m4
11283 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11284 instruct AndL_reg_RShift_reg(iRegLNoSp dst,
11285                          iRegL src1, iRegL src2,
11286                          immI src3) %{
11287   match(Set dst (AndL src1 (RShiftL src2 src3)));
11288 
11289   ins_cost(1.9 * INSN_COST);
11290   format %{ "andr  $dst, $src1, $src2, ASR $src3" %}
11291 
11292   ins_encode %{
11293     __ andr(as_Register($dst$$reg),
11294               as_Register($src1$$reg),
11295               as_Register($src2$$reg),
11296               Assembler::ASR,
11297               $src3$$constant & 0x3f);
11298   %}
11299 
11300   ins_pipe(ialu_reg_reg_shift);
11301 %}
11302 
11303 // This pattern is automatically generated from aarch64_ad.m4
11304 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11305 instruct AndI_reg_LShift_reg(iRegINoSp dst,
11306                          iRegIorL2I src1, iRegIorL2I src2,
11307                          immI src3) %{
11308   match(Set dst (AndI src1 (LShiftI src2 src3)));
11309 
11310   ins_cost(1.9 * INSN_COST);
11311   format %{ "andw  $dst, $src1, $src2, LSL $src3" %}
11312 
11313   ins_encode %{
11314     __ andw(as_Register($dst$$reg),
11315               as_Register($src1$$reg),
11316               as_Register($src2$$reg),
11317               Assembler::LSL,
11318               $src3$$constant & 0x1f);
11319   %}
11320 
11321   ins_pipe(ialu_reg_reg_shift);
11322 %}
11323 
11324 // This pattern is automatically generated from aarch64_ad.m4
11325 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11326 instruct AndL_reg_LShift_reg(iRegLNoSp dst,
11327                          iRegL src1, iRegL src2,
11328                          immI src3) %{
11329   match(Set dst (AndL src1 (LShiftL src2 src3)));
11330 
11331   ins_cost(1.9 * INSN_COST);
11332   format %{ "andr  $dst, $src1, $src2, LSL $src3" %}
11333 
11334   ins_encode %{
11335     __ andr(as_Register($dst$$reg),
11336               as_Register($src1$$reg),
11337               as_Register($src2$$reg),
11338               Assembler::LSL,
11339               $src3$$constant & 0x3f);
11340   %}
11341 
11342   ins_pipe(ialu_reg_reg_shift);
11343 %}
11344 
11345 // This pattern is automatically generated from aarch64_ad.m4
11346 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11347 instruct AndI_reg_RotateRight_reg(iRegINoSp dst,
11348                          iRegIorL2I src1, iRegIorL2I src2,
11349                          immI src3) %{
11350   match(Set dst (AndI src1 (RotateRight src2 src3)));
11351 
11352   ins_cost(1.9 * INSN_COST);
11353   format %{ "andw  $dst, $src1, $src2, ROR $src3" %}
11354 
11355   ins_encode %{
11356     __ andw(as_Register($dst$$reg),
11357               as_Register($src1$$reg),
11358               as_Register($src2$$reg),
11359               Assembler::ROR,
11360               $src3$$constant & 0x1f);
11361   %}
11362 
11363   ins_pipe(ialu_reg_reg_shift);
11364 %}
11365 
11366 // This pattern is automatically generated from aarch64_ad.m4
11367 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11368 instruct AndL_reg_RotateRight_reg(iRegLNoSp dst,
11369                          iRegL src1, iRegL src2,
11370                          immI src3) %{
11371   match(Set dst (AndL src1 (RotateRight src2 src3)));
11372 
11373   ins_cost(1.9 * INSN_COST);
11374   format %{ "andr  $dst, $src1, $src2, ROR $src3" %}
11375 
11376   ins_encode %{
11377     __ andr(as_Register($dst$$reg),
11378               as_Register($src1$$reg),
11379               as_Register($src2$$reg),
11380               Assembler::ROR,
11381               $src3$$constant & 0x3f);
11382   %}
11383 
11384   ins_pipe(ialu_reg_reg_shift);
11385 %}
11386 
11387 // This pattern is automatically generated from aarch64_ad.m4
11388 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11389 instruct XorI_reg_URShift_reg(iRegINoSp dst,
11390                          iRegIorL2I src1, iRegIorL2I src2,
11391                          immI src3) %{
11392   match(Set dst (XorI src1 (URShiftI src2 src3)));
11393 
11394   ins_cost(1.9 * INSN_COST);
11395   format %{ "eorw  $dst, $src1, $src2, LSR $src3" %}
11396 
11397   ins_encode %{
11398     __ eorw(as_Register($dst$$reg),
11399               as_Register($src1$$reg),
11400               as_Register($src2$$reg),
11401               Assembler::LSR,
11402               $src3$$constant & 0x1f);
11403   %}
11404 
11405   ins_pipe(ialu_reg_reg_shift);
11406 %}
11407 
11408 // This pattern is automatically generated from aarch64_ad.m4
11409 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11410 instruct XorL_reg_URShift_reg(iRegLNoSp dst,
11411                          iRegL src1, iRegL src2,
11412                          immI src3) %{
11413   match(Set dst (XorL src1 (URShiftL src2 src3)));
11414 
11415   ins_cost(1.9 * INSN_COST);
11416   format %{ "eor  $dst, $src1, $src2, LSR $src3" %}
11417 
11418   ins_encode %{
11419     __ eor(as_Register($dst$$reg),
11420               as_Register($src1$$reg),
11421               as_Register($src2$$reg),
11422               Assembler::LSR,
11423               $src3$$constant & 0x3f);
11424   %}
11425 
11426   ins_pipe(ialu_reg_reg_shift);
11427 %}
11428 
11429 // This pattern is automatically generated from aarch64_ad.m4
11430 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11431 instruct XorI_reg_RShift_reg(iRegINoSp dst,
11432                          iRegIorL2I src1, iRegIorL2I src2,
11433                          immI src3) %{
11434   match(Set dst (XorI src1 (RShiftI src2 src3)));
11435 
11436   ins_cost(1.9 * INSN_COST);
11437   format %{ "eorw  $dst, $src1, $src2, ASR $src3" %}
11438 
11439   ins_encode %{
11440     __ eorw(as_Register($dst$$reg),
11441               as_Register($src1$$reg),
11442               as_Register($src2$$reg),
11443               Assembler::ASR,
11444               $src3$$constant & 0x1f);
11445   %}
11446 
11447   ins_pipe(ialu_reg_reg_shift);
11448 %}
11449 
11450 // This pattern is automatically generated from aarch64_ad.m4
11451 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11452 instruct XorL_reg_RShift_reg(iRegLNoSp dst,
11453                          iRegL src1, iRegL src2,
11454                          immI src3) %{
11455   match(Set dst (XorL src1 (RShiftL src2 src3)));
11456 
11457   ins_cost(1.9 * INSN_COST);
11458   format %{ "eor  $dst, $src1, $src2, ASR $src3" %}
11459 
11460   ins_encode %{
11461     __ eor(as_Register($dst$$reg),
11462               as_Register($src1$$reg),
11463               as_Register($src2$$reg),
11464               Assembler::ASR,
11465               $src3$$constant & 0x3f);
11466   %}
11467 
11468   ins_pipe(ialu_reg_reg_shift);
11469 %}
11470 
11471 // This pattern is automatically generated from aarch64_ad.m4
11472 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11473 instruct XorI_reg_LShift_reg(iRegINoSp dst,
11474                          iRegIorL2I src1, iRegIorL2I src2,
11475                          immI src3) %{
11476   match(Set dst (XorI src1 (LShiftI src2 src3)));
11477 
11478   ins_cost(1.9 * INSN_COST);
11479   format %{ "eorw  $dst, $src1, $src2, LSL $src3" %}
11480 
11481   ins_encode %{
11482     __ eorw(as_Register($dst$$reg),
11483               as_Register($src1$$reg),
11484               as_Register($src2$$reg),
11485               Assembler::LSL,
11486               $src3$$constant & 0x1f);
11487   %}
11488 
11489   ins_pipe(ialu_reg_reg_shift);
11490 %}
11491 
11492 // This pattern is automatically generated from aarch64_ad.m4
11493 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11494 instruct XorL_reg_LShift_reg(iRegLNoSp dst,
11495                          iRegL src1, iRegL src2,
11496                          immI src3) %{
11497   match(Set dst (XorL src1 (LShiftL src2 src3)));
11498 
11499   ins_cost(1.9 * INSN_COST);
11500   format %{ "eor  $dst, $src1, $src2, LSL $src3" %}
11501 
11502   ins_encode %{
11503     __ eor(as_Register($dst$$reg),
11504               as_Register($src1$$reg),
11505               as_Register($src2$$reg),
11506               Assembler::LSL,
11507               $src3$$constant & 0x3f);
11508   %}
11509 
11510   ins_pipe(ialu_reg_reg_shift);
11511 %}
11512 
11513 // This pattern is automatically generated from aarch64_ad.m4
11514 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11515 instruct XorI_reg_RotateRight_reg(iRegINoSp dst,
11516                          iRegIorL2I src1, iRegIorL2I src2,
11517                          immI src3) %{
11518   match(Set dst (XorI src1 (RotateRight src2 src3)));
11519 
11520   ins_cost(1.9 * INSN_COST);
11521   format %{ "eorw  $dst, $src1, $src2, ROR $src3" %}
11522 
11523   ins_encode %{
11524     __ eorw(as_Register($dst$$reg),
11525               as_Register($src1$$reg),
11526               as_Register($src2$$reg),
11527               Assembler::ROR,
11528               $src3$$constant & 0x1f);
11529   %}
11530 
11531   ins_pipe(ialu_reg_reg_shift);
11532 %}
11533 
11534 // This pattern is automatically generated from aarch64_ad.m4
11535 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11536 instruct XorL_reg_RotateRight_reg(iRegLNoSp dst,
11537                          iRegL src1, iRegL src2,
11538                          immI src3) %{
11539   match(Set dst (XorL src1 (RotateRight src2 src3)));
11540 
11541   ins_cost(1.9 * INSN_COST);
11542   format %{ "eor  $dst, $src1, $src2, ROR $src3" %}
11543 
11544   ins_encode %{
11545     __ eor(as_Register($dst$$reg),
11546               as_Register($src1$$reg),
11547               as_Register($src2$$reg),
11548               Assembler::ROR,
11549               $src3$$constant & 0x3f);
11550   %}
11551 
11552   ins_pipe(ialu_reg_reg_shift);
11553 %}
11554 
11555 // This pattern is automatically generated from aarch64_ad.m4
11556 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11557 instruct OrI_reg_URShift_reg(iRegINoSp dst,
11558                          iRegIorL2I src1, iRegIorL2I src2,
11559                          immI src3) %{
11560   match(Set dst (OrI src1 (URShiftI src2 src3)));
11561 
11562   ins_cost(1.9 * INSN_COST);
11563   format %{ "orrw  $dst, $src1, $src2, LSR $src3" %}
11564 
11565   ins_encode %{
11566     __ orrw(as_Register($dst$$reg),
11567               as_Register($src1$$reg),
11568               as_Register($src2$$reg),
11569               Assembler::LSR,
11570               $src3$$constant & 0x1f);
11571   %}
11572 
11573   ins_pipe(ialu_reg_reg_shift);
11574 %}
11575 
11576 // This pattern is automatically generated from aarch64_ad.m4
11577 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11578 instruct OrL_reg_URShift_reg(iRegLNoSp dst,
11579                          iRegL src1, iRegL src2,
11580                          immI src3) %{
11581   match(Set dst (OrL src1 (URShiftL src2 src3)));
11582 
11583   ins_cost(1.9 * INSN_COST);
11584   format %{ "orr  $dst, $src1, $src2, LSR $src3" %}
11585 
11586   ins_encode %{
11587     __ orr(as_Register($dst$$reg),
11588               as_Register($src1$$reg),
11589               as_Register($src2$$reg),
11590               Assembler::LSR,
11591               $src3$$constant & 0x3f);
11592   %}
11593 
11594   ins_pipe(ialu_reg_reg_shift);
11595 %}
11596 
11597 // This pattern is automatically generated from aarch64_ad.m4
11598 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11599 instruct OrI_reg_RShift_reg(iRegINoSp dst,
11600                          iRegIorL2I src1, iRegIorL2I src2,
11601                          immI src3) %{
11602   match(Set dst (OrI src1 (RShiftI src2 src3)));
11603 
11604   ins_cost(1.9 * INSN_COST);
11605   format %{ "orrw  $dst, $src1, $src2, ASR $src3" %}
11606 
11607   ins_encode %{
11608     __ orrw(as_Register($dst$$reg),
11609               as_Register($src1$$reg),
11610               as_Register($src2$$reg),
11611               Assembler::ASR,
11612               $src3$$constant & 0x1f);
11613   %}
11614 
11615   ins_pipe(ialu_reg_reg_shift);
11616 %}
11617 
11618 // This pattern is automatically generated from aarch64_ad.m4
11619 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11620 instruct OrL_reg_RShift_reg(iRegLNoSp dst,
11621                          iRegL src1, iRegL src2,
11622                          immI src3) %{
11623   match(Set dst (OrL src1 (RShiftL src2 src3)));
11624 
11625   ins_cost(1.9 * INSN_COST);
11626   format %{ "orr  $dst, $src1, $src2, ASR $src3" %}
11627 
11628   ins_encode %{
11629     __ orr(as_Register($dst$$reg),
11630               as_Register($src1$$reg),
11631               as_Register($src2$$reg),
11632               Assembler::ASR,
11633               $src3$$constant & 0x3f);
11634   %}
11635 
11636   ins_pipe(ialu_reg_reg_shift);
11637 %}
11638 
11639 // This pattern is automatically generated from aarch64_ad.m4
11640 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11641 instruct OrI_reg_LShift_reg(iRegINoSp dst,
11642                          iRegIorL2I src1, iRegIorL2I src2,
11643                          immI src3) %{
11644   match(Set dst (OrI src1 (LShiftI src2 src3)));
11645 
11646   ins_cost(1.9 * INSN_COST);
11647   format %{ "orrw  $dst, $src1, $src2, LSL $src3" %}
11648 
11649   ins_encode %{
11650     __ orrw(as_Register($dst$$reg),
11651               as_Register($src1$$reg),
11652               as_Register($src2$$reg),
11653               Assembler::LSL,
11654               $src3$$constant & 0x1f);
11655   %}
11656 
11657   ins_pipe(ialu_reg_reg_shift);
11658 %}
11659 
11660 // This pattern is automatically generated from aarch64_ad.m4
11661 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11662 instruct OrL_reg_LShift_reg(iRegLNoSp dst,
11663                          iRegL src1, iRegL src2,
11664                          immI src3) %{
11665   match(Set dst (OrL src1 (LShiftL src2 src3)));
11666 
11667   ins_cost(1.9 * INSN_COST);
11668   format %{ "orr  $dst, $src1, $src2, LSL $src3" %}
11669 
11670   ins_encode %{
11671     __ orr(as_Register($dst$$reg),
11672               as_Register($src1$$reg),
11673               as_Register($src2$$reg),
11674               Assembler::LSL,
11675               $src3$$constant & 0x3f);
11676   %}
11677 
11678   ins_pipe(ialu_reg_reg_shift);
11679 %}
11680 
11681 // This pattern is automatically generated from aarch64_ad.m4
11682 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11683 instruct OrI_reg_RotateRight_reg(iRegINoSp dst,
11684                          iRegIorL2I src1, iRegIorL2I src2,
11685                          immI src3) %{
11686   match(Set dst (OrI src1 (RotateRight src2 src3)));
11687 
11688   ins_cost(1.9 * INSN_COST);
11689   format %{ "orrw  $dst, $src1, $src2, ROR $src3" %}
11690 
11691   ins_encode %{
11692     __ orrw(as_Register($dst$$reg),
11693               as_Register($src1$$reg),
11694               as_Register($src2$$reg),
11695               Assembler::ROR,
11696               $src3$$constant & 0x1f);
11697   %}
11698 
11699   ins_pipe(ialu_reg_reg_shift);
11700 %}
11701 
11702 // This pattern is automatically generated from aarch64_ad.m4
11703 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11704 instruct OrL_reg_RotateRight_reg(iRegLNoSp dst,
11705                          iRegL src1, iRegL src2,
11706                          immI src3) %{
11707   match(Set dst (OrL src1 (RotateRight src2 src3)));
11708 
11709   ins_cost(1.9 * INSN_COST);
11710   format %{ "orr  $dst, $src1, $src2, ROR $src3" %}
11711 
11712   ins_encode %{
11713     __ orr(as_Register($dst$$reg),
11714               as_Register($src1$$reg),
11715               as_Register($src2$$reg),
11716               Assembler::ROR,
11717               $src3$$constant & 0x3f);
11718   %}
11719 
11720   ins_pipe(ialu_reg_reg_shift);
11721 %}
11722 
11723 // This pattern is automatically generated from aarch64_ad.m4
11724 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11725 instruct AddI_reg_URShift_reg(iRegINoSp dst,
11726                          iRegIorL2I src1, iRegIorL2I src2,
11727                          immI src3) %{
11728   match(Set dst (AddI src1 (URShiftI src2 src3)));
11729 
11730   ins_cost(1.9 * INSN_COST);
11731   format %{ "addw  $dst, $src1, $src2, LSR $src3" %}
11732 
11733   ins_encode %{
11734     __ addw(as_Register($dst$$reg),
11735               as_Register($src1$$reg),
11736               as_Register($src2$$reg),
11737               Assembler::LSR,
11738               $src3$$constant & 0x1f);
11739   %}
11740 
11741   ins_pipe(ialu_reg_reg_shift);
11742 %}
11743 
11744 // This pattern is automatically generated from aarch64_ad.m4
11745 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11746 instruct AddL_reg_URShift_reg(iRegLNoSp dst,
11747                          iRegL src1, iRegL src2,
11748                          immI src3) %{
11749   match(Set dst (AddL src1 (URShiftL src2 src3)));
11750 
11751   ins_cost(1.9 * INSN_COST);
11752   format %{ "add  $dst, $src1, $src2, LSR $src3" %}
11753 
11754   ins_encode %{
11755     __ add(as_Register($dst$$reg),
11756               as_Register($src1$$reg),
11757               as_Register($src2$$reg),
11758               Assembler::LSR,
11759               $src3$$constant & 0x3f);
11760   %}
11761 
11762   ins_pipe(ialu_reg_reg_shift);
11763 %}
11764 
11765 // This pattern is automatically generated from aarch64_ad.m4
11766 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11767 instruct AddI_reg_RShift_reg(iRegINoSp dst,
11768                          iRegIorL2I src1, iRegIorL2I src2,
11769                          immI src3) %{
11770   match(Set dst (AddI src1 (RShiftI src2 src3)));
11771 
11772   ins_cost(1.9 * INSN_COST);
11773   format %{ "addw  $dst, $src1, $src2, ASR $src3" %}
11774 
11775   ins_encode %{
11776     __ addw(as_Register($dst$$reg),
11777               as_Register($src1$$reg),
11778               as_Register($src2$$reg),
11779               Assembler::ASR,
11780               $src3$$constant & 0x1f);
11781   %}
11782 
11783   ins_pipe(ialu_reg_reg_shift);
11784 %}
11785 
11786 // This pattern is automatically generated from aarch64_ad.m4
11787 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11788 instruct AddL_reg_RShift_reg(iRegLNoSp dst,
11789                          iRegL src1, iRegL src2,
11790                          immI src3) %{
11791   match(Set dst (AddL src1 (RShiftL src2 src3)));
11792 
11793   ins_cost(1.9 * INSN_COST);
11794   format %{ "add  $dst, $src1, $src2, ASR $src3" %}
11795 
11796   ins_encode %{
11797     __ add(as_Register($dst$$reg),
11798               as_Register($src1$$reg),
11799               as_Register($src2$$reg),
11800               Assembler::ASR,
11801               $src3$$constant & 0x3f);
11802   %}
11803 
11804   ins_pipe(ialu_reg_reg_shift);
11805 %}
11806 
11807 // This pattern is automatically generated from aarch64_ad.m4
11808 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11809 instruct AddI_reg_LShift_reg(iRegINoSp dst,
11810                          iRegIorL2I src1, iRegIorL2I src2,
11811                          immI src3) %{
11812   match(Set dst (AddI src1 (LShiftI src2 src3)));
11813 
11814   ins_cost(1.9 * INSN_COST);
11815   format %{ "addw  $dst, $src1, $src2, LSL $src3" %}
11816 
11817   ins_encode %{
11818     __ addw(as_Register($dst$$reg),
11819               as_Register($src1$$reg),
11820               as_Register($src2$$reg),
11821               Assembler::LSL,
11822               $src3$$constant & 0x1f);
11823   %}
11824 
11825   ins_pipe(ialu_reg_reg_shift);
11826 %}
11827 
11828 // This pattern is automatically generated from aarch64_ad.m4
11829 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11830 instruct AddL_reg_LShift_reg(iRegLNoSp dst,
11831                          iRegL src1, iRegL src2,
11832                          immI src3) %{
11833   match(Set dst (AddL src1 (LShiftL src2 src3)));
11834 
11835   ins_cost(1.9 * INSN_COST);
11836   format %{ "add  $dst, $src1, $src2, LSL $src3" %}
11837 
11838   ins_encode %{
11839     __ add(as_Register($dst$$reg),
11840               as_Register($src1$$reg),
11841               as_Register($src2$$reg),
11842               Assembler::LSL,
11843               $src3$$constant & 0x3f);
11844   %}
11845 
11846   ins_pipe(ialu_reg_reg_shift);
11847 %}
11848 
11849 // This pattern is automatically generated from aarch64_ad.m4
11850 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11851 instruct SubI_reg_URShift_reg(iRegINoSp dst,
11852                          iRegIorL2I src1, iRegIorL2I src2,
11853                          immI src3) %{
11854   match(Set dst (SubI src1 (URShiftI src2 src3)));
11855 
11856   ins_cost(1.9 * INSN_COST);
11857   format %{ "subw  $dst, $src1, $src2, LSR $src3" %}
11858 
11859   ins_encode %{
11860     __ subw(as_Register($dst$$reg),
11861               as_Register($src1$$reg),
11862               as_Register($src2$$reg),
11863               Assembler::LSR,
11864               $src3$$constant & 0x1f);
11865   %}
11866 
11867   ins_pipe(ialu_reg_reg_shift);
11868 %}
11869 
11870 // This pattern is automatically generated from aarch64_ad.m4
11871 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11872 instruct SubL_reg_URShift_reg(iRegLNoSp dst,
11873                          iRegL src1, iRegL src2,
11874                          immI src3) %{
11875   match(Set dst (SubL src1 (URShiftL src2 src3)));
11876 
11877   ins_cost(1.9 * INSN_COST);
11878   format %{ "sub  $dst, $src1, $src2, LSR $src3" %}
11879 
11880   ins_encode %{
11881     __ sub(as_Register($dst$$reg),
11882               as_Register($src1$$reg),
11883               as_Register($src2$$reg),
11884               Assembler::LSR,
11885               $src3$$constant & 0x3f);
11886   %}
11887 
11888   ins_pipe(ialu_reg_reg_shift);
11889 %}
11890 
11891 // This pattern is automatically generated from aarch64_ad.m4
11892 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11893 instruct SubI_reg_RShift_reg(iRegINoSp dst,
11894                          iRegIorL2I src1, iRegIorL2I src2,
11895                          immI src3) %{
11896   match(Set dst (SubI src1 (RShiftI src2 src3)));
11897 
11898   ins_cost(1.9 * INSN_COST);
11899   format %{ "subw  $dst, $src1, $src2, ASR $src3" %}
11900 
11901   ins_encode %{
11902     __ subw(as_Register($dst$$reg),
11903               as_Register($src1$$reg),
11904               as_Register($src2$$reg),
11905               Assembler::ASR,
11906               $src3$$constant & 0x1f);
11907   %}
11908 
11909   ins_pipe(ialu_reg_reg_shift);
11910 %}
11911 
11912 // This pattern is automatically generated from aarch64_ad.m4
11913 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11914 instruct SubL_reg_RShift_reg(iRegLNoSp dst,
11915                          iRegL src1, iRegL src2,
11916                          immI src3) %{
11917   match(Set dst (SubL src1 (RShiftL src2 src3)));
11918 
11919   ins_cost(1.9 * INSN_COST);
11920   format %{ "sub  $dst, $src1, $src2, ASR $src3" %}
11921 
11922   ins_encode %{
11923     __ sub(as_Register($dst$$reg),
11924               as_Register($src1$$reg),
11925               as_Register($src2$$reg),
11926               Assembler::ASR,
11927               $src3$$constant & 0x3f);
11928   %}
11929 
11930   ins_pipe(ialu_reg_reg_shift);
11931 %}
11932 
11933 // This pattern is automatically generated from aarch64_ad.m4
11934 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11935 instruct SubI_reg_LShift_reg(iRegINoSp dst,
11936                          iRegIorL2I src1, iRegIorL2I src2,
11937                          immI src3) %{
11938   match(Set dst (SubI src1 (LShiftI src2 src3)));
11939 
11940   ins_cost(1.9 * INSN_COST);
11941   format %{ "subw  $dst, $src1, $src2, LSL $src3" %}
11942 
11943   ins_encode %{
11944     __ subw(as_Register($dst$$reg),
11945               as_Register($src1$$reg),
11946               as_Register($src2$$reg),
11947               Assembler::LSL,
11948               $src3$$constant & 0x1f);
11949   %}
11950 
11951   ins_pipe(ialu_reg_reg_shift);
11952 %}
11953 
11954 // This pattern is automatically generated from aarch64_ad.m4
11955 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11956 instruct SubL_reg_LShift_reg(iRegLNoSp dst,
11957                          iRegL src1, iRegL src2,
11958                          immI src3) %{
11959   match(Set dst (SubL src1 (LShiftL src2 src3)));
11960 
11961   ins_cost(1.9 * INSN_COST);
11962   format %{ "sub  $dst, $src1, $src2, LSL $src3" %}
11963 
11964   ins_encode %{
11965     __ sub(as_Register($dst$$reg),
11966               as_Register($src1$$reg),
11967               as_Register($src2$$reg),
11968               Assembler::LSL,
11969               $src3$$constant & 0x3f);
11970   %}
11971 
11972   ins_pipe(ialu_reg_reg_shift);
11973 %}
11974 
11975 // This pattern is automatically generated from aarch64_ad.m4
11976 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11977 
11978 // Shift Left followed by Shift Right.
11979 // This idiom is used by the compiler for the i2b bytecode etc.
11980 instruct sbfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
11981 %{
11982   match(Set dst (RShiftL (LShiftL src lshift_count) rshift_count));
11983   ins_cost(INSN_COST * 2);
11984   format %{ "sbfm  $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
11985   ins_encode %{
11986     int lshift = $lshift_count$$constant & 63;
11987     int rshift = $rshift_count$$constant & 63;
11988     int s = 63 - lshift;
11989     int r = (rshift - lshift) & 63;
11990     __ sbfm(as_Register($dst$$reg),
11991             as_Register($src$$reg),
11992             r, s);
11993   %}
11994 
11995   ins_pipe(ialu_reg_shift);
11996 %}
11997 
11998 // This pattern is automatically generated from aarch64_ad.m4
11999 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12000 
12001 // Shift Left followed by Shift Right.
12002 // This idiom is used by the compiler for the i2b bytecode etc.
12003 instruct sbfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12004 %{
12005   match(Set dst (RShiftI (LShiftI src lshift_count) rshift_count));
12006   ins_cost(INSN_COST * 2);
12007   format %{ "sbfmw  $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12008   ins_encode %{
12009     int lshift = $lshift_count$$constant & 31;
12010     int rshift = $rshift_count$$constant & 31;
12011     int s = 31 - lshift;
12012     int r = (rshift - lshift) & 31;
12013     __ sbfmw(as_Register($dst$$reg),
12014             as_Register($src$$reg),
12015             r, s);
12016   %}
12017 
12018   ins_pipe(ialu_reg_shift);
12019 %}
12020 
12021 // This pattern is automatically generated from aarch64_ad.m4
12022 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12023 
12024 // Shift Left followed by Shift Right.
12025 // This idiom is used by the compiler for the i2b bytecode etc.
12026 instruct ubfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12027 %{
12028   match(Set dst (URShiftL (LShiftL src lshift_count) rshift_count));
12029   ins_cost(INSN_COST * 2);
12030   format %{ "ubfm  $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12031   ins_encode %{
12032     int lshift = $lshift_count$$constant & 63;
12033     int rshift = $rshift_count$$constant & 63;
12034     int s = 63 - lshift;
12035     int r = (rshift - lshift) & 63;
12036     __ ubfm(as_Register($dst$$reg),
12037             as_Register($src$$reg),
12038             r, s);
12039   %}
12040 
12041   ins_pipe(ialu_reg_shift);
12042 %}
12043 
12044 // This pattern is automatically generated from aarch64_ad.m4
12045 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12046 
12047 // Shift Left followed by Shift Right.
12048 // This idiom is used by the compiler for the i2b bytecode etc.
12049 instruct ubfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12050 %{
12051   match(Set dst (URShiftI (LShiftI src lshift_count) rshift_count));
12052   ins_cost(INSN_COST * 2);
12053   format %{ "ubfmw  $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12054   ins_encode %{
12055     int lshift = $lshift_count$$constant & 31;
12056     int rshift = $rshift_count$$constant & 31;
12057     int s = 31 - lshift;
12058     int r = (rshift - lshift) & 31;
12059     __ ubfmw(as_Register($dst$$reg),
12060             as_Register($src$$reg),
12061             r, s);
12062   %}
12063 
12064   ins_pipe(ialu_reg_shift);
12065 %}
12066 
12067 // Bitfield extract with shift & mask
12068 
12069 // This pattern is automatically generated from aarch64_ad.m4
12070 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12071 instruct ubfxwI(iRegINoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12072 %{
12073   match(Set dst (AndI (URShiftI src rshift) mask));
12074   // Make sure we are not going to exceed what ubfxw can do.
12075   predicate((exact_log2(n->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12076 
12077   ins_cost(INSN_COST);
12078   format %{ "ubfxw $dst, $src, $rshift, $mask" %}
12079   ins_encode %{
12080     int rshift = $rshift$$constant & 31;
12081     intptr_t mask = $mask$$constant;
12082     int width = exact_log2(mask+1);
12083     __ ubfxw(as_Register($dst$$reg),
12084             as_Register($src$$reg), rshift, width);
12085   %}
12086   ins_pipe(ialu_reg_shift);
12087 %}
12088 
12089 // This pattern is automatically generated from aarch64_ad.m4
12090 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12091 instruct ubfxL(iRegLNoSp dst, iRegL src, immI rshift, immL_bitmask mask)
12092 %{
12093   match(Set dst (AndL (URShiftL src rshift) mask));
12094   // Make sure we are not going to exceed what ubfx can do.
12095   predicate((exact_log2_long(n->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= (63 + 1));
12096 
12097   ins_cost(INSN_COST);
12098   format %{ "ubfx $dst, $src, $rshift, $mask" %}
12099   ins_encode %{
12100     int rshift = $rshift$$constant & 63;
12101     intptr_t mask = $mask$$constant;
12102     int width = exact_log2_long(mask+1);
12103     __ ubfx(as_Register($dst$$reg),
12104             as_Register($src$$reg), rshift, width);
12105   %}
12106   ins_pipe(ialu_reg_shift);
12107 %}
12108 
12109 
12110 // This pattern is automatically generated from aarch64_ad.m4
12111 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12112 
12113 // We can use ubfx when extending an And with a mask when we know mask
12114 // is positive.  We know that because immI_bitmask guarantees it.
12115 instruct ubfxIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12116 %{
12117   match(Set dst (ConvI2L (AndI (URShiftI src rshift) mask)));
12118   // Make sure we are not going to exceed what ubfxw can do.
12119   predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12120 
12121   ins_cost(INSN_COST * 2);
12122   format %{ "ubfx $dst, $src, $rshift, $mask" %}
12123   ins_encode %{
12124     int rshift = $rshift$$constant & 31;
12125     intptr_t mask = $mask$$constant;
12126     int width = exact_log2(mask+1);
12127     __ ubfx(as_Register($dst$$reg),
12128             as_Register($src$$reg), rshift, width);
12129   %}
12130   ins_pipe(ialu_reg_shift);
12131 %}
12132 
12133 
12134 // This pattern is automatically generated from aarch64_ad.m4
12135 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12136 
12137 // We can use ubfiz when masking by a positive number and then left shifting the result.
12138 // We know that the mask is positive because immI_bitmask guarantees it.
12139 instruct ubfizwI(iRegINoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12140 %{
12141   match(Set dst (LShiftI (AndI src mask) lshift));
12142   predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 31)) <= (31 + 1));
12143 
12144   ins_cost(INSN_COST);
12145   format %{ "ubfizw $dst, $src, $lshift, $mask" %}
12146   ins_encode %{
12147     int lshift = $lshift$$constant & 31;
12148     intptr_t mask = $mask$$constant;
12149     int width = exact_log2(mask+1);
12150     __ ubfizw(as_Register($dst$$reg),
12151           as_Register($src$$reg), lshift, width);
12152   %}
12153   ins_pipe(ialu_reg_shift);
12154 %}
12155 
12156 // This pattern is automatically generated from aarch64_ad.m4
12157 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12158 
12159 // We can use ubfiz when masking by a positive number and then left shifting the result.
12160 // We know that the mask is positive because immL_bitmask guarantees it.
12161 instruct ubfizL(iRegLNoSp dst, iRegL src, immI lshift, immL_bitmask mask)
12162 %{
12163   match(Set dst (LShiftL (AndL src mask) lshift));
12164   predicate((exact_log2_long(n->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
12165 
12166   ins_cost(INSN_COST);
12167   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12168   ins_encode %{
12169     int lshift = $lshift$$constant & 63;
12170     intptr_t mask = $mask$$constant;
12171     int width = exact_log2_long(mask+1);
12172     __ ubfiz(as_Register($dst$$reg),
12173           as_Register($src$$reg), lshift, width);
12174   %}
12175   ins_pipe(ialu_reg_shift);
12176 %}
12177 
12178 // This pattern is automatically generated from aarch64_ad.m4
12179 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12180 
12181 // We can use ubfiz when masking by a positive number and then left shifting the result.
12182 // We know that the mask is positive because immI_bitmask guarantees it.
12183 instruct ubfizwIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12184 %{
12185   match(Set dst (ConvI2L (LShiftI (AndI src mask) lshift)));
12186   predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= 31);
12187 
12188   ins_cost(INSN_COST);
12189   format %{ "ubfizw $dst, $src, $lshift, $mask" %}
12190   ins_encode %{
12191     int lshift = $lshift$$constant & 31;
12192     intptr_t mask = $mask$$constant;
12193     int width = exact_log2(mask+1);
12194     __ ubfizw(as_Register($dst$$reg),
12195           as_Register($src$$reg), lshift, width);
12196   %}
12197   ins_pipe(ialu_reg_shift);
12198 %}
12199 
12200 // This pattern is automatically generated from aarch64_ad.m4
12201 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12202 
12203 // We can use ubfiz when masking by a positive number and then left shifting the result.
12204 // We know that the mask is positive because immL_bitmask guarantees it.
12205 instruct ubfizLConvL2I(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
12206 %{
12207   match(Set dst (ConvL2I (LShiftL (AndL src mask) lshift)));
12208   predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= 31);
12209 
12210   ins_cost(INSN_COST);
12211   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12212   ins_encode %{
12213     int lshift = $lshift$$constant & 63;
12214     intptr_t mask = $mask$$constant;
12215     int width = exact_log2_long(mask+1);
12216     __ ubfiz(as_Register($dst$$reg),
12217           as_Register($src$$reg), lshift, width);
12218   %}
12219   ins_pipe(ialu_reg_shift);
12220 %}
12221 
12222 
12223 // This pattern is automatically generated from aarch64_ad.m4
12224 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12225 
12226 // If there is a convert I to L block between and AndI and a LShiftL, we can also match ubfiz
12227 instruct ubfizIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12228 %{
12229   match(Set dst (LShiftL (ConvI2L (AndI src mask)) lshift));
12230   predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
12231 
12232   ins_cost(INSN_COST);
12233   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12234   ins_encode %{
12235     int lshift = $lshift$$constant & 63;
12236     intptr_t mask = $mask$$constant;
12237     int width = exact_log2(mask+1);
12238     __ ubfiz(as_Register($dst$$reg),
12239              as_Register($src$$reg), lshift, width);
12240   %}
12241   ins_pipe(ialu_reg_shift);
12242 %}
12243 
12244 // This pattern is automatically generated from aarch64_ad.m4
12245 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12246 
12247 // If there is a convert L to I block between and AndL and a LShiftI, we can also match ubfiz
12248 instruct ubfizLConvL2Ix(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
12249 %{
12250   match(Set dst (LShiftI (ConvL2I (AndL src mask)) lshift));
12251   predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 31)) <= 31);
12252 
12253   ins_cost(INSN_COST);
12254   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12255   ins_encode %{
12256     int lshift = $lshift$$constant & 31;
12257     intptr_t mask = $mask$$constant;
12258     int width = exact_log2(mask+1);
12259     __ ubfiz(as_Register($dst$$reg),
12260              as_Register($src$$reg), lshift, width);
12261   %}
12262   ins_pipe(ialu_reg_shift);
12263 %}
12264 
12265 // This pattern is automatically generated from aarch64_ad.m4
12266 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12267 
12268 // Can skip int2long conversions after AND with small bitmask
12269 instruct ubfizIConvI2LAndI(iRegLNoSp dst, iRegI src, immI_bitmask msk)
12270 %{
12271   match(Set dst (ConvI2L (AndI src msk)));
12272   ins_cost(INSN_COST);
12273   format %{ "ubfiz $dst, $src, 0, exact_log2($msk + 1) " %}
12274   ins_encode %{
12275     __ ubfiz(as_Register($dst$$reg), as_Register($src$$reg), 0, exact_log2($msk$$constant + 1));
12276   %}
12277   ins_pipe(ialu_reg_shift);
12278 %}
12279 
12280 
12281 // Rotations
12282 
12283 // This pattern is automatically generated from aarch64_ad.m4
12284 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12285 instruct extrOrL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
12286 %{
12287   match(Set dst (OrL (LShiftL src1 lshift) (URShiftL src2 rshift)));
12288   predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
12289 
12290   ins_cost(INSN_COST);
12291   format %{ "extr $dst, $src1, $src2, #$rshift" %}
12292 
12293   ins_encode %{
12294     __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12295             $rshift$$constant & 63);
12296   %}
12297   ins_pipe(ialu_reg_reg_extr);
12298 %}
12299 
12300 
12301 // This pattern is automatically generated from aarch64_ad.m4
12302 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12303 instruct extrOrI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
12304 %{
12305   match(Set dst (OrI (LShiftI src1 lshift) (URShiftI src2 rshift)));
12306   predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
12307 
12308   ins_cost(INSN_COST);
12309   format %{ "extr $dst, $src1, $src2, #$rshift" %}
12310 
12311   ins_encode %{
12312     __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12313             $rshift$$constant & 31);
12314   %}
12315   ins_pipe(ialu_reg_reg_extr);
12316 %}
12317 
12318 
12319 // This pattern is automatically generated from aarch64_ad.m4
12320 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12321 instruct extrAddL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
12322 %{
12323   match(Set dst (AddL (LShiftL src1 lshift) (URShiftL src2 rshift)));
12324   predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
12325 
12326   ins_cost(INSN_COST);
12327   format %{ "extr $dst, $src1, $src2, #$rshift" %}
12328 
12329   ins_encode %{
12330     __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12331             $rshift$$constant & 63);
12332   %}
12333   ins_pipe(ialu_reg_reg_extr);
12334 %}
12335 
12336 
12337 // This pattern is automatically generated from aarch64_ad.m4
12338 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12339 instruct extrAddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
12340 %{
12341   match(Set dst (AddI (LShiftI src1 lshift) (URShiftI src2 rshift)));
12342   predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
12343 
12344   ins_cost(INSN_COST);
12345   format %{ "extr $dst, $src1, $src2, #$rshift" %}
12346 
12347   ins_encode %{
12348     __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12349             $rshift$$constant & 31);
12350   %}
12351   ins_pipe(ialu_reg_reg_extr);
12352 %}
12353 
12354 // This pattern is automatically generated from aarch64_ad.m4
12355 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12356 instruct rorI_imm(iRegINoSp dst, iRegI src, immI shift)
12357 %{
12358   match(Set dst (RotateRight src shift));
12359 
12360   ins_cost(INSN_COST);
12361   format %{ "ror    $dst, $src, $shift" %}
12362 
12363   ins_encode %{
12364      __ extrw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
12365                $shift$$constant & 0x1f);
12366   %}
12367   ins_pipe(ialu_reg_reg_vshift);
12368 %}
12369 
12370 // This pattern is automatically generated from aarch64_ad.m4
12371 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12372 instruct rorL_imm(iRegLNoSp dst, iRegL src, immI shift)
12373 %{
12374   match(Set dst (RotateRight src shift));
12375 
12376   ins_cost(INSN_COST);
12377   format %{ "ror    $dst, $src, $shift" %}
12378 
12379   ins_encode %{
12380      __ extr(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
12381                $shift$$constant & 0x3f);
12382   %}
12383   ins_pipe(ialu_reg_reg_vshift);
12384 %}
12385 
12386 // This pattern is automatically generated from aarch64_ad.m4
12387 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12388 instruct rorI_reg(iRegINoSp dst, iRegI src, iRegI shift)
12389 %{
12390   match(Set dst (RotateRight src shift));
12391 
12392   ins_cost(INSN_COST);
12393   format %{ "ror    $dst, $src, $shift" %}
12394 
12395   ins_encode %{
12396      __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
12397   %}
12398   ins_pipe(ialu_reg_reg_vshift);
12399 %}
12400 
12401 // This pattern is automatically generated from aarch64_ad.m4
12402 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12403 instruct rorL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
12404 %{
12405   match(Set dst (RotateRight src shift));
12406 
12407   ins_cost(INSN_COST);
12408   format %{ "ror    $dst, $src, $shift" %}
12409 
12410   ins_encode %{
12411      __ rorv(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
12412   %}
12413   ins_pipe(ialu_reg_reg_vshift);
12414 %}
12415 
12416 // This pattern is automatically generated from aarch64_ad.m4
12417 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12418 instruct rolI_reg(iRegINoSp dst, iRegI src, iRegI shift)
12419 %{
12420   match(Set dst (RotateLeft src shift));
12421 
12422   ins_cost(INSN_COST);
12423   format %{ "rol    $dst, $src, $shift" %}
12424 
12425   ins_encode %{
12426      __ subw(rscratch1, zr, as_Register($shift$$reg));
12427      __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
12428   %}
12429   ins_pipe(ialu_reg_reg_vshift);
12430 %}
12431 
12432 // This pattern is automatically generated from aarch64_ad.m4
12433 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12434 instruct rolL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
12435 %{
12436   match(Set dst (RotateLeft src shift));
12437 
12438   ins_cost(INSN_COST);
12439   format %{ "rol    $dst, $src, $shift" %}
12440 
12441   ins_encode %{
12442      __ subw(rscratch1, zr, as_Register($shift$$reg));
12443      __ rorv(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
12444   %}
12445   ins_pipe(ialu_reg_reg_vshift);
12446 %}
12447 
12448 
12449 // Add/subtract (extended)
12450 
12451 // This pattern is automatically generated from aarch64_ad.m4
12452 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12453 instruct AddExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
12454 %{
12455   match(Set dst (AddL src1 (ConvI2L src2)));
12456   ins_cost(INSN_COST);
12457   format %{ "add  $dst, $src1, $src2, sxtw" %}
12458 
12459    ins_encode %{
12460      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12461             as_Register($src2$$reg), ext::sxtw);
12462    %}
12463   ins_pipe(ialu_reg_reg);
12464 %}
12465 
12466 // This pattern is automatically generated from aarch64_ad.m4
12467 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12468 instruct SubExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
12469 %{
12470   match(Set dst (SubL src1 (ConvI2L src2)));
12471   ins_cost(INSN_COST);
12472   format %{ "sub  $dst, $src1, $src2, sxtw" %}
12473 
12474    ins_encode %{
12475      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12476             as_Register($src2$$reg), ext::sxtw);
12477    %}
12478   ins_pipe(ialu_reg_reg);
12479 %}
12480 
12481 // This pattern is automatically generated from aarch64_ad.m4
12482 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12483 instruct AddExtI_sxth(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_16 lshift, immI_16 rshift, rFlagsReg cr)
12484 %{
12485   match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
12486   ins_cost(INSN_COST);
12487   format %{ "add  $dst, $src1, $src2, sxth" %}
12488 
12489    ins_encode %{
12490      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12491             as_Register($src2$$reg), ext::sxth);
12492    %}
12493   ins_pipe(ialu_reg_reg);
12494 %}
12495 
12496 // This pattern is automatically generated from aarch64_ad.m4
12497 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12498 instruct AddExtI_sxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
12499 %{
12500   match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
12501   ins_cost(INSN_COST);
12502   format %{ "add  $dst, $src1, $src2, sxtb" %}
12503 
12504    ins_encode %{
12505      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12506             as_Register($src2$$reg), ext::sxtb);
12507    %}
12508   ins_pipe(ialu_reg_reg);
12509 %}
12510 
12511 // This pattern is automatically generated from aarch64_ad.m4
12512 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12513 instruct AddExtI_uxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
12514 %{
12515   match(Set dst (AddI src1 (URShiftI (LShiftI src2 lshift) rshift)));
12516   ins_cost(INSN_COST);
12517   format %{ "add  $dst, $src1, $src2, uxtb" %}
12518 
12519    ins_encode %{
12520      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12521             as_Register($src2$$reg), ext::uxtb);
12522    %}
12523   ins_pipe(ialu_reg_reg);
12524 %}
12525 
12526 // This pattern is automatically generated from aarch64_ad.m4
12527 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12528 instruct AddExtL_sxth(iRegLNoSp dst, iRegL src1, iRegL src2, immI_48 lshift, immI_48 rshift, rFlagsReg cr)
12529 %{
12530   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12531   ins_cost(INSN_COST);
12532   format %{ "add  $dst, $src1, $src2, sxth" %}
12533 
12534    ins_encode %{
12535      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12536             as_Register($src2$$reg), ext::sxth);
12537    %}
12538   ins_pipe(ialu_reg_reg);
12539 %}
12540 
12541 // This pattern is automatically generated from aarch64_ad.m4
12542 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12543 instruct AddExtL_sxtw(iRegLNoSp dst, iRegL src1, iRegL src2, immI_32 lshift, immI_32 rshift, rFlagsReg cr)
12544 %{
12545   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12546   ins_cost(INSN_COST);
12547   format %{ "add  $dst, $src1, $src2, sxtw" %}
12548 
12549    ins_encode %{
12550      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12551             as_Register($src2$$reg), ext::sxtw);
12552    %}
12553   ins_pipe(ialu_reg_reg);
12554 %}
12555 
12556 // This pattern is automatically generated from aarch64_ad.m4
12557 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12558 instruct AddExtL_sxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
12559 %{
12560   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12561   ins_cost(INSN_COST);
12562   format %{ "add  $dst, $src1, $src2, sxtb" %}
12563 
12564    ins_encode %{
12565      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12566             as_Register($src2$$reg), ext::sxtb);
12567    %}
12568   ins_pipe(ialu_reg_reg);
12569 %}
12570 
12571 // This pattern is automatically generated from aarch64_ad.m4
12572 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12573 instruct AddExtL_uxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
12574 %{
12575   match(Set dst (AddL src1 (URShiftL (LShiftL src2 lshift) rshift)));
12576   ins_cost(INSN_COST);
12577   format %{ "add  $dst, $src1, $src2, uxtb" %}
12578 
12579    ins_encode %{
12580      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12581             as_Register($src2$$reg), ext::uxtb);
12582    %}
12583   ins_pipe(ialu_reg_reg);
12584 %}
12585 
12586 // This pattern is automatically generated from aarch64_ad.m4
12587 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12588 instruct AddExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
12589 %{
12590   match(Set dst (AddI src1 (AndI src2 mask)));
12591   ins_cost(INSN_COST);
12592   format %{ "addw  $dst, $src1, $src2, uxtb" %}
12593 
12594    ins_encode %{
12595      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12596             as_Register($src2$$reg), ext::uxtb);
12597    %}
12598   ins_pipe(ialu_reg_reg);
12599 %}
12600 
12601 // This pattern is automatically generated from aarch64_ad.m4
12602 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12603 instruct AddExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
12604 %{
12605   match(Set dst (AddI src1 (AndI src2 mask)));
12606   ins_cost(INSN_COST);
12607   format %{ "addw  $dst, $src1, $src2, uxth" %}
12608 
12609    ins_encode %{
12610      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12611             as_Register($src2$$reg), ext::uxth);
12612    %}
12613   ins_pipe(ialu_reg_reg);
12614 %}
12615 
12616 // This pattern is automatically generated from aarch64_ad.m4
12617 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12618 instruct AddExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
12619 %{
12620   match(Set dst (AddL src1 (AndL src2 mask)));
12621   ins_cost(INSN_COST);
12622   format %{ "add  $dst, $src1, $src2, uxtb" %}
12623 
12624    ins_encode %{
12625      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12626             as_Register($src2$$reg), ext::uxtb);
12627    %}
12628   ins_pipe(ialu_reg_reg);
12629 %}
12630 
12631 // This pattern is automatically generated from aarch64_ad.m4
12632 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12633 instruct AddExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
12634 %{
12635   match(Set dst (AddL src1 (AndL src2 mask)));
12636   ins_cost(INSN_COST);
12637   format %{ "add  $dst, $src1, $src2, uxth" %}
12638 
12639    ins_encode %{
12640      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12641             as_Register($src2$$reg), ext::uxth);
12642    %}
12643   ins_pipe(ialu_reg_reg);
12644 %}
12645 
12646 // This pattern is automatically generated from aarch64_ad.m4
12647 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12648 instruct AddExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
12649 %{
12650   match(Set dst (AddL src1 (AndL src2 mask)));
12651   ins_cost(INSN_COST);
12652   format %{ "add  $dst, $src1, $src2, uxtw" %}
12653 
12654    ins_encode %{
12655      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12656             as_Register($src2$$reg), ext::uxtw);
12657    %}
12658   ins_pipe(ialu_reg_reg);
12659 %}
12660 
12661 // This pattern is automatically generated from aarch64_ad.m4
12662 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12663 instruct SubExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
12664 %{
12665   match(Set dst (SubI src1 (AndI src2 mask)));
12666   ins_cost(INSN_COST);
12667   format %{ "subw  $dst, $src1, $src2, uxtb" %}
12668 
12669    ins_encode %{
12670      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12671             as_Register($src2$$reg), ext::uxtb);
12672    %}
12673   ins_pipe(ialu_reg_reg);
12674 %}
12675 
12676 // This pattern is automatically generated from aarch64_ad.m4
12677 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12678 instruct SubExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
12679 %{
12680   match(Set dst (SubI src1 (AndI src2 mask)));
12681   ins_cost(INSN_COST);
12682   format %{ "subw  $dst, $src1, $src2, uxth" %}
12683 
12684    ins_encode %{
12685      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12686             as_Register($src2$$reg), ext::uxth);
12687    %}
12688   ins_pipe(ialu_reg_reg);
12689 %}
12690 
12691 // This pattern is automatically generated from aarch64_ad.m4
12692 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12693 instruct SubExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
12694 %{
12695   match(Set dst (SubL src1 (AndL src2 mask)));
12696   ins_cost(INSN_COST);
12697   format %{ "sub  $dst, $src1, $src2, uxtb" %}
12698 
12699    ins_encode %{
12700      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12701             as_Register($src2$$reg), ext::uxtb);
12702    %}
12703   ins_pipe(ialu_reg_reg);
12704 %}
12705 
12706 // This pattern is automatically generated from aarch64_ad.m4
12707 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12708 instruct SubExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
12709 %{
12710   match(Set dst (SubL src1 (AndL src2 mask)));
12711   ins_cost(INSN_COST);
12712   format %{ "sub  $dst, $src1, $src2, uxth" %}
12713 
12714    ins_encode %{
12715      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12716             as_Register($src2$$reg), ext::uxth);
12717    %}
12718   ins_pipe(ialu_reg_reg);
12719 %}
12720 
12721 // This pattern is automatically generated from aarch64_ad.m4
12722 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12723 instruct SubExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
12724 %{
12725   match(Set dst (SubL src1 (AndL src2 mask)));
12726   ins_cost(INSN_COST);
12727   format %{ "sub  $dst, $src1, $src2, uxtw" %}
12728 
12729    ins_encode %{
12730      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12731             as_Register($src2$$reg), ext::uxtw);
12732    %}
12733   ins_pipe(ialu_reg_reg);
12734 %}
12735 
12736 
12737 // This pattern is automatically generated from aarch64_ad.m4
12738 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12739 instruct AddExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
12740 %{
12741   match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12742   ins_cost(1.9 * INSN_COST);
12743   format %{ "add  $dst, $src1, $src2, sxtb #lshift2" %}
12744 
12745    ins_encode %{
12746      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12747             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12748    %}
12749   ins_pipe(ialu_reg_reg_shift);
12750 %}
12751 
12752 // This pattern is automatically generated from aarch64_ad.m4
12753 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12754 instruct AddExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
12755 %{
12756   match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12757   ins_cost(1.9 * INSN_COST);
12758   format %{ "add  $dst, $src1, $src2, sxth #lshift2" %}
12759 
12760    ins_encode %{
12761      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12762             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12763    %}
12764   ins_pipe(ialu_reg_reg_shift);
12765 %}
12766 
12767 // This pattern is automatically generated from aarch64_ad.m4
12768 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12769 instruct AddExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
12770 %{
12771   match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12772   ins_cost(1.9 * INSN_COST);
12773   format %{ "add  $dst, $src1, $src2, sxtw #lshift2" %}
12774 
12775    ins_encode %{
12776      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12777             as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
12778    %}
12779   ins_pipe(ialu_reg_reg_shift);
12780 %}
12781 
12782 // This pattern is automatically generated from aarch64_ad.m4
12783 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12784 instruct SubExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
12785 %{
12786   match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12787   ins_cost(1.9 * INSN_COST);
12788   format %{ "sub  $dst, $src1, $src2, sxtb #lshift2" %}
12789 
12790    ins_encode %{
12791      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12792             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12793    %}
12794   ins_pipe(ialu_reg_reg_shift);
12795 %}
12796 
12797 // This pattern is automatically generated from aarch64_ad.m4
12798 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12799 instruct SubExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
12800 %{
12801   match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12802   ins_cost(1.9 * INSN_COST);
12803   format %{ "sub  $dst, $src1, $src2, sxth #lshift2" %}
12804 
12805    ins_encode %{
12806      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12807             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12808    %}
12809   ins_pipe(ialu_reg_reg_shift);
12810 %}
12811 
12812 // This pattern is automatically generated from aarch64_ad.m4
12813 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12814 instruct SubExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
12815 %{
12816   match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12817   ins_cost(1.9 * INSN_COST);
12818   format %{ "sub  $dst, $src1, $src2, sxtw #lshift2" %}
12819 
12820    ins_encode %{
12821      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12822             as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
12823    %}
12824   ins_pipe(ialu_reg_reg_shift);
12825 %}
12826 
12827 // This pattern is automatically generated from aarch64_ad.m4
12828 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12829 instruct AddExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
12830 %{
12831   match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12832   ins_cost(1.9 * INSN_COST);
12833   format %{ "addw  $dst, $src1, $src2, sxtb #lshift2" %}
12834 
12835    ins_encode %{
12836      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12837             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12838    %}
12839   ins_pipe(ialu_reg_reg_shift);
12840 %}
12841 
12842 // This pattern is automatically generated from aarch64_ad.m4
12843 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12844 instruct AddExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
12845 %{
12846   match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12847   ins_cost(1.9 * INSN_COST);
12848   format %{ "addw  $dst, $src1, $src2, sxth #lshift2" %}
12849 
12850    ins_encode %{
12851      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12852             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12853    %}
12854   ins_pipe(ialu_reg_reg_shift);
12855 %}
12856 
12857 // This pattern is automatically generated from aarch64_ad.m4
12858 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12859 instruct SubExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
12860 %{
12861   match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12862   ins_cost(1.9 * INSN_COST);
12863   format %{ "subw  $dst, $src1, $src2, sxtb #lshift2" %}
12864 
12865    ins_encode %{
12866      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12867             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12868    %}
12869   ins_pipe(ialu_reg_reg_shift);
12870 %}
12871 
12872 // This pattern is automatically generated from aarch64_ad.m4
12873 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12874 instruct SubExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
12875 %{
12876   match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12877   ins_cost(1.9 * INSN_COST);
12878   format %{ "subw  $dst, $src1, $src2, sxth #lshift2" %}
12879 
12880    ins_encode %{
12881      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12882             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12883    %}
12884   ins_pipe(ialu_reg_reg_shift);
12885 %}
12886 
12887 // This pattern is automatically generated from aarch64_ad.m4
12888 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12889 instruct AddExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
12890 %{
12891   match(Set dst (AddL src1 (LShiftL (ConvI2L src2) lshift)));
12892   ins_cost(1.9 * INSN_COST);
12893   format %{ "add  $dst, $src1, $src2, sxtw #lshift" %}
12894 
12895    ins_encode %{
12896      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12897             as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
12898    %}
12899   ins_pipe(ialu_reg_reg_shift);
12900 %}
12901 
12902 // This pattern is automatically generated from aarch64_ad.m4
12903 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12904 instruct SubExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
12905 %{
12906   match(Set dst (SubL src1 (LShiftL (ConvI2L src2) lshift)));
12907   ins_cost(1.9 * INSN_COST);
12908   format %{ "sub  $dst, $src1, $src2, sxtw #lshift" %}
12909 
12910    ins_encode %{
12911      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12912             as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
12913    %}
12914   ins_pipe(ialu_reg_reg_shift);
12915 %}
12916 
12917 // This pattern is automatically generated from aarch64_ad.m4
12918 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12919 instruct AddExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
12920 %{
12921   match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
12922   ins_cost(1.9 * INSN_COST);
12923   format %{ "add  $dst, $src1, $src2, uxtb #lshift" %}
12924 
12925    ins_encode %{
12926      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12927             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12928    %}
12929   ins_pipe(ialu_reg_reg_shift);
12930 %}
12931 
12932 // This pattern is automatically generated from aarch64_ad.m4
12933 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12934 instruct AddExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
12935 %{
12936   match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
12937   ins_cost(1.9 * INSN_COST);
12938   format %{ "add  $dst, $src1, $src2, uxth #lshift" %}
12939 
12940    ins_encode %{
12941      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12942             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12943    %}
12944   ins_pipe(ialu_reg_reg_shift);
12945 %}
12946 
12947 // This pattern is automatically generated from aarch64_ad.m4
12948 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12949 instruct AddExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
12950 %{
12951   match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
12952   ins_cost(1.9 * INSN_COST);
12953   format %{ "add  $dst, $src1, $src2, uxtw #lshift" %}
12954 
12955    ins_encode %{
12956      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12957             as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
12958    %}
12959   ins_pipe(ialu_reg_reg_shift);
12960 %}
12961 
12962 // This pattern is automatically generated from aarch64_ad.m4
12963 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12964 instruct SubExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
12965 %{
12966   match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
12967   ins_cost(1.9 * INSN_COST);
12968   format %{ "sub  $dst, $src1, $src2, uxtb #lshift" %}
12969 
12970    ins_encode %{
12971      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12972             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12973    %}
12974   ins_pipe(ialu_reg_reg_shift);
12975 %}
12976 
12977 // This pattern is automatically generated from aarch64_ad.m4
12978 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12979 instruct SubExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
12980 %{
12981   match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
12982   ins_cost(1.9 * INSN_COST);
12983   format %{ "sub  $dst, $src1, $src2, uxth #lshift" %}
12984 
12985    ins_encode %{
12986      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12987             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12988    %}
12989   ins_pipe(ialu_reg_reg_shift);
12990 %}
12991 
12992 // This pattern is automatically generated from aarch64_ad.m4
12993 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12994 instruct SubExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
12995 %{
12996   match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
12997   ins_cost(1.9 * INSN_COST);
12998   format %{ "sub  $dst, $src1, $src2, uxtw #lshift" %}
12999 
13000    ins_encode %{
13001      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13002             as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13003    %}
13004   ins_pipe(ialu_reg_reg_shift);
13005 %}
13006 
13007 // This pattern is automatically generated from aarch64_ad.m4
13008 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13009 instruct AddExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13010 %{
13011   match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13012   ins_cost(1.9 * INSN_COST);
13013   format %{ "addw  $dst, $src1, $src2, uxtb #lshift" %}
13014 
13015    ins_encode %{
13016      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13017             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13018    %}
13019   ins_pipe(ialu_reg_reg_shift);
13020 %}
13021 
13022 // This pattern is automatically generated from aarch64_ad.m4
13023 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13024 instruct AddExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13025 %{
13026   match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13027   ins_cost(1.9 * INSN_COST);
13028   format %{ "addw  $dst, $src1, $src2, uxth #lshift" %}
13029 
13030    ins_encode %{
13031      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13032             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13033    %}
13034   ins_pipe(ialu_reg_reg_shift);
13035 %}
13036 
13037 // This pattern is automatically generated from aarch64_ad.m4
13038 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13039 instruct SubExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13040 %{
13041   match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13042   ins_cost(1.9 * INSN_COST);
13043   format %{ "subw  $dst, $src1, $src2, uxtb #lshift" %}
13044 
13045    ins_encode %{
13046      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13047             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13048    %}
13049   ins_pipe(ialu_reg_reg_shift);
13050 %}
13051 
13052 // This pattern is automatically generated from aarch64_ad.m4
13053 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13054 instruct SubExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13055 %{
13056   match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13057   ins_cost(1.9 * INSN_COST);
13058   format %{ "subw  $dst, $src1, $src2, uxth #lshift" %}
13059 
13060    ins_encode %{
13061      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13062             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13063    %}
13064   ins_pipe(ialu_reg_reg_shift);
13065 %}
13066 
13067 // This pattern is automatically generated from aarch64_ad.m4
13068 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13069 instruct cmovI_reg_reg_lt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
13070 %{
13071   effect(DEF dst, USE src1, USE src2, USE cr);
13072   ins_cost(INSN_COST * 2);
13073   format %{ "cselw $dst, $src1, $src2 lt\t"  %}
13074 
13075   ins_encode %{
13076     __ cselw($dst$$Register,
13077              $src1$$Register,
13078              $src2$$Register,
13079              Assembler::LT);
13080   %}
13081   ins_pipe(icond_reg_reg);
13082 %}
13083 
13084 // This pattern is automatically generated from aarch64_ad.m4
13085 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13086 instruct cmovI_reg_reg_gt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
13087 %{
13088   effect(DEF dst, USE src1, USE src2, USE cr);
13089   ins_cost(INSN_COST * 2);
13090   format %{ "cselw $dst, $src1, $src2 gt\t"  %}
13091 
13092   ins_encode %{
13093     __ cselw($dst$$Register,
13094              $src1$$Register,
13095              $src2$$Register,
13096              Assembler::GT);
13097   %}
13098   ins_pipe(icond_reg_reg);
13099 %}
13100 
13101 // This pattern is automatically generated from aarch64_ad.m4
13102 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13103 instruct cmovI_reg_imm0_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13104 %{
13105   effect(DEF dst, USE src1, USE cr);
13106   ins_cost(INSN_COST * 2);
13107   format %{ "cselw $dst, $src1, zr lt\t"  %}
13108 
13109   ins_encode %{
13110     __ cselw($dst$$Register,
13111              $src1$$Register,
13112              zr,
13113              Assembler::LT);
13114   %}
13115   ins_pipe(icond_reg);
13116 %}
13117 
13118 // This pattern is automatically generated from aarch64_ad.m4
13119 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13120 instruct cmovI_reg_imm0_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13121 %{
13122   effect(DEF dst, USE src1, USE cr);
13123   ins_cost(INSN_COST * 2);
13124   format %{ "cselw $dst, $src1, zr gt\t"  %}
13125 
13126   ins_encode %{
13127     __ cselw($dst$$Register,
13128              $src1$$Register,
13129              zr,
13130              Assembler::GT);
13131   %}
13132   ins_pipe(icond_reg);
13133 %}
13134 
13135 // This pattern is automatically generated from aarch64_ad.m4
13136 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13137 instruct cmovI_reg_imm1_le(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13138 %{
13139   effect(DEF dst, USE src1, USE cr);
13140   ins_cost(INSN_COST * 2);
13141   format %{ "csincw $dst, $src1, zr le\t"  %}
13142 
13143   ins_encode %{
13144     __ csincw($dst$$Register,
13145              $src1$$Register,
13146              zr,
13147              Assembler::LE);
13148   %}
13149   ins_pipe(icond_reg);
13150 %}
13151 
13152 // This pattern is automatically generated from aarch64_ad.m4
13153 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13154 instruct cmovI_reg_imm1_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13155 %{
13156   effect(DEF dst, USE src1, USE cr);
13157   ins_cost(INSN_COST * 2);
13158   format %{ "csincw $dst, $src1, zr gt\t"  %}
13159 
13160   ins_encode %{
13161     __ csincw($dst$$Register,
13162              $src1$$Register,
13163              zr,
13164              Assembler::GT);
13165   %}
13166   ins_pipe(icond_reg);
13167 %}
13168 
13169 // This pattern is automatically generated from aarch64_ad.m4
13170 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13171 instruct cmovI_reg_immM1_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13172 %{
13173   effect(DEF dst, USE src1, USE cr);
13174   ins_cost(INSN_COST * 2);
13175   format %{ "csinvw $dst, $src1, zr lt\t"  %}
13176 
13177   ins_encode %{
13178     __ csinvw($dst$$Register,
13179              $src1$$Register,
13180              zr,
13181              Assembler::LT);
13182   %}
13183   ins_pipe(icond_reg);
13184 %}
13185 
13186 // This pattern is automatically generated from aarch64_ad.m4
13187 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13188 instruct cmovI_reg_immM1_ge(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13189 %{
13190   effect(DEF dst, USE src1, USE cr);
13191   ins_cost(INSN_COST * 2);
13192   format %{ "csinvw $dst, $src1, zr ge\t"  %}
13193 
13194   ins_encode %{
13195     __ csinvw($dst$$Register,
13196              $src1$$Register,
13197              zr,
13198              Assembler::GE);
13199   %}
13200   ins_pipe(icond_reg);
13201 %}
13202 
13203 // This pattern is automatically generated from aarch64_ad.m4
13204 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13205 instruct minI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
13206 %{
13207   match(Set dst (MinI src imm));
13208   ins_cost(INSN_COST * 3);
13209   expand %{
13210     rFlagsReg cr;
13211     compI_reg_imm0(cr, src);
13212     cmovI_reg_imm0_lt(dst, src, cr);
13213   %}
13214 %}
13215 
13216 // This pattern is automatically generated from aarch64_ad.m4
13217 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13218 instruct minI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
13219 %{
13220   match(Set dst (MinI imm src));
13221   ins_cost(INSN_COST * 3);
13222   expand %{
13223     rFlagsReg cr;
13224     compI_reg_imm0(cr, src);
13225     cmovI_reg_imm0_lt(dst, src, cr);
13226   %}
13227 %}
13228 
13229 // This pattern is automatically generated from aarch64_ad.m4
13230 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13231 instruct minI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
13232 %{
13233   match(Set dst (MinI src imm));
13234   ins_cost(INSN_COST * 3);
13235   expand %{
13236     rFlagsReg cr;
13237     compI_reg_imm0(cr, src);
13238     cmovI_reg_imm1_le(dst, src, cr);
13239   %}
13240 %}
13241 
13242 // This pattern is automatically generated from aarch64_ad.m4
13243 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13244 instruct minI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
13245 %{
13246   match(Set dst (MinI imm src));
13247   ins_cost(INSN_COST * 3);
13248   expand %{
13249     rFlagsReg cr;
13250     compI_reg_imm0(cr, src);
13251     cmovI_reg_imm1_le(dst, src, cr);
13252   %}
13253 %}
13254 
13255 // This pattern is automatically generated from aarch64_ad.m4
13256 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13257 instruct minI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
13258 %{
13259   match(Set dst (MinI src imm));
13260   ins_cost(INSN_COST * 3);
13261   expand %{
13262     rFlagsReg cr;
13263     compI_reg_imm0(cr, src);
13264     cmovI_reg_immM1_lt(dst, src, cr);
13265   %}
13266 %}
13267 
13268 // This pattern is automatically generated from aarch64_ad.m4
13269 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13270 instruct minI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
13271 %{
13272   match(Set dst (MinI imm src));
13273   ins_cost(INSN_COST * 3);
13274   expand %{
13275     rFlagsReg cr;
13276     compI_reg_imm0(cr, src);
13277     cmovI_reg_immM1_lt(dst, src, cr);
13278   %}
13279 %}
13280 
13281 // This pattern is automatically generated from aarch64_ad.m4
13282 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13283 instruct maxI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
13284 %{
13285   match(Set dst (MaxI src imm));
13286   ins_cost(INSN_COST * 3);
13287   expand %{
13288     rFlagsReg cr;
13289     compI_reg_imm0(cr, src);
13290     cmovI_reg_imm0_gt(dst, src, cr);
13291   %}
13292 %}
13293 
13294 // This pattern is automatically generated from aarch64_ad.m4
13295 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13296 instruct maxI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
13297 %{
13298   match(Set dst (MaxI imm src));
13299   ins_cost(INSN_COST * 3);
13300   expand %{
13301     rFlagsReg cr;
13302     compI_reg_imm0(cr, src);
13303     cmovI_reg_imm0_gt(dst, src, cr);
13304   %}
13305 %}
13306 
13307 // This pattern is automatically generated from aarch64_ad.m4
13308 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13309 instruct maxI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
13310 %{
13311   match(Set dst (MaxI src imm));
13312   ins_cost(INSN_COST * 3);
13313   expand %{
13314     rFlagsReg cr;
13315     compI_reg_imm0(cr, src);
13316     cmovI_reg_imm1_gt(dst, src, cr);
13317   %}
13318 %}
13319 
13320 // This pattern is automatically generated from aarch64_ad.m4
13321 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13322 instruct maxI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
13323 %{
13324   match(Set dst (MaxI imm src));
13325   ins_cost(INSN_COST * 3);
13326   expand %{
13327     rFlagsReg cr;
13328     compI_reg_imm0(cr, src);
13329     cmovI_reg_imm1_gt(dst, src, cr);
13330   %}
13331 %}
13332 
13333 // This pattern is automatically generated from aarch64_ad.m4
13334 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13335 instruct maxI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
13336 %{
13337   match(Set dst (MaxI src imm));
13338   ins_cost(INSN_COST * 3);
13339   expand %{
13340     rFlagsReg cr;
13341     compI_reg_imm0(cr, src);
13342     cmovI_reg_immM1_ge(dst, src, cr);
13343   %}
13344 %}
13345 
13346 // This pattern is automatically generated from aarch64_ad.m4
13347 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13348 instruct maxI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
13349 %{
13350   match(Set dst (MaxI imm src));
13351   ins_cost(INSN_COST * 3);
13352   expand %{
13353     rFlagsReg cr;
13354     compI_reg_imm0(cr, src);
13355     cmovI_reg_immM1_ge(dst, src, cr);
13356   %}
13357 %}
13358 
13359 // This pattern is automatically generated from aarch64_ad.m4
13360 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13361 instruct bits_reverse_I(iRegINoSp dst, iRegIorL2I src)
13362 %{
13363   match(Set dst (ReverseI src));
13364   ins_cost(INSN_COST);
13365   format %{ "rbitw  $dst, $src" %}
13366   ins_encode %{
13367     __ rbitw($dst$$Register, $src$$Register);
13368   %}
13369   ins_pipe(ialu_reg);
13370 %}
13371 
13372 // This pattern is automatically generated from aarch64_ad.m4
13373 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13374 instruct bits_reverse_L(iRegLNoSp dst, iRegL src)
13375 %{
13376   match(Set dst (ReverseL src));
13377   ins_cost(INSN_COST);
13378   format %{ "rbit  $dst, $src" %}
13379   ins_encode %{
13380     __ rbit($dst$$Register, $src$$Register);
13381   %}
13382   ins_pipe(ialu_reg);
13383 %}
13384 
13385 
13386 // END This section of the file is automatically generated. Do not edit --------------
13387 
13388 
13389 // ============================================================================
13390 // Floating Point Arithmetic Instructions
13391 
13392 instruct addF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13393   match(Set dst (AddF src1 src2));
13394 
13395   ins_cost(INSN_COST * 5);
13396   format %{ "fadds   $dst, $src1, $src2" %}
13397 
13398   ins_encode %{
13399     __ fadds(as_FloatRegister($dst$$reg),
13400              as_FloatRegister($src1$$reg),
13401              as_FloatRegister($src2$$reg));
13402   %}
13403 
13404   ins_pipe(fp_dop_reg_reg_s);
13405 %}
13406 
13407 instruct addD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13408   match(Set dst (AddD src1 src2));
13409 
13410   ins_cost(INSN_COST * 5);
13411   format %{ "faddd   $dst, $src1, $src2" %}
13412 
13413   ins_encode %{
13414     __ faddd(as_FloatRegister($dst$$reg),
13415              as_FloatRegister($src1$$reg),
13416              as_FloatRegister($src2$$reg));
13417   %}
13418 
13419   ins_pipe(fp_dop_reg_reg_d);
13420 %}
13421 
13422 instruct subF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13423   match(Set dst (SubF src1 src2));
13424 
13425   ins_cost(INSN_COST * 5);
13426   format %{ "fsubs   $dst, $src1, $src2" %}
13427 
13428   ins_encode %{
13429     __ fsubs(as_FloatRegister($dst$$reg),
13430              as_FloatRegister($src1$$reg),
13431              as_FloatRegister($src2$$reg));
13432   %}
13433 
13434   ins_pipe(fp_dop_reg_reg_s);
13435 %}
13436 
13437 instruct subD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13438   match(Set dst (SubD src1 src2));
13439 
13440   ins_cost(INSN_COST * 5);
13441   format %{ "fsubd   $dst, $src1, $src2" %}
13442 
13443   ins_encode %{
13444     __ fsubd(as_FloatRegister($dst$$reg),
13445              as_FloatRegister($src1$$reg),
13446              as_FloatRegister($src2$$reg));
13447   %}
13448 
13449   ins_pipe(fp_dop_reg_reg_d);
13450 %}
13451 
13452 instruct mulF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13453   match(Set dst (MulF src1 src2));
13454 
13455   ins_cost(INSN_COST * 6);
13456   format %{ "fmuls   $dst, $src1, $src2" %}
13457 
13458   ins_encode %{
13459     __ fmuls(as_FloatRegister($dst$$reg),
13460              as_FloatRegister($src1$$reg),
13461              as_FloatRegister($src2$$reg));
13462   %}
13463 
13464   ins_pipe(fp_dop_reg_reg_s);
13465 %}
13466 
13467 instruct mulD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13468   match(Set dst (MulD src1 src2));
13469 
13470   ins_cost(INSN_COST * 6);
13471   format %{ "fmuld   $dst, $src1, $src2" %}
13472 
13473   ins_encode %{
13474     __ fmuld(as_FloatRegister($dst$$reg),
13475              as_FloatRegister($src1$$reg),
13476              as_FloatRegister($src2$$reg));
13477   %}
13478 
13479   ins_pipe(fp_dop_reg_reg_d);
13480 %}
13481 
13482 // src1 * src2 + src3
13483 instruct maddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13484   match(Set dst (FmaF src3 (Binary src1 src2)));
13485 
13486   format %{ "fmadds   $dst, $src1, $src2, $src3" %}
13487 
13488   ins_encode %{
13489     assert(UseFMA, "Needs FMA instructions support.");
13490     __ fmadds(as_FloatRegister($dst$$reg),
13491              as_FloatRegister($src1$$reg),
13492              as_FloatRegister($src2$$reg),
13493              as_FloatRegister($src3$$reg));
13494   %}
13495 
13496   ins_pipe(pipe_class_default);
13497 %}
13498 
13499 // src1 * src2 + src3
13500 instruct maddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13501   match(Set dst (FmaD src3 (Binary src1 src2)));
13502 
13503   format %{ "fmaddd   $dst, $src1, $src2, $src3" %}
13504 
13505   ins_encode %{
13506     assert(UseFMA, "Needs FMA instructions support.");
13507     __ fmaddd(as_FloatRegister($dst$$reg),
13508              as_FloatRegister($src1$$reg),
13509              as_FloatRegister($src2$$reg),
13510              as_FloatRegister($src3$$reg));
13511   %}
13512 
13513   ins_pipe(pipe_class_default);
13514 %}
13515 
13516 // src1 * (-src2) + src3
13517 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
13518 instruct msubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13519   match(Set dst (FmaF src3 (Binary src1 (NegF src2))));
13520 
13521   format %{ "fmsubs   $dst, $src1, $src2, $src3" %}
13522 
13523   ins_encode %{
13524     assert(UseFMA, "Needs FMA instructions support.");
13525     __ fmsubs(as_FloatRegister($dst$$reg),
13526               as_FloatRegister($src1$$reg),
13527               as_FloatRegister($src2$$reg),
13528               as_FloatRegister($src3$$reg));
13529   %}
13530 
13531   ins_pipe(pipe_class_default);
13532 %}
13533 
13534 // src1 * (-src2) + src3
13535 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
13536 instruct msubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13537   match(Set dst (FmaD src3 (Binary src1 (NegD src2))));
13538 
13539   format %{ "fmsubd   $dst, $src1, $src2, $src3" %}
13540 
13541   ins_encode %{
13542     assert(UseFMA, "Needs FMA instructions support.");
13543     __ fmsubd(as_FloatRegister($dst$$reg),
13544               as_FloatRegister($src1$$reg),
13545               as_FloatRegister($src2$$reg),
13546               as_FloatRegister($src3$$reg));
13547   %}
13548 
13549   ins_pipe(pipe_class_default);
13550 %}
13551 
13552 // src1 * (-src2) - src3
13553 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
13554 instruct mnaddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13555   match(Set dst (FmaF (NegF src3) (Binary src1 (NegF src2))));
13556 
13557   format %{ "fnmadds  $dst, $src1, $src2, $src3" %}
13558 
13559   ins_encode %{
13560     assert(UseFMA, "Needs FMA instructions support.");
13561     __ fnmadds(as_FloatRegister($dst$$reg),
13562                as_FloatRegister($src1$$reg),
13563                as_FloatRegister($src2$$reg),
13564                as_FloatRegister($src3$$reg));
13565   %}
13566 
13567   ins_pipe(pipe_class_default);
13568 %}
13569 
13570 // src1 * (-src2) - src3
13571 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
13572 instruct mnaddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13573   match(Set dst (FmaD (NegD src3) (Binary src1 (NegD src2))));
13574 
13575   format %{ "fnmaddd   $dst, $src1, $src2, $src3" %}
13576 
13577   ins_encode %{
13578     assert(UseFMA, "Needs FMA instructions support.");
13579     __ fnmaddd(as_FloatRegister($dst$$reg),
13580                as_FloatRegister($src1$$reg),
13581                as_FloatRegister($src2$$reg),
13582                as_FloatRegister($src3$$reg));
13583   %}
13584 
13585   ins_pipe(pipe_class_default);
13586 %}
13587 
13588 // src1 * src2 - src3
13589 instruct mnsubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3, immF0 zero) %{
13590   match(Set dst (FmaF (NegF src3) (Binary src1 src2)));
13591 
13592   format %{ "fnmsubs  $dst, $src1, $src2, $src3" %}
13593 
13594   ins_encode %{
13595     assert(UseFMA, "Needs FMA instructions support.");
13596     __ fnmsubs(as_FloatRegister($dst$$reg),
13597                as_FloatRegister($src1$$reg),
13598                as_FloatRegister($src2$$reg),
13599                as_FloatRegister($src3$$reg));
13600   %}
13601 
13602   ins_pipe(pipe_class_default);
13603 %}
13604 
13605 // src1 * src2 - src3
13606 instruct mnsubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3, immD0 zero) %{
13607   match(Set dst (FmaD (NegD src3) (Binary src1 src2)));
13608 
13609   format %{ "fnmsubd   $dst, $src1, $src2, $src3" %}
13610 
13611   ins_encode %{
13612     assert(UseFMA, "Needs FMA instructions support.");
13613     // n.b. insn name should be fnmsubd
13614     __ fnmsub(as_FloatRegister($dst$$reg),
13615               as_FloatRegister($src1$$reg),
13616               as_FloatRegister($src2$$reg),
13617               as_FloatRegister($src3$$reg));
13618   %}
13619 
13620   ins_pipe(pipe_class_default);
13621 %}
13622 
13623 
13624 // Math.max(FF)F
13625 instruct maxF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13626   match(Set dst (MaxF src1 src2));
13627 
13628   format %{ "fmaxs   $dst, $src1, $src2" %}
13629   ins_encode %{
13630     __ fmaxs(as_FloatRegister($dst$$reg),
13631              as_FloatRegister($src1$$reg),
13632              as_FloatRegister($src2$$reg));
13633   %}
13634 
13635   ins_pipe(fp_dop_reg_reg_s);
13636 %}
13637 
13638 // Math.min(FF)F
13639 instruct minF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13640   match(Set dst (MinF src1 src2));
13641 
13642   format %{ "fmins   $dst, $src1, $src2" %}
13643   ins_encode %{
13644     __ fmins(as_FloatRegister($dst$$reg),
13645              as_FloatRegister($src1$$reg),
13646              as_FloatRegister($src2$$reg));
13647   %}
13648 
13649   ins_pipe(fp_dop_reg_reg_s);
13650 %}
13651 
13652 // Math.max(DD)D
13653 instruct maxD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13654   match(Set dst (MaxD src1 src2));
13655 
13656   format %{ "fmaxd   $dst, $src1, $src2" %}
13657   ins_encode %{
13658     __ fmaxd(as_FloatRegister($dst$$reg),
13659              as_FloatRegister($src1$$reg),
13660              as_FloatRegister($src2$$reg));
13661   %}
13662 
13663   ins_pipe(fp_dop_reg_reg_d);
13664 %}
13665 
13666 // Math.min(DD)D
13667 instruct minD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13668   match(Set dst (MinD src1 src2));
13669 
13670   format %{ "fmind   $dst, $src1, $src2" %}
13671   ins_encode %{
13672     __ fmind(as_FloatRegister($dst$$reg),
13673              as_FloatRegister($src1$$reg),
13674              as_FloatRegister($src2$$reg));
13675   %}
13676 
13677   ins_pipe(fp_dop_reg_reg_d);
13678 %}
13679 
13680 
13681 instruct divF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13682   match(Set dst (DivF src1  src2));
13683 
13684   ins_cost(INSN_COST * 18);
13685   format %{ "fdivs   $dst, $src1, $src2" %}
13686 
13687   ins_encode %{
13688     __ fdivs(as_FloatRegister($dst$$reg),
13689              as_FloatRegister($src1$$reg),
13690              as_FloatRegister($src2$$reg));
13691   %}
13692 
13693   ins_pipe(fp_div_s);
13694 %}
13695 
13696 instruct divD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13697   match(Set dst (DivD src1  src2));
13698 
13699   ins_cost(INSN_COST * 32);
13700   format %{ "fdivd   $dst, $src1, $src2" %}
13701 
13702   ins_encode %{
13703     __ fdivd(as_FloatRegister($dst$$reg),
13704              as_FloatRegister($src1$$reg),
13705              as_FloatRegister($src2$$reg));
13706   %}
13707 
13708   ins_pipe(fp_div_d);
13709 %}
13710 
13711 instruct negF_reg_reg(vRegF dst, vRegF src) %{
13712   match(Set dst (NegF src));
13713 
13714   ins_cost(INSN_COST * 3);
13715   format %{ "fneg   $dst, $src" %}
13716 
13717   ins_encode %{
13718     __ fnegs(as_FloatRegister($dst$$reg),
13719              as_FloatRegister($src$$reg));
13720   %}
13721 
13722   ins_pipe(fp_uop_s);
13723 %}
13724 
13725 instruct negD_reg_reg(vRegD dst, vRegD src) %{
13726   match(Set dst (NegD src));
13727 
13728   ins_cost(INSN_COST * 3);
13729   format %{ "fnegd   $dst, $src" %}
13730 
13731   ins_encode %{
13732     __ fnegd(as_FloatRegister($dst$$reg),
13733              as_FloatRegister($src$$reg));
13734   %}
13735 
13736   ins_pipe(fp_uop_d);
13737 %}
13738 
13739 instruct absI_reg(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
13740 %{
13741   match(Set dst (AbsI src));
13742 
13743   effect(KILL cr);
13744   ins_cost(INSN_COST * 2);
13745   format %{ "cmpw  $src, zr\n\t"
13746             "cnegw $dst, $src, Assembler::LT\t# int abs"
13747   %}
13748 
13749   ins_encode %{
13750     __ cmpw(as_Register($src$$reg), zr);
13751     __ cnegw(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
13752   %}
13753   ins_pipe(pipe_class_default);
13754 %}
13755 
13756 instruct absL_reg(iRegLNoSp dst, iRegL src, rFlagsReg cr)
13757 %{
13758   match(Set dst (AbsL src));
13759 
13760   effect(KILL cr);
13761   ins_cost(INSN_COST * 2);
13762   format %{ "cmp  $src, zr\n\t"
13763             "cneg $dst, $src, Assembler::LT\t# long abs"
13764   %}
13765 
13766   ins_encode %{
13767     __ cmp(as_Register($src$$reg), zr);
13768     __ cneg(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
13769   %}
13770   ins_pipe(pipe_class_default);
13771 %}
13772 
13773 instruct absF_reg(vRegF dst, vRegF src) %{
13774   match(Set dst (AbsF src));
13775 
13776   ins_cost(INSN_COST * 3);
13777   format %{ "fabss   $dst, $src" %}
13778   ins_encode %{
13779     __ fabss(as_FloatRegister($dst$$reg),
13780              as_FloatRegister($src$$reg));
13781   %}
13782 
13783   ins_pipe(fp_uop_s);
13784 %}
13785 
13786 instruct absD_reg(vRegD dst, vRegD src) %{
13787   match(Set dst (AbsD src));
13788 
13789   ins_cost(INSN_COST * 3);
13790   format %{ "fabsd   $dst, $src" %}
13791   ins_encode %{
13792     __ fabsd(as_FloatRegister($dst$$reg),
13793              as_FloatRegister($src$$reg));
13794   %}
13795 
13796   ins_pipe(fp_uop_d);
13797 %}
13798 
13799 instruct absdF_reg(vRegF dst, vRegF src1, vRegF src2) %{
13800   match(Set dst (AbsF (SubF src1 src2)));
13801 
13802   ins_cost(INSN_COST * 3);
13803   format %{ "fabds   $dst, $src1, $src2" %}
13804   ins_encode %{
13805     __ fabds(as_FloatRegister($dst$$reg),
13806              as_FloatRegister($src1$$reg),
13807              as_FloatRegister($src2$$reg));
13808   %}
13809 
13810   ins_pipe(fp_uop_s);
13811 %}
13812 
13813 instruct absdD_reg(vRegD dst, vRegD src1, vRegD src2) %{
13814   match(Set dst (AbsD (SubD src1 src2)));
13815 
13816   ins_cost(INSN_COST * 3);
13817   format %{ "fabdd   $dst, $src1, $src2" %}
13818   ins_encode %{
13819     __ fabdd(as_FloatRegister($dst$$reg),
13820              as_FloatRegister($src1$$reg),
13821              as_FloatRegister($src2$$reg));
13822   %}
13823 
13824   ins_pipe(fp_uop_d);
13825 %}
13826 
13827 instruct sqrtD_reg(vRegD dst, vRegD src) %{
13828   match(Set dst (SqrtD src));
13829 
13830   ins_cost(INSN_COST * 50);
13831   format %{ "fsqrtd  $dst, $src" %}
13832   ins_encode %{
13833     __ fsqrtd(as_FloatRegister($dst$$reg),
13834              as_FloatRegister($src$$reg));
13835   %}
13836 
13837   ins_pipe(fp_div_s);
13838 %}
13839 
13840 instruct sqrtF_reg(vRegF dst, vRegF src) %{
13841   match(Set dst (SqrtF src));
13842 
13843   ins_cost(INSN_COST * 50);
13844   format %{ "fsqrts  $dst, $src" %}
13845   ins_encode %{
13846     __ fsqrts(as_FloatRegister($dst$$reg),
13847              as_FloatRegister($src$$reg));
13848   %}
13849 
13850   ins_pipe(fp_div_d);
13851 %}
13852 
13853 // Math.rint, floor, ceil
13854 instruct roundD_reg(vRegD dst, vRegD src, immI rmode) %{
13855   match(Set dst (RoundDoubleMode src rmode));
13856   format %{ "frint  $dst, $src, $rmode" %}
13857   ins_encode %{
13858     switch ($rmode$$constant) {
13859       case RoundDoubleModeNode::rmode_rint:
13860         __ frintnd(as_FloatRegister($dst$$reg),
13861                    as_FloatRegister($src$$reg));
13862         break;
13863       case RoundDoubleModeNode::rmode_floor:
13864         __ frintmd(as_FloatRegister($dst$$reg),
13865                    as_FloatRegister($src$$reg));
13866         break;
13867       case RoundDoubleModeNode::rmode_ceil:
13868         __ frintpd(as_FloatRegister($dst$$reg),
13869                    as_FloatRegister($src$$reg));
13870         break;
13871     }
13872   %}
13873   ins_pipe(fp_uop_d);
13874 %}
13875 
13876 instruct copySignD_reg(vRegD dst, vRegD src1, vRegD src2, vRegD zero) %{
13877   match(Set dst (CopySignD src1 (Binary src2 zero)));
13878   effect(TEMP_DEF dst, USE src1, USE src2, USE zero);
13879   format %{ "CopySignD  $dst $src1 $src2" %}
13880   ins_encode %{
13881     FloatRegister dst = as_FloatRegister($dst$$reg),
13882                   src1 = as_FloatRegister($src1$$reg),
13883                   src2 = as_FloatRegister($src2$$reg),
13884                   zero = as_FloatRegister($zero$$reg);
13885     __ fnegd(dst, zero);
13886     __ bsl(dst, __ T8B, src2, src1);
13887   %}
13888   ins_pipe(fp_uop_d);
13889 %}
13890 
13891 instruct copySignF_reg(vRegF dst, vRegF src1, vRegF src2) %{
13892   match(Set dst (CopySignF src1 src2));
13893   effect(TEMP_DEF dst, USE src1, USE src2);
13894   format %{ "CopySignF  $dst $src1 $src2" %}
13895   ins_encode %{
13896     FloatRegister dst = as_FloatRegister($dst$$reg),
13897                   src1 = as_FloatRegister($src1$$reg),
13898                   src2 = as_FloatRegister($src2$$reg);
13899     __ movi(dst, __ T2S, 0x80, 24);
13900     __ bsl(dst, __ T8B, src2, src1);
13901   %}
13902   ins_pipe(fp_uop_d);
13903 %}
13904 
13905 instruct signumD_reg(vRegD dst, vRegD src, vRegD zero, vRegD one) %{
13906   match(Set dst (SignumD src (Binary zero one)));
13907   effect(TEMP_DEF dst, USE src, USE zero, USE one);
13908   format %{ "signumD  $dst, $src" %}
13909   ins_encode %{
13910     FloatRegister src = as_FloatRegister($src$$reg),
13911                   dst = as_FloatRegister($dst$$reg),
13912                   zero = as_FloatRegister($zero$$reg),
13913                   one = as_FloatRegister($one$$reg);
13914     __ facgtd(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
13915     __ ushrd(dst, dst, 1);     // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
13916     // Bit selection instruction gets bit from "one" for each enabled bit in
13917     // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
13918     // NaN the whole "src" will be copied because "dst" is zero. For all other
13919     // "src" values dst is 0x7FF..F, which means only the sign bit is copied
13920     // from "src", and all other bits are copied from 1.0.
13921     __ bsl(dst, __ T8B, one, src);
13922   %}
13923   ins_pipe(fp_uop_d);
13924 %}
13925 
13926 instruct signumF_reg(vRegF dst, vRegF src, vRegF zero, vRegF one) %{
13927   match(Set dst (SignumF src (Binary zero one)));
13928   effect(TEMP_DEF dst, USE src, USE zero, USE one);
13929   format %{ "signumF  $dst, $src" %}
13930   ins_encode %{
13931     FloatRegister src = as_FloatRegister($src$$reg),
13932                   dst = as_FloatRegister($dst$$reg),
13933                   zero = as_FloatRegister($zero$$reg),
13934                   one = as_FloatRegister($one$$reg);
13935     __ facgts(dst, src, zero);    // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
13936     __ ushr(dst, __ T2S, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
13937     // Bit selection instruction gets bit from "one" for each enabled bit in
13938     // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
13939     // NaN the whole "src" will be copied because "dst" is zero. For all other
13940     // "src" values dst is 0x7FF..F, which means only the sign bit is copied
13941     // from "src", and all other bits are copied from 1.0.
13942     __ bsl(dst, __ T8B, one, src);
13943   %}
13944   ins_pipe(fp_uop_d);
13945 %}
13946 
13947 instruct onspinwait() %{
13948   match(OnSpinWait);
13949   ins_cost(INSN_COST);
13950 
13951   format %{ "onspinwait" %}
13952 
13953   ins_encode %{
13954     __ spin_wait();
13955   %}
13956   ins_pipe(pipe_class_empty);
13957 %}
13958 
13959 // ============================================================================
13960 // Logical Instructions
13961 
13962 // Integer Logical Instructions
13963 
13964 // And Instructions
13965 
13966 
13967 instruct andI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, rFlagsReg cr) %{
13968   match(Set dst (AndI src1 src2));
13969 
13970   format %{ "andw  $dst, $src1, $src2\t# int" %}
13971 
13972   ins_cost(INSN_COST);
13973   ins_encode %{
13974     __ andw(as_Register($dst$$reg),
13975             as_Register($src1$$reg),
13976             as_Register($src2$$reg));
13977   %}
13978 
13979   ins_pipe(ialu_reg_reg);
13980 %}
13981 
13982 instruct andI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2, rFlagsReg cr) %{
13983   match(Set dst (AndI src1 src2));
13984 
13985   format %{ "andsw  $dst, $src1, $src2\t# int" %}
13986 
13987   ins_cost(INSN_COST);
13988   ins_encode %{
13989     __ andw(as_Register($dst$$reg),
13990             as_Register($src1$$reg),
13991             (uint64_t)($src2$$constant));
13992   %}
13993 
13994   ins_pipe(ialu_reg_imm);
13995 %}
13996 
13997 // Or Instructions
13998 
13999 instruct orI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14000   match(Set dst (OrI src1 src2));
14001 
14002   format %{ "orrw  $dst, $src1, $src2\t# int" %}
14003 
14004   ins_cost(INSN_COST);
14005   ins_encode %{
14006     __ orrw(as_Register($dst$$reg),
14007             as_Register($src1$$reg),
14008             as_Register($src2$$reg));
14009   %}
14010 
14011   ins_pipe(ialu_reg_reg);
14012 %}
14013 
14014 instruct orI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14015   match(Set dst (OrI src1 src2));
14016 
14017   format %{ "orrw  $dst, $src1, $src2\t# int" %}
14018 
14019   ins_cost(INSN_COST);
14020   ins_encode %{
14021     __ orrw(as_Register($dst$$reg),
14022             as_Register($src1$$reg),
14023             (uint64_t)($src2$$constant));
14024   %}
14025 
14026   ins_pipe(ialu_reg_imm);
14027 %}
14028 
14029 // Xor Instructions
14030 
14031 instruct xorI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14032   match(Set dst (XorI src1 src2));
14033 
14034   format %{ "eorw  $dst, $src1, $src2\t# int" %}
14035 
14036   ins_cost(INSN_COST);
14037   ins_encode %{
14038     __ eorw(as_Register($dst$$reg),
14039             as_Register($src1$$reg),
14040             as_Register($src2$$reg));
14041   %}
14042 
14043   ins_pipe(ialu_reg_reg);
14044 %}
14045 
14046 instruct xorI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14047   match(Set dst (XorI src1 src2));
14048 
14049   format %{ "eorw  $dst, $src1, $src2\t# int" %}
14050 
14051   ins_cost(INSN_COST);
14052   ins_encode %{
14053     __ eorw(as_Register($dst$$reg),
14054             as_Register($src1$$reg),
14055             (uint64_t)($src2$$constant));
14056   %}
14057 
14058   ins_pipe(ialu_reg_imm);
14059 %}
14060 
14061 // Long Logical Instructions
14062 // TODO
14063 
14064 instruct andL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr) %{
14065   match(Set dst (AndL src1 src2));
14066 
14067   format %{ "and  $dst, $src1, $src2\t# int" %}
14068 
14069   ins_cost(INSN_COST);
14070   ins_encode %{
14071     __ andr(as_Register($dst$$reg),
14072             as_Register($src1$$reg),
14073             as_Register($src2$$reg));
14074   %}
14075 
14076   ins_pipe(ialu_reg_reg);
14077 %}
14078 
14079 instruct andL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2, rFlagsReg cr) %{
14080   match(Set dst (AndL src1 src2));
14081 
14082   format %{ "and  $dst, $src1, $src2\t# int" %}
14083 
14084   ins_cost(INSN_COST);
14085   ins_encode %{
14086     __ andr(as_Register($dst$$reg),
14087             as_Register($src1$$reg),
14088             (uint64_t)($src2$$constant));
14089   %}
14090 
14091   ins_pipe(ialu_reg_imm);
14092 %}
14093 
14094 // Or Instructions
14095 
14096 instruct orL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14097   match(Set dst (OrL src1 src2));
14098 
14099   format %{ "orr  $dst, $src1, $src2\t# int" %}
14100 
14101   ins_cost(INSN_COST);
14102   ins_encode %{
14103     __ orr(as_Register($dst$$reg),
14104            as_Register($src1$$reg),
14105            as_Register($src2$$reg));
14106   %}
14107 
14108   ins_pipe(ialu_reg_reg);
14109 %}
14110 
14111 instruct orL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14112   match(Set dst (OrL src1 src2));
14113 
14114   format %{ "orr  $dst, $src1, $src2\t# int" %}
14115 
14116   ins_cost(INSN_COST);
14117   ins_encode %{
14118     __ orr(as_Register($dst$$reg),
14119            as_Register($src1$$reg),
14120            (uint64_t)($src2$$constant));
14121   %}
14122 
14123   ins_pipe(ialu_reg_imm);
14124 %}
14125 
14126 // Xor Instructions
14127 
14128 instruct xorL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14129   match(Set dst (XorL src1 src2));
14130 
14131   format %{ "eor  $dst, $src1, $src2\t# int" %}
14132 
14133   ins_cost(INSN_COST);
14134   ins_encode %{
14135     __ eor(as_Register($dst$$reg),
14136            as_Register($src1$$reg),
14137            as_Register($src2$$reg));
14138   %}
14139 
14140   ins_pipe(ialu_reg_reg);
14141 %}
14142 
14143 instruct xorL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14144   match(Set dst (XorL src1 src2));
14145 
14146   ins_cost(INSN_COST);
14147   format %{ "eor  $dst, $src1, $src2\t# int" %}
14148 
14149   ins_encode %{
14150     __ eor(as_Register($dst$$reg),
14151            as_Register($src1$$reg),
14152            (uint64_t)($src2$$constant));
14153   %}
14154 
14155   ins_pipe(ialu_reg_imm);
14156 %}
14157 
14158 instruct convI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src)
14159 %{
14160   match(Set dst (ConvI2L src));
14161 
14162   ins_cost(INSN_COST);
14163   format %{ "sxtw  $dst, $src\t# i2l" %}
14164   ins_encode %{
14165     __ sbfm($dst$$Register, $src$$Register, 0, 31);
14166   %}
14167   ins_pipe(ialu_reg_shift);
14168 %}
14169 
14170 // this pattern occurs in bigmath arithmetic
14171 instruct convUI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src, immL_32bits mask)
14172 %{
14173   match(Set dst (AndL (ConvI2L src) mask));
14174 
14175   ins_cost(INSN_COST);
14176   format %{ "ubfm  $dst, $src, 0, 31\t# ui2l" %}
14177   ins_encode %{
14178     __ ubfm($dst$$Register, $src$$Register, 0, 31);
14179   %}
14180 
14181   ins_pipe(ialu_reg_shift);
14182 %}
14183 
14184 instruct convL2I_reg(iRegINoSp dst, iRegL src) %{
14185   match(Set dst (ConvL2I src));
14186 
14187   ins_cost(INSN_COST);
14188   format %{ "movw  $dst, $src \t// l2i" %}
14189 
14190   ins_encode %{
14191     __ movw(as_Register($dst$$reg), as_Register($src$$reg));
14192   %}
14193 
14194   ins_pipe(ialu_reg);
14195 %}
14196 
14197 instruct convD2F_reg(vRegF dst, vRegD src) %{
14198   match(Set dst (ConvD2F src));
14199 
14200   ins_cost(INSN_COST * 5);
14201   format %{ "fcvtd  $dst, $src \t// d2f" %}
14202 
14203   ins_encode %{
14204     __ fcvtd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14205   %}
14206 
14207   ins_pipe(fp_d2f);
14208 %}
14209 
14210 instruct convF2D_reg(vRegD dst, vRegF src) %{
14211   match(Set dst (ConvF2D src));
14212 
14213   ins_cost(INSN_COST * 5);
14214   format %{ "fcvts  $dst, $src \t// f2d" %}
14215 
14216   ins_encode %{
14217     __ fcvts(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14218   %}
14219 
14220   ins_pipe(fp_f2d);
14221 %}
14222 
14223 instruct convF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14224   match(Set dst (ConvF2I src));
14225 
14226   ins_cost(INSN_COST * 5);
14227   format %{ "fcvtzsw  $dst, $src \t// f2i" %}
14228 
14229   ins_encode %{
14230     __ fcvtzsw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14231   %}
14232 
14233   ins_pipe(fp_f2i);
14234 %}
14235 
14236 instruct convF2L_reg_reg(iRegLNoSp dst, vRegF src) %{
14237   match(Set dst (ConvF2L src));
14238 
14239   ins_cost(INSN_COST * 5);
14240   format %{ "fcvtzs  $dst, $src \t// f2l" %}
14241 
14242   ins_encode %{
14243     __ fcvtzs(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14244   %}
14245 
14246   ins_pipe(fp_f2l);
14247 %}
14248 
14249 instruct convF2HF_reg_reg(iRegINoSp dst, vRegF src, vRegF tmp) %{
14250   match(Set dst (ConvF2HF src));
14251   format %{ "fcvt $tmp, $src\t# convert single to half precision\n\t"
14252             "smov $dst, $tmp\t# move result from $tmp to $dst"
14253   %}
14254   effect(TEMP tmp);
14255   ins_encode %{
14256       __ flt_to_flt16($dst$$Register, $src$$FloatRegister, $tmp$$FloatRegister);
14257   %}
14258   ins_pipe(pipe_slow);
14259 %}
14260 
14261 instruct convHF2F_reg_reg(vRegF dst, iRegINoSp src, vRegF tmp) %{
14262   match(Set dst (ConvHF2F src));
14263   format %{ "mov $tmp, $src\t# move source from $src to $tmp\n\t"
14264             "fcvt $dst, $tmp\t# convert half to single precision"
14265   %}
14266   effect(TEMP tmp);
14267   ins_encode %{
14268       __ flt16_to_flt($dst$$FloatRegister, $src$$Register, $tmp$$FloatRegister);
14269   %}
14270   ins_pipe(pipe_slow);
14271 %}
14272 
14273 instruct convI2F_reg_reg(vRegF dst, iRegIorL2I src) %{
14274   match(Set dst (ConvI2F src));
14275 
14276   ins_cost(INSN_COST * 5);
14277   format %{ "scvtfws  $dst, $src \t// i2f" %}
14278 
14279   ins_encode %{
14280     __ scvtfws(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14281   %}
14282 
14283   ins_pipe(fp_i2f);
14284 %}
14285 
14286 instruct convL2F_reg_reg(vRegF dst, iRegL src) %{
14287   match(Set dst (ConvL2F src));
14288 
14289   ins_cost(INSN_COST * 5);
14290   format %{ "scvtfs  $dst, $src \t// l2f" %}
14291 
14292   ins_encode %{
14293     __ scvtfs(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14294   %}
14295 
14296   ins_pipe(fp_l2f);
14297 %}
14298 
14299 instruct convD2I_reg_reg(iRegINoSp dst, vRegD src) %{
14300   match(Set dst (ConvD2I src));
14301 
14302   ins_cost(INSN_COST * 5);
14303   format %{ "fcvtzdw  $dst, $src \t// d2i" %}
14304 
14305   ins_encode %{
14306     __ fcvtzdw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14307   %}
14308 
14309   ins_pipe(fp_d2i);
14310 %}
14311 
14312 instruct convD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14313   match(Set dst (ConvD2L src));
14314 
14315   ins_cost(INSN_COST * 5);
14316   format %{ "fcvtzd  $dst, $src \t// d2l" %}
14317 
14318   ins_encode %{
14319     __ fcvtzd(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14320   %}
14321 
14322   ins_pipe(fp_d2l);
14323 %}
14324 
14325 instruct convI2D_reg_reg(vRegD dst, iRegIorL2I src) %{
14326   match(Set dst (ConvI2D src));
14327 
14328   ins_cost(INSN_COST * 5);
14329   format %{ "scvtfwd  $dst, $src \t// i2d" %}
14330 
14331   ins_encode %{
14332     __ scvtfwd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14333   %}
14334 
14335   ins_pipe(fp_i2d);
14336 %}
14337 
14338 instruct convL2D_reg_reg(vRegD dst, iRegL src) %{
14339   match(Set dst (ConvL2D src));
14340 
14341   ins_cost(INSN_COST * 5);
14342   format %{ "scvtfd  $dst, $src \t// l2d" %}
14343 
14344   ins_encode %{
14345     __ scvtfd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14346   %}
14347 
14348   ins_pipe(fp_l2d);
14349 %}
14350 
14351 instruct round_double_reg(iRegLNoSp dst, vRegD src, vRegD ftmp, rFlagsReg cr)
14352 %{
14353   match(Set dst (RoundD src));
14354   effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
14355   format %{ "java_round_double $dst,$src"%}
14356   ins_encode %{
14357     __ java_round_double($dst$$Register, as_FloatRegister($src$$reg),
14358                          as_FloatRegister($ftmp$$reg));
14359   %}
14360   ins_pipe(pipe_slow);
14361 %}
14362 
14363 instruct round_float_reg(iRegINoSp dst, vRegF src, vRegF ftmp, rFlagsReg cr)
14364 %{
14365   match(Set dst (RoundF src));
14366   effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
14367   format %{ "java_round_float $dst,$src"%}
14368   ins_encode %{
14369     __ java_round_float($dst$$Register, as_FloatRegister($src$$reg),
14370                         as_FloatRegister($ftmp$$reg));
14371   %}
14372   ins_pipe(pipe_slow);
14373 %}
14374 
14375 // stack <-> reg and reg <-> reg shuffles with no conversion
14376 
14377 instruct MoveF2I_stack_reg(iRegINoSp dst, stackSlotF src) %{
14378 
14379   match(Set dst (MoveF2I src));
14380 
14381   effect(DEF dst, USE src);
14382 
14383   ins_cost(4 * INSN_COST);
14384 
14385   format %{ "ldrw $dst, $src\t# MoveF2I_stack_reg" %}
14386 
14387   ins_encode %{
14388     __ ldrw($dst$$Register, Address(sp, $src$$disp));
14389   %}
14390 
14391   ins_pipe(iload_reg_reg);
14392 
14393 %}
14394 
14395 instruct MoveI2F_stack_reg(vRegF dst, stackSlotI src) %{
14396 
14397   match(Set dst (MoveI2F src));
14398 
14399   effect(DEF dst, USE src);
14400 
14401   ins_cost(4 * INSN_COST);
14402 
14403   format %{ "ldrs $dst, $src\t# MoveI2F_stack_reg" %}
14404 
14405   ins_encode %{
14406     __ ldrs(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
14407   %}
14408 
14409   ins_pipe(pipe_class_memory);
14410 
14411 %}
14412 
14413 instruct MoveD2L_stack_reg(iRegLNoSp dst, stackSlotD src) %{
14414 
14415   match(Set dst (MoveD2L src));
14416 
14417   effect(DEF dst, USE src);
14418 
14419   ins_cost(4 * INSN_COST);
14420 
14421   format %{ "ldr $dst, $src\t# MoveD2L_stack_reg" %}
14422 
14423   ins_encode %{
14424     __ ldr($dst$$Register, Address(sp, $src$$disp));
14425   %}
14426 
14427   ins_pipe(iload_reg_reg);
14428 
14429 %}
14430 
14431 instruct MoveL2D_stack_reg(vRegD dst, stackSlotL src) %{
14432 
14433   match(Set dst (MoveL2D src));
14434 
14435   effect(DEF dst, USE src);
14436 
14437   ins_cost(4 * INSN_COST);
14438 
14439   format %{ "ldrd $dst, $src\t# MoveL2D_stack_reg" %}
14440 
14441   ins_encode %{
14442     __ ldrd(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
14443   %}
14444 
14445   ins_pipe(pipe_class_memory);
14446 
14447 %}
14448 
14449 instruct MoveF2I_reg_stack(stackSlotI dst, vRegF src) %{
14450 
14451   match(Set dst (MoveF2I src));
14452 
14453   effect(DEF dst, USE src);
14454 
14455   ins_cost(INSN_COST);
14456 
14457   format %{ "strs $src, $dst\t# MoveF2I_reg_stack" %}
14458 
14459   ins_encode %{
14460     __ strs(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
14461   %}
14462 
14463   ins_pipe(pipe_class_memory);
14464 
14465 %}
14466 
14467 instruct MoveI2F_reg_stack(stackSlotF dst, iRegI src) %{
14468 
14469   match(Set dst (MoveI2F src));
14470 
14471   effect(DEF dst, USE src);
14472 
14473   ins_cost(INSN_COST);
14474 
14475   format %{ "strw $src, $dst\t# MoveI2F_reg_stack" %}
14476 
14477   ins_encode %{
14478     __ strw($src$$Register, Address(sp, $dst$$disp));
14479   %}
14480 
14481   ins_pipe(istore_reg_reg);
14482 
14483 %}
14484 
14485 instruct MoveD2L_reg_stack(stackSlotL dst, vRegD src) %{
14486 
14487   match(Set dst (MoveD2L src));
14488 
14489   effect(DEF dst, USE src);
14490 
14491   ins_cost(INSN_COST);
14492 
14493   format %{ "strd $dst, $src\t# MoveD2L_reg_stack" %}
14494 
14495   ins_encode %{
14496     __ strd(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
14497   %}
14498 
14499   ins_pipe(pipe_class_memory);
14500 
14501 %}
14502 
14503 instruct MoveL2D_reg_stack(stackSlotD dst, iRegL src) %{
14504 
14505   match(Set dst (MoveL2D src));
14506 
14507   effect(DEF dst, USE src);
14508 
14509   ins_cost(INSN_COST);
14510 
14511   format %{ "str $src, $dst\t# MoveL2D_reg_stack" %}
14512 
14513   ins_encode %{
14514     __ str($src$$Register, Address(sp, $dst$$disp));
14515   %}
14516 
14517   ins_pipe(istore_reg_reg);
14518 
14519 %}
14520 
14521 instruct MoveF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14522 
14523   match(Set dst (MoveF2I src));
14524 
14525   effect(DEF dst, USE src);
14526 
14527   ins_cost(INSN_COST);
14528 
14529   format %{ "fmovs $dst, $src\t# MoveF2I_reg_reg" %}
14530 
14531   ins_encode %{
14532     __ fmovs($dst$$Register, as_FloatRegister($src$$reg));
14533   %}
14534 
14535   ins_pipe(fp_f2i);
14536 
14537 %}
14538 
14539 instruct MoveI2F_reg_reg(vRegF dst, iRegI src) %{
14540 
14541   match(Set dst (MoveI2F src));
14542 
14543   effect(DEF dst, USE src);
14544 
14545   ins_cost(INSN_COST);
14546 
14547   format %{ "fmovs $dst, $src\t# MoveI2F_reg_reg" %}
14548 
14549   ins_encode %{
14550     __ fmovs(as_FloatRegister($dst$$reg), $src$$Register);
14551   %}
14552 
14553   ins_pipe(fp_i2f);
14554 
14555 %}
14556 
14557 instruct MoveD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14558 
14559   match(Set dst (MoveD2L src));
14560 
14561   effect(DEF dst, USE src);
14562 
14563   ins_cost(INSN_COST);
14564 
14565   format %{ "fmovd $dst, $src\t# MoveD2L_reg_reg" %}
14566 
14567   ins_encode %{
14568     __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
14569   %}
14570 
14571   ins_pipe(fp_d2l);
14572 
14573 %}
14574 
14575 instruct MoveL2D_reg_reg(vRegD dst, iRegL src) %{
14576 
14577   match(Set dst (MoveL2D src));
14578 
14579   effect(DEF dst, USE src);
14580 
14581   ins_cost(INSN_COST);
14582 
14583   format %{ "fmovd $dst, $src\t# MoveL2D_reg_reg" %}
14584 
14585   ins_encode %{
14586     __ fmovd(as_FloatRegister($dst$$reg), $src$$Register);
14587   %}
14588 
14589   ins_pipe(fp_l2d);
14590 
14591 %}
14592 
14593 // ============================================================================
14594 // clearing of an array
14595 
14596 instruct clearArray_reg_reg_immL0(iRegL_R11 cnt, iRegP_R10 base, immL0 zero, Universe dummy, rFlagsReg cr)
14597 %{
14598   match(Set dummy (ClearArray (Binary cnt base) zero));
14599   effect(USE_KILL cnt, USE_KILL base, KILL cr);
14600 
14601   ins_cost(4 * INSN_COST);
14602   format %{ "ClearArray $cnt, $base" %}
14603 
14604   ins_encode %{
14605     address tpc = __ zero_words($base$$Register, $cnt$$Register);
14606     if (tpc == nullptr) {
14607       ciEnv::current()->record_failure("CodeCache is full");
14608       return;
14609     }
14610   %}
14611 
14612   ins_pipe(pipe_class_memory);
14613 %}
14614 
14615 instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, iRegL val, Universe dummy, rFlagsReg cr)
14616 %{
14617   predicate(((ClearArrayNode*)n)->word_copy_only());
14618   match(Set dummy (ClearArray (Binary cnt base) val));
14619   effect(USE_KILL cnt, USE_KILL base, KILL cr);
14620 
14621   ins_cost(4 * INSN_COST);
14622   format %{ "ClearArray $cnt, $base, $val" %}
14623 
14624   ins_encode %{
14625     __ fill_words($base$$Register, $cnt$$Register, $val$$Register);
14626   %}
14627 
14628   ins_pipe(pipe_class_memory);
14629 %}
14630 
14631 instruct clearArray_imm_reg(immL cnt, iRegP_R10 base, iRegL_R11 temp, Universe dummy, rFlagsReg cr)
14632 %{
14633   predicate((uint64_t)n->in(2)->get_long()
14634             < (uint64_t)(BlockZeroingLowLimit >> LogBytesPerWord)
14635             && !((ClearArrayNode*)n)->word_copy_only());
14636   match(Set dummy (ClearArray cnt base));
14637   effect(TEMP temp, USE_KILL base, KILL cr);
14638 
14639   ins_cost(4 * INSN_COST);
14640   format %{ "ClearArray $cnt, $base" %}
14641 
14642   ins_encode %{
14643     address tpc = __ zero_words($base$$Register, (uint64_t)$cnt$$constant);
14644     if (tpc == nullptr) {
14645       ciEnv::current()->record_failure("CodeCache is full");
14646       return;
14647     }
14648   %}
14649 
14650   ins_pipe(pipe_class_memory);
14651 %}
14652 
14653 // ============================================================================
14654 // Overflow Math Instructions
14655 
14656 instruct overflowAddI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14657 %{
14658   match(Set cr (OverflowAddI op1 op2));
14659 
14660   format %{ "cmnw  $op1, $op2\t# overflow check int" %}
14661   ins_cost(INSN_COST);
14662   ins_encode %{
14663     __ cmnw($op1$$Register, $op2$$Register);
14664   %}
14665 
14666   ins_pipe(icmp_reg_reg);
14667 %}
14668 
14669 instruct overflowAddI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
14670 %{
14671   match(Set cr (OverflowAddI op1 op2));
14672 
14673   format %{ "cmnw  $op1, $op2\t# overflow check int" %}
14674   ins_cost(INSN_COST);
14675   ins_encode %{
14676     __ cmnw($op1$$Register, $op2$$constant);
14677   %}
14678 
14679   ins_pipe(icmp_reg_imm);
14680 %}
14681 
14682 instruct overflowAddL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14683 %{
14684   match(Set cr (OverflowAddL op1 op2));
14685 
14686   format %{ "cmn   $op1, $op2\t# overflow check long" %}
14687   ins_cost(INSN_COST);
14688   ins_encode %{
14689     __ cmn($op1$$Register, $op2$$Register);
14690   %}
14691 
14692   ins_pipe(icmp_reg_reg);
14693 %}
14694 
14695 instruct overflowAddL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
14696 %{
14697   match(Set cr (OverflowAddL op1 op2));
14698 
14699   format %{ "adds  zr, $op1, $op2\t# overflow check long" %}
14700   ins_cost(INSN_COST);
14701   ins_encode %{
14702     __ adds(zr, $op1$$Register, $op2$$constant);
14703   %}
14704 
14705   ins_pipe(icmp_reg_imm);
14706 %}
14707 
14708 instruct overflowSubI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14709 %{
14710   match(Set cr (OverflowSubI op1 op2));
14711 
14712   format %{ "cmpw  $op1, $op2\t# overflow check int" %}
14713   ins_cost(INSN_COST);
14714   ins_encode %{
14715     __ cmpw($op1$$Register, $op2$$Register);
14716   %}
14717 
14718   ins_pipe(icmp_reg_reg);
14719 %}
14720 
14721 instruct overflowSubI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
14722 %{
14723   match(Set cr (OverflowSubI op1 op2));
14724 
14725   format %{ "cmpw  $op1, $op2\t# overflow check int" %}
14726   ins_cost(INSN_COST);
14727   ins_encode %{
14728     __ cmpw($op1$$Register, $op2$$constant);
14729   %}
14730 
14731   ins_pipe(icmp_reg_imm);
14732 %}
14733 
14734 instruct overflowSubL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14735 %{
14736   match(Set cr (OverflowSubL op1 op2));
14737 
14738   format %{ "cmp   $op1, $op2\t# overflow check long" %}
14739   ins_cost(INSN_COST);
14740   ins_encode %{
14741     __ cmp($op1$$Register, $op2$$Register);
14742   %}
14743 
14744   ins_pipe(icmp_reg_reg);
14745 %}
14746 
14747 instruct overflowSubL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
14748 %{
14749   match(Set cr (OverflowSubL op1 op2));
14750 
14751   format %{ "cmp   $op1, $op2\t# overflow check long" %}
14752   ins_cost(INSN_COST);
14753   ins_encode %{
14754     __ subs(zr, $op1$$Register, $op2$$constant);
14755   %}
14756 
14757   ins_pipe(icmp_reg_imm);
14758 %}
14759 
14760 instruct overflowNegI_reg(rFlagsReg cr, immI0 zero, iRegIorL2I op1)
14761 %{
14762   match(Set cr (OverflowSubI zero op1));
14763 
14764   format %{ "cmpw  zr, $op1\t# overflow check int" %}
14765   ins_cost(INSN_COST);
14766   ins_encode %{
14767     __ cmpw(zr, $op1$$Register);
14768   %}
14769 
14770   ins_pipe(icmp_reg_imm);
14771 %}
14772 
14773 instruct overflowNegL_reg(rFlagsReg cr, immI0 zero, iRegL op1)
14774 %{
14775   match(Set cr (OverflowSubL zero op1));
14776 
14777   format %{ "cmp   zr, $op1\t# overflow check long" %}
14778   ins_cost(INSN_COST);
14779   ins_encode %{
14780     __ cmp(zr, $op1$$Register);
14781   %}
14782 
14783   ins_pipe(icmp_reg_imm);
14784 %}
14785 
14786 instruct overflowMulI_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14787 %{
14788   match(Set cr (OverflowMulI op1 op2));
14789 
14790   format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
14791             "cmp   rscratch1, rscratch1, sxtw\n\t"
14792             "movw  rscratch1, #0x80000000\n\t"
14793             "cselw rscratch1, rscratch1, zr, NE\n\t"
14794             "cmpw  rscratch1, #1" %}
14795   ins_cost(5 * INSN_COST);
14796   ins_encode %{
14797     __ smull(rscratch1, $op1$$Register, $op2$$Register);
14798     __ subs(zr, rscratch1, rscratch1, ext::sxtw);      // NE => overflow
14799     __ movw(rscratch1, 0x80000000);                    // Develop 0 (EQ),
14800     __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
14801     __ cmpw(rscratch1, 1);                             // 0x80000000 - 1 => VS
14802   %}
14803 
14804   ins_pipe(pipe_slow);
14805 %}
14806 
14807 instruct overflowMulI_reg_branch(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, label labl, rFlagsReg cr)
14808 %{
14809   match(If cmp (OverflowMulI op1 op2));
14810   predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
14811             || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
14812   effect(USE labl, KILL cr);
14813 
14814   format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
14815             "cmp   rscratch1, rscratch1, sxtw\n\t"
14816             "b$cmp   $labl" %}
14817   ins_cost(3 * INSN_COST); // Branch is rare so treat as INSN_COST
14818   ins_encode %{
14819     Label* L = $labl$$label;
14820     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14821     __ smull(rscratch1, $op1$$Register, $op2$$Register);
14822     __ subs(zr, rscratch1, rscratch1, ext::sxtw);      // NE => overflow
14823     __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
14824   %}
14825 
14826   ins_pipe(pipe_serial);
14827 %}
14828 
14829 instruct overflowMulL_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14830 %{
14831   match(Set cr (OverflowMulL op1 op2));
14832 
14833   format %{ "mul   rscratch1, $op1, $op2\t#overflow check long\n\t"
14834             "smulh rscratch2, $op1, $op2\n\t"
14835             "cmp   rscratch2, rscratch1, ASR #63\n\t"
14836             "movw  rscratch1, #0x80000000\n\t"
14837             "cselw rscratch1, rscratch1, zr, NE\n\t"
14838             "cmpw  rscratch1, #1" %}
14839   ins_cost(6 * INSN_COST);
14840   ins_encode %{
14841     __ mul(rscratch1, $op1$$Register, $op2$$Register);   // Result bits 0..63
14842     __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
14843     __ cmp(rscratch2, rscratch1, Assembler::ASR, 63);    // Top is pure sign ext
14844     __ movw(rscratch1, 0x80000000);                    // Develop 0 (EQ),
14845     __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
14846     __ cmpw(rscratch1, 1);                             // 0x80000000 - 1 => VS
14847   %}
14848 
14849   ins_pipe(pipe_slow);
14850 %}
14851 
14852 instruct overflowMulL_reg_branch(cmpOp cmp, iRegL op1, iRegL op2, label labl, rFlagsReg cr)
14853 %{
14854   match(If cmp (OverflowMulL op1 op2));
14855   predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
14856             || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
14857   effect(USE labl, KILL cr);
14858 
14859   format %{ "mul   rscratch1, $op1, $op2\t#overflow check long\n\t"
14860             "smulh rscratch2, $op1, $op2\n\t"
14861             "cmp   rscratch2, rscratch1, ASR #63\n\t"
14862             "b$cmp $labl" %}
14863   ins_cost(4 * INSN_COST); // Branch is rare so treat as INSN_COST
14864   ins_encode %{
14865     Label* L = $labl$$label;
14866     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14867     __ mul(rscratch1, $op1$$Register, $op2$$Register);   // Result bits 0..63
14868     __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
14869     __ cmp(rscratch2, rscratch1, Assembler::ASR, 63);    // Top is pure sign ext
14870     __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
14871   %}
14872 
14873   ins_pipe(pipe_serial);
14874 %}
14875 
14876 // ============================================================================
14877 // Compare Instructions
14878 
14879 instruct compI_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
14880 %{
14881   match(Set cr (CmpI op1 op2));
14882 
14883   effect(DEF cr, USE op1, USE op2);
14884 
14885   ins_cost(INSN_COST);
14886   format %{ "cmpw  $op1, $op2" %}
14887 
14888   ins_encode(aarch64_enc_cmpw(op1, op2));
14889 
14890   ins_pipe(icmp_reg_reg);
14891 %}
14892 
14893 instruct compI_reg_immI0(rFlagsReg cr, iRegI op1, immI0 zero)
14894 %{
14895   match(Set cr (CmpI op1 zero));
14896 
14897   effect(DEF cr, USE op1);
14898 
14899   ins_cost(INSN_COST);
14900   format %{ "cmpw $op1, 0" %}
14901 
14902   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
14903 
14904   ins_pipe(icmp_reg_imm);
14905 %}
14906 
14907 instruct compI_reg_immIAddSub(rFlagsReg cr, iRegI op1, immIAddSub op2)
14908 %{
14909   match(Set cr (CmpI op1 op2));
14910 
14911   effect(DEF cr, USE op1);
14912 
14913   ins_cost(INSN_COST);
14914   format %{ "cmpw  $op1, $op2" %}
14915 
14916   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
14917 
14918   ins_pipe(icmp_reg_imm);
14919 %}
14920 
14921 instruct compI_reg_immI(rFlagsReg cr, iRegI op1, immI op2)
14922 %{
14923   match(Set cr (CmpI op1 op2));
14924 
14925   effect(DEF cr, USE op1);
14926 
14927   ins_cost(INSN_COST * 2);
14928   format %{ "cmpw  $op1, $op2" %}
14929 
14930   ins_encode(aarch64_enc_cmpw_imm(op1, op2));
14931 
14932   ins_pipe(icmp_reg_imm);
14933 %}
14934 
14935 // Unsigned compare Instructions; really, same as signed compare
14936 // except it should only be used to feed an If or a CMovI which takes a
14937 // cmpOpU.
14938 
14939 instruct compU_reg_reg(rFlagsRegU cr, iRegI op1, iRegI op2)
14940 %{
14941   match(Set cr (CmpU op1 op2));
14942 
14943   effect(DEF cr, USE op1, USE op2);
14944 
14945   ins_cost(INSN_COST);
14946   format %{ "cmpw  $op1, $op2\t# unsigned" %}
14947 
14948   ins_encode(aarch64_enc_cmpw(op1, op2));
14949 
14950   ins_pipe(icmp_reg_reg);
14951 %}
14952 
14953 instruct compU_reg_immI0(rFlagsRegU cr, iRegI op1, immI0 zero)
14954 %{
14955   match(Set cr (CmpU op1 zero));
14956 
14957   effect(DEF cr, USE op1);
14958 
14959   ins_cost(INSN_COST);
14960   format %{ "cmpw $op1, #0\t# unsigned" %}
14961 
14962   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
14963 
14964   ins_pipe(icmp_reg_imm);
14965 %}
14966 
14967 instruct compU_reg_immIAddSub(rFlagsRegU cr, iRegI op1, immIAddSub op2)
14968 %{
14969   match(Set cr (CmpU op1 op2));
14970 
14971   effect(DEF cr, USE op1);
14972 
14973   ins_cost(INSN_COST);
14974   format %{ "cmpw  $op1, $op2\t# unsigned" %}
14975 
14976   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
14977 
14978   ins_pipe(icmp_reg_imm);
14979 %}
14980 
14981 instruct compU_reg_immI(rFlagsRegU cr, iRegI op1, immI op2)
14982 %{
14983   match(Set cr (CmpU op1 op2));
14984 
14985   effect(DEF cr, USE op1);
14986 
14987   ins_cost(INSN_COST * 2);
14988   format %{ "cmpw  $op1, $op2\t# unsigned" %}
14989 
14990   ins_encode(aarch64_enc_cmpw_imm(op1, op2));
14991 
14992   ins_pipe(icmp_reg_imm);
14993 %}
14994 
14995 instruct compL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14996 %{
14997   match(Set cr (CmpL op1 op2));
14998 
14999   effect(DEF cr, USE op1, USE op2);
15000 
15001   ins_cost(INSN_COST);
15002   format %{ "cmp  $op1, $op2" %}
15003 
15004   ins_encode(aarch64_enc_cmp(op1, op2));
15005 
15006   ins_pipe(icmp_reg_reg);
15007 %}
15008 
15009 instruct compL_reg_immL0(rFlagsReg cr, iRegL op1, immL0 zero)
15010 %{
15011   match(Set cr (CmpL op1 zero));
15012 
15013   effect(DEF cr, USE op1);
15014 
15015   ins_cost(INSN_COST);
15016   format %{ "tst  $op1" %}
15017 
15018   ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15019 
15020   ins_pipe(icmp_reg_imm);
15021 %}
15022 
15023 instruct compL_reg_immLAddSub(rFlagsReg cr, iRegL op1, immLAddSub op2)
15024 %{
15025   match(Set cr (CmpL op1 op2));
15026 
15027   effect(DEF cr, USE op1);
15028 
15029   ins_cost(INSN_COST);
15030   format %{ "cmp  $op1, $op2" %}
15031 
15032   ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15033 
15034   ins_pipe(icmp_reg_imm);
15035 %}
15036 
15037 instruct compL_reg_immL(rFlagsReg cr, iRegL op1, immL op2)
15038 %{
15039   match(Set cr (CmpL op1 op2));
15040 
15041   effect(DEF cr, USE op1);
15042 
15043   ins_cost(INSN_COST * 2);
15044   format %{ "cmp  $op1, $op2" %}
15045 
15046   ins_encode(aarch64_enc_cmp_imm(op1, op2));
15047 
15048   ins_pipe(icmp_reg_imm);
15049 %}
15050 
15051 instruct compUL_reg_reg(rFlagsRegU cr, iRegL op1, iRegL op2)
15052 %{
15053   match(Set cr (CmpUL op1 op2));
15054 
15055   effect(DEF cr, USE op1, USE op2);
15056 
15057   ins_cost(INSN_COST);
15058   format %{ "cmp  $op1, $op2" %}
15059 
15060   ins_encode(aarch64_enc_cmp(op1, op2));
15061 
15062   ins_pipe(icmp_reg_reg);
15063 %}
15064 
15065 instruct compUL_reg_immL0(rFlagsRegU cr, iRegL op1, immL0 zero)
15066 %{
15067   match(Set cr (CmpUL op1 zero));
15068 
15069   effect(DEF cr, USE op1);
15070 
15071   ins_cost(INSN_COST);
15072   format %{ "tst  $op1" %}
15073 
15074   ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15075 
15076   ins_pipe(icmp_reg_imm);
15077 %}
15078 
15079 instruct compUL_reg_immLAddSub(rFlagsRegU cr, iRegL op1, immLAddSub op2)
15080 %{
15081   match(Set cr (CmpUL op1 op2));
15082 
15083   effect(DEF cr, USE op1);
15084 
15085   ins_cost(INSN_COST);
15086   format %{ "cmp  $op1, $op2" %}
15087 
15088   ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15089 
15090   ins_pipe(icmp_reg_imm);
15091 %}
15092 
15093 instruct compUL_reg_immL(rFlagsRegU cr, iRegL op1, immL op2)
15094 %{
15095   match(Set cr (CmpUL op1 op2));
15096 
15097   effect(DEF cr, USE op1);
15098 
15099   ins_cost(INSN_COST * 2);
15100   format %{ "cmp  $op1, $op2" %}
15101 
15102   ins_encode(aarch64_enc_cmp_imm(op1, op2));
15103 
15104   ins_pipe(icmp_reg_imm);
15105 %}
15106 
15107 instruct compP_reg_reg(rFlagsRegU cr, iRegP op1, iRegP op2)
15108 %{
15109   match(Set cr (CmpP op1 op2));
15110 
15111   effect(DEF cr, USE op1, USE op2);
15112 
15113   ins_cost(INSN_COST);
15114   format %{ "cmp  $op1, $op2\t // ptr" %}
15115 
15116   ins_encode(aarch64_enc_cmpp(op1, op2));
15117 
15118   ins_pipe(icmp_reg_reg);
15119 %}
15120 
15121 instruct compN_reg_reg(rFlagsRegU cr, iRegN op1, iRegN op2)
15122 %{
15123   match(Set cr (CmpN op1 op2));
15124 
15125   effect(DEF cr, USE op1, USE op2);
15126 
15127   ins_cost(INSN_COST);
15128   format %{ "cmp  $op1, $op2\t // compressed ptr" %}
15129 
15130   ins_encode(aarch64_enc_cmpn(op1, op2));
15131 
15132   ins_pipe(icmp_reg_reg);
15133 %}
15134 
15135 instruct testP_reg(rFlagsRegU cr, iRegP op1, immP0 zero)
15136 %{
15137   match(Set cr (CmpP op1 zero));
15138 
15139   effect(DEF cr, USE op1, USE zero);
15140 
15141   ins_cost(INSN_COST);
15142   format %{ "cmp  $op1, 0\t // ptr" %}
15143 
15144   ins_encode(aarch64_enc_testp(op1));
15145 
15146   ins_pipe(icmp_reg_imm);
15147 %}
15148 
15149 instruct testN_reg(rFlagsRegU cr, iRegN op1, immN0 zero)
15150 %{
15151   match(Set cr (CmpN op1 zero));
15152 
15153   effect(DEF cr, USE op1, USE zero);
15154 
15155   ins_cost(INSN_COST);
15156   format %{ "cmp  $op1, 0\t // compressed ptr" %}
15157 
15158   ins_encode(aarch64_enc_testn(op1));
15159 
15160   ins_pipe(icmp_reg_imm);
15161 %}
15162 
15163 // FP comparisons
15164 //
15165 // n.b. CmpF/CmpD set a normal flags reg which then gets compared
15166 // using normal cmpOp. See declaration of rFlagsReg for details.
15167 
15168 instruct compF_reg_reg(rFlagsReg cr, vRegF src1, vRegF src2)
15169 %{
15170   match(Set cr (CmpF src1 src2));
15171 
15172   ins_cost(3 * INSN_COST);
15173   format %{ "fcmps $src1, $src2" %}
15174 
15175   ins_encode %{
15176     __ fcmps(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15177   %}
15178 
15179   ins_pipe(pipe_class_compare);
15180 %}
15181 
15182 instruct compF_reg_zero(rFlagsReg cr, vRegF src1, immF0 src2)
15183 %{
15184   match(Set cr (CmpF src1 src2));
15185 
15186   ins_cost(3 * INSN_COST);
15187   format %{ "fcmps $src1, 0.0" %}
15188 
15189   ins_encode %{
15190     __ fcmps(as_FloatRegister($src1$$reg), 0.0);
15191   %}
15192 
15193   ins_pipe(pipe_class_compare);
15194 %}
15195 // FROM HERE
15196 
15197 instruct compD_reg_reg(rFlagsReg cr, vRegD src1, vRegD src2)
15198 %{
15199   match(Set cr (CmpD src1 src2));
15200 
15201   ins_cost(3 * INSN_COST);
15202   format %{ "fcmpd $src1, $src2" %}
15203 
15204   ins_encode %{
15205     __ fcmpd(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15206   %}
15207 
15208   ins_pipe(pipe_class_compare);
15209 %}
15210 
15211 instruct compD_reg_zero(rFlagsReg cr, vRegD src1, immD0 src2)
15212 %{
15213   match(Set cr (CmpD src1 src2));
15214 
15215   ins_cost(3 * INSN_COST);
15216   format %{ "fcmpd $src1, 0.0" %}
15217 
15218   ins_encode %{
15219     __ fcmpd(as_FloatRegister($src1$$reg), 0.0);
15220   %}
15221 
15222   ins_pipe(pipe_class_compare);
15223 %}
15224 
15225 instruct compF3_reg_reg(iRegINoSp dst, vRegF src1, vRegF src2, rFlagsReg cr)
15226 %{
15227   match(Set dst (CmpF3 src1 src2));
15228   effect(KILL cr);
15229 
15230   ins_cost(5 * INSN_COST);
15231   format %{ "fcmps $src1, $src2\n\t"
15232             "csinvw($dst, zr, zr, eq\n\t"
15233             "csnegw($dst, $dst, $dst, lt)"
15234   %}
15235 
15236   ins_encode %{
15237     Label done;
15238     FloatRegister s1 = as_FloatRegister($src1$$reg);
15239     FloatRegister s2 = as_FloatRegister($src2$$reg);
15240     Register d = as_Register($dst$$reg);
15241     __ fcmps(s1, s2);
15242     // installs 0 if EQ else -1
15243     __ csinvw(d, zr, zr, Assembler::EQ);
15244     // keeps -1 if less or unordered else installs 1
15245     __ csnegw(d, d, d, Assembler::LT);
15246     __ bind(done);
15247   %}
15248 
15249   ins_pipe(pipe_class_default);
15250 
15251 %}
15252 
15253 instruct compD3_reg_reg(iRegINoSp dst, vRegD src1, vRegD src2, rFlagsReg cr)
15254 %{
15255   match(Set dst (CmpD3 src1 src2));
15256   effect(KILL cr);
15257 
15258   ins_cost(5 * INSN_COST);
15259   format %{ "fcmpd $src1, $src2\n\t"
15260             "csinvw($dst, zr, zr, eq\n\t"
15261             "csnegw($dst, $dst, $dst, lt)"
15262   %}
15263 
15264   ins_encode %{
15265     Label done;
15266     FloatRegister s1 = as_FloatRegister($src1$$reg);
15267     FloatRegister s2 = as_FloatRegister($src2$$reg);
15268     Register d = as_Register($dst$$reg);
15269     __ fcmpd(s1, s2);
15270     // installs 0 if EQ else -1
15271     __ csinvw(d, zr, zr, Assembler::EQ);
15272     // keeps -1 if less or unordered else installs 1
15273     __ csnegw(d, d, d, Assembler::LT);
15274     __ bind(done);
15275   %}
15276   ins_pipe(pipe_class_default);
15277 
15278 %}
15279 
15280 instruct compF3_reg_immF0(iRegINoSp dst, vRegF src1, immF0 zero, rFlagsReg cr)
15281 %{
15282   match(Set dst (CmpF3 src1 zero));
15283   effect(KILL cr);
15284 
15285   ins_cost(5 * INSN_COST);
15286   format %{ "fcmps $src1, 0.0\n\t"
15287             "csinvw($dst, zr, zr, eq\n\t"
15288             "csnegw($dst, $dst, $dst, lt)"
15289   %}
15290 
15291   ins_encode %{
15292     Label done;
15293     FloatRegister s1 = as_FloatRegister($src1$$reg);
15294     Register d = as_Register($dst$$reg);
15295     __ fcmps(s1, 0.0);
15296     // installs 0 if EQ else -1
15297     __ csinvw(d, zr, zr, Assembler::EQ);
15298     // keeps -1 if less or unordered else installs 1
15299     __ csnegw(d, d, d, Assembler::LT);
15300     __ bind(done);
15301   %}
15302 
15303   ins_pipe(pipe_class_default);
15304 
15305 %}
15306 
15307 instruct compD3_reg_immD0(iRegINoSp dst, vRegD src1, immD0 zero, rFlagsReg cr)
15308 %{
15309   match(Set dst (CmpD3 src1 zero));
15310   effect(KILL cr);
15311 
15312   ins_cost(5 * INSN_COST);
15313   format %{ "fcmpd $src1, 0.0\n\t"
15314             "csinvw($dst, zr, zr, eq\n\t"
15315             "csnegw($dst, $dst, $dst, lt)"
15316   %}
15317 
15318   ins_encode %{
15319     Label done;
15320     FloatRegister s1 = as_FloatRegister($src1$$reg);
15321     Register d = as_Register($dst$$reg);
15322     __ fcmpd(s1, 0.0);
15323     // installs 0 if EQ else -1
15324     __ csinvw(d, zr, zr, Assembler::EQ);
15325     // keeps -1 if less or unordered else installs 1
15326     __ csnegw(d, d, d, Assembler::LT);
15327     __ bind(done);
15328   %}
15329   ins_pipe(pipe_class_default);
15330 
15331 %}
15332 
15333 instruct cmpLTMask_reg_reg(iRegINoSp dst, iRegIorL2I p, iRegIorL2I q, rFlagsReg cr)
15334 %{
15335   match(Set dst (CmpLTMask p q));
15336   effect(KILL cr);
15337 
15338   ins_cost(3 * INSN_COST);
15339 
15340   format %{ "cmpw $p, $q\t# cmpLTMask\n\t"
15341             "csetw $dst, lt\n\t"
15342             "subw $dst, zr, $dst"
15343   %}
15344 
15345   ins_encode %{
15346     __ cmpw(as_Register($p$$reg), as_Register($q$$reg));
15347     __ csetw(as_Register($dst$$reg), Assembler::LT);
15348     __ subw(as_Register($dst$$reg), zr, as_Register($dst$$reg));
15349   %}
15350 
15351   ins_pipe(ialu_reg_reg);
15352 %}
15353 
15354 instruct cmpLTMask_reg_zero(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr)
15355 %{
15356   match(Set dst (CmpLTMask src zero));
15357   effect(KILL cr);
15358 
15359   ins_cost(INSN_COST);
15360 
15361   format %{ "asrw $dst, $src, #31\t# cmpLTMask0" %}
15362 
15363   ins_encode %{
15364     __ asrw(as_Register($dst$$reg), as_Register($src$$reg), 31);
15365   %}
15366 
15367   ins_pipe(ialu_reg_shift);
15368 %}
15369 
15370 // ============================================================================
15371 // Max and Min
15372 
15373 // Like compI_reg_reg or compI_reg_immI0 but without match rule and second zero parameter.
15374 
15375 instruct compI_reg_imm0(rFlagsReg cr, iRegI src)
15376 %{
15377   effect(DEF cr, USE src);
15378   ins_cost(INSN_COST);
15379   format %{ "cmpw $src, 0" %}
15380 
15381   ins_encode %{
15382     __ cmpw($src$$Register, 0);
15383   %}
15384   ins_pipe(icmp_reg_imm);
15385 %}
15386 
15387 instruct minI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
15388 %{
15389   match(Set dst (MinI src1 src2));
15390   ins_cost(INSN_COST * 3);
15391 
15392   expand %{
15393     rFlagsReg cr;
15394     compI_reg_reg(cr, src1, src2);
15395     cmovI_reg_reg_lt(dst, src1, src2, cr);
15396   %}
15397 %}
15398 
15399 instruct maxI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
15400 %{
15401   match(Set dst (MaxI src1 src2));
15402   ins_cost(INSN_COST * 3);
15403 
15404   expand %{
15405     rFlagsReg cr;
15406     compI_reg_reg(cr, src1, src2);
15407     cmovI_reg_reg_gt(dst, src1, src2, cr);
15408   %}
15409 %}
15410 
15411 
15412 // ============================================================================
15413 // Branch Instructions
15414 
15415 // Direct Branch.
15416 instruct branch(label lbl)
15417 %{
15418   match(Goto);
15419 
15420   effect(USE lbl);
15421 
15422   ins_cost(BRANCH_COST);
15423   format %{ "b  $lbl" %}
15424 
15425   ins_encode(aarch64_enc_b(lbl));
15426 
15427   ins_pipe(pipe_branch);
15428 %}
15429 
15430 // Conditional Near Branch
15431 instruct branchCon(cmpOp cmp, rFlagsReg cr, label lbl)
15432 %{
15433   // Same match rule as `branchConFar'.
15434   match(If cmp cr);
15435 
15436   effect(USE lbl);
15437 
15438   ins_cost(BRANCH_COST);
15439   // If set to 1 this indicates that the current instruction is a
15440   // short variant of a long branch. This avoids using this
15441   // instruction in first-pass matching. It will then only be used in
15442   // the `Shorten_branches' pass.
15443   // ins_short_branch(1);
15444   format %{ "b$cmp  $lbl" %}
15445 
15446   ins_encode(aarch64_enc_br_con(cmp, lbl));
15447 
15448   ins_pipe(pipe_branch_cond);
15449 %}
15450 
15451 // Conditional Near Branch Unsigned
15452 instruct branchConU(cmpOpU cmp, rFlagsRegU cr, label lbl)
15453 %{
15454   // Same match rule as `branchConFar'.
15455   match(If cmp cr);
15456 
15457   effect(USE lbl);
15458 
15459   ins_cost(BRANCH_COST);
15460   // If set to 1 this indicates that the current instruction is a
15461   // short variant of a long branch. This avoids using this
15462   // instruction in first-pass matching. It will then only be used in
15463   // the `Shorten_branches' pass.
15464   // ins_short_branch(1);
15465   format %{ "b$cmp  $lbl\t# unsigned" %}
15466 
15467   ins_encode(aarch64_enc_br_conU(cmp, lbl));
15468 
15469   ins_pipe(pipe_branch_cond);
15470 %}
15471 
15472 // Make use of CBZ and CBNZ.  These instructions, as well as being
15473 // shorter than (cmp; branch), have the additional benefit of not
15474 // killing the flags.
15475 
15476 instruct cmpI_imm0_branch(cmpOpEqNe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsReg cr) %{
15477   match(If cmp (CmpI op1 op2));
15478   effect(USE labl);
15479 
15480   ins_cost(BRANCH_COST);
15481   format %{ "cbw$cmp   $op1, $labl" %}
15482   ins_encode %{
15483     Label* L = $labl$$label;
15484     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15485     if (cond == Assembler::EQ)
15486       __ cbzw($op1$$Register, *L);
15487     else
15488       __ cbnzw($op1$$Register, *L);
15489   %}
15490   ins_pipe(pipe_cmp_branch);
15491 %}
15492 
15493 instruct cmpL_imm0_branch(cmpOpEqNe cmp, iRegL op1, immL0 op2, label labl, rFlagsReg cr) %{
15494   match(If cmp (CmpL op1 op2));
15495   effect(USE labl);
15496 
15497   ins_cost(BRANCH_COST);
15498   format %{ "cb$cmp   $op1, $labl" %}
15499   ins_encode %{
15500     Label* L = $labl$$label;
15501     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15502     if (cond == Assembler::EQ)
15503       __ cbz($op1$$Register, *L);
15504     else
15505       __ cbnz($op1$$Register, *L);
15506   %}
15507   ins_pipe(pipe_cmp_branch);
15508 %}
15509 
15510 instruct cmpP_imm0_branch(cmpOpEqNe cmp, iRegP op1, immP0 op2, label labl, rFlagsReg cr) %{
15511   match(If cmp (CmpP op1 op2));
15512   effect(USE labl);
15513 
15514   ins_cost(BRANCH_COST);
15515   format %{ "cb$cmp   $op1, $labl" %}
15516   ins_encode %{
15517     Label* L = $labl$$label;
15518     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15519     if (cond == Assembler::EQ)
15520       __ cbz($op1$$Register, *L);
15521     else
15522       __ cbnz($op1$$Register, *L);
15523   %}
15524   ins_pipe(pipe_cmp_branch);
15525 %}
15526 
15527 instruct cmpN_imm0_branch(cmpOpEqNe cmp, iRegN op1, immN0 op2, label labl, rFlagsReg cr) %{
15528   match(If cmp (CmpN op1 op2));
15529   effect(USE labl);
15530 
15531   ins_cost(BRANCH_COST);
15532   format %{ "cbw$cmp   $op1, $labl" %}
15533   ins_encode %{
15534     Label* L = $labl$$label;
15535     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15536     if (cond == Assembler::EQ)
15537       __ cbzw($op1$$Register, *L);
15538     else
15539       __ cbnzw($op1$$Register, *L);
15540   %}
15541   ins_pipe(pipe_cmp_branch);
15542 %}
15543 
15544 instruct cmpP_narrowOop_imm0_branch(cmpOpEqNe cmp, iRegN oop, immP0 zero, label labl, rFlagsReg cr) %{
15545   match(If cmp (CmpP (DecodeN oop) zero));
15546   effect(USE labl);
15547 
15548   ins_cost(BRANCH_COST);
15549   format %{ "cb$cmp   $oop, $labl" %}
15550   ins_encode %{
15551     Label* L = $labl$$label;
15552     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15553     if (cond == Assembler::EQ)
15554       __ cbzw($oop$$Register, *L);
15555     else
15556       __ cbnzw($oop$$Register, *L);
15557   %}
15558   ins_pipe(pipe_cmp_branch);
15559 %}
15560 
15561 instruct cmpUI_imm0_branch(cmpOpUEqNeLeGt cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15562   match(If cmp (CmpU op1 op2));
15563   effect(USE labl);
15564 
15565   ins_cost(BRANCH_COST);
15566   format %{ "cbw$cmp   $op1, $labl" %}
15567   ins_encode %{
15568     Label* L = $labl$$label;
15569     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15570     if (cond == Assembler::EQ || cond == Assembler::LS) {
15571       __ cbzw($op1$$Register, *L);
15572     } else {
15573       assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
15574       __ cbnzw($op1$$Register, *L);
15575     }
15576   %}
15577   ins_pipe(pipe_cmp_branch);
15578 %}
15579 
15580 instruct cmpUL_imm0_branch(cmpOpUEqNeLeGt cmp, iRegL op1, immL0 op2, label labl) %{
15581   match(If cmp (CmpUL op1 op2));
15582   effect(USE labl);
15583 
15584   ins_cost(BRANCH_COST);
15585   format %{ "cb$cmp   $op1, $labl" %}
15586   ins_encode %{
15587     Label* L = $labl$$label;
15588     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15589     if (cond == Assembler::EQ || cond == Assembler::LS) {
15590       __ cbz($op1$$Register, *L);
15591     } else {
15592       assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
15593       __ cbnz($op1$$Register, *L);
15594     }
15595   %}
15596   ins_pipe(pipe_cmp_branch);
15597 %}
15598 
15599 // Test bit and Branch
15600 
15601 // Patterns for short (< 32KiB) variants
15602 instruct cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
15603   match(If cmp (CmpL op1 op2));
15604   effect(USE labl);
15605 
15606   ins_cost(BRANCH_COST);
15607   format %{ "cb$cmp   $op1, $labl # long" %}
15608   ins_encode %{
15609     Label* L = $labl$$label;
15610     Assembler::Condition cond =
15611       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15612     __ tbr(cond, $op1$$Register, 63, *L);
15613   %}
15614   ins_pipe(pipe_cmp_branch);
15615   ins_short_branch(1);
15616 %}
15617 
15618 instruct cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15619   match(If cmp (CmpI op1 op2));
15620   effect(USE labl);
15621 
15622   ins_cost(BRANCH_COST);
15623   format %{ "cb$cmp   $op1, $labl # int" %}
15624   ins_encode %{
15625     Label* L = $labl$$label;
15626     Assembler::Condition cond =
15627       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15628     __ tbr(cond, $op1$$Register, 31, *L);
15629   %}
15630   ins_pipe(pipe_cmp_branch);
15631   ins_short_branch(1);
15632 %}
15633 
15634 instruct cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
15635   match(If cmp (CmpL (AndL op1 op2) op3));
15636   predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
15637   effect(USE labl);
15638 
15639   ins_cost(BRANCH_COST);
15640   format %{ "tb$cmp   $op1, $op2, $labl" %}
15641   ins_encode %{
15642     Label* L = $labl$$label;
15643     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15644     int bit = exact_log2_long($op2$$constant);
15645     __ tbr(cond, $op1$$Register, bit, *L);
15646   %}
15647   ins_pipe(pipe_cmp_branch);
15648   ins_short_branch(1);
15649 %}
15650 
15651 instruct cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
15652   match(If cmp (CmpI (AndI op1 op2) op3));
15653   predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
15654   effect(USE labl);
15655 
15656   ins_cost(BRANCH_COST);
15657   format %{ "tb$cmp   $op1, $op2, $labl" %}
15658   ins_encode %{
15659     Label* L = $labl$$label;
15660     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15661     int bit = exact_log2((juint)$op2$$constant);
15662     __ tbr(cond, $op1$$Register, bit, *L);
15663   %}
15664   ins_pipe(pipe_cmp_branch);
15665   ins_short_branch(1);
15666 %}
15667 
15668 // And far variants
15669 instruct far_cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
15670   match(If cmp (CmpL op1 op2));
15671   effect(USE labl);
15672 
15673   ins_cost(BRANCH_COST);
15674   format %{ "cb$cmp   $op1, $labl # long" %}
15675   ins_encode %{
15676     Label* L = $labl$$label;
15677     Assembler::Condition cond =
15678       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15679     __ tbr(cond, $op1$$Register, 63, *L, /*far*/true);
15680   %}
15681   ins_pipe(pipe_cmp_branch);
15682 %}
15683 
15684 instruct far_cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15685   match(If cmp (CmpI op1 op2));
15686   effect(USE labl);
15687 
15688   ins_cost(BRANCH_COST);
15689   format %{ "cb$cmp   $op1, $labl # int" %}
15690   ins_encode %{
15691     Label* L = $labl$$label;
15692     Assembler::Condition cond =
15693       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15694     __ tbr(cond, $op1$$Register, 31, *L, /*far*/true);
15695   %}
15696   ins_pipe(pipe_cmp_branch);
15697 %}
15698 
15699 instruct far_cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
15700   match(If cmp (CmpL (AndL op1 op2) op3));
15701   predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
15702   effect(USE labl);
15703 
15704   ins_cost(BRANCH_COST);
15705   format %{ "tb$cmp   $op1, $op2, $labl" %}
15706   ins_encode %{
15707     Label* L = $labl$$label;
15708     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15709     int bit = exact_log2_long($op2$$constant);
15710     __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
15711   %}
15712   ins_pipe(pipe_cmp_branch);
15713 %}
15714 
15715 instruct far_cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
15716   match(If cmp (CmpI (AndI op1 op2) op3));
15717   predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
15718   effect(USE labl);
15719 
15720   ins_cost(BRANCH_COST);
15721   format %{ "tb$cmp   $op1, $op2, $labl" %}
15722   ins_encode %{
15723     Label* L = $labl$$label;
15724     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15725     int bit = exact_log2((juint)$op2$$constant);
15726     __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
15727   %}
15728   ins_pipe(pipe_cmp_branch);
15729 %}
15730 
15731 // Test bits
15732 
15733 instruct cmpL_and(cmpOp cmp, iRegL op1, immL op2, immL0 op3, rFlagsReg cr) %{
15734   match(Set cr (CmpL (AndL op1 op2) op3));
15735   predicate(Assembler::operand_valid_for_logical_immediate
15736             (/*is_32*/false, n->in(1)->in(2)->get_long()));
15737 
15738   ins_cost(INSN_COST);
15739   format %{ "tst $op1, $op2 # long" %}
15740   ins_encode %{
15741     __ tst($op1$$Register, $op2$$constant);
15742   %}
15743   ins_pipe(ialu_reg_reg);
15744 %}
15745 
15746 instruct cmpI_and(cmpOp cmp, iRegIorL2I op1, immI op2, immI0 op3, rFlagsReg cr) %{
15747   match(Set cr (CmpI (AndI op1 op2) op3));
15748   predicate(Assembler::operand_valid_for_logical_immediate
15749             (/*is_32*/true, n->in(1)->in(2)->get_int()));
15750 
15751   ins_cost(INSN_COST);
15752   format %{ "tst $op1, $op2 # int" %}
15753   ins_encode %{
15754     __ tstw($op1$$Register, $op2$$constant);
15755   %}
15756   ins_pipe(ialu_reg_reg);
15757 %}
15758 
15759 instruct cmpL_and_reg(cmpOp cmp, iRegL op1, iRegL op2, immL0 op3, rFlagsReg cr) %{
15760   match(Set cr (CmpL (AndL op1 op2) op3));
15761 
15762   ins_cost(INSN_COST);
15763   format %{ "tst $op1, $op2 # long" %}
15764   ins_encode %{
15765     __ tst($op1$$Register, $op2$$Register);
15766   %}
15767   ins_pipe(ialu_reg_reg);
15768 %}
15769 
15770 instruct cmpI_and_reg(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, immI0 op3, rFlagsReg cr) %{
15771   match(Set cr (CmpI (AndI op1 op2) op3));
15772 
15773   ins_cost(INSN_COST);
15774   format %{ "tstw $op1, $op2 # int" %}
15775   ins_encode %{
15776     __ tstw($op1$$Register, $op2$$Register);
15777   %}
15778   ins_pipe(ialu_reg_reg);
15779 %}
15780 
15781 
15782 // Conditional Far Branch
15783 // Conditional Far Branch Unsigned
15784 // TODO: fixme
15785 
15786 // counted loop end branch near
15787 instruct branchLoopEnd(cmpOp cmp, rFlagsReg cr, label lbl)
15788 %{
15789   match(CountedLoopEnd cmp cr);
15790 
15791   effect(USE lbl);
15792 
15793   ins_cost(BRANCH_COST);
15794   // short variant.
15795   // ins_short_branch(1);
15796   format %{ "b$cmp $lbl \t// counted loop end" %}
15797 
15798   ins_encode(aarch64_enc_br_con(cmp, lbl));
15799 
15800   ins_pipe(pipe_branch);
15801 %}
15802 
15803 // counted loop end branch far
15804 // TODO: fixme
15805 
15806 // ============================================================================
15807 // inlined locking and unlocking
15808 
15809 instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
15810 %{
15811   predicate(LockingMode != LM_LIGHTWEIGHT);
15812   match(Set cr (FastLock object box));
15813   effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
15814 
15815   ins_cost(5 * INSN_COST);
15816   format %{ "fastlock $object,$box\t! kills $tmp,$tmp2,$tmp3" %}
15817 
15818   ins_encode %{
15819     __ fast_lock($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
15820   %}
15821 
15822   ins_pipe(pipe_serial);
15823 %}
15824 
15825 instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
15826 %{
15827   predicate(LockingMode != LM_LIGHTWEIGHT);
15828   match(Set cr (FastUnlock object box));
15829   effect(TEMP tmp, TEMP tmp2);
15830 
15831   ins_cost(5 * INSN_COST);
15832   format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2" %}
15833 
15834   ins_encode %{
15835     __ fast_unlock($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register);
15836   %}
15837 
15838   ins_pipe(pipe_serial);
15839 %}
15840 
15841 instruct cmpFastLockLightweight(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
15842 %{
15843   predicate(LockingMode == LM_LIGHTWEIGHT);
15844   match(Set cr (FastLock object box));
15845   effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
15846 
15847   ins_cost(5 * INSN_COST);
15848   format %{ "fastlock $object,$box\t! kills $tmp,$tmp2,$tmp3" %}
15849 
15850   ins_encode %{
15851     __ fast_lock_lightweight($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
15852   %}
15853 
15854   ins_pipe(pipe_serial);
15855 %}
15856 
15857 instruct cmpFastUnlockLightweight(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
15858 %{
15859   predicate(LockingMode == LM_LIGHTWEIGHT);
15860   match(Set cr (FastUnlock object box));
15861   effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
15862 
15863   ins_cost(5 * INSN_COST);
15864   format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2, $tmp3" %}
15865 
15866   ins_encode %{
15867     __ fast_unlock_lightweight($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
15868   %}
15869 
15870   ins_pipe(pipe_serial);
15871 %}
15872 
15873 // ============================================================================
15874 // Safepoint Instructions
15875 
15876 // TODO
15877 // provide a near and far version of this code
15878 
15879 instruct safePoint(rFlagsReg cr, iRegP poll)
15880 %{
15881   match(SafePoint poll);
15882   effect(KILL cr);
15883 
15884   format %{
15885     "ldrw zr, [$poll]\t# Safepoint: poll for GC"
15886   %}
15887   ins_encode %{
15888     __ read_polling_page(as_Register($poll$$reg), relocInfo::poll_type);
15889   %}
15890   ins_pipe(pipe_serial); // ins_pipe(iload_reg_mem);
15891 %}
15892 
15893 
15894 // ============================================================================
15895 // Procedure Call/Return Instructions
15896 
15897 // Call Java Static Instruction
15898 
15899 instruct CallStaticJavaDirect(method meth)
15900 %{
15901   match(CallStaticJava);
15902 
15903   effect(USE meth);
15904 
15905   ins_cost(CALL_COST);
15906 
15907   format %{ "call,static $meth \t// ==> " %}
15908 
15909   ins_encode(aarch64_enc_java_static_call(meth),
15910              aarch64_enc_call_epilog);
15911 
15912   ins_pipe(pipe_class_call);
15913 %}
15914 
15915 // TO HERE
15916 
15917 // Call Java Dynamic Instruction
15918 instruct CallDynamicJavaDirect(method meth)
15919 %{
15920   match(CallDynamicJava);
15921 
15922   effect(USE meth);
15923 
15924   ins_cost(CALL_COST);
15925 
15926   format %{ "CALL,dynamic $meth \t// ==> " %}
15927 
15928   ins_encode(aarch64_enc_java_dynamic_call(meth),
15929              aarch64_enc_call_epilog);
15930 
15931   ins_pipe(pipe_class_call);
15932 %}
15933 
15934 // Call Runtime Instruction
15935 
15936 instruct CallRuntimeDirect(method meth)
15937 %{
15938   match(CallRuntime);
15939 
15940   effect(USE meth);
15941 
15942   ins_cost(CALL_COST);
15943 
15944   format %{ "CALL, runtime $meth" %}
15945 
15946   ins_encode( aarch64_enc_java_to_runtime(meth) );
15947 
15948   ins_pipe(pipe_class_call);
15949 %}
15950 
15951 // Call Runtime Instruction
15952 
15953 instruct CallLeafDirect(method meth)
15954 %{
15955   match(CallLeaf);
15956 
15957   effect(USE meth);
15958 
15959   ins_cost(CALL_COST);
15960 
15961   format %{ "CALL, runtime leaf $meth" %}
15962 
15963   ins_encode( aarch64_enc_java_to_runtime(meth) );
15964 
15965   ins_pipe(pipe_class_call);
15966 %}
15967 
15968 // Call Runtime Instruction
15969 
15970 // entry point is null, target holds the address to call
15971 instruct CallLeafNoFPIndirect(iRegP target)
15972 %{
15973   predicate(n->as_Call()->entry_point() == nullptr);
15974 
15975   match(CallLeafNoFP target);
15976 
15977   ins_cost(CALL_COST);
15978 
15979   format %{ "CALL, runtime leaf nofp indirect $target" %}
15980 
15981   ins_encode %{
15982     __ blr($target$$Register);
15983   %}
15984 
15985   ins_pipe(pipe_class_call);
15986 %}
15987 
15988 instruct CallLeafNoFPDirect(method meth)
15989 %{
15990   predicate(n->as_Call()->entry_point() != nullptr);
15991 
15992   match(CallLeafNoFP);
15993 
15994   effect(USE meth);
15995 
15996   ins_cost(CALL_COST);
15997 
15998   format %{ "CALL, runtime leaf nofp $meth" %}
15999 
16000   ins_encode( aarch64_enc_java_to_runtime(meth) );
16001 
16002   ins_pipe(pipe_class_call);
16003 %}
16004 
16005 // Tail Call; Jump from runtime stub to Java code.
16006 // Also known as an 'interprocedural jump'.
16007 // Target of jump will eventually return to caller.
16008 // TailJump below removes the return address.
16009 // Don't use rfp for 'jump_target' because a MachEpilogNode has already been
16010 // emitted just above the TailCall which has reset rfp to the caller state.
16011 instruct TailCalljmpInd(iRegPNoSpNoRfp jump_target, inline_cache_RegP method_ptr)
16012 %{
16013   match(TailCall jump_target method_ptr);
16014 
16015   ins_cost(CALL_COST);
16016 
16017   format %{ "br $jump_target\t# $method_ptr holds method" %}
16018 
16019   ins_encode(aarch64_enc_tail_call(jump_target));
16020 
16021   ins_pipe(pipe_class_call);
16022 %}
16023 
16024 instruct TailjmpInd(iRegPNoSpNoRfp jump_target, iRegP_R0 ex_oop)
16025 %{
16026   match(TailJump jump_target ex_oop);
16027 
16028   ins_cost(CALL_COST);
16029 
16030   format %{ "br $jump_target\t# $ex_oop holds exception oop" %}
16031 
16032   ins_encode(aarch64_enc_tail_jmp(jump_target));
16033 
16034   ins_pipe(pipe_class_call);
16035 %}
16036 
16037 // Forward exception.
16038 instruct ForwardExceptionjmp()
16039 %{
16040   match(ForwardException);
16041   ins_cost(CALL_COST);
16042 
16043   format %{ "b forward_exception_stub" %}
16044   ins_encode %{
16045     __ far_jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
16046   %}
16047   ins_pipe(pipe_class_call);
16048 %}
16049 
16050 // Create exception oop: created by stack-crawling runtime code.
16051 // Created exception is now available to this handler, and is setup
16052 // just prior to jumping to this handler. No code emitted.
16053 // TODO check
16054 // should ex_oop be in r0? intel uses rax, ppc cannot use r0 so uses rarg1
16055 instruct CreateException(iRegP_R0 ex_oop)
16056 %{
16057   match(Set ex_oop (CreateEx));
16058 
16059   format %{ " -- \t// exception oop; no code emitted" %}
16060 
16061   size(0);
16062 
16063   ins_encode( /*empty*/ );
16064 
16065   ins_pipe(pipe_class_empty);
16066 %}
16067 
16068 // Rethrow exception: The exception oop will come in the first
16069 // argument position. Then JUMP (not call) to the rethrow stub code.
16070 instruct RethrowException() %{
16071   match(Rethrow);
16072   ins_cost(CALL_COST);
16073 
16074   format %{ "b rethrow_stub" %}
16075 
16076   ins_encode( aarch64_enc_rethrow() );
16077 
16078   ins_pipe(pipe_class_call);
16079 %}
16080 
16081 
16082 // Return Instruction
16083 // epilog node loads ret address into lr as part of frame pop
16084 instruct Ret()
16085 %{
16086   match(Return);
16087 
16088   format %{ "ret\t// return register" %}
16089 
16090   ins_encode( aarch64_enc_ret() );
16091 
16092   ins_pipe(pipe_branch);
16093 %}
16094 
16095 // Die now.
16096 instruct ShouldNotReachHere() %{
16097   match(Halt);
16098 
16099   ins_cost(CALL_COST);
16100   format %{ "ShouldNotReachHere" %}
16101 
16102   ins_encode %{
16103     if (is_reachable()) {
16104       __ stop(_halt_reason);
16105     }
16106   %}
16107 
16108   ins_pipe(pipe_class_default);
16109 %}
16110 
16111 // ============================================================================
16112 // Partial Subtype Check
16113 //
16114 // superklass array for an instance of the superklass.  Set a hidden
16115 // internal cache on a hit (cache is checked with exposed code in
16116 // gen_subtype_check()).  Return NZ for a miss or zero for a hit.  The
16117 // encoding ALSO sets flags.
16118 
16119 instruct partialSubtypeCheck(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, rFlagsReg cr)
16120 %{
16121   match(Set result (PartialSubtypeCheck sub super));
16122   effect(KILL cr, KILL temp);
16123 
16124   ins_cost(1100);  // slightly larger than the next version
16125   format %{ "partialSubtypeCheck $result, $sub, $super" %}
16126 
16127   ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
16128 
16129   opcode(0x1); // Force zero of result reg on hit
16130 
16131   ins_pipe(pipe_class_memory);
16132 %}
16133 
16134 instruct partialSubtypeCheckConstSuper(iRegP_R4 sub, iRegP_R0 super_reg, immP super_con, vRegD_V0 vtemp, iRegP_R5 result,
16135                                        iRegP_R1 tempR1, iRegP_R2 tempR2, iRegP_R3 tempR3,
16136                                        rFlagsReg cr)
16137 %{
16138   match(Set result (PartialSubtypeCheck sub (Binary super_reg super_con)));
16139   predicate(UseSecondarySupersTable);
16140   effect(KILL cr, TEMP tempR1, TEMP tempR2, TEMP tempR3, TEMP vtemp);
16141 
16142   ins_cost(700);  // smaller than the next version
16143   format %{ "partialSubtypeCheck $result, $sub, $super_reg, $super_con" %}
16144 
16145   ins_encode %{
16146     bool success = false;
16147     u1 super_klass_slot = ((Klass*)$super_con$$constant)->hash_slot();
16148     if (InlineSecondarySupersTest) {
16149       success = __ lookup_secondary_supers_table($sub$$Register, $super_reg$$Register,
16150                                                  $tempR1$$Register, $tempR2$$Register, $tempR3$$Register,
16151                                                  $vtemp$$FloatRegister,
16152                                                  $result$$Register,
16153                                                  super_klass_slot);
16154     } else {
16155       address call = __ trampoline_call(RuntimeAddress(StubRoutines::lookup_secondary_supers_table_stub(super_klass_slot)));
16156       success = (call != nullptr);
16157     }
16158     if (!success) {
16159       ciEnv::current()->record_failure("CodeCache is full");
16160       return;
16161     }
16162   %}
16163 
16164   ins_pipe(pipe_class_memory);
16165 %}
16166 
16167 instruct partialSubtypeCheckVsZero(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, immP0 zero, rFlagsReg cr)
16168 %{
16169   match(Set cr (CmpP (PartialSubtypeCheck sub super) zero));
16170   effect(KILL temp, KILL result);
16171 
16172   ins_cost(1100);  // slightly larger than the next version
16173   format %{ "partialSubtypeCheck $result, $sub, $super == 0" %}
16174 
16175   ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
16176 
16177   opcode(0x0); // Don't zero result reg on hit
16178 
16179   ins_pipe(pipe_class_memory);
16180 %}
16181 
16182 // Intrisics for String.compareTo()
16183 
16184 instruct string_compareU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16185                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16186 %{
16187   predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
16188   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16189   effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16190 
16191   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1" %}
16192   ins_encode %{
16193     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16194     __ string_compare($str1$$Register, $str2$$Register,
16195                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16196                       $tmp1$$Register, $tmp2$$Register,
16197                       fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::UU);
16198   %}
16199   ins_pipe(pipe_class_memory);
16200 %}
16201 
16202 instruct string_compareL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16203                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16204 %{
16205   predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
16206   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16207   effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16208 
16209   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1" %}
16210   ins_encode %{
16211     __ string_compare($str1$$Register, $str2$$Register,
16212                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16213                       $tmp1$$Register, $tmp2$$Register,
16214                       fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::LL);
16215   %}
16216   ins_pipe(pipe_class_memory);
16217 %}
16218 
16219 instruct string_compareUL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16220                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16221                         vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16222 %{
16223   predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
16224   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16225   effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16226          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16227 
16228   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16229   ins_encode %{
16230     __ string_compare($str1$$Register, $str2$$Register,
16231                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16232                       $tmp1$$Register, $tmp2$$Register,
16233                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16234                       $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::UL);
16235   %}
16236   ins_pipe(pipe_class_memory);
16237 %}
16238 
16239 instruct string_compareLU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16240                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16241                         vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16242 %{
16243   predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
16244   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16245   effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16246          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16247 
16248   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16249   ins_encode %{
16250     __ string_compare($str1$$Register, $str2$$Register,
16251                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16252                       $tmp1$$Register, $tmp2$$Register,
16253                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16254                       $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::LU);
16255   %}
16256   ins_pipe(pipe_class_memory);
16257 %}
16258 
16259 // Note that Z registers alias the corresponding NEON registers, we declare the vector operands of
16260 // these string_compare variants as NEON register type for convenience so that the prototype of
16261 // string_compare can be shared with all variants.
16262 
16263 instruct string_compareLL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16264                               iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16265                               vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16266                               pRegGov_P1 pgtmp2, rFlagsReg cr)
16267 %{
16268   predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
16269   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16270   effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16271          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16272 
16273   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # USE sve" %}
16274   ins_encode %{
16275     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16276     __ string_compare($str1$$Register, $str2$$Register,
16277                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16278                       $tmp1$$Register, $tmp2$$Register,
16279                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16280                       as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16281                       StrIntrinsicNode::LL);
16282   %}
16283   ins_pipe(pipe_class_memory);
16284 %}
16285 
16286 instruct string_compareLU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16287                               iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16288                               vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16289                               pRegGov_P1 pgtmp2, rFlagsReg cr)
16290 %{
16291   predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
16292   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16293   effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16294          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16295 
16296   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # USE sve" %}
16297   ins_encode %{
16298     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16299     __ string_compare($str1$$Register, $str2$$Register,
16300                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16301                       $tmp1$$Register, $tmp2$$Register,
16302                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16303                       as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16304                       StrIntrinsicNode::LU);
16305   %}
16306   ins_pipe(pipe_class_memory);
16307 %}
16308 
16309 instruct string_compareUL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16310                               iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16311                               vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16312                               pRegGov_P1 pgtmp2, rFlagsReg cr)
16313 %{
16314   predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
16315   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16316   effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16317          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16318 
16319   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # USE sve" %}
16320   ins_encode %{
16321     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16322     __ string_compare($str1$$Register, $str2$$Register,
16323                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16324                       $tmp1$$Register, $tmp2$$Register,
16325                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16326                       as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16327                       StrIntrinsicNode::UL);
16328   %}
16329   ins_pipe(pipe_class_memory);
16330 %}
16331 
16332 instruct string_compareUU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16333                               iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16334                               vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16335                               pRegGov_P1 pgtmp2, rFlagsReg cr)
16336 %{
16337   predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
16338   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16339   effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16340          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16341 
16342   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # USE sve" %}
16343   ins_encode %{
16344     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16345     __ string_compare($str1$$Register, $str2$$Register,
16346                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16347                       $tmp1$$Register, $tmp2$$Register,
16348                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16349                       as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16350                       StrIntrinsicNode::UU);
16351   %}
16352   ins_pipe(pipe_class_memory);
16353 %}
16354 
16355 instruct string_indexofUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16356                           iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16357                           iRegINoSp tmp3, iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
16358                           vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
16359 %{
16360   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16361   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16362   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16363          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
16364          TEMP vtmp0, TEMP vtmp1, KILL cr);
16365   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UU) "
16366             "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
16367 
16368   ins_encode %{
16369     __ string_indexof($str1$$Register, $str2$$Register,
16370                       $cnt1$$Register, $cnt2$$Register,
16371                       $tmp1$$Register, $tmp2$$Register,
16372                       $tmp3$$Register, $tmp4$$Register,
16373                       $tmp5$$Register, $tmp6$$Register,
16374                       -1, $result$$Register, StrIntrinsicNode::UU);
16375   %}
16376   ins_pipe(pipe_class_memory);
16377 %}
16378 
16379 instruct string_indexofLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16380                           iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
16381                           iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
16382                           vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
16383 %{
16384   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16385   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16386   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16387          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
16388          TEMP vtmp0, TEMP vtmp1, KILL cr);
16389   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (LL) "
16390             "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
16391 
16392   ins_encode %{
16393     __ string_indexof($str1$$Register, $str2$$Register,
16394                       $cnt1$$Register, $cnt2$$Register,
16395                       $tmp1$$Register, $tmp2$$Register,
16396                       $tmp3$$Register, $tmp4$$Register,
16397                       $tmp5$$Register, $tmp6$$Register,
16398                       -1, $result$$Register, StrIntrinsicNode::LL);
16399   %}
16400   ins_pipe(pipe_class_memory);
16401 %}
16402 
16403 instruct string_indexofUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16404                           iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,iRegINoSp tmp3,
16405                           iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
16406                           vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
16407 %{
16408   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16409   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16410   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16411          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5,
16412          TEMP tmp6, TEMP vtmp0, TEMP vtmp1, KILL cr);
16413   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UL) "
16414             "# KILL $str1 cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
16415 
16416   ins_encode %{
16417     __ string_indexof($str1$$Register, $str2$$Register,
16418                       $cnt1$$Register, $cnt2$$Register,
16419                       $tmp1$$Register, $tmp2$$Register,
16420                       $tmp3$$Register, $tmp4$$Register,
16421                       $tmp5$$Register, $tmp6$$Register,
16422                       -1, $result$$Register, StrIntrinsicNode::UL);
16423   %}
16424   ins_pipe(pipe_class_memory);
16425 %}
16426 
16427 instruct string_indexof_conUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16428                               immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16429                               iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16430 %{
16431   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16432   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16433   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16434          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16435   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UU) "
16436             "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16437 
16438   ins_encode %{
16439     int icnt2 = (int)$int_cnt2$$constant;
16440     __ string_indexof($str1$$Register, $str2$$Register,
16441                       $cnt1$$Register, zr,
16442                       $tmp1$$Register, $tmp2$$Register,
16443                       $tmp3$$Register, $tmp4$$Register, zr, zr,
16444                       icnt2, $result$$Register, StrIntrinsicNode::UU);
16445   %}
16446   ins_pipe(pipe_class_memory);
16447 %}
16448 
16449 instruct string_indexof_conLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16450                               immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16451                               iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16452 %{
16453   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16454   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16455   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16456          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16457   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (LL) "
16458             "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16459 
16460   ins_encode %{
16461     int icnt2 = (int)$int_cnt2$$constant;
16462     __ string_indexof($str1$$Register, $str2$$Register,
16463                       $cnt1$$Register, zr,
16464                       $tmp1$$Register, $tmp2$$Register,
16465                       $tmp3$$Register, $tmp4$$Register, zr, zr,
16466                       icnt2, $result$$Register, StrIntrinsicNode::LL);
16467   %}
16468   ins_pipe(pipe_class_memory);
16469 %}
16470 
16471 instruct string_indexof_conUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16472                               immI_1 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16473                               iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16474 %{
16475   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16476   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16477   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16478          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16479   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UL) "
16480             "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16481 
16482   ins_encode %{
16483     int icnt2 = (int)$int_cnt2$$constant;
16484     __ string_indexof($str1$$Register, $str2$$Register,
16485                       $cnt1$$Register, zr,
16486                       $tmp1$$Register, $tmp2$$Register,
16487                       $tmp3$$Register, $tmp4$$Register, zr, zr,
16488                       icnt2, $result$$Register, StrIntrinsicNode::UL);
16489   %}
16490   ins_pipe(pipe_class_memory);
16491 %}
16492 
16493 instruct string_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16494                              iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16495                              iRegINoSp tmp3, rFlagsReg cr)
16496 %{
16497   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16498   predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
16499   effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16500          TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16501 
16502   format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16503 
16504   ins_encode %{
16505     __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16506                            $result$$Register, $tmp1$$Register, $tmp2$$Register,
16507                            $tmp3$$Register);
16508   %}
16509   ins_pipe(pipe_class_memory);
16510 %}
16511 
16512 instruct stringL_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16513                               iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16514                               iRegINoSp tmp3, rFlagsReg cr)
16515 %{
16516   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16517   predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
16518   effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16519          TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16520 
16521   format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16522 
16523   ins_encode %{
16524     __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16525                             $result$$Register, $tmp1$$Register, $tmp2$$Register,
16526                             $tmp3$$Register);
16527   %}
16528   ins_pipe(pipe_class_memory);
16529 %}
16530 
16531 instruct stringL_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16532                                   iRegI_R0 result, vecA ztmp1, vecA ztmp2,
16533                                   pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
16534   predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L);
16535   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16536   effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
16537   format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
16538   ins_encode %{
16539     __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
16540                                $result$$Register, $ztmp1$$FloatRegister,
16541                                $ztmp2$$FloatRegister, $pgtmp$$PRegister,
16542                                $ptmp$$PRegister, true /* isL */);
16543   %}
16544   ins_pipe(pipe_class_memory);
16545 %}
16546 
16547 instruct stringU_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16548                                   iRegI_R0 result, vecA ztmp1, vecA ztmp2,
16549                                   pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
16550   predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U);
16551   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16552   effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
16553   format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
16554   ins_encode %{
16555     __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
16556                                $result$$Register, $ztmp1$$FloatRegister,
16557                                $ztmp2$$FloatRegister, $pgtmp$$PRegister,
16558                                $ptmp$$PRegister, false /* isL */);
16559   %}
16560   ins_pipe(pipe_class_memory);
16561 %}
16562 
16563 instruct string_equalsL(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
16564                         iRegI_R0 result, rFlagsReg cr)
16565 %{
16566   predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::LL);
16567   match(Set result (StrEquals (Binary str1 str2) cnt));
16568   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
16569 
16570   format %{ "String Equals $str1,$str2,$cnt -> $result" %}
16571   ins_encode %{
16572     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16573     __ string_equals($str1$$Register, $str2$$Register,
16574                      $result$$Register, $cnt$$Register);
16575   %}
16576   ins_pipe(pipe_class_memory);
16577 %}
16578 
16579 instruct array_equalsB(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16580                        iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16581                        vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16582                        vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16583                        iRegP_R10 tmp, rFlagsReg cr)
16584 %{
16585   predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
16586   match(Set result (AryEq ary1 ary2));
16587   effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
16588          TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16589          TEMP vtmp6, TEMP vtmp7, KILL cr);
16590 
16591   format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
16592   ins_encode %{
16593     address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
16594                                    $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
16595                                    $result$$Register, $tmp$$Register, 1);
16596     if (tpc == nullptr) {
16597       ciEnv::current()->record_failure("CodeCache is full");
16598       return;
16599     }
16600   %}
16601   ins_pipe(pipe_class_memory);
16602 %}
16603 
16604 instruct array_equalsC(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16605                        iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16606                        vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16607                        vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16608                        iRegP_R10 tmp, rFlagsReg cr)
16609 %{
16610   predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
16611   match(Set result (AryEq ary1 ary2));
16612   effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
16613          TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16614          TEMP vtmp6, TEMP vtmp7, KILL cr);
16615 
16616   format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
16617   ins_encode %{
16618     address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
16619                                    $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
16620                                    $result$$Register, $tmp$$Register, 2);
16621     if (tpc == nullptr) {
16622       ciEnv::current()->record_failure("CodeCache is full");
16623       return;
16624     }
16625   %}
16626   ins_pipe(pipe_class_memory);
16627 %}
16628 
16629 instruct count_positives(iRegP_R1 ary1, iRegI_R2 len, iRegI_R0 result, rFlagsReg cr)
16630 %{
16631   match(Set result (CountPositives ary1 len));
16632   effect(USE_KILL ary1, USE_KILL len, KILL cr);
16633   format %{ "count positives byte[] $ary1,$len -> $result" %}
16634   ins_encode %{
16635     address tpc = __ count_positives($ary1$$Register, $len$$Register, $result$$Register);
16636     if (tpc == nullptr) {
16637       ciEnv::current()->record_failure("CodeCache is full");
16638       return;
16639     }
16640   %}
16641   ins_pipe( pipe_slow );
16642 %}
16643 
16644 // fast char[] to byte[] compression
16645 instruct string_compress(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16646                          vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16647                          vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16648                          iRegI_R0 result, rFlagsReg cr)
16649 %{
16650   match(Set result (StrCompressedCopy src (Binary dst len)));
16651   effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16652          USE_KILL src, USE_KILL dst, USE len, KILL cr);
16653 
16654   format %{ "String Compress $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16655   ins_encode %{
16656     __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
16657                            $result$$Register, $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16658                            $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
16659                            $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
16660   %}
16661   ins_pipe(pipe_slow);
16662 %}
16663 
16664 // fast byte[] to char[] inflation
16665 instruct string_inflate(Universe dummy, iRegP_R0 src, iRegP_R1 dst, iRegI_R2 len, iRegP_R3 tmp,
16666                         vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16667                         vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, rFlagsReg cr)
16668 %{
16669   match(Set dummy (StrInflatedCopy src (Binary dst len)));
16670   effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3,
16671          TEMP vtmp4, TEMP vtmp5, TEMP vtmp6, TEMP tmp,
16672          USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
16673 
16674   format %{ "String Inflate $src,$dst # KILL $tmp $src $dst $len V0-V6 cr" %}
16675   ins_encode %{
16676     address tpc = __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
16677                                         $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16678                                         $vtmp2$$FloatRegister, $tmp$$Register);
16679     if (tpc == nullptr) {
16680       ciEnv::current()->record_failure("CodeCache is full");
16681       return;
16682     }
16683   %}
16684   ins_pipe(pipe_class_memory);
16685 %}
16686 
16687 // encode char[] to byte[] in ISO_8859_1
16688 instruct encode_iso_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16689                           vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16690                           vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16691                           iRegI_R0 result, rFlagsReg cr)
16692 %{
16693   predicate(!((EncodeISOArrayNode*)n)->is_ascii());
16694   match(Set result (EncodeISOArray src (Binary dst len)));
16695   effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
16696          KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
16697 
16698   format %{ "Encode ISO array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16699   ins_encode %{
16700     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
16701                         $result$$Register, false,
16702                         $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16703                         $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
16704                         $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
16705   %}
16706   ins_pipe(pipe_class_memory);
16707 %}
16708 
16709 instruct encode_ascii_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16710                             vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16711                             vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16712                             iRegI_R0 result, rFlagsReg cr)
16713 %{
16714   predicate(((EncodeISOArrayNode*)n)->is_ascii());
16715   match(Set result (EncodeISOArray src (Binary dst len)));
16716   effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
16717          KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
16718 
16719   format %{ "Encode ASCII array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16720   ins_encode %{
16721     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
16722                         $result$$Register, true,
16723                         $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16724                         $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
16725                         $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
16726   %}
16727   ins_pipe(pipe_class_memory);
16728 %}
16729 
16730 //----------------------------- CompressBits/ExpandBits ------------------------
16731 
16732 instruct compressBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
16733                            vRegF tdst, vRegF tsrc, vRegF tmask) %{
16734   match(Set dst (CompressBits src mask));
16735   effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16736   format %{ "mov    $tsrc, $src\n\t"
16737             "mov    $tmask, $mask\n\t"
16738             "bext   $tdst, $tsrc, $tmask\n\t"
16739             "mov    $dst, $tdst"
16740           %}
16741   ins_encode %{
16742     __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
16743     __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
16744     __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16745     __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16746   %}
16747   ins_pipe(pipe_slow);
16748 %}
16749 
16750 instruct compressBitsI_memcon(iRegINoSp dst, memory mem, immI mask,
16751                            vRegF tdst, vRegF tsrc, vRegF tmask) %{
16752   match(Set dst (CompressBits (LoadI mem) mask));
16753   effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16754   format %{ "ldrs   $tsrc, $mem\n\t"
16755             "ldrs   $tmask, $mask\n\t"
16756             "bext   $tdst, $tsrc, $tmask\n\t"
16757             "mov    $dst, $tdst"
16758           %}
16759   ins_encode %{
16760     loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
16761               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
16762     __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
16763     __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16764     __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16765   %}
16766   ins_pipe(pipe_slow);
16767 %}
16768 
16769 instruct compressBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
16770                            vRegD tdst, vRegD tsrc, vRegD tmask) %{
16771   match(Set dst (CompressBits src mask));
16772   effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16773   format %{ "mov    $tsrc, $src\n\t"
16774             "mov    $tmask, $mask\n\t"
16775             "bext   $tdst, $tsrc, $tmask\n\t"
16776             "mov    $dst, $tdst"
16777           %}
16778   ins_encode %{
16779     __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
16780     __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
16781     __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16782     __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16783   %}
16784   ins_pipe(pipe_slow);
16785 %}
16786 
16787 instruct compressBitsL_memcon(iRegLNoSp dst, memory mem, immL mask,
16788                            vRegF tdst, vRegF tsrc, vRegF tmask) %{
16789   match(Set dst (CompressBits (LoadL mem) mask));
16790   effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16791   format %{ "ldrd   $tsrc, $mem\n\t"
16792             "ldrd   $tmask, $mask\n\t"
16793             "bext   $tdst, $tsrc, $tmask\n\t"
16794             "mov    $dst, $tdst"
16795           %}
16796   ins_encode %{
16797     loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
16798               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
16799     __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
16800     __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16801     __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16802   %}
16803   ins_pipe(pipe_slow);
16804 %}
16805 
16806 instruct expandBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
16807                          vRegF tdst, vRegF tsrc, vRegF tmask) %{
16808   match(Set dst (ExpandBits src mask));
16809   effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16810   format %{ "mov    $tsrc, $src\n\t"
16811             "mov    $tmask, $mask\n\t"
16812             "bdep   $tdst, $tsrc, $tmask\n\t"
16813             "mov    $dst, $tdst"
16814           %}
16815   ins_encode %{
16816     __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
16817     __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
16818     __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16819     __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16820   %}
16821   ins_pipe(pipe_slow);
16822 %}
16823 
16824 instruct expandBitsI_memcon(iRegINoSp dst, memory mem, immI mask,
16825                          vRegF tdst, vRegF tsrc, vRegF tmask) %{
16826   match(Set dst (ExpandBits (LoadI mem) mask));
16827   effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16828   format %{ "ldrs   $tsrc, $mem\n\t"
16829             "ldrs   $tmask, $mask\n\t"
16830             "bdep   $tdst, $tsrc, $tmask\n\t"
16831             "mov    $dst, $tdst"
16832           %}
16833   ins_encode %{
16834     loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
16835               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
16836     __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
16837     __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16838     __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16839   %}
16840   ins_pipe(pipe_slow);
16841 %}
16842 
16843 instruct expandBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
16844                          vRegD tdst, vRegD tsrc, vRegD tmask) %{
16845   match(Set dst (ExpandBits src mask));
16846   effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16847   format %{ "mov    $tsrc, $src\n\t"
16848             "mov    $tmask, $mask\n\t"
16849             "bdep   $tdst, $tsrc, $tmask\n\t"
16850             "mov    $dst, $tdst"
16851           %}
16852   ins_encode %{
16853     __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
16854     __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
16855     __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16856     __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16857   %}
16858   ins_pipe(pipe_slow);
16859 %}
16860 
16861 
16862 instruct expandBitsL_memcon(iRegINoSp dst, memory mem, immL mask,
16863                          vRegF tdst, vRegF tsrc, vRegF tmask) %{
16864   match(Set dst (ExpandBits (LoadL mem) mask));
16865   effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16866   format %{ "ldrd   $tsrc, $mem\n\t"
16867             "ldrd   $tmask, $mask\n\t"
16868             "bdep   $tdst, $tsrc, $tmask\n\t"
16869             "mov    $dst, $tdst"
16870           %}
16871   ins_encode %{
16872     loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
16873               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
16874     __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
16875     __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16876     __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16877   %}
16878   ins_pipe(pipe_slow);
16879 %}
16880 
16881 // ============================================================================
16882 // This name is KNOWN by the ADLC and cannot be changed.
16883 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
16884 // for this guy.
16885 instruct tlsLoadP(thread_RegP dst)
16886 %{
16887   match(Set dst (ThreadLocal));
16888 
16889   ins_cost(0);
16890 
16891   format %{ " -- \t// $dst=Thread::current(), empty" %}
16892 
16893   size(0);
16894 
16895   ins_encode( /*empty*/ );
16896 
16897   ins_pipe(pipe_class_empty);
16898 %}
16899 
16900 //----------PEEPHOLE RULES-----------------------------------------------------
16901 // These must follow all instruction definitions as they use the names
16902 // defined in the instructions definitions.
16903 //
16904 // peepmatch ( root_instr_name [preceding_instruction]* );
16905 //
16906 // peepconstraint %{
16907 // (instruction_number.operand_name relational_op instruction_number.operand_name
16908 //  [, ...] );
16909 // // instruction numbers are zero-based using left to right order in peepmatch
16910 //
16911 // peepreplace ( instr_name  ( [instruction_number.operand_name]* ) );
16912 // // provide an instruction_number.operand_name for each operand that appears
16913 // // in the replacement instruction's match rule
16914 //
16915 // ---------VM FLAGS---------------------------------------------------------
16916 //
16917 // All peephole optimizations can be turned off using -XX:-OptoPeephole
16918 //
16919 // Each peephole rule is given an identifying number starting with zero and
16920 // increasing by one in the order seen by the parser.  An individual peephole
16921 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
16922 // on the command-line.
16923 //
16924 // ---------CURRENT LIMITATIONS----------------------------------------------
16925 //
16926 // Only match adjacent instructions in same basic block
16927 // Only equality constraints
16928 // Only constraints between operands, not (0.dest_reg == RAX_enc)
16929 // Only one replacement instruction
16930 //
16931 // ---------EXAMPLE----------------------------------------------------------
16932 //
16933 // // pertinent parts of existing instructions in architecture description
16934 // instruct movI(iRegINoSp dst, iRegI src)
16935 // %{
16936 //   match(Set dst (CopyI src));
16937 // %}
16938 //
16939 // instruct incI_iReg(iRegINoSp dst, immI1 src, rFlagsReg cr)
16940 // %{
16941 //   match(Set dst (AddI dst src));
16942 //   effect(KILL cr);
16943 // %}
16944 //
16945 // // Change (inc mov) to lea
16946 // peephole %{
16947 //   // increment preceded by register-register move
16948 //   peepmatch ( incI_iReg movI );
16949 //   // require that the destination register of the increment
16950 //   // match the destination register of the move
16951 //   peepconstraint ( 0.dst == 1.dst );
16952 //   // construct a replacement instruction that sets
16953 //   // the destination to ( move's source register + one )
16954 //   peepreplace ( leaI_iReg_immI( 0.dst 1.src 0.src ) );
16955 // %}
16956 //
16957 
16958 // Implementation no longer uses movX instructions since
16959 // machine-independent system no longer uses CopyX nodes.
16960 //
16961 // peephole
16962 // %{
16963 //   peepmatch (incI_iReg movI);
16964 //   peepconstraint (0.dst == 1.dst);
16965 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
16966 // %}
16967 
16968 // peephole
16969 // %{
16970 //   peepmatch (decI_iReg movI);
16971 //   peepconstraint (0.dst == 1.dst);
16972 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
16973 // %}
16974 
16975 // peephole
16976 // %{
16977 //   peepmatch (addI_iReg_imm movI);
16978 //   peepconstraint (0.dst == 1.dst);
16979 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
16980 // %}
16981 
16982 // peephole
16983 // %{
16984 //   peepmatch (incL_iReg movL);
16985 //   peepconstraint (0.dst == 1.dst);
16986 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
16987 // %}
16988 
16989 // peephole
16990 // %{
16991 //   peepmatch (decL_iReg movL);
16992 //   peepconstraint (0.dst == 1.dst);
16993 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
16994 // %}
16995 
16996 // peephole
16997 // %{
16998 //   peepmatch (addL_iReg_imm movL);
16999 //   peepconstraint (0.dst == 1.dst);
17000 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17001 // %}
17002 
17003 // peephole
17004 // %{
17005 //   peepmatch (addP_iReg_imm movP);
17006 //   peepconstraint (0.dst == 1.dst);
17007 //   peepreplace (leaP_iReg_imm(0.dst 1.src 0.src));
17008 // %}
17009 
17010 // // Change load of spilled value to only a spill
17011 // instruct storeI(memory mem, iRegI src)
17012 // %{
17013 //   match(Set mem (StoreI mem src));
17014 // %}
17015 //
17016 // instruct loadI(iRegINoSp dst, memory mem)
17017 // %{
17018 //   match(Set dst (LoadI mem));
17019 // %}
17020 //
17021 
17022 //----------SMARTSPILL RULES---------------------------------------------------
17023 // These must follow all instruction definitions as they use the names
17024 // defined in the instructions definitions.
17025 
17026 // Local Variables:
17027 // mode: c++
17028 // End: