1 //
    2 // Copyright (c) 2003, 2024, Oracle and/or its affiliates. All rights reserved.
    3 // Copyright (c) 2014, 2024, Red Hat, Inc. All rights reserved.
    4 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
    5 //
    6 // This code is free software; you can redistribute it and/or modify it
    7 // under the terms of the GNU General Public License version 2 only, as
    8 // published by the Free Software Foundation.
    9 //
   10 // This code is distributed in the hope that it will be useful, but WITHOUT
   11 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
   12 // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
   13 // version 2 for more details (a copy is included in the LICENSE file that
   14 // accompanied this code).
   15 //
   16 // You should have received a copy of the GNU General Public License version
   17 // 2 along with this work; if not, write to the Free Software Foundation,
   18 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
   19 //
   20 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
   21 // or visit www.oracle.com if you need additional information or have any
   22 // questions.
   23 //
   24 //
   25 
   26 // AArch64 Architecture Description File
   27 
   28 //----------REGISTER DEFINITION BLOCK------------------------------------------
   29 // This information is used by the matcher and the register allocator to
   30 // describe individual registers and classes of registers within the target
   31 // architecture.
   32 
   33 register %{
   34 //----------Architecture Description Register Definitions----------------------
   35 // General Registers
   36 // "reg_def"  name ( register save type, C convention save type,
   37 //                   ideal register type, encoding );
   38 // Register Save Types:
   39 //
   40 // NS  = No-Save:       The register allocator assumes that these registers
   41 //                      can be used without saving upon entry to the method, &
   42 //                      that they do not need to be saved at call sites.
   43 //
   44 // SOC = Save-On-Call:  The register allocator assumes that these registers
   45 //                      can be used without saving upon entry to the method,
   46 //                      but that they must be saved at call sites.
   47 //
   48 // SOE = Save-On-Entry: The register allocator assumes that these registers
   49 //                      must be saved before using them upon entry to the
   50 //                      method, but they do not need to be saved at call
   51 //                      sites.
   52 //
   53 // AS  = Always-Save:   The register allocator assumes that these registers
   54 //                      must be saved before using them upon entry to the
   55 //                      method, & that they must be saved at call sites.
   56 //
   57 // Ideal Register Type is used to determine how to save & restore a
   58 // register.  Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
   59 // spilled with LoadP/StoreP.  If the register supports both, use Op_RegI.
   60 //
   61 // The encoding number is the actual bit-pattern placed into the opcodes.
   62 
   63 // We must define the 64 bit int registers in two 32 bit halves, the
   64 // real lower register and a virtual upper half register. upper halves
   65 // are used by the register allocator but are not actually supplied as
   66 // operands to memory ops.
   67 //
   68 // follow the C1 compiler in making registers
   69 //
   70 //   r0-r7,r10-r26 volatile (caller save)
   71 //   r27-r32 system (no save, no allocate)
   72 //   r8-r9 non-allocatable (so we can use them as scratch regs)
   73 //
   74 // as regards Java usage. we don't use any callee save registers
   75 // because this makes it difficult to de-optimise a frame (see comment
   76 // in x86 implementation of Deoptimization::unwind_callee_save_values)
   77 //
   78 
   79 // General Registers
   80 
   81 reg_def R0      ( SOC, SOC, Op_RegI,  0, r0->as_VMReg()         );
   82 reg_def R0_H    ( SOC, SOC, Op_RegI,  0, r0->as_VMReg()->next() );
   83 reg_def R1      ( SOC, SOC, Op_RegI,  1, r1->as_VMReg()         );
   84 reg_def R1_H    ( SOC, SOC, Op_RegI,  1, r1->as_VMReg()->next() );
   85 reg_def R2      ( SOC, SOC, Op_RegI,  2, r2->as_VMReg()         );
   86 reg_def R2_H    ( SOC, SOC, Op_RegI,  2, r2->as_VMReg()->next() );
   87 reg_def R3      ( SOC, SOC, Op_RegI,  3, r3->as_VMReg()         );
   88 reg_def R3_H    ( SOC, SOC, Op_RegI,  3, r3->as_VMReg()->next() );
   89 reg_def R4      ( SOC, SOC, Op_RegI,  4, r4->as_VMReg()         );
   90 reg_def R4_H    ( SOC, SOC, Op_RegI,  4, r4->as_VMReg()->next() );
   91 reg_def R5      ( SOC, SOC, Op_RegI,  5, r5->as_VMReg()         );
   92 reg_def R5_H    ( SOC, SOC, Op_RegI,  5, r5->as_VMReg()->next() );
   93 reg_def R6      ( SOC, SOC, Op_RegI,  6, r6->as_VMReg()         );
   94 reg_def R6_H    ( SOC, SOC, Op_RegI,  6, r6->as_VMReg()->next() );
   95 reg_def R7      ( SOC, SOC, Op_RegI,  7, r7->as_VMReg()         );
   96 reg_def R7_H    ( SOC, SOC, Op_RegI,  7, r7->as_VMReg()->next() );
   97 reg_def R8      ( NS,  SOC, Op_RegI,  8, r8->as_VMReg()         ); // rscratch1, non-allocatable
   98 reg_def R8_H    ( NS,  SOC, Op_RegI,  8, r8->as_VMReg()->next() );
   99 reg_def R9      ( NS,  SOC, Op_RegI,  9, r9->as_VMReg()         ); // rscratch2, non-allocatable
  100 reg_def R9_H    ( NS,  SOC, Op_RegI,  9, r9->as_VMReg()->next() );
  101 reg_def R10     ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()        );
  102 reg_def R10_H   ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
  103 reg_def R11     ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()        );
  104 reg_def R11_H   ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
  105 reg_def R12     ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()        );
  106 reg_def R12_H   ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()->next());
  107 reg_def R13     ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()        );
  108 reg_def R13_H   ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()->next());
  109 reg_def R14     ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()        );
  110 reg_def R14_H   ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()->next());
  111 reg_def R15     ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()        );
  112 reg_def R15_H   ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()->next());
  113 reg_def R16     ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()        );
  114 reg_def R16_H   ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
  115 reg_def R17     ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()        );
  116 reg_def R17_H   ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
  117 reg_def R18     ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()        );
  118 reg_def R18_H   ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()->next());
  119 reg_def R19     ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()        );
  120 reg_def R19_H   ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()->next());
  121 reg_def R20     ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()        ); // caller esp
  122 reg_def R20_H   ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()->next());
  123 reg_def R21     ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()        );
  124 reg_def R21_H   ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()->next());
  125 reg_def R22     ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()        );
  126 reg_def R22_H   ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()->next());
  127 reg_def R23     ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()        );
  128 reg_def R23_H   ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()->next());
  129 reg_def R24     ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()        );
  130 reg_def R24_H   ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()->next());
  131 reg_def R25     ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()        );
  132 reg_def R25_H   ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()->next());
  133 reg_def R26     ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()        );
  134 reg_def R26_H   ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()->next());
  135 reg_def R27     ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()        ); // heapbase
  136 reg_def R27_H   ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()->next());
  137 reg_def R28     (  NS, SOE, Op_RegI, 28, r28->as_VMReg()        ); // thread
  138 reg_def R28_H   (  NS, SOE, Op_RegI, 28, r28->as_VMReg()->next());
  139 reg_def R29     (  NS,  NS, Op_RegI, 29, r29->as_VMReg()        ); // fp
  140 reg_def R29_H   (  NS,  NS, Op_RegI, 29, r29->as_VMReg()->next());
  141 reg_def R30     (  NS,  NS, Op_RegI, 30, r30->as_VMReg()        ); // lr
  142 reg_def R30_H   (  NS,  NS, Op_RegI, 30, r30->as_VMReg()->next());
  143 reg_def R31     (  NS,  NS, Op_RegI, 31, r31_sp->as_VMReg()     ); // sp
  144 reg_def R31_H   (  NS,  NS, Op_RegI, 31, r31_sp->as_VMReg()->next());
  145 
  146 // ----------------------------
  147 // Float/Double/Vector Registers
  148 // ----------------------------
  149 
  150 // Double Registers
  151 
  152 // The rules of ADL require that double registers be defined in pairs.
  153 // Each pair must be two 32-bit values, but not necessarily a pair of
  154 // single float registers. In each pair, ADLC-assigned register numbers
  155 // must be adjacent, with the lower number even. Finally, when the
  156 // CPU stores such a register pair to memory, the word associated with
  157 // the lower ADLC-assigned number must be stored to the lower address.
  158 
  159 // AArch64 has 32 floating-point registers. Each can store a vector of
  160 // single or double precision floating-point values up to 8 * 32
  161 // floats, 4 * 64 bit floats or 2 * 128 bit floats.  We currently only
  162 // use the first float or double element of the vector.
  163 
  164 // for Java use float registers v0-v15 are always save on call whereas
  165 // the platform ABI treats v8-v15 as callee save). float registers
  166 // v16-v31 are SOC as per the platform spec
  167 
  168 // For SVE vector registers, we simply extend vector register size to 8
  169 // 'logical' slots. This is nominally 256 bits but it actually covers
  170 // all possible 'physical' SVE vector register lengths from 128 ~ 2048
  171 // bits. The 'physical' SVE vector register length is detected during
  172 // startup, so the register allocator is able to identify the correct
  173 // number of bytes needed for an SVE spill/unspill.
  174 // Note that a vector register with 4 slots denotes a 128-bit NEON
  175 // register allowing it to be distinguished from the corresponding SVE
  176 // vector register when the SVE vector length is 128 bits.
  177 
  178   reg_def V0   ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()          );
  179   reg_def V0_H ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next()  );
  180   reg_def V0_J ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(2) );
  181   reg_def V0_K ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(3) );
  182 
  183   reg_def V1   ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()          );
  184   reg_def V1_H ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next()  );
  185   reg_def V1_J ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(2) );
  186   reg_def V1_K ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(3) );
  187 
  188   reg_def V2   ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()          );
  189   reg_def V2_H ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next()  );
  190   reg_def V2_J ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(2) );
  191   reg_def V2_K ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(3) );
  192 
  193   reg_def V3   ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()          );
  194   reg_def V3_H ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next()  );
  195   reg_def V3_J ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(2) );
  196   reg_def V3_K ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(3) );
  197 
  198   reg_def V4   ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()          );
  199   reg_def V4_H ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next()  );
  200   reg_def V4_J ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(2) );
  201   reg_def V4_K ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(3) );
  202 
  203   reg_def V5   ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()          );
  204   reg_def V5_H ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next()  );
  205   reg_def V5_J ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(2) );
  206   reg_def V5_K ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(3) );
  207 
  208   reg_def V6   ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()          );
  209   reg_def V6_H ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next()  );
  210   reg_def V6_J ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(2) );
  211   reg_def V6_K ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(3) );
  212 
  213   reg_def V7   ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()          );
  214   reg_def V7_H ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next()  );
  215   reg_def V7_J ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(2) );
  216   reg_def V7_K ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(3) );
  217 
  218   reg_def V8   ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()          );
  219   reg_def V8_H ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()->next()  );
  220   reg_def V8_J ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(2) );
  221   reg_def V8_K ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(3) );
  222 
  223   reg_def V9   ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()          );
  224   reg_def V9_H ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()->next()  );
  225   reg_def V9_J ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(2) );
  226   reg_def V9_K ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(3) );
  227 
  228   reg_def V10   ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()          );
  229   reg_def V10_H ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()->next()  );
  230   reg_def V10_J ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(2) );
  231   reg_def V10_K ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(3) );
  232 
  233   reg_def V11   ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()          );
  234   reg_def V11_H ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()->next()  );
  235   reg_def V11_J ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(2) );
  236   reg_def V11_K ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(3) );
  237 
  238   reg_def V12   ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()          );
  239   reg_def V12_H ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()->next()  );
  240   reg_def V12_J ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(2) );
  241   reg_def V12_K ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(3) );
  242 
  243   reg_def V13   ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()          );
  244   reg_def V13_H ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()->next()  );
  245   reg_def V13_J ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(2) );
  246   reg_def V13_K ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(3) );
  247 
  248   reg_def V14   ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()          );
  249   reg_def V14_H ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()->next()  );
  250   reg_def V14_J ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(2) );
  251   reg_def V14_K ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(3) );
  252 
  253   reg_def V15   ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()          );
  254   reg_def V15_H ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()->next()  );
  255   reg_def V15_J ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(2) );
  256   reg_def V15_K ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(3) );
  257 
  258   reg_def V16   ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()          );
  259   reg_def V16_H ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next()  );
  260   reg_def V16_J ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(2) );
  261   reg_def V16_K ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(3) );
  262 
  263   reg_def V17   ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()          );
  264   reg_def V17_H ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next()  );
  265   reg_def V17_J ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(2) );
  266   reg_def V17_K ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(3) );
  267 
  268   reg_def V18   ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()          );
  269   reg_def V18_H ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next()  );
  270   reg_def V18_J ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(2) );
  271   reg_def V18_K ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(3) );
  272 
  273   reg_def V19   ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()          );
  274   reg_def V19_H ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next()  );
  275   reg_def V19_J ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(2) );
  276   reg_def V19_K ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(3) );
  277 
  278   reg_def V20   ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()          );
  279   reg_def V20_H ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next()  );
  280   reg_def V20_J ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(2) );
  281   reg_def V20_K ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(3) );
  282 
  283   reg_def V21   ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()          );
  284   reg_def V21_H ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next()  );
  285   reg_def V21_J ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(2) );
  286   reg_def V21_K ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(3) );
  287 
  288   reg_def V22   ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()          );
  289   reg_def V22_H ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next()  );
  290   reg_def V22_J ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(2) );
  291   reg_def V22_K ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(3) );
  292 
  293   reg_def V23   ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()          );
  294   reg_def V23_H ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next()  );
  295   reg_def V23_J ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(2) );
  296   reg_def V23_K ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(3) );
  297 
  298   reg_def V24   ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()          );
  299   reg_def V24_H ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next()  );
  300   reg_def V24_J ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(2) );
  301   reg_def V24_K ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(3) );
  302 
  303   reg_def V25   ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()          );
  304   reg_def V25_H ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next()  );
  305   reg_def V25_J ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(2) );
  306   reg_def V25_K ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(3) );
  307 
  308   reg_def V26   ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()          );
  309   reg_def V26_H ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next()  );
  310   reg_def V26_J ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(2) );
  311   reg_def V26_K ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(3) );
  312 
  313   reg_def V27   ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()          );
  314   reg_def V27_H ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next()  );
  315   reg_def V27_J ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(2) );
  316   reg_def V27_K ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(3) );
  317 
  318   reg_def V28   ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()          );
  319   reg_def V28_H ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next()  );
  320   reg_def V28_J ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(2) );
  321   reg_def V28_K ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(3) );
  322 
  323   reg_def V29   ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()          );
  324   reg_def V29_H ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next()  );
  325   reg_def V29_J ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(2) );
  326   reg_def V29_K ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(3) );
  327 
  328   reg_def V30   ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()          );
  329   reg_def V30_H ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next()  );
  330   reg_def V30_J ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(2) );
  331   reg_def V30_K ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(3) );
  332 
  333   reg_def V31   ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()          );
  334   reg_def V31_H ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next()  );
  335   reg_def V31_J ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(2) );
  336   reg_def V31_K ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(3) );
  337 
  338 // ----------------------------
  339 // SVE Predicate Registers
  340 // ----------------------------
  341   reg_def P0 (SOC, SOC, Op_RegVectMask, 0, p0->as_VMReg());
  342   reg_def P1 (SOC, SOC, Op_RegVectMask, 1, p1->as_VMReg());
  343   reg_def P2 (SOC, SOC, Op_RegVectMask, 2, p2->as_VMReg());
  344   reg_def P3 (SOC, SOC, Op_RegVectMask, 3, p3->as_VMReg());
  345   reg_def P4 (SOC, SOC, Op_RegVectMask, 4, p4->as_VMReg());
  346   reg_def P5 (SOC, SOC, Op_RegVectMask, 5, p5->as_VMReg());
  347   reg_def P6 (SOC, SOC, Op_RegVectMask, 6, p6->as_VMReg());
  348   reg_def P7 (SOC, SOC, Op_RegVectMask, 7, p7->as_VMReg());
  349   reg_def P8 (SOC, SOC, Op_RegVectMask, 8, p8->as_VMReg());
  350   reg_def P9 (SOC, SOC, Op_RegVectMask, 9, p9->as_VMReg());
  351   reg_def P10 (SOC, SOC, Op_RegVectMask, 10, p10->as_VMReg());
  352   reg_def P11 (SOC, SOC, Op_RegVectMask, 11, p11->as_VMReg());
  353   reg_def P12 (SOC, SOC, Op_RegVectMask, 12, p12->as_VMReg());
  354   reg_def P13 (SOC, SOC, Op_RegVectMask, 13, p13->as_VMReg());
  355   reg_def P14 (SOC, SOC, Op_RegVectMask, 14, p14->as_VMReg());
  356   reg_def P15 (SOC, SOC, Op_RegVectMask, 15, p15->as_VMReg());
  357 
  358 // ----------------------------
  359 // Special Registers
  360 // ----------------------------
  361 
  362 // the AArch64 CSPR status flag register is not directly accessible as
  363 // instruction operand. the FPSR status flag register is a system
  364 // register which can be written/read using MSR/MRS but again does not
  365 // appear as an operand (a code identifying the FSPR occurs as an
  366 // immediate value in the instruction).
  367 
  368 reg_def RFLAGS(SOC, SOC, 0, 32, VMRegImpl::Bad());
  369 
  370 // Specify priority of register selection within phases of register
  371 // allocation.  Highest priority is first.  A useful heuristic is to
  372 // give registers a low priority when they are required by machine
  373 // instructions, like EAX and EDX on I486, and choose no-save registers
  374 // before save-on-call, & save-on-call before save-on-entry.  Registers
  375 // which participate in fixed calling sequences should come last.
  376 // Registers which are used as pairs must fall on an even boundary.
  377 
  378 alloc_class chunk0(
  379     // volatiles
  380     R10, R10_H,
  381     R11, R11_H,
  382     R12, R12_H,
  383     R13, R13_H,
  384     R14, R14_H,
  385     R15, R15_H,
  386     R16, R16_H,
  387     R17, R17_H,
  388     R18, R18_H,
  389 
  390     // arg registers
  391     R0, R0_H,
  392     R1, R1_H,
  393     R2, R2_H,
  394     R3, R3_H,
  395     R4, R4_H,
  396     R5, R5_H,
  397     R6, R6_H,
  398     R7, R7_H,
  399 
  400     // non-volatiles
  401     R19, R19_H,
  402     R20, R20_H,
  403     R21, R21_H,
  404     R22, R22_H,
  405     R23, R23_H,
  406     R24, R24_H,
  407     R25, R25_H,
  408     R26, R26_H,
  409 
  410     // non-allocatable registers
  411 
  412     R27, R27_H, // heapbase
  413     R28, R28_H, // thread
  414     R29, R29_H, // fp
  415     R30, R30_H, // lr
  416     R31, R31_H, // sp
  417     R8, R8_H,   // rscratch1
  418     R9, R9_H,   // rscratch2
  419 );
  420 
  421 alloc_class chunk1(
  422 
  423     // no save
  424     V16, V16_H, V16_J, V16_K,
  425     V17, V17_H, V17_J, V17_K,
  426     V18, V18_H, V18_J, V18_K,
  427     V19, V19_H, V19_J, V19_K,
  428     V20, V20_H, V20_J, V20_K,
  429     V21, V21_H, V21_J, V21_K,
  430     V22, V22_H, V22_J, V22_K,
  431     V23, V23_H, V23_J, V23_K,
  432     V24, V24_H, V24_J, V24_K,
  433     V25, V25_H, V25_J, V25_K,
  434     V26, V26_H, V26_J, V26_K,
  435     V27, V27_H, V27_J, V27_K,
  436     V28, V28_H, V28_J, V28_K,
  437     V29, V29_H, V29_J, V29_K,
  438     V30, V30_H, V30_J, V30_K,
  439     V31, V31_H, V31_J, V31_K,
  440 
  441     // arg registers
  442     V0, V0_H, V0_J, V0_K,
  443     V1, V1_H, V1_J, V1_K,
  444     V2, V2_H, V2_J, V2_K,
  445     V3, V3_H, V3_J, V3_K,
  446     V4, V4_H, V4_J, V4_K,
  447     V5, V5_H, V5_J, V5_K,
  448     V6, V6_H, V6_J, V6_K,
  449     V7, V7_H, V7_J, V7_K,
  450 
  451     // non-volatiles
  452     V8, V8_H, V8_J, V8_K,
  453     V9, V9_H, V9_J, V9_K,
  454     V10, V10_H, V10_J, V10_K,
  455     V11, V11_H, V11_J, V11_K,
  456     V12, V12_H, V12_J, V12_K,
  457     V13, V13_H, V13_J, V13_K,
  458     V14, V14_H, V14_J, V14_K,
  459     V15, V15_H, V15_J, V15_K,
  460 );
  461 
  462 alloc_class chunk2 (
  463     // Governing predicates for load/store and arithmetic
  464     P0,
  465     P1,
  466     P2,
  467     P3,
  468     P4,
  469     P5,
  470     P6,
  471 
  472     // Extra predicates
  473     P8,
  474     P9,
  475     P10,
  476     P11,
  477     P12,
  478     P13,
  479     P14,
  480     P15,
  481 
  482     // Preserved for all-true predicate
  483     P7,
  484 );
  485 
  486 alloc_class chunk3(RFLAGS);
  487 
  488 //----------Architecture Description Register Classes--------------------------
  489 // Several register classes are automatically defined based upon information in
  490 // this architecture description.
  491 // 1) reg_class inline_cache_reg           ( /* as def'd in frame section */ )
  492 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
  493 //
  494 
  495 // Class for all 32 bit general purpose registers
  496 reg_class all_reg32(
  497     R0,
  498     R1,
  499     R2,
  500     R3,
  501     R4,
  502     R5,
  503     R6,
  504     R7,
  505     R10,
  506     R11,
  507     R12,
  508     R13,
  509     R14,
  510     R15,
  511     R16,
  512     R17,
  513     R18,
  514     R19,
  515     R20,
  516     R21,
  517     R22,
  518     R23,
  519     R24,
  520     R25,
  521     R26,
  522     R27,
  523     R28,
  524     R29,
  525     R30,
  526     R31
  527 );
  528 
  529 
  530 // Class for all 32 bit integer registers (excluding SP which
  531 // will never be used as an integer register)
  532 reg_class any_reg32 %{
  533   return _ANY_REG32_mask;
  534 %}
  535 
  536 // Singleton class for R0 int register
  537 reg_class int_r0_reg(R0);
  538 
  539 // Singleton class for R2 int register
  540 reg_class int_r2_reg(R2);
  541 
  542 // Singleton class for R3 int register
  543 reg_class int_r3_reg(R3);
  544 
  545 // Singleton class for R4 int register
  546 reg_class int_r4_reg(R4);
  547 
  548 // Singleton class for R31 int register
  549 reg_class int_r31_reg(R31);
  550 
  551 // Class for all 64 bit general purpose registers
  552 reg_class all_reg(
  553     R0, R0_H,
  554     R1, R1_H,
  555     R2, R2_H,
  556     R3, R3_H,
  557     R4, R4_H,
  558     R5, R5_H,
  559     R6, R6_H,
  560     R7, R7_H,
  561     R10, R10_H,
  562     R11, R11_H,
  563     R12, R12_H,
  564     R13, R13_H,
  565     R14, R14_H,
  566     R15, R15_H,
  567     R16, R16_H,
  568     R17, R17_H,
  569     R18, R18_H,
  570     R19, R19_H,
  571     R20, R20_H,
  572     R21, R21_H,
  573     R22, R22_H,
  574     R23, R23_H,
  575     R24, R24_H,
  576     R25, R25_H,
  577     R26, R26_H,
  578     R27, R27_H,
  579     R28, R28_H,
  580     R29, R29_H,
  581     R30, R30_H,
  582     R31, R31_H
  583 );
  584 
  585 // Class for all long integer registers (including SP)
  586 reg_class any_reg %{
  587   return _ANY_REG_mask;
  588 %}
  589 
  590 // Class for non-allocatable 32 bit registers
  591 reg_class non_allocatable_reg32(
  592 #ifdef R18_RESERVED
  593     // See comment in register_aarch64.hpp
  594     R18,                        // tls on Windows
  595 #endif
  596     R28,                        // thread
  597     R30,                        // lr
  598     R31                         // sp
  599 );
  600 
  601 // Class for non-allocatable 64 bit registers
  602 reg_class non_allocatable_reg(
  603 #ifdef R18_RESERVED
  604     // See comment in register_aarch64.hpp
  605     R18, R18_H,                 // tls on Windows, platform register on macOS
  606 #endif
  607     R28, R28_H,                 // thread
  608     R30, R30_H,                 // lr
  609     R31, R31_H                  // sp
  610 );
  611 
  612 // Class for all non-special integer registers
  613 reg_class no_special_reg32 %{
  614   return _NO_SPECIAL_REG32_mask;
  615 %}
  616 
  617 // Class for all non-special long integer registers
  618 reg_class no_special_reg %{
  619   return _NO_SPECIAL_REG_mask;
  620 %}
  621 
  622 // Class for 64 bit register r0
  623 reg_class r0_reg(
  624     R0, R0_H
  625 );
  626 
  627 // Class for 64 bit register r1
  628 reg_class r1_reg(
  629     R1, R1_H
  630 );
  631 
  632 // Class for 64 bit register r2
  633 reg_class r2_reg(
  634     R2, R2_H
  635 );
  636 
  637 // Class for 64 bit register r3
  638 reg_class r3_reg(
  639     R3, R3_H
  640 );
  641 
  642 // Class for 64 bit register r4
  643 reg_class r4_reg(
  644     R4, R4_H
  645 );
  646 
  647 // Class for 64 bit register r5
  648 reg_class r5_reg(
  649     R5, R5_H
  650 );
  651 
  652 // Class for 64 bit register r10
  653 reg_class r10_reg(
  654     R10, R10_H
  655 );
  656 
  657 // Class for 64 bit register r11
  658 reg_class r11_reg(
  659     R11, R11_H
  660 );
  661 
  662 // Class for method register
  663 reg_class method_reg(
  664     R12, R12_H
  665 );
  666 
  667 // Class for thread register
  668 reg_class thread_reg(
  669     R28, R28_H
  670 );
  671 
  672 // Class for frame pointer register
  673 reg_class fp_reg(
  674     R29, R29_H
  675 );
  676 
  677 // Class for link register
  678 reg_class lr_reg(
  679     R30, R30_H
  680 );
  681 
  682 // Class for long sp register
  683 reg_class sp_reg(
  684   R31, R31_H
  685 );
  686 
  687 // Class for all pointer registers
  688 reg_class ptr_reg %{
  689   return _PTR_REG_mask;
  690 %}
  691 
  692 // Class for all non_special pointer registers
  693 reg_class no_special_ptr_reg %{
  694   return _NO_SPECIAL_PTR_REG_mask;
  695 %}
  696 
  697 // Class for all non_special pointer registers (excluding rfp)
  698 reg_class no_special_no_rfp_ptr_reg %{
  699   return _NO_SPECIAL_NO_RFP_PTR_REG_mask;
  700 %}
  701 
  702 // Class for all float registers
  703 reg_class float_reg(
  704     V0,
  705     V1,
  706     V2,
  707     V3,
  708     V4,
  709     V5,
  710     V6,
  711     V7,
  712     V8,
  713     V9,
  714     V10,
  715     V11,
  716     V12,
  717     V13,
  718     V14,
  719     V15,
  720     V16,
  721     V17,
  722     V18,
  723     V19,
  724     V20,
  725     V21,
  726     V22,
  727     V23,
  728     V24,
  729     V25,
  730     V26,
  731     V27,
  732     V28,
  733     V29,
  734     V30,
  735     V31
  736 );
  737 
  738 // Double precision float registers have virtual `high halves' that
  739 // are needed by the allocator.
  740 // Class for all double registers
  741 reg_class double_reg(
  742     V0, V0_H,
  743     V1, V1_H,
  744     V2, V2_H,
  745     V3, V3_H,
  746     V4, V4_H,
  747     V5, V5_H,
  748     V6, V6_H,
  749     V7, V7_H,
  750     V8, V8_H,
  751     V9, V9_H,
  752     V10, V10_H,
  753     V11, V11_H,
  754     V12, V12_H,
  755     V13, V13_H,
  756     V14, V14_H,
  757     V15, V15_H,
  758     V16, V16_H,
  759     V17, V17_H,
  760     V18, V18_H,
  761     V19, V19_H,
  762     V20, V20_H,
  763     V21, V21_H,
  764     V22, V22_H,
  765     V23, V23_H,
  766     V24, V24_H,
  767     V25, V25_H,
  768     V26, V26_H,
  769     V27, V27_H,
  770     V28, V28_H,
  771     V29, V29_H,
  772     V30, V30_H,
  773     V31, V31_H
  774 );
  775 
  776 // Class for all SVE vector registers.
  777 reg_class vectora_reg (
  778     V0, V0_H, V0_J, V0_K,
  779     V1, V1_H, V1_J, V1_K,
  780     V2, V2_H, V2_J, V2_K,
  781     V3, V3_H, V3_J, V3_K,
  782     V4, V4_H, V4_J, V4_K,
  783     V5, V5_H, V5_J, V5_K,
  784     V6, V6_H, V6_J, V6_K,
  785     V7, V7_H, V7_J, V7_K,
  786     V8, V8_H, V8_J, V8_K,
  787     V9, V9_H, V9_J, V9_K,
  788     V10, V10_H, V10_J, V10_K,
  789     V11, V11_H, V11_J, V11_K,
  790     V12, V12_H, V12_J, V12_K,
  791     V13, V13_H, V13_J, V13_K,
  792     V14, V14_H, V14_J, V14_K,
  793     V15, V15_H, V15_J, V15_K,
  794     V16, V16_H, V16_J, V16_K,
  795     V17, V17_H, V17_J, V17_K,
  796     V18, V18_H, V18_J, V18_K,
  797     V19, V19_H, V19_J, V19_K,
  798     V20, V20_H, V20_J, V20_K,
  799     V21, V21_H, V21_J, V21_K,
  800     V22, V22_H, V22_J, V22_K,
  801     V23, V23_H, V23_J, V23_K,
  802     V24, V24_H, V24_J, V24_K,
  803     V25, V25_H, V25_J, V25_K,
  804     V26, V26_H, V26_J, V26_K,
  805     V27, V27_H, V27_J, V27_K,
  806     V28, V28_H, V28_J, V28_K,
  807     V29, V29_H, V29_J, V29_K,
  808     V30, V30_H, V30_J, V30_K,
  809     V31, V31_H, V31_J, V31_K,
  810 );
  811 
  812 // Class for all 64bit vector registers
  813 reg_class vectord_reg(
  814     V0, V0_H,
  815     V1, V1_H,
  816     V2, V2_H,
  817     V3, V3_H,
  818     V4, V4_H,
  819     V5, V5_H,
  820     V6, V6_H,
  821     V7, V7_H,
  822     V8, V8_H,
  823     V9, V9_H,
  824     V10, V10_H,
  825     V11, V11_H,
  826     V12, V12_H,
  827     V13, V13_H,
  828     V14, V14_H,
  829     V15, V15_H,
  830     V16, V16_H,
  831     V17, V17_H,
  832     V18, V18_H,
  833     V19, V19_H,
  834     V20, V20_H,
  835     V21, V21_H,
  836     V22, V22_H,
  837     V23, V23_H,
  838     V24, V24_H,
  839     V25, V25_H,
  840     V26, V26_H,
  841     V27, V27_H,
  842     V28, V28_H,
  843     V29, V29_H,
  844     V30, V30_H,
  845     V31, V31_H
  846 );
  847 
  848 // Class for all 128bit vector registers
  849 reg_class vectorx_reg(
  850     V0, V0_H, V0_J, V0_K,
  851     V1, V1_H, V1_J, V1_K,
  852     V2, V2_H, V2_J, V2_K,
  853     V3, V3_H, V3_J, V3_K,
  854     V4, V4_H, V4_J, V4_K,
  855     V5, V5_H, V5_J, V5_K,
  856     V6, V6_H, V6_J, V6_K,
  857     V7, V7_H, V7_J, V7_K,
  858     V8, V8_H, V8_J, V8_K,
  859     V9, V9_H, V9_J, V9_K,
  860     V10, V10_H, V10_J, V10_K,
  861     V11, V11_H, V11_J, V11_K,
  862     V12, V12_H, V12_J, V12_K,
  863     V13, V13_H, V13_J, V13_K,
  864     V14, V14_H, V14_J, V14_K,
  865     V15, V15_H, V15_J, V15_K,
  866     V16, V16_H, V16_J, V16_K,
  867     V17, V17_H, V17_J, V17_K,
  868     V18, V18_H, V18_J, V18_K,
  869     V19, V19_H, V19_J, V19_K,
  870     V20, V20_H, V20_J, V20_K,
  871     V21, V21_H, V21_J, V21_K,
  872     V22, V22_H, V22_J, V22_K,
  873     V23, V23_H, V23_J, V23_K,
  874     V24, V24_H, V24_J, V24_K,
  875     V25, V25_H, V25_J, V25_K,
  876     V26, V26_H, V26_J, V26_K,
  877     V27, V27_H, V27_J, V27_K,
  878     V28, V28_H, V28_J, V28_K,
  879     V29, V29_H, V29_J, V29_K,
  880     V30, V30_H, V30_J, V30_K,
  881     V31, V31_H, V31_J, V31_K
  882 );
  883 
  884 // Class for 128 bit register v0
  885 reg_class v0_reg(
  886     V0, V0_H
  887 );
  888 
  889 // Class for 128 bit register v1
  890 reg_class v1_reg(
  891     V1, V1_H
  892 );
  893 
  894 // Class for 128 bit register v2
  895 reg_class v2_reg(
  896     V2, V2_H
  897 );
  898 
  899 // Class for 128 bit register v3
  900 reg_class v3_reg(
  901     V3, V3_H
  902 );
  903 
  904 // Class for 128 bit register v4
  905 reg_class v4_reg(
  906     V4, V4_H
  907 );
  908 
  909 // Class for 128 bit register v5
  910 reg_class v5_reg(
  911     V5, V5_H
  912 );
  913 
  914 // Class for 128 bit register v6
  915 reg_class v6_reg(
  916     V6, V6_H
  917 );
  918 
  919 // Class for 128 bit register v7
  920 reg_class v7_reg(
  921     V7, V7_H
  922 );
  923 
  924 // Class for 128 bit register v8
  925 reg_class v8_reg(
  926     V8, V8_H
  927 );
  928 
  929 // Class for 128 bit register v9
  930 reg_class v9_reg(
  931     V9, V9_H
  932 );
  933 
  934 // Class for 128 bit register v10
  935 reg_class v10_reg(
  936     V10, V10_H
  937 );
  938 
  939 // Class for 128 bit register v11
  940 reg_class v11_reg(
  941     V11, V11_H
  942 );
  943 
  944 // Class for 128 bit register v12
  945 reg_class v12_reg(
  946     V12, V12_H
  947 );
  948 
  949 // Class for 128 bit register v13
  950 reg_class v13_reg(
  951     V13, V13_H
  952 );
  953 
  954 // Class for 128 bit register v14
  955 reg_class v14_reg(
  956     V14, V14_H
  957 );
  958 
  959 // Class for 128 bit register v15
  960 reg_class v15_reg(
  961     V15, V15_H
  962 );
  963 
  964 // Class for 128 bit register v16
  965 reg_class v16_reg(
  966     V16, V16_H
  967 );
  968 
  969 // Class for 128 bit register v17
  970 reg_class v17_reg(
  971     V17, V17_H
  972 );
  973 
  974 // Class for 128 bit register v18
  975 reg_class v18_reg(
  976     V18, V18_H
  977 );
  978 
  979 // Class for 128 bit register v19
  980 reg_class v19_reg(
  981     V19, V19_H
  982 );
  983 
  984 // Class for 128 bit register v20
  985 reg_class v20_reg(
  986     V20, V20_H
  987 );
  988 
  989 // Class for 128 bit register v21
  990 reg_class v21_reg(
  991     V21, V21_H
  992 );
  993 
  994 // Class for 128 bit register v22
  995 reg_class v22_reg(
  996     V22, V22_H
  997 );
  998 
  999 // Class for 128 bit register v23
 1000 reg_class v23_reg(
 1001     V23, V23_H
 1002 );
 1003 
 1004 // Class for 128 bit register v24
 1005 reg_class v24_reg(
 1006     V24, V24_H
 1007 );
 1008 
 1009 // Class for 128 bit register v25
 1010 reg_class v25_reg(
 1011     V25, V25_H
 1012 );
 1013 
 1014 // Class for 128 bit register v26
 1015 reg_class v26_reg(
 1016     V26, V26_H
 1017 );
 1018 
 1019 // Class for 128 bit register v27
 1020 reg_class v27_reg(
 1021     V27, V27_H
 1022 );
 1023 
 1024 // Class for 128 bit register v28
 1025 reg_class v28_reg(
 1026     V28, V28_H
 1027 );
 1028 
 1029 // Class for 128 bit register v29
 1030 reg_class v29_reg(
 1031     V29, V29_H
 1032 );
 1033 
 1034 // Class for 128 bit register v30
 1035 reg_class v30_reg(
 1036     V30, V30_H
 1037 );
 1038 
 1039 // Class for 128 bit register v31
 1040 reg_class v31_reg(
 1041     V31, V31_H
 1042 );
 1043 
 1044 // Class for all SVE predicate registers.
 1045 reg_class pr_reg (
 1046     P0,
 1047     P1,
 1048     P2,
 1049     P3,
 1050     P4,
 1051     P5,
 1052     P6,
 1053     // P7, non-allocatable, preserved with all elements preset to TRUE.
 1054     P8,
 1055     P9,
 1056     P10,
 1057     P11,
 1058     P12,
 1059     P13,
 1060     P14,
 1061     P15
 1062 );
 1063 
 1064 // Class for SVE governing predicate registers, which are used
 1065 // to determine the active elements of a predicated instruction.
 1066 reg_class gov_pr (
 1067     P0,
 1068     P1,
 1069     P2,
 1070     P3,
 1071     P4,
 1072     P5,
 1073     P6,
 1074     // P7, non-allocatable, preserved with all elements preset to TRUE.
 1075 );
 1076 
 1077 reg_class p0_reg(P0);
 1078 reg_class p1_reg(P1);
 1079 
 1080 // Singleton class for condition codes
 1081 reg_class int_flags(RFLAGS);
 1082 
 1083 %}
 1084 
 1085 //----------DEFINITION BLOCK---------------------------------------------------
 1086 // Define name --> value mappings to inform the ADLC of an integer valued name
 1087 // Current support includes integer values in the range [0, 0x7FFFFFFF]
 1088 // Format:
 1089 //        int_def  <name>         ( <int_value>, <expression>);
 1090 // Generated Code in ad_<arch>.hpp
 1091 //        #define  <name>   (<expression>)
 1092 //        // value == <int_value>
 1093 // Generated code in ad_<arch>.cpp adlc_verification()
 1094 //        assert( <name> == <int_value>, "Expect (<expression>) to equal <int_value>");
 1095 //
 1096 
 1097 // we follow the ppc-aix port in using a simple cost model which ranks
 1098 // register operations as cheap, memory ops as more expensive and
 1099 // branches as most expensive. the first two have a low as well as a
 1100 // normal cost. huge cost appears to be a way of saying don't do
 1101 // something
 1102 
 1103 definitions %{
 1104   // The default cost (of a register move instruction).
 1105   int_def INSN_COST            (    100,     100);
 1106   int_def BRANCH_COST          (    200,     2 * INSN_COST);
 1107   int_def CALL_COST            (    200,     2 * INSN_COST);
 1108   int_def VOLATILE_REF_COST    (   1000,     10 * INSN_COST);
 1109 %}
 1110 
 1111 
 1112 //----------SOURCE BLOCK-------------------------------------------------------
 1113 // This is a block of C++ code which provides values, functions, and
 1114 // definitions necessary in the rest of the architecture description
 1115 
 1116 source_hpp %{
 1117 
 1118 #include "asm/macroAssembler.hpp"
 1119 #include "gc/shared/barrierSetAssembler.hpp"
 1120 #include "gc/shared/cardTable.hpp"
 1121 #include "gc/shared/cardTableBarrierSet.hpp"
 1122 #include "gc/shared/collectedHeap.hpp"
 1123 #include "opto/addnode.hpp"
 1124 #include "opto/convertnode.hpp"
 1125 #include "runtime/objectMonitor.hpp"
 1126 
 1127 extern RegMask _ANY_REG32_mask;
 1128 extern RegMask _ANY_REG_mask;
 1129 extern RegMask _PTR_REG_mask;
 1130 extern RegMask _NO_SPECIAL_REG32_mask;
 1131 extern RegMask _NO_SPECIAL_REG_mask;
 1132 extern RegMask _NO_SPECIAL_PTR_REG_mask;
 1133 extern RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
 1134 
 1135 class CallStubImpl {
 1136 
 1137   //--------------------------------------------------------------
 1138   //---<  Used for optimization in Compile::shorten_branches  >---
 1139   //--------------------------------------------------------------
 1140 
 1141  public:
 1142   // Size of call trampoline stub.
 1143   static uint size_call_trampoline() {
 1144     return MacroAssembler::max_trampoline_stub_size(); // no call trampolines on this platform
 1145   }
 1146 
 1147   // number of relocations needed by a call trampoline stub
 1148   static uint reloc_call_trampoline() {
 1149     return 0; // no call trampolines on this platform
 1150   }
 1151 };
 1152 
 1153 class HandlerImpl {
 1154 
 1155  public:
 1156 
 1157   static int emit_exception_handler(C2_MacroAssembler *masm);
 1158   static int emit_deopt_handler(C2_MacroAssembler* masm);
 1159 
 1160   static uint size_exception_handler() {
 1161     return MacroAssembler::far_codestub_branch_size();
 1162   }
 1163 
 1164   static uint size_deopt_handler() {
 1165     // count one adr and one far branch instruction
 1166     return NativeInstruction::instruction_size + MacroAssembler::far_codestub_branch_size();
 1167   }
 1168 };
 1169 
 1170 class Node::PD {
 1171 public:
 1172   enum NodeFlags {
 1173     _last_flag = Node::_last_flag
 1174   };
 1175 };
 1176 
 1177   bool is_CAS(int opcode, bool maybe_volatile);
 1178 
 1179   // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
 1180 
 1181   bool unnecessary_acquire(const Node *barrier);
 1182   bool needs_acquiring_load(const Node *load);
 1183 
 1184   // predicates controlling emit of str<x>/stlr<x> and associated dmbs
 1185 
 1186   bool unnecessary_release(const Node *barrier);
 1187   bool unnecessary_volatile(const Node *barrier);
 1188   bool needs_releasing_store(const Node *store);
 1189 
 1190   // predicate controlling translation of CompareAndSwapX
 1191   bool needs_acquiring_load_exclusive(const Node *load);
 1192 
 1193   // predicate controlling addressing modes
 1194   bool size_fits_all_mem_uses(AddPNode* addp, int shift);
 1195 
 1196   // Convert BootTest condition to Assembler condition.
 1197   // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
 1198   Assembler::Condition to_assembler_cond(BoolTest::mask cond);
 1199 %}
 1200 
 1201 source %{
 1202 
 1203   // Derived RegMask with conditionally allocatable registers
 1204 
 1205   void PhaseOutput::pd_perform_mach_node_analysis() {
 1206   }
 1207 
 1208   int MachNode::pd_alignment_required() const {
 1209     return 1;
 1210   }
 1211 
 1212   int MachNode::compute_padding(int current_offset) const {
 1213     return 0;
 1214   }
 1215 
 1216   RegMask _ANY_REG32_mask;
 1217   RegMask _ANY_REG_mask;
 1218   RegMask _PTR_REG_mask;
 1219   RegMask _NO_SPECIAL_REG32_mask;
 1220   RegMask _NO_SPECIAL_REG_mask;
 1221   RegMask _NO_SPECIAL_PTR_REG_mask;
 1222   RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
 1223 
 1224   void reg_mask_init() {
 1225     // We derive below RegMask(s) from the ones which are auto-generated from
 1226     // adlc register classes to make AArch64 rheapbase (r27) and rfp (r29)
 1227     // registers conditionally reserved.
 1228 
 1229     _ANY_REG32_mask = _ALL_REG32_mask;
 1230     _ANY_REG32_mask.Remove(OptoReg::as_OptoReg(r31_sp->as_VMReg()));
 1231 
 1232     _ANY_REG_mask = _ALL_REG_mask;
 1233 
 1234     _PTR_REG_mask = _ALL_REG_mask;
 1235 
 1236     _NO_SPECIAL_REG32_mask = _ALL_REG32_mask;
 1237     _NO_SPECIAL_REG32_mask.SUBTRACT(_NON_ALLOCATABLE_REG32_mask);
 1238 
 1239     _NO_SPECIAL_REG_mask = _ALL_REG_mask;
 1240     _NO_SPECIAL_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
 1241 
 1242     _NO_SPECIAL_PTR_REG_mask = _ALL_REG_mask;
 1243     _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
 1244 
 1245     // r27 is not allocatable when compressed oops is on and heapbase is not
 1246     // zero, compressed klass pointers doesn't use r27 after JDK-8234794
 1247     if (UseCompressedOops && (CompressedOops::ptrs_base() != nullptr)) {
 1248       _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
 1249       _NO_SPECIAL_REG_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
 1250       _NO_SPECIAL_PTR_REG_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
 1251     }
 1252 
 1253     // r29 is not allocatable when PreserveFramePointer is on
 1254     if (PreserveFramePointer) {
 1255       _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
 1256       _NO_SPECIAL_REG_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
 1257       _NO_SPECIAL_PTR_REG_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
 1258     }
 1259 
 1260     _NO_SPECIAL_NO_RFP_PTR_REG_mask = _NO_SPECIAL_PTR_REG_mask;
 1261     _NO_SPECIAL_NO_RFP_PTR_REG_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
 1262   }
 1263 
 1264   // Optimizaton of volatile gets and puts
 1265   // -------------------------------------
 1266   //
 1267   // AArch64 has ldar<x> and stlr<x> instructions which we can safely
 1268   // use to implement volatile reads and writes. For a volatile read
 1269   // we simply need
 1270   //
 1271   //   ldar<x>
 1272   //
 1273   // and for a volatile write we need
 1274   //
 1275   //   stlr<x>
 1276   //
 1277   // Alternatively, we can implement them by pairing a normal
 1278   // load/store with a memory barrier. For a volatile read we need
 1279   //
 1280   //   ldr<x>
 1281   //   dmb ishld
 1282   //
 1283   // for a volatile write
 1284   //
 1285   //   dmb ish
 1286   //   str<x>
 1287   //   dmb ish
 1288   //
 1289   // We can also use ldaxr and stlxr to implement compare and swap CAS
 1290   // sequences. These are normally translated to an instruction
 1291   // sequence like the following
 1292   //
 1293   //   dmb      ish
 1294   // retry:
 1295   //   ldxr<x>   rval raddr
 1296   //   cmp       rval rold
 1297   //   b.ne done
 1298   //   stlxr<x>  rval, rnew, rold
 1299   //   cbnz      rval retry
 1300   // done:
 1301   //   cset      r0, eq
 1302   //   dmb ishld
 1303   //
 1304   // Note that the exclusive store is already using an stlxr
 1305   // instruction. That is required to ensure visibility to other
 1306   // threads of the exclusive write (assuming it succeeds) before that
 1307   // of any subsequent writes.
 1308   //
 1309   // The following instruction sequence is an improvement on the above
 1310   //
 1311   // retry:
 1312   //   ldaxr<x>  rval raddr
 1313   //   cmp       rval rold
 1314   //   b.ne done
 1315   //   stlxr<x>  rval, rnew, rold
 1316   //   cbnz      rval retry
 1317   // done:
 1318   //   cset      r0, eq
 1319   //
 1320   // We don't need the leading dmb ish since the stlxr guarantees
 1321   // visibility of prior writes in the case that the swap is
 1322   // successful. Crucially we don't have to worry about the case where
 1323   // the swap is not successful since no valid program should be
 1324   // relying on visibility of prior changes by the attempting thread
 1325   // in the case where the CAS fails.
 1326   //
 1327   // Similarly, we don't need the trailing dmb ishld if we substitute
 1328   // an ldaxr instruction since that will provide all the guarantees we
 1329   // require regarding observation of changes made by other threads
 1330   // before any change to the CAS address observed by the load.
 1331   //
 1332   // In order to generate the desired instruction sequence we need to
 1333   // be able to identify specific 'signature' ideal graph node
 1334   // sequences which i) occur as a translation of a volatile reads or
 1335   // writes or CAS operations and ii) do not occur through any other
 1336   // translation or graph transformation. We can then provide
 1337   // alternative aldc matching rules which translate these node
 1338   // sequences to the desired machine code sequences. Selection of the
 1339   // alternative rules can be implemented by predicates which identify
 1340   // the relevant node sequences.
 1341   //
 1342   // The ideal graph generator translates a volatile read to the node
 1343   // sequence
 1344   //
 1345   //   LoadX[mo_acquire]
 1346   //   MemBarAcquire
 1347   //
 1348   // As a special case when using the compressed oops optimization we
 1349   // may also see this variant
 1350   //
 1351   //   LoadN[mo_acquire]
 1352   //   DecodeN
 1353   //   MemBarAcquire
 1354   //
 1355   // A volatile write is translated to the node sequence
 1356   //
 1357   //   MemBarRelease
 1358   //   StoreX[mo_release] {CardMark}-optional
 1359   //   MemBarVolatile
 1360   //
 1361   // n.b. the above node patterns are generated with a strict
 1362   // 'signature' configuration of input and output dependencies (see
 1363   // the predicates below for exact details). The card mark may be as
 1364   // simple as a few extra nodes or, in a few GC configurations, may
 1365   // include more complex control flow between the leading and
 1366   // trailing memory barriers. However, whatever the card mark
 1367   // configuration these signatures are unique to translated volatile
 1368   // reads/stores -- they will not appear as a result of any other
 1369   // bytecode translation or inlining nor as a consequence of
 1370   // optimizing transforms.
 1371   //
 1372   // We also want to catch inlined unsafe volatile gets and puts and
 1373   // be able to implement them using either ldar<x>/stlr<x> or some
 1374   // combination of ldr<x>/stlr<x> and dmb instructions.
 1375   //
 1376   // Inlined unsafe volatiles puts manifest as a minor variant of the
 1377   // normal volatile put node sequence containing an extra cpuorder
 1378   // membar
 1379   //
 1380   //   MemBarRelease
 1381   //   MemBarCPUOrder
 1382   //   StoreX[mo_release] {CardMark}-optional
 1383   //   MemBarCPUOrder
 1384   //   MemBarVolatile
 1385   //
 1386   // n.b. as an aside, a cpuorder membar is not itself subject to
 1387   // matching and translation by adlc rules.  However, the rule
 1388   // predicates need to detect its presence in order to correctly
 1389   // select the desired adlc rules.
 1390   //
 1391   // Inlined unsafe volatile gets manifest as a slightly different
 1392   // node sequence to a normal volatile get because of the
 1393   // introduction of some CPUOrder memory barriers to bracket the
 1394   // Load. However, but the same basic skeleton of a LoadX feeding a
 1395   // MemBarAcquire, possibly through an optional DecodeN, is still
 1396   // present
 1397   //
 1398   //   MemBarCPUOrder
 1399   //        ||       \\
 1400   //   MemBarCPUOrder LoadX[mo_acquire]
 1401   //        ||            |
 1402   //        ||       {DecodeN} optional
 1403   //        ||       /
 1404   //     MemBarAcquire
 1405   //
 1406   // In this case the acquire membar does not directly depend on the
 1407   // load. However, we can be sure that the load is generated from an
 1408   // inlined unsafe volatile get if we see it dependent on this unique
 1409   // sequence of membar nodes. Similarly, given an acquire membar we
 1410   // can know that it was added because of an inlined unsafe volatile
 1411   // get if it is fed and feeds a cpuorder membar and if its feed
 1412   // membar also feeds an acquiring load.
 1413   //
 1414   // Finally an inlined (Unsafe) CAS operation is translated to the
 1415   // following ideal graph
 1416   //
 1417   //   MemBarRelease
 1418   //   MemBarCPUOrder
 1419   //   CompareAndSwapX {CardMark}-optional
 1420   //   MemBarCPUOrder
 1421   //   MemBarAcquire
 1422   //
 1423   // So, where we can identify these volatile read and write
 1424   // signatures we can choose to plant either of the above two code
 1425   // sequences. For a volatile read we can simply plant a normal
 1426   // ldr<x> and translate the MemBarAcquire to a dmb. However, we can
 1427   // also choose to inhibit translation of the MemBarAcquire and
 1428   // inhibit planting of the ldr<x>, instead planting an ldar<x>.
 1429   //
 1430   // When we recognise a volatile store signature we can choose to
 1431   // plant at a dmb ish as a translation for the MemBarRelease, a
 1432   // normal str<x> and then a dmb ish for the MemBarVolatile.
 1433   // Alternatively, we can inhibit translation of the MemBarRelease
 1434   // and MemBarVolatile and instead plant a simple stlr<x>
 1435   // instruction.
 1436   //
 1437   // when we recognise a CAS signature we can choose to plant a dmb
 1438   // ish as a translation for the MemBarRelease, the conventional
 1439   // macro-instruction sequence for the CompareAndSwap node (which
 1440   // uses ldxr<x>) and then a dmb ishld for the MemBarAcquire.
 1441   // Alternatively, we can elide generation of the dmb instructions
 1442   // and plant the alternative CompareAndSwap macro-instruction
 1443   // sequence (which uses ldaxr<x>).
 1444   //
 1445   // Of course, the above only applies when we see these signature
 1446   // configurations. We still want to plant dmb instructions in any
 1447   // other cases where we may see a MemBarAcquire, MemBarRelease or
 1448   // MemBarVolatile. For example, at the end of a constructor which
 1449   // writes final/volatile fields we will see a MemBarRelease
 1450   // instruction and this needs a 'dmb ish' lest we risk the
 1451   // constructed object being visible without making the
 1452   // final/volatile field writes visible.
 1453   //
 1454   // n.b. the translation rules below which rely on detection of the
 1455   // volatile signatures and insert ldar<x> or stlr<x> are failsafe.
 1456   // If we see anything other than the signature configurations we
 1457   // always just translate the loads and stores to ldr<x> and str<x>
 1458   // and translate acquire, release and volatile membars to the
 1459   // relevant dmb instructions.
 1460   //
 1461 
 1462   // is_CAS(int opcode, bool maybe_volatile)
 1463   //
 1464   // return true if opcode is one of the possible CompareAndSwapX
 1465   // values otherwise false.
 1466 
 1467   bool is_CAS(int opcode, bool maybe_volatile)
 1468   {
 1469     switch(opcode) {
 1470       // We handle these
 1471     case Op_CompareAndSwapI:
 1472     case Op_CompareAndSwapL:
 1473     case Op_CompareAndSwapP:
 1474     case Op_CompareAndSwapN:
 1475     case Op_ShenandoahCompareAndSwapP:
 1476     case Op_ShenandoahCompareAndSwapN:
 1477     case Op_CompareAndSwapB:
 1478     case Op_CompareAndSwapS:
 1479     case Op_GetAndSetI:
 1480     case Op_GetAndSetL:
 1481     case Op_GetAndSetP:
 1482     case Op_GetAndSetN:
 1483     case Op_GetAndAddI:
 1484     case Op_GetAndAddL:
 1485       return true;
 1486     case Op_CompareAndExchangeI:
 1487     case Op_CompareAndExchangeN:
 1488     case Op_CompareAndExchangeB:
 1489     case Op_CompareAndExchangeS:
 1490     case Op_CompareAndExchangeL:
 1491     case Op_CompareAndExchangeP:
 1492     case Op_WeakCompareAndSwapB:
 1493     case Op_WeakCompareAndSwapS:
 1494     case Op_WeakCompareAndSwapI:
 1495     case Op_WeakCompareAndSwapL:
 1496     case Op_WeakCompareAndSwapP:
 1497     case Op_WeakCompareAndSwapN:
 1498     case Op_ShenandoahWeakCompareAndSwapP:
 1499     case Op_ShenandoahWeakCompareAndSwapN:
 1500     case Op_ShenandoahCompareAndExchangeP:
 1501     case Op_ShenandoahCompareAndExchangeN:
 1502       return maybe_volatile;
 1503     default:
 1504       return false;
 1505     }
 1506   }
 1507 
 1508   // helper to determine the maximum number of Phi nodes we may need to
 1509   // traverse when searching from a card mark membar for the merge mem
 1510   // feeding a trailing membar or vice versa
 1511 
 1512 // predicates controlling emit of ldr<x>/ldar<x>
 1513 
 1514 bool unnecessary_acquire(const Node *barrier)
 1515 {
 1516   assert(barrier->is_MemBar(), "expecting a membar");
 1517 
 1518   MemBarNode* mb = barrier->as_MemBar();
 1519 
 1520   if (mb->trailing_load()) {
 1521     return true;
 1522   }
 1523 
 1524   if (mb->trailing_load_store()) {
 1525     Node* load_store = mb->in(MemBarNode::Precedent);
 1526     assert(load_store->is_LoadStore(), "unexpected graph shape");
 1527     return is_CAS(load_store->Opcode(), true);
 1528   }
 1529 
 1530   return false;
 1531 }
 1532 
 1533 bool needs_acquiring_load(const Node *n)
 1534 {
 1535   assert(n->is_Load(), "expecting a load");
 1536   LoadNode *ld = n->as_Load();
 1537   return ld->is_acquire();
 1538 }
 1539 
 1540 bool unnecessary_release(const Node *n)
 1541 {
 1542   assert((n->is_MemBar() &&
 1543           n->Opcode() == Op_MemBarRelease),
 1544          "expecting a release membar");
 1545 
 1546   MemBarNode *barrier = n->as_MemBar();
 1547   if (!barrier->leading()) {
 1548     return false;
 1549   } else {
 1550     Node* trailing = barrier->trailing_membar();
 1551     MemBarNode* trailing_mb = trailing->as_MemBar();
 1552     assert(trailing_mb->trailing(), "Not a trailing membar?");
 1553     assert(trailing_mb->leading_membar() == n, "inconsistent leading/trailing membars");
 1554 
 1555     Node* mem = trailing_mb->in(MemBarNode::Precedent);
 1556     if (mem->is_Store()) {
 1557       assert(mem->as_Store()->is_release(), "");
 1558       assert(trailing_mb->Opcode() == Op_MemBarVolatile, "");
 1559       return true;
 1560     } else {
 1561       assert(mem->is_LoadStore(), "");
 1562       assert(trailing_mb->Opcode() == Op_MemBarAcquire, "");
 1563       return is_CAS(mem->Opcode(), true);
 1564     }
 1565   }
 1566   return false;
 1567 }
 1568 
 1569 bool unnecessary_volatile(const Node *n)
 1570 {
 1571   // assert n->is_MemBar();
 1572   MemBarNode *mbvol = n->as_MemBar();
 1573 
 1574   bool release = mbvol->trailing_store();
 1575   assert(!release || (mbvol->in(MemBarNode::Precedent)->is_Store() && mbvol->in(MemBarNode::Precedent)->as_Store()->is_release()), "");
 1576 #ifdef ASSERT
 1577   if (release) {
 1578     Node* leading = mbvol->leading_membar();
 1579     assert(leading->Opcode() == Op_MemBarRelease, "");
 1580     assert(leading->as_MemBar()->leading_store(), "");
 1581     assert(leading->as_MemBar()->trailing_membar() == mbvol, "");
 1582   }
 1583 #endif
 1584 
 1585   return release;
 1586 }
 1587 
 1588 // predicates controlling emit of str<x>/stlr<x>
 1589 
 1590 bool needs_releasing_store(const Node *n)
 1591 {
 1592   // assert n->is_Store();
 1593   StoreNode *st = n->as_Store();
 1594   return st->trailing_membar() != nullptr;
 1595 }
 1596 
 1597 // predicate controlling translation of CAS
 1598 //
 1599 // returns true if CAS needs to use an acquiring load otherwise false
 1600 
 1601 bool needs_acquiring_load_exclusive(const Node *n)
 1602 {
 1603   assert(is_CAS(n->Opcode(), true), "expecting a compare and swap");
 1604   LoadStoreNode* ldst = n->as_LoadStore();
 1605   if (is_CAS(n->Opcode(), false)) {
 1606     assert(ldst->trailing_membar() != nullptr, "expected trailing membar");
 1607   } else {
 1608     return ldst->trailing_membar() != nullptr;
 1609   }
 1610 
 1611   // so we can just return true here
 1612   return true;
 1613 }
 1614 
 1615 #define __ masm->
 1616 
 1617 // advance declarations for helper functions to convert register
 1618 // indices to register objects
 1619 
 1620 // the ad file has to provide implementations of certain methods
 1621 // expected by the generic code
 1622 //
 1623 // REQUIRED FUNCTIONALITY
 1624 
 1625 //=============================================================================
 1626 
 1627 // !!!!! Special hack to get all types of calls to specify the byte offset
 1628 //       from the start of the call to the point where the return address
 1629 //       will point.
 1630 
 1631 int MachCallStaticJavaNode::ret_addr_offset()
 1632 {
 1633   // call should be a simple bl
 1634   int off = 4;
 1635   return off;
 1636 }
 1637 
 1638 int MachCallDynamicJavaNode::ret_addr_offset()
 1639 {
 1640   return 16; // movz, movk, movk, bl
 1641 }
 1642 
 1643 int MachCallRuntimeNode::ret_addr_offset() {
 1644   // for generated stubs the call will be
 1645   //   bl(addr)
 1646   // or with far branches
 1647   //   bl(trampoline_stub)
 1648   // for real runtime callouts it will be six instructions
 1649   // see aarch64_enc_java_to_runtime
 1650   //   adr(rscratch2, retaddr)
 1651   //   lea(rscratch1, RuntimeAddress(addr)
 1652   //   stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)))
 1653   //   blr(rscratch1)
 1654   CodeBlob *cb = CodeCache::find_blob(_entry_point);
 1655   if (cb) {
 1656     return 1 * NativeInstruction::instruction_size;
 1657   } else {
 1658     return 6 * NativeInstruction::instruction_size;
 1659   }
 1660 }
 1661 
 1662 //=============================================================================
 1663 
 1664 #ifndef PRODUCT
 1665 void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 1666   st->print("BREAKPOINT");
 1667 }
 1668 #endif
 1669 
 1670 void MachBreakpointNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
 1671   __ brk(0);
 1672 }
 1673 
 1674 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
 1675   return MachNode::size(ra_);
 1676 }
 1677 
 1678 //=============================================================================
 1679 
 1680 #ifndef PRODUCT
 1681   void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
 1682     st->print("nop \t# %d bytes pad for loops and calls", _count);
 1683   }
 1684 #endif
 1685 
 1686   void MachNopNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc*) const {
 1687     for (int i = 0; i < _count; i++) {
 1688       __ nop();
 1689     }
 1690   }
 1691 
 1692   uint MachNopNode::size(PhaseRegAlloc*) const {
 1693     return _count * NativeInstruction::instruction_size;
 1694   }
 1695 
 1696 //=============================================================================
 1697 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
 1698 
 1699 int ConstantTable::calculate_table_base_offset() const {
 1700   return 0;  // absolute addressing, no offset
 1701 }
 1702 
 1703 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
 1704 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
 1705   ShouldNotReachHere();
 1706 }
 1707 
 1708 void MachConstantBaseNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const {
 1709   // Empty encoding
 1710 }
 1711 
 1712 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
 1713   return 0;
 1714 }
 1715 
 1716 #ifndef PRODUCT
 1717 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
 1718   st->print("-- \t// MachConstantBaseNode (empty encoding)");
 1719 }
 1720 #endif
 1721 
 1722 #ifndef PRODUCT
 1723 void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 1724   Compile* C = ra_->C;
 1725 
 1726   int framesize = C->output()->frame_slots() << LogBytesPerInt;
 1727 
 1728   if (C->output()->need_stack_bang(framesize))
 1729     st->print("# stack bang size=%d\n\t", framesize);
 1730 
 1731   if (VM_Version::use_rop_protection()) {
 1732     st->print("ldr  zr, [lr]\n\t");
 1733     st->print("paciaz\n\t");
 1734   }
 1735   if (framesize < ((1 << 9) + 2 * wordSize)) {
 1736     st->print("sub  sp, sp, #%d\n\t", framesize);
 1737     st->print("stp  rfp, lr, [sp, #%d]", framesize - 2 * wordSize);
 1738     if (PreserveFramePointer) st->print("\n\tadd  rfp, sp, #%d", framesize - 2 * wordSize);
 1739   } else {
 1740     st->print("stp  lr, rfp, [sp, #%d]!\n\t", -(2 * wordSize));
 1741     if (PreserveFramePointer) st->print("mov  rfp, sp\n\t");
 1742     st->print("mov  rscratch1, #%d\n\t", framesize - 2 * wordSize);
 1743     st->print("sub  sp, sp, rscratch1");
 1744   }
 1745   if (C->stub_function() == nullptr && BarrierSet::barrier_set()->barrier_set_nmethod() != nullptr) {
 1746     st->print("\n\t");
 1747     st->print("ldr  rscratch1, [guard]\n\t");
 1748     st->print("dmb ishld\n\t");
 1749     st->print("ldr  rscratch2, [rthread, #thread_disarmed_guard_value_offset]\n\t");
 1750     st->print("cmp  rscratch1, rscratch2\n\t");
 1751     st->print("b.eq skip");
 1752     st->print("\n\t");
 1753     st->print("blr #nmethod_entry_barrier_stub\n\t");
 1754     st->print("b skip\n\t");
 1755     st->print("guard: int\n\t");
 1756     st->print("\n\t");
 1757     st->print("skip:\n\t");
 1758   }
 1759 }
 1760 #endif
 1761 
 1762 void MachPrologNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
 1763   Compile* C = ra_->C;
 1764 
 1765   // n.b. frame size includes space for return pc and rfp
 1766   const int framesize = C->output()->frame_size_in_bytes();
 1767 
 1768   // insert a nop at the start of the prolog so we can patch in a
 1769   // branch if we need to invalidate the method later
 1770   __ nop();
 1771 
 1772   if (C->clinit_barrier_on_entry()) {
 1773     assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
 1774 
 1775     Label L_skip_barrier;
 1776 
 1777     __ mov_metadata(rscratch2, C->method()->holder()->constant_encoding());
 1778     __ clinit_barrier(rscratch2, rscratch1, &L_skip_barrier);
 1779     __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
 1780     __ bind(L_skip_barrier);
 1781   }
 1782 
 1783   if (C->max_vector_size() > 0) {
 1784     __ reinitialize_ptrue();
 1785   }
 1786 
 1787   int bangsize = C->output()->bang_size_in_bytes();
 1788   if (C->output()->need_stack_bang(bangsize))
 1789     __ generate_stack_overflow_check(bangsize);
 1790 
 1791   __ build_frame(framesize);
 1792 
 1793   if (C->stub_function() == nullptr) {
 1794     BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
 1795     if (BarrierSet::barrier_set()->barrier_set_nmethod() != nullptr) {
 1796       // Dummy labels for just measuring the code size
 1797       Label dummy_slow_path;
 1798       Label dummy_continuation;
 1799       Label dummy_guard;
 1800       Label* slow_path = &dummy_slow_path;
 1801       Label* continuation = &dummy_continuation;
 1802       Label* guard = &dummy_guard;
 1803       if (!Compile::current()->output()->in_scratch_emit_size()) {
 1804         // Use real labels from actual stub when not emitting code for the purpose of measuring its size
 1805         C2EntryBarrierStub* stub = new (Compile::current()->comp_arena()) C2EntryBarrierStub();
 1806         Compile::current()->output()->add_stub(stub);
 1807         slow_path = &stub->entry();
 1808         continuation = &stub->continuation();
 1809         guard = &stub->guard();
 1810       }
 1811       // In the C2 code, we move the non-hot part of nmethod entry barriers out-of-line to a stub.
 1812       bs->nmethod_entry_barrier(masm, slow_path, continuation, guard);
 1813     }
 1814   }
 1815 
 1816   if (VerifyStackAtCalls) {
 1817     Unimplemented();
 1818   }
 1819 
 1820   C->output()->set_frame_complete(__ offset());
 1821 
 1822   if (C->has_mach_constant_base_node()) {
 1823     // NOTE: We set the table base offset here because users might be
 1824     // emitted before MachConstantBaseNode.
 1825     ConstantTable& constant_table = C->output()->constant_table();
 1826     constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
 1827   }
 1828 }
 1829 
 1830 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
 1831 {
 1832   return MachNode::size(ra_); // too many variables; just compute it
 1833                               // the hard way
 1834 }
 1835 
 1836 int MachPrologNode::reloc() const
 1837 {
 1838   return 0;
 1839 }
 1840 
 1841 //=============================================================================
 1842 
 1843 #ifndef PRODUCT
 1844 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 1845   Compile* C = ra_->C;
 1846   int framesize = C->output()->frame_slots() << LogBytesPerInt;
 1847 
 1848   st->print("# pop frame %d\n\t",framesize);
 1849 
 1850   if (framesize == 0) {
 1851     st->print("ldp  lr, rfp, [sp],#%d\n\t", (2 * wordSize));
 1852   } else if (framesize < ((1 << 9) + 2 * wordSize)) {
 1853     st->print("ldp  lr, rfp, [sp,#%d]\n\t", framesize - 2 * wordSize);
 1854     st->print("add  sp, sp, #%d\n\t", framesize);
 1855   } else {
 1856     st->print("mov  rscratch1, #%d\n\t", framesize - 2 * wordSize);
 1857     st->print("add  sp, sp, rscratch1\n\t");
 1858     st->print("ldp  lr, rfp, [sp],#%d\n\t", (2 * wordSize));
 1859   }
 1860   if (VM_Version::use_rop_protection()) {
 1861     st->print("autiaz\n\t");
 1862     st->print("ldr  zr, [lr]\n\t");
 1863   }
 1864 
 1865   if (do_polling() && C->is_method_compilation()) {
 1866     st->print("# test polling word\n\t");
 1867     st->print("ldr  rscratch1, [rthread],#%d\n\t", in_bytes(JavaThread::polling_word_offset()));
 1868     st->print("cmp  sp, rscratch1\n\t");
 1869     st->print("bhi #slow_path");
 1870   }
 1871 }
 1872 #endif
 1873 
 1874 void MachEpilogNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
 1875   Compile* C = ra_->C;
 1876   int framesize = C->output()->frame_slots() << LogBytesPerInt;
 1877 
 1878   __ remove_frame(framesize);
 1879 
 1880   if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
 1881     __ reserved_stack_check();
 1882   }
 1883 
 1884   if (do_polling() && C->is_method_compilation()) {
 1885     Label dummy_label;
 1886     Label* code_stub = &dummy_label;
 1887     if (!C->output()->in_scratch_emit_size()) {
 1888       C2SafepointPollStub* stub = new (C->comp_arena()) C2SafepointPollStub(__ offset());
 1889       C->output()->add_stub(stub);
 1890       code_stub = &stub->entry();
 1891     }
 1892     __ relocate(relocInfo::poll_return_type);
 1893     __ safepoint_poll(*code_stub, true /* at_return */, false /* acquire */, true /* in_nmethod */);
 1894   }
 1895 }
 1896 
 1897 uint MachEpilogNode::size(PhaseRegAlloc *ra_) const {
 1898   // Variable size. Determine dynamically.
 1899   return MachNode::size(ra_);
 1900 }
 1901 
 1902 int MachEpilogNode::reloc() const {
 1903   // Return number of relocatable values contained in this instruction.
 1904   return 1; // 1 for polling page.
 1905 }
 1906 
 1907 const Pipeline * MachEpilogNode::pipeline() const {
 1908   return MachNode::pipeline_class();
 1909 }
 1910 
 1911 //=============================================================================
 1912 
 1913 static enum RC rc_class(OptoReg::Name reg) {
 1914 
 1915   if (reg == OptoReg::Bad) {
 1916     return rc_bad;
 1917   }
 1918 
 1919   // we have 32 int registers * 2 halves
 1920   int slots_of_int_registers = Register::number_of_registers * Register::max_slots_per_register;
 1921 
 1922   if (reg < slots_of_int_registers) {
 1923     return rc_int;
 1924   }
 1925 
 1926   // we have 32 float register * 8 halves
 1927   int slots_of_float_registers = FloatRegister::number_of_registers * FloatRegister::max_slots_per_register;
 1928   if (reg < slots_of_int_registers + slots_of_float_registers) {
 1929     return rc_float;
 1930   }
 1931 
 1932   int slots_of_predicate_registers = PRegister::number_of_registers * PRegister::max_slots_per_register;
 1933   if (reg < slots_of_int_registers + slots_of_float_registers + slots_of_predicate_registers) {
 1934     return rc_predicate;
 1935   }
 1936 
 1937   // Between predicate regs & stack is the flags.
 1938   assert(OptoReg::is_stack(reg), "blow up if spilling flags");
 1939 
 1940   return rc_stack;
 1941 }
 1942 
 1943 uint MachSpillCopyNode::implementation(C2_MacroAssembler *masm, PhaseRegAlloc *ra_, bool do_size, outputStream *st) const {
 1944   Compile* C = ra_->C;
 1945 
 1946   // Get registers to move.
 1947   OptoReg::Name src_hi = ra_->get_reg_second(in(1));
 1948   OptoReg::Name src_lo = ra_->get_reg_first(in(1));
 1949   OptoReg::Name dst_hi = ra_->get_reg_second(this);
 1950   OptoReg::Name dst_lo = ra_->get_reg_first(this);
 1951 
 1952   enum RC src_hi_rc = rc_class(src_hi);
 1953   enum RC src_lo_rc = rc_class(src_lo);
 1954   enum RC dst_hi_rc = rc_class(dst_hi);
 1955   enum RC dst_lo_rc = rc_class(dst_lo);
 1956 
 1957   assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
 1958 
 1959   if (src_hi != OptoReg::Bad && !bottom_type()->isa_vectmask()) {
 1960     assert((src_lo&1)==0 && src_lo+1==src_hi &&
 1961            (dst_lo&1)==0 && dst_lo+1==dst_hi,
 1962            "expected aligned-adjacent pairs");
 1963   }
 1964 
 1965   if (src_lo == dst_lo && src_hi == dst_hi) {
 1966     return 0;            // Self copy, no move.
 1967   }
 1968 
 1969   bool is64 = (src_lo & 1) == 0 && src_lo + 1 == src_hi &&
 1970               (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi;
 1971   int src_offset = ra_->reg2offset(src_lo);
 1972   int dst_offset = ra_->reg2offset(dst_lo);
 1973 
 1974   if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
 1975     uint ireg = ideal_reg();
 1976     if (ireg == Op_VecA && masm) {
 1977       int sve_vector_reg_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
 1978       if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
 1979         // stack->stack
 1980         __ spill_copy_sve_vector_stack_to_stack(src_offset, dst_offset,
 1981                                                 sve_vector_reg_size_in_bytes);
 1982       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
 1983         __ spill_sve_vector(as_FloatRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
 1984                             sve_vector_reg_size_in_bytes);
 1985       } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
 1986         __ unspill_sve_vector(as_FloatRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
 1987                               sve_vector_reg_size_in_bytes);
 1988       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
 1989         __ sve_orr(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 1990                    as_FloatRegister(Matcher::_regEncode[src_lo]),
 1991                    as_FloatRegister(Matcher::_regEncode[src_lo]));
 1992       } else {
 1993         ShouldNotReachHere();
 1994       }
 1995     } else if (masm) {
 1996       assert(ireg == Op_VecD || ireg == Op_VecX, "must be 64 bit or 128 bit vector");
 1997       assert((src_lo_rc != rc_int && dst_lo_rc != rc_int), "sanity");
 1998       if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
 1999         // stack->stack
 2000         assert((src_offset & 7) == 0 && (dst_offset & 7) == 0, "unaligned stack offset");
 2001         if (ireg == Op_VecD) {
 2002           __ unspill(rscratch1, true, src_offset);
 2003           __ spill(rscratch1, true, dst_offset);
 2004         } else {
 2005           __ spill_copy128(src_offset, dst_offset);
 2006         }
 2007       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
 2008         __ mov(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2009                ireg == Op_VecD ? __ T8B : __ T16B,
 2010                as_FloatRegister(Matcher::_regEncode[src_lo]));
 2011       } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
 2012         __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
 2013                  ireg == Op_VecD ? __ D : __ Q,
 2014                  ra_->reg2offset(dst_lo));
 2015       } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
 2016         __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2017                    ireg == Op_VecD ? __ D : __ Q,
 2018                    ra_->reg2offset(src_lo));
 2019       } else {
 2020         ShouldNotReachHere();
 2021       }
 2022     }
 2023   } else if (masm) {
 2024     switch (src_lo_rc) {
 2025     case rc_int:
 2026       if (dst_lo_rc == rc_int) {  // gpr --> gpr copy
 2027         if (is64) {
 2028             __ mov(as_Register(Matcher::_regEncode[dst_lo]),
 2029                    as_Register(Matcher::_regEncode[src_lo]));
 2030         } else {
 2031             __ movw(as_Register(Matcher::_regEncode[dst_lo]),
 2032                     as_Register(Matcher::_regEncode[src_lo]));
 2033         }
 2034       } else if (dst_lo_rc == rc_float) { // gpr --> fpr copy
 2035         if (is64) {
 2036             __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2037                      as_Register(Matcher::_regEncode[src_lo]));
 2038         } else {
 2039             __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2040                      as_Register(Matcher::_regEncode[src_lo]));
 2041         }
 2042       } else {                    // gpr --> stack spill
 2043         assert(dst_lo_rc == rc_stack, "spill to bad register class");
 2044         __ spill(as_Register(Matcher::_regEncode[src_lo]), is64, dst_offset);
 2045       }
 2046       break;
 2047     case rc_float:
 2048       if (dst_lo_rc == rc_int) {  // fpr --> gpr copy
 2049         if (is64) {
 2050             __ fmovd(as_Register(Matcher::_regEncode[dst_lo]),
 2051                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2052         } else {
 2053             __ fmovs(as_Register(Matcher::_regEncode[dst_lo]),
 2054                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2055         }
 2056       } else if (dst_lo_rc == rc_float) { // fpr --> fpr copy
 2057         if (is64) {
 2058             __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2059                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2060         } else {
 2061             __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2062                      as_FloatRegister(Matcher::_regEncode[src_lo]));
 2063         }
 2064       } else {                    // fpr --> stack spill
 2065         assert(dst_lo_rc == rc_stack, "spill to bad register class");
 2066         __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
 2067                  is64 ? __ D : __ S, dst_offset);
 2068       }
 2069       break;
 2070     case rc_stack:
 2071       if (dst_lo_rc == rc_int) {  // stack --> gpr load
 2072         __ unspill(as_Register(Matcher::_regEncode[dst_lo]), is64, src_offset);
 2073       } else if (dst_lo_rc == rc_float) { // stack --> fpr load
 2074         __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
 2075                    is64 ? __ D : __ S, src_offset);
 2076       } else if (dst_lo_rc == rc_predicate) {
 2077         __ unspill_sve_predicate(as_PRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
 2078                                  Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
 2079       } else {                    // stack --> stack copy
 2080         assert(dst_lo_rc == rc_stack, "spill to bad register class");
 2081         if (ideal_reg() == Op_RegVectMask) {
 2082           __ spill_copy_sve_predicate_stack_to_stack(src_offset, dst_offset,
 2083                                                      Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
 2084         } else {
 2085           __ unspill(rscratch1, is64, src_offset);
 2086           __ spill(rscratch1, is64, dst_offset);
 2087         }
 2088       }
 2089       break;
 2090     case rc_predicate:
 2091       if (dst_lo_rc == rc_predicate) {
 2092         __ sve_mov(as_PRegister(Matcher::_regEncode[dst_lo]), as_PRegister(Matcher::_regEncode[src_lo]));
 2093       } else if (dst_lo_rc == rc_stack) {
 2094         __ spill_sve_predicate(as_PRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
 2095                                Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
 2096       } else {
 2097         assert(false, "bad src and dst rc_class combination.");
 2098         ShouldNotReachHere();
 2099       }
 2100       break;
 2101     default:
 2102       assert(false, "bad rc_class for spill");
 2103       ShouldNotReachHere();
 2104     }
 2105   }
 2106 
 2107   if (st) {
 2108     st->print("spill ");
 2109     if (src_lo_rc == rc_stack) {
 2110       st->print("[sp, #%d] -> ", ra_->reg2offset(src_lo));
 2111     } else {
 2112       st->print("%s -> ", Matcher::regName[src_lo]);
 2113     }
 2114     if (dst_lo_rc == rc_stack) {
 2115       st->print("[sp, #%d]", ra_->reg2offset(dst_lo));
 2116     } else {
 2117       st->print("%s", Matcher::regName[dst_lo]);
 2118     }
 2119     if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
 2120       int vsize = 0;
 2121       switch (ideal_reg()) {
 2122       case Op_VecD:
 2123         vsize = 64;
 2124         break;
 2125       case Op_VecX:
 2126         vsize = 128;
 2127         break;
 2128       case Op_VecA:
 2129         vsize = Matcher::scalable_vector_reg_size(T_BYTE) * 8;
 2130         break;
 2131       default:
 2132         assert(false, "bad register type for spill");
 2133         ShouldNotReachHere();
 2134       }
 2135       st->print("\t# vector spill size = %d", vsize);
 2136     } else if (ideal_reg() == Op_RegVectMask) {
 2137       assert(Matcher::supports_scalable_vector(), "bad register type for spill");
 2138       int vsize = Matcher::scalable_predicate_reg_slots() * 32;
 2139       st->print("\t# predicate spill size = %d", vsize);
 2140     } else {
 2141       st->print("\t# spill size = %d", is64 ? 64 : 32);
 2142     }
 2143   }
 2144 
 2145   return 0;
 2146 
 2147 }
 2148 
 2149 #ifndef PRODUCT
 2150 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 2151   if (!ra_)
 2152     st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
 2153   else
 2154     implementation(nullptr, ra_, false, st);
 2155 }
 2156 #endif
 2157 
 2158 void MachSpillCopyNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
 2159   implementation(masm, ra_, false, nullptr);
 2160 }
 2161 
 2162 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
 2163   return MachNode::size(ra_);
 2164 }
 2165 
 2166 //=============================================================================
 2167 
 2168 #ifndef PRODUCT
 2169 void BoxLockNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 2170   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 2171   int reg = ra_->get_reg_first(this);
 2172   st->print("add %s, rsp, #%d]\t# box lock",
 2173             Matcher::regName[reg], offset);
 2174 }
 2175 #endif
 2176 
 2177 void BoxLockNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
 2178   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 2179   int reg    = ra_->get_encode(this);
 2180 
 2181   // This add will handle any 24-bit signed offset. 24 bits allows an
 2182   // 8 megabyte stack frame.
 2183   __ add(as_Register(reg), sp, offset);
 2184 }
 2185 
 2186 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
 2187   // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_).
 2188   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 2189 
 2190   if (Assembler::operand_valid_for_add_sub_immediate(offset)) {
 2191     return NativeInstruction::instruction_size;
 2192   } else {
 2193     return 2 * NativeInstruction::instruction_size;
 2194   }
 2195 }
 2196 
 2197 //=============================================================================
 2198 
 2199 #ifndef PRODUCT
 2200 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 2201 {
 2202   st->print_cr("# MachUEPNode");
 2203   if (UseCompressedClassPointers) {
 2204     st->print_cr("\tldrw rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
 2205     st->print_cr("\tldrw r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
 2206     st->print_cr("\tcmpw rscratch1, r10");
 2207   } else {
 2208     st->print_cr("\tldr rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
 2209     st->print_cr("\tldr r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
 2210     st->print_cr("\tcmp rscratch1, r10");
 2211   }
 2212   st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
 2213 }
 2214 #endif
 2215 
 2216 void MachUEPNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
 2217 {
 2218   __ ic_check(InteriorEntryAlignment);
 2219 }
 2220 
 2221 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
 2222 {
 2223   return MachNode::size(ra_);
 2224 }
 2225 
 2226 // REQUIRED EMIT CODE
 2227 
 2228 //=============================================================================
 2229 
 2230 // Emit exception handler code.
 2231 int HandlerImpl::emit_exception_handler(C2_MacroAssembler* masm)
 2232 {
 2233   // mov rscratch1 #exception_blob_entry_point
 2234   // br rscratch1
 2235   // Note that the code buffer's insts_mark is always relative to insts.
 2236   // That's why we must use the macroassembler to generate a handler.
 2237   address base = __ start_a_stub(size_exception_handler());
 2238   if (base == nullptr) {
 2239     ciEnv::current()->record_failure("CodeCache is full");
 2240     return 0;  // CodeBuffer::expand failed
 2241   }
 2242   int offset = __ offset();
 2243   __ far_jump(RuntimeAddress(OptoRuntime::exception_blob()->entry_point()));
 2244   assert(__ offset() - offset <= (int) size_exception_handler(), "overflow");
 2245   __ end_a_stub();
 2246   return offset;
 2247 }
 2248 
 2249 // Emit deopt handler code.
 2250 int HandlerImpl::emit_deopt_handler(C2_MacroAssembler* masm)
 2251 {
 2252   // Note that the code buffer's insts_mark is always relative to insts.
 2253   // That's why we must use the macroassembler to generate a handler.
 2254   address base = __ start_a_stub(size_deopt_handler());
 2255   if (base == nullptr) {
 2256     ciEnv::current()->record_failure("CodeCache is full");
 2257     return 0;  // CodeBuffer::expand failed
 2258   }
 2259   int offset = __ offset();
 2260 
 2261   __ adr(lr, __ pc());
 2262   __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
 2263 
 2264   assert(__ offset() - offset == (int) size_deopt_handler(), "overflow");
 2265   __ end_a_stub();
 2266   return offset;
 2267 }
 2268 
 2269 // REQUIRED MATCHER CODE
 2270 
 2271 //=============================================================================
 2272 
 2273 bool Matcher::match_rule_supported(int opcode) {
 2274   if (!has_match_rule(opcode))
 2275     return false;
 2276 
 2277   switch (opcode) {
 2278     case Op_OnSpinWait:
 2279       return VM_Version::supports_on_spin_wait();
 2280     case Op_CacheWB:
 2281     case Op_CacheWBPreSync:
 2282     case Op_CacheWBPostSync:
 2283       if (!VM_Version::supports_data_cache_line_flush()) {
 2284         return false;
 2285       }
 2286       break;
 2287     case Op_ExpandBits:
 2288     case Op_CompressBits:
 2289       if (!VM_Version::supports_svebitperm()) {
 2290         return false;
 2291       }
 2292       break;
 2293     case Op_FmaF:
 2294     case Op_FmaD:
 2295     case Op_FmaVF:
 2296     case Op_FmaVD:
 2297       if (!UseFMA) {
 2298         return false;
 2299       }
 2300       break;
 2301   }
 2302 
 2303   return true; // Per default match rules are supported.
 2304 }
 2305 
 2306 const RegMask* Matcher::predicate_reg_mask(void) {
 2307   return &_PR_REG_mask;
 2308 }
 2309 
 2310 const TypeVectMask* Matcher::predicate_reg_type(const Type* elemTy, int length) {
 2311   return new TypeVectMask(elemTy, length);
 2312 }
 2313 
 2314 // Vector calling convention not yet implemented.
 2315 bool Matcher::supports_vector_calling_convention(void) {
 2316   return false;
 2317 }
 2318 
 2319 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
 2320   Unimplemented();
 2321   return OptoRegPair(0, 0);
 2322 }
 2323 
 2324 // Is this branch offset short enough that a short branch can be used?
 2325 //
 2326 // NOTE: If the platform does not provide any short branch variants, then
 2327 //       this method should return false for offset 0.
 2328 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
 2329   // The passed offset is relative to address of the branch.
 2330 
 2331   return (-32768 <= offset && offset < 32768);
 2332 }
 2333 
 2334 // Vector width in bytes.
 2335 int Matcher::vector_width_in_bytes(BasicType bt) {
 2336   // The MaxVectorSize should have been set by detecting SVE max vector register size.
 2337   int size = MIN2((UseSVE > 0) ? 256 : 16, (int)MaxVectorSize);
 2338   // Minimum 2 values in vector
 2339   if (size < 2*type2aelembytes(bt)) size = 0;
 2340   // But never < 4
 2341   if (size < 4) size = 0;
 2342   return size;
 2343 }
 2344 
 2345 // Limits on vector size (number of elements) loaded into vector.
 2346 int Matcher::max_vector_size(const BasicType bt) {
 2347   return vector_width_in_bytes(bt)/type2aelembytes(bt);
 2348 }
 2349 
 2350 int Matcher::min_vector_size(const BasicType bt) {
 2351   int max_size = max_vector_size(bt);
 2352   // Limit the min vector size to 8 bytes.
 2353   int size = 8 / type2aelembytes(bt);
 2354   if (bt == T_BYTE) {
 2355     // To support vector api shuffle/rearrange.
 2356     size = 4;
 2357   } else if (bt == T_BOOLEAN) {
 2358     // To support vector api load/store mask.
 2359     size = 2;
 2360   }
 2361   if (size < 2) size = 2;
 2362   return MIN2(size, max_size);
 2363 }
 2364 
 2365 int Matcher::max_vector_size_auto_vectorization(const BasicType bt) {
 2366   return Matcher::max_vector_size(bt);
 2367 }
 2368 
 2369 // Actual max scalable vector register length.
 2370 int Matcher::scalable_vector_reg_size(const BasicType bt) {
 2371   return Matcher::max_vector_size(bt);
 2372 }
 2373 
 2374 // Vector ideal reg.
 2375 uint Matcher::vector_ideal_reg(int len) {
 2376   if (UseSVE > 0 && 16 < len && len <= 256) {
 2377     return Op_VecA;
 2378   }
 2379   switch(len) {
 2380     // For 16-bit/32-bit mask vector, reuse VecD.
 2381     case  2:
 2382     case  4:
 2383     case  8: return Op_VecD;
 2384     case 16: return Op_VecX;
 2385   }
 2386   ShouldNotReachHere();
 2387   return 0;
 2388 }
 2389 
 2390 MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* generic_opnd, uint ideal_reg, bool is_temp) {
 2391   assert(Matcher::is_generic_vector(generic_opnd), "not generic");
 2392   switch (ideal_reg) {
 2393     case Op_VecA: return new vecAOper();
 2394     case Op_VecD: return new vecDOper();
 2395     case Op_VecX: return new vecXOper();
 2396   }
 2397   ShouldNotReachHere();
 2398   return nullptr;
 2399 }
 2400 
 2401 bool Matcher::is_reg2reg_move(MachNode* m) {
 2402   return false;
 2403 }
 2404 
 2405 bool Matcher::is_generic_vector(MachOper* opnd)  {
 2406   return opnd->opcode() == VREG;
 2407 }
 2408 
 2409 // Return whether or not this register is ever used as an argument.
 2410 // This function is used on startup to build the trampoline stubs in
 2411 // generateOptoStub.  Registers not mentioned will be killed by the VM
 2412 // call in the trampoline, and arguments in those registers not be
 2413 // available to the callee.
 2414 bool Matcher::can_be_java_arg(int reg)
 2415 {
 2416   return
 2417     reg ==  R0_num || reg == R0_H_num ||
 2418     reg ==  R1_num || reg == R1_H_num ||
 2419     reg ==  R2_num || reg == R2_H_num ||
 2420     reg ==  R3_num || reg == R3_H_num ||
 2421     reg ==  R4_num || reg == R4_H_num ||
 2422     reg ==  R5_num || reg == R5_H_num ||
 2423     reg ==  R6_num || reg == R6_H_num ||
 2424     reg ==  R7_num || reg == R7_H_num ||
 2425     reg ==  V0_num || reg == V0_H_num ||
 2426     reg ==  V1_num || reg == V1_H_num ||
 2427     reg ==  V2_num || reg == V2_H_num ||
 2428     reg ==  V3_num || reg == V3_H_num ||
 2429     reg ==  V4_num || reg == V4_H_num ||
 2430     reg ==  V5_num || reg == V5_H_num ||
 2431     reg ==  V6_num || reg == V6_H_num ||
 2432     reg ==  V7_num || reg == V7_H_num;
 2433 }
 2434 
 2435 bool Matcher::is_spillable_arg(int reg)
 2436 {
 2437   return can_be_java_arg(reg);
 2438 }
 2439 
 2440 uint Matcher::int_pressure_limit()
 2441 {
 2442   // JDK-8183543: When taking the number of available registers as int
 2443   // register pressure threshold, the jtreg test:
 2444   // test/hotspot/jtreg/compiler/regalloc/TestC2IntPressure.java
 2445   // failed due to C2 compilation failure with
 2446   // "COMPILE SKIPPED: failed spill-split-recycle sanity check".
 2447   //
 2448   // A derived pointer is live at CallNode and then is flagged by RA
 2449   // as a spilled LRG. Spilling heuristics(Spill-USE) explicitly skip
 2450   // derived pointers and lastly fail to spill after reaching maximum
 2451   // number of iterations. Lowering the default pressure threshold to
 2452   // (_NO_SPECIAL_REG32_mask.Size() minus 1) forces CallNode to become
 2453   // a high register pressure area of the code so that split_DEF can
 2454   // generate DefinitionSpillCopy for the derived pointer.
 2455   uint default_int_pressure_threshold = _NO_SPECIAL_REG32_mask.Size() - 1;
 2456   if (!PreserveFramePointer) {
 2457     // When PreserveFramePointer is off, frame pointer is allocatable,
 2458     // but different from other SOC registers, it is excluded from
 2459     // fatproj's mask because its save type is No-Save. Decrease 1 to
 2460     // ensure high pressure at fatproj when PreserveFramePointer is off.
 2461     // See check_pressure_at_fatproj().
 2462     default_int_pressure_threshold--;
 2463   }
 2464   return (INTPRESSURE == -1) ? default_int_pressure_threshold : INTPRESSURE;
 2465 }
 2466 
 2467 uint Matcher::float_pressure_limit()
 2468 {
 2469   // _FLOAT_REG_mask is generated by adlc from the float_reg register class.
 2470   return (FLOATPRESSURE == -1) ? _FLOAT_REG_mask.Size() : FLOATPRESSURE;
 2471 }
 2472 
 2473 bool Matcher::use_asm_for_ldiv_by_con(jlong divisor) {
 2474   return false;
 2475 }
 2476 
 2477 RegMask Matcher::divI_proj_mask() {
 2478   ShouldNotReachHere();
 2479   return RegMask();
 2480 }
 2481 
 2482 // Register for MODI projection of divmodI.
 2483 RegMask Matcher::modI_proj_mask() {
 2484   ShouldNotReachHere();
 2485   return RegMask();
 2486 }
 2487 
 2488 // Register for DIVL projection of divmodL.
 2489 RegMask Matcher::divL_proj_mask() {
 2490   ShouldNotReachHere();
 2491   return RegMask();
 2492 }
 2493 
 2494 // Register for MODL projection of divmodL.
 2495 RegMask Matcher::modL_proj_mask() {
 2496   ShouldNotReachHere();
 2497   return RegMask();
 2498 }
 2499 
 2500 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
 2501   return FP_REG_mask();
 2502 }
 2503 
 2504 bool size_fits_all_mem_uses(AddPNode* addp, int shift) {
 2505   for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) {
 2506     Node* u = addp->fast_out(i);
 2507     if (u->is_LoadStore()) {
 2508       // On AArch64, LoadStoreNodes (i.e. compare and swap
 2509       // instructions) only take register indirect as an operand, so
 2510       // any attempt to use an AddPNode as an input to a LoadStoreNode
 2511       // must fail.
 2512       return false;
 2513     }
 2514     if (u->is_Mem()) {
 2515       int opsize = u->as_Mem()->memory_size();
 2516       assert(opsize > 0, "unexpected memory operand size");
 2517       if (u->as_Mem()->memory_size() != (1<<shift)) {
 2518         return false;
 2519       }
 2520     }
 2521   }
 2522   return true;
 2523 }
 2524 
 2525 // Convert BootTest condition to Assembler condition.
 2526 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
 2527 Assembler::Condition to_assembler_cond(BoolTest::mask cond) {
 2528   Assembler::Condition result;
 2529   switch(cond) {
 2530     case BoolTest::eq:
 2531       result = Assembler::EQ; break;
 2532     case BoolTest::ne:
 2533       result = Assembler::NE; break;
 2534     case BoolTest::le:
 2535       result = Assembler::LE; break;
 2536     case BoolTest::ge:
 2537       result = Assembler::GE; break;
 2538     case BoolTest::lt:
 2539       result = Assembler::LT; break;
 2540     case BoolTest::gt:
 2541       result = Assembler::GT; break;
 2542     case BoolTest::ule:
 2543       result = Assembler::LS; break;
 2544     case BoolTest::uge:
 2545       result = Assembler::HS; break;
 2546     case BoolTest::ult:
 2547       result = Assembler::LO; break;
 2548     case BoolTest::ugt:
 2549       result = Assembler::HI; break;
 2550     case BoolTest::overflow:
 2551       result = Assembler::VS; break;
 2552     case BoolTest::no_overflow:
 2553       result = Assembler::VC; break;
 2554     default:
 2555       ShouldNotReachHere();
 2556       return Assembler::Condition(-1);
 2557   }
 2558 
 2559   // Check conversion
 2560   if (cond & BoolTest::unsigned_compare) {
 2561     assert(cmpOpUOper((BoolTest::mask)((int)cond & ~(BoolTest::unsigned_compare))).ccode() == result, "Invalid conversion");
 2562   } else {
 2563     assert(cmpOpOper(cond).ccode() == result, "Invalid conversion");
 2564   }
 2565 
 2566   return result;
 2567 }
 2568 
 2569 // Binary src (Replicate con)
 2570 static bool is_valid_sve_arith_imm_pattern(Node* n, Node* m) {
 2571   if (n == nullptr || m == nullptr) {
 2572     return false;
 2573   }
 2574 
 2575   if (UseSVE == 0 || m->Opcode() != Op_Replicate) {
 2576     return false;
 2577   }
 2578 
 2579   Node* imm_node = m->in(1);
 2580   if (!imm_node->is_Con()) {
 2581     return false;
 2582   }
 2583 
 2584   const Type* t = imm_node->bottom_type();
 2585   if (!(t->isa_int() || t->isa_long())) {
 2586     return false;
 2587   }
 2588 
 2589   switch (n->Opcode()) {
 2590   case Op_AndV:
 2591   case Op_OrV:
 2592   case Op_XorV: {
 2593     Assembler::SIMD_RegVariant T = Assembler::elemType_to_regVariant(Matcher::vector_element_basic_type(n));
 2594     uint64_t value = t->isa_long() ? (uint64_t)imm_node->get_long() : (uint64_t)imm_node->get_int();
 2595     return Assembler::operand_valid_for_sve_logical_immediate(Assembler::regVariant_to_elemBits(T), value);
 2596   }
 2597   case Op_AddVB:
 2598     return (imm_node->get_int() <= 255 && imm_node->get_int() >= -255);
 2599   case Op_AddVS:
 2600   case Op_AddVI:
 2601     return Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)imm_node->get_int());
 2602   case Op_AddVL:
 2603     return Assembler::operand_valid_for_sve_add_sub_immediate(imm_node->get_long());
 2604   default:
 2605     return false;
 2606   }
 2607 }
 2608 
 2609 // (XorV src (Replicate m1))
 2610 // (XorVMask src (MaskAll m1))
 2611 static bool is_vector_bitwise_not_pattern(Node* n, Node* m) {
 2612   if (n != nullptr && m != nullptr) {
 2613     return (n->Opcode() == Op_XorV || n->Opcode() == Op_XorVMask) &&
 2614            VectorNode::is_all_ones_vector(m);
 2615   }
 2616   return false;
 2617 }
 2618 
 2619 // Should the matcher clone input 'm' of node 'n'?
 2620 bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
 2621   if (is_vshift_con_pattern(n, m) ||
 2622       is_vector_bitwise_not_pattern(n, m) ||
 2623       is_valid_sve_arith_imm_pattern(n, m)) {
 2624     mstack.push(m, Visit);
 2625     return true;
 2626   }
 2627   return false;
 2628 }
 2629 
 2630 // Should the Matcher clone shifts on addressing modes, expecting them
 2631 // to be subsumed into complex addressing expressions or compute them
 2632 // into registers?
 2633 bool Matcher::pd_clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
 2634   if (clone_base_plus_offset_address(m, mstack, address_visited)) {
 2635     return true;
 2636   }
 2637 
 2638   Node *off = m->in(AddPNode::Offset);
 2639   if (off->Opcode() == Op_LShiftL && off->in(2)->is_Con() &&
 2640       size_fits_all_mem_uses(m, off->in(2)->get_int()) &&
 2641       // Are there other uses besides address expressions?
 2642       !is_visited(off)) {
 2643     address_visited.set(off->_idx); // Flag as address_visited
 2644     mstack.push(off->in(2), Visit);
 2645     Node *conv = off->in(1);
 2646     if (conv->Opcode() == Op_ConvI2L &&
 2647         // Are there other uses besides address expressions?
 2648         !is_visited(conv)) {
 2649       address_visited.set(conv->_idx); // Flag as address_visited
 2650       mstack.push(conv->in(1), Pre_Visit);
 2651     } else {
 2652       mstack.push(conv, Pre_Visit);
 2653     }
 2654     address_visited.test_set(m->_idx); // Flag as address_visited
 2655     mstack.push(m->in(AddPNode::Address), Pre_Visit);
 2656     mstack.push(m->in(AddPNode::Base), Pre_Visit);
 2657     return true;
 2658   } else if (off->Opcode() == Op_ConvI2L &&
 2659              // Are there other uses besides address expressions?
 2660              !is_visited(off)) {
 2661     address_visited.test_set(m->_idx); // Flag as address_visited
 2662     address_visited.set(off->_idx); // Flag as address_visited
 2663     mstack.push(off->in(1), Pre_Visit);
 2664     mstack.push(m->in(AddPNode::Address), Pre_Visit);
 2665     mstack.push(m->in(AddPNode::Base), Pre_Visit);
 2666     return true;
 2667   }
 2668   return false;
 2669 }
 2670 
 2671 #define MOV_VOLATILE(REG, BASE, INDEX, SCALE, DISP, SCRATCH, INSN)      \
 2672   {                                                                     \
 2673     guarantee(INDEX == -1, "mode not permitted for volatile");          \
 2674     guarantee(DISP == 0, "mode not permitted for volatile");            \
 2675     guarantee(SCALE == 0, "mode not permitted for volatile");           \
 2676     __ INSN(REG, as_Register(BASE));                                    \
 2677   }
 2678 
 2679 
 2680 static Address mem2address(int opcode, Register base, int index, int size, int disp)
 2681   {
 2682     Address::extend scale;
 2683 
 2684     // Hooboy, this is fugly.  We need a way to communicate to the
 2685     // encoder that the index needs to be sign extended, so we have to
 2686     // enumerate all the cases.
 2687     switch (opcode) {
 2688     case INDINDEXSCALEDI2L:
 2689     case INDINDEXSCALEDI2LN:
 2690     case INDINDEXI2L:
 2691     case INDINDEXI2LN:
 2692       scale = Address::sxtw(size);
 2693       break;
 2694     default:
 2695       scale = Address::lsl(size);
 2696     }
 2697 
 2698     if (index == -1) {
 2699       return Address(base, disp);
 2700     } else {
 2701       assert(disp == 0, "unsupported address mode: disp = %d", disp);
 2702       return Address(base, as_Register(index), scale);
 2703     }
 2704   }
 2705 
 2706 
 2707 typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr);
 2708 typedef void (MacroAssembler::* mem_insn2)(Register Rt, Register adr);
 2709 typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr);
 2710 typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt,
 2711                                   MacroAssembler::SIMD_RegVariant T, const Address &adr);
 2712 
 2713   // Used for all non-volatile memory accesses.  The use of
 2714   // $mem->opcode() to discover whether this pattern uses sign-extended
 2715   // offsets is something of a kludge.
 2716   static void loadStore(C2_MacroAssembler* masm, mem_insn insn,
 2717                         Register reg, int opcode,
 2718                         Register base, int index, int scale, int disp,
 2719                         int size_in_memory)
 2720   {
 2721     Address addr = mem2address(opcode, base, index, scale, disp);
 2722     if (addr.getMode() == Address::base_plus_offset) {
 2723       /* Fix up any out-of-range offsets. */
 2724       assert_different_registers(rscratch1, base);
 2725       assert_different_registers(rscratch1, reg);
 2726       addr = __ legitimize_address(addr, size_in_memory, rscratch1);
 2727     }
 2728     (masm->*insn)(reg, addr);
 2729   }
 2730 
 2731   static void loadStore(C2_MacroAssembler* masm, mem_float_insn insn,
 2732                         FloatRegister reg, int opcode,
 2733                         Register base, int index, int size, int disp,
 2734                         int size_in_memory)
 2735   {
 2736     Address::extend scale;
 2737 
 2738     switch (opcode) {
 2739     case INDINDEXSCALEDI2L:
 2740     case INDINDEXSCALEDI2LN:
 2741       scale = Address::sxtw(size);
 2742       break;
 2743     default:
 2744       scale = Address::lsl(size);
 2745     }
 2746 
 2747     if (index == -1) {
 2748       /* If we get an out-of-range offset it is a bug in the compiler,
 2749          so we assert here. */
 2750       assert(Address::offset_ok_for_immed(disp, exact_log2(size_in_memory)), "c2 compiler bug");
 2751       /* Fix up any out-of-range offsets. */
 2752       assert_different_registers(rscratch1, base);
 2753       Address addr = Address(base, disp);
 2754       addr = __ legitimize_address(addr, size_in_memory, rscratch1);
 2755       (masm->*insn)(reg, addr);
 2756     } else {
 2757       assert(disp == 0, "unsupported address mode: disp = %d", disp);
 2758       (masm->*insn)(reg, Address(base, as_Register(index), scale));
 2759     }
 2760   }
 2761 
 2762   static void loadStore(C2_MacroAssembler* masm, mem_vector_insn insn,
 2763                         FloatRegister reg, MacroAssembler::SIMD_RegVariant T,
 2764                         int opcode, Register base, int index, int size, int disp)
 2765   {
 2766     if (index == -1) {
 2767       (masm->*insn)(reg, T, Address(base, disp));
 2768     } else {
 2769       assert(disp == 0, "unsupported address mode");
 2770       (masm->*insn)(reg, T, Address(base, as_Register(index), Address::lsl(size)));
 2771     }
 2772   }
 2773 
 2774 %}
 2775 
 2776 
 2777 
 2778 //----------ENCODING BLOCK-----------------------------------------------------
 2779 // This block specifies the encoding classes used by the compiler to
 2780 // output byte streams.  Encoding classes are parameterized macros
 2781 // used by Machine Instruction Nodes in order to generate the bit
 2782 // encoding of the instruction.  Operands specify their base encoding
 2783 // interface with the interface keyword.  There are currently
 2784 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
 2785 // COND_INTER.  REG_INTER causes an operand to generate a function
 2786 // which returns its register number when queried.  CONST_INTER causes
 2787 // an operand to generate a function which returns the value of the
 2788 // constant when queried.  MEMORY_INTER causes an operand to generate
 2789 // four functions which return the Base Register, the Index Register,
 2790 // the Scale Value, and the Offset Value of the operand when queried.
 2791 // COND_INTER causes an operand to generate six functions which return
 2792 // the encoding code (ie - encoding bits for the instruction)
 2793 // associated with each basic boolean condition for a conditional
 2794 // instruction.
 2795 //
 2796 // Instructions specify two basic values for encoding.  Again, a
 2797 // function is available to check if the constant displacement is an
 2798 // oop. They use the ins_encode keyword to specify their encoding
 2799 // classes (which must be a sequence of enc_class names, and their
 2800 // parameters, specified in the encoding block), and they use the
 2801 // opcode keyword to specify, in order, their primary, secondary, and
 2802 // tertiary opcode.  Only the opcode sections which a particular
 2803 // instruction needs for encoding need to be specified.
 2804 encode %{
 2805   // Build emit functions for each basic byte or larger field in the
 2806   // intel encoding scheme (opcode, rm, sib, immediate), and call them
 2807   // from C++ code in the enc_class source block.  Emit functions will
 2808   // live in the main source block for now.  In future, we can
 2809   // generalize this by adding a syntax that specifies the sizes of
 2810   // fields in an order, so that the adlc can build the emit functions
 2811   // automagically
 2812 
 2813   // catch all for unimplemented encodings
 2814   enc_class enc_unimplemented %{
 2815     __ unimplemented("C2 catch all");
 2816   %}
 2817 
 2818   // BEGIN Non-volatile memory access
 2819 
 2820   // This encoding class is generated automatically from ad_encode.m4.
 2821   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2822   enc_class aarch64_enc_ldrsbw(iRegI dst, memory1 mem) %{
 2823     Register dst_reg = as_Register($dst$$reg);
 2824     loadStore(masm, &MacroAssembler::ldrsbw, dst_reg, $mem->opcode(),
 2825                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2826   %}
 2827 
 2828   // This encoding class is generated automatically from ad_encode.m4.
 2829   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2830   enc_class aarch64_enc_ldrsb(iRegI dst, memory1 mem) %{
 2831     Register dst_reg = as_Register($dst$$reg);
 2832     loadStore(masm, &MacroAssembler::ldrsb, dst_reg, $mem->opcode(),
 2833                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2834   %}
 2835 
 2836   // This encoding class is generated automatically from ad_encode.m4.
 2837   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2838   enc_class aarch64_enc_ldrb(iRegI dst, memory1 mem) %{
 2839     Register dst_reg = as_Register($dst$$reg);
 2840     loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
 2841                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2842   %}
 2843 
 2844   // This encoding class is generated automatically from ad_encode.m4.
 2845   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2846   enc_class aarch64_enc_ldrb(iRegL dst, memory1 mem) %{
 2847     Register dst_reg = as_Register($dst$$reg);
 2848     loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
 2849                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2850   %}
 2851 
 2852   // This encoding class is generated automatically from ad_encode.m4.
 2853   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2854   enc_class aarch64_enc_ldrshw(iRegI dst, memory2 mem) %{
 2855     Register dst_reg = as_Register($dst$$reg);
 2856     loadStore(masm, &MacroAssembler::ldrshw, dst_reg, $mem->opcode(),
 2857                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2858   %}
 2859 
 2860   // This encoding class is generated automatically from ad_encode.m4.
 2861   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2862   enc_class aarch64_enc_ldrsh(iRegI dst, memory2 mem) %{
 2863     Register dst_reg = as_Register($dst$$reg);
 2864     loadStore(masm, &MacroAssembler::ldrsh, dst_reg, $mem->opcode(),
 2865                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2866   %}
 2867 
 2868   // This encoding class is generated automatically from ad_encode.m4.
 2869   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2870   enc_class aarch64_enc_ldrh(iRegI dst, memory2 mem) %{
 2871     Register dst_reg = as_Register($dst$$reg);
 2872     loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
 2873                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2874   %}
 2875 
 2876   // This encoding class is generated automatically from ad_encode.m4.
 2877   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2878   enc_class aarch64_enc_ldrh(iRegL dst, memory2 mem) %{
 2879     Register dst_reg = as_Register($dst$$reg);
 2880     loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
 2881                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2882   %}
 2883 
 2884   // This encoding class is generated automatically from ad_encode.m4.
 2885   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2886   enc_class aarch64_enc_ldrw(iRegI dst, memory4 mem) %{
 2887     Register dst_reg = as_Register($dst$$reg);
 2888     loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
 2889                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2890   %}
 2891 
 2892   // This encoding class is generated automatically from ad_encode.m4.
 2893   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2894   enc_class aarch64_enc_ldrw(iRegL dst, memory4 mem) %{
 2895     Register dst_reg = as_Register($dst$$reg);
 2896     loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
 2897                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2898   %}
 2899 
 2900   // This encoding class is generated automatically from ad_encode.m4.
 2901   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2902   enc_class aarch64_enc_ldrsw(iRegL dst, memory4 mem) %{
 2903     Register dst_reg = as_Register($dst$$reg);
 2904     loadStore(masm, &MacroAssembler::ldrsw, dst_reg, $mem->opcode(),
 2905                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2906   %}
 2907 
 2908   // This encoding class is generated automatically from ad_encode.m4.
 2909   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2910   enc_class aarch64_enc_ldr(iRegL dst, memory8 mem) %{
 2911     Register dst_reg = as_Register($dst$$reg);
 2912     loadStore(masm, &MacroAssembler::ldr, dst_reg, $mem->opcode(),
 2913                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 2914   %}
 2915 
 2916   // This encoding class is generated automatically from ad_encode.m4.
 2917   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2918   enc_class aarch64_enc_ldrs(vRegF dst, memory4 mem) %{
 2919     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 2920     loadStore(masm, &MacroAssembler::ldrs, dst_reg, $mem->opcode(),
 2921                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2922   %}
 2923 
 2924   // This encoding class is generated automatically from ad_encode.m4.
 2925   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2926   enc_class aarch64_enc_ldrd(vRegD dst, memory8 mem) %{
 2927     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 2928     loadStore(masm, &MacroAssembler::ldrd, dst_reg, $mem->opcode(),
 2929                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 2930   %}
 2931 
 2932   // This encoding class is generated automatically from ad_encode.m4.
 2933   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2934   enc_class aarch64_enc_strb(iRegI src, memory1 mem) %{
 2935     Register src_reg = as_Register($src$$reg);
 2936     loadStore(masm, &MacroAssembler::strb, src_reg, $mem->opcode(),
 2937                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2938   %}
 2939 
 2940   // This encoding class is generated automatically from ad_encode.m4.
 2941   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2942   enc_class aarch64_enc_strb0(memory1 mem) %{
 2943     loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
 2944                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 2945   %}
 2946 
 2947   // This encoding class is generated automatically from ad_encode.m4.
 2948   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2949   enc_class aarch64_enc_strh(iRegI src, memory2 mem) %{
 2950     Register src_reg = as_Register($src$$reg);
 2951     loadStore(masm, &MacroAssembler::strh, src_reg, $mem->opcode(),
 2952                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2953   %}
 2954 
 2955   // This encoding class is generated automatically from ad_encode.m4.
 2956   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2957   enc_class aarch64_enc_strh0(memory2 mem) %{
 2958     loadStore(masm, &MacroAssembler::strh, zr, $mem->opcode(),
 2959                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
 2960   %}
 2961 
 2962   // This encoding class is generated automatically from ad_encode.m4.
 2963   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2964   enc_class aarch64_enc_strw(iRegI src, memory4 mem) %{
 2965     Register src_reg = as_Register($src$$reg);
 2966     loadStore(masm, &MacroAssembler::strw, src_reg, $mem->opcode(),
 2967                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2968   %}
 2969 
 2970   // This encoding class is generated automatically from ad_encode.m4.
 2971   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2972   enc_class aarch64_enc_strw0(memory4 mem) %{
 2973     loadStore(masm, &MacroAssembler::strw, zr, $mem->opcode(),
 2974                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 2975   %}
 2976 
 2977   // This encoding class is generated automatically from ad_encode.m4.
 2978   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2979   enc_class aarch64_enc_str(iRegL src, memory8 mem) %{
 2980     Register src_reg = as_Register($src$$reg);
 2981     // we sometimes get asked to store the stack pointer into the
 2982     // current thread -- we cannot do that directly on AArch64
 2983     if (src_reg == r31_sp) {
 2984       assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
 2985       __ mov(rscratch2, sp);
 2986       src_reg = rscratch2;
 2987     }
 2988     loadStore(masm, &MacroAssembler::str, src_reg, $mem->opcode(),
 2989                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 2990   %}
 2991 
 2992   // This encoding class is generated automatically from ad_encode.m4.
 2993   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 2994   enc_class aarch64_enc_str0(memory8 mem) %{
 2995     loadStore(masm, &MacroAssembler::str, zr, $mem->opcode(),
 2996                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 2997   %}
 2998 
 2999   // This encoding class is generated automatically from ad_encode.m4.
 3000   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3001   enc_class aarch64_enc_strs(vRegF src, memory4 mem) %{
 3002     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3003     loadStore(masm, &MacroAssembler::strs, src_reg, $mem->opcode(),
 3004                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 3005   %}
 3006 
 3007   // This encoding class is generated automatically from ad_encode.m4.
 3008   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3009   enc_class aarch64_enc_strd(vRegD src, memory8 mem) %{
 3010     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3011     loadStore(masm, &MacroAssembler::strd, src_reg, $mem->opcode(),
 3012                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 3013   %}
 3014 
 3015   // This encoding class is generated automatically from ad_encode.m4.
 3016   // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 3017   enc_class aarch64_enc_strb0_ordered(memory4 mem) %{
 3018       __ membar(Assembler::StoreStore);
 3019       loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
 3020                as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
 3021   %}
 3022 
 3023   // END Non-volatile memory access
 3024 
 3025   // Vector loads and stores
 3026   enc_class aarch64_enc_ldrvH(vReg dst, memory mem) %{
 3027     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3028     loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::H,
 3029        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3030   %}
 3031 
 3032   enc_class aarch64_enc_ldrvS(vReg dst, memory mem) %{
 3033     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3034     loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::S,
 3035        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3036   %}
 3037 
 3038   enc_class aarch64_enc_ldrvD(vReg dst, memory mem) %{
 3039     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3040     loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::D,
 3041        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3042   %}
 3043 
 3044   enc_class aarch64_enc_ldrvQ(vReg dst, memory mem) %{
 3045     FloatRegister dst_reg = as_FloatRegister($dst$$reg);
 3046     loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::Q,
 3047        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3048   %}
 3049 
 3050   enc_class aarch64_enc_strvH(vReg src, memory mem) %{
 3051     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3052     loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::H,
 3053        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3054   %}
 3055 
 3056   enc_class aarch64_enc_strvS(vReg src, memory mem) %{
 3057     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3058     loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::S,
 3059        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3060   %}
 3061 
 3062   enc_class aarch64_enc_strvD(vReg src, memory mem) %{
 3063     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3064     loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::D,
 3065        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3066   %}
 3067 
 3068   enc_class aarch64_enc_strvQ(vReg src, memory mem) %{
 3069     FloatRegister src_reg = as_FloatRegister($src$$reg);
 3070     loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::Q,
 3071        $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
 3072   %}
 3073 
 3074   // volatile loads and stores
 3075 
 3076   enc_class aarch64_enc_stlrb(iRegI src, memory mem) %{
 3077     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3078                  rscratch1, stlrb);
 3079   %}
 3080 
 3081   enc_class aarch64_enc_stlrb0(memory mem) %{
 3082     MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3083                  rscratch1, stlrb);
 3084   %}
 3085 
 3086   enc_class aarch64_enc_stlrh(iRegI src, memory mem) %{
 3087     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3088                  rscratch1, stlrh);
 3089   %}
 3090 
 3091   enc_class aarch64_enc_stlrh0(memory mem) %{
 3092     MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3093                  rscratch1, stlrh);
 3094   %}
 3095 
 3096   enc_class aarch64_enc_stlrw(iRegI src, memory mem) %{
 3097     MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3098                  rscratch1, stlrw);
 3099   %}
 3100 
 3101   enc_class aarch64_enc_stlrw0(memory mem) %{
 3102     MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3103                  rscratch1, stlrw);
 3104   %}
 3105 
 3106   enc_class aarch64_enc_ldarsbw(iRegI dst, memory mem) %{
 3107     Register dst_reg = as_Register($dst$$reg);
 3108     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3109              rscratch1, ldarb);
 3110     __ sxtbw(dst_reg, dst_reg);
 3111   %}
 3112 
 3113   enc_class aarch64_enc_ldarsb(iRegL dst, memory mem) %{
 3114     Register dst_reg = as_Register($dst$$reg);
 3115     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3116              rscratch1, ldarb);
 3117     __ sxtb(dst_reg, dst_reg);
 3118   %}
 3119 
 3120   enc_class aarch64_enc_ldarbw(iRegI dst, memory mem) %{
 3121     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3122              rscratch1, ldarb);
 3123   %}
 3124 
 3125   enc_class aarch64_enc_ldarb(iRegL dst, memory mem) %{
 3126     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3127              rscratch1, ldarb);
 3128   %}
 3129 
 3130   enc_class aarch64_enc_ldarshw(iRegI dst, memory mem) %{
 3131     Register dst_reg = as_Register($dst$$reg);
 3132     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3133              rscratch1, ldarh);
 3134     __ sxthw(dst_reg, dst_reg);
 3135   %}
 3136 
 3137   enc_class aarch64_enc_ldarsh(iRegL dst, memory mem) %{
 3138     Register dst_reg = as_Register($dst$$reg);
 3139     MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3140              rscratch1, ldarh);
 3141     __ sxth(dst_reg, dst_reg);
 3142   %}
 3143 
 3144   enc_class aarch64_enc_ldarhw(iRegI dst, memory mem) %{
 3145     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3146              rscratch1, ldarh);
 3147   %}
 3148 
 3149   enc_class aarch64_enc_ldarh(iRegL dst, memory mem) %{
 3150     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3151              rscratch1, ldarh);
 3152   %}
 3153 
 3154   enc_class aarch64_enc_ldarw(iRegI dst, memory mem) %{
 3155     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3156              rscratch1, ldarw);
 3157   %}
 3158 
 3159   enc_class aarch64_enc_ldarw(iRegL dst, memory mem) %{
 3160     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3161              rscratch1, ldarw);
 3162   %}
 3163 
 3164   enc_class aarch64_enc_ldar(iRegL dst, memory mem) %{
 3165     MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3166              rscratch1, ldar);
 3167   %}
 3168 
 3169   enc_class aarch64_enc_fldars(vRegF dst, memory mem) %{
 3170     MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3171              rscratch1, ldarw);
 3172     __ fmovs(as_FloatRegister($dst$$reg), rscratch1);
 3173   %}
 3174 
 3175   enc_class aarch64_enc_fldard(vRegD dst, memory mem) %{
 3176     MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3177              rscratch1, ldar);
 3178     __ fmovd(as_FloatRegister($dst$$reg), rscratch1);
 3179   %}
 3180 
 3181   enc_class aarch64_enc_stlr(iRegL src, memory mem) %{
 3182     Register src_reg = as_Register($src$$reg);
 3183     // we sometimes get asked to store the stack pointer into the
 3184     // current thread -- we cannot do that directly on AArch64
 3185     if (src_reg == r31_sp) {
 3186       assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
 3187       __ mov(rscratch2, sp);
 3188       src_reg = rscratch2;
 3189     }
 3190     MOV_VOLATILE(src_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3191                  rscratch1, stlr);
 3192   %}
 3193 
 3194   enc_class aarch64_enc_stlr0(memory mem) %{
 3195     MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3196                  rscratch1, stlr);
 3197   %}
 3198 
 3199   enc_class aarch64_enc_fstlrs(vRegF src, memory mem) %{
 3200     {
 3201       FloatRegister src_reg = as_FloatRegister($src$$reg);
 3202       __ fmovs(rscratch2, src_reg);
 3203     }
 3204     MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3205                  rscratch1, stlrw);
 3206   %}
 3207 
 3208   enc_class aarch64_enc_fstlrd(vRegD src, memory mem) %{
 3209     {
 3210       FloatRegister src_reg = as_FloatRegister($src$$reg);
 3211       __ fmovd(rscratch2, src_reg);
 3212     }
 3213     MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
 3214                  rscratch1, stlr);
 3215   %}
 3216 
 3217   // synchronized read/update encodings
 3218 
 3219   enc_class aarch64_enc_ldaxr(iRegL dst, memory8 mem) %{
 3220     Register dst_reg = as_Register($dst$$reg);
 3221     Register base = as_Register($mem$$base);
 3222     int index = $mem$$index;
 3223     int scale = $mem$$scale;
 3224     int disp = $mem$$disp;
 3225     if (index == -1) {
 3226        if (disp != 0) {
 3227         __ lea(rscratch1, Address(base, disp));
 3228         __ ldaxr(dst_reg, rscratch1);
 3229       } else {
 3230         // TODO
 3231         // should we ever get anything other than this case?
 3232         __ ldaxr(dst_reg, base);
 3233       }
 3234     } else {
 3235       Register index_reg = as_Register(index);
 3236       if (disp == 0) {
 3237         __ lea(rscratch1, Address(base, index_reg, Address::lsl(scale)));
 3238         __ ldaxr(dst_reg, rscratch1);
 3239       } else {
 3240         __ lea(rscratch1, Address(base, disp));
 3241         __ lea(rscratch1, Address(rscratch1, index_reg, Address::lsl(scale)));
 3242         __ ldaxr(dst_reg, rscratch1);
 3243       }
 3244     }
 3245   %}
 3246 
 3247   enc_class aarch64_enc_stlxr(iRegLNoSp src, memory8 mem) %{
 3248     Register src_reg = as_Register($src$$reg);
 3249     Register base = as_Register($mem$$base);
 3250     int index = $mem$$index;
 3251     int scale = $mem$$scale;
 3252     int disp = $mem$$disp;
 3253     if (index == -1) {
 3254        if (disp != 0) {
 3255         __ lea(rscratch2, Address(base, disp));
 3256         __ stlxr(rscratch1, src_reg, rscratch2);
 3257       } else {
 3258         // TODO
 3259         // should we ever get anything other than this case?
 3260         __ stlxr(rscratch1, src_reg, base);
 3261       }
 3262     } else {
 3263       Register index_reg = as_Register(index);
 3264       if (disp == 0) {
 3265         __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
 3266         __ stlxr(rscratch1, src_reg, rscratch2);
 3267       } else {
 3268         __ lea(rscratch2, Address(base, disp));
 3269         __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
 3270         __ stlxr(rscratch1, src_reg, rscratch2);
 3271       }
 3272     }
 3273     __ cmpw(rscratch1, zr);
 3274   %}
 3275 
 3276   enc_class aarch64_enc_cmpxchg(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
 3277     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3278     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3279                Assembler::xword, /*acquire*/ false, /*release*/ true,
 3280                /*weak*/ false, noreg);
 3281   %}
 3282 
 3283   enc_class aarch64_enc_cmpxchgw(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3284     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3285     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3286                Assembler::word, /*acquire*/ false, /*release*/ true,
 3287                /*weak*/ false, noreg);
 3288   %}
 3289 
 3290   enc_class aarch64_enc_cmpxchgs(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3291     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3292     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3293                Assembler::halfword, /*acquire*/ false, /*release*/ true,
 3294                /*weak*/ false, noreg);
 3295   %}
 3296 
 3297   enc_class aarch64_enc_cmpxchgb(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3298     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3299     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3300                Assembler::byte, /*acquire*/ false, /*release*/ true,
 3301                /*weak*/ false, noreg);
 3302   %}
 3303 
 3304 
 3305   // The only difference between aarch64_enc_cmpxchg and
 3306   // aarch64_enc_cmpxchg_acq is that we use load-acquire in the
 3307   // CompareAndSwap sequence to serve as a barrier on acquiring a
 3308   // lock.
 3309   enc_class aarch64_enc_cmpxchg_acq(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
 3310     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3311     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3312                Assembler::xword, /*acquire*/ true, /*release*/ true,
 3313                /*weak*/ false, noreg);
 3314   %}
 3315 
 3316   enc_class aarch64_enc_cmpxchgw_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3317     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3318     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3319                Assembler::word, /*acquire*/ true, /*release*/ true,
 3320                /*weak*/ false, noreg);
 3321   %}
 3322 
 3323   enc_class aarch64_enc_cmpxchgs_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3324     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3325     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3326                Assembler::halfword, /*acquire*/ true, /*release*/ true,
 3327                /*weak*/ false, noreg);
 3328   %}
 3329 
 3330   enc_class aarch64_enc_cmpxchgb_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
 3331     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
 3332     __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
 3333                Assembler::byte, /*acquire*/ true, /*release*/ true,
 3334                /*weak*/ false, noreg);
 3335   %}
 3336 
 3337   // auxiliary used for CompareAndSwapX to set result register
 3338   enc_class aarch64_enc_cset_eq(iRegINoSp res) %{
 3339     Register res_reg = as_Register($res$$reg);
 3340     __ cset(res_reg, Assembler::EQ);
 3341   %}
 3342 
 3343   // prefetch encodings
 3344 
 3345   enc_class aarch64_enc_prefetchw(memory mem) %{
 3346     Register base = as_Register($mem$$base);
 3347     int index = $mem$$index;
 3348     int scale = $mem$$scale;
 3349     int disp = $mem$$disp;
 3350     if (index == -1) {
 3351       __ prfm(Address(base, disp), PSTL1KEEP);
 3352     } else {
 3353       Register index_reg = as_Register(index);
 3354       if (disp == 0) {
 3355         __ prfm(Address(base, index_reg, Address::lsl(scale)), PSTL1KEEP);
 3356       } else {
 3357         __ lea(rscratch1, Address(base, disp));
 3358 	__ prfm(Address(rscratch1, index_reg, Address::lsl(scale)), PSTL1KEEP);
 3359       }
 3360     }
 3361   %}
 3362 
 3363   /// mov envcodings
 3364 
 3365   enc_class aarch64_enc_movw_imm(iRegI dst, immI src) %{
 3366     uint32_t con = (uint32_t)$src$$constant;
 3367     Register dst_reg = as_Register($dst$$reg);
 3368     if (con == 0) {
 3369       __ movw(dst_reg, zr);
 3370     } else {
 3371       __ movw(dst_reg, con);
 3372     }
 3373   %}
 3374 
 3375   enc_class aarch64_enc_mov_imm(iRegL dst, immL src) %{
 3376     Register dst_reg = as_Register($dst$$reg);
 3377     uint64_t con = (uint64_t)$src$$constant;
 3378     if (con == 0) {
 3379       __ mov(dst_reg, zr);
 3380     } else {
 3381       __ mov(dst_reg, con);
 3382     }
 3383   %}
 3384 
 3385   enc_class aarch64_enc_mov_p(iRegP dst, immP src) %{
 3386     Register dst_reg = as_Register($dst$$reg);
 3387     address con = (address)$src$$constant;
 3388     if (con == nullptr || con == (address)1) {
 3389       ShouldNotReachHere();
 3390     } else {
 3391       relocInfo::relocType rtype = $src->constant_reloc();
 3392       if (rtype == relocInfo::oop_type) {
 3393         __ movoop(dst_reg, (jobject)con);
 3394       } else if (rtype == relocInfo::metadata_type) {
 3395         __ mov_metadata(dst_reg, (Metadata*)con);
 3396       } else {
 3397         assert(rtype == relocInfo::none || rtype == relocInfo::external_word_type, "unexpected reloc type");
 3398         if (! __ is_valid_AArch64_address(con) ||
 3399             con < (address)(uintptr_t)os::vm_page_size()) {
 3400           __ mov(dst_reg, con);
 3401         } else {
 3402           uint64_t offset;
 3403           __ adrp(dst_reg, con, offset);
 3404           __ add(dst_reg, dst_reg, offset);
 3405         }
 3406       }
 3407     }
 3408   %}
 3409 
 3410   enc_class aarch64_enc_mov_p0(iRegP dst, immP0 src) %{
 3411     Register dst_reg = as_Register($dst$$reg);
 3412     __ mov(dst_reg, zr);
 3413   %}
 3414 
 3415   enc_class aarch64_enc_mov_p1(iRegP dst, immP_1 src) %{
 3416     Register dst_reg = as_Register($dst$$reg);
 3417     __ mov(dst_reg, (uint64_t)1);
 3418   %}
 3419 
 3420   enc_class aarch64_enc_mov_byte_map_base(iRegP dst, immByteMapBase src) %{
 3421     __ load_byte_map_base($dst$$Register);
 3422   %}
 3423 
 3424   enc_class aarch64_enc_mov_n(iRegN dst, immN src) %{
 3425     Register dst_reg = as_Register($dst$$reg);
 3426     address con = (address)$src$$constant;
 3427     if (con == nullptr) {
 3428       ShouldNotReachHere();
 3429     } else {
 3430       relocInfo::relocType rtype = $src->constant_reloc();
 3431       assert(rtype == relocInfo::oop_type, "unexpected reloc type");
 3432       __ set_narrow_oop(dst_reg, (jobject)con);
 3433     }
 3434   %}
 3435 
 3436   enc_class aarch64_enc_mov_n0(iRegN dst, immN0 src) %{
 3437     Register dst_reg = as_Register($dst$$reg);
 3438     __ mov(dst_reg, zr);
 3439   %}
 3440 
 3441   enc_class aarch64_enc_mov_nk(iRegN dst, immNKlass src) %{
 3442     Register dst_reg = as_Register($dst$$reg);
 3443     address con = (address)$src$$constant;
 3444     if (con == nullptr) {
 3445       ShouldNotReachHere();
 3446     } else {
 3447       relocInfo::relocType rtype = $src->constant_reloc();
 3448       assert(rtype == relocInfo::metadata_type, "unexpected reloc type");
 3449       __ set_narrow_klass(dst_reg, (Klass *)con);
 3450     }
 3451   %}
 3452 
 3453   // arithmetic encodings
 3454 
 3455   enc_class aarch64_enc_addsubw_imm(iRegI dst, iRegI src1, immIAddSub src2) %{
 3456     Register dst_reg = as_Register($dst$$reg);
 3457     Register src_reg = as_Register($src1$$reg);
 3458     int32_t con = (int32_t)$src2$$constant;
 3459     // add has primary == 0, subtract has primary == 1
 3460     if ($primary) { con = -con; }
 3461     if (con < 0) {
 3462       __ subw(dst_reg, src_reg, -con);
 3463     } else {
 3464       __ addw(dst_reg, src_reg, con);
 3465     }
 3466   %}
 3467 
 3468   enc_class aarch64_enc_addsub_imm(iRegL dst, iRegL src1, immLAddSub src2) %{
 3469     Register dst_reg = as_Register($dst$$reg);
 3470     Register src_reg = as_Register($src1$$reg);
 3471     int32_t con = (int32_t)$src2$$constant;
 3472     // add has primary == 0, subtract has primary == 1
 3473     if ($primary) { con = -con; }
 3474     if (con < 0) {
 3475       __ sub(dst_reg, src_reg, -con);
 3476     } else {
 3477       __ add(dst_reg, src_reg, con);
 3478     }
 3479   %}
 3480 
 3481   enc_class aarch64_enc_divw(iRegI dst, iRegI src1, iRegI src2) %{
 3482    Register dst_reg = as_Register($dst$$reg);
 3483    Register src1_reg = as_Register($src1$$reg);
 3484    Register src2_reg = as_Register($src2$$reg);
 3485     __ corrected_idivl(dst_reg, src1_reg, src2_reg, false, rscratch1);
 3486   %}
 3487 
 3488   enc_class aarch64_enc_div(iRegI dst, iRegI src1, iRegI src2) %{
 3489    Register dst_reg = as_Register($dst$$reg);
 3490    Register src1_reg = as_Register($src1$$reg);
 3491    Register src2_reg = as_Register($src2$$reg);
 3492     __ corrected_idivq(dst_reg, src1_reg, src2_reg, false, rscratch1);
 3493   %}
 3494 
 3495   enc_class aarch64_enc_modw(iRegI dst, iRegI src1, iRegI src2) %{
 3496    Register dst_reg = as_Register($dst$$reg);
 3497    Register src1_reg = as_Register($src1$$reg);
 3498    Register src2_reg = as_Register($src2$$reg);
 3499     __ corrected_idivl(dst_reg, src1_reg, src2_reg, true, rscratch1);
 3500   %}
 3501 
 3502   enc_class aarch64_enc_mod(iRegI dst, iRegI src1, iRegI src2) %{
 3503    Register dst_reg = as_Register($dst$$reg);
 3504    Register src1_reg = as_Register($src1$$reg);
 3505    Register src2_reg = as_Register($src2$$reg);
 3506     __ corrected_idivq(dst_reg, src1_reg, src2_reg, true, rscratch1);
 3507   %}
 3508 
 3509   // compare instruction encodings
 3510 
 3511   enc_class aarch64_enc_cmpw(iRegI src1, iRegI src2) %{
 3512     Register reg1 = as_Register($src1$$reg);
 3513     Register reg2 = as_Register($src2$$reg);
 3514     __ cmpw(reg1, reg2);
 3515   %}
 3516 
 3517   enc_class aarch64_enc_cmpw_imm_addsub(iRegI src1, immIAddSub src2) %{
 3518     Register reg = as_Register($src1$$reg);
 3519     int32_t val = $src2$$constant;
 3520     if (val >= 0) {
 3521       __ subsw(zr, reg, val);
 3522     } else {
 3523       __ addsw(zr, reg, -val);
 3524     }
 3525   %}
 3526 
 3527   enc_class aarch64_enc_cmpw_imm(iRegI src1, immI src2) %{
 3528     Register reg1 = as_Register($src1$$reg);
 3529     uint32_t val = (uint32_t)$src2$$constant;
 3530     __ movw(rscratch1, val);
 3531     __ cmpw(reg1, rscratch1);
 3532   %}
 3533 
 3534   enc_class aarch64_enc_cmp(iRegL src1, iRegL src2) %{
 3535     Register reg1 = as_Register($src1$$reg);
 3536     Register reg2 = as_Register($src2$$reg);
 3537     __ cmp(reg1, reg2);
 3538   %}
 3539 
 3540   enc_class aarch64_enc_cmp_imm_addsub(iRegL src1, immL12 src2) %{
 3541     Register reg = as_Register($src1$$reg);
 3542     int64_t val = $src2$$constant;
 3543     if (val >= 0) {
 3544       __ subs(zr, reg, val);
 3545     } else if (val != -val) {
 3546       __ adds(zr, reg, -val);
 3547     } else {
 3548     // aargh, Long.MIN_VALUE is a special case
 3549       __ orr(rscratch1, zr, (uint64_t)val);
 3550       __ subs(zr, reg, rscratch1);
 3551     }
 3552   %}
 3553 
 3554   enc_class aarch64_enc_cmp_imm(iRegL src1, immL src2) %{
 3555     Register reg1 = as_Register($src1$$reg);
 3556     uint64_t val = (uint64_t)$src2$$constant;
 3557     __ mov(rscratch1, val);
 3558     __ cmp(reg1, rscratch1);
 3559   %}
 3560 
 3561   enc_class aarch64_enc_cmpp(iRegP src1, iRegP src2) %{
 3562     Register reg1 = as_Register($src1$$reg);
 3563     Register reg2 = as_Register($src2$$reg);
 3564     __ cmp(reg1, reg2);
 3565   %}
 3566 
 3567   enc_class aarch64_enc_cmpn(iRegN src1, iRegN src2) %{
 3568     Register reg1 = as_Register($src1$$reg);
 3569     Register reg2 = as_Register($src2$$reg);
 3570     __ cmpw(reg1, reg2);
 3571   %}
 3572 
 3573   enc_class aarch64_enc_testp(iRegP src) %{
 3574     Register reg = as_Register($src$$reg);
 3575     __ cmp(reg, zr);
 3576   %}
 3577 
 3578   enc_class aarch64_enc_testn(iRegN src) %{
 3579     Register reg = as_Register($src$$reg);
 3580     __ cmpw(reg, zr);
 3581   %}
 3582 
 3583   enc_class aarch64_enc_b(label lbl) %{
 3584     Label *L = $lbl$$label;
 3585     __ b(*L);
 3586   %}
 3587 
 3588   enc_class aarch64_enc_br_con(cmpOp cmp, label lbl) %{
 3589     Label *L = $lbl$$label;
 3590     __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
 3591   %}
 3592 
 3593   enc_class aarch64_enc_br_conU(cmpOpU cmp, label lbl) %{
 3594     Label *L = $lbl$$label;
 3595     __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
 3596   %}
 3597 
 3598   enc_class aarch64_enc_partial_subtype_check(iRegP sub, iRegP super, iRegP temp, iRegP result)
 3599   %{
 3600      Register sub_reg = as_Register($sub$$reg);
 3601      Register super_reg = as_Register($super$$reg);
 3602      Register temp_reg = as_Register($temp$$reg);
 3603      Register result_reg = as_Register($result$$reg);
 3604 
 3605      Label miss;
 3606      __ check_klass_subtype_slow_path(sub_reg, super_reg, temp_reg, result_reg,
 3607                                      nullptr, &miss,
 3608                                      /*set_cond_codes:*/ true);
 3609      if ($primary) {
 3610        __ mov(result_reg, zr);
 3611      }
 3612      __ bind(miss);
 3613   %}
 3614 
 3615   enc_class aarch64_enc_java_static_call(method meth) %{
 3616     address addr = (address)$meth$$method;
 3617     address call;
 3618     if (!_method) {
 3619       // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
 3620       call = __ trampoline_call(Address(addr, relocInfo::runtime_call_type));
 3621       if (call == nullptr) {
 3622         ciEnv::current()->record_failure("CodeCache is full");
 3623         return;
 3624       }
 3625     } else if (_method->intrinsic_id() == vmIntrinsicID::_ensureMaterializedForStackWalk) {
 3626       // The NOP here is purely to ensure that eliding a call to
 3627       // JVM_EnsureMaterializedForStackWalk doesn't change the code size.
 3628       __ nop();
 3629       __ block_comment("call JVM_EnsureMaterializedForStackWalk (elided)");
 3630     } else {
 3631       int method_index = resolved_method_index(masm);
 3632       RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
 3633                                                   : static_call_Relocation::spec(method_index);
 3634       call = __ trampoline_call(Address(addr, rspec));
 3635       if (call == nullptr) {
 3636         ciEnv::current()->record_failure("CodeCache is full");
 3637         return;
 3638       }
 3639       if (CodeBuffer::supports_shared_stubs() && _method->can_be_statically_bound()) {
 3640         // Calls of the same statically bound method can share
 3641         // a stub to the interpreter.
 3642         __ code()->shared_stub_to_interp_for(_method, call - __ begin());
 3643       } else {
 3644         // Emit stub for static call
 3645         address stub = CompiledDirectCall::emit_to_interp_stub(masm, call);
 3646         if (stub == nullptr) {
 3647           ciEnv::current()->record_failure("CodeCache is full");
 3648           return;
 3649         }
 3650       }
 3651     }
 3652 
 3653     __ post_call_nop();
 3654 
 3655     // Only non uncommon_trap calls need to reinitialize ptrue.
 3656     if (Compile::current()->max_vector_size() > 0 && uncommon_trap_request() == 0) {
 3657       __ reinitialize_ptrue();
 3658     }
 3659   %}
 3660 
 3661   enc_class aarch64_enc_java_dynamic_call(method meth) %{
 3662     int method_index = resolved_method_index(masm);
 3663     address call = __ ic_call((address)$meth$$method, method_index);
 3664     if (call == nullptr) {
 3665       ciEnv::current()->record_failure("CodeCache is full");
 3666       return;
 3667     }
 3668     __ post_call_nop();
 3669     if (Compile::current()->max_vector_size() > 0) {
 3670       __ reinitialize_ptrue();
 3671     }
 3672   %}
 3673 
 3674   enc_class aarch64_enc_call_epilog() %{
 3675     if (VerifyStackAtCalls) {
 3676       // Check that stack depth is unchanged: find majik cookie on stack
 3677       __ call_Unimplemented();
 3678     }
 3679   %}
 3680 
 3681   enc_class aarch64_enc_java_to_runtime(method meth) %{
 3682     // some calls to generated routines (arraycopy code) are scheduled
 3683     // by C2 as runtime calls. if so we can call them using a br (they
 3684     // will be in a reachable segment) otherwise we have to use a blr
 3685     // which loads the absolute address into a register.
 3686     address entry = (address)$meth$$method;
 3687     CodeBlob *cb = CodeCache::find_blob(entry);
 3688     if (cb) {
 3689       address call = __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
 3690       if (call == nullptr) {
 3691         ciEnv::current()->record_failure("CodeCache is full");
 3692         return;
 3693       }
 3694       __ post_call_nop();
 3695     } else {
 3696       Label retaddr;
 3697       __ adr(rscratch2, retaddr);
 3698       __ lea(rscratch1, RuntimeAddress(entry));
 3699       // Leave a breadcrumb for JavaFrameAnchor::capture_last_Java_pc()
 3700       __ stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)));
 3701       __ blr(rscratch1);
 3702       __ bind(retaddr);
 3703       __ post_call_nop();
 3704       __ add(sp, sp, 2 * wordSize);
 3705     }
 3706     if (Compile::current()->max_vector_size() > 0) {
 3707       __ reinitialize_ptrue();
 3708     }
 3709   %}
 3710 
 3711   enc_class aarch64_enc_rethrow() %{
 3712     __ far_jump(RuntimeAddress(OptoRuntime::rethrow_stub()));
 3713   %}
 3714 
 3715   enc_class aarch64_enc_ret() %{
 3716 #ifdef ASSERT
 3717     if (Compile::current()->max_vector_size() > 0) {
 3718       __ verify_ptrue();
 3719     }
 3720 #endif
 3721     __ ret(lr);
 3722   %}
 3723 
 3724   enc_class aarch64_enc_tail_call(iRegP jump_target) %{
 3725     Register target_reg = as_Register($jump_target$$reg);
 3726     __ br(target_reg);
 3727   %}
 3728 
 3729   enc_class aarch64_enc_tail_jmp(iRegP jump_target) %{
 3730     Register target_reg = as_Register($jump_target$$reg);
 3731     // exception oop should be in r0
 3732     // ret addr has been popped into lr
 3733     // callee expects it in r3
 3734     __ mov(r3, lr);
 3735     __ br(target_reg);
 3736   %}
 3737 
 3738 %}
 3739 
 3740 //----------FRAME--------------------------------------------------------------
 3741 // Definition of frame structure and management information.
 3742 //
 3743 //  S T A C K   L A Y O U T    Allocators stack-slot number
 3744 //                             |   (to get allocators register number
 3745 //  G  Owned by    |        |  v    add OptoReg::stack0())
 3746 //  r   CALLER     |        |
 3747 //  o     |        +--------+      pad to even-align allocators stack-slot
 3748 //  w     V        |  pad0  |        numbers; owned by CALLER
 3749 //  t   -----------+--------+----> Matcher::_in_arg_limit, unaligned
 3750 //  h     ^        |   in   |  5
 3751 //        |        |  args  |  4   Holes in incoming args owned by SELF
 3752 //  |     |        |        |  3
 3753 //  |     |        +--------+
 3754 //  V     |        | old out|      Empty on Intel, window on Sparc
 3755 //        |    old |preserve|      Must be even aligned.
 3756 //        |     SP-+--------+----> Matcher::_old_SP, even aligned
 3757 //        |        |   in   |  3   area for Intel ret address
 3758 //     Owned by    |preserve|      Empty on Sparc.
 3759 //       SELF      +--------+
 3760 //        |        |  pad2  |  2   pad to align old SP
 3761 //        |        +--------+  1
 3762 //        |        | locks  |  0
 3763 //        |        +--------+----> OptoReg::stack0(), even aligned
 3764 //        |        |  pad1  | 11   pad to align new SP
 3765 //        |        +--------+
 3766 //        |        |        | 10
 3767 //        |        | spills |  9   spills
 3768 //        V        |        |  8   (pad0 slot for callee)
 3769 //      -----------+--------+----> Matcher::_out_arg_limit, unaligned
 3770 //        ^        |  out   |  7
 3771 //        |        |  args  |  6   Holes in outgoing args owned by CALLEE
 3772 //     Owned by    +--------+
 3773 //      CALLEE     | new out|  6   Empty on Intel, window on Sparc
 3774 //        |    new |preserve|      Must be even-aligned.
 3775 //        |     SP-+--------+----> Matcher::_new_SP, even aligned
 3776 //        |        |        |
 3777 //
 3778 // Note 1: Only region 8-11 is determined by the allocator.  Region 0-5 is
 3779 //         known from SELF's arguments and the Java calling convention.
 3780 //         Region 6-7 is determined per call site.
 3781 // Note 2: If the calling convention leaves holes in the incoming argument
 3782 //         area, those holes are owned by SELF.  Holes in the outgoing area
 3783 //         are owned by the CALLEE.  Holes should not be necessary in the
 3784 //         incoming area, as the Java calling convention is completely under
 3785 //         the control of the AD file.  Doubles can be sorted and packed to
 3786 //         avoid holes.  Holes in the outgoing arguments may be necessary for
 3787 //         varargs C calling conventions.
 3788 // Note 3: Region 0-3 is even aligned, with pad2 as needed.  Region 3-5 is
 3789 //         even aligned with pad0 as needed.
 3790 //         Region 6 is even aligned.  Region 6-7 is NOT even aligned;
 3791 //           (the latter is true on Intel but is it false on AArch64?)
 3792 //         region 6-11 is even aligned; it may be padded out more so that
 3793 //         the region from SP to FP meets the minimum stack alignment.
 3794 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
 3795 //         alignment.  Region 11, pad1, may be dynamically extended so that
 3796 //         SP meets the minimum alignment.
 3797 
 3798 frame %{
 3799   // These three registers define part of the calling convention
 3800   // between compiled code and the interpreter.
 3801 
 3802   // Inline Cache Register or Method for I2C.
 3803   inline_cache_reg(R12);
 3804 
 3805   // Number of stack slots consumed by locking an object
 3806   sync_stack_slots(2);
 3807 
 3808   // Compiled code's Frame Pointer
 3809   frame_pointer(R31);
 3810 
 3811   // Interpreter stores its frame pointer in a register which is
 3812   // stored to the stack by I2CAdaptors.
 3813   // I2CAdaptors convert from interpreted java to compiled java.
 3814   interpreter_frame_pointer(R29);
 3815 
 3816   // Stack alignment requirement
 3817   stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
 3818 
 3819   // Number of outgoing stack slots killed above the out_preserve_stack_slots
 3820   // for calls to C.  Supports the var-args backing area for register parms.
 3821   varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
 3822 
 3823   // The after-PROLOG location of the return address.  Location of
 3824   // return address specifies a type (REG or STACK) and a number
 3825   // representing the register number (i.e. - use a register name) or
 3826   // stack slot.
 3827   // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
 3828   // Otherwise, it is above the locks and verification slot and alignment word
 3829   // TODO this may well be correct but need to check why that - 2 is there
 3830   // ppc port uses 0 but we definitely need to allow for fixed_slots
 3831   // which folds in the space used for monitors
 3832   return_addr(STACK - 2 +
 3833               align_up((Compile::current()->in_preserve_stack_slots() +
 3834                         Compile::current()->fixed_slots()),
 3835                        stack_alignment_in_slots()));
 3836 
 3837   // Location of compiled Java return values.  Same as C for now.
 3838   return_value
 3839   %{
 3840     // TODO do we allow ideal_reg == Op_RegN???
 3841     assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
 3842            "only return normal values");
 3843 
 3844     static const int lo[Op_RegL + 1] = { // enum name
 3845       0,                                 // Op_Node
 3846       0,                                 // Op_Set
 3847       R0_num,                            // Op_RegN
 3848       R0_num,                            // Op_RegI
 3849       R0_num,                            // Op_RegP
 3850       V0_num,                            // Op_RegF
 3851       V0_num,                            // Op_RegD
 3852       R0_num                             // Op_RegL
 3853     };
 3854 
 3855     static const int hi[Op_RegL + 1] = { // enum name
 3856       0,                                 // Op_Node
 3857       0,                                 // Op_Set
 3858       OptoReg::Bad,                      // Op_RegN
 3859       OptoReg::Bad,                      // Op_RegI
 3860       R0_H_num,                          // Op_RegP
 3861       OptoReg::Bad,                      // Op_RegF
 3862       V0_H_num,                          // Op_RegD
 3863       R0_H_num                           // Op_RegL
 3864     };
 3865 
 3866     return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
 3867   %}
 3868 %}
 3869 
 3870 //----------ATTRIBUTES---------------------------------------------------------
 3871 //----------Operand Attributes-------------------------------------------------
 3872 op_attrib op_cost(1);        // Required cost attribute
 3873 
 3874 //----------Instruction Attributes---------------------------------------------
 3875 ins_attrib ins_cost(INSN_COST); // Required cost attribute
 3876 ins_attrib ins_size(32);        // Required size attribute (in bits)
 3877 ins_attrib ins_short_branch(0); // Required flag: is this instruction
 3878                                 // a non-matching short branch variant
 3879                                 // of some long branch?
 3880 ins_attrib ins_alignment(4);    // Required alignment attribute (must
 3881                                 // be a power of 2) specifies the
 3882                                 // alignment that some part of the
 3883                                 // instruction (not necessarily the
 3884                                 // start) requires.  If > 1, a
 3885                                 // compute_padding() function must be
 3886                                 // provided for the instruction
 3887 
 3888 //----------OPERANDS-----------------------------------------------------------
 3889 // Operand definitions must precede instruction definitions for correct parsing
 3890 // in the ADLC because operands constitute user defined types which are used in
 3891 // instruction definitions.
 3892 
 3893 //----------Simple Operands----------------------------------------------------
 3894 
 3895 // Integer operands 32 bit
 3896 // 32 bit immediate
 3897 operand immI()
 3898 %{
 3899   match(ConI);
 3900 
 3901   op_cost(0);
 3902   format %{ %}
 3903   interface(CONST_INTER);
 3904 %}
 3905 
 3906 // 32 bit zero
 3907 operand immI0()
 3908 %{
 3909   predicate(n->get_int() == 0);
 3910   match(ConI);
 3911 
 3912   op_cost(0);
 3913   format %{ %}
 3914   interface(CONST_INTER);
 3915 %}
 3916 
 3917 // 32 bit unit increment
 3918 operand immI_1()
 3919 %{
 3920   predicate(n->get_int() == 1);
 3921   match(ConI);
 3922 
 3923   op_cost(0);
 3924   format %{ %}
 3925   interface(CONST_INTER);
 3926 %}
 3927 
 3928 // 32 bit unit decrement
 3929 operand immI_M1()
 3930 %{
 3931   predicate(n->get_int() == -1);
 3932   match(ConI);
 3933 
 3934   op_cost(0);
 3935   format %{ %}
 3936   interface(CONST_INTER);
 3937 %}
 3938 
 3939 // Shift values for add/sub extension shift
 3940 operand immIExt()
 3941 %{
 3942   predicate(0 <= n->get_int() && (n->get_int() <= 4));
 3943   match(ConI);
 3944 
 3945   op_cost(0);
 3946   format %{ %}
 3947   interface(CONST_INTER);
 3948 %}
 3949 
 3950 operand immI_gt_1()
 3951 %{
 3952   predicate(n->get_int() > 1);
 3953   match(ConI);
 3954 
 3955   op_cost(0);
 3956   format %{ %}
 3957   interface(CONST_INTER);
 3958 %}
 3959 
 3960 operand immI_le_4()
 3961 %{
 3962   predicate(n->get_int() <= 4);
 3963   match(ConI);
 3964 
 3965   op_cost(0);
 3966   format %{ %}
 3967   interface(CONST_INTER);
 3968 %}
 3969 
 3970 operand immI_16()
 3971 %{
 3972   predicate(n->get_int() == 16);
 3973   match(ConI);
 3974 
 3975   op_cost(0);
 3976   format %{ %}
 3977   interface(CONST_INTER);
 3978 %}
 3979 
 3980 operand immI_24()
 3981 %{
 3982   predicate(n->get_int() == 24);
 3983   match(ConI);
 3984 
 3985   op_cost(0);
 3986   format %{ %}
 3987   interface(CONST_INTER);
 3988 %}
 3989 
 3990 operand immI_32()
 3991 %{
 3992   predicate(n->get_int() == 32);
 3993   match(ConI);
 3994 
 3995   op_cost(0);
 3996   format %{ %}
 3997   interface(CONST_INTER);
 3998 %}
 3999 
 4000 operand immI_48()
 4001 %{
 4002   predicate(n->get_int() == 48);
 4003   match(ConI);
 4004 
 4005   op_cost(0);
 4006   format %{ %}
 4007   interface(CONST_INTER);
 4008 %}
 4009 
 4010 operand immI_56()
 4011 %{
 4012   predicate(n->get_int() == 56);
 4013   match(ConI);
 4014 
 4015   op_cost(0);
 4016   format %{ %}
 4017   interface(CONST_INTER);
 4018 %}
 4019 
 4020 operand immI_255()
 4021 %{
 4022   predicate(n->get_int() == 255);
 4023   match(ConI);
 4024 
 4025   op_cost(0);
 4026   format %{ %}
 4027   interface(CONST_INTER);
 4028 %}
 4029 
 4030 operand immI_65535()
 4031 %{
 4032   predicate(n->get_int() == 65535);
 4033   match(ConI);
 4034 
 4035   op_cost(0);
 4036   format %{ %}
 4037   interface(CONST_INTER);
 4038 %}
 4039 
 4040 operand immI_positive()
 4041 %{
 4042   predicate(n->get_int() > 0);
 4043   match(ConI);
 4044 
 4045   op_cost(0);
 4046   format %{ %}
 4047   interface(CONST_INTER);
 4048 %}
 4049 
 4050 // BoolTest condition for signed compare
 4051 operand immI_cmp_cond()
 4052 %{
 4053   predicate(!Matcher::is_unsigned_booltest_pred(n->get_int()));
 4054   match(ConI);
 4055 
 4056   op_cost(0);
 4057   format %{ %}
 4058   interface(CONST_INTER);
 4059 %}
 4060 
 4061 // BoolTest condition for unsigned compare
 4062 operand immI_cmpU_cond()
 4063 %{
 4064   predicate(Matcher::is_unsigned_booltest_pred(n->get_int()));
 4065   match(ConI);
 4066 
 4067   op_cost(0);
 4068   format %{ %}
 4069   interface(CONST_INTER);
 4070 %}
 4071 
 4072 operand immL_255()
 4073 %{
 4074   predicate(n->get_long() == 255L);
 4075   match(ConL);
 4076 
 4077   op_cost(0);
 4078   format %{ %}
 4079   interface(CONST_INTER);
 4080 %}
 4081 
 4082 operand immL_65535()
 4083 %{
 4084   predicate(n->get_long() == 65535L);
 4085   match(ConL);
 4086 
 4087   op_cost(0);
 4088   format %{ %}
 4089   interface(CONST_INTER);
 4090 %}
 4091 
 4092 operand immL_4294967295()
 4093 %{
 4094   predicate(n->get_long() == 4294967295L);
 4095   match(ConL);
 4096 
 4097   op_cost(0);
 4098   format %{ %}
 4099   interface(CONST_INTER);
 4100 %}
 4101 
 4102 operand immL_bitmask()
 4103 %{
 4104   predicate((n->get_long() != 0)
 4105             && ((n->get_long() & 0xc000000000000000l) == 0)
 4106             && is_power_of_2(n->get_long() + 1));
 4107   match(ConL);
 4108 
 4109   op_cost(0);
 4110   format %{ %}
 4111   interface(CONST_INTER);
 4112 %}
 4113 
 4114 operand immI_bitmask()
 4115 %{
 4116   predicate((n->get_int() != 0)
 4117             && ((n->get_int() & 0xc0000000) == 0)
 4118             && is_power_of_2(n->get_int() + 1));
 4119   match(ConI);
 4120 
 4121   op_cost(0);
 4122   format %{ %}
 4123   interface(CONST_INTER);
 4124 %}
 4125 
 4126 operand immL_positive_bitmaskI()
 4127 %{
 4128   predicate((n->get_long() != 0)
 4129             && ((julong)n->get_long() < 0x80000000ULL)
 4130             && is_power_of_2(n->get_long() + 1));
 4131   match(ConL);
 4132 
 4133   op_cost(0);
 4134   format %{ %}
 4135   interface(CONST_INTER);
 4136 %}
 4137 
 4138 // Scale values for scaled offset addressing modes (up to long but not quad)
 4139 operand immIScale()
 4140 %{
 4141   predicate(0 <= n->get_int() && (n->get_int() <= 3));
 4142   match(ConI);
 4143 
 4144   op_cost(0);
 4145   format %{ %}
 4146   interface(CONST_INTER);
 4147 %}
 4148 
 4149 // 5 bit signed integer
 4150 operand immI5()
 4151 %{
 4152   predicate(Assembler::is_simm(n->get_int(), 5));
 4153   match(ConI);
 4154 
 4155   op_cost(0);
 4156   format %{ %}
 4157   interface(CONST_INTER);
 4158 %}
 4159 
 4160 // 7 bit unsigned integer
 4161 operand immIU7()
 4162 %{
 4163   predicate(Assembler::is_uimm(n->get_int(), 7));
 4164   match(ConI);
 4165 
 4166   op_cost(0);
 4167   format %{ %}
 4168   interface(CONST_INTER);
 4169 %}
 4170 
 4171 // Offset for scaled or unscaled immediate loads and stores
 4172 operand immIOffset()
 4173 %{
 4174   predicate(Address::offset_ok_for_immed(n->get_int(), 0));
 4175   match(ConI);
 4176 
 4177   op_cost(0);
 4178   format %{ %}
 4179   interface(CONST_INTER);
 4180 %}
 4181 
 4182 operand immIOffset1()
 4183 %{
 4184   predicate(Address::offset_ok_for_immed(n->get_int(), 0));
 4185   match(ConI);
 4186 
 4187   op_cost(0);
 4188   format %{ %}
 4189   interface(CONST_INTER);
 4190 %}
 4191 
 4192 operand immIOffset2()
 4193 %{
 4194   predicate(Address::offset_ok_for_immed(n->get_int(), 1));
 4195   match(ConI);
 4196 
 4197   op_cost(0);
 4198   format %{ %}
 4199   interface(CONST_INTER);
 4200 %}
 4201 
 4202 operand immIOffset4()
 4203 %{
 4204   predicate(Address::offset_ok_for_immed(n->get_int(), 2));
 4205   match(ConI);
 4206 
 4207   op_cost(0);
 4208   format %{ %}
 4209   interface(CONST_INTER);
 4210 %}
 4211 
 4212 operand immIOffset8()
 4213 %{
 4214   predicate(Address::offset_ok_for_immed(n->get_int(), 3));
 4215   match(ConI);
 4216 
 4217   op_cost(0);
 4218   format %{ %}
 4219   interface(CONST_INTER);
 4220 %}
 4221 
 4222 operand immIOffset16()
 4223 %{
 4224   predicate(Address::offset_ok_for_immed(n->get_int(), 4));
 4225   match(ConI);
 4226 
 4227   op_cost(0);
 4228   format %{ %}
 4229   interface(CONST_INTER);
 4230 %}
 4231 
 4232 operand immLoffset()
 4233 %{
 4234   predicate(Address::offset_ok_for_immed(n->get_long(), 0));
 4235   match(ConL);
 4236 
 4237   op_cost(0);
 4238   format %{ %}
 4239   interface(CONST_INTER);
 4240 %}
 4241 
 4242 operand immLoffset1()
 4243 %{
 4244   predicate(Address::offset_ok_for_immed(n->get_long(), 0));
 4245   match(ConL);
 4246 
 4247   op_cost(0);
 4248   format %{ %}
 4249   interface(CONST_INTER);
 4250 %}
 4251 
 4252 operand immLoffset2()
 4253 %{
 4254   predicate(Address::offset_ok_for_immed(n->get_long(), 1));
 4255   match(ConL);
 4256 
 4257   op_cost(0);
 4258   format %{ %}
 4259   interface(CONST_INTER);
 4260 %}
 4261 
 4262 operand immLoffset4()
 4263 %{
 4264   predicate(Address::offset_ok_for_immed(n->get_long(), 2));
 4265   match(ConL);
 4266 
 4267   op_cost(0);
 4268   format %{ %}
 4269   interface(CONST_INTER);
 4270 %}
 4271 
 4272 operand immLoffset8()
 4273 %{
 4274   predicate(Address::offset_ok_for_immed(n->get_long(), 3));
 4275   match(ConL);
 4276 
 4277   op_cost(0);
 4278   format %{ %}
 4279   interface(CONST_INTER);
 4280 %}
 4281 
 4282 operand immLoffset16()
 4283 %{
 4284   predicate(Address::offset_ok_for_immed(n->get_long(), 4));
 4285   match(ConL);
 4286 
 4287   op_cost(0);
 4288   format %{ %}
 4289   interface(CONST_INTER);
 4290 %}
 4291 
 4292 // 5 bit signed long integer
 4293 operand immL5()
 4294 %{
 4295   predicate(Assembler::is_simm(n->get_long(), 5));
 4296   match(ConL);
 4297 
 4298   op_cost(0);
 4299   format %{ %}
 4300   interface(CONST_INTER);
 4301 %}
 4302 
 4303 // 7 bit unsigned long integer
 4304 operand immLU7()
 4305 %{
 4306   predicate(Assembler::is_uimm(n->get_long(), 7));
 4307   match(ConL);
 4308 
 4309   op_cost(0);
 4310   format %{ %}
 4311   interface(CONST_INTER);
 4312 %}
 4313 
 4314 // 8 bit signed value.
 4315 operand immI8()
 4316 %{
 4317   predicate(n->get_int() <= 127 && n->get_int() >= -128);
 4318   match(ConI);
 4319 
 4320   op_cost(0);
 4321   format %{ %}
 4322   interface(CONST_INTER);
 4323 %}
 4324 
 4325 // 8 bit signed value (simm8), or #simm8 LSL 8.
 4326 operand immI8_shift8()
 4327 %{
 4328   predicate((n->get_int() <= 127 && n->get_int() >= -128) ||
 4329             (n->get_int() <= 32512 && n->get_int() >= -32768 && (n->get_int() & 0xff) == 0));
 4330   match(ConI);
 4331 
 4332   op_cost(0);
 4333   format %{ %}
 4334   interface(CONST_INTER);
 4335 %}
 4336 
 4337 // 8 bit signed value (simm8), or #simm8 LSL 8.
 4338 operand immL8_shift8()
 4339 %{
 4340   predicate((n->get_long() <= 127 && n->get_long() >= -128) ||
 4341             (n->get_long() <= 32512 && n->get_long() >= -32768 && (n->get_long() & 0xff) == 0));
 4342   match(ConL);
 4343 
 4344   op_cost(0);
 4345   format %{ %}
 4346   interface(CONST_INTER);
 4347 %}
 4348 
 4349 // 8 bit integer valid for vector add sub immediate
 4350 operand immBAddSubV()
 4351 %{
 4352   predicate(n->get_int() <= 255 && n->get_int() >= -255);
 4353   match(ConI);
 4354 
 4355   op_cost(0);
 4356   format %{ %}
 4357   interface(CONST_INTER);
 4358 %}
 4359 
 4360 // 32 bit integer valid for add sub immediate
 4361 operand immIAddSub()
 4362 %{
 4363   predicate(Assembler::operand_valid_for_add_sub_immediate((int64_t)n->get_int()));
 4364   match(ConI);
 4365   op_cost(0);
 4366   format %{ %}
 4367   interface(CONST_INTER);
 4368 %}
 4369 
 4370 // 32 bit integer valid for vector add sub immediate
 4371 operand immIAddSubV()
 4372 %{
 4373   predicate(Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)n->get_int()));
 4374   match(ConI);
 4375 
 4376   op_cost(0);
 4377   format %{ %}
 4378   interface(CONST_INTER);
 4379 %}
 4380 
 4381 // 32 bit unsigned integer valid for logical immediate
 4382 
 4383 operand immBLog()
 4384 %{
 4385   predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerByte, (uint64_t)n->get_int()));
 4386   match(ConI);
 4387 
 4388   op_cost(0);
 4389   format %{ %}
 4390   interface(CONST_INTER);
 4391 %}
 4392 
 4393 operand immSLog()
 4394 %{
 4395   predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerShort, (uint64_t)n->get_int()));
 4396   match(ConI);
 4397 
 4398   op_cost(0);
 4399   format %{ %}
 4400   interface(CONST_INTER);
 4401 %}
 4402 
 4403 operand immILog()
 4404 %{
 4405   predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/true, (uint64_t)n->get_int()));
 4406   match(ConI);
 4407 
 4408   op_cost(0);
 4409   format %{ %}
 4410   interface(CONST_INTER);
 4411 %}
 4412 
 4413 // Integer operands 64 bit
 4414 // 64 bit immediate
 4415 operand immL()
 4416 %{
 4417   match(ConL);
 4418 
 4419   op_cost(0);
 4420   format %{ %}
 4421   interface(CONST_INTER);
 4422 %}
 4423 
 4424 // 64 bit zero
 4425 operand immL0()
 4426 %{
 4427   predicate(n->get_long() == 0);
 4428   match(ConL);
 4429 
 4430   op_cost(0);
 4431   format %{ %}
 4432   interface(CONST_INTER);
 4433 %}
 4434 
 4435 // 64 bit unit decrement
 4436 operand immL_M1()
 4437 %{
 4438   predicate(n->get_long() == -1);
 4439   match(ConL);
 4440 
 4441   op_cost(0);
 4442   format %{ %}
 4443   interface(CONST_INTER);
 4444 %}
 4445 
 4446 // 64 bit integer valid for add sub immediate
 4447 operand immLAddSub()
 4448 %{
 4449   predicate(Assembler::operand_valid_for_add_sub_immediate(n->get_long()));
 4450   match(ConL);
 4451   op_cost(0);
 4452   format %{ %}
 4453   interface(CONST_INTER);
 4454 %}
 4455 
 4456 // 64 bit integer valid for addv subv immediate
 4457 operand immLAddSubV()
 4458 %{
 4459   predicate(Assembler::operand_valid_for_sve_add_sub_immediate(n->get_long()));
 4460   match(ConL);
 4461 
 4462   op_cost(0);
 4463   format %{ %}
 4464   interface(CONST_INTER);
 4465 %}
 4466 
 4467 // 64 bit integer valid for logical immediate
 4468 operand immLLog()
 4469 %{
 4470   predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/false, (uint64_t)n->get_long()));
 4471   match(ConL);
 4472   op_cost(0);
 4473   format %{ %}
 4474   interface(CONST_INTER);
 4475 %}
 4476 
 4477 // Long Immediate: low 32-bit mask
 4478 operand immL_32bits()
 4479 %{
 4480   predicate(n->get_long() == 0xFFFFFFFFL);
 4481   match(ConL);
 4482   op_cost(0);
 4483   format %{ %}
 4484   interface(CONST_INTER);
 4485 %}
 4486 
 4487 // Pointer operands
 4488 // Pointer Immediate
 4489 operand immP()
 4490 %{
 4491   match(ConP);
 4492 
 4493   op_cost(0);
 4494   format %{ %}
 4495   interface(CONST_INTER);
 4496 %}
 4497 
 4498 // nullptr Pointer Immediate
 4499 operand immP0()
 4500 %{
 4501   predicate(n->get_ptr() == 0);
 4502   match(ConP);
 4503 
 4504   op_cost(0);
 4505   format %{ %}
 4506   interface(CONST_INTER);
 4507 %}
 4508 
 4509 // Pointer Immediate One
 4510 // this is used in object initialization (initial object header)
 4511 operand immP_1()
 4512 %{
 4513   predicate(n->get_ptr() == 1);
 4514   match(ConP);
 4515 
 4516   op_cost(0);
 4517   format %{ %}
 4518   interface(CONST_INTER);
 4519 %}
 4520 
 4521 // Card Table Byte Map Base
 4522 operand immByteMapBase()
 4523 %{
 4524   // Get base of card map
 4525   predicate(BarrierSet::barrier_set()->is_a(BarrierSet::CardTableBarrierSet) &&
 4526             (CardTable::CardValue*)n->get_ptr() == ((CardTableBarrierSet*)(BarrierSet::barrier_set()))->card_table()->byte_map_base());
 4527   match(ConP);
 4528 
 4529   op_cost(0);
 4530   format %{ %}
 4531   interface(CONST_INTER);
 4532 %}
 4533 
 4534 // Float and Double operands
 4535 // Double Immediate
 4536 operand immD()
 4537 %{
 4538   match(ConD);
 4539   op_cost(0);
 4540   format %{ %}
 4541   interface(CONST_INTER);
 4542 %}
 4543 
 4544 // Double Immediate: +0.0d
 4545 operand immD0()
 4546 %{
 4547   predicate(jlong_cast(n->getd()) == 0);
 4548   match(ConD);
 4549 
 4550   op_cost(0);
 4551   format %{ %}
 4552   interface(CONST_INTER);
 4553 %}
 4554 
 4555 // constant 'double +0.0'.
 4556 operand immDPacked()
 4557 %{
 4558   predicate(Assembler::operand_valid_for_float_immediate(n->getd()));
 4559   match(ConD);
 4560   op_cost(0);
 4561   format %{ %}
 4562   interface(CONST_INTER);
 4563 %}
 4564 
 4565 // Float Immediate
 4566 operand immF()
 4567 %{
 4568   match(ConF);
 4569   op_cost(0);
 4570   format %{ %}
 4571   interface(CONST_INTER);
 4572 %}
 4573 
 4574 // Float Immediate: +0.0f.
 4575 operand immF0()
 4576 %{
 4577   predicate(jint_cast(n->getf()) == 0);
 4578   match(ConF);
 4579 
 4580   op_cost(0);
 4581   format %{ %}
 4582   interface(CONST_INTER);
 4583 %}
 4584 
 4585 //
 4586 operand immFPacked()
 4587 %{
 4588   predicate(Assembler::operand_valid_for_float_immediate((double)n->getf()));
 4589   match(ConF);
 4590   op_cost(0);
 4591   format %{ %}
 4592   interface(CONST_INTER);
 4593 %}
 4594 
 4595 // Narrow pointer operands
 4596 // Narrow Pointer Immediate
 4597 operand immN()
 4598 %{
 4599   match(ConN);
 4600 
 4601   op_cost(0);
 4602   format %{ %}
 4603   interface(CONST_INTER);
 4604 %}
 4605 
 4606 // Narrow nullptr Pointer Immediate
 4607 operand immN0()
 4608 %{
 4609   predicate(n->get_narrowcon() == 0);
 4610   match(ConN);
 4611 
 4612   op_cost(0);
 4613   format %{ %}
 4614   interface(CONST_INTER);
 4615 %}
 4616 
 4617 operand immNKlass()
 4618 %{
 4619   match(ConNKlass);
 4620 
 4621   op_cost(0);
 4622   format %{ %}
 4623   interface(CONST_INTER);
 4624 %}
 4625 
 4626 // Integer 32 bit Register Operands
 4627 // Integer 32 bitRegister (excludes SP)
 4628 operand iRegI()
 4629 %{
 4630   constraint(ALLOC_IN_RC(any_reg32));
 4631   match(RegI);
 4632   match(iRegINoSp);
 4633   op_cost(0);
 4634   format %{ %}
 4635   interface(REG_INTER);
 4636 %}
 4637 
 4638 // Integer 32 bit Register not Special
 4639 operand iRegINoSp()
 4640 %{
 4641   constraint(ALLOC_IN_RC(no_special_reg32));
 4642   match(RegI);
 4643   op_cost(0);
 4644   format %{ %}
 4645   interface(REG_INTER);
 4646 %}
 4647 
 4648 // Integer 64 bit Register Operands
 4649 // Integer 64 bit Register (includes SP)
 4650 operand iRegL()
 4651 %{
 4652   constraint(ALLOC_IN_RC(any_reg));
 4653   match(RegL);
 4654   match(iRegLNoSp);
 4655   op_cost(0);
 4656   format %{ %}
 4657   interface(REG_INTER);
 4658 %}
 4659 
 4660 // Integer 64 bit Register not Special
 4661 operand iRegLNoSp()
 4662 %{
 4663   constraint(ALLOC_IN_RC(no_special_reg));
 4664   match(RegL);
 4665   match(iRegL_R0);
 4666   format %{ %}
 4667   interface(REG_INTER);
 4668 %}
 4669 
 4670 // Pointer Register Operands
 4671 // Pointer Register
 4672 operand iRegP()
 4673 %{
 4674   constraint(ALLOC_IN_RC(ptr_reg));
 4675   match(RegP);
 4676   match(iRegPNoSp);
 4677   match(iRegP_R0);
 4678   //match(iRegP_R2);
 4679   //match(iRegP_R4);
 4680   match(iRegP_R5);
 4681   match(thread_RegP);
 4682   op_cost(0);
 4683   format %{ %}
 4684   interface(REG_INTER);
 4685 %}
 4686 
 4687 // Pointer 64 bit Register not Special
 4688 operand iRegPNoSp()
 4689 %{
 4690   constraint(ALLOC_IN_RC(no_special_ptr_reg));
 4691   match(RegP);
 4692   // match(iRegP);
 4693   // match(iRegP_R0);
 4694   // match(iRegP_R2);
 4695   // match(iRegP_R4);
 4696   // match(iRegP_R5);
 4697   // match(thread_RegP);
 4698   op_cost(0);
 4699   format %{ %}
 4700   interface(REG_INTER);
 4701 %}
 4702 
 4703 // This operand is not allowed to use rfp even if
 4704 // rfp is not used to hold the frame pointer.
 4705 operand iRegPNoSpNoRfp()
 4706 %{
 4707   constraint(ALLOC_IN_RC(no_special_no_rfp_ptr_reg));
 4708   match(RegP);
 4709   match(iRegPNoSp);
 4710   op_cost(0);
 4711   format %{ %}
 4712   interface(REG_INTER);
 4713 %}
 4714 
 4715 // Pointer 64 bit Register R0 only
 4716 operand iRegP_R0()
 4717 %{
 4718   constraint(ALLOC_IN_RC(r0_reg));
 4719   match(RegP);
 4720   // match(iRegP);
 4721   match(iRegPNoSp);
 4722   op_cost(0);
 4723   format %{ %}
 4724   interface(REG_INTER);
 4725 %}
 4726 
 4727 // Pointer 64 bit Register R1 only
 4728 operand iRegP_R1()
 4729 %{
 4730   constraint(ALLOC_IN_RC(r1_reg));
 4731   match(RegP);
 4732   // match(iRegP);
 4733   match(iRegPNoSp);
 4734   op_cost(0);
 4735   format %{ %}
 4736   interface(REG_INTER);
 4737 %}
 4738 
 4739 // Pointer 64 bit Register R2 only
 4740 operand iRegP_R2()
 4741 %{
 4742   constraint(ALLOC_IN_RC(r2_reg));
 4743   match(RegP);
 4744   // match(iRegP);
 4745   match(iRegPNoSp);
 4746   op_cost(0);
 4747   format %{ %}
 4748   interface(REG_INTER);
 4749 %}
 4750 
 4751 // Pointer 64 bit Register R3 only
 4752 operand iRegP_R3()
 4753 %{
 4754   constraint(ALLOC_IN_RC(r3_reg));
 4755   match(RegP);
 4756   // match(iRegP);
 4757   match(iRegPNoSp);
 4758   op_cost(0);
 4759   format %{ %}
 4760   interface(REG_INTER);
 4761 %}
 4762 
 4763 // Pointer 64 bit Register R4 only
 4764 operand iRegP_R4()
 4765 %{
 4766   constraint(ALLOC_IN_RC(r4_reg));
 4767   match(RegP);
 4768   // match(iRegP);
 4769   match(iRegPNoSp);
 4770   op_cost(0);
 4771   format %{ %}
 4772   interface(REG_INTER);
 4773 %}
 4774 
 4775 // Pointer 64 bit Register R5 only
 4776 operand iRegP_R5()
 4777 %{
 4778   constraint(ALLOC_IN_RC(r5_reg));
 4779   match(RegP);
 4780   // match(iRegP);
 4781   match(iRegPNoSp);
 4782   op_cost(0);
 4783   format %{ %}
 4784   interface(REG_INTER);
 4785 %}
 4786 
 4787 // Pointer 64 bit Register R10 only
 4788 operand iRegP_R10()
 4789 %{
 4790   constraint(ALLOC_IN_RC(r10_reg));
 4791   match(RegP);
 4792   // match(iRegP);
 4793   match(iRegPNoSp);
 4794   op_cost(0);
 4795   format %{ %}
 4796   interface(REG_INTER);
 4797 %}
 4798 
 4799 // Long 64 bit Register R0 only
 4800 operand iRegL_R0()
 4801 %{
 4802   constraint(ALLOC_IN_RC(r0_reg));
 4803   match(RegL);
 4804   match(iRegLNoSp);
 4805   op_cost(0);
 4806   format %{ %}
 4807   interface(REG_INTER);
 4808 %}
 4809 
 4810 // Long 64 bit Register R11 only
 4811 operand iRegL_R11()
 4812 %{
 4813   constraint(ALLOC_IN_RC(r11_reg));
 4814   match(RegL);
 4815   match(iRegLNoSp);
 4816   op_cost(0);
 4817   format %{ %}
 4818   interface(REG_INTER);
 4819 %}
 4820 
 4821 // Register R0 only
 4822 operand iRegI_R0()
 4823 %{
 4824   constraint(ALLOC_IN_RC(int_r0_reg));
 4825   match(RegI);
 4826   match(iRegINoSp);
 4827   op_cost(0);
 4828   format %{ %}
 4829   interface(REG_INTER);
 4830 %}
 4831 
 4832 // Register R2 only
 4833 operand iRegI_R2()
 4834 %{
 4835   constraint(ALLOC_IN_RC(int_r2_reg));
 4836   match(RegI);
 4837   match(iRegINoSp);
 4838   op_cost(0);
 4839   format %{ %}
 4840   interface(REG_INTER);
 4841 %}
 4842 
 4843 // Register R3 only
 4844 operand iRegI_R3()
 4845 %{
 4846   constraint(ALLOC_IN_RC(int_r3_reg));
 4847   match(RegI);
 4848   match(iRegINoSp);
 4849   op_cost(0);
 4850   format %{ %}
 4851   interface(REG_INTER);
 4852 %}
 4853 
 4854 
 4855 // Register R4 only
 4856 operand iRegI_R4()
 4857 %{
 4858   constraint(ALLOC_IN_RC(int_r4_reg));
 4859   match(RegI);
 4860   match(iRegINoSp);
 4861   op_cost(0);
 4862   format %{ %}
 4863   interface(REG_INTER);
 4864 %}
 4865 
 4866 
 4867 // Pointer Register Operands
 4868 // Narrow Pointer Register
 4869 operand iRegN()
 4870 %{
 4871   constraint(ALLOC_IN_RC(any_reg32));
 4872   match(RegN);
 4873   match(iRegNNoSp);
 4874   op_cost(0);
 4875   format %{ %}
 4876   interface(REG_INTER);
 4877 %}
 4878 
 4879 // Integer 64 bit Register not Special
 4880 operand iRegNNoSp()
 4881 %{
 4882   constraint(ALLOC_IN_RC(no_special_reg32));
 4883   match(RegN);
 4884   op_cost(0);
 4885   format %{ %}
 4886   interface(REG_INTER);
 4887 %}
 4888 
 4889 // Float Register
 4890 // Float register operands
 4891 operand vRegF()
 4892 %{
 4893   constraint(ALLOC_IN_RC(float_reg));
 4894   match(RegF);
 4895 
 4896   op_cost(0);
 4897   format %{ %}
 4898   interface(REG_INTER);
 4899 %}
 4900 
 4901 // Double Register
 4902 // Double register operands
 4903 operand vRegD()
 4904 %{
 4905   constraint(ALLOC_IN_RC(double_reg));
 4906   match(RegD);
 4907 
 4908   op_cost(0);
 4909   format %{ %}
 4910   interface(REG_INTER);
 4911 %}
 4912 
 4913 // Generic vector class. This will be used for
 4914 // all vector operands, including NEON and SVE.
 4915 operand vReg()
 4916 %{
 4917   constraint(ALLOC_IN_RC(dynamic));
 4918   match(VecA);
 4919   match(VecD);
 4920   match(VecX);
 4921 
 4922   op_cost(0);
 4923   format %{ %}
 4924   interface(REG_INTER);
 4925 %}
 4926 
 4927 operand vecA()
 4928 %{
 4929   constraint(ALLOC_IN_RC(vectora_reg));
 4930   match(VecA);
 4931 
 4932   op_cost(0);
 4933   format %{ %}
 4934   interface(REG_INTER);
 4935 %}
 4936 
 4937 operand vecD()
 4938 %{
 4939   constraint(ALLOC_IN_RC(vectord_reg));
 4940   match(VecD);
 4941 
 4942   op_cost(0);
 4943   format %{ %}
 4944   interface(REG_INTER);
 4945 %}
 4946 
 4947 operand vecX()
 4948 %{
 4949   constraint(ALLOC_IN_RC(vectorx_reg));
 4950   match(VecX);
 4951 
 4952   op_cost(0);
 4953   format %{ %}
 4954   interface(REG_INTER);
 4955 %}
 4956 
 4957 operand vRegD_V0()
 4958 %{
 4959   constraint(ALLOC_IN_RC(v0_reg));
 4960   match(RegD);
 4961   op_cost(0);
 4962   format %{ %}
 4963   interface(REG_INTER);
 4964 %}
 4965 
 4966 operand vRegD_V1()
 4967 %{
 4968   constraint(ALLOC_IN_RC(v1_reg));
 4969   match(RegD);
 4970   op_cost(0);
 4971   format %{ %}
 4972   interface(REG_INTER);
 4973 %}
 4974 
 4975 operand vRegD_V2()
 4976 %{
 4977   constraint(ALLOC_IN_RC(v2_reg));
 4978   match(RegD);
 4979   op_cost(0);
 4980   format %{ %}
 4981   interface(REG_INTER);
 4982 %}
 4983 
 4984 operand vRegD_V3()
 4985 %{
 4986   constraint(ALLOC_IN_RC(v3_reg));
 4987   match(RegD);
 4988   op_cost(0);
 4989   format %{ %}
 4990   interface(REG_INTER);
 4991 %}
 4992 
 4993 operand vRegD_V4()
 4994 %{
 4995   constraint(ALLOC_IN_RC(v4_reg));
 4996   match(RegD);
 4997   op_cost(0);
 4998   format %{ %}
 4999   interface(REG_INTER);
 5000 %}
 5001 
 5002 operand vRegD_V5()
 5003 %{
 5004   constraint(ALLOC_IN_RC(v5_reg));
 5005   match(RegD);
 5006   op_cost(0);
 5007   format %{ %}
 5008   interface(REG_INTER);
 5009 %}
 5010 
 5011 operand vRegD_V6()
 5012 %{
 5013   constraint(ALLOC_IN_RC(v6_reg));
 5014   match(RegD);
 5015   op_cost(0);
 5016   format %{ %}
 5017   interface(REG_INTER);
 5018 %}
 5019 
 5020 operand vRegD_V7()
 5021 %{
 5022   constraint(ALLOC_IN_RC(v7_reg));
 5023   match(RegD);
 5024   op_cost(0);
 5025   format %{ %}
 5026   interface(REG_INTER);
 5027 %}
 5028 
 5029 operand pReg()
 5030 %{
 5031   constraint(ALLOC_IN_RC(pr_reg));
 5032   match(RegVectMask);
 5033   match(pRegGov);
 5034   op_cost(0);
 5035   format %{ %}
 5036   interface(REG_INTER);
 5037 %}
 5038 
 5039 operand pRegGov()
 5040 %{
 5041   constraint(ALLOC_IN_RC(gov_pr));
 5042   match(RegVectMask);
 5043   match(pReg);
 5044   op_cost(0);
 5045   format %{ %}
 5046   interface(REG_INTER);
 5047 %}
 5048 
 5049 operand pRegGov_P0()
 5050 %{
 5051   constraint(ALLOC_IN_RC(p0_reg));
 5052   match(RegVectMask);
 5053   op_cost(0);
 5054   format %{ %}
 5055   interface(REG_INTER);
 5056 %}
 5057 
 5058 operand pRegGov_P1()
 5059 %{
 5060   constraint(ALLOC_IN_RC(p1_reg));
 5061   match(RegVectMask);
 5062   op_cost(0);
 5063   format %{ %}
 5064   interface(REG_INTER);
 5065 %}
 5066 
 5067 // Flags register, used as output of signed compare instructions
 5068 
 5069 // note that on AArch64 we also use this register as the output for
 5070 // for floating point compare instructions (CmpF CmpD). this ensures
 5071 // that ordered inequality tests use GT, GE, LT or LE none of which
 5072 // pass through cases where the result is unordered i.e. one or both
 5073 // inputs to the compare is a NaN. this means that the ideal code can
 5074 // replace e.g. a GT with an LE and not end up capturing the NaN case
 5075 // (where the comparison should always fail). EQ and NE tests are
 5076 // always generated in ideal code so that unordered folds into the NE
 5077 // case, matching the behaviour of AArch64 NE.
 5078 //
 5079 // This differs from x86 where the outputs of FP compares use a
 5080 // special FP flags registers and where compares based on this
 5081 // register are distinguished into ordered inequalities (cmpOpUCF) and
 5082 // EQ/NEQ tests (cmpOpUCF2). x86 has to special case the latter tests
 5083 // to explicitly handle the unordered case in branches. x86 also has
 5084 // to include extra CMoveX rules to accept a cmpOpUCF input.
 5085 
 5086 operand rFlagsReg()
 5087 %{
 5088   constraint(ALLOC_IN_RC(int_flags));
 5089   match(RegFlags);
 5090 
 5091   op_cost(0);
 5092   format %{ "RFLAGS" %}
 5093   interface(REG_INTER);
 5094 %}
 5095 
 5096 // Flags register, used as output of unsigned compare instructions
 5097 operand rFlagsRegU()
 5098 %{
 5099   constraint(ALLOC_IN_RC(int_flags));
 5100   match(RegFlags);
 5101 
 5102   op_cost(0);
 5103   format %{ "RFLAGSU" %}
 5104   interface(REG_INTER);
 5105 %}
 5106 
 5107 // Special Registers
 5108 
 5109 // Method Register
 5110 operand inline_cache_RegP(iRegP reg)
 5111 %{
 5112   constraint(ALLOC_IN_RC(method_reg)); // inline_cache_reg
 5113   match(reg);
 5114   match(iRegPNoSp);
 5115   op_cost(0);
 5116   format %{ %}
 5117   interface(REG_INTER);
 5118 %}
 5119 
 5120 // Thread Register
 5121 operand thread_RegP(iRegP reg)
 5122 %{
 5123   constraint(ALLOC_IN_RC(thread_reg)); // link_reg
 5124   match(reg);
 5125   op_cost(0);
 5126   format %{ %}
 5127   interface(REG_INTER);
 5128 %}
 5129 
 5130 //----------Memory Operands----------------------------------------------------
 5131 
 5132 operand indirect(iRegP reg)
 5133 %{
 5134   constraint(ALLOC_IN_RC(ptr_reg));
 5135   match(reg);
 5136   op_cost(0);
 5137   format %{ "[$reg]" %}
 5138   interface(MEMORY_INTER) %{
 5139     base($reg);
 5140     index(0xffffffff);
 5141     scale(0x0);
 5142     disp(0x0);
 5143   %}
 5144 %}
 5145 
 5146 operand indIndexScaledI2L(iRegP reg, iRegI ireg, immIScale scale)
 5147 %{
 5148   constraint(ALLOC_IN_RC(ptr_reg));
 5149   predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 5150   match(AddP reg (LShiftL (ConvI2L ireg) scale));
 5151   op_cost(0);
 5152   format %{ "$reg, $ireg sxtw($scale), 0, I2L" %}
 5153   interface(MEMORY_INTER) %{
 5154     base($reg);
 5155     index($ireg);
 5156     scale($scale);
 5157     disp(0x0);
 5158   %}
 5159 %}
 5160 
 5161 operand indIndexScaled(iRegP reg, iRegL lreg, immIScale scale)
 5162 %{
 5163   constraint(ALLOC_IN_RC(ptr_reg));
 5164   predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 5165   match(AddP reg (LShiftL lreg scale));
 5166   op_cost(0);
 5167   format %{ "$reg, $lreg lsl($scale)" %}
 5168   interface(MEMORY_INTER) %{
 5169     base($reg);
 5170     index($lreg);
 5171     scale($scale);
 5172     disp(0x0);
 5173   %}
 5174 %}
 5175 
 5176 operand indIndexI2L(iRegP reg, iRegI ireg)
 5177 %{
 5178   constraint(ALLOC_IN_RC(ptr_reg));
 5179   match(AddP reg (ConvI2L ireg));
 5180   op_cost(0);
 5181   format %{ "$reg, $ireg, 0, I2L" %}
 5182   interface(MEMORY_INTER) %{
 5183     base($reg);
 5184     index($ireg);
 5185     scale(0x0);
 5186     disp(0x0);
 5187   %}
 5188 %}
 5189 
 5190 operand indIndex(iRegP reg, iRegL lreg)
 5191 %{
 5192   constraint(ALLOC_IN_RC(ptr_reg));
 5193   match(AddP reg lreg);
 5194   op_cost(0);
 5195   format %{ "$reg, $lreg" %}
 5196   interface(MEMORY_INTER) %{
 5197     base($reg);
 5198     index($lreg);
 5199     scale(0x0);
 5200     disp(0x0);
 5201   %}
 5202 %}
 5203 
 5204 operand indOffI1(iRegP reg, immIOffset1 off)
 5205 %{
 5206   constraint(ALLOC_IN_RC(ptr_reg));
 5207   match(AddP reg off);
 5208   op_cost(0);
 5209   format %{ "[$reg, $off]" %}
 5210   interface(MEMORY_INTER) %{
 5211     base($reg);
 5212     index(0xffffffff);
 5213     scale(0x0);
 5214     disp($off);
 5215   %}
 5216 %}
 5217 
 5218 operand indOffI2(iRegP reg, immIOffset2 off)
 5219 %{
 5220   constraint(ALLOC_IN_RC(ptr_reg));
 5221   match(AddP reg off);
 5222   op_cost(0);
 5223   format %{ "[$reg, $off]" %}
 5224   interface(MEMORY_INTER) %{
 5225     base($reg);
 5226     index(0xffffffff);
 5227     scale(0x0);
 5228     disp($off);
 5229   %}
 5230 %}
 5231 
 5232 operand indOffI4(iRegP reg, immIOffset4 off)
 5233 %{
 5234   constraint(ALLOC_IN_RC(ptr_reg));
 5235   match(AddP reg off);
 5236   op_cost(0);
 5237   format %{ "[$reg, $off]" %}
 5238   interface(MEMORY_INTER) %{
 5239     base($reg);
 5240     index(0xffffffff);
 5241     scale(0x0);
 5242     disp($off);
 5243   %}
 5244 %}
 5245 
 5246 operand indOffI8(iRegP reg, immIOffset8 off)
 5247 %{
 5248   constraint(ALLOC_IN_RC(ptr_reg));
 5249   match(AddP reg off);
 5250   op_cost(0);
 5251   format %{ "[$reg, $off]" %}
 5252   interface(MEMORY_INTER) %{
 5253     base($reg);
 5254     index(0xffffffff);
 5255     scale(0x0);
 5256     disp($off);
 5257   %}
 5258 %}
 5259 
 5260 operand indOffI16(iRegP reg, immIOffset16 off)
 5261 %{
 5262   constraint(ALLOC_IN_RC(ptr_reg));
 5263   match(AddP reg off);
 5264   op_cost(0);
 5265   format %{ "[$reg, $off]" %}
 5266   interface(MEMORY_INTER) %{
 5267     base($reg);
 5268     index(0xffffffff);
 5269     scale(0x0);
 5270     disp($off);
 5271   %}
 5272 %}
 5273 
 5274 operand indOffL1(iRegP reg, immLoffset1 off)
 5275 %{
 5276   constraint(ALLOC_IN_RC(ptr_reg));
 5277   match(AddP reg off);
 5278   op_cost(0);
 5279   format %{ "[$reg, $off]" %}
 5280   interface(MEMORY_INTER) %{
 5281     base($reg);
 5282     index(0xffffffff);
 5283     scale(0x0);
 5284     disp($off);
 5285   %}
 5286 %}
 5287 
 5288 operand indOffL2(iRegP reg, immLoffset2 off)
 5289 %{
 5290   constraint(ALLOC_IN_RC(ptr_reg));
 5291   match(AddP reg off);
 5292   op_cost(0);
 5293   format %{ "[$reg, $off]" %}
 5294   interface(MEMORY_INTER) %{
 5295     base($reg);
 5296     index(0xffffffff);
 5297     scale(0x0);
 5298     disp($off);
 5299   %}
 5300 %}
 5301 
 5302 operand indOffL4(iRegP reg, immLoffset4 off)
 5303 %{
 5304   constraint(ALLOC_IN_RC(ptr_reg));
 5305   match(AddP reg off);
 5306   op_cost(0);
 5307   format %{ "[$reg, $off]" %}
 5308   interface(MEMORY_INTER) %{
 5309     base($reg);
 5310     index(0xffffffff);
 5311     scale(0x0);
 5312     disp($off);
 5313   %}
 5314 %}
 5315 
 5316 operand indOffL8(iRegP reg, immLoffset8 off)
 5317 %{
 5318   constraint(ALLOC_IN_RC(ptr_reg));
 5319   match(AddP reg off);
 5320   op_cost(0);
 5321   format %{ "[$reg, $off]" %}
 5322   interface(MEMORY_INTER) %{
 5323     base($reg);
 5324     index(0xffffffff);
 5325     scale(0x0);
 5326     disp($off);
 5327   %}
 5328 %}
 5329 
 5330 operand indOffL16(iRegP reg, immLoffset16 off)
 5331 %{
 5332   constraint(ALLOC_IN_RC(ptr_reg));
 5333   match(AddP reg off);
 5334   op_cost(0);
 5335   format %{ "[$reg, $off]" %}
 5336   interface(MEMORY_INTER) %{
 5337     base($reg);
 5338     index(0xffffffff);
 5339     scale(0x0);
 5340     disp($off);
 5341   %}
 5342 %}
 5343 
 5344 operand indirectN(iRegN reg)
 5345 %{
 5346   predicate(CompressedOops::shift() == 0);
 5347   constraint(ALLOC_IN_RC(ptr_reg));
 5348   match(DecodeN reg);
 5349   op_cost(0);
 5350   format %{ "[$reg]\t# narrow" %}
 5351   interface(MEMORY_INTER) %{
 5352     base($reg);
 5353     index(0xffffffff);
 5354     scale(0x0);
 5355     disp(0x0);
 5356   %}
 5357 %}
 5358 
 5359 operand indIndexScaledI2LN(iRegN reg, iRegI ireg, immIScale scale)
 5360 %{
 5361   predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 5362   constraint(ALLOC_IN_RC(ptr_reg));
 5363   match(AddP (DecodeN reg) (LShiftL (ConvI2L ireg) scale));
 5364   op_cost(0);
 5365   format %{ "$reg, $ireg sxtw($scale), 0, I2L\t# narrow" %}
 5366   interface(MEMORY_INTER) %{
 5367     base($reg);
 5368     index($ireg);
 5369     scale($scale);
 5370     disp(0x0);
 5371   %}
 5372 %}
 5373 
 5374 operand indIndexScaledN(iRegN reg, iRegL lreg, immIScale scale)
 5375 %{
 5376   predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
 5377   constraint(ALLOC_IN_RC(ptr_reg));
 5378   match(AddP (DecodeN reg) (LShiftL lreg scale));
 5379   op_cost(0);
 5380   format %{ "$reg, $lreg lsl($scale)\t# narrow" %}
 5381   interface(MEMORY_INTER) %{
 5382     base($reg);
 5383     index($lreg);
 5384     scale($scale);
 5385     disp(0x0);
 5386   %}
 5387 %}
 5388 
 5389 operand indIndexI2LN(iRegN reg, iRegI ireg)
 5390 %{
 5391   predicate(CompressedOops::shift() == 0);
 5392   constraint(ALLOC_IN_RC(ptr_reg));
 5393   match(AddP (DecodeN reg) (ConvI2L ireg));
 5394   op_cost(0);
 5395   format %{ "$reg, $ireg, 0, I2L\t# narrow" %}
 5396   interface(MEMORY_INTER) %{
 5397     base($reg);
 5398     index($ireg);
 5399     scale(0x0);
 5400     disp(0x0);
 5401   %}
 5402 %}
 5403 
 5404 operand indIndexN(iRegN reg, iRegL lreg)
 5405 %{
 5406   predicate(CompressedOops::shift() == 0);
 5407   constraint(ALLOC_IN_RC(ptr_reg));
 5408   match(AddP (DecodeN reg) lreg);
 5409   op_cost(0);
 5410   format %{ "$reg, $lreg\t# narrow" %}
 5411   interface(MEMORY_INTER) %{
 5412     base($reg);
 5413     index($lreg);
 5414     scale(0x0);
 5415     disp(0x0);
 5416   %}
 5417 %}
 5418 
 5419 operand indOffIN(iRegN reg, immIOffset off)
 5420 %{
 5421   predicate(CompressedOops::shift() == 0);
 5422   constraint(ALLOC_IN_RC(ptr_reg));
 5423   match(AddP (DecodeN reg) off);
 5424   op_cost(0);
 5425   format %{ "[$reg, $off]\t# narrow" %}
 5426   interface(MEMORY_INTER) %{
 5427     base($reg);
 5428     index(0xffffffff);
 5429     scale(0x0);
 5430     disp($off);
 5431   %}
 5432 %}
 5433 
 5434 operand indOffLN(iRegN reg, immLoffset off)
 5435 %{
 5436   predicate(CompressedOops::shift() == 0);
 5437   constraint(ALLOC_IN_RC(ptr_reg));
 5438   match(AddP (DecodeN reg) off);
 5439   op_cost(0);
 5440   format %{ "[$reg, $off]\t# narrow" %}
 5441   interface(MEMORY_INTER) %{
 5442     base($reg);
 5443     index(0xffffffff);
 5444     scale(0x0);
 5445     disp($off);
 5446   %}
 5447 %}
 5448 
 5449 
 5450 //----------Special Memory Operands--------------------------------------------
 5451 // Stack Slot Operand - This operand is used for loading and storing temporary
 5452 //                      values on the stack where a match requires a value to
 5453 //                      flow through memory.
 5454 operand stackSlotP(sRegP reg)
 5455 %{
 5456   constraint(ALLOC_IN_RC(stack_slots));
 5457   op_cost(100);
 5458   // No match rule because this operand is only generated in matching
 5459   // match(RegP);
 5460   format %{ "[$reg]" %}
 5461   interface(MEMORY_INTER) %{
 5462     base(0x1e);  // RSP
 5463     index(0x0);  // No Index
 5464     scale(0x0);  // No Scale
 5465     disp($reg);  // Stack Offset
 5466   %}
 5467 %}
 5468 
 5469 operand stackSlotI(sRegI reg)
 5470 %{
 5471   constraint(ALLOC_IN_RC(stack_slots));
 5472   // No match rule because this operand is only generated in matching
 5473   // match(RegI);
 5474   format %{ "[$reg]" %}
 5475   interface(MEMORY_INTER) %{
 5476     base(0x1e);  // RSP
 5477     index(0x0);  // No Index
 5478     scale(0x0);  // No Scale
 5479     disp($reg);  // Stack Offset
 5480   %}
 5481 %}
 5482 
 5483 operand stackSlotF(sRegF reg)
 5484 %{
 5485   constraint(ALLOC_IN_RC(stack_slots));
 5486   // No match rule because this operand is only generated in matching
 5487   // match(RegF);
 5488   format %{ "[$reg]" %}
 5489   interface(MEMORY_INTER) %{
 5490     base(0x1e);  // RSP
 5491     index(0x0);  // No Index
 5492     scale(0x0);  // No Scale
 5493     disp($reg);  // Stack Offset
 5494   %}
 5495 %}
 5496 
 5497 operand stackSlotD(sRegD reg)
 5498 %{
 5499   constraint(ALLOC_IN_RC(stack_slots));
 5500   // No match rule because this operand is only generated in matching
 5501   // match(RegD);
 5502   format %{ "[$reg]" %}
 5503   interface(MEMORY_INTER) %{
 5504     base(0x1e);  // RSP
 5505     index(0x0);  // No Index
 5506     scale(0x0);  // No Scale
 5507     disp($reg);  // Stack Offset
 5508   %}
 5509 %}
 5510 
 5511 operand stackSlotL(sRegL reg)
 5512 %{
 5513   constraint(ALLOC_IN_RC(stack_slots));
 5514   // No match rule because this operand is only generated in matching
 5515   // match(RegL);
 5516   format %{ "[$reg]" %}
 5517   interface(MEMORY_INTER) %{
 5518     base(0x1e);  // RSP
 5519     index(0x0);  // No Index
 5520     scale(0x0);  // No Scale
 5521     disp($reg);  // Stack Offset
 5522   %}
 5523 %}
 5524 
 5525 // Operands for expressing Control Flow
 5526 // NOTE: Label is a predefined operand which should not be redefined in
 5527 //       the AD file. It is generically handled within the ADLC.
 5528 
 5529 //----------Conditional Branch Operands----------------------------------------
 5530 // Comparison Op  - This is the operation of the comparison, and is limited to
 5531 //                  the following set of codes:
 5532 //                  L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
 5533 //
 5534 // Other attributes of the comparison, such as unsignedness, are specified
 5535 // by the comparison instruction that sets a condition code flags register.
 5536 // That result is represented by a flags operand whose subtype is appropriate
 5537 // to the unsignedness (etc.) of the comparison.
 5538 //
 5539 // Later, the instruction which matches both the Comparison Op (a Bool) and
 5540 // the flags (produced by the Cmp) specifies the coding of the comparison op
 5541 // by matching a specific subtype of Bool operand below, such as cmpOpU.
 5542 
 5543 // used for signed integral comparisons and fp comparisons
 5544 
 5545 operand cmpOp()
 5546 %{
 5547   match(Bool);
 5548 
 5549   format %{ "" %}
 5550   interface(COND_INTER) %{
 5551     equal(0x0, "eq");
 5552     not_equal(0x1, "ne");
 5553     less(0xb, "lt");
 5554     greater_equal(0xa, "ge");
 5555     less_equal(0xd, "le");
 5556     greater(0xc, "gt");
 5557     overflow(0x6, "vs");
 5558     no_overflow(0x7, "vc");
 5559   %}
 5560 %}
 5561 
 5562 // used for unsigned integral comparisons
 5563 
 5564 operand cmpOpU()
 5565 %{
 5566   match(Bool);
 5567 
 5568   format %{ "" %}
 5569   interface(COND_INTER) %{
 5570     equal(0x0, "eq");
 5571     not_equal(0x1, "ne");
 5572     less(0x3, "lo");
 5573     greater_equal(0x2, "hs");
 5574     less_equal(0x9, "ls");
 5575     greater(0x8, "hi");
 5576     overflow(0x6, "vs");
 5577     no_overflow(0x7, "vc");
 5578   %}
 5579 %}
 5580 
 5581 // used for certain integral comparisons which can be
 5582 // converted to cbxx or tbxx instructions
 5583 
 5584 operand cmpOpEqNe()
 5585 %{
 5586   match(Bool);
 5587   op_cost(0);
 5588   predicate(n->as_Bool()->_test._test == BoolTest::ne
 5589             || n->as_Bool()->_test._test == BoolTest::eq);
 5590 
 5591   format %{ "" %}
 5592   interface(COND_INTER) %{
 5593     equal(0x0, "eq");
 5594     not_equal(0x1, "ne");
 5595     less(0xb, "lt");
 5596     greater_equal(0xa, "ge");
 5597     less_equal(0xd, "le");
 5598     greater(0xc, "gt");
 5599     overflow(0x6, "vs");
 5600     no_overflow(0x7, "vc");
 5601   %}
 5602 %}
 5603 
 5604 // used for certain integral comparisons which can be
 5605 // converted to cbxx or tbxx instructions
 5606 
 5607 operand cmpOpLtGe()
 5608 %{
 5609   match(Bool);
 5610   op_cost(0);
 5611 
 5612   predicate(n->as_Bool()->_test._test == BoolTest::lt
 5613             || n->as_Bool()->_test._test == BoolTest::ge);
 5614 
 5615   format %{ "" %}
 5616   interface(COND_INTER) %{
 5617     equal(0x0, "eq");
 5618     not_equal(0x1, "ne");
 5619     less(0xb, "lt");
 5620     greater_equal(0xa, "ge");
 5621     less_equal(0xd, "le");
 5622     greater(0xc, "gt");
 5623     overflow(0x6, "vs");
 5624     no_overflow(0x7, "vc");
 5625   %}
 5626 %}
 5627 
 5628 // used for certain unsigned integral comparisons which can be
 5629 // converted to cbxx or tbxx instructions
 5630 
 5631 operand cmpOpUEqNeLeGt()
 5632 %{
 5633   match(Bool);
 5634   op_cost(0);
 5635 
 5636   predicate(n->as_Bool()->_test._test == BoolTest::eq ||
 5637             n->as_Bool()->_test._test == BoolTest::ne ||
 5638             n->as_Bool()->_test._test == BoolTest::le ||
 5639             n->as_Bool()->_test._test == BoolTest::gt);
 5640 
 5641   format %{ "" %}
 5642   interface(COND_INTER) %{
 5643     equal(0x0, "eq");
 5644     not_equal(0x1, "ne");
 5645     less(0x3, "lo");
 5646     greater_equal(0x2, "hs");
 5647     less_equal(0x9, "ls");
 5648     greater(0x8, "hi");
 5649     overflow(0x6, "vs");
 5650     no_overflow(0x7, "vc");
 5651   %}
 5652 %}
 5653 
 5654 // Special operand allowing long args to int ops to be truncated for free
 5655 
 5656 operand iRegL2I(iRegL reg) %{
 5657 
 5658   op_cost(0);
 5659 
 5660   match(ConvL2I reg);
 5661 
 5662   format %{ "l2i($reg)" %}
 5663 
 5664   interface(REG_INTER)
 5665 %}
 5666 
 5667 opclass vmem2(indirect, indIndex, indOffI2, indOffL2);
 5668 opclass vmem4(indirect, indIndex, indOffI4, indOffL4);
 5669 opclass vmem8(indirect, indIndex, indOffI8, indOffL8);
 5670 opclass vmem16(indirect, indIndex, indOffI16, indOffL16);
 5671 
 5672 //----------OPERAND CLASSES----------------------------------------------------
 5673 // Operand Classes are groups of operands that are used as to simplify
 5674 // instruction definitions by not requiring the AD writer to specify
 5675 // separate instructions for every form of operand when the
 5676 // instruction accepts multiple operand types with the same basic
 5677 // encoding and format. The classic case of this is memory operands.
 5678 
 5679 // memory is used to define read/write location for load/store
 5680 // instruction defs. we can turn a memory op into an Address
 5681 
 5682 opclass memory1(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI1, indOffL1,
 5683                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN);
 5684 
 5685 opclass memory2(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI2, indOffL2,
 5686                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN);
 5687 
 5688 opclass memory4(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI4, indOffL4,
 5689                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
 5690 
 5691 opclass memory8(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI8, indOffL8,
 5692                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
 5693 
 5694 // All of the memory operands. For the pipeline description.
 5695 opclass memory(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex,
 5696                indOffI1, indOffL1, indOffI2, indOffL2, indOffI4, indOffL4, indOffI8, indOffL8,
 5697                indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
 5698 
 5699 
 5700 // iRegIorL2I is used for src inputs in rules for 32 bit int (I)
 5701 // operations. it allows the src to be either an iRegI or a (ConvL2I
 5702 // iRegL). in the latter case the l2i normally planted for a ConvL2I
 5703 // can be elided because the 32-bit instruction will just employ the
 5704 // lower 32 bits anyway.
 5705 //
 5706 // n.b. this does not elide all L2I conversions. if the truncated
 5707 // value is consumed by more than one operation then the ConvL2I
 5708 // cannot be bundled into the consuming nodes so an l2i gets planted
 5709 // (actually a movw $dst $src) and the downstream instructions consume
 5710 // the result of the l2i as an iRegI input. That's a shame since the
 5711 // movw is actually redundant but its not too costly.
 5712 
 5713 opclass iRegIorL2I(iRegI, iRegL2I);
 5714 
 5715 //----------PIPELINE-----------------------------------------------------------
 5716 // Rules which define the behavior of the target architectures pipeline.
 5717 
 5718 // For specific pipelines, eg A53, define the stages of that pipeline
 5719 //pipe_desc(ISS, EX1, EX2, WR);
 5720 #define ISS S0
 5721 #define EX1 S1
 5722 #define EX2 S2
 5723 #define WR  S3
 5724 
 5725 // Integer ALU reg operation
 5726 pipeline %{
 5727 
 5728 attributes %{
 5729   // ARM instructions are of fixed length
 5730   fixed_size_instructions;        // Fixed size instructions TODO does
 5731   max_instructions_per_bundle = 4;   // A53 = 2, A57 = 4
 5732   // ARM instructions come in 32-bit word units
 5733   instruction_unit_size = 4;         // An instruction is 4 bytes long
 5734   instruction_fetch_unit_size = 64;  // The processor fetches one line
 5735   instruction_fetch_units = 1;       // of 64 bytes
 5736 
 5737   // List of nop instructions
 5738   nops( MachNop );
 5739 %}
 5740 
 5741 // We don't use an actual pipeline model so don't care about resources
 5742 // or description. we do use pipeline classes to introduce fixed
 5743 // latencies
 5744 
 5745 //----------RESOURCES----------------------------------------------------------
 5746 // Resources are the functional units available to the machine
 5747 
 5748 resources( INS0, INS1, INS01 = INS0 | INS1,
 5749            ALU0, ALU1, ALU = ALU0 | ALU1,
 5750            MAC,
 5751            DIV,
 5752            BRANCH,
 5753            LDST,
 5754            NEON_FP);
 5755 
 5756 //----------PIPELINE DESCRIPTION-----------------------------------------------
 5757 // Pipeline Description specifies the stages in the machine's pipeline
 5758 
 5759 // Define the pipeline as a generic 6 stage pipeline
 5760 pipe_desc(S0, S1, S2, S3, S4, S5);
 5761 
 5762 //----------PIPELINE CLASSES---------------------------------------------------
 5763 // Pipeline Classes describe the stages in which input and output are
 5764 // referenced by the hardware pipeline.
 5765 
 5766 pipe_class fp_dop_reg_reg_s(vRegF dst, vRegF src1, vRegF src2)
 5767 %{
 5768   single_instruction;
 5769   src1   : S1(read);
 5770   src2   : S2(read);
 5771   dst    : S5(write);
 5772   INS01  : ISS;
 5773   NEON_FP : S5;
 5774 %}
 5775 
 5776 pipe_class fp_dop_reg_reg_d(vRegD dst, vRegD src1, vRegD src2)
 5777 %{
 5778   single_instruction;
 5779   src1   : S1(read);
 5780   src2   : S2(read);
 5781   dst    : S5(write);
 5782   INS01  : ISS;
 5783   NEON_FP : S5;
 5784 %}
 5785 
 5786 pipe_class fp_uop_s(vRegF dst, vRegF src)
 5787 %{
 5788   single_instruction;
 5789   src    : S1(read);
 5790   dst    : S5(write);
 5791   INS01  : ISS;
 5792   NEON_FP : S5;
 5793 %}
 5794 
 5795 pipe_class fp_uop_d(vRegD dst, vRegD src)
 5796 %{
 5797   single_instruction;
 5798   src    : S1(read);
 5799   dst    : S5(write);
 5800   INS01  : ISS;
 5801   NEON_FP : S5;
 5802 %}
 5803 
 5804 pipe_class fp_d2f(vRegF dst, vRegD src)
 5805 %{
 5806   single_instruction;
 5807   src    : S1(read);
 5808   dst    : S5(write);
 5809   INS01  : ISS;
 5810   NEON_FP : S5;
 5811 %}
 5812 
 5813 pipe_class fp_f2d(vRegD dst, vRegF src)
 5814 %{
 5815   single_instruction;
 5816   src    : S1(read);
 5817   dst    : S5(write);
 5818   INS01  : ISS;
 5819   NEON_FP : S5;
 5820 %}
 5821 
 5822 pipe_class fp_f2i(iRegINoSp dst, vRegF src)
 5823 %{
 5824   single_instruction;
 5825   src    : S1(read);
 5826   dst    : S5(write);
 5827   INS01  : ISS;
 5828   NEON_FP : S5;
 5829 %}
 5830 
 5831 pipe_class fp_f2l(iRegLNoSp dst, vRegF src)
 5832 %{
 5833   single_instruction;
 5834   src    : S1(read);
 5835   dst    : S5(write);
 5836   INS01  : ISS;
 5837   NEON_FP : S5;
 5838 %}
 5839 
 5840 pipe_class fp_i2f(vRegF dst, iRegIorL2I src)
 5841 %{
 5842   single_instruction;
 5843   src    : S1(read);
 5844   dst    : S5(write);
 5845   INS01  : ISS;
 5846   NEON_FP : S5;
 5847 %}
 5848 
 5849 pipe_class fp_l2f(vRegF dst, iRegL src)
 5850 %{
 5851   single_instruction;
 5852   src    : S1(read);
 5853   dst    : S5(write);
 5854   INS01  : ISS;
 5855   NEON_FP : S5;
 5856 %}
 5857 
 5858 pipe_class fp_d2i(iRegINoSp dst, vRegD src)
 5859 %{
 5860   single_instruction;
 5861   src    : S1(read);
 5862   dst    : S5(write);
 5863   INS01  : ISS;
 5864   NEON_FP : S5;
 5865 %}
 5866 
 5867 pipe_class fp_d2l(iRegLNoSp dst, vRegD src)
 5868 %{
 5869   single_instruction;
 5870   src    : S1(read);
 5871   dst    : S5(write);
 5872   INS01  : ISS;
 5873   NEON_FP : S5;
 5874 %}
 5875 
 5876 pipe_class fp_i2d(vRegD dst, iRegIorL2I src)
 5877 %{
 5878   single_instruction;
 5879   src    : S1(read);
 5880   dst    : S5(write);
 5881   INS01  : ISS;
 5882   NEON_FP : S5;
 5883 %}
 5884 
 5885 pipe_class fp_l2d(vRegD dst, iRegIorL2I src)
 5886 %{
 5887   single_instruction;
 5888   src    : S1(read);
 5889   dst    : S5(write);
 5890   INS01  : ISS;
 5891   NEON_FP : S5;
 5892 %}
 5893 
 5894 pipe_class fp_div_s(vRegF dst, vRegF src1, vRegF src2)
 5895 %{
 5896   single_instruction;
 5897   src1   : S1(read);
 5898   src2   : S2(read);
 5899   dst    : S5(write);
 5900   INS0   : ISS;
 5901   NEON_FP : S5;
 5902 %}
 5903 
 5904 pipe_class fp_div_d(vRegD dst, vRegD src1, vRegD src2)
 5905 %{
 5906   single_instruction;
 5907   src1   : S1(read);
 5908   src2   : S2(read);
 5909   dst    : S5(write);
 5910   INS0   : ISS;
 5911   NEON_FP : S5;
 5912 %}
 5913 
 5914 pipe_class fp_cond_reg_reg_s(vRegF dst, vRegF src1, vRegF src2, rFlagsReg cr)
 5915 %{
 5916   single_instruction;
 5917   cr     : S1(read);
 5918   src1   : S1(read);
 5919   src2   : S1(read);
 5920   dst    : S3(write);
 5921   INS01  : ISS;
 5922   NEON_FP : S3;
 5923 %}
 5924 
 5925 pipe_class fp_cond_reg_reg_d(vRegD dst, vRegD src1, vRegD src2, rFlagsReg cr)
 5926 %{
 5927   single_instruction;
 5928   cr     : S1(read);
 5929   src1   : S1(read);
 5930   src2   : S1(read);
 5931   dst    : S3(write);
 5932   INS01  : ISS;
 5933   NEON_FP : S3;
 5934 %}
 5935 
 5936 pipe_class fp_imm_s(vRegF dst)
 5937 %{
 5938   single_instruction;
 5939   dst    : S3(write);
 5940   INS01  : ISS;
 5941   NEON_FP : S3;
 5942 %}
 5943 
 5944 pipe_class fp_imm_d(vRegD dst)
 5945 %{
 5946   single_instruction;
 5947   dst    : S3(write);
 5948   INS01  : ISS;
 5949   NEON_FP : S3;
 5950 %}
 5951 
 5952 pipe_class fp_load_constant_s(vRegF dst)
 5953 %{
 5954   single_instruction;
 5955   dst    : S4(write);
 5956   INS01  : ISS;
 5957   NEON_FP : S4;
 5958 %}
 5959 
 5960 pipe_class fp_load_constant_d(vRegD dst)
 5961 %{
 5962   single_instruction;
 5963   dst    : S4(write);
 5964   INS01  : ISS;
 5965   NEON_FP : S4;
 5966 %}
 5967 
 5968 //------- Integer ALU operations --------------------------
 5969 
 5970 // Integer ALU reg-reg operation
 5971 // Operands needed in EX1, result generated in EX2
 5972 // Eg.  ADD     x0, x1, x2
 5973 pipe_class ialu_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 5974 %{
 5975   single_instruction;
 5976   dst    : EX2(write);
 5977   src1   : EX1(read);
 5978   src2   : EX1(read);
 5979   INS01  : ISS; // Dual issue as instruction 0 or 1
 5980   ALU    : EX2;
 5981 %}
 5982 
 5983 // Integer ALU reg-reg operation with constant shift
 5984 // Shifted register must be available in LATE_ISS instead of EX1
 5985 // Eg.  ADD     x0, x1, x2, LSL #2
 5986 pipe_class ialu_reg_reg_shift(iRegI dst, iRegI src1, iRegI src2, immI shift)
 5987 %{
 5988   single_instruction;
 5989   dst    : EX2(write);
 5990   src1   : EX1(read);
 5991   src2   : ISS(read);
 5992   INS01  : ISS;
 5993   ALU    : EX2;
 5994 %}
 5995 
 5996 // Integer ALU reg operation with constant shift
 5997 // Eg.  LSL     x0, x1, #shift
 5998 pipe_class ialu_reg_shift(iRegI dst, iRegI src1)
 5999 %{
 6000   single_instruction;
 6001   dst    : EX2(write);
 6002   src1   : ISS(read);
 6003   INS01  : ISS;
 6004   ALU    : EX2;
 6005 %}
 6006 
 6007 // Integer ALU reg-reg operation with variable shift
 6008 // Both operands must be available in LATE_ISS instead of EX1
 6009 // Result is available in EX1 instead of EX2
 6010 // Eg.  LSLV    x0, x1, x2
 6011 pipe_class ialu_reg_reg_vshift(iRegI dst, iRegI src1, iRegI src2)
 6012 %{
 6013   single_instruction;
 6014   dst    : EX1(write);
 6015   src1   : ISS(read);
 6016   src2   : ISS(read);
 6017   INS01  : ISS;
 6018   ALU    : EX1;
 6019 %}
 6020 
 6021 // Integer ALU reg-reg operation with extract
 6022 // As for _vshift above, but result generated in EX2
 6023 // Eg.  EXTR    x0, x1, x2, #N
 6024 pipe_class ialu_reg_reg_extr(iRegI dst, iRegI src1, iRegI src2)
 6025 %{
 6026   single_instruction;
 6027   dst    : EX2(write);
 6028   src1   : ISS(read);
 6029   src2   : ISS(read);
 6030   INS1   : ISS; // Can only dual issue as Instruction 1
 6031   ALU    : EX1;
 6032 %}
 6033 
 6034 // Integer ALU reg operation
 6035 // Eg.  NEG     x0, x1
 6036 pipe_class ialu_reg(iRegI dst, iRegI src)
 6037 %{
 6038   single_instruction;
 6039   dst    : EX2(write);
 6040   src    : EX1(read);
 6041   INS01  : ISS;
 6042   ALU    : EX2;
 6043 %}
 6044 
 6045 // Integer ALU reg mmediate operation
 6046 // Eg.  ADD     x0, x1, #N
 6047 pipe_class ialu_reg_imm(iRegI dst, iRegI src1)
 6048 %{
 6049   single_instruction;
 6050   dst    : EX2(write);
 6051   src1   : EX1(read);
 6052   INS01  : ISS;
 6053   ALU    : EX2;
 6054 %}
 6055 
 6056 // Integer ALU immediate operation (no source operands)
 6057 // Eg.  MOV     x0, #N
 6058 pipe_class ialu_imm(iRegI dst)
 6059 %{
 6060   single_instruction;
 6061   dst    : EX1(write);
 6062   INS01  : ISS;
 6063   ALU    : EX1;
 6064 %}
 6065 
 6066 //------- Compare operation -------------------------------
 6067 
 6068 // Compare reg-reg
 6069 // Eg.  CMP     x0, x1
 6070 pipe_class icmp_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
 6071 %{
 6072   single_instruction;
 6073 //  fixed_latency(16);
 6074   cr     : EX2(write);
 6075   op1    : EX1(read);
 6076   op2    : EX1(read);
 6077   INS01  : ISS;
 6078   ALU    : EX2;
 6079 %}
 6080 
 6081 // Compare reg-reg
 6082 // Eg.  CMP     x0, #N
 6083 pipe_class icmp_reg_imm(rFlagsReg cr, iRegI op1)
 6084 %{
 6085   single_instruction;
 6086 //  fixed_latency(16);
 6087   cr     : EX2(write);
 6088   op1    : EX1(read);
 6089   INS01  : ISS;
 6090   ALU    : EX2;
 6091 %}
 6092 
 6093 //------- Conditional instructions ------------------------
 6094 
 6095 // Conditional no operands
 6096 // Eg.  CSINC   x0, zr, zr, <cond>
 6097 pipe_class icond_none(iRegI dst, rFlagsReg cr)
 6098 %{
 6099   single_instruction;
 6100   cr     : EX1(read);
 6101   dst    : EX2(write);
 6102   INS01  : ISS;
 6103   ALU    : EX2;
 6104 %}
 6105 
 6106 // Conditional 2 operand
 6107 // EG.  CSEL    X0, X1, X2, <cond>
 6108 pipe_class icond_reg_reg(iRegI dst, iRegI src1, iRegI src2, rFlagsReg cr)
 6109 %{
 6110   single_instruction;
 6111   cr     : EX1(read);
 6112   src1   : EX1(read);
 6113   src2   : EX1(read);
 6114   dst    : EX2(write);
 6115   INS01  : ISS;
 6116   ALU    : EX2;
 6117 %}
 6118 
 6119 // Conditional 2 operand
 6120 // EG.  CSEL    X0, X1, X2, <cond>
 6121 pipe_class icond_reg(iRegI dst, iRegI src, rFlagsReg cr)
 6122 %{
 6123   single_instruction;
 6124   cr     : EX1(read);
 6125   src    : EX1(read);
 6126   dst    : EX2(write);
 6127   INS01  : ISS;
 6128   ALU    : EX2;
 6129 %}
 6130 
 6131 //------- Multiply pipeline operations --------------------
 6132 
 6133 // Multiply reg-reg
 6134 // Eg.  MUL     w0, w1, w2
 6135 pipe_class imul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 6136 %{
 6137   single_instruction;
 6138   dst    : WR(write);
 6139   src1   : ISS(read);
 6140   src2   : ISS(read);
 6141   INS01  : ISS;
 6142   MAC    : WR;
 6143 %}
 6144 
 6145 // Multiply accumulate
 6146 // Eg.  MADD    w0, w1, w2, w3
 6147 pipe_class imac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
 6148 %{
 6149   single_instruction;
 6150   dst    : WR(write);
 6151   src1   : ISS(read);
 6152   src2   : ISS(read);
 6153   src3   : ISS(read);
 6154   INS01  : ISS;
 6155   MAC    : WR;
 6156 %}
 6157 
 6158 // Eg.  MUL     w0, w1, w2
 6159 pipe_class lmul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 6160 %{
 6161   single_instruction;
 6162   fixed_latency(3); // Maximum latency for 64 bit mul
 6163   dst    : WR(write);
 6164   src1   : ISS(read);
 6165   src2   : ISS(read);
 6166   INS01  : ISS;
 6167   MAC    : WR;
 6168 %}
 6169 
 6170 // Multiply accumulate
 6171 // Eg.  MADD    w0, w1, w2, w3
 6172 pipe_class lmac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
 6173 %{
 6174   single_instruction;
 6175   fixed_latency(3); // Maximum latency for 64 bit mul
 6176   dst    : WR(write);
 6177   src1   : ISS(read);
 6178   src2   : ISS(read);
 6179   src3   : ISS(read);
 6180   INS01  : ISS;
 6181   MAC    : WR;
 6182 %}
 6183 
 6184 //------- Divide pipeline operations --------------------
 6185 
 6186 // Eg.  SDIV    w0, w1, w2
 6187 pipe_class idiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 6188 %{
 6189   single_instruction;
 6190   fixed_latency(8); // Maximum latency for 32 bit divide
 6191   dst    : WR(write);
 6192   src1   : ISS(read);
 6193   src2   : ISS(read);
 6194   INS0   : ISS; // Can only dual issue as instruction 0
 6195   DIV    : WR;
 6196 %}
 6197 
 6198 // Eg.  SDIV    x0, x1, x2
 6199 pipe_class ldiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
 6200 %{
 6201   single_instruction;
 6202   fixed_latency(16); // Maximum latency for 64 bit divide
 6203   dst    : WR(write);
 6204   src1   : ISS(read);
 6205   src2   : ISS(read);
 6206   INS0   : ISS; // Can only dual issue as instruction 0
 6207   DIV    : WR;
 6208 %}
 6209 
 6210 //------- Load pipeline operations ------------------------
 6211 
 6212 // Load - prefetch
 6213 // Eg.  PFRM    <mem>
 6214 pipe_class iload_prefetch(memory mem)
 6215 %{
 6216   single_instruction;
 6217   mem    : ISS(read);
 6218   INS01  : ISS;
 6219   LDST   : WR;
 6220 %}
 6221 
 6222 // Load - reg, mem
 6223 // Eg.  LDR     x0, <mem>
 6224 pipe_class iload_reg_mem(iRegI dst, memory mem)
 6225 %{
 6226   single_instruction;
 6227   dst    : WR(write);
 6228   mem    : ISS(read);
 6229   INS01  : ISS;
 6230   LDST   : WR;
 6231 %}
 6232 
 6233 // Load - reg, reg
 6234 // Eg.  LDR     x0, [sp, x1]
 6235 pipe_class iload_reg_reg(iRegI dst, iRegI src)
 6236 %{
 6237   single_instruction;
 6238   dst    : WR(write);
 6239   src    : ISS(read);
 6240   INS01  : ISS;
 6241   LDST   : WR;
 6242 %}
 6243 
 6244 //------- Store pipeline operations -----------------------
 6245 
 6246 // Store - zr, mem
 6247 // Eg.  STR     zr, <mem>
 6248 pipe_class istore_mem(memory mem)
 6249 %{
 6250   single_instruction;
 6251   mem    : ISS(read);
 6252   INS01  : ISS;
 6253   LDST   : WR;
 6254 %}
 6255 
 6256 // Store - reg, mem
 6257 // Eg.  STR     x0, <mem>
 6258 pipe_class istore_reg_mem(iRegI src, memory mem)
 6259 %{
 6260   single_instruction;
 6261   mem    : ISS(read);
 6262   src    : EX2(read);
 6263   INS01  : ISS;
 6264   LDST   : WR;
 6265 %}
 6266 
 6267 // Store - reg, reg
 6268 // Eg. STR      x0, [sp, x1]
 6269 pipe_class istore_reg_reg(iRegI dst, iRegI src)
 6270 %{
 6271   single_instruction;
 6272   dst    : ISS(read);
 6273   src    : EX2(read);
 6274   INS01  : ISS;
 6275   LDST   : WR;
 6276 %}
 6277 
 6278 //------- Store pipeline operations -----------------------
 6279 
 6280 // Branch
 6281 pipe_class pipe_branch()
 6282 %{
 6283   single_instruction;
 6284   INS01  : ISS;
 6285   BRANCH : EX1;
 6286 %}
 6287 
 6288 // Conditional branch
 6289 pipe_class pipe_branch_cond(rFlagsReg cr)
 6290 %{
 6291   single_instruction;
 6292   cr     : EX1(read);
 6293   INS01  : ISS;
 6294   BRANCH : EX1;
 6295 %}
 6296 
 6297 // Compare & Branch
 6298 // EG.  CBZ/CBNZ
 6299 pipe_class pipe_cmp_branch(iRegI op1)
 6300 %{
 6301   single_instruction;
 6302   op1    : EX1(read);
 6303   INS01  : ISS;
 6304   BRANCH : EX1;
 6305 %}
 6306 
 6307 //------- Synchronisation operations ----------------------
 6308 
 6309 // Any operation requiring serialization.
 6310 // EG.  DMB/Atomic Ops/Load Acquire/Str Release
 6311 pipe_class pipe_serial()
 6312 %{
 6313   single_instruction;
 6314   force_serialization;
 6315   fixed_latency(16);
 6316   INS01  : ISS(2); // Cannot dual issue with any other instruction
 6317   LDST   : WR;
 6318 %}
 6319 
 6320 // Generic big/slow expanded idiom - also serialized
 6321 pipe_class pipe_slow()
 6322 %{
 6323   instruction_count(10);
 6324   multiple_bundles;
 6325   force_serialization;
 6326   fixed_latency(16);
 6327   INS01  : ISS(2); // Cannot dual issue with any other instruction
 6328   LDST   : WR;
 6329 %}
 6330 
 6331 // Empty pipeline class
 6332 pipe_class pipe_class_empty()
 6333 %{
 6334   single_instruction;
 6335   fixed_latency(0);
 6336 %}
 6337 
 6338 // Default pipeline class.
 6339 pipe_class pipe_class_default()
 6340 %{
 6341   single_instruction;
 6342   fixed_latency(2);
 6343 %}
 6344 
 6345 // Pipeline class for compares.
 6346 pipe_class pipe_class_compare()
 6347 %{
 6348   single_instruction;
 6349   fixed_latency(16);
 6350 %}
 6351 
 6352 // Pipeline class for memory operations.
 6353 pipe_class pipe_class_memory()
 6354 %{
 6355   single_instruction;
 6356   fixed_latency(16);
 6357 %}
 6358 
 6359 // Pipeline class for call.
 6360 pipe_class pipe_class_call()
 6361 %{
 6362   single_instruction;
 6363   fixed_latency(100);
 6364 %}
 6365 
 6366 // Define the class for the Nop node.
 6367 define %{
 6368    MachNop = pipe_class_empty;
 6369 %}
 6370 
 6371 %}
 6372 //----------INSTRUCTIONS-------------------------------------------------------
 6373 //
 6374 // match      -- States which machine-independent subtree may be replaced
 6375 //               by this instruction.
 6376 // ins_cost   -- The estimated cost of this instruction is used by instruction
 6377 //               selection to identify a minimum cost tree of machine
 6378 //               instructions that matches a tree of machine-independent
 6379 //               instructions.
 6380 // format     -- A string providing the disassembly for this instruction.
 6381 //               The value of an instruction's operand may be inserted
 6382 //               by referring to it with a '$' prefix.
 6383 // opcode     -- Three instruction opcodes may be provided.  These are referred
 6384 //               to within an encode class as $primary, $secondary, and $tertiary
 6385 //               rrspectively.  The primary opcode is commonly used to
 6386 //               indicate the type of machine instruction, while secondary
 6387 //               and tertiary are often used for prefix options or addressing
 6388 //               modes.
 6389 // ins_encode -- A list of encode classes with parameters. The encode class
 6390 //               name must have been defined in an 'enc_class' specification
 6391 //               in the encode section of the architecture description.
 6392 
 6393 // ============================================================================
 6394 // Memory (Load/Store) Instructions
 6395 
 6396 // Load Instructions
 6397 
 6398 // Load Byte (8 bit signed)
 6399 instruct loadB(iRegINoSp dst, memory1 mem)
 6400 %{
 6401   match(Set dst (LoadB mem));
 6402   predicate(!needs_acquiring_load(n));
 6403 
 6404   ins_cost(4 * INSN_COST);
 6405   format %{ "ldrsbw  $dst, $mem\t# byte" %}
 6406 
 6407   ins_encode(aarch64_enc_ldrsbw(dst, mem));
 6408 
 6409   ins_pipe(iload_reg_mem);
 6410 %}
 6411 
 6412 // Load Byte (8 bit signed) into long
 6413 instruct loadB2L(iRegLNoSp dst, memory1 mem)
 6414 %{
 6415   match(Set dst (ConvI2L (LoadB mem)));
 6416   predicate(!needs_acquiring_load(n->in(1)));
 6417 
 6418   ins_cost(4 * INSN_COST);
 6419   format %{ "ldrsb  $dst, $mem\t# byte" %}
 6420 
 6421   ins_encode(aarch64_enc_ldrsb(dst, mem));
 6422 
 6423   ins_pipe(iload_reg_mem);
 6424 %}
 6425 
 6426 // Load Byte (8 bit unsigned)
 6427 instruct loadUB(iRegINoSp dst, memory1 mem)
 6428 %{
 6429   match(Set dst (LoadUB mem));
 6430   predicate(!needs_acquiring_load(n));
 6431 
 6432   ins_cost(4 * INSN_COST);
 6433   format %{ "ldrbw  $dst, $mem\t# byte" %}
 6434 
 6435   ins_encode(aarch64_enc_ldrb(dst, mem));
 6436 
 6437   ins_pipe(iload_reg_mem);
 6438 %}
 6439 
 6440 // Load Byte (8 bit unsigned) into long
 6441 instruct loadUB2L(iRegLNoSp dst, memory1 mem)
 6442 %{
 6443   match(Set dst (ConvI2L (LoadUB mem)));
 6444   predicate(!needs_acquiring_load(n->in(1)));
 6445 
 6446   ins_cost(4 * INSN_COST);
 6447   format %{ "ldrb  $dst, $mem\t# byte" %}
 6448 
 6449   ins_encode(aarch64_enc_ldrb(dst, mem));
 6450 
 6451   ins_pipe(iload_reg_mem);
 6452 %}
 6453 
 6454 // Load Short (16 bit signed)
 6455 instruct loadS(iRegINoSp dst, memory2 mem)
 6456 %{
 6457   match(Set dst (LoadS mem));
 6458   predicate(!needs_acquiring_load(n));
 6459 
 6460   ins_cost(4 * INSN_COST);
 6461   format %{ "ldrshw  $dst, $mem\t# short" %}
 6462 
 6463   ins_encode(aarch64_enc_ldrshw(dst, mem));
 6464 
 6465   ins_pipe(iload_reg_mem);
 6466 %}
 6467 
 6468 // Load Short (16 bit signed) into long
 6469 instruct loadS2L(iRegLNoSp dst, memory2 mem)
 6470 %{
 6471   match(Set dst (ConvI2L (LoadS mem)));
 6472   predicate(!needs_acquiring_load(n->in(1)));
 6473 
 6474   ins_cost(4 * INSN_COST);
 6475   format %{ "ldrsh  $dst, $mem\t# short" %}
 6476 
 6477   ins_encode(aarch64_enc_ldrsh(dst, mem));
 6478 
 6479   ins_pipe(iload_reg_mem);
 6480 %}
 6481 
 6482 // Load Char (16 bit unsigned)
 6483 instruct loadUS(iRegINoSp dst, memory2 mem)
 6484 %{
 6485   match(Set dst (LoadUS mem));
 6486   predicate(!needs_acquiring_load(n));
 6487 
 6488   ins_cost(4 * INSN_COST);
 6489   format %{ "ldrh  $dst, $mem\t# short" %}
 6490 
 6491   ins_encode(aarch64_enc_ldrh(dst, mem));
 6492 
 6493   ins_pipe(iload_reg_mem);
 6494 %}
 6495 
 6496 // Load Short/Char (16 bit unsigned) into long
 6497 instruct loadUS2L(iRegLNoSp dst, memory2 mem)
 6498 %{
 6499   match(Set dst (ConvI2L (LoadUS mem)));
 6500   predicate(!needs_acquiring_load(n->in(1)));
 6501 
 6502   ins_cost(4 * INSN_COST);
 6503   format %{ "ldrh  $dst, $mem\t# short" %}
 6504 
 6505   ins_encode(aarch64_enc_ldrh(dst, mem));
 6506 
 6507   ins_pipe(iload_reg_mem);
 6508 %}
 6509 
 6510 // Load Integer (32 bit signed)
 6511 instruct loadI(iRegINoSp dst, memory4 mem)
 6512 %{
 6513   match(Set dst (LoadI mem));
 6514   predicate(!needs_acquiring_load(n));
 6515 
 6516   ins_cost(4 * INSN_COST);
 6517   format %{ "ldrw  $dst, $mem\t# int" %}
 6518 
 6519   ins_encode(aarch64_enc_ldrw(dst, mem));
 6520 
 6521   ins_pipe(iload_reg_mem);
 6522 %}
 6523 
 6524 // Load Integer (32 bit signed) into long
 6525 instruct loadI2L(iRegLNoSp dst, memory4 mem)
 6526 %{
 6527   match(Set dst (ConvI2L (LoadI mem)));
 6528   predicate(!needs_acquiring_load(n->in(1)));
 6529 
 6530   ins_cost(4 * INSN_COST);
 6531   format %{ "ldrsw  $dst, $mem\t# int" %}
 6532 
 6533   ins_encode(aarch64_enc_ldrsw(dst, mem));
 6534 
 6535   ins_pipe(iload_reg_mem);
 6536 %}
 6537 
 6538 // Load Integer (32 bit unsigned) into long
 6539 instruct loadUI2L(iRegLNoSp dst, memory4 mem, immL_32bits mask)
 6540 %{
 6541   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
 6542   predicate(!needs_acquiring_load(n->in(1)->in(1)->as_Load()));
 6543 
 6544   ins_cost(4 * INSN_COST);
 6545   format %{ "ldrw  $dst, $mem\t# int" %}
 6546 
 6547   ins_encode(aarch64_enc_ldrw(dst, mem));
 6548 
 6549   ins_pipe(iload_reg_mem);
 6550 %}
 6551 
 6552 // Load Long (64 bit signed)
 6553 instruct loadL(iRegLNoSp dst, memory8 mem)
 6554 %{
 6555   match(Set dst (LoadL mem));
 6556   predicate(!needs_acquiring_load(n));
 6557 
 6558   ins_cost(4 * INSN_COST);
 6559   format %{ "ldr  $dst, $mem\t# int" %}
 6560 
 6561   ins_encode(aarch64_enc_ldr(dst, mem));
 6562 
 6563   ins_pipe(iload_reg_mem);
 6564 %}
 6565 
 6566 // Load Range
 6567 instruct loadRange(iRegINoSp dst, memory4 mem)
 6568 %{
 6569   match(Set dst (LoadRange mem));
 6570 
 6571   ins_cost(4 * INSN_COST);
 6572   format %{ "ldrw  $dst, $mem\t# range" %}
 6573 
 6574   ins_encode(aarch64_enc_ldrw(dst, mem));
 6575 
 6576   ins_pipe(iload_reg_mem);
 6577 %}
 6578 
 6579 // Load Pointer
 6580 instruct loadP(iRegPNoSp dst, memory8 mem)
 6581 %{
 6582   match(Set dst (LoadP mem));
 6583   predicate(!needs_acquiring_load(n) && (n->as_Load()->barrier_data() == 0));
 6584 
 6585   ins_cost(4 * INSN_COST);
 6586   format %{ "ldr  $dst, $mem\t# ptr" %}
 6587 
 6588   ins_encode(aarch64_enc_ldr(dst, mem));
 6589 
 6590   ins_pipe(iload_reg_mem);
 6591 %}
 6592 
 6593 // Load Compressed Pointer
 6594 instruct loadN(iRegNNoSp dst, memory4 mem)
 6595 %{
 6596   match(Set dst (LoadN mem));
 6597   predicate(!needs_acquiring_load(n));
 6598 
 6599   ins_cost(4 * INSN_COST);
 6600   format %{ "ldrw  $dst, $mem\t# compressed ptr" %}
 6601 
 6602   ins_encode(aarch64_enc_ldrw(dst, mem));
 6603 
 6604   ins_pipe(iload_reg_mem);
 6605 %}
 6606 
 6607 // Load Klass Pointer
 6608 instruct loadKlass(iRegPNoSp dst, memory8 mem)
 6609 %{
 6610   match(Set dst (LoadKlass mem));
 6611   predicate(!needs_acquiring_load(n));
 6612 
 6613   ins_cost(4 * INSN_COST);
 6614   format %{ "ldr  $dst, $mem\t# class" %}
 6615 
 6616   ins_encode(aarch64_enc_ldr(dst, mem));
 6617 
 6618   ins_pipe(iload_reg_mem);
 6619 %}
 6620 
 6621 // Load Narrow Klass Pointer
 6622 instruct loadNKlass(iRegNNoSp dst, memory4 mem)
 6623 %{
 6624   match(Set dst (LoadNKlass mem));
 6625   predicate(!needs_acquiring_load(n));
 6626 
 6627   ins_cost(4 * INSN_COST);
 6628   format %{ "ldrw  $dst, $mem\t# compressed class ptr" %}
 6629 
 6630   ins_encode(aarch64_enc_ldrw(dst, mem));
 6631 
 6632   ins_pipe(iload_reg_mem);
 6633 %}
 6634 
 6635 // Load Float
 6636 instruct loadF(vRegF dst, memory4 mem)
 6637 %{
 6638   match(Set dst (LoadF mem));
 6639   predicate(!needs_acquiring_load(n));
 6640 
 6641   ins_cost(4 * INSN_COST);
 6642   format %{ "ldrs  $dst, $mem\t# float" %}
 6643 
 6644   ins_encode( aarch64_enc_ldrs(dst, mem) );
 6645 
 6646   ins_pipe(pipe_class_memory);
 6647 %}
 6648 
 6649 // Load Double
 6650 instruct loadD(vRegD dst, memory8 mem)
 6651 %{
 6652   match(Set dst (LoadD mem));
 6653   predicate(!needs_acquiring_load(n));
 6654 
 6655   ins_cost(4 * INSN_COST);
 6656   format %{ "ldrd  $dst, $mem\t# double" %}
 6657 
 6658   ins_encode( aarch64_enc_ldrd(dst, mem) );
 6659 
 6660   ins_pipe(pipe_class_memory);
 6661 %}
 6662 
 6663 
 6664 // Load Int Constant
 6665 instruct loadConI(iRegINoSp dst, immI src)
 6666 %{
 6667   match(Set dst src);
 6668 
 6669   ins_cost(INSN_COST);
 6670   format %{ "mov $dst, $src\t# int" %}
 6671 
 6672   ins_encode( aarch64_enc_movw_imm(dst, src) );
 6673 
 6674   ins_pipe(ialu_imm);
 6675 %}
 6676 
 6677 // Load Long Constant
 6678 instruct loadConL(iRegLNoSp dst, immL src)
 6679 %{
 6680   match(Set dst src);
 6681 
 6682   ins_cost(INSN_COST);
 6683   format %{ "mov $dst, $src\t# long" %}
 6684 
 6685   ins_encode( aarch64_enc_mov_imm(dst, src) );
 6686 
 6687   ins_pipe(ialu_imm);
 6688 %}
 6689 
 6690 // Load Pointer Constant
 6691 
 6692 instruct loadConP(iRegPNoSp dst, immP con)
 6693 %{
 6694   match(Set dst con);
 6695 
 6696   ins_cost(INSN_COST * 4);
 6697   format %{
 6698     "mov  $dst, $con\t# ptr\n\t"
 6699   %}
 6700 
 6701   ins_encode(aarch64_enc_mov_p(dst, con));
 6702 
 6703   ins_pipe(ialu_imm);
 6704 %}
 6705 
 6706 // Load Null Pointer Constant
 6707 
 6708 instruct loadConP0(iRegPNoSp dst, immP0 con)
 6709 %{
 6710   match(Set dst con);
 6711 
 6712   ins_cost(INSN_COST);
 6713   format %{ "mov  $dst, $con\t# nullptr ptr" %}
 6714 
 6715   ins_encode(aarch64_enc_mov_p0(dst, con));
 6716 
 6717   ins_pipe(ialu_imm);
 6718 %}
 6719 
 6720 // Load Pointer Constant One
 6721 
 6722 instruct loadConP1(iRegPNoSp dst, immP_1 con)
 6723 %{
 6724   match(Set dst con);
 6725 
 6726   ins_cost(INSN_COST);
 6727   format %{ "mov  $dst, $con\t# nullptr ptr" %}
 6728 
 6729   ins_encode(aarch64_enc_mov_p1(dst, con));
 6730 
 6731   ins_pipe(ialu_imm);
 6732 %}
 6733 
 6734 // Load Byte Map Base Constant
 6735 
 6736 instruct loadByteMapBase(iRegPNoSp dst, immByteMapBase con)
 6737 %{
 6738   match(Set dst con);
 6739 
 6740   ins_cost(INSN_COST);
 6741   format %{ "adr  $dst, $con\t# Byte Map Base" %}
 6742 
 6743   ins_encode(aarch64_enc_mov_byte_map_base(dst, con));
 6744 
 6745   ins_pipe(ialu_imm);
 6746 %}
 6747 
 6748 // Load Narrow Pointer Constant
 6749 
 6750 instruct loadConN(iRegNNoSp dst, immN con)
 6751 %{
 6752   match(Set dst con);
 6753 
 6754   ins_cost(INSN_COST * 4);
 6755   format %{ "mov  $dst, $con\t# compressed ptr" %}
 6756 
 6757   ins_encode(aarch64_enc_mov_n(dst, con));
 6758 
 6759   ins_pipe(ialu_imm);
 6760 %}
 6761 
 6762 // Load Narrow Null Pointer Constant
 6763 
 6764 instruct loadConN0(iRegNNoSp dst, immN0 con)
 6765 %{
 6766   match(Set dst con);
 6767 
 6768   ins_cost(INSN_COST);
 6769   format %{ "mov  $dst, $con\t# compressed nullptr ptr" %}
 6770 
 6771   ins_encode(aarch64_enc_mov_n0(dst, con));
 6772 
 6773   ins_pipe(ialu_imm);
 6774 %}
 6775 
 6776 // Load Narrow Klass Constant
 6777 
 6778 instruct loadConNKlass(iRegNNoSp dst, immNKlass con)
 6779 %{
 6780   match(Set dst con);
 6781 
 6782   ins_cost(INSN_COST);
 6783   format %{ "mov  $dst, $con\t# compressed klass ptr" %}
 6784 
 6785   ins_encode(aarch64_enc_mov_nk(dst, con));
 6786 
 6787   ins_pipe(ialu_imm);
 6788 %}
 6789 
 6790 // Load Packed Float Constant
 6791 
 6792 instruct loadConF_packed(vRegF dst, immFPacked con) %{
 6793   match(Set dst con);
 6794   ins_cost(INSN_COST * 4);
 6795   format %{ "fmovs  $dst, $con"%}
 6796   ins_encode %{
 6797     __ fmovs(as_FloatRegister($dst$$reg), (double)$con$$constant);
 6798   %}
 6799 
 6800   ins_pipe(fp_imm_s);
 6801 %}
 6802 
 6803 // Load Float Constant
 6804 
 6805 instruct loadConF(vRegF dst, immF con) %{
 6806   match(Set dst con);
 6807 
 6808   ins_cost(INSN_COST * 4);
 6809 
 6810   format %{
 6811     "ldrs $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
 6812   %}
 6813 
 6814   ins_encode %{
 6815     __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
 6816   %}
 6817 
 6818   ins_pipe(fp_load_constant_s);
 6819 %}
 6820 
 6821 // Load Packed Double Constant
 6822 
 6823 instruct loadConD_packed(vRegD dst, immDPacked con) %{
 6824   match(Set dst con);
 6825   ins_cost(INSN_COST);
 6826   format %{ "fmovd  $dst, $con"%}
 6827   ins_encode %{
 6828     __ fmovd(as_FloatRegister($dst$$reg), $con$$constant);
 6829   %}
 6830 
 6831   ins_pipe(fp_imm_d);
 6832 %}
 6833 
 6834 // Load Double Constant
 6835 
 6836 instruct loadConD(vRegD dst, immD con) %{
 6837   match(Set dst con);
 6838 
 6839   ins_cost(INSN_COST * 5);
 6840   format %{
 6841     "ldrd $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
 6842   %}
 6843 
 6844   ins_encode %{
 6845     __ ldrd(as_FloatRegister($dst$$reg), $constantaddress($con));
 6846   %}
 6847 
 6848   ins_pipe(fp_load_constant_d);
 6849 %}
 6850 
 6851 // Store Instructions
 6852 
 6853 // Store CMS card-mark Immediate
 6854 instruct storeimmCM0(immI0 zero, memory1 mem)
 6855 %{
 6856   match(Set mem (StoreCM mem zero));
 6857 
 6858   ins_cost(INSN_COST);
 6859   format %{ "storestore (elided)\n\t"
 6860             "strb zr, $mem\t# byte" %}
 6861 
 6862   ins_encode(aarch64_enc_strb0(mem));
 6863 
 6864   ins_pipe(istore_mem);
 6865 %}
 6866 
 6867 // Store CMS card-mark Immediate with intervening StoreStore
 6868 // needed when using CMS with no conditional card marking
 6869 instruct storeimmCM0_ordered(immI0 zero, memory1 mem)
 6870 %{
 6871   match(Set mem (StoreCM mem zero));
 6872 
 6873   ins_cost(INSN_COST * 2);
 6874   format %{ "storestore\n\t"
 6875             "dmb ishst"
 6876             "\n\tstrb zr, $mem\t# byte" %}
 6877 
 6878   ins_encode(aarch64_enc_strb0_ordered(mem));
 6879 
 6880   ins_pipe(istore_mem);
 6881 %}
 6882 
 6883 // Store Byte
 6884 instruct storeB(iRegIorL2I src, memory1 mem)
 6885 %{
 6886   match(Set mem (StoreB mem src));
 6887   predicate(!needs_releasing_store(n));
 6888 
 6889   ins_cost(INSN_COST);
 6890   format %{ "strb  $src, $mem\t# byte" %}
 6891 
 6892   ins_encode(aarch64_enc_strb(src, mem));
 6893 
 6894   ins_pipe(istore_reg_mem);
 6895 %}
 6896 
 6897 
 6898 instruct storeimmB0(immI0 zero, memory1 mem)
 6899 %{
 6900   match(Set mem (StoreB mem zero));
 6901   predicate(!needs_releasing_store(n));
 6902 
 6903   ins_cost(INSN_COST);
 6904   format %{ "strb rscractch2, $mem\t# byte" %}
 6905 
 6906   ins_encode(aarch64_enc_strb0(mem));
 6907 
 6908   ins_pipe(istore_mem);
 6909 %}
 6910 
 6911 // Store Char/Short
 6912 instruct storeC(iRegIorL2I src, memory2 mem)
 6913 %{
 6914   match(Set mem (StoreC mem src));
 6915   predicate(!needs_releasing_store(n));
 6916 
 6917   ins_cost(INSN_COST);
 6918   format %{ "strh  $src, $mem\t# short" %}
 6919 
 6920   ins_encode(aarch64_enc_strh(src, mem));
 6921 
 6922   ins_pipe(istore_reg_mem);
 6923 %}
 6924 
 6925 instruct storeimmC0(immI0 zero, memory2 mem)
 6926 %{
 6927   match(Set mem (StoreC mem zero));
 6928   predicate(!needs_releasing_store(n));
 6929 
 6930   ins_cost(INSN_COST);
 6931   format %{ "strh  zr, $mem\t# short" %}
 6932 
 6933   ins_encode(aarch64_enc_strh0(mem));
 6934 
 6935   ins_pipe(istore_mem);
 6936 %}
 6937 
 6938 // Store Integer
 6939 
 6940 instruct storeI(iRegIorL2I src, memory4 mem)
 6941 %{
 6942   match(Set mem(StoreI mem src));
 6943   predicate(!needs_releasing_store(n));
 6944 
 6945   ins_cost(INSN_COST);
 6946   format %{ "strw  $src, $mem\t# int" %}
 6947 
 6948   ins_encode(aarch64_enc_strw(src, mem));
 6949 
 6950   ins_pipe(istore_reg_mem);
 6951 %}
 6952 
 6953 instruct storeimmI0(immI0 zero, memory4 mem)
 6954 %{
 6955   match(Set mem(StoreI mem zero));
 6956   predicate(!needs_releasing_store(n));
 6957 
 6958   ins_cost(INSN_COST);
 6959   format %{ "strw  zr, $mem\t# int" %}
 6960 
 6961   ins_encode(aarch64_enc_strw0(mem));
 6962 
 6963   ins_pipe(istore_mem);
 6964 %}
 6965 
 6966 // Store Long (64 bit signed)
 6967 instruct storeL(iRegL src, memory8 mem)
 6968 %{
 6969   match(Set mem (StoreL mem src));
 6970   predicate(!needs_releasing_store(n));
 6971 
 6972   ins_cost(INSN_COST);
 6973   format %{ "str  $src, $mem\t# int" %}
 6974 
 6975   ins_encode(aarch64_enc_str(src, mem));
 6976 
 6977   ins_pipe(istore_reg_mem);
 6978 %}
 6979 
 6980 // Store Long (64 bit signed)
 6981 instruct storeimmL0(immL0 zero, memory8 mem)
 6982 %{
 6983   match(Set mem (StoreL mem zero));
 6984   predicate(!needs_releasing_store(n));
 6985 
 6986   ins_cost(INSN_COST);
 6987   format %{ "str  zr, $mem\t# int" %}
 6988 
 6989   ins_encode(aarch64_enc_str0(mem));
 6990 
 6991   ins_pipe(istore_mem);
 6992 %}
 6993 
 6994 // Store Pointer
 6995 instruct storeP(iRegP src, memory8 mem)
 6996 %{
 6997   match(Set mem (StoreP mem src));
 6998   predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
 6999 
 7000   ins_cost(INSN_COST);
 7001   format %{ "str  $src, $mem\t# ptr" %}
 7002 
 7003   ins_encode(aarch64_enc_str(src, mem));
 7004 
 7005   ins_pipe(istore_reg_mem);
 7006 %}
 7007 
 7008 // Store Pointer
 7009 instruct storeimmP0(immP0 zero, memory8 mem)
 7010 %{
 7011   match(Set mem (StoreP mem zero));
 7012   predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
 7013 
 7014   ins_cost(INSN_COST);
 7015   format %{ "str zr, $mem\t# ptr" %}
 7016 
 7017   ins_encode(aarch64_enc_str0(mem));
 7018 
 7019   ins_pipe(istore_mem);
 7020 %}
 7021 
 7022 // Store Compressed Pointer
 7023 instruct storeN(iRegN src, memory4 mem)
 7024 %{
 7025   match(Set mem (StoreN mem src));
 7026   predicate(!needs_releasing_store(n));
 7027 
 7028   ins_cost(INSN_COST);
 7029   format %{ "strw  $src, $mem\t# compressed ptr" %}
 7030 
 7031   ins_encode(aarch64_enc_strw(src, mem));
 7032 
 7033   ins_pipe(istore_reg_mem);
 7034 %}
 7035 
 7036 instruct storeImmN0(immN0 zero, memory4 mem)
 7037 %{
 7038   match(Set mem (StoreN mem zero));
 7039   predicate(!needs_releasing_store(n));
 7040 
 7041   ins_cost(INSN_COST);
 7042   format %{ "strw  zr, $mem\t# compressed ptr" %}
 7043 
 7044   ins_encode(aarch64_enc_strw0(mem));
 7045 
 7046   ins_pipe(istore_mem);
 7047 %}
 7048 
 7049 // Store Float
 7050 instruct storeF(vRegF src, memory4 mem)
 7051 %{
 7052   match(Set mem (StoreF mem src));
 7053   predicate(!needs_releasing_store(n));
 7054 
 7055   ins_cost(INSN_COST);
 7056   format %{ "strs  $src, $mem\t# float" %}
 7057 
 7058   ins_encode( aarch64_enc_strs(src, mem) );
 7059 
 7060   ins_pipe(pipe_class_memory);
 7061 %}
 7062 
 7063 // TODO
 7064 // implement storeImmF0 and storeFImmPacked
 7065 
 7066 // Store Double
 7067 instruct storeD(vRegD src, memory8 mem)
 7068 %{
 7069   match(Set mem (StoreD mem src));
 7070   predicate(!needs_releasing_store(n));
 7071 
 7072   ins_cost(INSN_COST);
 7073   format %{ "strd  $src, $mem\t# double" %}
 7074 
 7075   ins_encode( aarch64_enc_strd(src, mem) );
 7076 
 7077   ins_pipe(pipe_class_memory);
 7078 %}
 7079 
 7080 // Store Compressed Klass Pointer
 7081 instruct storeNKlass(iRegN src, memory4 mem)
 7082 %{
 7083   predicate(!needs_releasing_store(n));
 7084   match(Set mem (StoreNKlass mem src));
 7085 
 7086   ins_cost(INSN_COST);
 7087   format %{ "strw  $src, $mem\t# compressed klass ptr" %}
 7088 
 7089   ins_encode(aarch64_enc_strw(src, mem));
 7090 
 7091   ins_pipe(istore_reg_mem);
 7092 %}
 7093 
 7094 // TODO
 7095 // implement storeImmD0 and storeDImmPacked
 7096 
 7097 // prefetch instructions
 7098 // Must be safe to execute with invalid address (cannot fault).
 7099 
 7100 instruct prefetchalloc( memory8 mem ) %{
 7101   match(PrefetchAllocation mem);
 7102 
 7103   ins_cost(INSN_COST);
 7104   format %{ "prfm $mem, PSTL1KEEP\t# Prefetch into level 1 cache write keep" %}
 7105 
 7106   ins_encode( aarch64_enc_prefetchw(mem) );
 7107 
 7108   ins_pipe(iload_prefetch);
 7109 %}
 7110 
 7111 //  ---------------- volatile loads and stores ----------------
 7112 
 7113 // Load Byte (8 bit signed)
 7114 instruct loadB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 7115 %{
 7116   match(Set dst (LoadB mem));
 7117 
 7118   ins_cost(VOLATILE_REF_COST);
 7119   format %{ "ldarsb  $dst, $mem\t# byte" %}
 7120 
 7121   ins_encode(aarch64_enc_ldarsb(dst, mem));
 7122 
 7123   ins_pipe(pipe_serial);
 7124 %}
 7125 
 7126 // Load Byte (8 bit signed) into long
 7127 instruct loadB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 7128 %{
 7129   match(Set dst (ConvI2L (LoadB mem)));
 7130 
 7131   ins_cost(VOLATILE_REF_COST);
 7132   format %{ "ldarsb  $dst, $mem\t# byte" %}
 7133 
 7134   ins_encode(aarch64_enc_ldarsb(dst, mem));
 7135 
 7136   ins_pipe(pipe_serial);
 7137 %}
 7138 
 7139 // Load Byte (8 bit unsigned)
 7140 instruct loadUB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 7141 %{
 7142   match(Set dst (LoadUB mem));
 7143 
 7144   ins_cost(VOLATILE_REF_COST);
 7145   format %{ "ldarb  $dst, $mem\t# byte" %}
 7146 
 7147   ins_encode(aarch64_enc_ldarb(dst, mem));
 7148 
 7149   ins_pipe(pipe_serial);
 7150 %}
 7151 
 7152 // Load Byte (8 bit unsigned) into long
 7153 instruct loadUB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 7154 %{
 7155   match(Set dst (ConvI2L (LoadUB mem)));
 7156 
 7157   ins_cost(VOLATILE_REF_COST);
 7158   format %{ "ldarb  $dst, $mem\t# byte" %}
 7159 
 7160   ins_encode(aarch64_enc_ldarb(dst, mem));
 7161 
 7162   ins_pipe(pipe_serial);
 7163 %}
 7164 
 7165 // Load Short (16 bit signed)
 7166 instruct loadS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 7167 %{
 7168   match(Set dst (LoadS mem));
 7169 
 7170   ins_cost(VOLATILE_REF_COST);
 7171   format %{ "ldarshw  $dst, $mem\t# short" %}
 7172 
 7173   ins_encode(aarch64_enc_ldarshw(dst, mem));
 7174 
 7175   ins_pipe(pipe_serial);
 7176 %}
 7177 
 7178 instruct loadUS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 7179 %{
 7180   match(Set dst (LoadUS mem));
 7181 
 7182   ins_cost(VOLATILE_REF_COST);
 7183   format %{ "ldarhw  $dst, $mem\t# short" %}
 7184 
 7185   ins_encode(aarch64_enc_ldarhw(dst, mem));
 7186 
 7187   ins_pipe(pipe_serial);
 7188 %}
 7189 
 7190 // Load Short/Char (16 bit unsigned) into long
 7191 instruct loadUS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 7192 %{
 7193   match(Set dst (ConvI2L (LoadUS mem)));
 7194 
 7195   ins_cost(VOLATILE_REF_COST);
 7196   format %{ "ldarh  $dst, $mem\t# short" %}
 7197 
 7198   ins_encode(aarch64_enc_ldarh(dst, mem));
 7199 
 7200   ins_pipe(pipe_serial);
 7201 %}
 7202 
 7203 // Load Short/Char (16 bit signed) into long
 7204 instruct loadS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 7205 %{
 7206   match(Set dst (ConvI2L (LoadS mem)));
 7207 
 7208   ins_cost(VOLATILE_REF_COST);
 7209   format %{ "ldarh  $dst, $mem\t# short" %}
 7210 
 7211   ins_encode(aarch64_enc_ldarsh(dst, mem));
 7212 
 7213   ins_pipe(pipe_serial);
 7214 %}
 7215 
 7216 // Load Integer (32 bit signed)
 7217 instruct loadI_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
 7218 %{
 7219   match(Set dst (LoadI mem));
 7220 
 7221   ins_cost(VOLATILE_REF_COST);
 7222   format %{ "ldarw  $dst, $mem\t# int" %}
 7223 
 7224   ins_encode(aarch64_enc_ldarw(dst, mem));
 7225 
 7226   ins_pipe(pipe_serial);
 7227 %}
 7228 
 7229 // Load Integer (32 bit unsigned) into long
 7230 instruct loadUI2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem, immL_32bits mask)
 7231 %{
 7232   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
 7233 
 7234   ins_cost(VOLATILE_REF_COST);
 7235   format %{ "ldarw  $dst, $mem\t# int" %}
 7236 
 7237   ins_encode(aarch64_enc_ldarw(dst, mem));
 7238 
 7239   ins_pipe(pipe_serial);
 7240 %}
 7241 
 7242 // Load Long (64 bit signed)
 7243 instruct loadL_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
 7244 %{
 7245   match(Set dst (LoadL mem));
 7246 
 7247   ins_cost(VOLATILE_REF_COST);
 7248   format %{ "ldar  $dst, $mem\t# int" %}
 7249 
 7250   ins_encode(aarch64_enc_ldar(dst, mem));
 7251 
 7252   ins_pipe(pipe_serial);
 7253 %}
 7254 
 7255 // Load Pointer
 7256 instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem)
 7257 %{
 7258   match(Set dst (LoadP mem));
 7259   predicate(n->as_Load()->barrier_data() == 0);
 7260 
 7261   ins_cost(VOLATILE_REF_COST);
 7262   format %{ "ldar  $dst, $mem\t# ptr" %}
 7263 
 7264   ins_encode(aarch64_enc_ldar(dst, mem));
 7265 
 7266   ins_pipe(pipe_serial);
 7267 %}
 7268 
 7269 // Load Compressed Pointer
 7270 instruct loadN_volatile(iRegNNoSp dst, /* sync_memory*/indirect mem)
 7271 %{
 7272   match(Set dst (LoadN mem));
 7273 
 7274   ins_cost(VOLATILE_REF_COST);
 7275   format %{ "ldarw  $dst, $mem\t# compressed ptr" %}
 7276 
 7277   ins_encode(aarch64_enc_ldarw(dst, mem));
 7278 
 7279   ins_pipe(pipe_serial);
 7280 %}
 7281 
 7282 // Load Float
 7283 instruct loadF_volatile(vRegF dst, /* sync_memory*/indirect mem)
 7284 %{
 7285   match(Set dst (LoadF mem));
 7286 
 7287   ins_cost(VOLATILE_REF_COST);
 7288   format %{ "ldars  $dst, $mem\t# float" %}
 7289 
 7290   ins_encode( aarch64_enc_fldars(dst, mem) );
 7291 
 7292   ins_pipe(pipe_serial);
 7293 %}
 7294 
 7295 // Load Double
 7296 instruct loadD_volatile(vRegD dst, /* sync_memory*/indirect mem)
 7297 %{
 7298   match(Set dst (LoadD mem));
 7299 
 7300   ins_cost(VOLATILE_REF_COST);
 7301   format %{ "ldard  $dst, $mem\t# double" %}
 7302 
 7303   ins_encode( aarch64_enc_fldard(dst, mem) );
 7304 
 7305   ins_pipe(pipe_serial);
 7306 %}
 7307 
 7308 // Store Byte
 7309 instruct storeB_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
 7310 %{
 7311   match(Set mem (StoreB mem src));
 7312 
 7313   ins_cost(VOLATILE_REF_COST);
 7314   format %{ "stlrb  $src, $mem\t# byte" %}
 7315 
 7316   ins_encode(aarch64_enc_stlrb(src, mem));
 7317 
 7318   ins_pipe(pipe_class_memory);
 7319 %}
 7320 
 7321 instruct storeimmB0_volatile(immI0 zero, /* sync_memory*/indirect mem)
 7322 %{
 7323   match(Set mem (StoreB mem zero));
 7324 
 7325   ins_cost(VOLATILE_REF_COST);
 7326   format %{ "stlrb  zr, $mem\t# byte" %}
 7327 
 7328   ins_encode(aarch64_enc_stlrb0(mem));
 7329 
 7330   ins_pipe(pipe_class_memory);
 7331 %}
 7332 
 7333 // Store Char/Short
 7334 instruct storeC_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
 7335 %{
 7336   match(Set mem (StoreC mem src));
 7337 
 7338   ins_cost(VOLATILE_REF_COST);
 7339   format %{ "stlrh  $src, $mem\t# short" %}
 7340 
 7341   ins_encode(aarch64_enc_stlrh(src, mem));
 7342 
 7343   ins_pipe(pipe_class_memory);
 7344 %}
 7345 
 7346 instruct storeimmC0_volatile(immI0 zero, /* sync_memory*/indirect mem)
 7347 %{
 7348   match(Set mem (StoreC mem zero));
 7349 
 7350   ins_cost(VOLATILE_REF_COST);
 7351   format %{ "stlrh  zr, $mem\t# short" %}
 7352 
 7353   ins_encode(aarch64_enc_stlrh0(mem));
 7354 
 7355   ins_pipe(pipe_class_memory);
 7356 %}
 7357 
 7358 // Store Integer
 7359 
 7360 instruct storeI_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
 7361 %{
 7362   match(Set mem(StoreI mem src));
 7363 
 7364   ins_cost(VOLATILE_REF_COST);
 7365   format %{ "stlrw  $src, $mem\t# int" %}
 7366 
 7367   ins_encode(aarch64_enc_stlrw(src, mem));
 7368 
 7369   ins_pipe(pipe_class_memory);
 7370 %}
 7371 
 7372 instruct storeimmI0_volatile(immI0 zero, /* sync_memory*/indirect mem)
 7373 %{
 7374   match(Set mem(StoreI mem zero));
 7375 
 7376   ins_cost(VOLATILE_REF_COST);
 7377   format %{ "stlrw  zr, $mem\t# int" %}
 7378 
 7379   ins_encode(aarch64_enc_stlrw0(mem));
 7380 
 7381   ins_pipe(pipe_class_memory);
 7382 %}
 7383 
 7384 // Store Long (64 bit signed)
 7385 instruct storeL_volatile(iRegL src, /* sync_memory*/indirect mem)
 7386 %{
 7387   match(Set mem (StoreL mem src));
 7388 
 7389   ins_cost(VOLATILE_REF_COST);
 7390   format %{ "stlr  $src, $mem\t# int" %}
 7391 
 7392   ins_encode(aarch64_enc_stlr(src, mem));
 7393 
 7394   ins_pipe(pipe_class_memory);
 7395 %}
 7396 
 7397 instruct storeimmL0_volatile(immL0 zero, /* sync_memory*/indirect mem)
 7398 %{
 7399   match(Set mem (StoreL mem zero));
 7400 
 7401   ins_cost(VOLATILE_REF_COST);
 7402   format %{ "stlr  zr, $mem\t# int" %}
 7403 
 7404   ins_encode(aarch64_enc_stlr0(mem));
 7405 
 7406   ins_pipe(pipe_class_memory);
 7407 %}
 7408 
 7409 // Store Pointer
 7410 instruct storeP_volatile(iRegP src, /* sync_memory*/indirect mem)
 7411 %{
 7412   match(Set mem (StoreP mem src));
 7413   predicate(n->as_Store()->barrier_data() == 0);
 7414 
 7415   ins_cost(VOLATILE_REF_COST);
 7416   format %{ "stlr  $src, $mem\t# ptr" %}
 7417 
 7418   ins_encode(aarch64_enc_stlr(src, mem));
 7419 
 7420   ins_pipe(pipe_class_memory);
 7421 %}
 7422 
 7423 instruct storeimmP0_volatile(immP0 zero, /* sync_memory*/indirect mem)
 7424 %{
 7425   match(Set mem (StoreP mem zero));
 7426   predicate(n->as_Store()->barrier_data() == 0);
 7427 
 7428   ins_cost(VOLATILE_REF_COST);
 7429   format %{ "stlr  zr, $mem\t# ptr" %}
 7430 
 7431   ins_encode(aarch64_enc_stlr0(mem));
 7432 
 7433   ins_pipe(pipe_class_memory);
 7434 %}
 7435 
 7436 // Store Compressed Pointer
 7437 instruct storeN_volatile(iRegN src, /* sync_memory*/indirect mem)
 7438 %{
 7439   match(Set mem (StoreN mem src));
 7440 
 7441   ins_cost(VOLATILE_REF_COST);
 7442   format %{ "stlrw  $src, $mem\t# compressed ptr" %}
 7443 
 7444   ins_encode(aarch64_enc_stlrw(src, mem));
 7445 
 7446   ins_pipe(pipe_class_memory);
 7447 %}
 7448 
 7449 instruct storeimmN0_volatile(immN0 zero, /* sync_memory*/indirect mem)
 7450 %{
 7451   match(Set mem (StoreN mem zero));
 7452 
 7453   ins_cost(VOLATILE_REF_COST);
 7454   format %{ "stlrw  zr, $mem\t# compressed ptr" %}
 7455 
 7456   ins_encode(aarch64_enc_stlrw0(mem));
 7457 
 7458   ins_pipe(pipe_class_memory);
 7459 %}
 7460 
 7461 // Store Float
 7462 instruct storeF_volatile(vRegF src, /* sync_memory*/indirect mem)
 7463 %{
 7464   match(Set mem (StoreF mem src));
 7465 
 7466   ins_cost(VOLATILE_REF_COST);
 7467   format %{ "stlrs  $src, $mem\t# float" %}
 7468 
 7469   ins_encode( aarch64_enc_fstlrs(src, mem) );
 7470 
 7471   ins_pipe(pipe_class_memory);
 7472 %}
 7473 
 7474 // TODO
 7475 // implement storeImmF0 and storeFImmPacked
 7476 
 7477 // Store Double
 7478 instruct storeD_volatile(vRegD src, /* sync_memory*/indirect mem)
 7479 %{
 7480   match(Set mem (StoreD mem src));
 7481 
 7482   ins_cost(VOLATILE_REF_COST);
 7483   format %{ "stlrd  $src, $mem\t# double" %}
 7484 
 7485   ins_encode( aarch64_enc_fstlrd(src, mem) );
 7486 
 7487   ins_pipe(pipe_class_memory);
 7488 %}
 7489 
 7490 //  ---------------- end of volatile loads and stores ----------------
 7491 
 7492 instruct cacheWB(indirect addr)
 7493 %{
 7494   predicate(VM_Version::supports_data_cache_line_flush());
 7495   match(CacheWB addr);
 7496 
 7497   ins_cost(100);
 7498   format %{"cache wb $addr" %}
 7499   ins_encode %{
 7500     assert($addr->index_position() < 0, "should be");
 7501     assert($addr$$disp == 0, "should be");
 7502     __ cache_wb(Address($addr$$base$$Register, 0));
 7503   %}
 7504   ins_pipe(pipe_slow); // XXX
 7505 %}
 7506 
 7507 instruct cacheWBPreSync()
 7508 %{
 7509   predicate(VM_Version::supports_data_cache_line_flush());
 7510   match(CacheWBPreSync);
 7511 
 7512   ins_cost(100);
 7513   format %{"cache wb presync" %}
 7514   ins_encode %{
 7515     __ cache_wbsync(true);
 7516   %}
 7517   ins_pipe(pipe_slow); // XXX
 7518 %}
 7519 
 7520 instruct cacheWBPostSync()
 7521 %{
 7522   predicate(VM_Version::supports_data_cache_line_flush());
 7523   match(CacheWBPostSync);
 7524 
 7525   ins_cost(100);
 7526   format %{"cache wb postsync" %}
 7527   ins_encode %{
 7528     __ cache_wbsync(false);
 7529   %}
 7530   ins_pipe(pipe_slow); // XXX
 7531 %}
 7532 
 7533 // ============================================================================
 7534 // BSWAP Instructions
 7535 
 7536 instruct bytes_reverse_int(iRegINoSp dst, iRegIorL2I src) %{
 7537   match(Set dst (ReverseBytesI src));
 7538 
 7539   ins_cost(INSN_COST);
 7540   format %{ "revw  $dst, $src" %}
 7541 
 7542   ins_encode %{
 7543     __ revw(as_Register($dst$$reg), as_Register($src$$reg));
 7544   %}
 7545 
 7546   ins_pipe(ialu_reg);
 7547 %}
 7548 
 7549 instruct bytes_reverse_long(iRegLNoSp dst, iRegL src) %{
 7550   match(Set dst (ReverseBytesL src));
 7551 
 7552   ins_cost(INSN_COST);
 7553   format %{ "rev  $dst, $src" %}
 7554 
 7555   ins_encode %{
 7556     __ rev(as_Register($dst$$reg), as_Register($src$$reg));
 7557   %}
 7558 
 7559   ins_pipe(ialu_reg);
 7560 %}
 7561 
 7562 instruct bytes_reverse_unsigned_short(iRegINoSp dst, iRegIorL2I src) %{
 7563   match(Set dst (ReverseBytesUS src));
 7564 
 7565   ins_cost(INSN_COST);
 7566   format %{ "rev16w  $dst, $src" %}
 7567 
 7568   ins_encode %{
 7569     __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
 7570   %}
 7571 
 7572   ins_pipe(ialu_reg);
 7573 %}
 7574 
 7575 instruct bytes_reverse_short(iRegINoSp dst, iRegIorL2I src) %{
 7576   match(Set dst (ReverseBytesS src));
 7577 
 7578   ins_cost(INSN_COST);
 7579   format %{ "rev16w  $dst, $src\n\t"
 7580             "sbfmw $dst, $dst, #0, #15" %}
 7581 
 7582   ins_encode %{
 7583     __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
 7584     __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 15U);
 7585   %}
 7586 
 7587   ins_pipe(ialu_reg);
 7588 %}
 7589 
 7590 // ============================================================================
 7591 // Zero Count Instructions
 7592 
 7593 instruct countLeadingZerosI(iRegINoSp dst, iRegIorL2I src) %{
 7594   match(Set dst (CountLeadingZerosI src));
 7595 
 7596   ins_cost(INSN_COST);
 7597   format %{ "clzw  $dst, $src" %}
 7598   ins_encode %{
 7599     __ clzw(as_Register($dst$$reg), as_Register($src$$reg));
 7600   %}
 7601 
 7602   ins_pipe(ialu_reg);
 7603 %}
 7604 
 7605 instruct countLeadingZerosL(iRegINoSp dst, iRegL src) %{
 7606   match(Set dst (CountLeadingZerosL src));
 7607 
 7608   ins_cost(INSN_COST);
 7609   format %{ "clz   $dst, $src" %}
 7610   ins_encode %{
 7611     __ clz(as_Register($dst$$reg), as_Register($src$$reg));
 7612   %}
 7613 
 7614   ins_pipe(ialu_reg);
 7615 %}
 7616 
 7617 instruct countTrailingZerosI(iRegINoSp dst, iRegIorL2I src) %{
 7618   match(Set dst (CountTrailingZerosI src));
 7619 
 7620   ins_cost(INSN_COST * 2);
 7621   format %{ "rbitw  $dst, $src\n\t"
 7622             "clzw   $dst, $dst" %}
 7623   ins_encode %{
 7624     __ rbitw(as_Register($dst$$reg), as_Register($src$$reg));
 7625     __ clzw(as_Register($dst$$reg), as_Register($dst$$reg));
 7626   %}
 7627 
 7628   ins_pipe(ialu_reg);
 7629 %}
 7630 
 7631 instruct countTrailingZerosL(iRegINoSp dst, iRegL src) %{
 7632   match(Set dst (CountTrailingZerosL src));
 7633 
 7634   ins_cost(INSN_COST * 2);
 7635   format %{ "rbit   $dst, $src\n\t"
 7636             "clz    $dst, $dst" %}
 7637   ins_encode %{
 7638     __ rbit(as_Register($dst$$reg), as_Register($src$$reg));
 7639     __ clz(as_Register($dst$$reg), as_Register($dst$$reg));
 7640   %}
 7641 
 7642   ins_pipe(ialu_reg);
 7643 %}
 7644 
 7645 //---------- Population Count Instructions -------------------------------------
 7646 //
 7647 
 7648 instruct popCountI(iRegINoSp dst, iRegIorL2I src, vRegF tmp) %{
 7649   match(Set dst (PopCountI src));
 7650   effect(TEMP tmp);
 7651   ins_cost(INSN_COST * 13);
 7652 
 7653   format %{ "movw   $src, $src\n\t"
 7654             "mov    $tmp, $src\t# vector (1D)\n\t"
 7655             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 7656             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 7657             "mov    $dst, $tmp\t# vector (1D)" %}
 7658   ins_encode %{
 7659     __ movw($src$$Register, $src$$Register); // ensure top 32 bits 0
 7660     __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
 7661     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 7662     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 7663     __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
 7664   %}
 7665 
 7666   ins_pipe(pipe_class_default);
 7667 %}
 7668 
 7669 instruct popCountI_mem(iRegINoSp dst, memory4 mem, vRegF tmp) %{
 7670   match(Set dst (PopCountI (LoadI mem)));
 7671   effect(TEMP tmp);
 7672   ins_cost(INSN_COST * 13);
 7673 
 7674   format %{ "ldrs   $tmp, $mem\n\t"
 7675             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 7676             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 7677             "mov    $dst, $tmp\t# vector (1D)" %}
 7678   ins_encode %{
 7679     FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
 7680     loadStore(masm, &MacroAssembler::ldrs, tmp_reg, $mem->opcode(),
 7681               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
 7682     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 7683     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 7684     __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
 7685   %}
 7686 
 7687   ins_pipe(pipe_class_default);
 7688 %}
 7689 
 7690 // Note: Long.bitCount(long) returns an int.
 7691 instruct popCountL(iRegINoSp dst, iRegL src, vRegD tmp) %{
 7692   match(Set dst (PopCountL src));
 7693   effect(TEMP tmp);
 7694   ins_cost(INSN_COST * 13);
 7695 
 7696   format %{ "mov    $tmp, $src\t# vector (1D)\n\t"
 7697             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 7698             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 7699             "mov    $dst, $tmp\t# vector (1D)" %}
 7700   ins_encode %{
 7701     __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
 7702     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 7703     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 7704     __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
 7705   %}
 7706 
 7707   ins_pipe(pipe_class_default);
 7708 %}
 7709 
 7710 instruct popCountL_mem(iRegINoSp dst, memory8 mem, vRegD tmp) %{
 7711   match(Set dst (PopCountL (LoadL mem)));
 7712   effect(TEMP tmp);
 7713   ins_cost(INSN_COST * 13);
 7714 
 7715   format %{ "ldrd   $tmp, $mem\n\t"
 7716             "cnt    $tmp, $tmp\t# vector (8B)\n\t"
 7717             "addv   $tmp, $tmp\t# vector (8B)\n\t"
 7718             "mov    $dst, $tmp\t# vector (1D)" %}
 7719   ins_encode %{
 7720     FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
 7721     loadStore(masm, &MacroAssembler::ldrd, tmp_reg, $mem->opcode(),
 7722               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
 7723     __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 7724     __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
 7725     __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
 7726   %}
 7727 
 7728   ins_pipe(pipe_class_default);
 7729 %}
 7730 
 7731 // ============================================================================
 7732 // VerifyVectorAlignment Instruction
 7733 
 7734 instruct verify_vector_alignment(iRegP addr, immL_positive_bitmaskI mask, rFlagsReg cr) %{
 7735   match(Set addr (VerifyVectorAlignment addr mask));
 7736   effect(KILL cr);
 7737   format %{ "verify_vector_alignment $addr $mask \t! verify alignment" %}
 7738   ins_encode %{
 7739     Label Lskip;
 7740     // check if masked bits of addr are zero
 7741     __ tst($addr$$Register, $mask$$constant);
 7742     __ br(Assembler::EQ, Lskip);
 7743     __ stop("verify_vector_alignment found a misaligned vector memory access");
 7744     __ bind(Lskip);
 7745   %}
 7746   ins_pipe(pipe_slow);
 7747 %}
 7748 
 7749 // ============================================================================
 7750 // MemBar Instruction
 7751 
 7752 instruct load_fence() %{
 7753   match(LoadFence);
 7754   ins_cost(VOLATILE_REF_COST);
 7755 
 7756   format %{ "load_fence" %}
 7757 
 7758   ins_encode %{
 7759     __ membar(Assembler::LoadLoad|Assembler::LoadStore);
 7760   %}
 7761   ins_pipe(pipe_serial);
 7762 %}
 7763 
 7764 instruct unnecessary_membar_acquire() %{
 7765   predicate(unnecessary_acquire(n));
 7766   match(MemBarAcquire);
 7767   ins_cost(0);
 7768 
 7769   format %{ "membar_acquire (elided)" %}
 7770 
 7771   ins_encode %{
 7772     __ block_comment("membar_acquire (elided)");
 7773   %}
 7774 
 7775   ins_pipe(pipe_class_empty);
 7776 %}
 7777 
 7778 instruct membar_acquire() %{
 7779   match(MemBarAcquire);
 7780   ins_cost(VOLATILE_REF_COST);
 7781 
 7782   format %{ "membar_acquire\n\t"
 7783             "dmb ishld" %}
 7784 
 7785   ins_encode %{
 7786     __ block_comment("membar_acquire");
 7787     __ membar(Assembler::LoadLoad|Assembler::LoadStore);
 7788   %}
 7789 
 7790   ins_pipe(pipe_serial);
 7791 %}
 7792 
 7793 
 7794 instruct membar_acquire_lock() %{
 7795   match(MemBarAcquireLock);
 7796   ins_cost(VOLATILE_REF_COST);
 7797 
 7798   format %{ "membar_acquire_lock (elided)" %}
 7799 
 7800   ins_encode %{
 7801     __ block_comment("membar_acquire_lock (elided)");
 7802   %}
 7803 
 7804   ins_pipe(pipe_serial);
 7805 %}
 7806 
 7807 instruct store_fence() %{
 7808   match(StoreFence);
 7809   ins_cost(VOLATILE_REF_COST);
 7810 
 7811   format %{ "store_fence" %}
 7812 
 7813   ins_encode %{
 7814     __ membar(Assembler::LoadStore|Assembler::StoreStore);
 7815   %}
 7816   ins_pipe(pipe_serial);
 7817 %}
 7818 
 7819 instruct unnecessary_membar_release() %{
 7820   predicate(unnecessary_release(n));
 7821   match(MemBarRelease);
 7822   ins_cost(0);
 7823 
 7824   format %{ "membar_release (elided)" %}
 7825 
 7826   ins_encode %{
 7827     __ block_comment("membar_release (elided)");
 7828   %}
 7829   ins_pipe(pipe_serial);
 7830 %}
 7831 
 7832 instruct membar_release() %{
 7833   match(MemBarRelease);
 7834   ins_cost(VOLATILE_REF_COST);
 7835 
 7836   format %{ "membar_release\n\t"
 7837             "dmb ishst\n\tdmb ishld" %}
 7838 
 7839   ins_encode %{
 7840     __ block_comment("membar_release");
 7841     // These will be merged if AlwaysMergeDMB is enabled.
 7842     __ membar(Assembler::StoreStore);
 7843     __ membar(Assembler::LoadStore);
 7844   %}
 7845   ins_pipe(pipe_serial);
 7846 %}
 7847 
 7848 instruct membar_storestore() %{
 7849   match(MemBarStoreStore);
 7850   match(StoreStoreFence);
 7851   ins_cost(VOLATILE_REF_COST);
 7852 
 7853   format %{ "MEMBAR-store-store" %}
 7854 
 7855   ins_encode %{
 7856     __ membar(Assembler::StoreStore);
 7857   %}
 7858   ins_pipe(pipe_serial);
 7859 %}
 7860 
 7861 instruct membar_release_lock() %{
 7862   match(MemBarReleaseLock);
 7863   ins_cost(VOLATILE_REF_COST);
 7864 
 7865   format %{ "membar_release_lock (elided)" %}
 7866 
 7867   ins_encode %{
 7868     __ block_comment("membar_release_lock (elided)");
 7869   %}
 7870 
 7871   ins_pipe(pipe_serial);
 7872 %}
 7873 
 7874 instruct unnecessary_membar_volatile() %{
 7875   predicate(unnecessary_volatile(n));
 7876   match(MemBarVolatile);
 7877   ins_cost(0);
 7878 
 7879   format %{ "membar_volatile (elided)" %}
 7880 
 7881   ins_encode %{
 7882     __ block_comment("membar_volatile (elided)");
 7883   %}
 7884 
 7885   ins_pipe(pipe_serial);
 7886 %}
 7887 
 7888 instruct membar_volatile() %{
 7889   match(MemBarVolatile);
 7890   ins_cost(VOLATILE_REF_COST*100);
 7891 
 7892   format %{ "membar_volatile\n\t"
 7893              "dmb ish"%}
 7894 
 7895   ins_encode %{
 7896     __ block_comment("membar_volatile");
 7897     __ membar(Assembler::StoreLoad);
 7898   %}
 7899 
 7900   ins_pipe(pipe_serial);
 7901 %}
 7902 
 7903 // ============================================================================
 7904 // Cast/Convert Instructions
 7905 
 7906 instruct castX2P(iRegPNoSp dst, iRegL src) %{
 7907   match(Set dst (CastX2P src));
 7908 
 7909   ins_cost(INSN_COST);
 7910   format %{ "mov $dst, $src\t# long -> ptr" %}
 7911 
 7912   ins_encode %{
 7913     if ($dst$$reg != $src$$reg) {
 7914       __ mov(as_Register($dst$$reg), as_Register($src$$reg));
 7915     }
 7916   %}
 7917 
 7918   ins_pipe(ialu_reg);
 7919 %}
 7920 
 7921 instruct castP2X(iRegLNoSp dst, iRegP src) %{
 7922   match(Set dst (CastP2X src));
 7923 
 7924   ins_cost(INSN_COST);
 7925   format %{ "mov $dst, $src\t# ptr -> long" %}
 7926 
 7927   ins_encode %{
 7928     if ($dst$$reg != $src$$reg) {
 7929       __ mov(as_Register($dst$$reg), as_Register($src$$reg));
 7930     }
 7931   %}
 7932 
 7933   ins_pipe(ialu_reg);
 7934 %}
 7935 
 7936 // Convert oop into int for vectors alignment masking
 7937 instruct convP2I(iRegINoSp dst, iRegP src) %{
 7938   match(Set dst (ConvL2I (CastP2X src)));
 7939 
 7940   ins_cost(INSN_COST);
 7941   format %{ "movw $dst, $src\t# ptr -> int" %}
 7942   ins_encode %{
 7943     __ movw($dst$$Register, $src$$Register);
 7944   %}
 7945 
 7946   ins_pipe(ialu_reg);
 7947 %}
 7948 
 7949 // Convert compressed oop into int for vectors alignment masking
 7950 // in case of 32bit oops (heap < 4Gb).
 7951 instruct convN2I(iRegINoSp dst, iRegN src)
 7952 %{
 7953   predicate(CompressedOops::shift() == 0);
 7954   match(Set dst (ConvL2I (CastP2X (DecodeN src))));
 7955 
 7956   ins_cost(INSN_COST);
 7957   format %{ "mov dst, $src\t# compressed ptr -> int" %}
 7958   ins_encode %{
 7959     __ movw($dst$$Register, $src$$Register);
 7960   %}
 7961 
 7962   ins_pipe(ialu_reg);
 7963 %}
 7964 
 7965 
 7966 // Convert oop pointer into compressed form
 7967 instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
 7968   predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
 7969   match(Set dst (EncodeP src));
 7970   effect(KILL cr);
 7971   ins_cost(INSN_COST * 3);
 7972   format %{ "encode_heap_oop $dst, $src" %}
 7973   ins_encode %{
 7974     Register s = $src$$Register;
 7975     Register d = $dst$$Register;
 7976     __ encode_heap_oop(d, s);
 7977   %}
 7978   ins_pipe(ialu_reg);
 7979 %}
 7980 
 7981 instruct encodeHeapOop_not_null(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
 7982   predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
 7983   match(Set dst (EncodeP src));
 7984   ins_cost(INSN_COST * 3);
 7985   format %{ "encode_heap_oop_not_null $dst, $src" %}
 7986   ins_encode %{
 7987     __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
 7988   %}
 7989   ins_pipe(ialu_reg);
 7990 %}
 7991 
 7992 instruct decodeHeapOop(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
 7993   predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
 7994             n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
 7995   match(Set dst (DecodeN src));
 7996   ins_cost(INSN_COST * 3);
 7997   format %{ "decode_heap_oop $dst, $src" %}
 7998   ins_encode %{
 7999     Register s = $src$$Register;
 8000     Register d = $dst$$Register;
 8001     __ decode_heap_oop(d, s);
 8002   %}
 8003   ins_pipe(ialu_reg);
 8004 %}
 8005 
 8006 instruct decodeHeapOop_not_null(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
 8007   predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
 8008             n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
 8009   match(Set dst (DecodeN src));
 8010   ins_cost(INSN_COST * 3);
 8011   format %{ "decode_heap_oop_not_null $dst, $src" %}
 8012   ins_encode %{
 8013     Register s = $src$$Register;
 8014     Register d = $dst$$Register;
 8015     __ decode_heap_oop_not_null(d, s);
 8016   %}
 8017   ins_pipe(ialu_reg);
 8018 %}
 8019 
 8020 // n.b. AArch64 implementations of encode_klass_not_null and
 8021 // decode_klass_not_null do not modify the flags register so, unlike
 8022 // Intel, we don't kill CR as a side effect here
 8023 
 8024 instruct encodeKlass_not_null(iRegNNoSp dst, iRegP src) %{
 8025   match(Set dst (EncodePKlass src));
 8026 
 8027   ins_cost(INSN_COST * 3);
 8028   format %{ "encode_klass_not_null $dst,$src" %}
 8029 
 8030   ins_encode %{
 8031     Register src_reg = as_Register($src$$reg);
 8032     Register dst_reg = as_Register($dst$$reg);
 8033     __ encode_klass_not_null(dst_reg, src_reg);
 8034   %}
 8035 
 8036    ins_pipe(ialu_reg);
 8037 %}
 8038 
 8039 instruct decodeKlass_not_null(iRegPNoSp dst, iRegN src) %{
 8040   match(Set dst (DecodeNKlass src));
 8041 
 8042   ins_cost(INSN_COST * 3);
 8043   format %{ "decode_klass_not_null $dst,$src" %}
 8044 
 8045   ins_encode %{
 8046     Register src_reg = as_Register($src$$reg);
 8047     Register dst_reg = as_Register($dst$$reg);
 8048     if (dst_reg != src_reg) {
 8049       __ decode_klass_not_null(dst_reg, src_reg);
 8050     } else {
 8051       __ decode_klass_not_null(dst_reg);
 8052     }
 8053   %}
 8054 
 8055    ins_pipe(ialu_reg);
 8056 %}
 8057 
 8058 instruct checkCastPP(iRegPNoSp dst)
 8059 %{
 8060   match(Set dst (CheckCastPP dst));
 8061 
 8062   size(0);
 8063   format %{ "# checkcastPP of $dst" %}
 8064   ins_encode(/* empty encoding */);
 8065   ins_pipe(pipe_class_empty);
 8066 %}
 8067 
 8068 instruct castPP(iRegPNoSp dst)
 8069 %{
 8070   match(Set dst (CastPP dst));
 8071 
 8072   size(0);
 8073   format %{ "# castPP of $dst" %}
 8074   ins_encode(/* empty encoding */);
 8075   ins_pipe(pipe_class_empty);
 8076 %}
 8077 
 8078 instruct castII(iRegI dst)
 8079 %{
 8080   match(Set dst (CastII dst));
 8081 
 8082   size(0);
 8083   format %{ "# castII of $dst" %}
 8084   ins_encode(/* empty encoding */);
 8085   ins_cost(0);
 8086   ins_pipe(pipe_class_empty);
 8087 %}
 8088 
 8089 instruct castLL(iRegL dst)
 8090 %{
 8091   match(Set dst (CastLL dst));
 8092 
 8093   size(0);
 8094   format %{ "# castLL of $dst" %}
 8095   ins_encode(/* empty encoding */);
 8096   ins_cost(0);
 8097   ins_pipe(pipe_class_empty);
 8098 %}
 8099 
 8100 instruct castFF(vRegF dst)
 8101 %{
 8102   match(Set dst (CastFF dst));
 8103 
 8104   size(0);
 8105   format %{ "# castFF of $dst" %}
 8106   ins_encode(/* empty encoding */);
 8107   ins_cost(0);
 8108   ins_pipe(pipe_class_empty);
 8109 %}
 8110 
 8111 instruct castDD(vRegD dst)
 8112 %{
 8113   match(Set dst (CastDD dst));
 8114 
 8115   size(0);
 8116   format %{ "# castDD of $dst" %}
 8117   ins_encode(/* empty encoding */);
 8118   ins_cost(0);
 8119   ins_pipe(pipe_class_empty);
 8120 %}
 8121 
 8122 instruct castVV(vReg dst)
 8123 %{
 8124   match(Set dst (CastVV dst));
 8125 
 8126   size(0);
 8127   format %{ "# castVV of $dst" %}
 8128   ins_encode(/* empty encoding */);
 8129   ins_cost(0);
 8130   ins_pipe(pipe_class_empty);
 8131 %}
 8132 
 8133 instruct castVVMask(pRegGov dst)
 8134 %{
 8135   match(Set dst (CastVV dst));
 8136 
 8137   size(0);
 8138   format %{ "# castVV of $dst" %}
 8139   ins_encode(/* empty encoding */);
 8140   ins_cost(0);
 8141   ins_pipe(pipe_class_empty);
 8142 %}
 8143 
 8144 // ============================================================================
 8145 // Atomic operation instructions
 8146 //
 8147 
 8148 // standard CompareAndSwapX when we are using barriers
 8149 // these have higher priority than the rules selected by a predicate
 8150 
 8151 // XXX No flag versions for CompareAndSwap{I,L,P,N} because matcher
 8152 // can't match them
 8153 
 8154 instruct compareAndSwapB(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 8155 
 8156   match(Set res (CompareAndSwapB mem (Binary oldval newval)));
 8157   ins_cost(2 * VOLATILE_REF_COST);
 8158 
 8159   effect(KILL cr);
 8160 
 8161   format %{
 8162     "cmpxchgb $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 8163     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8164   %}
 8165 
 8166   ins_encode(aarch64_enc_cmpxchgb(mem, oldval, newval),
 8167             aarch64_enc_cset_eq(res));
 8168 
 8169   ins_pipe(pipe_slow);
 8170 %}
 8171 
 8172 instruct compareAndSwapS(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 8173 
 8174   match(Set res (CompareAndSwapS mem (Binary oldval newval)));
 8175   ins_cost(2 * VOLATILE_REF_COST);
 8176 
 8177   effect(KILL cr);
 8178 
 8179   format %{
 8180     "cmpxchgs $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 8181     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8182   %}
 8183 
 8184   ins_encode(aarch64_enc_cmpxchgs(mem, oldval, newval),
 8185             aarch64_enc_cset_eq(res));
 8186 
 8187   ins_pipe(pipe_slow);
 8188 %}
 8189 
 8190 instruct compareAndSwapI(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 8191 
 8192   match(Set res (CompareAndSwapI mem (Binary oldval newval)));
 8193   ins_cost(2 * VOLATILE_REF_COST);
 8194 
 8195   effect(KILL cr);
 8196 
 8197  format %{
 8198     "cmpxchgw $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 8199     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8200  %}
 8201 
 8202  ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
 8203             aarch64_enc_cset_eq(res));
 8204 
 8205   ins_pipe(pipe_slow);
 8206 %}
 8207 
 8208 instruct compareAndSwapL(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
 8209 
 8210   match(Set res (CompareAndSwapL mem (Binary oldval newval)));
 8211   ins_cost(2 * VOLATILE_REF_COST);
 8212 
 8213   effect(KILL cr);
 8214 
 8215  format %{
 8216     "cmpxchg $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
 8217     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8218  %}
 8219 
 8220  ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
 8221             aarch64_enc_cset_eq(res));
 8222 
 8223   ins_pipe(pipe_slow);
 8224 %}
 8225 
 8226 instruct compareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 8227 
 8228   match(Set res (CompareAndSwapP mem (Binary oldval newval)));
 8229   predicate(n->as_LoadStore()->barrier_data() == 0);
 8230   ins_cost(2 * VOLATILE_REF_COST);
 8231 
 8232   effect(KILL cr);
 8233 
 8234  format %{
 8235     "cmpxchg $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
 8236     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8237  %}
 8238 
 8239  ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
 8240             aarch64_enc_cset_eq(res));
 8241 
 8242   ins_pipe(pipe_slow);
 8243 %}
 8244 
 8245 instruct compareAndSwapN(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
 8246 
 8247   match(Set res (CompareAndSwapN mem (Binary oldval newval)));
 8248   ins_cost(2 * VOLATILE_REF_COST);
 8249 
 8250   effect(KILL cr);
 8251 
 8252  format %{
 8253     "cmpxchgw $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
 8254     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8255  %}
 8256 
 8257  ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
 8258             aarch64_enc_cset_eq(res));
 8259 
 8260   ins_pipe(pipe_slow);
 8261 %}
 8262 
 8263 // alternative CompareAndSwapX when we are eliding barriers
 8264 
 8265 instruct compareAndSwapBAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 8266 
 8267   predicate(needs_acquiring_load_exclusive(n));
 8268   match(Set res (CompareAndSwapB mem (Binary oldval newval)));
 8269   ins_cost(VOLATILE_REF_COST);
 8270 
 8271   effect(KILL cr);
 8272 
 8273   format %{
 8274     "cmpxchgb_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 8275     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8276   %}
 8277 
 8278   ins_encode(aarch64_enc_cmpxchgb_acq(mem, oldval, newval),
 8279             aarch64_enc_cset_eq(res));
 8280 
 8281   ins_pipe(pipe_slow);
 8282 %}
 8283 
 8284 instruct compareAndSwapSAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 8285 
 8286   predicate(needs_acquiring_load_exclusive(n));
 8287   match(Set res (CompareAndSwapS mem (Binary oldval newval)));
 8288   ins_cost(VOLATILE_REF_COST);
 8289 
 8290   effect(KILL cr);
 8291 
 8292   format %{
 8293     "cmpxchgs_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 8294     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8295   %}
 8296 
 8297   ins_encode(aarch64_enc_cmpxchgs_acq(mem, oldval, newval),
 8298             aarch64_enc_cset_eq(res));
 8299 
 8300   ins_pipe(pipe_slow);
 8301 %}
 8302 
 8303 instruct compareAndSwapIAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
 8304 
 8305   predicate(needs_acquiring_load_exclusive(n));
 8306   match(Set res (CompareAndSwapI mem (Binary oldval newval)));
 8307   ins_cost(VOLATILE_REF_COST);
 8308 
 8309   effect(KILL cr);
 8310 
 8311  format %{
 8312     "cmpxchgw_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
 8313     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8314  %}
 8315 
 8316  ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
 8317             aarch64_enc_cset_eq(res));
 8318 
 8319   ins_pipe(pipe_slow);
 8320 %}
 8321 
 8322 instruct compareAndSwapLAcq(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
 8323 
 8324   predicate(needs_acquiring_load_exclusive(n));
 8325   match(Set res (CompareAndSwapL mem (Binary oldval newval)));
 8326   ins_cost(VOLATILE_REF_COST);
 8327 
 8328   effect(KILL cr);
 8329 
 8330  format %{
 8331     "cmpxchg_acq $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
 8332     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8333  %}
 8334 
 8335  ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
 8336             aarch64_enc_cset_eq(res));
 8337 
 8338   ins_pipe(pipe_slow);
 8339 %}
 8340 
 8341 instruct compareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 8342 
 8343   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 8344   match(Set res (CompareAndSwapP mem (Binary oldval newval)));
 8345   ins_cost(VOLATILE_REF_COST);
 8346 
 8347   effect(KILL cr);
 8348 
 8349  format %{
 8350     "cmpxchg_acq $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
 8351     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8352  %}
 8353 
 8354  ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
 8355             aarch64_enc_cset_eq(res));
 8356 
 8357   ins_pipe(pipe_slow);
 8358 %}
 8359 
 8360 instruct compareAndSwapNAcq(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
 8361 
 8362   predicate(needs_acquiring_load_exclusive(n));
 8363   match(Set res (CompareAndSwapN mem (Binary oldval newval)));
 8364   ins_cost(VOLATILE_REF_COST);
 8365 
 8366   effect(KILL cr);
 8367 
 8368  format %{
 8369     "cmpxchgw_acq $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
 8370     "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8371  %}
 8372 
 8373  ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
 8374             aarch64_enc_cset_eq(res));
 8375 
 8376   ins_pipe(pipe_slow);
 8377 %}
 8378 
 8379 
 8380 // ---------------------------------------------------------------------
 8381 
 8382 // BEGIN This section of the file is automatically generated. Do not edit --------------
 8383 
 8384 // Sundry CAS operations.  Note that release is always true,
 8385 // regardless of the memory ordering of the CAS.  This is because we
 8386 // need the volatile case to be sequentially consistent but there is
 8387 // no trailing StoreLoad barrier emitted by C2.  Unfortunately we
 8388 // can't check the type of memory ordering here, so we always emit a
 8389 // STLXR.
 8390 
 8391 // This section is generated from cas.m4
 8392 
 8393 
 8394 // This pattern is generated automatically from cas.m4.
 8395 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8396 instruct compareAndExchangeB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 8397   match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
 8398   ins_cost(2 * VOLATILE_REF_COST);
 8399   effect(TEMP_DEF res, KILL cr);
 8400   format %{
 8401     "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 8402   %}
 8403   ins_encode %{
 8404     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8405                Assembler::byte, /*acquire*/ false, /*release*/ true,
 8406                /*weak*/ false, $res$$Register);
 8407     __ sxtbw($res$$Register, $res$$Register);
 8408   %}
 8409   ins_pipe(pipe_slow);
 8410 %}
 8411 
 8412 // This pattern is generated automatically from cas.m4.
 8413 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8414 instruct compareAndExchangeS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 8415   match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
 8416   ins_cost(2 * VOLATILE_REF_COST);
 8417   effect(TEMP_DEF res, KILL cr);
 8418   format %{
 8419     "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 8420   %}
 8421   ins_encode %{
 8422     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8423                Assembler::halfword, /*acquire*/ false, /*release*/ true,
 8424                /*weak*/ false, $res$$Register);
 8425     __ sxthw($res$$Register, $res$$Register);
 8426   %}
 8427   ins_pipe(pipe_slow);
 8428 %}
 8429 
 8430 // This pattern is generated automatically from cas.m4.
 8431 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8432 instruct compareAndExchangeI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 8433   match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
 8434   ins_cost(2 * VOLATILE_REF_COST);
 8435   effect(TEMP_DEF res, KILL cr);
 8436   format %{
 8437     "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 8438   %}
 8439   ins_encode %{
 8440     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8441                Assembler::word, /*acquire*/ false, /*release*/ true,
 8442                /*weak*/ false, $res$$Register);
 8443   %}
 8444   ins_pipe(pipe_slow);
 8445 %}
 8446 
 8447 // This pattern is generated automatically from cas.m4.
 8448 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8449 instruct compareAndExchangeL(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 8450   match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
 8451   ins_cost(2 * VOLATILE_REF_COST);
 8452   effect(TEMP_DEF res, KILL cr);
 8453   format %{
 8454     "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 8455   %}
 8456   ins_encode %{
 8457     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8458                Assembler::xword, /*acquire*/ false, /*release*/ true,
 8459                /*weak*/ false, $res$$Register);
 8460   %}
 8461   ins_pipe(pipe_slow);
 8462 %}
 8463 
 8464 // This pattern is generated automatically from cas.m4.
 8465 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8466 instruct compareAndExchangeN(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 8467   match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
 8468   ins_cost(2 * VOLATILE_REF_COST);
 8469   effect(TEMP_DEF res, KILL cr);
 8470   format %{
 8471     "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 8472   %}
 8473   ins_encode %{
 8474     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8475                Assembler::word, /*acquire*/ false, /*release*/ true,
 8476                /*weak*/ false, $res$$Register);
 8477   %}
 8478   ins_pipe(pipe_slow);
 8479 %}
 8480 
 8481 // This pattern is generated automatically from cas.m4.
 8482 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8483 instruct compareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 8484   predicate(n->as_LoadStore()->barrier_data() == 0);
 8485   match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
 8486   ins_cost(2 * VOLATILE_REF_COST);
 8487   effect(TEMP_DEF res, KILL cr);
 8488   format %{
 8489     "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 8490   %}
 8491   ins_encode %{
 8492     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8493                Assembler::xword, /*acquire*/ false, /*release*/ true,
 8494                /*weak*/ false, $res$$Register);
 8495   %}
 8496   ins_pipe(pipe_slow);
 8497 %}
 8498 
 8499 // This pattern is generated automatically from cas.m4.
 8500 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8501 instruct compareAndExchangeBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 8502   predicate(needs_acquiring_load_exclusive(n));
 8503   match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
 8504   ins_cost(VOLATILE_REF_COST);
 8505   effect(TEMP_DEF res, KILL cr);
 8506   format %{
 8507     "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 8508   %}
 8509   ins_encode %{
 8510     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8511                Assembler::byte, /*acquire*/ true, /*release*/ true,
 8512                /*weak*/ false, $res$$Register);
 8513     __ sxtbw($res$$Register, $res$$Register);
 8514   %}
 8515   ins_pipe(pipe_slow);
 8516 %}
 8517 
 8518 // This pattern is generated automatically from cas.m4.
 8519 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8520 instruct compareAndExchangeSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 8521   predicate(needs_acquiring_load_exclusive(n));
 8522   match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
 8523   ins_cost(VOLATILE_REF_COST);
 8524   effect(TEMP_DEF res, KILL cr);
 8525   format %{
 8526     "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 8527   %}
 8528   ins_encode %{
 8529     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8530                Assembler::halfword, /*acquire*/ true, /*release*/ true,
 8531                /*weak*/ false, $res$$Register);
 8532     __ sxthw($res$$Register, $res$$Register);
 8533   %}
 8534   ins_pipe(pipe_slow);
 8535 %}
 8536 
 8537 // This pattern is generated automatically from cas.m4.
 8538 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8539 instruct compareAndExchangeIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 8540   predicate(needs_acquiring_load_exclusive(n));
 8541   match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
 8542   ins_cost(VOLATILE_REF_COST);
 8543   effect(TEMP_DEF res, KILL cr);
 8544   format %{
 8545     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 8546   %}
 8547   ins_encode %{
 8548     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8549                Assembler::word, /*acquire*/ true, /*release*/ true,
 8550                /*weak*/ false, $res$$Register);
 8551   %}
 8552   ins_pipe(pipe_slow);
 8553 %}
 8554 
 8555 // This pattern is generated automatically from cas.m4.
 8556 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8557 instruct compareAndExchangeLAcq(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 8558   predicate(needs_acquiring_load_exclusive(n));
 8559   match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
 8560   ins_cost(VOLATILE_REF_COST);
 8561   effect(TEMP_DEF res, KILL cr);
 8562   format %{
 8563     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 8564   %}
 8565   ins_encode %{
 8566     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8567                Assembler::xword, /*acquire*/ true, /*release*/ true,
 8568                /*weak*/ false, $res$$Register);
 8569   %}
 8570   ins_pipe(pipe_slow);
 8571 %}
 8572 
 8573 // This pattern is generated automatically from cas.m4.
 8574 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8575 instruct compareAndExchangeNAcq(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 8576   predicate(needs_acquiring_load_exclusive(n));
 8577   match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
 8578   ins_cost(VOLATILE_REF_COST);
 8579   effect(TEMP_DEF res, KILL cr);
 8580   format %{
 8581     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 8582   %}
 8583   ins_encode %{
 8584     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8585                Assembler::word, /*acquire*/ true, /*release*/ true,
 8586                /*weak*/ false, $res$$Register);
 8587   %}
 8588   ins_pipe(pipe_slow);
 8589 %}
 8590 
 8591 // This pattern is generated automatically from cas.m4.
 8592 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8593 instruct compareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 8594   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 8595   match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
 8596   ins_cost(VOLATILE_REF_COST);
 8597   effect(TEMP_DEF res, KILL cr);
 8598   format %{
 8599     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 8600   %}
 8601   ins_encode %{
 8602     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8603                Assembler::xword, /*acquire*/ true, /*release*/ true,
 8604                /*weak*/ false, $res$$Register);
 8605   %}
 8606   ins_pipe(pipe_slow);
 8607 %}
 8608 
 8609 // This pattern is generated automatically from cas.m4.
 8610 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8611 instruct weakCompareAndSwapB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 8612   match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
 8613   ins_cost(2 * VOLATILE_REF_COST);
 8614   effect(KILL cr);
 8615   format %{
 8616     "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 8617     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8618   %}
 8619   ins_encode %{
 8620     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8621                Assembler::byte, /*acquire*/ false, /*release*/ true,
 8622                /*weak*/ true, noreg);
 8623     __ csetw($res$$Register, Assembler::EQ);
 8624   %}
 8625   ins_pipe(pipe_slow);
 8626 %}
 8627 
 8628 // This pattern is generated automatically from cas.m4.
 8629 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8630 instruct weakCompareAndSwapS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 8631   match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
 8632   ins_cost(2 * VOLATILE_REF_COST);
 8633   effect(KILL cr);
 8634   format %{
 8635     "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 8636     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8637   %}
 8638   ins_encode %{
 8639     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8640                Assembler::halfword, /*acquire*/ false, /*release*/ true,
 8641                /*weak*/ true, noreg);
 8642     __ csetw($res$$Register, Assembler::EQ);
 8643   %}
 8644   ins_pipe(pipe_slow);
 8645 %}
 8646 
 8647 // This pattern is generated automatically from cas.m4.
 8648 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8649 instruct weakCompareAndSwapI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 8650   match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
 8651   ins_cost(2 * VOLATILE_REF_COST);
 8652   effect(KILL cr);
 8653   format %{
 8654     "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 8655     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8656   %}
 8657   ins_encode %{
 8658     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8659                Assembler::word, /*acquire*/ false, /*release*/ true,
 8660                /*weak*/ true, noreg);
 8661     __ csetw($res$$Register, Assembler::EQ);
 8662   %}
 8663   ins_pipe(pipe_slow);
 8664 %}
 8665 
 8666 // This pattern is generated automatically from cas.m4.
 8667 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8668 instruct weakCompareAndSwapL(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 8669   match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
 8670   ins_cost(2 * VOLATILE_REF_COST);
 8671   effect(KILL cr);
 8672   format %{
 8673     "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 8674     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8675   %}
 8676   ins_encode %{
 8677     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8678                Assembler::xword, /*acquire*/ false, /*release*/ true,
 8679                /*weak*/ true, noreg);
 8680     __ csetw($res$$Register, Assembler::EQ);
 8681   %}
 8682   ins_pipe(pipe_slow);
 8683 %}
 8684 
 8685 // This pattern is generated automatically from cas.m4.
 8686 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8687 instruct weakCompareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 8688   match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
 8689   ins_cost(2 * VOLATILE_REF_COST);
 8690   effect(KILL cr);
 8691   format %{
 8692     "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 8693     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8694   %}
 8695   ins_encode %{
 8696     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8697                Assembler::word, /*acquire*/ false, /*release*/ true,
 8698                /*weak*/ true, noreg);
 8699     __ csetw($res$$Register, Assembler::EQ);
 8700   %}
 8701   ins_pipe(pipe_slow);
 8702 %}
 8703 
 8704 // This pattern is generated automatically from cas.m4.
 8705 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8706 instruct weakCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 8707   predicate(n->as_LoadStore()->barrier_data() == 0);
 8708   match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
 8709   ins_cost(2 * VOLATILE_REF_COST);
 8710   effect(KILL cr);
 8711   format %{
 8712     "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 8713     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8714   %}
 8715   ins_encode %{
 8716     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8717                Assembler::xword, /*acquire*/ false, /*release*/ true,
 8718                /*weak*/ true, noreg);
 8719     __ csetw($res$$Register, Assembler::EQ);
 8720   %}
 8721   ins_pipe(pipe_slow);
 8722 %}
 8723 
 8724 // This pattern is generated automatically from cas.m4.
 8725 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8726 instruct weakCompareAndSwapBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 8727   predicate(needs_acquiring_load_exclusive(n));
 8728   match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
 8729   ins_cost(VOLATILE_REF_COST);
 8730   effect(KILL cr);
 8731   format %{
 8732     "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
 8733     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8734   %}
 8735   ins_encode %{
 8736     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8737                Assembler::byte, /*acquire*/ true, /*release*/ true,
 8738                /*weak*/ true, noreg);
 8739     __ csetw($res$$Register, Assembler::EQ);
 8740   %}
 8741   ins_pipe(pipe_slow);
 8742 %}
 8743 
 8744 // This pattern is generated automatically from cas.m4.
 8745 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8746 instruct weakCompareAndSwapSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 8747   predicate(needs_acquiring_load_exclusive(n));
 8748   match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
 8749   ins_cost(VOLATILE_REF_COST);
 8750   effect(KILL cr);
 8751   format %{
 8752     "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
 8753     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8754   %}
 8755   ins_encode %{
 8756     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8757                Assembler::halfword, /*acquire*/ true, /*release*/ true,
 8758                /*weak*/ true, noreg);
 8759     __ csetw($res$$Register, Assembler::EQ);
 8760   %}
 8761   ins_pipe(pipe_slow);
 8762 %}
 8763 
 8764 // This pattern is generated automatically from cas.m4.
 8765 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8766 instruct weakCompareAndSwapIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
 8767   predicate(needs_acquiring_load_exclusive(n));
 8768   match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
 8769   ins_cost(VOLATILE_REF_COST);
 8770   effect(KILL cr);
 8771   format %{
 8772     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
 8773     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8774   %}
 8775   ins_encode %{
 8776     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8777                Assembler::word, /*acquire*/ true, /*release*/ true,
 8778                /*weak*/ true, noreg);
 8779     __ csetw($res$$Register, Assembler::EQ);
 8780   %}
 8781   ins_pipe(pipe_slow);
 8782 %}
 8783 
 8784 // This pattern is generated automatically from cas.m4.
 8785 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8786 instruct weakCompareAndSwapLAcq(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
 8787   predicate(needs_acquiring_load_exclusive(n));
 8788   match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
 8789   ins_cost(VOLATILE_REF_COST);
 8790   effect(KILL cr);
 8791   format %{
 8792     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
 8793     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8794   %}
 8795   ins_encode %{
 8796     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8797                Assembler::xword, /*acquire*/ true, /*release*/ true,
 8798                /*weak*/ true, noreg);
 8799     __ csetw($res$$Register, Assembler::EQ);
 8800   %}
 8801   ins_pipe(pipe_slow);
 8802 %}
 8803 
 8804 // This pattern is generated automatically from cas.m4.
 8805 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8806 instruct weakCompareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
 8807   predicate(needs_acquiring_load_exclusive(n));
 8808   match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
 8809   ins_cost(VOLATILE_REF_COST);
 8810   effect(KILL cr);
 8811   format %{
 8812     "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
 8813     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8814   %}
 8815   ins_encode %{
 8816     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8817                Assembler::word, /*acquire*/ true, /*release*/ true,
 8818                /*weak*/ true, noreg);
 8819     __ csetw($res$$Register, Assembler::EQ);
 8820   %}
 8821   ins_pipe(pipe_slow);
 8822 %}
 8823 
 8824 // This pattern is generated automatically from cas.m4.
 8825 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 8826 instruct weakCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
 8827   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 8828   match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
 8829   ins_cost(VOLATILE_REF_COST);
 8830   effect(KILL cr);
 8831   format %{
 8832     "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
 8833     "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
 8834   %}
 8835   ins_encode %{
 8836     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
 8837                Assembler::xword, /*acquire*/ true, /*release*/ true,
 8838                /*weak*/ true, noreg);
 8839     __ csetw($res$$Register, Assembler::EQ);
 8840   %}
 8841   ins_pipe(pipe_slow);
 8842 %}
 8843 
 8844 // END This section of the file is automatically generated. Do not edit --------------
 8845 // ---------------------------------------------------------------------
 8846 
 8847 instruct get_and_setI(indirect mem, iRegI newv, iRegINoSp prev) %{
 8848   match(Set prev (GetAndSetI mem newv));
 8849   ins_cost(2 * VOLATILE_REF_COST);
 8850   format %{ "atomic_xchgw  $prev, $newv, [$mem]" %}
 8851   ins_encode %{
 8852     __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 8853   %}
 8854   ins_pipe(pipe_serial);
 8855 %}
 8856 
 8857 instruct get_and_setL(indirect mem, iRegL newv, iRegLNoSp prev) %{
 8858   match(Set prev (GetAndSetL mem newv));
 8859   ins_cost(2 * VOLATILE_REF_COST);
 8860   format %{ "atomic_xchg  $prev, $newv, [$mem]" %}
 8861   ins_encode %{
 8862     __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
 8863   %}
 8864   ins_pipe(pipe_serial);
 8865 %}
 8866 
 8867 instruct get_and_setN(indirect mem, iRegN newv, iRegINoSp prev) %{
 8868   match(Set prev (GetAndSetN mem newv));
 8869   ins_cost(2 * VOLATILE_REF_COST);
 8870   format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
 8871   ins_encode %{
 8872     __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 8873   %}
 8874   ins_pipe(pipe_serial);
 8875 %}
 8876 
 8877 instruct get_and_setP(indirect mem, iRegP newv, iRegPNoSp prev) %{
 8878   predicate(n->as_LoadStore()->barrier_data() == 0);
 8879   match(Set prev (GetAndSetP mem newv));
 8880   ins_cost(2 * VOLATILE_REF_COST);
 8881   format %{ "atomic_xchg  $prev, $newv, [$mem]" %}
 8882   ins_encode %{
 8883     __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
 8884   %}
 8885   ins_pipe(pipe_serial);
 8886 %}
 8887 
 8888 instruct get_and_setIAcq(indirect mem, iRegI newv, iRegINoSp prev) %{
 8889   predicate(needs_acquiring_load_exclusive(n));
 8890   match(Set prev (GetAndSetI mem newv));
 8891   ins_cost(VOLATILE_REF_COST);
 8892   format %{ "atomic_xchgw_acq  $prev, $newv, [$mem]" %}
 8893   ins_encode %{
 8894     __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 8895   %}
 8896   ins_pipe(pipe_serial);
 8897 %}
 8898 
 8899 instruct get_and_setLAcq(indirect mem, iRegL newv, iRegLNoSp prev) %{
 8900   predicate(needs_acquiring_load_exclusive(n));
 8901   match(Set prev (GetAndSetL mem newv));
 8902   ins_cost(VOLATILE_REF_COST);
 8903   format %{ "atomic_xchg_acq  $prev, $newv, [$mem]" %}
 8904   ins_encode %{
 8905     __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
 8906   %}
 8907   ins_pipe(pipe_serial);
 8908 %}
 8909 
 8910 instruct get_and_setNAcq(indirect mem, iRegN newv, iRegINoSp prev) %{
 8911   predicate(needs_acquiring_load_exclusive(n));
 8912   match(Set prev (GetAndSetN mem newv));
 8913   ins_cost(VOLATILE_REF_COST);
 8914   format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
 8915   ins_encode %{
 8916     __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
 8917   %}
 8918   ins_pipe(pipe_serial);
 8919 %}
 8920 
 8921 instruct get_and_setPAcq(indirect mem, iRegP newv, iRegPNoSp prev) %{
 8922   predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
 8923   match(Set prev (GetAndSetP mem newv));
 8924   ins_cost(VOLATILE_REF_COST);
 8925   format %{ "atomic_xchg_acq  $prev, $newv, [$mem]" %}
 8926   ins_encode %{
 8927     __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
 8928   %}
 8929   ins_pipe(pipe_serial);
 8930 %}
 8931 
 8932 
 8933 instruct get_and_addL(indirect mem, iRegLNoSp newval, iRegL incr) %{
 8934   match(Set newval (GetAndAddL mem incr));
 8935   ins_cost(2 * VOLATILE_REF_COST + 1);
 8936   format %{ "get_and_addL $newval, [$mem], $incr" %}
 8937   ins_encode %{
 8938     __ atomic_add($newval$$Register, $incr$$Register, as_Register($mem$$base));
 8939   %}
 8940   ins_pipe(pipe_serial);
 8941 %}
 8942 
 8943 instruct get_and_addL_no_res(indirect mem, Universe dummy, iRegL incr) %{
 8944   predicate(n->as_LoadStore()->result_not_used());
 8945   match(Set dummy (GetAndAddL mem incr));
 8946   ins_cost(2 * VOLATILE_REF_COST);
 8947   format %{ "get_and_addL [$mem], $incr" %}
 8948   ins_encode %{
 8949     __ atomic_add(noreg, $incr$$Register, as_Register($mem$$base));
 8950   %}
 8951   ins_pipe(pipe_serial);
 8952 %}
 8953 
 8954 instruct get_and_addLi(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
 8955   match(Set newval (GetAndAddL mem incr));
 8956   ins_cost(2 * VOLATILE_REF_COST + 1);
 8957   format %{ "get_and_addL $newval, [$mem], $incr" %}
 8958   ins_encode %{
 8959     __ atomic_add($newval$$Register, $incr$$constant, as_Register($mem$$base));
 8960   %}
 8961   ins_pipe(pipe_serial);
 8962 %}
 8963 
 8964 instruct get_and_addLi_no_res(indirect mem, Universe dummy, immLAddSub incr) %{
 8965   predicate(n->as_LoadStore()->result_not_used());
 8966   match(Set dummy (GetAndAddL mem incr));
 8967   ins_cost(2 * VOLATILE_REF_COST);
 8968   format %{ "get_and_addL [$mem], $incr" %}
 8969   ins_encode %{
 8970     __ atomic_add(noreg, $incr$$constant, as_Register($mem$$base));
 8971   %}
 8972   ins_pipe(pipe_serial);
 8973 %}
 8974 
 8975 instruct get_and_addI(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
 8976   match(Set newval (GetAndAddI mem incr));
 8977   ins_cost(2 * VOLATILE_REF_COST + 1);
 8978   format %{ "get_and_addI $newval, [$mem], $incr" %}
 8979   ins_encode %{
 8980     __ atomic_addw($newval$$Register, $incr$$Register, as_Register($mem$$base));
 8981   %}
 8982   ins_pipe(pipe_serial);
 8983 %}
 8984 
 8985 instruct get_and_addI_no_res(indirect mem, Universe dummy, iRegIorL2I incr) %{
 8986   predicate(n->as_LoadStore()->result_not_used());
 8987   match(Set dummy (GetAndAddI mem incr));
 8988   ins_cost(2 * VOLATILE_REF_COST);
 8989   format %{ "get_and_addI [$mem], $incr" %}
 8990   ins_encode %{
 8991     __ atomic_addw(noreg, $incr$$Register, as_Register($mem$$base));
 8992   %}
 8993   ins_pipe(pipe_serial);
 8994 %}
 8995 
 8996 instruct get_and_addIi(indirect mem, iRegINoSp newval, immIAddSub incr) %{
 8997   match(Set newval (GetAndAddI mem incr));
 8998   ins_cost(2 * VOLATILE_REF_COST + 1);
 8999   format %{ "get_and_addI $newval, [$mem], $incr" %}
 9000   ins_encode %{
 9001     __ atomic_addw($newval$$Register, $incr$$constant, as_Register($mem$$base));
 9002   %}
 9003   ins_pipe(pipe_serial);
 9004 %}
 9005 
 9006 instruct get_and_addIi_no_res(indirect mem, Universe dummy, immIAddSub incr) %{
 9007   predicate(n->as_LoadStore()->result_not_used());
 9008   match(Set dummy (GetAndAddI mem incr));
 9009   ins_cost(2 * VOLATILE_REF_COST);
 9010   format %{ "get_and_addI [$mem], $incr" %}
 9011   ins_encode %{
 9012     __ atomic_addw(noreg, $incr$$constant, as_Register($mem$$base));
 9013   %}
 9014   ins_pipe(pipe_serial);
 9015 %}
 9016 
 9017 instruct get_and_addLAcq(indirect mem, iRegLNoSp newval, iRegL incr) %{
 9018   predicate(needs_acquiring_load_exclusive(n));
 9019   match(Set newval (GetAndAddL mem incr));
 9020   ins_cost(VOLATILE_REF_COST + 1);
 9021   format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
 9022   ins_encode %{
 9023     __ atomic_addal($newval$$Register, $incr$$Register, as_Register($mem$$base));
 9024   %}
 9025   ins_pipe(pipe_serial);
 9026 %}
 9027 
 9028 instruct get_and_addL_no_resAcq(indirect mem, Universe dummy, iRegL incr) %{
 9029   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
 9030   match(Set dummy (GetAndAddL mem incr));
 9031   ins_cost(VOLATILE_REF_COST);
 9032   format %{ "get_and_addL_acq [$mem], $incr" %}
 9033   ins_encode %{
 9034     __ atomic_addal(noreg, $incr$$Register, as_Register($mem$$base));
 9035   %}
 9036   ins_pipe(pipe_serial);
 9037 %}
 9038 
 9039 instruct get_and_addLiAcq(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
 9040   predicate(needs_acquiring_load_exclusive(n));
 9041   match(Set newval (GetAndAddL mem incr));
 9042   ins_cost(VOLATILE_REF_COST + 1);
 9043   format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
 9044   ins_encode %{
 9045     __ atomic_addal($newval$$Register, $incr$$constant, as_Register($mem$$base));
 9046   %}
 9047   ins_pipe(pipe_serial);
 9048 %}
 9049 
 9050 instruct get_and_addLi_no_resAcq(indirect mem, Universe dummy, immLAddSub incr) %{
 9051   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
 9052   match(Set dummy (GetAndAddL mem incr));
 9053   ins_cost(VOLATILE_REF_COST);
 9054   format %{ "get_and_addL_acq [$mem], $incr" %}
 9055   ins_encode %{
 9056     __ atomic_addal(noreg, $incr$$constant, as_Register($mem$$base));
 9057   %}
 9058   ins_pipe(pipe_serial);
 9059 %}
 9060 
 9061 instruct get_and_addIAcq(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
 9062   predicate(needs_acquiring_load_exclusive(n));
 9063   match(Set newval (GetAndAddI mem incr));
 9064   ins_cost(VOLATILE_REF_COST + 1);
 9065   format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
 9066   ins_encode %{
 9067     __ atomic_addalw($newval$$Register, $incr$$Register, as_Register($mem$$base));
 9068   %}
 9069   ins_pipe(pipe_serial);
 9070 %}
 9071 
 9072 instruct get_and_addI_no_resAcq(indirect mem, Universe dummy, iRegIorL2I incr) %{
 9073   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
 9074   match(Set dummy (GetAndAddI mem incr));
 9075   ins_cost(VOLATILE_REF_COST);
 9076   format %{ "get_and_addI_acq [$mem], $incr" %}
 9077   ins_encode %{
 9078     __ atomic_addalw(noreg, $incr$$Register, as_Register($mem$$base));
 9079   %}
 9080   ins_pipe(pipe_serial);
 9081 %}
 9082 
 9083 instruct get_and_addIiAcq(indirect mem, iRegINoSp newval, immIAddSub incr) %{
 9084   predicate(needs_acquiring_load_exclusive(n));
 9085   match(Set newval (GetAndAddI mem incr));
 9086   ins_cost(VOLATILE_REF_COST + 1);
 9087   format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
 9088   ins_encode %{
 9089     __ atomic_addalw($newval$$Register, $incr$$constant, as_Register($mem$$base));
 9090   %}
 9091   ins_pipe(pipe_serial);
 9092 %}
 9093 
 9094 instruct get_and_addIi_no_resAcq(indirect mem, Universe dummy, immIAddSub incr) %{
 9095   predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
 9096   match(Set dummy (GetAndAddI mem incr));
 9097   ins_cost(VOLATILE_REF_COST);
 9098   format %{ "get_and_addI_acq [$mem], $incr" %}
 9099   ins_encode %{
 9100     __ atomic_addalw(noreg, $incr$$constant, as_Register($mem$$base));
 9101   %}
 9102   ins_pipe(pipe_serial);
 9103 %}
 9104 
 9105 // Manifest a CmpU result in an integer register.
 9106 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
 9107 instruct cmpU3_reg_reg(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg flags)
 9108 %{
 9109   match(Set dst (CmpU3 src1 src2));
 9110   effect(KILL flags);
 9111 
 9112   ins_cost(INSN_COST * 3);
 9113   format %{
 9114       "cmpw $src1, $src2\n\t"
 9115       "csetw $dst, ne\n\t"
 9116       "cnegw $dst, lo\t# CmpU3(reg)"
 9117   %}
 9118   ins_encode %{
 9119     __ cmpw($src1$$Register, $src2$$Register);
 9120     __ csetw($dst$$Register, Assembler::NE);
 9121     __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
 9122   %}
 9123 
 9124   ins_pipe(pipe_class_default);
 9125 %}
 9126 
 9127 instruct cmpU3_reg_imm(iRegINoSp dst, iRegI src1, immIAddSub src2, rFlagsReg flags)
 9128 %{
 9129   match(Set dst (CmpU3 src1 src2));
 9130   effect(KILL flags);
 9131 
 9132   ins_cost(INSN_COST * 3);
 9133   format %{
 9134       "subsw zr, $src1, $src2\n\t"
 9135       "csetw $dst, ne\n\t"
 9136       "cnegw $dst, lo\t# CmpU3(imm)"
 9137   %}
 9138   ins_encode %{
 9139     __ subsw(zr, $src1$$Register, (int32_t)$src2$$constant);
 9140     __ csetw($dst$$Register, Assembler::NE);
 9141     __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
 9142   %}
 9143 
 9144   ins_pipe(pipe_class_default);
 9145 %}
 9146 
 9147 // Manifest a CmpUL result in an integer register.
 9148 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
 9149 instruct cmpUL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
 9150 %{
 9151   match(Set dst (CmpUL3 src1 src2));
 9152   effect(KILL flags);
 9153 
 9154   ins_cost(INSN_COST * 3);
 9155   format %{
 9156       "cmp $src1, $src2\n\t"
 9157       "csetw $dst, ne\n\t"
 9158       "cnegw $dst, lo\t# CmpUL3(reg)"
 9159   %}
 9160   ins_encode %{
 9161     __ cmp($src1$$Register, $src2$$Register);
 9162     __ csetw($dst$$Register, Assembler::NE);
 9163     __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
 9164   %}
 9165 
 9166   ins_pipe(pipe_class_default);
 9167 %}
 9168 
 9169 instruct cmpUL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
 9170 %{
 9171   match(Set dst (CmpUL3 src1 src2));
 9172   effect(KILL flags);
 9173 
 9174   ins_cost(INSN_COST * 3);
 9175   format %{
 9176       "subs zr, $src1, $src2\n\t"
 9177       "csetw $dst, ne\n\t"
 9178       "cnegw $dst, lo\t# CmpUL3(imm)"
 9179   %}
 9180   ins_encode %{
 9181     __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
 9182     __ csetw($dst$$Register, Assembler::NE);
 9183     __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
 9184   %}
 9185 
 9186   ins_pipe(pipe_class_default);
 9187 %}
 9188 
 9189 // Manifest a CmpL result in an integer register.
 9190 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
 9191 instruct cmpL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
 9192 %{
 9193   match(Set dst (CmpL3 src1 src2));
 9194   effect(KILL flags);
 9195 
 9196   ins_cost(INSN_COST * 3);
 9197   format %{
 9198       "cmp $src1, $src2\n\t"
 9199       "csetw $dst, ne\n\t"
 9200       "cnegw $dst, lt\t# CmpL3(reg)"
 9201   %}
 9202   ins_encode %{
 9203     __ cmp($src1$$Register, $src2$$Register);
 9204     __ csetw($dst$$Register, Assembler::NE);
 9205     __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
 9206   %}
 9207 
 9208   ins_pipe(pipe_class_default);
 9209 %}
 9210 
 9211 instruct cmpL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
 9212 %{
 9213   match(Set dst (CmpL3 src1 src2));
 9214   effect(KILL flags);
 9215 
 9216   ins_cost(INSN_COST * 3);
 9217   format %{
 9218       "subs zr, $src1, $src2\n\t"
 9219       "csetw $dst, ne\n\t"
 9220       "cnegw $dst, lt\t# CmpL3(imm)"
 9221   %}
 9222   ins_encode %{
 9223     __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
 9224     __ csetw($dst$$Register, Assembler::NE);
 9225     __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
 9226   %}
 9227 
 9228   ins_pipe(pipe_class_default);
 9229 %}
 9230 
 9231 // ============================================================================
 9232 // Conditional Move Instructions
 9233 
 9234 // n.b. we have identical rules for both a signed compare op (cmpOp)
 9235 // and an unsigned compare op (cmpOpU). it would be nice if we could
 9236 // define an op class which merged both inputs and use it to type the
 9237 // argument to a single rule. unfortunatelyt his fails because the
 9238 // opclass does not live up to the COND_INTER interface of its
 9239 // component operands. When the generic code tries to negate the
 9240 // operand it ends up running the generci Machoper::negate method
 9241 // which throws a ShouldNotHappen. So, we have to provide two flavours
 9242 // of each rule, one for a cmpOp and a second for a cmpOpU (sigh).
 9243 
 9244 instruct cmovI_reg_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
 9245   match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
 9246 
 9247   ins_cost(INSN_COST * 2);
 9248   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, int"  %}
 9249 
 9250   ins_encode %{
 9251     __ cselw(as_Register($dst$$reg),
 9252              as_Register($src2$$reg),
 9253              as_Register($src1$$reg),
 9254              (Assembler::Condition)$cmp$$cmpcode);
 9255   %}
 9256 
 9257   ins_pipe(icond_reg_reg);
 9258 %}
 9259 
 9260 instruct cmovUI_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
 9261   match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
 9262 
 9263   ins_cost(INSN_COST * 2);
 9264   format %{ "cselw $dst, $src2, $src1 $cmp\t# unsigned, int"  %}
 9265 
 9266   ins_encode %{
 9267     __ cselw(as_Register($dst$$reg),
 9268              as_Register($src2$$reg),
 9269              as_Register($src1$$reg),
 9270              (Assembler::Condition)$cmp$$cmpcode);
 9271   %}
 9272 
 9273   ins_pipe(icond_reg_reg);
 9274 %}
 9275 
 9276 // special cases where one arg is zero
 9277 
 9278 // n.b. this is selected in preference to the rule above because it
 9279 // avoids loading constant 0 into a source register
 9280 
 9281 // TODO
 9282 // we ought only to be able to cull one of these variants as the ideal
 9283 // transforms ought always to order the zero consistently (to left/right?)
 9284 
 9285 instruct cmovI_zero_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
 9286   match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
 9287 
 9288   ins_cost(INSN_COST * 2);
 9289   format %{ "cselw $dst, $src, zr $cmp\t# signed, int"  %}
 9290 
 9291   ins_encode %{
 9292     __ cselw(as_Register($dst$$reg),
 9293              as_Register($src$$reg),
 9294              zr,
 9295              (Assembler::Condition)$cmp$$cmpcode);
 9296   %}
 9297 
 9298   ins_pipe(icond_reg);
 9299 %}
 9300 
 9301 instruct cmovUI_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
 9302   match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
 9303 
 9304   ins_cost(INSN_COST * 2);
 9305   format %{ "cselw $dst, $src, zr $cmp\t# unsigned, int"  %}
 9306 
 9307   ins_encode %{
 9308     __ cselw(as_Register($dst$$reg),
 9309              as_Register($src$$reg),
 9310              zr,
 9311              (Assembler::Condition)$cmp$$cmpcode);
 9312   %}
 9313 
 9314   ins_pipe(icond_reg);
 9315 %}
 9316 
 9317 instruct cmovI_reg_zero(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
 9318   match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
 9319 
 9320   ins_cost(INSN_COST * 2);
 9321   format %{ "cselw $dst, zr, $src $cmp\t# signed, int"  %}
 9322 
 9323   ins_encode %{
 9324     __ cselw(as_Register($dst$$reg),
 9325              zr,
 9326              as_Register($src$$reg),
 9327              (Assembler::Condition)$cmp$$cmpcode);
 9328   %}
 9329 
 9330   ins_pipe(icond_reg);
 9331 %}
 9332 
 9333 instruct cmovUI_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
 9334   match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
 9335 
 9336   ins_cost(INSN_COST * 2);
 9337   format %{ "cselw $dst, zr, $src $cmp\t# unsigned, int"  %}
 9338 
 9339   ins_encode %{
 9340     __ cselw(as_Register($dst$$reg),
 9341              zr,
 9342              as_Register($src$$reg),
 9343              (Assembler::Condition)$cmp$$cmpcode);
 9344   %}
 9345 
 9346   ins_pipe(icond_reg);
 9347 %}
 9348 
 9349 // special case for creating a boolean 0 or 1
 9350 
 9351 // n.b. this is selected in preference to the rule above because it
 9352 // avoids loading constants 0 and 1 into a source register
 9353 
 9354 instruct cmovI_reg_zero_one(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
 9355   match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
 9356 
 9357   ins_cost(INSN_COST * 2);
 9358   format %{ "csincw $dst, zr, zr $cmp\t# signed, int"  %}
 9359 
 9360   ins_encode %{
 9361     // equivalently
 9362     // cset(as_Register($dst$$reg),
 9363     //      negate_condition((Assembler::Condition)$cmp$$cmpcode));
 9364     __ csincw(as_Register($dst$$reg),
 9365              zr,
 9366              zr,
 9367              (Assembler::Condition)$cmp$$cmpcode);
 9368   %}
 9369 
 9370   ins_pipe(icond_none);
 9371 %}
 9372 
 9373 instruct cmovUI_reg_zero_one(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
 9374   match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
 9375 
 9376   ins_cost(INSN_COST * 2);
 9377   format %{ "csincw $dst, zr, zr $cmp\t# unsigned, int"  %}
 9378 
 9379   ins_encode %{
 9380     // equivalently
 9381     // cset(as_Register($dst$$reg),
 9382     //      negate_condition((Assembler::Condition)$cmp$$cmpcode));
 9383     __ csincw(as_Register($dst$$reg),
 9384              zr,
 9385              zr,
 9386              (Assembler::Condition)$cmp$$cmpcode);
 9387   %}
 9388 
 9389   ins_pipe(icond_none);
 9390 %}
 9391 
 9392 instruct cmovL_reg_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
 9393   match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
 9394 
 9395   ins_cost(INSN_COST * 2);
 9396   format %{ "csel $dst, $src2, $src1 $cmp\t# signed, long"  %}
 9397 
 9398   ins_encode %{
 9399     __ csel(as_Register($dst$$reg),
 9400             as_Register($src2$$reg),
 9401             as_Register($src1$$reg),
 9402             (Assembler::Condition)$cmp$$cmpcode);
 9403   %}
 9404 
 9405   ins_pipe(icond_reg_reg);
 9406 %}
 9407 
 9408 instruct cmovUL_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
 9409   match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
 9410 
 9411   ins_cost(INSN_COST * 2);
 9412   format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, long"  %}
 9413 
 9414   ins_encode %{
 9415     __ csel(as_Register($dst$$reg),
 9416             as_Register($src2$$reg),
 9417             as_Register($src1$$reg),
 9418             (Assembler::Condition)$cmp$$cmpcode);
 9419   %}
 9420 
 9421   ins_pipe(icond_reg_reg);
 9422 %}
 9423 
 9424 // special cases where one arg is zero
 9425 
 9426 instruct cmovL_reg_zero(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
 9427   match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
 9428 
 9429   ins_cost(INSN_COST * 2);
 9430   format %{ "csel $dst, zr, $src $cmp\t# signed, long"  %}
 9431 
 9432   ins_encode %{
 9433     __ csel(as_Register($dst$$reg),
 9434             zr,
 9435             as_Register($src$$reg),
 9436             (Assembler::Condition)$cmp$$cmpcode);
 9437   %}
 9438 
 9439   ins_pipe(icond_reg);
 9440 %}
 9441 
 9442 instruct cmovUL_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
 9443   match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
 9444 
 9445   ins_cost(INSN_COST * 2);
 9446   format %{ "csel $dst, zr, $src $cmp\t# unsigned, long"  %}
 9447 
 9448   ins_encode %{
 9449     __ csel(as_Register($dst$$reg),
 9450             zr,
 9451             as_Register($src$$reg),
 9452             (Assembler::Condition)$cmp$$cmpcode);
 9453   %}
 9454 
 9455   ins_pipe(icond_reg);
 9456 %}
 9457 
 9458 instruct cmovL_zero_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
 9459   match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
 9460 
 9461   ins_cost(INSN_COST * 2);
 9462   format %{ "csel $dst, $src, zr $cmp\t# signed, long"  %}
 9463 
 9464   ins_encode %{
 9465     __ csel(as_Register($dst$$reg),
 9466             as_Register($src$$reg),
 9467             zr,
 9468             (Assembler::Condition)$cmp$$cmpcode);
 9469   %}
 9470 
 9471   ins_pipe(icond_reg);
 9472 %}
 9473 
 9474 instruct cmovUL_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
 9475   match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
 9476 
 9477   ins_cost(INSN_COST * 2);
 9478   format %{ "csel $dst, $src, zr $cmp\t# unsigned, long"  %}
 9479 
 9480   ins_encode %{
 9481     __ csel(as_Register($dst$$reg),
 9482             as_Register($src$$reg),
 9483             zr,
 9484             (Assembler::Condition)$cmp$$cmpcode);
 9485   %}
 9486 
 9487   ins_pipe(icond_reg);
 9488 %}
 9489 
 9490 instruct cmovP_reg_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
 9491   match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
 9492 
 9493   ins_cost(INSN_COST * 2);
 9494   format %{ "csel $dst, $src2, $src1 $cmp\t# signed, ptr"  %}
 9495 
 9496   ins_encode %{
 9497     __ csel(as_Register($dst$$reg),
 9498             as_Register($src2$$reg),
 9499             as_Register($src1$$reg),
 9500             (Assembler::Condition)$cmp$$cmpcode);
 9501   %}
 9502 
 9503   ins_pipe(icond_reg_reg);
 9504 %}
 9505 
 9506 instruct cmovUP_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
 9507   match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
 9508 
 9509   ins_cost(INSN_COST * 2);
 9510   format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, ptr"  %}
 9511 
 9512   ins_encode %{
 9513     __ csel(as_Register($dst$$reg),
 9514             as_Register($src2$$reg),
 9515             as_Register($src1$$reg),
 9516             (Assembler::Condition)$cmp$$cmpcode);
 9517   %}
 9518 
 9519   ins_pipe(icond_reg_reg);
 9520 %}
 9521 
 9522 // special cases where one arg is zero
 9523 
 9524 instruct cmovP_reg_zero(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
 9525   match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
 9526 
 9527   ins_cost(INSN_COST * 2);
 9528   format %{ "csel $dst, zr, $src $cmp\t# signed, ptr"  %}
 9529 
 9530   ins_encode %{
 9531     __ csel(as_Register($dst$$reg),
 9532             zr,
 9533             as_Register($src$$reg),
 9534             (Assembler::Condition)$cmp$$cmpcode);
 9535   %}
 9536 
 9537   ins_pipe(icond_reg);
 9538 %}
 9539 
 9540 instruct cmovUP_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
 9541   match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
 9542 
 9543   ins_cost(INSN_COST * 2);
 9544   format %{ "csel $dst, zr, $src $cmp\t# unsigned, ptr"  %}
 9545 
 9546   ins_encode %{
 9547     __ csel(as_Register($dst$$reg),
 9548             zr,
 9549             as_Register($src$$reg),
 9550             (Assembler::Condition)$cmp$$cmpcode);
 9551   %}
 9552 
 9553   ins_pipe(icond_reg);
 9554 %}
 9555 
 9556 instruct cmovP_zero_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
 9557   match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
 9558 
 9559   ins_cost(INSN_COST * 2);
 9560   format %{ "csel $dst, $src, zr $cmp\t# signed, ptr"  %}
 9561 
 9562   ins_encode %{
 9563     __ csel(as_Register($dst$$reg),
 9564             as_Register($src$$reg),
 9565             zr,
 9566             (Assembler::Condition)$cmp$$cmpcode);
 9567   %}
 9568 
 9569   ins_pipe(icond_reg);
 9570 %}
 9571 
 9572 instruct cmovUP_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
 9573   match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
 9574 
 9575   ins_cost(INSN_COST * 2);
 9576   format %{ "csel $dst, $src, zr $cmp\t# unsigned, ptr"  %}
 9577 
 9578   ins_encode %{
 9579     __ csel(as_Register($dst$$reg),
 9580             as_Register($src$$reg),
 9581             zr,
 9582             (Assembler::Condition)$cmp$$cmpcode);
 9583   %}
 9584 
 9585   ins_pipe(icond_reg);
 9586 %}
 9587 
 9588 instruct cmovN_reg_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
 9589   match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
 9590 
 9591   ins_cost(INSN_COST * 2);
 9592   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr"  %}
 9593 
 9594   ins_encode %{
 9595     __ cselw(as_Register($dst$$reg),
 9596              as_Register($src2$$reg),
 9597              as_Register($src1$$reg),
 9598              (Assembler::Condition)$cmp$$cmpcode);
 9599   %}
 9600 
 9601   ins_pipe(icond_reg_reg);
 9602 %}
 9603 
 9604 instruct cmovUN_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
 9605   match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
 9606 
 9607   ins_cost(INSN_COST * 2);
 9608   format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr"  %}
 9609 
 9610   ins_encode %{
 9611     __ cselw(as_Register($dst$$reg),
 9612              as_Register($src2$$reg),
 9613              as_Register($src1$$reg),
 9614              (Assembler::Condition)$cmp$$cmpcode);
 9615   %}
 9616 
 9617   ins_pipe(icond_reg_reg);
 9618 %}
 9619 
 9620 // special cases where one arg is zero
 9621 
 9622 instruct cmovN_reg_zero(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
 9623   match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
 9624 
 9625   ins_cost(INSN_COST * 2);
 9626   format %{ "cselw $dst, zr, $src $cmp\t# signed, compressed ptr"  %}
 9627 
 9628   ins_encode %{
 9629     __ cselw(as_Register($dst$$reg),
 9630              zr,
 9631              as_Register($src$$reg),
 9632              (Assembler::Condition)$cmp$$cmpcode);
 9633   %}
 9634 
 9635   ins_pipe(icond_reg);
 9636 %}
 9637 
 9638 instruct cmovUN_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
 9639   match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
 9640 
 9641   ins_cost(INSN_COST * 2);
 9642   format %{ "cselw $dst, zr, $src $cmp\t# unsigned, compressed ptr"  %}
 9643 
 9644   ins_encode %{
 9645     __ cselw(as_Register($dst$$reg),
 9646              zr,
 9647              as_Register($src$$reg),
 9648              (Assembler::Condition)$cmp$$cmpcode);
 9649   %}
 9650 
 9651   ins_pipe(icond_reg);
 9652 %}
 9653 
 9654 instruct cmovN_zero_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
 9655   match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
 9656 
 9657   ins_cost(INSN_COST * 2);
 9658   format %{ "cselw $dst, $src, zr $cmp\t# signed, compressed ptr"  %}
 9659 
 9660   ins_encode %{
 9661     __ cselw(as_Register($dst$$reg),
 9662              as_Register($src$$reg),
 9663              zr,
 9664              (Assembler::Condition)$cmp$$cmpcode);
 9665   %}
 9666 
 9667   ins_pipe(icond_reg);
 9668 %}
 9669 
 9670 instruct cmovUN_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
 9671   match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
 9672 
 9673   ins_cost(INSN_COST * 2);
 9674   format %{ "cselw $dst, $src, zr $cmp\t# unsigned, compressed ptr"  %}
 9675 
 9676   ins_encode %{
 9677     __ cselw(as_Register($dst$$reg),
 9678              as_Register($src$$reg),
 9679              zr,
 9680              (Assembler::Condition)$cmp$$cmpcode);
 9681   %}
 9682 
 9683   ins_pipe(icond_reg);
 9684 %}
 9685 
 9686 instruct cmovF_reg(cmpOp cmp, rFlagsReg cr, vRegF dst, vRegF src1,  vRegF src2)
 9687 %{
 9688   match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
 9689 
 9690   ins_cost(INSN_COST * 3);
 9691 
 9692   format %{ "fcsels $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
 9693   ins_encode %{
 9694     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
 9695     __ fcsels(as_FloatRegister($dst$$reg),
 9696               as_FloatRegister($src2$$reg),
 9697               as_FloatRegister($src1$$reg),
 9698               cond);
 9699   %}
 9700 
 9701   ins_pipe(fp_cond_reg_reg_s);
 9702 %}
 9703 
 9704 instruct cmovUF_reg(cmpOpU cmp, rFlagsRegU cr, vRegF dst, vRegF src1,  vRegF src2)
 9705 %{
 9706   match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
 9707 
 9708   ins_cost(INSN_COST * 3);
 9709 
 9710   format %{ "fcsels $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
 9711   ins_encode %{
 9712     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
 9713     __ fcsels(as_FloatRegister($dst$$reg),
 9714               as_FloatRegister($src2$$reg),
 9715               as_FloatRegister($src1$$reg),
 9716               cond);
 9717   %}
 9718 
 9719   ins_pipe(fp_cond_reg_reg_s);
 9720 %}
 9721 
 9722 instruct cmovD_reg(cmpOp cmp, rFlagsReg cr, vRegD dst, vRegD src1,  vRegD src2)
 9723 %{
 9724   match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
 9725 
 9726   ins_cost(INSN_COST * 3);
 9727 
 9728   format %{ "fcseld $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
 9729   ins_encode %{
 9730     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
 9731     __ fcseld(as_FloatRegister($dst$$reg),
 9732               as_FloatRegister($src2$$reg),
 9733               as_FloatRegister($src1$$reg),
 9734               cond);
 9735   %}
 9736 
 9737   ins_pipe(fp_cond_reg_reg_d);
 9738 %}
 9739 
 9740 instruct cmovUD_reg(cmpOpU cmp, rFlagsRegU cr, vRegD dst, vRegD src1,  vRegD src2)
 9741 %{
 9742   match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
 9743 
 9744   ins_cost(INSN_COST * 3);
 9745 
 9746   format %{ "fcseld $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
 9747   ins_encode %{
 9748     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
 9749     __ fcseld(as_FloatRegister($dst$$reg),
 9750               as_FloatRegister($src2$$reg),
 9751               as_FloatRegister($src1$$reg),
 9752               cond);
 9753   %}
 9754 
 9755   ins_pipe(fp_cond_reg_reg_d);
 9756 %}
 9757 
 9758 // ============================================================================
 9759 // Arithmetic Instructions
 9760 //
 9761 
 9762 // Integer Addition
 9763 
 9764 // TODO
 9765 // these currently employ operations which do not set CR and hence are
 9766 // not flagged as killing CR but we would like to isolate the cases
 9767 // where we want to set flags from those where we don't. need to work
 9768 // out how to do that.
 9769 
 9770 instruct addI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
 9771   match(Set dst (AddI src1 src2));
 9772 
 9773   ins_cost(INSN_COST);
 9774   format %{ "addw  $dst, $src1, $src2" %}
 9775 
 9776   ins_encode %{
 9777     __ addw(as_Register($dst$$reg),
 9778             as_Register($src1$$reg),
 9779             as_Register($src2$$reg));
 9780   %}
 9781 
 9782   ins_pipe(ialu_reg_reg);
 9783 %}
 9784 
 9785 instruct addI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
 9786   match(Set dst (AddI src1 src2));
 9787 
 9788   ins_cost(INSN_COST);
 9789   format %{ "addw $dst, $src1, $src2" %}
 9790 
 9791   // use opcode to indicate that this is an add not a sub
 9792   opcode(0x0);
 9793 
 9794   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
 9795 
 9796   ins_pipe(ialu_reg_imm);
 9797 %}
 9798 
 9799 instruct addI_reg_imm_i2l(iRegINoSp dst, iRegL src1, immIAddSub src2) %{
 9800   match(Set dst (AddI (ConvL2I src1) src2));
 9801 
 9802   ins_cost(INSN_COST);
 9803   format %{ "addw $dst, $src1, $src2" %}
 9804 
 9805   // use opcode to indicate that this is an add not a sub
 9806   opcode(0x0);
 9807 
 9808   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
 9809 
 9810   ins_pipe(ialu_reg_imm);
 9811 %}
 9812 
 9813 // Pointer Addition
 9814 instruct addP_reg_reg(iRegPNoSp dst, iRegP src1, iRegL src2) %{
 9815   match(Set dst (AddP src1 src2));
 9816 
 9817   ins_cost(INSN_COST);
 9818   format %{ "add $dst, $src1, $src2\t# ptr" %}
 9819 
 9820   ins_encode %{
 9821     __ add(as_Register($dst$$reg),
 9822            as_Register($src1$$reg),
 9823            as_Register($src2$$reg));
 9824   %}
 9825 
 9826   ins_pipe(ialu_reg_reg);
 9827 %}
 9828 
 9829 instruct addP_reg_reg_ext(iRegPNoSp dst, iRegP src1, iRegIorL2I src2) %{
 9830   match(Set dst (AddP src1 (ConvI2L src2)));
 9831 
 9832   ins_cost(1.9 * INSN_COST);
 9833   format %{ "add $dst, $src1, $src2, sxtw\t# ptr" %}
 9834 
 9835   ins_encode %{
 9836     __ add(as_Register($dst$$reg),
 9837            as_Register($src1$$reg),
 9838            as_Register($src2$$reg), ext::sxtw);
 9839   %}
 9840 
 9841   ins_pipe(ialu_reg_reg);
 9842 %}
 9843 
 9844 instruct addP_reg_reg_lsl(iRegPNoSp dst, iRegP src1, iRegL src2, immIScale scale) %{
 9845   match(Set dst (AddP src1 (LShiftL src2 scale)));
 9846 
 9847   ins_cost(1.9 * INSN_COST);
 9848   format %{ "add $dst, $src1, $src2, LShiftL $scale\t# ptr" %}
 9849 
 9850   ins_encode %{
 9851     __ lea(as_Register($dst$$reg),
 9852            Address(as_Register($src1$$reg), as_Register($src2$$reg),
 9853                    Address::lsl($scale$$constant)));
 9854   %}
 9855 
 9856   ins_pipe(ialu_reg_reg_shift);
 9857 %}
 9858 
 9859 instruct addP_reg_reg_ext_shift(iRegPNoSp dst, iRegP src1, iRegIorL2I src2, immIScale scale) %{
 9860   match(Set dst (AddP src1 (LShiftL (ConvI2L src2) scale)));
 9861 
 9862   ins_cost(1.9 * INSN_COST);
 9863   format %{ "add $dst, $src1, $src2, I2L $scale\t# ptr" %}
 9864 
 9865   ins_encode %{
 9866     __ lea(as_Register($dst$$reg),
 9867            Address(as_Register($src1$$reg), as_Register($src2$$reg),
 9868                    Address::sxtw($scale$$constant)));
 9869   %}
 9870 
 9871   ins_pipe(ialu_reg_reg_shift);
 9872 %}
 9873 
 9874 instruct lshift_ext(iRegLNoSp dst, iRegIorL2I src, immI scale, rFlagsReg cr) %{
 9875   match(Set dst (LShiftL (ConvI2L src) scale));
 9876 
 9877   ins_cost(INSN_COST);
 9878   format %{ "sbfiz $dst, $src, $scale & 63, -$scale & 63\t" %}
 9879 
 9880   ins_encode %{
 9881     __ sbfiz(as_Register($dst$$reg),
 9882           as_Register($src$$reg),
 9883           $scale$$constant & 63, MIN2(32, (int)((-$scale$$constant) & 63)));
 9884   %}
 9885 
 9886   ins_pipe(ialu_reg_shift);
 9887 %}
 9888 
 9889 // Pointer Immediate Addition
 9890 // n.b. this needs to be more expensive than using an indirect memory
 9891 // operand
 9892 instruct addP_reg_imm(iRegPNoSp dst, iRegP src1, immLAddSub src2) %{
 9893   match(Set dst (AddP src1 src2));
 9894 
 9895   ins_cost(INSN_COST);
 9896   format %{ "add $dst, $src1, $src2\t# ptr" %}
 9897 
 9898   // use opcode to indicate that this is an add not a sub
 9899   opcode(0x0);
 9900 
 9901   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
 9902 
 9903   ins_pipe(ialu_reg_imm);
 9904 %}
 9905 
 9906 // Long Addition
 9907 instruct addL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
 9908 
 9909   match(Set dst (AddL src1 src2));
 9910 
 9911   ins_cost(INSN_COST);
 9912   format %{ "add  $dst, $src1, $src2" %}
 9913 
 9914   ins_encode %{
 9915     __ add(as_Register($dst$$reg),
 9916            as_Register($src1$$reg),
 9917            as_Register($src2$$reg));
 9918   %}
 9919 
 9920   ins_pipe(ialu_reg_reg);
 9921 %}
 9922 
 9923 // No constant pool entries requiredLong Immediate Addition.
 9924 instruct addL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
 9925   match(Set dst (AddL src1 src2));
 9926 
 9927   ins_cost(INSN_COST);
 9928   format %{ "add $dst, $src1, $src2" %}
 9929 
 9930   // use opcode to indicate that this is an add not a sub
 9931   opcode(0x0);
 9932 
 9933   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
 9934 
 9935   ins_pipe(ialu_reg_imm);
 9936 %}
 9937 
 9938 // Integer Subtraction
 9939 instruct subI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
 9940   match(Set dst (SubI src1 src2));
 9941 
 9942   ins_cost(INSN_COST);
 9943   format %{ "subw  $dst, $src1, $src2" %}
 9944 
 9945   ins_encode %{
 9946     __ subw(as_Register($dst$$reg),
 9947             as_Register($src1$$reg),
 9948             as_Register($src2$$reg));
 9949   %}
 9950 
 9951   ins_pipe(ialu_reg_reg);
 9952 %}
 9953 
 9954 // Immediate Subtraction
 9955 instruct subI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
 9956   match(Set dst (SubI src1 src2));
 9957 
 9958   ins_cost(INSN_COST);
 9959   format %{ "subw $dst, $src1, $src2" %}
 9960 
 9961   // use opcode to indicate that this is a sub not an add
 9962   opcode(0x1);
 9963 
 9964   ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
 9965 
 9966   ins_pipe(ialu_reg_imm);
 9967 %}
 9968 
 9969 // Long Subtraction
 9970 instruct subL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
 9971 
 9972   match(Set dst (SubL src1 src2));
 9973 
 9974   ins_cost(INSN_COST);
 9975   format %{ "sub  $dst, $src1, $src2" %}
 9976 
 9977   ins_encode %{
 9978     __ sub(as_Register($dst$$reg),
 9979            as_Register($src1$$reg),
 9980            as_Register($src2$$reg));
 9981   %}
 9982 
 9983   ins_pipe(ialu_reg_reg);
 9984 %}
 9985 
 9986 // No constant pool entries requiredLong Immediate Subtraction.
 9987 instruct subL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
 9988   match(Set dst (SubL src1 src2));
 9989 
 9990   ins_cost(INSN_COST);
 9991   format %{ "sub$dst, $src1, $src2" %}
 9992 
 9993   // use opcode to indicate that this is a sub not an add
 9994   opcode(0x1);
 9995 
 9996   ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
 9997 
 9998   ins_pipe(ialu_reg_imm);
 9999 %}
10000 
10001 // Integer Negation (special case for sub)
10002 
10003 instruct negI_reg(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr) %{
10004   match(Set dst (SubI zero src));
10005 
10006   ins_cost(INSN_COST);
10007   format %{ "negw $dst, $src\t# int" %}
10008 
10009   ins_encode %{
10010     __ negw(as_Register($dst$$reg),
10011             as_Register($src$$reg));
10012   %}
10013 
10014   ins_pipe(ialu_reg);
10015 %}
10016 
10017 // Long Negation
10018 
10019 instruct negL_reg(iRegLNoSp dst, iRegL src, immL0 zero, rFlagsReg cr) %{
10020   match(Set dst (SubL zero src));
10021 
10022   ins_cost(INSN_COST);
10023   format %{ "neg $dst, $src\t# long" %}
10024 
10025   ins_encode %{
10026     __ neg(as_Register($dst$$reg),
10027            as_Register($src$$reg));
10028   %}
10029 
10030   ins_pipe(ialu_reg);
10031 %}
10032 
10033 // Integer Multiply
10034 
10035 instruct mulI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10036   match(Set dst (MulI src1 src2));
10037 
10038   ins_cost(INSN_COST * 3);
10039   format %{ "mulw  $dst, $src1, $src2" %}
10040 
10041   ins_encode %{
10042     __ mulw(as_Register($dst$$reg),
10043             as_Register($src1$$reg),
10044             as_Register($src2$$reg));
10045   %}
10046 
10047   ins_pipe(imul_reg_reg);
10048 %}
10049 
10050 instruct smulI(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10051   match(Set dst (MulL (ConvI2L src1) (ConvI2L src2)));
10052 
10053   ins_cost(INSN_COST * 3);
10054   format %{ "smull  $dst, $src1, $src2" %}
10055 
10056   ins_encode %{
10057     __ smull(as_Register($dst$$reg),
10058              as_Register($src1$$reg),
10059              as_Register($src2$$reg));
10060   %}
10061 
10062   ins_pipe(imul_reg_reg);
10063 %}
10064 
10065 // Long Multiply
10066 
10067 instruct mulL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10068   match(Set dst (MulL src1 src2));
10069 
10070   ins_cost(INSN_COST * 5);
10071   format %{ "mul  $dst, $src1, $src2" %}
10072 
10073   ins_encode %{
10074     __ mul(as_Register($dst$$reg),
10075            as_Register($src1$$reg),
10076            as_Register($src2$$reg));
10077   %}
10078 
10079   ins_pipe(lmul_reg_reg);
10080 %}
10081 
10082 instruct mulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10083 %{
10084   match(Set dst (MulHiL src1 src2));
10085 
10086   ins_cost(INSN_COST * 7);
10087   format %{ "smulh   $dst, $src1, $src2\t# mulhi" %}
10088 
10089   ins_encode %{
10090     __ smulh(as_Register($dst$$reg),
10091              as_Register($src1$$reg),
10092              as_Register($src2$$reg));
10093   %}
10094 
10095   ins_pipe(lmul_reg_reg);
10096 %}
10097 
10098 instruct umulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10099 %{
10100   match(Set dst (UMulHiL src1 src2));
10101 
10102   ins_cost(INSN_COST * 7);
10103   format %{ "umulh   $dst, $src1, $src2\t# umulhi" %}
10104 
10105   ins_encode %{
10106     __ umulh(as_Register($dst$$reg),
10107              as_Register($src1$$reg),
10108              as_Register($src2$$reg));
10109   %}
10110 
10111   ins_pipe(lmul_reg_reg);
10112 %}
10113 
10114 // Combined Integer Multiply & Add/Sub
10115 
10116 instruct maddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10117   match(Set dst (AddI src3 (MulI src1 src2)));
10118 
10119   ins_cost(INSN_COST * 3);
10120   format %{ "madd  $dst, $src1, $src2, $src3" %}
10121 
10122   ins_encode %{
10123     __ maddw(as_Register($dst$$reg),
10124              as_Register($src1$$reg),
10125              as_Register($src2$$reg),
10126              as_Register($src3$$reg));
10127   %}
10128 
10129   ins_pipe(imac_reg_reg);
10130 %}
10131 
10132 instruct msubI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10133   match(Set dst (SubI src3 (MulI src1 src2)));
10134 
10135   ins_cost(INSN_COST * 3);
10136   format %{ "msub  $dst, $src1, $src2, $src3" %}
10137 
10138   ins_encode %{
10139     __ msubw(as_Register($dst$$reg),
10140              as_Register($src1$$reg),
10141              as_Register($src2$$reg),
10142              as_Register($src3$$reg));
10143   %}
10144 
10145   ins_pipe(imac_reg_reg);
10146 %}
10147 
10148 // Combined Integer Multiply & Neg
10149 
10150 instruct mnegI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI0 zero) %{
10151   match(Set dst (MulI (SubI zero src1) src2));
10152 
10153   ins_cost(INSN_COST * 3);
10154   format %{ "mneg  $dst, $src1, $src2" %}
10155 
10156   ins_encode %{
10157     __ mnegw(as_Register($dst$$reg),
10158              as_Register($src1$$reg),
10159              as_Register($src2$$reg));
10160   %}
10161 
10162   ins_pipe(imac_reg_reg);
10163 %}
10164 
10165 // Combined Long Multiply & Add/Sub
10166 
10167 instruct maddL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10168   match(Set dst (AddL src3 (MulL src1 src2)));
10169 
10170   ins_cost(INSN_COST * 5);
10171   format %{ "madd  $dst, $src1, $src2, $src3" %}
10172 
10173   ins_encode %{
10174     __ madd(as_Register($dst$$reg),
10175             as_Register($src1$$reg),
10176             as_Register($src2$$reg),
10177             as_Register($src3$$reg));
10178   %}
10179 
10180   ins_pipe(lmac_reg_reg);
10181 %}
10182 
10183 instruct msubL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10184   match(Set dst (SubL src3 (MulL src1 src2)));
10185 
10186   ins_cost(INSN_COST * 5);
10187   format %{ "msub  $dst, $src1, $src2, $src3" %}
10188 
10189   ins_encode %{
10190     __ msub(as_Register($dst$$reg),
10191             as_Register($src1$$reg),
10192             as_Register($src2$$reg),
10193             as_Register($src3$$reg));
10194   %}
10195 
10196   ins_pipe(lmac_reg_reg);
10197 %}
10198 
10199 // Combined Long Multiply & Neg
10200 
10201 instruct mnegL(iRegLNoSp dst, iRegL src1, iRegL src2, immL0 zero) %{
10202   match(Set dst (MulL (SubL zero src1) src2));
10203 
10204   ins_cost(INSN_COST * 5);
10205   format %{ "mneg  $dst, $src1, $src2" %}
10206 
10207   ins_encode %{
10208     __ mneg(as_Register($dst$$reg),
10209             as_Register($src1$$reg),
10210             as_Register($src2$$reg));
10211   %}
10212 
10213   ins_pipe(lmac_reg_reg);
10214 %}
10215 
10216 // Combine Integer Signed Multiply & Add/Sub/Neg Long
10217 
10218 instruct smaddL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10219   match(Set dst (AddL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10220 
10221   ins_cost(INSN_COST * 3);
10222   format %{ "smaddl  $dst, $src1, $src2, $src3" %}
10223 
10224   ins_encode %{
10225     __ smaddl(as_Register($dst$$reg),
10226               as_Register($src1$$reg),
10227               as_Register($src2$$reg),
10228               as_Register($src3$$reg));
10229   %}
10230 
10231   ins_pipe(imac_reg_reg);
10232 %}
10233 
10234 instruct smsubL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10235   match(Set dst (SubL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10236 
10237   ins_cost(INSN_COST * 3);
10238   format %{ "smsubl  $dst, $src1, $src2, $src3" %}
10239 
10240   ins_encode %{
10241     __ smsubl(as_Register($dst$$reg),
10242               as_Register($src1$$reg),
10243               as_Register($src2$$reg),
10244               as_Register($src3$$reg));
10245   %}
10246 
10247   ins_pipe(imac_reg_reg);
10248 %}
10249 
10250 instruct smnegL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, immL0 zero) %{
10251   match(Set dst (MulL (SubL zero (ConvI2L src1)) (ConvI2L src2)));
10252 
10253   ins_cost(INSN_COST * 3);
10254   format %{ "smnegl  $dst, $src1, $src2" %}
10255 
10256   ins_encode %{
10257     __ smnegl(as_Register($dst$$reg),
10258               as_Register($src1$$reg),
10259               as_Register($src2$$reg));
10260   %}
10261 
10262   ins_pipe(imac_reg_reg);
10263 %}
10264 
10265 // Combined Multiply-Add Shorts into Integer (dst = src1 * src2 + src3 * src4)
10266 
10267 instruct muladdS2I(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3, iRegIorL2I src4) %{
10268   match(Set dst (MulAddS2I (Binary src1 src2) (Binary src3 src4)));
10269 
10270   ins_cost(INSN_COST * 5);
10271   format %{ "mulw  rscratch1, $src1, $src2\n\t"
10272             "maddw $dst, $src3, $src4, rscratch1" %}
10273 
10274   ins_encode %{
10275     __ mulw(rscratch1, as_Register($src1$$reg), as_Register($src2$$reg));
10276     __ maddw(as_Register($dst$$reg), as_Register($src3$$reg), as_Register($src4$$reg), rscratch1); %}
10277 
10278   ins_pipe(imac_reg_reg);
10279 %}
10280 
10281 // Integer Divide
10282 
10283 instruct divI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10284   match(Set dst (DivI src1 src2));
10285 
10286   ins_cost(INSN_COST * 19);
10287   format %{ "sdivw  $dst, $src1, $src2" %}
10288 
10289   ins_encode(aarch64_enc_divw(dst, src1, src2));
10290   ins_pipe(idiv_reg_reg);
10291 %}
10292 
10293 // Long Divide
10294 
10295 instruct divL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10296   match(Set dst (DivL src1 src2));
10297 
10298   ins_cost(INSN_COST * 35);
10299   format %{ "sdiv   $dst, $src1, $src2" %}
10300 
10301   ins_encode(aarch64_enc_div(dst, src1, src2));
10302   ins_pipe(ldiv_reg_reg);
10303 %}
10304 
10305 // Integer Remainder
10306 
10307 instruct modI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10308   match(Set dst (ModI src1 src2));
10309 
10310   ins_cost(INSN_COST * 22);
10311   format %{ "sdivw  rscratch1, $src1, $src2\n\t"
10312             "msubw  $dst, rscratch1, $src2, $src1" %}
10313 
10314   ins_encode(aarch64_enc_modw(dst, src1, src2));
10315   ins_pipe(idiv_reg_reg);
10316 %}
10317 
10318 // Long Remainder
10319 
10320 instruct modL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10321   match(Set dst (ModL src1 src2));
10322 
10323   ins_cost(INSN_COST * 38);
10324   format %{ "sdiv   rscratch1, $src1, $src2\n"
10325             "msub   $dst, rscratch1, $src2, $src1" %}
10326 
10327   ins_encode(aarch64_enc_mod(dst, src1, src2));
10328   ins_pipe(ldiv_reg_reg);
10329 %}
10330 
10331 // Unsigned Integer Divide
10332 
10333 instruct UdivI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10334   match(Set dst (UDivI src1 src2));
10335 
10336   ins_cost(INSN_COST * 19);
10337   format %{ "udivw  $dst, $src1, $src2" %}
10338 
10339   ins_encode %{
10340     __ udivw($dst$$Register, $src1$$Register, $src2$$Register);
10341   %}
10342 
10343   ins_pipe(idiv_reg_reg);
10344 %}
10345 
10346 //  Unsigned Long Divide
10347 
10348 instruct UdivL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10349   match(Set dst (UDivL src1 src2));
10350 
10351   ins_cost(INSN_COST * 35);
10352   format %{ "udiv   $dst, $src1, $src2" %}
10353 
10354   ins_encode %{
10355     __ udiv($dst$$Register, $src1$$Register, $src2$$Register);
10356   %}
10357 
10358   ins_pipe(ldiv_reg_reg);
10359 %}
10360 
10361 // Unsigned Integer Remainder
10362 
10363 instruct UmodI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10364   match(Set dst (UModI src1 src2));
10365 
10366   ins_cost(INSN_COST * 22);
10367   format %{ "udivw  rscratch1, $src1, $src2\n\t"
10368             "msubw  $dst, rscratch1, $src2, $src1" %}
10369 
10370   ins_encode %{
10371     __ udivw(rscratch1, $src1$$Register, $src2$$Register);
10372     __ msubw($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
10373   %}
10374 
10375   ins_pipe(idiv_reg_reg);
10376 %}
10377 
10378 // Unsigned Long Remainder
10379 
10380 instruct UModL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10381   match(Set dst (UModL src1 src2));
10382 
10383   ins_cost(INSN_COST * 38);
10384   format %{ "udiv   rscratch1, $src1, $src2\n"
10385             "msub   $dst, rscratch1, $src2, $src1" %}
10386 
10387   ins_encode %{
10388     __ udiv(rscratch1, $src1$$Register, $src2$$Register);
10389     __ msub($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
10390   %}
10391 
10392   ins_pipe(ldiv_reg_reg);
10393 %}
10394 
10395 // Integer Shifts
10396 
10397 // Shift Left Register
10398 instruct lShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10399   match(Set dst (LShiftI src1 src2));
10400 
10401   ins_cost(INSN_COST * 2);
10402   format %{ "lslvw  $dst, $src1, $src2" %}
10403 
10404   ins_encode %{
10405     __ lslvw(as_Register($dst$$reg),
10406              as_Register($src1$$reg),
10407              as_Register($src2$$reg));
10408   %}
10409 
10410   ins_pipe(ialu_reg_reg_vshift);
10411 %}
10412 
10413 // Shift Left Immediate
10414 instruct lShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10415   match(Set dst (LShiftI src1 src2));
10416 
10417   ins_cost(INSN_COST);
10418   format %{ "lslw $dst, $src1, ($src2 & 0x1f)" %}
10419 
10420   ins_encode %{
10421     __ lslw(as_Register($dst$$reg),
10422             as_Register($src1$$reg),
10423             $src2$$constant & 0x1f);
10424   %}
10425 
10426   ins_pipe(ialu_reg_shift);
10427 %}
10428 
10429 // Shift Right Logical Register
10430 instruct urShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10431   match(Set dst (URShiftI src1 src2));
10432 
10433   ins_cost(INSN_COST * 2);
10434   format %{ "lsrvw  $dst, $src1, $src2" %}
10435 
10436   ins_encode %{
10437     __ lsrvw(as_Register($dst$$reg),
10438              as_Register($src1$$reg),
10439              as_Register($src2$$reg));
10440   %}
10441 
10442   ins_pipe(ialu_reg_reg_vshift);
10443 %}
10444 
10445 // Shift Right Logical Immediate
10446 instruct urShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10447   match(Set dst (URShiftI src1 src2));
10448 
10449   ins_cost(INSN_COST);
10450   format %{ "lsrw $dst, $src1, ($src2 & 0x1f)" %}
10451 
10452   ins_encode %{
10453     __ lsrw(as_Register($dst$$reg),
10454             as_Register($src1$$reg),
10455             $src2$$constant & 0x1f);
10456   %}
10457 
10458   ins_pipe(ialu_reg_shift);
10459 %}
10460 
10461 // Shift Right Arithmetic Register
10462 instruct rShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10463   match(Set dst (RShiftI src1 src2));
10464 
10465   ins_cost(INSN_COST * 2);
10466   format %{ "asrvw  $dst, $src1, $src2" %}
10467 
10468   ins_encode %{
10469     __ asrvw(as_Register($dst$$reg),
10470              as_Register($src1$$reg),
10471              as_Register($src2$$reg));
10472   %}
10473 
10474   ins_pipe(ialu_reg_reg_vshift);
10475 %}
10476 
10477 // Shift Right Arithmetic Immediate
10478 instruct rShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10479   match(Set dst (RShiftI src1 src2));
10480 
10481   ins_cost(INSN_COST);
10482   format %{ "asrw $dst, $src1, ($src2 & 0x1f)" %}
10483 
10484   ins_encode %{
10485     __ asrw(as_Register($dst$$reg),
10486             as_Register($src1$$reg),
10487             $src2$$constant & 0x1f);
10488   %}
10489 
10490   ins_pipe(ialu_reg_shift);
10491 %}
10492 
10493 // Combined Int Mask and Right Shift (using UBFM)
10494 // TODO
10495 
10496 // Long Shifts
10497 
10498 // Shift Left Register
10499 instruct lShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10500   match(Set dst (LShiftL src1 src2));
10501 
10502   ins_cost(INSN_COST * 2);
10503   format %{ "lslv  $dst, $src1, $src2" %}
10504 
10505   ins_encode %{
10506     __ lslv(as_Register($dst$$reg),
10507             as_Register($src1$$reg),
10508             as_Register($src2$$reg));
10509   %}
10510 
10511   ins_pipe(ialu_reg_reg_vshift);
10512 %}
10513 
10514 // Shift Left Immediate
10515 instruct lShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10516   match(Set dst (LShiftL src1 src2));
10517 
10518   ins_cost(INSN_COST);
10519   format %{ "lsl $dst, $src1, ($src2 & 0x3f)" %}
10520 
10521   ins_encode %{
10522     __ lsl(as_Register($dst$$reg),
10523             as_Register($src1$$reg),
10524             $src2$$constant & 0x3f);
10525   %}
10526 
10527   ins_pipe(ialu_reg_shift);
10528 %}
10529 
10530 // Shift Right Logical Register
10531 instruct urShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10532   match(Set dst (URShiftL src1 src2));
10533 
10534   ins_cost(INSN_COST * 2);
10535   format %{ "lsrv  $dst, $src1, $src2" %}
10536 
10537   ins_encode %{
10538     __ lsrv(as_Register($dst$$reg),
10539             as_Register($src1$$reg),
10540             as_Register($src2$$reg));
10541   %}
10542 
10543   ins_pipe(ialu_reg_reg_vshift);
10544 %}
10545 
10546 // Shift Right Logical Immediate
10547 instruct urShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10548   match(Set dst (URShiftL src1 src2));
10549 
10550   ins_cost(INSN_COST);
10551   format %{ "lsr $dst, $src1, ($src2 & 0x3f)" %}
10552 
10553   ins_encode %{
10554     __ lsr(as_Register($dst$$reg),
10555            as_Register($src1$$reg),
10556            $src2$$constant & 0x3f);
10557   %}
10558 
10559   ins_pipe(ialu_reg_shift);
10560 %}
10561 
10562 // A special-case pattern for card table stores.
10563 instruct urShiftP_reg_imm(iRegLNoSp dst, iRegP src1, immI src2) %{
10564   match(Set dst (URShiftL (CastP2X src1) src2));
10565 
10566   ins_cost(INSN_COST);
10567   format %{ "lsr $dst, p2x($src1), ($src2 & 0x3f)" %}
10568 
10569   ins_encode %{
10570     __ lsr(as_Register($dst$$reg),
10571            as_Register($src1$$reg),
10572            $src2$$constant & 0x3f);
10573   %}
10574 
10575   ins_pipe(ialu_reg_shift);
10576 %}
10577 
10578 // Shift Right Arithmetic Register
10579 instruct rShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10580   match(Set dst (RShiftL src1 src2));
10581 
10582   ins_cost(INSN_COST * 2);
10583   format %{ "asrv  $dst, $src1, $src2" %}
10584 
10585   ins_encode %{
10586     __ asrv(as_Register($dst$$reg),
10587             as_Register($src1$$reg),
10588             as_Register($src2$$reg));
10589   %}
10590 
10591   ins_pipe(ialu_reg_reg_vshift);
10592 %}
10593 
10594 // Shift Right Arithmetic Immediate
10595 instruct rShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10596   match(Set dst (RShiftL src1 src2));
10597 
10598   ins_cost(INSN_COST);
10599   format %{ "asr $dst, $src1, ($src2 & 0x3f)" %}
10600 
10601   ins_encode %{
10602     __ asr(as_Register($dst$$reg),
10603            as_Register($src1$$reg),
10604            $src2$$constant & 0x3f);
10605   %}
10606 
10607   ins_pipe(ialu_reg_shift);
10608 %}
10609 
10610 // BEGIN This section of the file is automatically generated. Do not edit --------------
10611 // This section is generated from aarch64_ad.m4
10612 
10613 // This pattern is automatically generated from aarch64_ad.m4
10614 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10615 instruct regL_not_reg(iRegLNoSp dst,
10616                          iRegL src1, immL_M1 m1,
10617                          rFlagsReg cr) %{
10618   match(Set dst (XorL src1 m1));
10619   ins_cost(INSN_COST);
10620   format %{ "eon  $dst, $src1, zr" %}
10621 
10622   ins_encode %{
10623     __ eon(as_Register($dst$$reg),
10624               as_Register($src1$$reg),
10625               zr,
10626               Assembler::LSL, 0);
10627   %}
10628 
10629   ins_pipe(ialu_reg);
10630 %}
10631 
10632 // This pattern is automatically generated from aarch64_ad.m4
10633 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10634 instruct regI_not_reg(iRegINoSp dst,
10635                          iRegIorL2I src1, immI_M1 m1,
10636                          rFlagsReg cr) %{
10637   match(Set dst (XorI src1 m1));
10638   ins_cost(INSN_COST);
10639   format %{ "eonw  $dst, $src1, zr" %}
10640 
10641   ins_encode %{
10642     __ eonw(as_Register($dst$$reg),
10643               as_Register($src1$$reg),
10644               zr,
10645               Assembler::LSL, 0);
10646   %}
10647 
10648   ins_pipe(ialu_reg);
10649 %}
10650 
10651 // This pattern is automatically generated from aarch64_ad.m4
10652 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10653 instruct NegI_reg_URShift_reg(iRegINoSp dst,
10654                               immI0 zero, iRegIorL2I src1, immI src2) %{
10655   match(Set dst (SubI zero (URShiftI src1 src2)));
10656 
10657   ins_cost(1.9 * INSN_COST);
10658   format %{ "negw  $dst, $src1, LSR $src2" %}
10659 
10660   ins_encode %{
10661     __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
10662             Assembler::LSR, $src2$$constant & 0x1f);
10663   %}
10664 
10665   ins_pipe(ialu_reg_shift);
10666 %}
10667 
10668 // This pattern is automatically generated from aarch64_ad.m4
10669 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10670 instruct NegI_reg_RShift_reg(iRegINoSp dst,
10671                               immI0 zero, iRegIorL2I src1, immI src2) %{
10672   match(Set dst (SubI zero (RShiftI src1 src2)));
10673 
10674   ins_cost(1.9 * INSN_COST);
10675   format %{ "negw  $dst, $src1, ASR $src2" %}
10676 
10677   ins_encode %{
10678     __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
10679             Assembler::ASR, $src2$$constant & 0x1f);
10680   %}
10681 
10682   ins_pipe(ialu_reg_shift);
10683 %}
10684 
10685 // This pattern is automatically generated from aarch64_ad.m4
10686 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10687 instruct NegI_reg_LShift_reg(iRegINoSp dst,
10688                               immI0 zero, iRegIorL2I src1, immI src2) %{
10689   match(Set dst (SubI zero (LShiftI src1 src2)));
10690 
10691   ins_cost(1.9 * INSN_COST);
10692   format %{ "negw  $dst, $src1, LSL $src2" %}
10693 
10694   ins_encode %{
10695     __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
10696             Assembler::LSL, $src2$$constant & 0x1f);
10697   %}
10698 
10699   ins_pipe(ialu_reg_shift);
10700 %}
10701 
10702 // This pattern is automatically generated from aarch64_ad.m4
10703 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10704 instruct NegL_reg_URShift_reg(iRegLNoSp dst,
10705                               immL0 zero, iRegL src1, immI src2) %{
10706   match(Set dst (SubL zero (URShiftL src1 src2)));
10707 
10708   ins_cost(1.9 * INSN_COST);
10709   format %{ "neg  $dst, $src1, LSR $src2" %}
10710 
10711   ins_encode %{
10712     __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10713             Assembler::LSR, $src2$$constant & 0x3f);
10714   %}
10715 
10716   ins_pipe(ialu_reg_shift);
10717 %}
10718 
10719 // This pattern is automatically generated from aarch64_ad.m4
10720 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10721 instruct NegL_reg_RShift_reg(iRegLNoSp dst,
10722                               immL0 zero, iRegL src1, immI src2) %{
10723   match(Set dst (SubL zero (RShiftL src1 src2)));
10724 
10725   ins_cost(1.9 * INSN_COST);
10726   format %{ "neg  $dst, $src1, ASR $src2" %}
10727 
10728   ins_encode %{
10729     __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10730             Assembler::ASR, $src2$$constant & 0x3f);
10731   %}
10732 
10733   ins_pipe(ialu_reg_shift);
10734 %}
10735 
10736 // This pattern is automatically generated from aarch64_ad.m4
10737 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10738 instruct NegL_reg_LShift_reg(iRegLNoSp dst,
10739                               immL0 zero, iRegL src1, immI src2) %{
10740   match(Set dst (SubL zero (LShiftL src1 src2)));
10741 
10742   ins_cost(1.9 * INSN_COST);
10743   format %{ "neg  $dst, $src1, LSL $src2" %}
10744 
10745   ins_encode %{
10746     __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10747             Assembler::LSL, $src2$$constant & 0x3f);
10748   %}
10749 
10750   ins_pipe(ialu_reg_shift);
10751 %}
10752 
10753 // This pattern is automatically generated from aarch64_ad.m4
10754 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10755 instruct AndI_reg_not_reg(iRegINoSp dst,
10756                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10757   match(Set dst (AndI src1 (XorI src2 m1)));
10758   ins_cost(INSN_COST);
10759   format %{ "bicw  $dst, $src1, $src2" %}
10760 
10761   ins_encode %{
10762     __ bicw(as_Register($dst$$reg),
10763               as_Register($src1$$reg),
10764               as_Register($src2$$reg),
10765               Assembler::LSL, 0);
10766   %}
10767 
10768   ins_pipe(ialu_reg_reg);
10769 %}
10770 
10771 // This pattern is automatically generated from aarch64_ad.m4
10772 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10773 instruct AndL_reg_not_reg(iRegLNoSp dst,
10774                          iRegL src1, iRegL src2, immL_M1 m1) %{
10775   match(Set dst (AndL src1 (XorL src2 m1)));
10776   ins_cost(INSN_COST);
10777   format %{ "bic  $dst, $src1, $src2" %}
10778 
10779   ins_encode %{
10780     __ bic(as_Register($dst$$reg),
10781               as_Register($src1$$reg),
10782               as_Register($src2$$reg),
10783               Assembler::LSL, 0);
10784   %}
10785 
10786   ins_pipe(ialu_reg_reg);
10787 %}
10788 
10789 // This pattern is automatically generated from aarch64_ad.m4
10790 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10791 instruct OrI_reg_not_reg(iRegINoSp dst,
10792                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10793   match(Set dst (OrI src1 (XorI src2 m1)));
10794   ins_cost(INSN_COST);
10795   format %{ "ornw  $dst, $src1, $src2" %}
10796 
10797   ins_encode %{
10798     __ ornw(as_Register($dst$$reg),
10799               as_Register($src1$$reg),
10800               as_Register($src2$$reg),
10801               Assembler::LSL, 0);
10802   %}
10803 
10804   ins_pipe(ialu_reg_reg);
10805 %}
10806 
10807 // This pattern is automatically generated from aarch64_ad.m4
10808 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10809 instruct OrL_reg_not_reg(iRegLNoSp dst,
10810                          iRegL src1, iRegL src2, immL_M1 m1) %{
10811   match(Set dst (OrL src1 (XorL src2 m1)));
10812   ins_cost(INSN_COST);
10813   format %{ "orn  $dst, $src1, $src2" %}
10814 
10815   ins_encode %{
10816     __ orn(as_Register($dst$$reg),
10817               as_Register($src1$$reg),
10818               as_Register($src2$$reg),
10819               Assembler::LSL, 0);
10820   %}
10821 
10822   ins_pipe(ialu_reg_reg);
10823 %}
10824 
10825 // This pattern is automatically generated from aarch64_ad.m4
10826 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10827 instruct XorI_reg_not_reg(iRegINoSp dst,
10828                          iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10829   match(Set dst (XorI m1 (XorI src2 src1)));
10830   ins_cost(INSN_COST);
10831   format %{ "eonw  $dst, $src1, $src2" %}
10832 
10833   ins_encode %{
10834     __ eonw(as_Register($dst$$reg),
10835               as_Register($src1$$reg),
10836               as_Register($src2$$reg),
10837               Assembler::LSL, 0);
10838   %}
10839 
10840   ins_pipe(ialu_reg_reg);
10841 %}
10842 
10843 // This pattern is automatically generated from aarch64_ad.m4
10844 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10845 instruct XorL_reg_not_reg(iRegLNoSp dst,
10846                          iRegL src1, iRegL src2, immL_M1 m1) %{
10847   match(Set dst (XorL m1 (XorL src2 src1)));
10848   ins_cost(INSN_COST);
10849   format %{ "eon  $dst, $src1, $src2" %}
10850 
10851   ins_encode %{
10852     __ eon(as_Register($dst$$reg),
10853               as_Register($src1$$reg),
10854               as_Register($src2$$reg),
10855               Assembler::LSL, 0);
10856   %}
10857 
10858   ins_pipe(ialu_reg_reg);
10859 %}
10860 
10861 // This pattern is automatically generated from aarch64_ad.m4
10862 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10863 // val & (-1 ^ (val >>> shift)) ==> bicw
10864 instruct AndI_reg_URShift_not_reg(iRegINoSp dst,
10865                          iRegIorL2I src1, iRegIorL2I src2,
10866                          immI src3, immI_M1 src4) %{
10867   match(Set dst (AndI src1 (XorI(URShiftI src2 src3) src4)));
10868   ins_cost(1.9 * INSN_COST);
10869   format %{ "bicw  $dst, $src1, $src2, LSR $src3" %}
10870 
10871   ins_encode %{
10872     __ bicw(as_Register($dst$$reg),
10873               as_Register($src1$$reg),
10874               as_Register($src2$$reg),
10875               Assembler::LSR,
10876               $src3$$constant & 0x1f);
10877   %}
10878 
10879   ins_pipe(ialu_reg_reg_shift);
10880 %}
10881 
10882 // This pattern is automatically generated from aarch64_ad.m4
10883 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10884 // val & (-1 ^ (val >>> shift)) ==> bic
10885 instruct AndL_reg_URShift_not_reg(iRegLNoSp dst,
10886                          iRegL src1, iRegL src2,
10887                          immI src3, immL_M1 src4) %{
10888   match(Set dst (AndL src1 (XorL(URShiftL src2 src3) src4)));
10889   ins_cost(1.9 * INSN_COST);
10890   format %{ "bic  $dst, $src1, $src2, LSR $src3" %}
10891 
10892   ins_encode %{
10893     __ bic(as_Register($dst$$reg),
10894               as_Register($src1$$reg),
10895               as_Register($src2$$reg),
10896               Assembler::LSR,
10897               $src3$$constant & 0x3f);
10898   %}
10899 
10900   ins_pipe(ialu_reg_reg_shift);
10901 %}
10902 
10903 // This pattern is automatically generated from aarch64_ad.m4
10904 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10905 // val & (-1 ^ (val >> shift)) ==> bicw
10906 instruct AndI_reg_RShift_not_reg(iRegINoSp dst,
10907                          iRegIorL2I src1, iRegIorL2I src2,
10908                          immI src3, immI_M1 src4) %{
10909   match(Set dst (AndI src1 (XorI(RShiftI src2 src3) src4)));
10910   ins_cost(1.9 * INSN_COST);
10911   format %{ "bicw  $dst, $src1, $src2, ASR $src3" %}
10912 
10913   ins_encode %{
10914     __ bicw(as_Register($dst$$reg),
10915               as_Register($src1$$reg),
10916               as_Register($src2$$reg),
10917               Assembler::ASR,
10918               $src3$$constant & 0x1f);
10919   %}
10920 
10921   ins_pipe(ialu_reg_reg_shift);
10922 %}
10923 
10924 // This pattern is automatically generated from aarch64_ad.m4
10925 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10926 // val & (-1 ^ (val >> shift)) ==> bic
10927 instruct AndL_reg_RShift_not_reg(iRegLNoSp dst,
10928                          iRegL src1, iRegL src2,
10929                          immI src3, immL_M1 src4) %{
10930   match(Set dst (AndL src1 (XorL(RShiftL src2 src3) src4)));
10931   ins_cost(1.9 * INSN_COST);
10932   format %{ "bic  $dst, $src1, $src2, ASR $src3" %}
10933 
10934   ins_encode %{
10935     __ bic(as_Register($dst$$reg),
10936               as_Register($src1$$reg),
10937               as_Register($src2$$reg),
10938               Assembler::ASR,
10939               $src3$$constant & 0x3f);
10940   %}
10941 
10942   ins_pipe(ialu_reg_reg_shift);
10943 %}
10944 
10945 // This pattern is automatically generated from aarch64_ad.m4
10946 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10947 // val & (-1 ^ (val ror shift)) ==> bicw
10948 instruct AndI_reg_RotateRight_not_reg(iRegINoSp dst,
10949                          iRegIorL2I src1, iRegIorL2I src2,
10950                          immI src3, immI_M1 src4) %{
10951   match(Set dst (AndI src1 (XorI(RotateRight src2 src3) src4)));
10952   ins_cost(1.9 * INSN_COST);
10953   format %{ "bicw  $dst, $src1, $src2, ROR $src3" %}
10954 
10955   ins_encode %{
10956     __ bicw(as_Register($dst$$reg),
10957               as_Register($src1$$reg),
10958               as_Register($src2$$reg),
10959               Assembler::ROR,
10960               $src3$$constant & 0x1f);
10961   %}
10962 
10963   ins_pipe(ialu_reg_reg_shift);
10964 %}
10965 
10966 // This pattern is automatically generated from aarch64_ad.m4
10967 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10968 // val & (-1 ^ (val ror shift)) ==> bic
10969 instruct AndL_reg_RotateRight_not_reg(iRegLNoSp dst,
10970                          iRegL src1, iRegL src2,
10971                          immI src3, immL_M1 src4) %{
10972   match(Set dst (AndL src1 (XorL(RotateRight src2 src3) src4)));
10973   ins_cost(1.9 * INSN_COST);
10974   format %{ "bic  $dst, $src1, $src2, ROR $src3" %}
10975 
10976   ins_encode %{
10977     __ bic(as_Register($dst$$reg),
10978               as_Register($src1$$reg),
10979               as_Register($src2$$reg),
10980               Assembler::ROR,
10981               $src3$$constant & 0x3f);
10982   %}
10983 
10984   ins_pipe(ialu_reg_reg_shift);
10985 %}
10986 
10987 // This pattern is automatically generated from aarch64_ad.m4
10988 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10989 // val & (-1 ^ (val << shift)) ==> bicw
10990 instruct AndI_reg_LShift_not_reg(iRegINoSp dst,
10991                          iRegIorL2I src1, iRegIorL2I src2,
10992                          immI src3, immI_M1 src4) %{
10993   match(Set dst (AndI src1 (XorI(LShiftI src2 src3) src4)));
10994   ins_cost(1.9 * INSN_COST);
10995   format %{ "bicw  $dst, $src1, $src2, LSL $src3" %}
10996 
10997   ins_encode %{
10998     __ bicw(as_Register($dst$$reg),
10999               as_Register($src1$$reg),
11000               as_Register($src2$$reg),
11001               Assembler::LSL,
11002               $src3$$constant & 0x1f);
11003   %}
11004 
11005   ins_pipe(ialu_reg_reg_shift);
11006 %}
11007 
11008 // This pattern is automatically generated from aarch64_ad.m4
11009 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11010 // val & (-1 ^ (val << shift)) ==> bic
11011 instruct AndL_reg_LShift_not_reg(iRegLNoSp dst,
11012                          iRegL src1, iRegL src2,
11013                          immI src3, immL_M1 src4) %{
11014   match(Set dst (AndL src1 (XorL(LShiftL src2 src3) src4)));
11015   ins_cost(1.9 * INSN_COST);
11016   format %{ "bic  $dst, $src1, $src2, LSL $src3" %}
11017 
11018   ins_encode %{
11019     __ bic(as_Register($dst$$reg),
11020               as_Register($src1$$reg),
11021               as_Register($src2$$reg),
11022               Assembler::LSL,
11023               $src3$$constant & 0x3f);
11024   %}
11025 
11026   ins_pipe(ialu_reg_reg_shift);
11027 %}
11028 
11029 // This pattern is automatically generated from aarch64_ad.m4
11030 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11031 // val ^ (-1 ^ (val >>> shift)) ==> eonw
11032 instruct XorI_reg_URShift_not_reg(iRegINoSp dst,
11033                          iRegIorL2I src1, iRegIorL2I src2,
11034                          immI src3, immI_M1 src4) %{
11035   match(Set dst (XorI src4 (XorI(URShiftI src2 src3) src1)));
11036   ins_cost(1.9 * INSN_COST);
11037   format %{ "eonw  $dst, $src1, $src2, LSR $src3" %}
11038 
11039   ins_encode %{
11040     __ eonw(as_Register($dst$$reg),
11041               as_Register($src1$$reg),
11042               as_Register($src2$$reg),
11043               Assembler::LSR,
11044               $src3$$constant & 0x1f);
11045   %}
11046 
11047   ins_pipe(ialu_reg_reg_shift);
11048 %}
11049 
11050 // This pattern is automatically generated from aarch64_ad.m4
11051 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11052 // val ^ (-1 ^ (val >>> shift)) ==> eon
11053 instruct XorL_reg_URShift_not_reg(iRegLNoSp dst,
11054                          iRegL src1, iRegL src2,
11055                          immI src3, immL_M1 src4) %{
11056   match(Set dst (XorL src4 (XorL(URShiftL src2 src3) src1)));
11057   ins_cost(1.9 * INSN_COST);
11058   format %{ "eon  $dst, $src1, $src2, LSR $src3" %}
11059 
11060   ins_encode %{
11061     __ eon(as_Register($dst$$reg),
11062               as_Register($src1$$reg),
11063               as_Register($src2$$reg),
11064               Assembler::LSR,
11065               $src3$$constant & 0x3f);
11066   %}
11067 
11068   ins_pipe(ialu_reg_reg_shift);
11069 %}
11070 
11071 // This pattern is automatically generated from aarch64_ad.m4
11072 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11073 // val ^ (-1 ^ (val >> shift)) ==> eonw
11074 instruct XorI_reg_RShift_not_reg(iRegINoSp dst,
11075                          iRegIorL2I src1, iRegIorL2I src2,
11076                          immI src3, immI_M1 src4) %{
11077   match(Set dst (XorI src4 (XorI(RShiftI src2 src3) src1)));
11078   ins_cost(1.9 * INSN_COST);
11079   format %{ "eonw  $dst, $src1, $src2, ASR $src3" %}
11080 
11081   ins_encode %{
11082     __ eonw(as_Register($dst$$reg),
11083               as_Register($src1$$reg),
11084               as_Register($src2$$reg),
11085               Assembler::ASR,
11086               $src3$$constant & 0x1f);
11087   %}
11088 
11089   ins_pipe(ialu_reg_reg_shift);
11090 %}
11091 
11092 // This pattern is automatically generated from aarch64_ad.m4
11093 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11094 // val ^ (-1 ^ (val >> shift)) ==> eon
11095 instruct XorL_reg_RShift_not_reg(iRegLNoSp dst,
11096                          iRegL src1, iRegL src2,
11097                          immI src3, immL_M1 src4) %{
11098   match(Set dst (XorL src4 (XorL(RShiftL src2 src3) src1)));
11099   ins_cost(1.9 * INSN_COST);
11100   format %{ "eon  $dst, $src1, $src2, ASR $src3" %}
11101 
11102   ins_encode %{
11103     __ eon(as_Register($dst$$reg),
11104               as_Register($src1$$reg),
11105               as_Register($src2$$reg),
11106               Assembler::ASR,
11107               $src3$$constant & 0x3f);
11108   %}
11109 
11110   ins_pipe(ialu_reg_reg_shift);
11111 %}
11112 
11113 // This pattern is automatically generated from aarch64_ad.m4
11114 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11115 // val ^ (-1 ^ (val ror shift)) ==> eonw
11116 instruct XorI_reg_RotateRight_not_reg(iRegINoSp dst,
11117                          iRegIorL2I src1, iRegIorL2I src2,
11118                          immI src3, immI_M1 src4) %{
11119   match(Set dst (XorI src4 (XorI(RotateRight src2 src3) src1)));
11120   ins_cost(1.9 * INSN_COST);
11121   format %{ "eonw  $dst, $src1, $src2, ROR $src3" %}
11122 
11123   ins_encode %{
11124     __ eonw(as_Register($dst$$reg),
11125               as_Register($src1$$reg),
11126               as_Register($src2$$reg),
11127               Assembler::ROR,
11128               $src3$$constant & 0x1f);
11129   %}
11130 
11131   ins_pipe(ialu_reg_reg_shift);
11132 %}
11133 
11134 // This pattern is automatically generated from aarch64_ad.m4
11135 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11136 // val ^ (-1 ^ (val ror shift)) ==> eon
11137 instruct XorL_reg_RotateRight_not_reg(iRegLNoSp dst,
11138                          iRegL src1, iRegL src2,
11139                          immI src3, immL_M1 src4) %{
11140   match(Set dst (XorL src4 (XorL(RotateRight src2 src3) src1)));
11141   ins_cost(1.9 * INSN_COST);
11142   format %{ "eon  $dst, $src1, $src2, ROR $src3" %}
11143 
11144   ins_encode %{
11145     __ eon(as_Register($dst$$reg),
11146               as_Register($src1$$reg),
11147               as_Register($src2$$reg),
11148               Assembler::ROR,
11149               $src3$$constant & 0x3f);
11150   %}
11151 
11152   ins_pipe(ialu_reg_reg_shift);
11153 %}
11154 
11155 // This pattern is automatically generated from aarch64_ad.m4
11156 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11157 // val ^ (-1 ^ (val << shift)) ==> eonw
11158 instruct XorI_reg_LShift_not_reg(iRegINoSp dst,
11159                          iRegIorL2I src1, iRegIorL2I src2,
11160                          immI src3, immI_M1 src4) %{
11161   match(Set dst (XorI src4 (XorI(LShiftI src2 src3) src1)));
11162   ins_cost(1.9 * INSN_COST);
11163   format %{ "eonw  $dst, $src1, $src2, LSL $src3" %}
11164 
11165   ins_encode %{
11166     __ eonw(as_Register($dst$$reg),
11167               as_Register($src1$$reg),
11168               as_Register($src2$$reg),
11169               Assembler::LSL,
11170               $src3$$constant & 0x1f);
11171   %}
11172 
11173   ins_pipe(ialu_reg_reg_shift);
11174 %}
11175 
11176 // This pattern is automatically generated from aarch64_ad.m4
11177 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11178 // val ^ (-1 ^ (val << shift)) ==> eon
11179 instruct XorL_reg_LShift_not_reg(iRegLNoSp dst,
11180                          iRegL src1, iRegL src2,
11181                          immI src3, immL_M1 src4) %{
11182   match(Set dst (XorL src4 (XorL(LShiftL src2 src3) src1)));
11183   ins_cost(1.9 * INSN_COST);
11184   format %{ "eon  $dst, $src1, $src2, LSL $src3" %}
11185 
11186   ins_encode %{
11187     __ eon(as_Register($dst$$reg),
11188               as_Register($src1$$reg),
11189               as_Register($src2$$reg),
11190               Assembler::LSL,
11191               $src3$$constant & 0x3f);
11192   %}
11193 
11194   ins_pipe(ialu_reg_reg_shift);
11195 %}
11196 
11197 // This pattern is automatically generated from aarch64_ad.m4
11198 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11199 // val | (-1 ^ (val >>> shift)) ==> ornw
11200 instruct OrI_reg_URShift_not_reg(iRegINoSp dst,
11201                          iRegIorL2I src1, iRegIorL2I src2,
11202                          immI src3, immI_M1 src4) %{
11203   match(Set dst (OrI src1 (XorI(URShiftI src2 src3) src4)));
11204   ins_cost(1.9 * INSN_COST);
11205   format %{ "ornw  $dst, $src1, $src2, LSR $src3" %}
11206 
11207   ins_encode %{
11208     __ ornw(as_Register($dst$$reg),
11209               as_Register($src1$$reg),
11210               as_Register($src2$$reg),
11211               Assembler::LSR,
11212               $src3$$constant & 0x1f);
11213   %}
11214 
11215   ins_pipe(ialu_reg_reg_shift);
11216 %}
11217 
11218 // This pattern is automatically generated from aarch64_ad.m4
11219 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11220 // val | (-1 ^ (val >>> shift)) ==> orn
11221 instruct OrL_reg_URShift_not_reg(iRegLNoSp dst,
11222                          iRegL src1, iRegL src2,
11223                          immI src3, immL_M1 src4) %{
11224   match(Set dst (OrL src1 (XorL(URShiftL src2 src3) src4)));
11225   ins_cost(1.9 * INSN_COST);
11226   format %{ "orn  $dst, $src1, $src2, LSR $src3" %}
11227 
11228   ins_encode %{
11229     __ orn(as_Register($dst$$reg),
11230               as_Register($src1$$reg),
11231               as_Register($src2$$reg),
11232               Assembler::LSR,
11233               $src3$$constant & 0x3f);
11234   %}
11235 
11236   ins_pipe(ialu_reg_reg_shift);
11237 %}
11238 
11239 // This pattern is automatically generated from aarch64_ad.m4
11240 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11241 // val | (-1 ^ (val >> shift)) ==> ornw
11242 instruct OrI_reg_RShift_not_reg(iRegINoSp dst,
11243                          iRegIorL2I src1, iRegIorL2I src2,
11244                          immI src3, immI_M1 src4) %{
11245   match(Set dst (OrI src1 (XorI(RShiftI src2 src3) src4)));
11246   ins_cost(1.9 * INSN_COST);
11247   format %{ "ornw  $dst, $src1, $src2, ASR $src3" %}
11248 
11249   ins_encode %{
11250     __ ornw(as_Register($dst$$reg),
11251               as_Register($src1$$reg),
11252               as_Register($src2$$reg),
11253               Assembler::ASR,
11254               $src3$$constant & 0x1f);
11255   %}
11256 
11257   ins_pipe(ialu_reg_reg_shift);
11258 %}
11259 
11260 // This pattern is automatically generated from aarch64_ad.m4
11261 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11262 // val | (-1 ^ (val >> shift)) ==> orn
11263 instruct OrL_reg_RShift_not_reg(iRegLNoSp dst,
11264                          iRegL src1, iRegL src2,
11265                          immI src3, immL_M1 src4) %{
11266   match(Set dst (OrL src1 (XorL(RShiftL src2 src3) src4)));
11267   ins_cost(1.9 * INSN_COST);
11268   format %{ "orn  $dst, $src1, $src2, ASR $src3" %}
11269 
11270   ins_encode %{
11271     __ orn(as_Register($dst$$reg),
11272               as_Register($src1$$reg),
11273               as_Register($src2$$reg),
11274               Assembler::ASR,
11275               $src3$$constant & 0x3f);
11276   %}
11277 
11278   ins_pipe(ialu_reg_reg_shift);
11279 %}
11280 
11281 // This pattern is automatically generated from aarch64_ad.m4
11282 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11283 // val | (-1 ^ (val ror shift)) ==> ornw
11284 instruct OrI_reg_RotateRight_not_reg(iRegINoSp dst,
11285                          iRegIorL2I src1, iRegIorL2I src2,
11286                          immI src3, immI_M1 src4) %{
11287   match(Set dst (OrI src1 (XorI(RotateRight src2 src3) src4)));
11288   ins_cost(1.9 * INSN_COST);
11289   format %{ "ornw  $dst, $src1, $src2, ROR $src3" %}
11290 
11291   ins_encode %{
11292     __ ornw(as_Register($dst$$reg),
11293               as_Register($src1$$reg),
11294               as_Register($src2$$reg),
11295               Assembler::ROR,
11296               $src3$$constant & 0x1f);
11297   %}
11298 
11299   ins_pipe(ialu_reg_reg_shift);
11300 %}
11301 
11302 // This pattern is automatically generated from aarch64_ad.m4
11303 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11304 // val | (-1 ^ (val ror shift)) ==> orn
11305 instruct OrL_reg_RotateRight_not_reg(iRegLNoSp dst,
11306                          iRegL src1, iRegL src2,
11307                          immI src3, immL_M1 src4) %{
11308   match(Set dst (OrL src1 (XorL(RotateRight src2 src3) src4)));
11309   ins_cost(1.9 * INSN_COST);
11310   format %{ "orn  $dst, $src1, $src2, ROR $src3" %}
11311 
11312   ins_encode %{
11313     __ orn(as_Register($dst$$reg),
11314               as_Register($src1$$reg),
11315               as_Register($src2$$reg),
11316               Assembler::ROR,
11317               $src3$$constant & 0x3f);
11318   %}
11319 
11320   ins_pipe(ialu_reg_reg_shift);
11321 %}
11322 
11323 // This pattern is automatically generated from aarch64_ad.m4
11324 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11325 // val | (-1 ^ (val << shift)) ==> ornw
11326 instruct OrI_reg_LShift_not_reg(iRegINoSp dst,
11327                          iRegIorL2I src1, iRegIorL2I src2,
11328                          immI src3, immI_M1 src4) %{
11329   match(Set dst (OrI src1 (XorI(LShiftI src2 src3) src4)));
11330   ins_cost(1.9 * INSN_COST);
11331   format %{ "ornw  $dst, $src1, $src2, LSL $src3" %}
11332 
11333   ins_encode %{
11334     __ ornw(as_Register($dst$$reg),
11335               as_Register($src1$$reg),
11336               as_Register($src2$$reg),
11337               Assembler::LSL,
11338               $src3$$constant & 0x1f);
11339   %}
11340 
11341   ins_pipe(ialu_reg_reg_shift);
11342 %}
11343 
11344 // This pattern is automatically generated from aarch64_ad.m4
11345 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11346 // val | (-1 ^ (val << shift)) ==> orn
11347 instruct OrL_reg_LShift_not_reg(iRegLNoSp dst,
11348                          iRegL src1, iRegL src2,
11349                          immI src3, immL_M1 src4) %{
11350   match(Set dst (OrL src1 (XorL(LShiftL src2 src3) src4)));
11351   ins_cost(1.9 * INSN_COST);
11352   format %{ "orn  $dst, $src1, $src2, LSL $src3" %}
11353 
11354   ins_encode %{
11355     __ orn(as_Register($dst$$reg),
11356               as_Register($src1$$reg),
11357               as_Register($src2$$reg),
11358               Assembler::LSL,
11359               $src3$$constant & 0x3f);
11360   %}
11361 
11362   ins_pipe(ialu_reg_reg_shift);
11363 %}
11364 
11365 // This pattern is automatically generated from aarch64_ad.m4
11366 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11367 instruct AndI_reg_URShift_reg(iRegINoSp dst,
11368                          iRegIorL2I src1, iRegIorL2I src2,
11369                          immI src3) %{
11370   match(Set dst (AndI src1 (URShiftI src2 src3)));
11371 
11372   ins_cost(1.9 * INSN_COST);
11373   format %{ "andw  $dst, $src1, $src2, LSR $src3" %}
11374 
11375   ins_encode %{
11376     __ andw(as_Register($dst$$reg),
11377               as_Register($src1$$reg),
11378               as_Register($src2$$reg),
11379               Assembler::LSR,
11380               $src3$$constant & 0x1f);
11381   %}
11382 
11383   ins_pipe(ialu_reg_reg_shift);
11384 %}
11385 
11386 // This pattern is automatically generated from aarch64_ad.m4
11387 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11388 instruct AndL_reg_URShift_reg(iRegLNoSp dst,
11389                          iRegL src1, iRegL src2,
11390                          immI src3) %{
11391   match(Set dst (AndL src1 (URShiftL src2 src3)));
11392 
11393   ins_cost(1.9 * INSN_COST);
11394   format %{ "andr  $dst, $src1, $src2, LSR $src3" %}
11395 
11396   ins_encode %{
11397     __ andr(as_Register($dst$$reg),
11398               as_Register($src1$$reg),
11399               as_Register($src2$$reg),
11400               Assembler::LSR,
11401               $src3$$constant & 0x3f);
11402   %}
11403 
11404   ins_pipe(ialu_reg_reg_shift);
11405 %}
11406 
11407 // This pattern is automatically generated from aarch64_ad.m4
11408 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11409 instruct AndI_reg_RShift_reg(iRegINoSp dst,
11410                          iRegIorL2I src1, iRegIorL2I src2,
11411                          immI src3) %{
11412   match(Set dst (AndI src1 (RShiftI src2 src3)));
11413 
11414   ins_cost(1.9 * INSN_COST);
11415   format %{ "andw  $dst, $src1, $src2, ASR $src3" %}
11416 
11417   ins_encode %{
11418     __ andw(as_Register($dst$$reg),
11419               as_Register($src1$$reg),
11420               as_Register($src2$$reg),
11421               Assembler::ASR,
11422               $src3$$constant & 0x1f);
11423   %}
11424 
11425   ins_pipe(ialu_reg_reg_shift);
11426 %}
11427 
11428 // This pattern is automatically generated from aarch64_ad.m4
11429 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11430 instruct AndL_reg_RShift_reg(iRegLNoSp dst,
11431                          iRegL src1, iRegL src2,
11432                          immI src3) %{
11433   match(Set dst (AndL src1 (RShiftL src2 src3)));
11434 
11435   ins_cost(1.9 * INSN_COST);
11436   format %{ "andr  $dst, $src1, $src2, ASR $src3" %}
11437 
11438   ins_encode %{
11439     __ andr(as_Register($dst$$reg),
11440               as_Register($src1$$reg),
11441               as_Register($src2$$reg),
11442               Assembler::ASR,
11443               $src3$$constant & 0x3f);
11444   %}
11445 
11446   ins_pipe(ialu_reg_reg_shift);
11447 %}
11448 
11449 // This pattern is automatically generated from aarch64_ad.m4
11450 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11451 instruct AndI_reg_LShift_reg(iRegINoSp dst,
11452                          iRegIorL2I src1, iRegIorL2I src2,
11453                          immI src3) %{
11454   match(Set dst (AndI src1 (LShiftI src2 src3)));
11455 
11456   ins_cost(1.9 * INSN_COST);
11457   format %{ "andw  $dst, $src1, $src2, LSL $src3" %}
11458 
11459   ins_encode %{
11460     __ andw(as_Register($dst$$reg),
11461               as_Register($src1$$reg),
11462               as_Register($src2$$reg),
11463               Assembler::LSL,
11464               $src3$$constant & 0x1f);
11465   %}
11466 
11467   ins_pipe(ialu_reg_reg_shift);
11468 %}
11469 
11470 // This pattern is automatically generated from aarch64_ad.m4
11471 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11472 instruct AndL_reg_LShift_reg(iRegLNoSp dst,
11473                          iRegL src1, iRegL src2,
11474                          immI src3) %{
11475   match(Set dst (AndL src1 (LShiftL src2 src3)));
11476 
11477   ins_cost(1.9 * INSN_COST);
11478   format %{ "andr  $dst, $src1, $src2, LSL $src3" %}
11479 
11480   ins_encode %{
11481     __ andr(as_Register($dst$$reg),
11482               as_Register($src1$$reg),
11483               as_Register($src2$$reg),
11484               Assembler::LSL,
11485               $src3$$constant & 0x3f);
11486   %}
11487 
11488   ins_pipe(ialu_reg_reg_shift);
11489 %}
11490 
11491 // This pattern is automatically generated from aarch64_ad.m4
11492 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11493 instruct AndI_reg_RotateRight_reg(iRegINoSp dst,
11494                          iRegIorL2I src1, iRegIorL2I src2,
11495                          immI src3) %{
11496   match(Set dst (AndI src1 (RotateRight src2 src3)));
11497 
11498   ins_cost(1.9 * INSN_COST);
11499   format %{ "andw  $dst, $src1, $src2, ROR $src3" %}
11500 
11501   ins_encode %{
11502     __ andw(as_Register($dst$$reg),
11503               as_Register($src1$$reg),
11504               as_Register($src2$$reg),
11505               Assembler::ROR,
11506               $src3$$constant & 0x1f);
11507   %}
11508 
11509   ins_pipe(ialu_reg_reg_shift);
11510 %}
11511 
11512 // This pattern is automatically generated from aarch64_ad.m4
11513 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11514 instruct AndL_reg_RotateRight_reg(iRegLNoSp dst,
11515                          iRegL src1, iRegL src2,
11516                          immI src3) %{
11517   match(Set dst (AndL src1 (RotateRight src2 src3)));
11518 
11519   ins_cost(1.9 * INSN_COST);
11520   format %{ "andr  $dst, $src1, $src2, ROR $src3" %}
11521 
11522   ins_encode %{
11523     __ andr(as_Register($dst$$reg),
11524               as_Register($src1$$reg),
11525               as_Register($src2$$reg),
11526               Assembler::ROR,
11527               $src3$$constant & 0x3f);
11528   %}
11529 
11530   ins_pipe(ialu_reg_reg_shift);
11531 %}
11532 
11533 // This pattern is automatically generated from aarch64_ad.m4
11534 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11535 instruct XorI_reg_URShift_reg(iRegINoSp dst,
11536                          iRegIorL2I src1, iRegIorL2I src2,
11537                          immI src3) %{
11538   match(Set dst (XorI src1 (URShiftI src2 src3)));
11539 
11540   ins_cost(1.9 * INSN_COST);
11541   format %{ "eorw  $dst, $src1, $src2, LSR $src3" %}
11542 
11543   ins_encode %{
11544     __ eorw(as_Register($dst$$reg),
11545               as_Register($src1$$reg),
11546               as_Register($src2$$reg),
11547               Assembler::LSR,
11548               $src3$$constant & 0x1f);
11549   %}
11550 
11551   ins_pipe(ialu_reg_reg_shift);
11552 %}
11553 
11554 // This pattern is automatically generated from aarch64_ad.m4
11555 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11556 instruct XorL_reg_URShift_reg(iRegLNoSp dst,
11557                          iRegL src1, iRegL src2,
11558                          immI src3) %{
11559   match(Set dst (XorL src1 (URShiftL src2 src3)));
11560 
11561   ins_cost(1.9 * INSN_COST);
11562   format %{ "eor  $dst, $src1, $src2, LSR $src3" %}
11563 
11564   ins_encode %{
11565     __ eor(as_Register($dst$$reg),
11566               as_Register($src1$$reg),
11567               as_Register($src2$$reg),
11568               Assembler::LSR,
11569               $src3$$constant & 0x3f);
11570   %}
11571 
11572   ins_pipe(ialu_reg_reg_shift);
11573 %}
11574 
11575 // This pattern is automatically generated from aarch64_ad.m4
11576 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11577 instruct XorI_reg_RShift_reg(iRegINoSp dst,
11578                          iRegIorL2I src1, iRegIorL2I src2,
11579                          immI src3) %{
11580   match(Set dst (XorI src1 (RShiftI src2 src3)));
11581 
11582   ins_cost(1.9 * INSN_COST);
11583   format %{ "eorw  $dst, $src1, $src2, ASR $src3" %}
11584 
11585   ins_encode %{
11586     __ eorw(as_Register($dst$$reg),
11587               as_Register($src1$$reg),
11588               as_Register($src2$$reg),
11589               Assembler::ASR,
11590               $src3$$constant & 0x1f);
11591   %}
11592 
11593   ins_pipe(ialu_reg_reg_shift);
11594 %}
11595 
11596 // This pattern is automatically generated from aarch64_ad.m4
11597 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11598 instruct XorL_reg_RShift_reg(iRegLNoSp dst,
11599                          iRegL src1, iRegL src2,
11600                          immI src3) %{
11601   match(Set dst (XorL src1 (RShiftL src2 src3)));
11602 
11603   ins_cost(1.9 * INSN_COST);
11604   format %{ "eor  $dst, $src1, $src2, ASR $src3" %}
11605 
11606   ins_encode %{
11607     __ eor(as_Register($dst$$reg),
11608               as_Register($src1$$reg),
11609               as_Register($src2$$reg),
11610               Assembler::ASR,
11611               $src3$$constant & 0x3f);
11612   %}
11613 
11614   ins_pipe(ialu_reg_reg_shift);
11615 %}
11616 
11617 // This pattern is automatically generated from aarch64_ad.m4
11618 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11619 instruct XorI_reg_LShift_reg(iRegINoSp dst,
11620                          iRegIorL2I src1, iRegIorL2I src2,
11621                          immI src3) %{
11622   match(Set dst (XorI src1 (LShiftI src2 src3)));
11623 
11624   ins_cost(1.9 * INSN_COST);
11625   format %{ "eorw  $dst, $src1, $src2, LSL $src3" %}
11626 
11627   ins_encode %{
11628     __ eorw(as_Register($dst$$reg),
11629               as_Register($src1$$reg),
11630               as_Register($src2$$reg),
11631               Assembler::LSL,
11632               $src3$$constant & 0x1f);
11633   %}
11634 
11635   ins_pipe(ialu_reg_reg_shift);
11636 %}
11637 
11638 // This pattern is automatically generated from aarch64_ad.m4
11639 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11640 instruct XorL_reg_LShift_reg(iRegLNoSp dst,
11641                          iRegL src1, iRegL src2,
11642                          immI src3) %{
11643   match(Set dst (XorL src1 (LShiftL src2 src3)));
11644 
11645   ins_cost(1.9 * INSN_COST);
11646   format %{ "eor  $dst, $src1, $src2, LSL $src3" %}
11647 
11648   ins_encode %{
11649     __ eor(as_Register($dst$$reg),
11650               as_Register($src1$$reg),
11651               as_Register($src2$$reg),
11652               Assembler::LSL,
11653               $src3$$constant & 0x3f);
11654   %}
11655 
11656   ins_pipe(ialu_reg_reg_shift);
11657 %}
11658 
11659 // This pattern is automatically generated from aarch64_ad.m4
11660 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11661 instruct XorI_reg_RotateRight_reg(iRegINoSp dst,
11662                          iRegIorL2I src1, iRegIorL2I src2,
11663                          immI src3) %{
11664   match(Set dst (XorI src1 (RotateRight src2 src3)));
11665 
11666   ins_cost(1.9 * INSN_COST);
11667   format %{ "eorw  $dst, $src1, $src2, ROR $src3" %}
11668 
11669   ins_encode %{
11670     __ eorw(as_Register($dst$$reg),
11671               as_Register($src1$$reg),
11672               as_Register($src2$$reg),
11673               Assembler::ROR,
11674               $src3$$constant & 0x1f);
11675   %}
11676 
11677   ins_pipe(ialu_reg_reg_shift);
11678 %}
11679 
11680 // This pattern is automatically generated from aarch64_ad.m4
11681 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11682 instruct XorL_reg_RotateRight_reg(iRegLNoSp dst,
11683                          iRegL src1, iRegL src2,
11684                          immI src3) %{
11685   match(Set dst (XorL src1 (RotateRight src2 src3)));
11686 
11687   ins_cost(1.9 * INSN_COST);
11688   format %{ "eor  $dst, $src1, $src2, ROR $src3" %}
11689 
11690   ins_encode %{
11691     __ eor(as_Register($dst$$reg),
11692               as_Register($src1$$reg),
11693               as_Register($src2$$reg),
11694               Assembler::ROR,
11695               $src3$$constant & 0x3f);
11696   %}
11697 
11698   ins_pipe(ialu_reg_reg_shift);
11699 %}
11700 
11701 // This pattern is automatically generated from aarch64_ad.m4
11702 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11703 instruct OrI_reg_URShift_reg(iRegINoSp dst,
11704                          iRegIorL2I src1, iRegIorL2I src2,
11705                          immI src3) %{
11706   match(Set dst (OrI src1 (URShiftI src2 src3)));
11707 
11708   ins_cost(1.9 * INSN_COST);
11709   format %{ "orrw  $dst, $src1, $src2, LSR $src3" %}
11710 
11711   ins_encode %{
11712     __ orrw(as_Register($dst$$reg),
11713               as_Register($src1$$reg),
11714               as_Register($src2$$reg),
11715               Assembler::LSR,
11716               $src3$$constant & 0x1f);
11717   %}
11718 
11719   ins_pipe(ialu_reg_reg_shift);
11720 %}
11721 
11722 // This pattern is automatically generated from aarch64_ad.m4
11723 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11724 instruct OrL_reg_URShift_reg(iRegLNoSp dst,
11725                          iRegL src1, iRegL src2,
11726                          immI src3) %{
11727   match(Set dst (OrL src1 (URShiftL src2 src3)));
11728 
11729   ins_cost(1.9 * INSN_COST);
11730   format %{ "orr  $dst, $src1, $src2, LSR $src3" %}
11731 
11732   ins_encode %{
11733     __ orr(as_Register($dst$$reg),
11734               as_Register($src1$$reg),
11735               as_Register($src2$$reg),
11736               Assembler::LSR,
11737               $src3$$constant & 0x3f);
11738   %}
11739 
11740   ins_pipe(ialu_reg_reg_shift);
11741 %}
11742 
11743 // This pattern is automatically generated from aarch64_ad.m4
11744 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11745 instruct OrI_reg_RShift_reg(iRegINoSp dst,
11746                          iRegIorL2I src1, iRegIorL2I src2,
11747                          immI src3) %{
11748   match(Set dst (OrI src1 (RShiftI src2 src3)));
11749 
11750   ins_cost(1.9 * INSN_COST);
11751   format %{ "orrw  $dst, $src1, $src2, ASR $src3" %}
11752 
11753   ins_encode %{
11754     __ orrw(as_Register($dst$$reg),
11755               as_Register($src1$$reg),
11756               as_Register($src2$$reg),
11757               Assembler::ASR,
11758               $src3$$constant & 0x1f);
11759   %}
11760 
11761   ins_pipe(ialu_reg_reg_shift);
11762 %}
11763 
11764 // This pattern is automatically generated from aarch64_ad.m4
11765 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11766 instruct OrL_reg_RShift_reg(iRegLNoSp dst,
11767                          iRegL src1, iRegL src2,
11768                          immI src3) %{
11769   match(Set dst (OrL src1 (RShiftL src2 src3)));
11770 
11771   ins_cost(1.9 * INSN_COST);
11772   format %{ "orr  $dst, $src1, $src2, ASR $src3" %}
11773 
11774   ins_encode %{
11775     __ orr(as_Register($dst$$reg),
11776               as_Register($src1$$reg),
11777               as_Register($src2$$reg),
11778               Assembler::ASR,
11779               $src3$$constant & 0x3f);
11780   %}
11781 
11782   ins_pipe(ialu_reg_reg_shift);
11783 %}
11784 
11785 // This pattern is automatically generated from aarch64_ad.m4
11786 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11787 instruct OrI_reg_LShift_reg(iRegINoSp dst,
11788                          iRegIorL2I src1, iRegIorL2I src2,
11789                          immI src3) %{
11790   match(Set dst (OrI src1 (LShiftI src2 src3)));
11791 
11792   ins_cost(1.9 * INSN_COST);
11793   format %{ "orrw  $dst, $src1, $src2, LSL $src3" %}
11794 
11795   ins_encode %{
11796     __ orrw(as_Register($dst$$reg),
11797               as_Register($src1$$reg),
11798               as_Register($src2$$reg),
11799               Assembler::LSL,
11800               $src3$$constant & 0x1f);
11801   %}
11802 
11803   ins_pipe(ialu_reg_reg_shift);
11804 %}
11805 
11806 // This pattern is automatically generated from aarch64_ad.m4
11807 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11808 instruct OrL_reg_LShift_reg(iRegLNoSp dst,
11809                          iRegL src1, iRegL src2,
11810                          immI src3) %{
11811   match(Set dst (OrL src1 (LShiftL src2 src3)));
11812 
11813   ins_cost(1.9 * INSN_COST);
11814   format %{ "orr  $dst, $src1, $src2, LSL $src3" %}
11815 
11816   ins_encode %{
11817     __ orr(as_Register($dst$$reg),
11818               as_Register($src1$$reg),
11819               as_Register($src2$$reg),
11820               Assembler::LSL,
11821               $src3$$constant & 0x3f);
11822   %}
11823 
11824   ins_pipe(ialu_reg_reg_shift);
11825 %}
11826 
11827 // This pattern is automatically generated from aarch64_ad.m4
11828 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11829 instruct OrI_reg_RotateRight_reg(iRegINoSp dst,
11830                          iRegIorL2I src1, iRegIorL2I src2,
11831                          immI src3) %{
11832   match(Set dst (OrI src1 (RotateRight src2 src3)));
11833 
11834   ins_cost(1.9 * INSN_COST);
11835   format %{ "orrw  $dst, $src1, $src2, ROR $src3" %}
11836 
11837   ins_encode %{
11838     __ orrw(as_Register($dst$$reg),
11839               as_Register($src1$$reg),
11840               as_Register($src2$$reg),
11841               Assembler::ROR,
11842               $src3$$constant & 0x1f);
11843   %}
11844 
11845   ins_pipe(ialu_reg_reg_shift);
11846 %}
11847 
11848 // This pattern is automatically generated from aarch64_ad.m4
11849 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11850 instruct OrL_reg_RotateRight_reg(iRegLNoSp dst,
11851                          iRegL src1, iRegL src2,
11852                          immI src3) %{
11853   match(Set dst (OrL src1 (RotateRight src2 src3)));
11854 
11855   ins_cost(1.9 * INSN_COST);
11856   format %{ "orr  $dst, $src1, $src2, ROR $src3" %}
11857 
11858   ins_encode %{
11859     __ orr(as_Register($dst$$reg),
11860               as_Register($src1$$reg),
11861               as_Register($src2$$reg),
11862               Assembler::ROR,
11863               $src3$$constant & 0x3f);
11864   %}
11865 
11866   ins_pipe(ialu_reg_reg_shift);
11867 %}
11868 
11869 // This pattern is automatically generated from aarch64_ad.m4
11870 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11871 instruct AddI_reg_URShift_reg(iRegINoSp dst,
11872                          iRegIorL2I src1, iRegIorL2I src2,
11873                          immI src3) %{
11874   match(Set dst (AddI src1 (URShiftI src2 src3)));
11875 
11876   ins_cost(1.9 * INSN_COST);
11877   format %{ "addw  $dst, $src1, $src2, LSR $src3" %}
11878 
11879   ins_encode %{
11880     __ addw(as_Register($dst$$reg),
11881               as_Register($src1$$reg),
11882               as_Register($src2$$reg),
11883               Assembler::LSR,
11884               $src3$$constant & 0x1f);
11885   %}
11886 
11887   ins_pipe(ialu_reg_reg_shift);
11888 %}
11889 
11890 // This pattern is automatically generated from aarch64_ad.m4
11891 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11892 instruct AddL_reg_URShift_reg(iRegLNoSp dst,
11893                          iRegL src1, iRegL src2,
11894                          immI src3) %{
11895   match(Set dst (AddL src1 (URShiftL src2 src3)));
11896 
11897   ins_cost(1.9 * INSN_COST);
11898   format %{ "add  $dst, $src1, $src2, LSR $src3" %}
11899 
11900   ins_encode %{
11901     __ add(as_Register($dst$$reg),
11902               as_Register($src1$$reg),
11903               as_Register($src2$$reg),
11904               Assembler::LSR,
11905               $src3$$constant & 0x3f);
11906   %}
11907 
11908   ins_pipe(ialu_reg_reg_shift);
11909 %}
11910 
11911 // This pattern is automatically generated from aarch64_ad.m4
11912 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11913 instruct AddI_reg_RShift_reg(iRegINoSp dst,
11914                          iRegIorL2I src1, iRegIorL2I src2,
11915                          immI src3) %{
11916   match(Set dst (AddI src1 (RShiftI src2 src3)));
11917 
11918   ins_cost(1.9 * INSN_COST);
11919   format %{ "addw  $dst, $src1, $src2, ASR $src3" %}
11920 
11921   ins_encode %{
11922     __ addw(as_Register($dst$$reg),
11923               as_Register($src1$$reg),
11924               as_Register($src2$$reg),
11925               Assembler::ASR,
11926               $src3$$constant & 0x1f);
11927   %}
11928 
11929   ins_pipe(ialu_reg_reg_shift);
11930 %}
11931 
11932 // This pattern is automatically generated from aarch64_ad.m4
11933 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11934 instruct AddL_reg_RShift_reg(iRegLNoSp dst,
11935                          iRegL src1, iRegL src2,
11936                          immI src3) %{
11937   match(Set dst (AddL src1 (RShiftL src2 src3)));
11938 
11939   ins_cost(1.9 * INSN_COST);
11940   format %{ "add  $dst, $src1, $src2, ASR $src3" %}
11941 
11942   ins_encode %{
11943     __ add(as_Register($dst$$reg),
11944               as_Register($src1$$reg),
11945               as_Register($src2$$reg),
11946               Assembler::ASR,
11947               $src3$$constant & 0x3f);
11948   %}
11949 
11950   ins_pipe(ialu_reg_reg_shift);
11951 %}
11952 
11953 // This pattern is automatically generated from aarch64_ad.m4
11954 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11955 instruct AddI_reg_LShift_reg(iRegINoSp dst,
11956                          iRegIorL2I src1, iRegIorL2I src2,
11957                          immI src3) %{
11958   match(Set dst (AddI src1 (LShiftI src2 src3)));
11959 
11960   ins_cost(1.9 * INSN_COST);
11961   format %{ "addw  $dst, $src1, $src2, LSL $src3" %}
11962 
11963   ins_encode %{
11964     __ addw(as_Register($dst$$reg),
11965               as_Register($src1$$reg),
11966               as_Register($src2$$reg),
11967               Assembler::LSL,
11968               $src3$$constant & 0x1f);
11969   %}
11970 
11971   ins_pipe(ialu_reg_reg_shift);
11972 %}
11973 
11974 // This pattern is automatically generated from aarch64_ad.m4
11975 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11976 instruct AddL_reg_LShift_reg(iRegLNoSp dst,
11977                          iRegL src1, iRegL src2,
11978                          immI src3) %{
11979   match(Set dst (AddL src1 (LShiftL src2 src3)));
11980 
11981   ins_cost(1.9 * INSN_COST);
11982   format %{ "add  $dst, $src1, $src2, LSL $src3" %}
11983 
11984   ins_encode %{
11985     __ add(as_Register($dst$$reg),
11986               as_Register($src1$$reg),
11987               as_Register($src2$$reg),
11988               Assembler::LSL,
11989               $src3$$constant & 0x3f);
11990   %}
11991 
11992   ins_pipe(ialu_reg_reg_shift);
11993 %}
11994 
11995 // This pattern is automatically generated from aarch64_ad.m4
11996 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11997 instruct SubI_reg_URShift_reg(iRegINoSp dst,
11998                          iRegIorL2I src1, iRegIorL2I src2,
11999                          immI src3) %{
12000   match(Set dst (SubI src1 (URShiftI src2 src3)));
12001 
12002   ins_cost(1.9 * INSN_COST);
12003   format %{ "subw  $dst, $src1, $src2, LSR $src3" %}
12004 
12005   ins_encode %{
12006     __ subw(as_Register($dst$$reg),
12007               as_Register($src1$$reg),
12008               as_Register($src2$$reg),
12009               Assembler::LSR,
12010               $src3$$constant & 0x1f);
12011   %}
12012 
12013   ins_pipe(ialu_reg_reg_shift);
12014 %}
12015 
12016 // This pattern is automatically generated from aarch64_ad.m4
12017 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12018 instruct SubL_reg_URShift_reg(iRegLNoSp dst,
12019                          iRegL src1, iRegL src2,
12020                          immI src3) %{
12021   match(Set dst (SubL src1 (URShiftL src2 src3)));
12022 
12023   ins_cost(1.9 * INSN_COST);
12024   format %{ "sub  $dst, $src1, $src2, LSR $src3" %}
12025 
12026   ins_encode %{
12027     __ sub(as_Register($dst$$reg),
12028               as_Register($src1$$reg),
12029               as_Register($src2$$reg),
12030               Assembler::LSR,
12031               $src3$$constant & 0x3f);
12032   %}
12033 
12034   ins_pipe(ialu_reg_reg_shift);
12035 %}
12036 
12037 // This pattern is automatically generated from aarch64_ad.m4
12038 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12039 instruct SubI_reg_RShift_reg(iRegINoSp dst,
12040                          iRegIorL2I src1, iRegIorL2I src2,
12041                          immI src3) %{
12042   match(Set dst (SubI src1 (RShiftI src2 src3)));
12043 
12044   ins_cost(1.9 * INSN_COST);
12045   format %{ "subw  $dst, $src1, $src2, ASR $src3" %}
12046 
12047   ins_encode %{
12048     __ subw(as_Register($dst$$reg),
12049               as_Register($src1$$reg),
12050               as_Register($src2$$reg),
12051               Assembler::ASR,
12052               $src3$$constant & 0x1f);
12053   %}
12054 
12055   ins_pipe(ialu_reg_reg_shift);
12056 %}
12057 
12058 // This pattern is automatically generated from aarch64_ad.m4
12059 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12060 instruct SubL_reg_RShift_reg(iRegLNoSp dst,
12061                          iRegL src1, iRegL src2,
12062                          immI src3) %{
12063   match(Set dst (SubL src1 (RShiftL src2 src3)));
12064 
12065   ins_cost(1.9 * INSN_COST);
12066   format %{ "sub  $dst, $src1, $src2, ASR $src3" %}
12067 
12068   ins_encode %{
12069     __ sub(as_Register($dst$$reg),
12070               as_Register($src1$$reg),
12071               as_Register($src2$$reg),
12072               Assembler::ASR,
12073               $src3$$constant & 0x3f);
12074   %}
12075 
12076   ins_pipe(ialu_reg_reg_shift);
12077 %}
12078 
12079 // This pattern is automatically generated from aarch64_ad.m4
12080 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12081 instruct SubI_reg_LShift_reg(iRegINoSp dst,
12082                          iRegIorL2I src1, iRegIorL2I src2,
12083                          immI src3) %{
12084   match(Set dst (SubI src1 (LShiftI src2 src3)));
12085 
12086   ins_cost(1.9 * INSN_COST);
12087   format %{ "subw  $dst, $src1, $src2, LSL $src3" %}
12088 
12089   ins_encode %{
12090     __ subw(as_Register($dst$$reg),
12091               as_Register($src1$$reg),
12092               as_Register($src2$$reg),
12093               Assembler::LSL,
12094               $src3$$constant & 0x1f);
12095   %}
12096 
12097   ins_pipe(ialu_reg_reg_shift);
12098 %}
12099 
12100 // This pattern is automatically generated from aarch64_ad.m4
12101 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12102 instruct SubL_reg_LShift_reg(iRegLNoSp dst,
12103                          iRegL src1, iRegL src2,
12104                          immI src3) %{
12105   match(Set dst (SubL src1 (LShiftL src2 src3)));
12106 
12107   ins_cost(1.9 * INSN_COST);
12108   format %{ "sub  $dst, $src1, $src2, LSL $src3" %}
12109 
12110   ins_encode %{
12111     __ sub(as_Register($dst$$reg),
12112               as_Register($src1$$reg),
12113               as_Register($src2$$reg),
12114               Assembler::LSL,
12115               $src3$$constant & 0x3f);
12116   %}
12117 
12118   ins_pipe(ialu_reg_reg_shift);
12119 %}
12120 
12121 // This pattern is automatically generated from aarch64_ad.m4
12122 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12123 
12124 // Shift Left followed by Shift Right.
12125 // This idiom is used by the compiler for the i2b bytecode etc.
12126 instruct sbfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12127 %{
12128   match(Set dst (RShiftL (LShiftL src lshift_count) rshift_count));
12129   ins_cost(INSN_COST * 2);
12130   format %{ "sbfm  $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12131   ins_encode %{
12132     int lshift = $lshift_count$$constant & 63;
12133     int rshift = $rshift_count$$constant & 63;
12134     int s = 63 - lshift;
12135     int r = (rshift - lshift) & 63;
12136     __ sbfm(as_Register($dst$$reg),
12137             as_Register($src$$reg),
12138             r, s);
12139   %}
12140 
12141   ins_pipe(ialu_reg_shift);
12142 %}
12143 
12144 // This pattern is automatically generated from aarch64_ad.m4
12145 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12146 
12147 // Shift Left followed by Shift Right.
12148 // This idiom is used by the compiler for the i2b bytecode etc.
12149 instruct sbfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12150 %{
12151   match(Set dst (RShiftI (LShiftI src lshift_count) rshift_count));
12152   ins_cost(INSN_COST * 2);
12153   format %{ "sbfmw  $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12154   ins_encode %{
12155     int lshift = $lshift_count$$constant & 31;
12156     int rshift = $rshift_count$$constant & 31;
12157     int s = 31 - lshift;
12158     int r = (rshift - lshift) & 31;
12159     __ sbfmw(as_Register($dst$$reg),
12160             as_Register($src$$reg),
12161             r, s);
12162   %}
12163 
12164   ins_pipe(ialu_reg_shift);
12165 %}
12166 
12167 // This pattern is automatically generated from aarch64_ad.m4
12168 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12169 
12170 // Shift Left followed by Shift Right.
12171 // This idiom is used by the compiler for the i2b bytecode etc.
12172 instruct ubfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12173 %{
12174   match(Set dst (URShiftL (LShiftL src lshift_count) rshift_count));
12175   ins_cost(INSN_COST * 2);
12176   format %{ "ubfm  $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12177   ins_encode %{
12178     int lshift = $lshift_count$$constant & 63;
12179     int rshift = $rshift_count$$constant & 63;
12180     int s = 63 - lshift;
12181     int r = (rshift - lshift) & 63;
12182     __ ubfm(as_Register($dst$$reg),
12183             as_Register($src$$reg),
12184             r, s);
12185   %}
12186 
12187   ins_pipe(ialu_reg_shift);
12188 %}
12189 
12190 // This pattern is automatically generated from aarch64_ad.m4
12191 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12192 
12193 // Shift Left followed by Shift Right.
12194 // This idiom is used by the compiler for the i2b bytecode etc.
12195 instruct ubfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12196 %{
12197   match(Set dst (URShiftI (LShiftI src lshift_count) rshift_count));
12198   ins_cost(INSN_COST * 2);
12199   format %{ "ubfmw  $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12200   ins_encode %{
12201     int lshift = $lshift_count$$constant & 31;
12202     int rshift = $rshift_count$$constant & 31;
12203     int s = 31 - lshift;
12204     int r = (rshift - lshift) & 31;
12205     __ ubfmw(as_Register($dst$$reg),
12206             as_Register($src$$reg),
12207             r, s);
12208   %}
12209 
12210   ins_pipe(ialu_reg_shift);
12211 %}
12212 
12213 // Bitfield extract with shift & mask
12214 
12215 // This pattern is automatically generated from aarch64_ad.m4
12216 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12217 instruct ubfxwI(iRegINoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12218 %{
12219   match(Set dst (AndI (URShiftI src rshift) mask));
12220   // Make sure we are not going to exceed what ubfxw can do.
12221   predicate((exact_log2(n->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12222 
12223   ins_cost(INSN_COST);
12224   format %{ "ubfxw $dst, $src, $rshift, $mask" %}
12225   ins_encode %{
12226     int rshift = $rshift$$constant & 31;
12227     intptr_t mask = $mask$$constant;
12228     int width = exact_log2(mask+1);
12229     __ ubfxw(as_Register($dst$$reg),
12230             as_Register($src$$reg), rshift, width);
12231   %}
12232   ins_pipe(ialu_reg_shift);
12233 %}
12234 
12235 // This pattern is automatically generated from aarch64_ad.m4
12236 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12237 instruct ubfxL(iRegLNoSp dst, iRegL src, immI rshift, immL_bitmask mask)
12238 %{
12239   match(Set dst (AndL (URShiftL src rshift) mask));
12240   // Make sure we are not going to exceed what ubfx can do.
12241   predicate((exact_log2_long(n->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= (63 + 1));
12242 
12243   ins_cost(INSN_COST);
12244   format %{ "ubfx $dst, $src, $rshift, $mask" %}
12245   ins_encode %{
12246     int rshift = $rshift$$constant & 63;
12247     intptr_t mask = $mask$$constant;
12248     int width = exact_log2_long(mask+1);
12249     __ ubfx(as_Register($dst$$reg),
12250             as_Register($src$$reg), rshift, width);
12251   %}
12252   ins_pipe(ialu_reg_shift);
12253 %}
12254 
12255 
12256 // This pattern is automatically generated from aarch64_ad.m4
12257 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12258 
12259 // We can use ubfx when extending an And with a mask when we know mask
12260 // is positive.  We know that because immI_bitmask guarantees it.
12261 instruct ubfxIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12262 %{
12263   match(Set dst (ConvI2L (AndI (URShiftI src rshift) mask)));
12264   // Make sure we are not going to exceed what ubfxw can do.
12265   predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12266 
12267   ins_cost(INSN_COST * 2);
12268   format %{ "ubfx $dst, $src, $rshift, $mask" %}
12269   ins_encode %{
12270     int rshift = $rshift$$constant & 31;
12271     intptr_t mask = $mask$$constant;
12272     int width = exact_log2(mask+1);
12273     __ ubfx(as_Register($dst$$reg),
12274             as_Register($src$$reg), rshift, width);
12275   %}
12276   ins_pipe(ialu_reg_shift);
12277 %}
12278 
12279 
12280 // This pattern is automatically generated from aarch64_ad.m4
12281 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12282 
12283 // We can use ubfiz when masking by a positive number and then left shifting the result.
12284 // We know that the mask is positive because immI_bitmask guarantees it.
12285 instruct ubfizwI(iRegINoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12286 %{
12287   match(Set dst (LShiftI (AndI src mask) lshift));
12288   predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 31)) <= (31 + 1));
12289 
12290   ins_cost(INSN_COST);
12291   format %{ "ubfizw $dst, $src, $lshift, $mask" %}
12292   ins_encode %{
12293     int lshift = $lshift$$constant & 31;
12294     intptr_t mask = $mask$$constant;
12295     int width = exact_log2(mask+1);
12296     __ ubfizw(as_Register($dst$$reg),
12297           as_Register($src$$reg), lshift, width);
12298   %}
12299   ins_pipe(ialu_reg_shift);
12300 %}
12301 
12302 // This pattern is automatically generated from aarch64_ad.m4
12303 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12304 
12305 // We can use ubfiz when masking by a positive number and then left shifting the result.
12306 // We know that the mask is positive because immL_bitmask guarantees it.
12307 instruct ubfizL(iRegLNoSp dst, iRegL src, immI lshift, immL_bitmask mask)
12308 %{
12309   match(Set dst (LShiftL (AndL src mask) lshift));
12310   predicate((exact_log2_long(n->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
12311 
12312   ins_cost(INSN_COST);
12313   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12314   ins_encode %{
12315     int lshift = $lshift$$constant & 63;
12316     intptr_t mask = $mask$$constant;
12317     int width = exact_log2_long(mask+1);
12318     __ ubfiz(as_Register($dst$$reg),
12319           as_Register($src$$reg), lshift, width);
12320   %}
12321   ins_pipe(ialu_reg_shift);
12322 %}
12323 
12324 // This pattern is automatically generated from aarch64_ad.m4
12325 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12326 
12327 // We can use ubfiz when masking by a positive number and then left shifting the result.
12328 // We know that the mask is positive because immI_bitmask guarantees it.
12329 instruct ubfizwIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12330 %{
12331   match(Set dst (ConvI2L (LShiftI (AndI src mask) lshift)));
12332   predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= 31);
12333 
12334   ins_cost(INSN_COST);
12335   format %{ "ubfizw $dst, $src, $lshift, $mask" %}
12336   ins_encode %{
12337     int lshift = $lshift$$constant & 31;
12338     intptr_t mask = $mask$$constant;
12339     int width = exact_log2(mask+1);
12340     __ ubfizw(as_Register($dst$$reg),
12341           as_Register($src$$reg), lshift, width);
12342   %}
12343   ins_pipe(ialu_reg_shift);
12344 %}
12345 
12346 // This pattern is automatically generated from aarch64_ad.m4
12347 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12348 
12349 // We can use ubfiz when masking by a positive number and then left shifting the result.
12350 // We know that the mask is positive because immL_bitmask guarantees it.
12351 instruct ubfizLConvL2I(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
12352 %{
12353   match(Set dst (ConvL2I (LShiftL (AndL src mask) lshift)));
12354   predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= 31);
12355 
12356   ins_cost(INSN_COST);
12357   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12358   ins_encode %{
12359     int lshift = $lshift$$constant & 63;
12360     intptr_t mask = $mask$$constant;
12361     int width = exact_log2_long(mask+1);
12362     __ ubfiz(as_Register($dst$$reg),
12363           as_Register($src$$reg), lshift, width);
12364   %}
12365   ins_pipe(ialu_reg_shift);
12366 %}
12367 
12368 
12369 // This pattern is automatically generated from aarch64_ad.m4
12370 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12371 
12372 // If there is a convert I to L block between and AndI and a LShiftL, we can also match ubfiz
12373 instruct ubfizIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12374 %{
12375   match(Set dst (LShiftL (ConvI2L (AndI src mask)) lshift));
12376   predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
12377 
12378   ins_cost(INSN_COST);
12379   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12380   ins_encode %{
12381     int lshift = $lshift$$constant & 63;
12382     intptr_t mask = $mask$$constant;
12383     int width = exact_log2(mask+1);
12384     __ ubfiz(as_Register($dst$$reg),
12385              as_Register($src$$reg), lshift, width);
12386   %}
12387   ins_pipe(ialu_reg_shift);
12388 %}
12389 
12390 // This pattern is automatically generated from aarch64_ad.m4
12391 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12392 
12393 // If there is a convert L to I block between and AndL and a LShiftI, we can also match ubfiz
12394 instruct ubfizLConvL2Ix(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
12395 %{
12396   match(Set dst (LShiftI (ConvL2I (AndL src mask)) lshift));
12397   predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 31)) <= 31);
12398 
12399   ins_cost(INSN_COST);
12400   format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12401   ins_encode %{
12402     int lshift = $lshift$$constant & 31;
12403     intptr_t mask = $mask$$constant;
12404     int width = exact_log2(mask+1);
12405     __ ubfiz(as_Register($dst$$reg),
12406              as_Register($src$$reg), lshift, width);
12407   %}
12408   ins_pipe(ialu_reg_shift);
12409 %}
12410 
12411 // This pattern is automatically generated from aarch64_ad.m4
12412 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12413 
12414 // Can skip int2long conversions after AND with small bitmask
12415 instruct ubfizIConvI2LAndI(iRegLNoSp dst, iRegI src, immI_bitmask msk)
12416 %{
12417   match(Set dst (ConvI2L (AndI src msk)));
12418   ins_cost(INSN_COST);
12419   format %{ "ubfiz $dst, $src, 0, exact_log2($msk + 1) " %}
12420   ins_encode %{
12421     __ ubfiz(as_Register($dst$$reg), as_Register($src$$reg), 0, exact_log2($msk$$constant + 1));
12422   %}
12423   ins_pipe(ialu_reg_shift);
12424 %}
12425 
12426 
12427 // Rotations
12428 
12429 // This pattern is automatically generated from aarch64_ad.m4
12430 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12431 instruct extrOrL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
12432 %{
12433   match(Set dst (OrL (LShiftL src1 lshift) (URShiftL src2 rshift)));
12434   predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
12435 
12436   ins_cost(INSN_COST);
12437   format %{ "extr $dst, $src1, $src2, #$rshift" %}
12438 
12439   ins_encode %{
12440     __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12441             $rshift$$constant & 63);
12442   %}
12443   ins_pipe(ialu_reg_reg_extr);
12444 %}
12445 
12446 
12447 // This pattern is automatically generated from aarch64_ad.m4
12448 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12449 instruct extrOrI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
12450 %{
12451   match(Set dst (OrI (LShiftI src1 lshift) (URShiftI src2 rshift)));
12452   predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
12453 
12454   ins_cost(INSN_COST);
12455   format %{ "extr $dst, $src1, $src2, #$rshift" %}
12456 
12457   ins_encode %{
12458     __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12459             $rshift$$constant & 31);
12460   %}
12461   ins_pipe(ialu_reg_reg_extr);
12462 %}
12463 
12464 
12465 // This pattern is automatically generated from aarch64_ad.m4
12466 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12467 instruct extrAddL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
12468 %{
12469   match(Set dst (AddL (LShiftL src1 lshift) (URShiftL src2 rshift)));
12470   predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
12471 
12472   ins_cost(INSN_COST);
12473   format %{ "extr $dst, $src1, $src2, #$rshift" %}
12474 
12475   ins_encode %{
12476     __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12477             $rshift$$constant & 63);
12478   %}
12479   ins_pipe(ialu_reg_reg_extr);
12480 %}
12481 
12482 
12483 // This pattern is automatically generated from aarch64_ad.m4
12484 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12485 instruct extrAddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
12486 %{
12487   match(Set dst (AddI (LShiftI src1 lshift) (URShiftI src2 rshift)));
12488   predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
12489 
12490   ins_cost(INSN_COST);
12491   format %{ "extr $dst, $src1, $src2, #$rshift" %}
12492 
12493   ins_encode %{
12494     __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12495             $rshift$$constant & 31);
12496   %}
12497   ins_pipe(ialu_reg_reg_extr);
12498 %}
12499 
12500 // This pattern is automatically generated from aarch64_ad.m4
12501 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12502 instruct rorI_imm(iRegINoSp dst, iRegI src, immI shift)
12503 %{
12504   match(Set dst (RotateRight src shift));
12505 
12506   ins_cost(INSN_COST);
12507   format %{ "ror    $dst, $src, $shift" %}
12508 
12509   ins_encode %{
12510      __ extrw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
12511                $shift$$constant & 0x1f);
12512   %}
12513   ins_pipe(ialu_reg_reg_vshift);
12514 %}
12515 
12516 // This pattern is automatically generated from aarch64_ad.m4
12517 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12518 instruct rorL_imm(iRegLNoSp dst, iRegL src, immI shift)
12519 %{
12520   match(Set dst (RotateRight src shift));
12521 
12522   ins_cost(INSN_COST);
12523   format %{ "ror    $dst, $src, $shift" %}
12524 
12525   ins_encode %{
12526      __ extr(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
12527                $shift$$constant & 0x3f);
12528   %}
12529   ins_pipe(ialu_reg_reg_vshift);
12530 %}
12531 
12532 // This pattern is automatically generated from aarch64_ad.m4
12533 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12534 instruct rorI_reg(iRegINoSp dst, iRegI src, iRegI shift)
12535 %{
12536   match(Set dst (RotateRight src shift));
12537 
12538   ins_cost(INSN_COST);
12539   format %{ "ror    $dst, $src, $shift" %}
12540 
12541   ins_encode %{
12542      __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
12543   %}
12544   ins_pipe(ialu_reg_reg_vshift);
12545 %}
12546 
12547 // This pattern is automatically generated from aarch64_ad.m4
12548 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12549 instruct rorL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
12550 %{
12551   match(Set dst (RotateRight src shift));
12552 
12553   ins_cost(INSN_COST);
12554   format %{ "ror    $dst, $src, $shift" %}
12555 
12556   ins_encode %{
12557      __ rorv(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
12558   %}
12559   ins_pipe(ialu_reg_reg_vshift);
12560 %}
12561 
12562 // This pattern is automatically generated from aarch64_ad.m4
12563 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12564 instruct rolI_reg(iRegINoSp dst, iRegI src, iRegI shift)
12565 %{
12566   match(Set dst (RotateLeft src shift));
12567 
12568   ins_cost(INSN_COST);
12569   format %{ "rol    $dst, $src, $shift" %}
12570 
12571   ins_encode %{
12572      __ subw(rscratch1, zr, as_Register($shift$$reg));
12573      __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
12574   %}
12575   ins_pipe(ialu_reg_reg_vshift);
12576 %}
12577 
12578 // This pattern is automatically generated from aarch64_ad.m4
12579 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12580 instruct rolL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
12581 %{
12582   match(Set dst (RotateLeft src shift));
12583 
12584   ins_cost(INSN_COST);
12585   format %{ "rol    $dst, $src, $shift" %}
12586 
12587   ins_encode %{
12588      __ subw(rscratch1, zr, as_Register($shift$$reg));
12589      __ rorv(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
12590   %}
12591   ins_pipe(ialu_reg_reg_vshift);
12592 %}
12593 
12594 
12595 // Add/subtract (extended)
12596 
12597 // This pattern is automatically generated from aarch64_ad.m4
12598 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12599 instruct AddExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
12600 %{
12601   match(Set dst (AddL src1 (ConvI2L src2)));
12602   ins_cost(INSN_COST);
12603   format %{ "add  $dst, $src1, $src2, sxtw" %}
12604 
12605    ins_encode %{
12606      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12607             as_Register($src2$$reg), ext::sxtw);
12608    %}
12609   ins_pipe(ialu_reg_reg);
12610 %}
12611 
12612 // This pattern is automatically generated from aarch64_ad.m4
12613 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12614 instruct SubExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
12615 %{
12616   match(Set dst (SubL src1 (ConvI2L src2)));
12617   ins_cost(INSN_COST);
12618   format %{ "sub  $dst, $src1, $src2, sxtw" %}
12619 
12620    ins_encode %{
12621      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12622             as_Register($src2$$reg), ext::sxtw);
12623    %}
12624   ins_pipe(ialu_reg_reg);
12625 %}
12626 
12627 // This pattern is automatically generated from aarch64_ad.m4
12628 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12629 instruct AddExtI_sxth(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_16 lshift, immI_16 rshift, rFlagsReg cr)
12630 %{
12631   match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
12632   ins_cost(INSN_COST);
12633   format %{ "add  $dst, $src1, $src2, sxth" %}
12634 
12635    ins_encode %{
12636      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12637             as_Register($src2$$reg), ext::sxth);
12638    %}
12639   ins_pipe(ialu_reg_reg);
12640 %}
12641 
12642 // This pattern is automatically generated from aarch64_ad.m4
12643 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12644 instruct AddExtI_sxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
12645 %{
12646   match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
12647   ins_cost(INSN_COST);
12648   format %{ "add  $dst, $src1, $src2, sxtb" %}
12649 
12650    ins_encode %{
12651      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12652             as_Register($src2$$reg), ext::sxtb);
12653    %}
12654   ins_pipe(ialu_reg_reg);
12655 %}
12656 
12657 // This pattern is automatically generated from aarch64_ad.m4
12658 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12659 instruct AddExtI_uxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
12660 %{
12661   match(Set dst (AddI src1 (URShiftI (LShiftI src2 lshift) rshift)));
12662   ins_cost(INSN_COST);
12663   format %{ "add  $dst, $src1, $src2, uxtb" %}
12664 
12665    ins_encode %{
12666      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12667             as_Register($src2$$reg), ext::uxtb);
12668    %}
12669   ins_pipe(ialu_reg_reg);
12670 %}
12671 
12672 // This pattern is automatically generated from aarch64_ad.m4
12673 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12674 instruct AddExtL_sxth(iRegLNoSp dst, iRegL src1, iRegL src2, immI_48 lshift, immI_48 rshift, rFlagsReg cr)
12675 %{
12676   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12677   ins_cost(INSN_COST);
12678   format %{ "add  $dst, $src1, $src2, sxth" %}
12679 
12680    ins_encode %{
12681      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12682             as_Register($src2$$reg), ext::sxth);
12683    %}
12684   ins_pipe(ialu_reg_reg);
12685 %}
12686 
12687 // This pattern is automatically generated from aarch64_ad.m4
12688 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12689 instruct AddExtL_sxtw(iRegLNoSp dst, iRegL src1, iRegL src2, immI_32 lshift, immI_32 rshift, rFlagsReg cr)
12690 %{
12691   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12692   ins_cost(INSN_COST);
12693   format %{ "add  $dst, $src1, $src2, sxtw" %}
12694 
12695    ins_encode %{
12696      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12697             as_Register($src2$$reg), ext::sxtw);
12698    %}
12699   ins_pipe(ialu_reg_reg);
12700 %}
12701 
12702 // This pattern is automatically generated from aarch64_ad.m4
12703 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12704 instruct AddExtL_sxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
12705 %{
12706   match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12707   ins_cost(INSN_COST);
12708   format %{ "add  $dst, $src1, $src2, sxtb" %}
12709 
12710    ins_encode %{
12711      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12712             as_Register($src2$$reg), ext::sxtb);
12713    %}
12714   ins_pipe(ialu_reg_reg);
12715 %}
12716 
12717 // This pattern is automatically generated from aarch64_ad.m4
12718 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12719 instruct AddExtL_uxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
12720 %{
12721   match(Set dst (AddL src1 (URShiftL (LShiftL src2 lshift) rshift)));
12722   ins_cost(INSN_COST);
12723   format %{ "add  $dst, $src1, $src2, uxtb" %}
12724 
12725    ins_encode %{
12726      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12727             as_Register($src2$$reg), ext::uxtb);
12728    %}
12729   ins_pipe(ialu_reg_reg);
12730 %}
12731 
12732 // This pattern is automatically generated from aarch64_ad.m4
12733 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12734 instruct AddExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
12735 %{
12736   match(Set dst (AddI src1 (AndI src2 mask)));
12737   ins_cost(INSN_COST);
12738   format %{ "addw  $dst, $src1, $src2, uxtb" %}
12739 
12740    ins_encode %{
12741      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12742             as_Register($src2$$reg), ext::uxtb);
12743    %}
12744   ins_pipe(ialu_reg_reg);
12745 %}
12746 
12747 // This pattern is automatically generated from aarch64_ad.m4
12748 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12749 instruct AddExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
12750 %{
12751   match(Set dst (AddI src1 (AndI src2 mask)));
12752   ins_cost(INSN_COST);
12753   format %{ "addw  $dst, $src1, $src2, uxth" %}
12754 
12755    ins_encode %{
12756      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12757             as_Register($src2$$reg), ext::uxth);
12758    %}
12759   ins_pipe(ialu_reg_reg);
12760 %}
12761 
12762 // This pattern is automatically generated from aarch64_ad.m4
12763 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12764 instruct AddExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
12765 %{
12766   match(Set dst (AddL src1 (AndL src2 mask)));
12767   ins_cost(INSN_COST);
12768   format %{ "add  $dst, $src1, $src2, uxtb" %}
12769 
12770    ins_encode %{
12771      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12772             as_Register($src2$$reg), ext::uxtb);
12773    %}
12774   ins_pipe(ialu_reg_reg);
12775 %}
12776 
12777 // This pattern is automatically generated from aarch64_ad.m4
12778 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12779 instruct AddExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
12780 %{
12781   match(Set dst (AddL src1 (AndL src2 mask)));
12782   ins_cost(INSN_COST);
12783   format %{ "add  $dst, $src1, $src2, uxth" %}
12784 
12785    ins_encode %{
12786      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12787             as_Register($src2$$reg), ext::uxth);
12788    %}
12789   ins_pipe(ialu_reg_reg);
12790 %}
12791 
12792 // This pattern is automatically generated from aarch64_ad.m4
12793 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12794 instruct AddExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
12795 %{
12796   match(Set dst (AddL src1 (AndL src2 mask)));
12797   ins_cost(INSN_COST);
12798   format %{ "add  $dst, $src1, $src2, uxtw" %}
12799 
12800    ins_encode %{
12801      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12802             as_Register($src2$$reg), ext::uxtw);
12803    %}
12804   ins_pipe(ialu_reg_reg);
12805 %}
12806 
12807 // This pattern is automatically generated from aarch64_ad.m4
12808 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12809 instruct SubExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
12810 %{
12811   match(Set dst (SubI src1 (AndI src2 mask)));
12812   ins_cost(INSN_COST);
12813   format %{ "subw  $dst, $src1, $src2, uxtb" %}
12814 
12815    ins_encode %{
12816      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12817             as_Register($src2$$reg), ext::uxtb);
12818    %}
12819   ins_pipe(ialu_reg_reg);
12820 %}
12821 
12822 // This pattern is automatically generated from aarch64_ad.m4
12823 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12824 instruct SubExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
12825 %{
12826   match(Set dst (SubI src1 (AndI src2 mask)));
12827   ins_cost(INSN_COST);
12828   format %{ "subw  $dst, $src1, $src2, uxth" %}
12829 
12830    ins_encode %{
12831      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12832             as_Register($src2$$reg), ext::uxth);
12833    %}
12834   ins_pipe(ialu_reg_reg);
12835 %}
12836 
12837 // This pattern is automatically generated from aarch64_ad.m4
12838 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12839 instruct SubExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
12840 %{
12841   match(Set dst (SubL src1 (AndL src2 mask)));
12842   ins_cost(INSN_COST);
12843   format %{ "sub  $dst, $src1, $src2, uxtb" %}
12844 
12845    ins_encode %{
12846      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12847             as_Register($src2$$reg), ext::uxtb);
12848    %}
12849   ins_pipe(ialu_reg_reg);
12850 %}
12851 
12852 // This pattern is automatically generated from aarch64_ad.m4
12853 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12854 instruct SubExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
12855 %{
12856   match(Set dst (SubL src1 (AndL src2 mask)));
12857   ins_cost(INSN_COST);
12858   format %{ "sub  $dst, $src1, $src2, uxth" %}
12859 
12860    ins_encode %{
12861      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12862             as_Register($src2$$reg), ext::uxth);
12863    %}
12864   ins_pipe(ialu_reg_reg);
12865 %}
12866 
12867 // This pattern is automatically generated from aarch64_ad.m4
12868 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12869 instruct SubExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
12870 %{
12871   match(Set dst (SubL src1 (AndL src2 mask)));
12872   ins_cost(INSN_COST);
12873   format %{ "sub  $dst, $src1, $src2, uxtw" %}
12874 
12875    ins_encode %{
12876      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12877             as_Register($src2$$reg), ext::uxtw);
12878    %}
12879   ins_pipe(ialu_reg_reg);
12880 %}
12881 
12882 
12883 // This pattern is automatically generated from aarch64_ad.m4
12884 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12885 instruct AddExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
12886 %{
12887   match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12888   ins_cost(1.9 * INSN_COST);
12889   format %{ "add  $dst, $src1, $src2, sxtb #lshift2" %}
12890 
12891    ins_encode %{
12892      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12893             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12894    %}
12895   ins_pipe(ialu_reg_reg_shift);
12896 %}
12897 
12898 // This pattern is automatically generated from aarch64_ad.m4
12899 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12900 instruct AddExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
12901 %{
12902   match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12903   ins_cost(1.9 * INSN_COST);
12904   format %{ "add  $dst, $src1, $src2, sxth #lshift2" %}
12905 
12906    ins_encode %{
12907      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12908             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12909    %}
12910   ins_pipe(ialu_reg_reg_shift);
12911 %}
12912 
12913 // This pattern is automatically generated from aarch64_ad.m4
12914 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12915 instruct AddExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
12916 %{
12917   match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12918   ins_cost(1.9 * INSN_COST);
12919   format %{ "add  $dst, $src1, $src2, sxtw #lshift2" %}
12920 
12921    ins_encode %{
12922      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12923             as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
12924    %}
12925   ins_pipe(ialu_reg_reg_shift);
12926 %}
12927 
12928 // This pattern is automatically generated from aarch64_ad.m4
12929 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12930 instruct SubExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
12931 %{
12932   match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12933   ins_cost(1.9 * INSN_COST);
12934   format %{ "sub  $dst, $src1, $src2, sxtb #lshift2" %}
12935 
12936    ins_encode %{
12937      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12938             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12939    %}
12940   ins_pipe(ialu_reg_reg_shift);
12941 %}
12942 
12943 // This pattern is automatically generated from aarch64_ad.m4
12944 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12945 instruct SubExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
12946 %{
12947   match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12948   ins_cost(1.9 * INSN_COST);
12949   format %{ "sub  $dst, $src1, $src2, sxth #lshift2" %}
12950 
12951    ins_encode %{
12952      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12953             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12954    %}
12955   ins_pipe(ialu_reg_reg_shift);
12956 %}
12957 
12958 // This pattern is automatically generated from aarch64_ad.m4
12959 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12960 instruct SubExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
12961 %{
12962   match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12963   ins_cost(1.9 * INSN_COST);
12964   format %{ "sub  $dst, $src1, $src2, sxtw #lshift2" %}
12965 
12966    ins_encode %{
12967      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12968             as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
12969    %}
12970   ins_pipe(ialu_reg_reg_shift);
12971 %}
12972 
12973 // This pattern is automatically generated from aarch64_ad.m4
12974 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12975 instruct AddExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
12976 %{
12977   match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12978   ins_cost(1.9 * INSN_COST);
12979   format %{ "addw  $dst, $src1, $src2, sxtb #lshift2" %}
12980 
12981    ins_encode %{
12982      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12983             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12984    %}
12985   ins_pipe(ialu_reg_reg_shift);
12986 %}
12987 
12988 // This pattern is automatically generated from aarch64_ad.m4
12989 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12990 instruct AddExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
12991 %{
12992   match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12993   ins_cost(1.9 * INSN_COST);
12994   format %{ "addw  $dst, $src1, $src2, sxth #lshift2" %}
12995 
12996    ins_encode %{
12997      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12998             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12999    %}
13000   ins_pipe(ialu_reg_reg_shift);
13001 %}
13002 
13003 // This pattern is automatically generated from aarch64_ad.m4
13004 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13005 instruct SubExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13006 %{
13007   match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13008   ins_cost(1.9 * INSN_COST);
13009   format %{ "subw  $dst, $src1, $src2, sxtb #lshift2" %}
13010 
13011    ins_encode %{
13012      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13013             as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13014    %}
13015   ins_pipe(ialu_reg_reg_shift);
13016 %}
13017 
13018 // This pattern is automatically generated from aarch64_ad.m4
13019 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13020 instruct SubExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13021 %{
13022   match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13023   ins_cost(1.9 * INSN_COST);
13024   format %{ "subw  $dst, $src1, $src2, sxth #lshift2" %}
13025 
13026    ins_encode %{
13027      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13028             as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13029    %}
13030   ins_pipe(ialu_reg_reg_shift);
13031 %}
13032 
13033 // This pattern is automatically generated from aarch64_ad.m4
13034 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13035 instruct AddExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13036 %{
13037   match(Set dst (AddL src1 (LShiftL (ConvI2L src2) lshift)));
13038   ins_cost(1.9 * INSN_COST);
13039   format %{ "add  $dst, $src1, $src2, sxtw #lshift" %}
13040 
13041    ins_encode %{
13042      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13043             as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13044    %}
13045   ins_pipe(ialu_reg_reg_shift);
13046 %}
13047 
13048 // This pattern is automatically generated from aarch64_ad.m4
13049 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13050 instruct SubExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13051 %{
13052   match(Set dst (SubL src1 (LShiftL (ConvI2L src2) lshift)));
13053   ins_cost(1.9 * INSN_COST);
13054   format %{ "sub  $dst, $src1, $src2, sxtw #lshift" %}
13055 
13056    ins_encode %{
13057      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13058             as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13059    %}
13060   ins_pipe(ialu_reg_reg_shift);
13061 %}
13062 
13063 // This pattern is automatically generated from aarch64_ad.m4
13064 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13065 instruct AddExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13066 %{
13067   match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13068   ins_cost(1.9 * INSN_COST);
13069   format %{ "add  $dst, $src1, $src2, uxtb #lshift" %}
13070 
13071    ins_encode %{
13072      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13073             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13074    %}
13075   ins_pipe(ialu_reg_reg_shift);
13076 %}
13077 
13078 // This pattern is automatically generated from aarch64_ad.m4
13079 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13080 instruct AddExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13081 %{
13082   match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13083   ins_cost(1.9 * INSN_COST);
13084   format %{ "add  $dst, $src1, $src2, uxth #lshift" %}
13085 
13086    ins_encode %{
13087      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13088             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13089    %}
13090   ins_pipe(ialu_reg_reg_shift);
13091 %}
13092 
13093 // This pattern is automatically generated from aarch64_ad.m4
13094 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13095 instruct AddExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13096 %{
13097   match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13098   ins_cost(1.9 * INSN_COST);
13099   format %{ "add  $dst, $src1, $src2, uxtw #lshift" %}
13100 
13101    ins_encode %{
13102      __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13103             as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13104    %}
13105   ins_pipe(ialu_reg_reg_shift);
13106 %}
13107 
13108 // This pattern is automatically generated from aarch64_ad.m4
13109 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13110 instruct SubExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13111 %{
13112   match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13113   ins_cost(1.9 * INSN_COST);
13114   format %{ "sub  $dst, $src1, $src2, uxtb #lshift" %}
13115 
13116    ins_encode %{
13117      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13118             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13119    %}
13120   ins_pipe(ialu_reg_reg_shift);
13121 %}
13122 
13123 // This pattern is automatically generated from aarch64_ad.m4
13124 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13125 instruct SubExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13126 %{
13127   match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13128   ins_cost(1.9 * INSN_COST);
13129   format %{ "sub  $dst, $src1, $src2, uxth #lshift" %}
13130 
13131    ins_encode %{
13132      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13133             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13134    %}
13135   ins_pipe(ialu_reg_reg_shift);
13136 %}
13137 
13138 // This pattern is automatically generated from aarch64_ad.m4
13139 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13140 instruct SubExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13141 %{
13142   match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13143   ins_cost(1.9 * INSN_COST);
13144   format %{ "sub  $dst, $src1, $src2, uxtw #lshift" %}
13145 
13146    ins_encode %{
13147      __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13148             as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13149    %}
13150   ins_pipe(ialu_reg_reg_shift);
13151 %}
13152 
13153 // This pattern is automatically generated from aarch64_ad.m4
13154 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13155 instruct AddExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13156 %{
13157   match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13158   ins_cost(1.9 * INSN_COST);
13159   format %{ "addw  $dst, $src1, $src2, uxtb #lshift" %}
13160 
13161    ins_encode %{
13162      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13163             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13164    %}
13165   ins_pipe(ialu_reg_reg_shift);
13166 %}
13167 
13168 // This pattern is automatically generated from aarch64_ad.m4
13169 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13170 instruct AddExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13171 %{
13172   match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13173   ins_cost(1.9 * INSN_COST);
13174   format %{ "addw  $dst, $src1, $src2, uxth #lshift" %}
13175 
13176    ins_encode %{
13177      __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13178             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13179    %}
13180   ins_pipe(ialu_reg_reg_shift);
13181 %}
13182 
13183 // This pattern is automatically generated from aarch64_ad.m4
13184 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13185 instruct SubExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13186 %{
13187   match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13188   ins_cost(1.9 * INSN_COST);
13189   format %{ "subw  $dst, $src1, $src2, uxtb #lshift" %}
13190 
13191    ins_encode %{
13192      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13193             as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13194    %}
13195   ins_pipe(ialu_reg_reg_shift);
13196 %}
13197 
13198 // This pattern is automatically generated from aarch64_ad.m4
13199 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13200 instruct SubExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13201 %{
13202   match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13203   ins_cost(1.9 * INSN_COST);
13204   format %{ "subw  $dst, $src1, $src2, uxth #lshift" %}
13205 
13206    ins_encode %{
13207      __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13208             as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13209    %}
13210   ins_pipe(ialu_reg_reg_shift);
13211 %}
13212 
13213 // This pattern is automatically generated from aarch64_ad.m4
13214 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13215 instruct cmovI_reg_reg_lt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
13216 %{
13217   effect(DEF dst, USE src1, USE src2, USE cr);
13218   ins_cost(INSN_COST * 2);
13219   format %{ "cselw $dst, $src1, $src2 lt\t"  %}
13220 
13221   ins_encode %{
13222     __ cselw($dst$$Register,
13223              $src1$$Register,
13224              $src2$$Register,
13225              Assembler::LT);
13226   %}
13227   ins_pipe(icond_reg_reg);
13228 %}
13229 
13230 // This pattern is automatically generated from aarch64_ad.m4
13231 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13232 instruct cmovI_reg_reg_gt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
13233 %{
13234   effect(DEF dst, USE src1, USE src2, USE cr);
13235   ins_cost(INSN_COST * 2);
13236   format %{ "cselw $dst, $src1, $src2 gt\t"  %}
13237 
13238   ins_encode %{
13239     __ cselw($dst$$Register,
13240              $src1$$Register,
13241              $src2$$Register,
13242              Assembler::GT);
13243   %}
13244   ins_pipe(icond_reg_reg);
13245 %}
13246 
13247 // This pattern is automatically generated from aarch64_ad.m4
13248 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13249 instruct cmovI_reg_imm0_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13250 %{
13251   effect(DEF dst, USE src1, USE cr);
13252   ins_cost(INSN_COST * 2);
13253   format %{ "cselw $dst, $src1, zr lt\t"  %}
13254 
13255   ins_encode %{
13256     __ cselw($dst$$Register,
13257              $src1$$Register,
13258              zr,
13259              Assembler::LT);
13260   %}
13261   ins_pipe(icond_reg);
13262 %}
13263 
13264 // This pattern is automatically generated from aarch64_ad.m4
13265 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13266 instruct cmovI_reg_imm0_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13267 %{
13268   effect(DEF dst, USE src1, USE cr);
13269   ins_cost(INSN_COST * 2);
13270   format %{ "cselw $dst, $src1, zr gt\t"  %}
13271 
13272   ins_encode %{
13273     __ cselw($dst$$Register,
13274              $src1$$Register,
13275              zr,
13276              Assembler::GT);
13277   %}
13278   ins_pipe(icond_reg);
13279 %}
13280 
13281 // This pattern is automatically generated from aarch64_ad.m4
13282 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13283 instruct cmovI_reg_imm1_le(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13284 %{
13285   effect(DEF dst, USE src1, USE cr);
13286   ins_cost(INSN_COST * 2);
13287   format %{ "csincw $dst, $src1, zr le\t"  %}
13288 
13289   ins_encode %{
13290     __ csincw($dst$$Register,
13291              $src1$$Register,
13292              zr,
13293              Assembler::LE);
13294   %}
13295   ins_pipe(icond_reg);
13296 %}
13297 
13298 // This pattern is automatically generated from aarch64_ad.m4
13299 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13300 instruct cmovI_reg_imm1_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13301 %{
13302   effect(DEF dst, USE src1, USE cr);
13303   ins_cost(INSN_COST * 2);
13304   format %{ "csincw $dst, $src1, zr gt\t"  %}
13305 
13306   ins_encode %{
13307     __ csincw($dst$$Register,
13308              $src1$$Register,
13309              zr,
13310              Assembler::GT);
13311   %}
13312   ins_pipe(icond_reg);
13313 %}
13314 
13315 // This pattern is automatically generated from aarch64_ad.m4
13316 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13317 instruct cmovI_reg_immM1_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13318 %{
13319   effect(DEF dst, USE src1, USE cr);
13320   ins_cost(INSN_COST * 2);
13321   format %{ "csinvw $dst, $src1, zr lt\t"  %}
13322 
13323   ins_encode %{
13324     __ csinvw($dst$$Register,
13325              $src1$$Register,
13326              zr,
13327              Assembler::LT);
13328   %}
13329   ins_pipe(icond_reg);
13330 %}
13331 
13332 // This pattern is automatically generated from aarch64_ad.m4
13333 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13334 instruct cmovI_reg_immM1_ge(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13335 %{
13336   effect(DEF dst, USE src1, USE cr);
13337   ins_cost(INSN_COST * 2);
13338   format %{ "csinvw $dst, $src1, zr ge\t"  %}
13339 
13340   ins_encode %{
13341     __ csinvw($dst$$Register,
13342              $src1$$Register,
13343              zr,
13344              Assembler::GE);
13345   %}
13346   ins_pipe(icond_reg);
13347 %}
13348 
13349 // This pattern is automatically generated from aarch64_ad.m4
13350 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13351 instruct minI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
13352 %{
13353   match(Set dst (MinI src imm));
13354   ins_cost(INSN_COST * 3);
13355   expand %{
13356     rFlagsReg cr;
13357     compI_reg_imm0(cr, src);
13358     cmovI_reg_imm0_lt(dst, src, cr);
13359   %}
13360 %}
13361 
13362 // This pattern is automatically generated from aarch64_ad.m4
13363 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13364 instruct minI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
13365 %{
13366   match(Set dst (MinI imm src));
13367   ins_cost(INSN_COST * 3);
13368   expand %{
13369     rFlagsReg cr;
13370     compI_reg_imm0(cr, src);
13371     cmovI_reg_imm0_lt(dst, src, cr);
13372   %}
13373 %}
13374 
13375 // This pattern is automatically generated from aarch64_ad.m4
13376 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13377 instruct minI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
13378 %{
13379   match(Set dst (MinI src imm));
13380   ins_cost(INSN_COST * 3);
13381   expand %{
13382     rFlagsReg cr;
13383     compI_reg_imm0(cr, src);
13384     cmovI_reg_imm1_le(dst, src, cr);
13385   %}
13386 %}
13387 
13388 // This pattern is automatically generated from aarch64_ad.m4
13389 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13390 instruct minI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
13391 %{
13392   match(Set dst (MinI imm src));
13393   ins_cost(INSN_COST * 3);
13394   expand %{
13395     rFlagsReg cr;
13396     compI_reg_imm0(cr, src);
13397     cmovI_reg_imm1_le(dst, src, cr);
13398   %}
13399 %}
13400 
13401 // This pattern is automatically generated from aarch64_ad.m4
13402 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13403 instruct minI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
13404 %{
13405   match(Set dst (MinI src imm));
13406   ins_cost(INSN_COST * 3);
13407   expand %{
13408     rFlagsReg cr;
13409     compI_reg_imm0(cr, src);
13410     cmovI_reg_immM1_lt(dst, src, cr);
13411   %}
13412 %}
13413 
13414 // This pattern is automatically generated from aarch64_ad.m4
13415 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13416 instruct minI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
13417 %{
13418   match(Set dst (MinI imm src));
13419   ins_cost(INSN_COST * 3);
13420   expand %{
13421     rFlagsReg cr;
13422     compI_reg_imm0(cr, src);
13423     cmovI_reg_immM1_lt(dst, src, cr);
13424   %}
13425 %}
13426 
13427 // This pattern is automatically generated from aarch64_ad.m4
13428 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13429 instruct maxI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
13430 %{
13431   match(Set dst (MaxI src imm));
13432   ins_cost(INSN_COST * 3);
13433   expand %{
13434     rFlagsReg cr;
13435     compI_reg_imm0(cr, src);
13436     cmovI_reg_imm0_gt(dst, src, cr);
13437   %}
13438 %}
13439 
13440 // This pattern is automatically generated from aarch64_ad.m4
13441 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13442 instruct maxI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
13443 %{
13444   match(Set dst (MaxI imm src));
13445   ins_cost(INSN_COST * 3);
13446   expand %{
13447     rFlagsReg cr;
13448     compI_reg_imm0(cr, src);
13449     cmovI_reg_imm0_gt(dst, src, cr);
13450   %}
13451 %}
13452 
13453 // This pattern is automatically generated from aarch64_ad.m4
13454 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13455 instruct maxI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
13456 %{
13457   match(Set dst (MaxI src imm));
13458   ins_cost(INSN_COST * 3);
13459   expand %{
13460     rFlagsReg cr;
13461     compI_reg_imm0(cr, src);
13462     cmovI_reg_imm1_gt(dst, src, cr);
13463   %}
13464 %}
13465 
13466 // This pattern is automatically generated from aarch64_ad.m4
13467 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13468 instruct maxI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
13469 %{
13470   match(Set dst (MaxI imm src));
13471   ins_cost(INSN_COST * 3);
13472   expand %{
13473     rFlagsReg cr;
13474     compI_reg_imm0(cr, src);
13475     cmovI_reg_imm1_gt(dst, src, cr);
13476   %}
13477 %}
13478 
13479 // This pattern is automatically generated from aarch64_ad.m4
13480 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13481 instruct maxI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
13482 %{
13483   match(Set dst (MaxI src imm));
13484   ins_cost(INSN_COST * 3);
13485   expand %{
13486     rFlagsReg cr;
13487     compI_reg_imm0(cr, src);
13488     cmovI_reg_immM1_ge(dst, src, cr);
13489   %}
13490 %}
13491 
13492 // This pattern is automatically generated from aarch64_ad.m4
13493 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13494 instruct maxI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
13495 %{
13496   match(Set dst (MaxI imm src));
13497   ins_cost(INSN_COST * 3);
13498   expand %{
13499     rFlagsReg cr;
13500     compI_reg_imm0(cr, src);
13501     cmovI_reg_immM1_ge(dst, src, cr);
13502   %}
13503 %}
13504 
13505 // This pattern is automatically generated from aarch64_ad.m4
13506 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13507 instruct bits_reverse_I(iRegINoSp dst, iRegIorL2I src)
13508 %{
13509   match(Set dst (ReverseI src));
13510   ins_cost(INSN_COST);
13511   format %{ "rbitw  $dst, $src" %}
13512   ins_encode %{
13513     __ rbitw($dst$$Register, $src$$Register);
13514   %}
13515   ins_pipe(ialu_reg);
13516 %}
13517 
13518 // This pattern is automatically generated from aarch64_ad.m4
13519 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13520 instruct bits_reverse_L(iRegLNoSp dst, iRegL src)
13521 %{
13522   match(Set dst (ReverseL src));
13523   ins_cost(INSN_COST);
13524   format %{ "rbit  $dst, $src" %}
13525   ins_encode %{
13526     __ rbit($dst$$Register, $src$$Register);
13527   %}
13528   ins_pipe(ialu_reg);
13529 %}
13530 
13531 
13532 // END This section of the file is automatically generated. Do not edit --------------
13533 
13534 
13535 // ============================================================================
13536 // Floating Point Arithmetic Instructions
13537 
13538 instruct addF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13539   match(Set dst (AddF src1 src2));
13540 
13541   ins_cost(INSN_COST * 5);
13542   format %{ "fadds   $dst, $src1, $src2" %}
13543 
13544   ins_encode %{
13545     __ fadds(as_FloatRegister($dst$$reg),
13546              as_FloatRegister($src1$$reg),
13547              as_FloatRegister($src2$$reg));
13548   %}
13549 
13550   ins_pipe(fp_dop_reg_reg_s);
13551 %}
13552 
13553 instruct addD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13554   match(Set dst (AddD src1 src2));
13555 
13556   ins_cost(INSN_COST * 5);
13557   format %{ "faddd   $dst, $src1, $src2" %}
13558 
13559   ins_encode %{
13560     __ faddd(as_FloatRegister($dst$$reg),
13561              as_FloatRegister($src1$$reg),
13562              as_FloatRegister($src2$$reg));
13563   %}
13564 
13565   ins_pipe(fp_dop_reg_reg_d);
13566 %}
13567 
13568 instruct subF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13569   match(Set dst (SubF src1 src2));
13570 
13571   ins_cost(INSN_COST * 5);
13572   format %{ "fsubs   $dst, $src1, $src2" %}
13573 
13574   ins_encode %{
13575     __ fsubs(as_FloatRegister($dst$$reg),
13576              as_FloatRegister($src1$$reg),
13577              as_FloatRegister($src2$$reg));
13578   %}
13579 
13580   ins_pipe(fp_dop_reg_reg_s);
13581 %}
13582 
13583 instruct subD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13584   match(Set dst (SubD src1 src2));
13585 
13586   ins_cost(INSN_COST * 5);
13587   format %{ "fsubd   $dst, $src1, $src2" %}
13588 
13589   ins_encode %{
13590     __ fsubd(as_FloatRegister($dst$$reg),
13591              as_FloatRegister($src1$$reg),
13592              as_FloatRegister($src2$$reg));
13593   %}
13594 
13595   ins_pipe(fp_dop_reg_reg_d);
13596 %}
13597 
13598 instruct mulF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13599   match(Set dst (MulF src1 src2));
13600 
13601   ins_cost(INSN_COST * 6);
13602   format %{ "fmuls   $dst, $src1, $src2" %}
13603 
13604   ins_encode %{
13605     __ fmuls(as_FloatRegister($dst$$reg),
13606              as_FloatRegister($src1$$reg),
13607              as_FloatRegister($src2$$reg));
13608   %}
13609 
13610   ins_pipe(fp_dop_reg_reg_s);
13611 %}
13612 
13613 instruct mulD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13614   match(Set dst (MulD src1 src2));
13615 
13616   ins_cost(INSN_COST * 6);
13617   format %{ "fmuld   $dst, $src1, $src2" %}
13618 
13619   ins_encode %{
13620     __ fmuld(as_FloatRegister($dst$$reg),
13621              as_FloatRegister($src1$$reg),
13622              as_FloatRegister($src2$$reg));
13623   %}
13624 
13625   ins_pipe(fp_dop_reg_reg_d);
13626 %}
13627 
13628 // src1 * src2 + src3
13629 instruct maddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13630   match(Set dst (FmaF src3 (Binary src1 src2)));
13631 
13632   format %{ "fmadds   $dst, $src1, $src2, $src3" %}
13633 
13634   ins_encode %{
13635     assert(UseFMA, "Needs FMA instructions support.");
13636     __ fmadds(as_FloatRegister($dst$$reg),
13637              as_FloatRegister($src1$$reg),
13638              as_FloatRegister($src2$$reg),
13639              as_FloatRegister($src3$$reg));
13640   %}
13641 
13642   ins_pipe(pipe_class_default);
13643 %}
13644 
13645 // src1 * src2 + src3
13646 instruct maddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13647   match(Set dst (FmaD src3 (Binary src1 src2)));
13648 
13649   format %{ "fmaddd   $dst, $src1, $src2, $src3" %}
13650 
13651   ins_encode %{
13652     assert(UseFMA, "Needs FMA instructions support.");
13653     __ fmaddd(as_FloatRegister($dst$$reg),
13654              as_FloatRegister($src1$$reg),
13655              as_FloatRegister($src2$$reg),
13656              as_FloatRegister($src3$$reg));
13657   %}
13658 
13659   ins_pipe(pipe_class_default);
13660 %}
13661 
13662 // src1 * (-src2) + src3
13663 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
13664 instruct msubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13665   match(Set dst (FmaF src3 (Binary src1 (NegF src2))));
13666 
13667   format %{ "fmsubs   $dst, $src1, $src2, $src3" %}
13668 
13669   ins_encode %{
13670     assert(UseFMA, "Needs FMA instructions support.");
13671     __ fmsubs(as_FloatRegister($dst$$reg),
13672               as_FloatRegister($src1$$reg),
13673               as_FloatRegister($src2$$reg),
13674               as_FloatRegister($src3$$reg));
13675   %}
13676 
13677   ins_pipe(pipe_class_default);
13678 %}
13679 
13680 // src1 * (-src2) + src3
13681 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
13682 instruct msubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13683   match(Set dst (FmaD src3 (Binary src1 (NegD src2))));
13684 
13685   format %{ "fmsubd   $dst, $src1, $src2, $src3" %}
13686 
13687   ins_encode %{
13688     assert(UseFMA, "Needs FMA instructions support.");
13689     __ fmsubd(as_FloatRegister($dst$$reg),
13690               as_FloatRegister($src1$$reg),
13691               as_FloatRegister($src2$$reg),
13692               as_FloatRegister($src3$$reg));
13693   %}
13694 
13695   ins_pipe(pipe_class_default);
13696 %}
13697 
13698 // src1 * (-src2) - src3
13699 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
13700 instruct mnaddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13701   match(Set dst (FmaF (NegF src3) (Binary src1 (NegF src2))));
13702 
13703   format %{ "fnmadds  $dst, $src1, $src2, $src3" %}
13704 
13705   ins_encode %{
13706     assert(UseFMA, "Needs FMA instructions support.");
13707     __ fnmadds(as_FloatRegister($dst$$reg),
13708                as_FloatRegister($src1$$reg),
13709                as_FloatRegister($src2$$reg),
13710                as_FloatRegister($src3$$reg));
13711   %}
13712 
13713   ins_pipe(pipe_class_default);
13714 %}
13715 
13716 // src1 * (-src2) - src3
13717 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
13718 instruct mnaddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13719   match(Set dst (FmaD (NegD src3) (Binary src1 (NegD src2))));
13720 
13721   format %{ "fnmaddd   $dst, $src1, $src2, $src3" %}
13722 
13723   ins_encode %{
13724     assert(UseFMA, "Needs FMA instructions support.");
13725     __ fnmaddd(as_FloatRegister($dst$$reg),
13726                as_FloatRegister($src1$$reg),
13727                as_FloatRegister($src2$$reg),
13728                as_FloatRegister($src3$$reg));
13729   %}
13730 
13731   ins_pipe(pipe_class_default);
13732 %}
13733 
13734 // src1 * src2 - src3
13735 instruct mnsubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3, immF0 zero) %{
13736   match(Set dst (FmaF (NegF src3) (Binary src1 src2)));
13737 
13738   format %{ "fnmsubs  $dst, $src1, $src2, $src3" %}
13739 
13740   ins_encode %{
13741     assert(UseFMA, "Needs FMA instructions support.");
13742     __ fnmsubs(as_FloatRegister($dst$$reg),
13743                as_FloatRegister($src1$$reg),
13744                as_FloatRegister($src2$$reg),
13745                as_FloatRegister($src3$$reg));
13746   %}
13747 
13748   ins_pipe(pipe_class_default);
13749 %}
13750 
13751 // src1 * src2 - src3
13752 instruct mnsubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3, immD0 zero) %{
13753   match(Set dst (FmaD (NegD src3) (Binary src1 src2)));
13754 
13755   format %{ "fnmsubd   $dst, $src1, $src2, $src3" %}
13756 
13757   ins_encode %{
13758     assert(UseFMA, "Needs FMA instructions support.");
13759     // n.b. insn name should be fnmsubd
13760     __ fnmsub(as_FloatRegister($dst$$reg),
13761               as_FloatRegister($src1$$reg),
13762               as_FloatRegister($src2$$reg),
13763               as_FloatRegister($src3$$reg));
13764   %}
13765 
13766   ins_pipe(pipe_class_default);
13767 %}
13768 
13769 
13770 // Math.max(FF)F
13771 instruct maxF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13772   match(Set dst (MaxF src1 src2));
13773 
13774   format %{ "fmaxs   $dst, $src1, $src2" %}
13775   ins_encode %{
13776     __ fmaxs(as_FloatRegister($dst$$reg),
13777              as_FloatRegister($src1$$reg),
13778              as_FloatRegister($src2$$reg));
13779   %}
13780 
13781   ins_pipe(fp_dop_reg_reg_s);
13782 %}
13783 
13784 // Math.min(FF)F
13785 instruct minF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13786   match(Set dst (MinF src1 src2));
13787 
13788   format %{ "fmins   $dst, $src1, $src2" %}
13789   ins_encode %{
13790     __ fmins(as_FloatRegister($dst$$reg),
13791              as_FloatRegister($src1$$reg),
13792              as_FloatRegister($src2$$reg));
13793   %}
13794 
13795   ins_pipe(fp_dop_reg_reg_s);
13796 %}
13797 
13798 // Math.max(DD)D
13799 instruct maxD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13800   match(Set dst (MaxD src1 src2));
13801 
13802   format %{ "fmaxd   $dst, $src1, $src2" %}
13803   ins_encode %{
13804     __ fmaxd(as_FloatRegister($dst$$reg),
13805              as_FloatRegister($src1$$reg),
13806              as_FloatRegister($src2$$reg));
13807   %}
13808 
13809   ins_pipe(fp_dop_reg_reg_d);
13810 %}
13811 
13812 // Math.min(DD)D
13813 instruct minD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13814   match(Set dst (MinD src1 src2));
13815 
13816   format %{ "fmind   $dst, $src1, $src2" %}
13817   ins_encode %{
13818     __ fmind(as_FloatRegister($dst$$reg),
13819              as_FloatRegister($src1$$reg),
13820              as_FloatRegister($src2$$reg));
13821   %}
13822 
13823   ins_pipe(fp_dop_reg_reg_d);
13824 %}
13825 
13826 
13827 instruct divF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13828   match(Set dst (DivF src1  src2));
13829 
13830   ins_cost(INSN_COST * 18);
13831   format %{ "fdivs   $dst, $src1, $src2" %}
13832 
13833   ins_encode %{
13834     __ fdivs(as_FloatRegister($dst$$reg),
13835              as_FloatRegister($src1$$reg),
13836              as_FloatRegister($src2$$reg));
13837   %}
13838 
13839   ins_pipe(fp_div_s);
13840 %}
13841 
13842 instruct divD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13843   match(Set dst (DivD src1  src2));
13844 
13845   ins_cost(INSN_COST * 32);
13846   format %{ "fdivd   $dst, $src1, $src2" %}
13847 
13848   ins_encode %{
13849     __ fdivd(as_FloatRegister($dst$$reg),
13850              as_FloatRegister($src1$$reg),
13851              as_FloatRegister($src2$$reg));
13852   %}
13853 
13854   ins_pipe(fp_div_d);
13855 %}
13856 
13857 instruct negF_reg_reg(vRegF dst, vRegF src) %{
13858   match(Set dst (NegF src));
13859 
13860   ins_cost(INSN_COST * 3);
13861   format %{ "fneg   $dst, $src" %}
13862 
13863   ins_encode %{
13864     __ fnegs(as_FloatRegister($dst$$reg),
13865              as_FloatRegister($src$$reg));
13866   %}
13867 
13868   ins_pipe(fp_uop_s);
13869 %}
13870 
13871 instruct negD_reg_reg(vRegD dst, vRegD src) %{
13872   match(Set dst (NegD src));
13873 
13874   ins_cost(INSN_COST * 3);
13875   format %{ "fnegd   $dst, $src" %}
13876 
13877   ins_encode %{
13878     __ fnegd(as_FloatRegister($dst$$reg),
13879              as_FloatRegister($src$$reg));
13880   %}
13881 
13882   ins_pipe(fp_uop_d);
13883 %}
13884 
13885 instruct absI_reg(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
13886 %{
13887   match(Set dst (AbsI src));
13888 
13889   effect(KILL cr);
13890   ins_cost(INSN_COST * 2);
13891   format %{ "cmpw  $src, zr\n\t"
13892             "cnegw $dst, $src, Assembler::LT\t# int abs"
13893   %}
13894 
13895   ins_encode %{
13896     __ cmpw(as_Register($src$$reg), zr);
13897     __ cnegw(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
13898   %}
13899   ins_pipe(pipe_class_default);
13900 %}
13901 
13902 instruct absL_reg(iRegLNoSp dst, iRegL src, rFlagsReg cr)
13903 %{
13904   match(Set dst (AbsL src));
13905 
13906   effect(KILL cr);
13907   ins_cost(INSN_COST * 2);
13908   format %{ "cmp  $src, zr\n\t"
13909             "cneg $dst, $src, Assembler::LT\t# long abs"
13910   %}
13911 
13912   ins_encode %{
13913     __ cmp(as_Register($src$$reg), zr);
13914     __ cneg(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
13915   %}
13916   ins_pipe(pipe_class_default);
13917 %}
13918 
13919 instruct absF_reg(vRegF dst, vRegF src) %{
13920   match(Set dst (AbsF src));
13921 
13922   ins_cost(INSN_COST * 3);
13923   format %{ "fabss   $dst, $src" %}
13924   ins_encode %{
13925     __ fabss(as_FloatRegister($dst$$reg),
13926              as_FloatRegister($src$$reg));
13927   %}
13928 
13929   ins_pipe(fp_uop_s);
13930 %}
13931 
13932 instruct absD_reg(vRegD dst, vRegD src) %{
13933   match(Set dst (AbsD src));
13934 
13935   ins_cost(INSN_COST * 3);
13936   format %{ "fabsd   $dst, $src" %}
13937   ins_encode %{
13938     __ fabsd(as_FloatRegister($dst$$reg),
13939              as_FloatRegister($src$$reg));
13940   %}
13941 
13942   ins_pipe(fp_uop_d);
13943 %}
13944 
13945 instruct absdF_reg(vRegF dst, vRegF src1, vRegF src2) %{
13946   match(Set dst (AbsF (SubF src1 src2)));
13947 
13948   ins_cost(INSN_COST * 3);
13949   format %{ "fabds   $dst, $src1, $src2" %}
13950   ins_encode %{
13951     __ fabds(as_FloatRegister($dst$$reg),
13952              as_FloatRegister($src1$$reg),
13953              as_FloatRegister($src2$$reg));
13954   %}
13955 
13956   ins_pipe(fp_uop_s);
13957 %}
13958 
13959 instruct absdD_reg(vRegD dst, vRegD src1, vRegD src2) %{
13960   match(Set dst (AbsD (SubD src1 src2)));
13961 
13962   ins_cost(INSN_COST * 3);
13963   format %{ "fabdd   $dst, $src1, $src2" %}
13964   ins_encode %{
13965     __ fabdd(as_FloatRegister($dst$$reg),
13966              as_FloatRegister($src1$$reg),
13967              as_FloatRegister($src2$$reg));
13968   %}
13969 
13970   ins_pipe(fp_uop_d);
13971 %}
13972 
13973 instruct sqrtD_reg(vRegD dst, vRegD src) %{
13974   match(Set dst (SqrtD src));
13975 
13976   ins_cost(INSN_COST * 50);
13977   format %{ "fsqrtd  $dst, $src" %}
13978   ins_encode %{
13979     __ fsqrtd(as_FloatRegister($dst$$reg),
13980              as_FloatRegister($src$$reg));
13981   %}
13982 
13983   ins_pipe(fp_div_s);
13984 %}
13985 
13986 instruct sqrtF_reg(vRegF dst, vRegF src) %{
13987   match(Set dst (SqrtF src));
13988 
13989   ins_cost(INSN_COST * 50);
13990   format %{ "fsqrts  $dst, $src" %}
13991   ins_encode %{
13992     __ fsqrts(as_FloatRegister($dst$$reg),
13993              as_FloatRegister($src$$reg));
13994   %}
13995 
13996   ins_pipe(fp_div_d);
13997 %}
13998 
13999 // Math.rint, floor, ceil
14000 instruct roundD_reg(vRegD dst, vRegD src, immI rmode) %{
14001   match(Set dst (RoundDoubleMode src rmode));
14002   format %{ "frint  $dst, $src, $rmode" %}
14003   ins_encode %{
14004     switch ($rmode$$constant) {
14005       case RoundDoubleModeNode::rmode_rint:
14006         __ frintnd(as_FloatRegister($dst$$reg),
14007                    as_FloatRegister($src$$reg));
14008         break;
14009       case RoundDoubleModeNode::rmode_floor:
14010         __ frintmd(as_FloatRegister($dst$$reg),
14011                    as_FloatRegister($src$$reg));
14012         break;
14013       case RoundDoubleModeNode::rmode_ceil:
14014         __ frintpd(as_FloatRegister($dst$$reg),
14015                    as_FloatRegister($src$$reg));
14016         break;
14017     }
14018   %}
14019   ins_pipe(fp_uop_d);
14020 %}
14021 
14022 instruct copySignD_reg(vRegD dst, vRegD src1, vRegD src2, vRegD zero) %{
14023   match(Set dst (CopySignD src1 (Binary src2 zero)));
14024   effect(TEMP_DEF dst, USE src1, USE src2, USE zero);
14025   format %{ "CopySignD  $dst $src1 $src2" %}
14026   ins_encode %{
14027     FloatRegister dst = as_FloatRegister($dst$$reg),
14028                   src1 = as_FloatRegister($src1$$reg),
14029                   src2 = as_FloatRegister($src2$$reg),
14030                   zero = as_FloatRegister($zero$$reg);
14031     __ fnegd(dst, zero);
14032     __ bsl(dst, __ T8B, src2, src1);
14033   %}
14034   ins_pipe(fp_uop_d);
14035 %}
14036 
14037 instruct copySignF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14038   match(Set dst (CopySignF src1 src2));
14039   effect(TEMP_DEF dst, USE src1, USE src2);
14040   format %{ "CopySignF  $dst $src1 $src2" %}
14041   ins_encode %{
14042     FloatRegister dst = as_FloatRegister($dst$$reg),
14043                   src1 = as_FloatRegister($src1$$reg),
14044                   src2 = as_FloatRegister($src2$$reg);
14045     __ movi(dst, __ T2S, 0x80, 24);
14046     __ bsl(dst, __ T8B, src2, src1);
14047   %}
14048   ins_pipe(fp_uop_d);
14049 %}
14050 
14051 instruct signumD_reg(vRegD dst, vRegD src, vRegD zero, vRegD one) %{
14052   match(Set dst (SignumD src (Binary zero one)));
14053   effect(TEMP_DEF dst, USE src, USE zero, USE one);
14054   format %{ "signumD  $dst, $src" %}
14055   ins_encode %{
14056     FloatRegister src = as_FloatRegister($src$$reg),
14057                   dst = as_FloatRegister($dst$$reg),
14058                   zero = as_FloatRegister($zero$$reg),
14059                   one = as_FloatRegister($one$$reg);
14060     __ facgtd(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14061     __ ushrd(dst, dst, 1);     // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14062     // Bit selection instruction gets bit from "one" for each enabled bit in
14063     // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14064     // NaN the whole "src" will be copied because "dst" is zero. For all other
14065     // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14066     // from "src", and all other bits are copied from 1.0.
14067     __ bsl(dst, __ T8B, one, src);
14068   %}
14069   ins_pipe(fp_uop_d);
14070 %}
14071 
14072 instruct signumF_reg(vRegF dst, vRegF src, vRegF zero, vRegF one) %{
14073   match(Set dst (SignumF src (Binary zero one)));
14074   effect(TEMP_DEF dst, USE src, USE zero, USE one);
14075   format %{ "signumF  $dst, $src" %}
14076   ins_encode %{
14077     FloatRegister src = as_FloatRegister($src$$reg),
14078                   dst = as_FloatRegister($dst$$reg),
14079                   zero = as_FloatRegister($zero$$reg),
14080                   one = as_FloatRegister($one$$reg);
14081     __ facgts(dst, src, zero);    // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14082     __ ushr(dst, __ T2S, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14083     // Bit selection instruction gets bit from "one" for each enabled bit in
14084     // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14085     // NaN the whole "src" will be copied because "dst" is zero. For all other
14086     // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14087     // from "src", and all other bits are copied from 1.0.
14088     __ bsl(dst, __ T8B, one, src);
14089   %}
14090   ins_pipe(fp_uop_d);
14091 %}
14092 
14093 instruct onspinwait() %{
14094   match(OnSpinWait);
14095   ins_cost(INSN_COST);
14096 
14097   format %{ "onspinwait" %}
14098 
14099   ins_encode %{
14100     __ spin_wait();
14101   %}
14102   ins_pipe(pipe_class_empty);
14103 %}
14104 
14105 // ============================================================================
14106 // Logical Instructions
14107 
14108 // Integer Logical Instructions
14109 
14110 // And Instructions
14111 
14112 
14113 instruct andI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, rFlagsReg cr) %{
14114   match(Set dst (AndI src1 src2));
14115 
14116   format %{ "andw  $dst, $src1, $src2\t# int" %}
14117 
14118   ins_cost(INSN_COST);
14119   ins_encode %{
14120     __ andw(as_Register($dst$$reg),
14121             as_Register($src1$$reg),
14122             as_Register($src2$$reg));
14123   %}
14124 
14125   ins_pipe(ialu_reg_reg);
14126 %}
14127 
14128 instruct andI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2, rFlagsReg cr) %{
14129   match(Set dst (AndI src1 src2));
14130 
14131   format %{ "andsw  $dst, $src1, $src2\t# int" %}
14132 
14133   ins_cost(INSN_COST);
14134   ins_encode %{
14135     __ andw(as_Register($dst$$reg),
14136             as_Register($src1$$reg),
14137             (uint64_t)($src2$$constant));
14138   %}
14139 
14140   ins_pipe(ialu_reg_imm);
14141 %}
14142 
14143 // Or Instructions
14144 
14145 instruct orI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14146   match(Set dst (OrI src1 src2));
14147 
14148   format %{ "orrw  $dst, $src1, $src2\t# int" %}
14149 
14150   ins_cost(INSN_COST);
14151   ins_encode %{
14152     __ orrw(as_Register($dst$$reg),
14153             as_Register($src1$$reg),
14154             as_Register($src2$$reg));
14155   %}
14156 
14157   ins_pipe(ialu_reg_reg);
14158 %}
14159 
14160 instruct orI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14161   match(Set dst (OrI src1 src2));
14162 
14163   format %{ "orrw  $dst, $src1, $src2\t# int" %}
14164 
14165   ins_cost(INSN_COST);
14166   ins_encode %{
14167     __ orrw(as_Register($dst$$reg),
14168             as_Register($src1$$reg),
14169             (uint64_t)($src2$$constant));
14170   %}
14171 
14172   ins_pipe(ialu_reg_imm);
14173 %}
14174 
14175 // Xor Instructions
14176 
14177 instruct xorI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14178   match(Set dst (XorI src1 src2));
14179 
14180   format %{ "eorw  $dst, $src1, $src2\t# int" %}
14181 
14182   ins_cost(INSN_COST);
14183   ins_encode %{
14184     __ eorw(as_Register($dst$$reg),
14185             as_Register($src1$$reg),
14186             as_Register($src2$$reg));
14187   %}
14188 
14189   ins_pipe(ialu_reg_reg);
14190 %}
14191 
14192 instruct xorI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14193   match(Set dst (XorI src1 src2));
14194 
14195   format %{ "eorw  $dst, $src1, $src2\t# int" %}
14196 
14197   ins_cost(INSN_COST);
14198   ins_encode %{
14199     __ eorw(as_Register($dst$$reg),
14200             as_Register($src1$$reg),
14201             (uint64_t)($src2$$constant));
14202   %}
14203 
14204   ins_pipe(ialu_reg_imm);
14205 %}
14206 
14207 // Long Logical Instructions
14208 // TODO
14209 
14210 instruct andL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr) %{
14211   match(Set dst (AndL src1 src2));
14212 
14213   format %{ "and  $dst, $src1, $src2\t# int" %}
14214 
14215   ins_cost(INSN_COST);
14216   ins_encode %{
14217     __ andr(as_Register($dst$$reg),
14218             as_Register($src1$$reg),
14219             as_Register($src2$$reg));
14220   %}
14221 
14222   ins_pipe(ialu_reg_reg);
14223 %}
14224 
14225 instruct andL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2, rFlagsReg cr) %{
14226   match(Set dst (AndL src1 src2));
14227 
14228   format %{ "and  $dst, $src1, $src2\t# int" %}
14229 
14230   ins_cost(INSN_COST);
14231   ins_encode %{
14232     __ andr(as_Register($dst$$reg),
14233             as_Register($src1$$reg),
14234             (uint64_t)($src2$$constant));
14235   %}
14236 
14237   ins_pipe(ialu_reg_imm);
14238 %}
14239 
14240 // Or Instructions
14241 
14242 instruct orL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14243   match(Set dst (OrL src1 src2));
14244 
14245   format %{ "orr  $dst, $src1, $src2\t# int" %}
14246 
14247   ins_cost(INSN_COST);
14248   ins_encode %{
14249     __ orr(as_Register($dst$$reg),
14250            as_Register($src1$$reg),
14251            as_Register($src2$$reg));
14252   %}
14253 
14254   ins_pipe(ialu_reg_reg);
14255 %}
14256 
14257 instruct orL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14258   match(Set dst (OrL src1 src2));
14259 
14260   format %{ "orr  $dst, $src1, $src2\t# int" %}
14261 
14262   ins_cost(INSN_COST);
14263   ins_encode %{
14264     __ orr(as_Register($dst$$reg),
14265            as_Register($src1$$reg),
14266            (uint64_t)($src2$$constant));
14267   %}
14268 
14269   ins_pipe(ialu_reg_imm);
14270 %}
14271 
14272 // Xor Instructions
14273 
14274 instruct xorL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14275   match(Set dst (XorL src1 src2));
14276 
14277   format %{ "eor  $dst, $src1, $src2\t# int" %}
14278 
14279   ins_cost(INSN_COST);
14280   ins_encode %{
14281     __ eor(as_Register($dst$$reg),
14282            as_Register($src1$$reg),
14283            as_Register($src2$$reg));
14284   %}
14285 
14286   ins_pipe(ialu_reg_reg);
14287 %}
14288 
14289 instruct xorL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14290   match(Set dst (XorL src1 src2));
14291 
14292   ins_cost(INSN_COST);
14293   format %{ "eor  $dst, $src1, $src2\t# int" %}
14294 
14295   ins_encode %{
14296     __ eor(as_Register($dst$$reg),
14297            as_Register($src1$$reg),
14298            (uint64_t)($src2$$constant));
14299   %}
14300 
14301   ins_pipe(ialu_reg_imm);
14302 %}
14303 
14304 instruct convI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src)
14305 %{
14306   match(Set dst (ConvI2L src));
14307 
14308   ins_cost(INSN_COST);
14309   format %{ "sxtw  $dst, $src\t# i2l" %}
14310   ins_encode %{
14311     __ sbfm($dst$$Register, $src$$Register, 0, 31);
14312   %}
14313   ins_pipe(ialu_reg_shift);
14314 %}
14315 
14316 // this pattern occurs in bigmath arithmetic
14317 instruct convUI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src, immL_32bits mask)
14318 %{
14319   match(Set dst (AndL (ConvI2L src) mask));
14320 
14321   ins_cost(INSN_COST);
14322   format %{ "ubfm  $dst, $src, 0, 31\t# ui2l" %}
14323   ins_encode %{
14324     __ ubfm($dst$$Register, $src$$Register, 0, 31);
14325   %}
14326 
14327   ins_pipe(ialu_reg_shift);
14328 %}
14329 
14330 instruct convL2I_reg(iRegINoSp dst, iRegL src) %{
14331   match(Set dst (ConvL2I src));
14332 
14333   ins_cost(INSN_COST);
14334   format %{ "movw  $dst, $src \t// l2i" %}
14335 
14336   ins_encode %{
14337     __ movw(as_Register($dst$$reg), as_Register($src$$reg));
14338   %}
14339 
14340   ins_pipe(ialu_reg);
14341 %}
14342 
14343 instruct convD2F_reg(vRegF dst, vRegD src) %{
14344   match(Set dst (ConvD2F src));
14345 
14346   ins_cost(INSN_COST * 5);
14347   format %{ "fcvtd  $dst, $src \t// d2f" %}
14348 
14349   ins_encode %{
14350     __ fcvtd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14351   %}
14352 
14353   ins_pipe(fp_d2f);
14354 %}
14355 
14356 instruct convF2D_reg(vRegD dst, vRegF src) %{
14357   match(Set dst (ConvF2D src));
14358 
14359   ins_cost(INSN_COST * 5);
14360   format %{ "fcvts  $dst, $src \t// f2d" %}
14361 
14362   ins_encode %{
14363     __ fcvts(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14364   %}
14365 
14366   ins_pipe(fp_f2d);
14367 %}
14368 
14369 instruct convF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14370   match(Set dst (ConvF2I src));
14371 
14372   ins_cost(INSN_COST * 5);
14373   format %{ "fcvtzsw  $dst, $src \t// f2i" %}
14374 
14375   ins_encode %{
14376     __ fcvtzsw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14377   %}
14378 
14379   ins_pipe(fp_f2i);
14380 %}
14381 
14382 instruct convF2L_reg_reg(iRegLNoSp dst, vRegF src) %{
14383   match(Set dst (ConvF2L src));
14384 
14385   ins_cost(INSN_COST * 5);
14386   format %{ "fcvtzs  $dst, $src \t// f2l" %}
14387 
14388   ins_encode %{
14389     __ fcvtzs(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14390   %}
14391 
14392   ins_pipe(fp_f2l);
14393 %}
14394 
14395 instruct convF2HF_reg_reg(iRegINoSp dst, vRegF src, vRegF tmp) %{
14396   match(Set dst (ConvF2HF src));
14397   format %{ "fcvt $tmp, $src\t# convert single to half precision\n\t"
14398             "smov $dst, $tmp\t# move result from $tmp to $dst"
14399   %}
14400   effect(TEMP tmp);
14401   ins_encode %{
14402       __ flt_to_flt16($dst$$Register, $src$$FloatRegister, $tmp$$FloatRegister);
14403   %}
14404   ins_pipe(pipe_slow);
14405 %}
14406 
14407 instruct convHF2F_reg_reg(vRegF dst, iRegINoSp src, vRegF tmp) %{
14408   match(Set dst (ConvHF2F src));
14409   format %{ "mov $tmp, $src\t# move source from $src to $tmp\n\t"
14410             "fcvt $dst, $tmp\t# convert half to single precision"
14411   %}
14412   effect(TEMP tmp);
14413   ins_encode %{
14414       __ flt16_to_flt($dst$$FloatRegister, $src$$Register, $tmp$$FloatRegister);
14415   %}
14416   ins_pipe(pipe_slow);
14417 %}
14418 
14419 instruct convI2F_reg_reg(vRegF dst, iRegIorL2I src) %{
14420   match(Set dst (ConvI2F src));
14421 
14422   ins_cost(INSN_COST * 5);
14423   format %{ "scvtfws  $dst, $src \t// i2f" %}
14424 
14425   ins_encode %{
14426     __ scvtfws(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14427   %}
14428 
14429   ins_pipe(fp_i2f);
14430 %}
14431 
14432 instruct convL2F_reg_reg(vRegF dst, iRegL src) %{
14433   match(Set dst (ConvL2F src));
14434 
14435   ins_cost(INSN_COST * 5);
14436   format %{ "scvtfs  $dst, $src \t// l2f" %}
14437 
14438   ins_encode %{
14439     __ scvtfs(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14440   %}
14441 
14442   ins_pipe(fp_l2f);
14443 %}
14444 
14445 instruct convD2I_reg_reg(iRegINoSp dst, vRegD src) %{
14446   match(Set dst (ConvD2I src));
14447 
14448   ins_cost(INSN_COST * 5);
14449   format %{ "fcvtzdw  $dst, $src \t// d2i" %}
14450 
14451   ins_encode %{
14452     __ fcvtzdw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14453   %}
14454 
14455   ins_pipe(fp_d2i);
14456 %}
14457 
14458 instruct convD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14459   match(Set dst (ConvD2L src));
14460 
14461   ins_cost(INSN_COST * 5);
14462   format %{ "fcvtzd  $dst, $src \t// d2l" %}
14463 
14464   ins_encode %{
14465     __ fcvtzd(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14466   %}
14467 
14468   ins_pipe(fp_d2l);
14469 %}
14470 
14471 instruct convI2D_reg_reg(vRegD dst, iRegIorL2I src) %{
14472   match(Set dst (ConvI2D src));
14473 
14474   ins_cost(INSN_COST * 5);
14475   format %{ "scvtfwd  $dst, $src \t// i2d" %}
14476 
14477   ins_encode %{
14478     __ scvtfwd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14479   %}
14480 
14481   ins_pipe(fp_i2d);
14482 %}
14483 
14484 instruct convL2D_reg_reg(vRegD dst, iRegL src) %{
14485   match(Set dst (ConvL2D src));
14486 
14487   ins_cost(INSN_COST * 5);
14488   format %{ "scvtfd  $dst, $src \t// l2d" %}
14489 
14490   ins_encode %{
14491     __ scvtfd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14492   %}
14493 
14494   ins_pipe(fp_l2d);
14495 %}
14496 
14497 instruct round_double_reg(iRegLNoSp dst, vRegD src, vRegD ftmp, rFlagsReg cr)
14498 %{
14499   match(Set dst (RoundD src));
14500   effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
14501   format %{ "java_round_double $dst,$src"%}
14502   ins_encode %{
14503     __ java_round_double($dst$$Register, as_FloatRegister($src$$reg),
14504                          as_FloatRegister($ftmp$$reg));
14505   %}
14506   ins_pipe(pipe_slow);
14507 %}
14508 
14509 instruct round_float_reg(iRegINoSp dst, vRegF src, vRegF ftmp, rFlagsReg cr)
14510 %{
14511   match(Set dst (RoundF src));
14512   effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
14513   format %{ "java_round_float $dst,$src"%}
14514   ins_encode %{
14515     __ java_round_float($dst$$Register, as_FloatRegister($src$$reg),
14516                         as_FloatRegister($ftmp$$reg));
14517   %}
14518   ins_pipe(pipe_slow);
14519 %}
14520 
14521 // stack <-> reg and reg <-> reg shuffles with no conversion
14522 
14523 instruct MoveF2I_stack_reg(iRegINoSp dst, stackSlotF src) %{
14524 
14525   match(Set dst (MoveF2I src));
14526 
14527   effect(DEF dst, USE src);
14528 
14529   ins_cost(4 * INSN_COST);
14530 
14531   format %{ "ldrw $dst, $src\t# MoveF2I_stack_reg" %}
14532 
14533   ins_encode %{
14534     __ ldrw($dst$$Register, Address(sp, $src$$disp));
14535   %}
14536 
14537   ins_pipe(iload_reg_reg);
14538 
14539 %}
14540 
14541 instruct MoveI2F_stack_reg(vRegF dst, stackSlotI src) %{
14542 
14543   match(Set dst (MoveI2F src));
14544 
14545   effect(DEF dst, USE src);
14546 
14547   ins_cost(4 * INSN_COST);
14548 
14549   format %{ "ldrs $dst, $src\t# MoveI2F_stack_reg" %}
14550 
14551   ins_encode %{
14552     __ ldrs(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
14553   %}
14554 
14555   ins_pipe(pipe_class_memory);
14556 
14557 %}
14558 
14559 instruct MoveD2L_stack_reg(iRegLNoSp dst, stackSlotD src) %{
14560 
14561   match(Set dst (MoveD2L src));
14562 
14563   effect(DEF dst, USE src);
14564 
14565   ins_cost(4 * INSN_COST);
14566 
14567   format %{ "ldr $dst, $src\t# MoveD2L_stack_reg" %}
14568 
14569   ins_encode %{
14570     __ ldr($dst$$Register, Address(sp, $src$$disp));
14571   %}
14572 
14573   ins_pipe(iload_reg_reg);
14574 
14575 %}
14576 
14577 instruct MoveL2D_stack_reg(vRegD dst, stackSlotL src) %{
14578 
14579   match(Set dst (MoveL2D src));
14580 
14581   effect(DEF dst, USE src);
14582 
14583   ins_cost(4 * INSN_COST);
14584 
14585   format %{ "ldrd $dst, $src\t# MoveL2D_stack_reg" %}
14586 
14587   ins_encode %{
14588     __ ldrd(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
14589   %}
14590 
14591   ins_pipe(pipe_class_memory);
14592 
14593 %}
14594 
14595 instruct MoveF2I_reg_stack(stackSlotI dst, vRegF src) %{
14596 
14597   match(Set dst (MoveF2I src));
14598 
14599   effect(DEF dst, USE src);
14600 
14601   ins_cost(INSN_COST);
14602 
14603   format %{ "strs $src, $dst\t# MoveF2I_reg_stack" %}
14604 
14605   ins_encode %{
14606     __ strs(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
14607   %}
14608 
14609   ins_pipe(pipe_class_memory);
14610 
14611 %}
14612 
14613 instruct MoveI2F_reg_stack(stackSlotF dst, iRegI src) %{
14614 
14615   match(Set dst (MoveI2F src));
14616 
14617   effect(DEF dst, USE src);
14618 
14619   ins_cost(INSN_COST);
14620 
14621   format %{ "strw $src, $dst\t# MoveI2F_reg_stack" %}
14622 
14623   ins_encode %{
14624     __ strw($src$$Register, Address(sp, $dst$$disp));
14625   %}
14626 
14627   ins_pipe(istore_reg_reg);
14628 
14629 %}
14630 
14631 instruct MoveD2L_reg_stack(stackSlotL dst, vRegD src) %{
14632 
14633   match(Set dst (MoveD2L src));
14634 
14635   effect(DEF dst, USE src);
14636 
14637   ins_cost(INSN_COST);
14638 
14639   format %{ "strd $dst, $src\t# MoveD2L_reg_stack" %}
14640 
14641   ins_encode %{
14642     __ strd(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
14643   %}
14644 
14645   ins_pipe(pipe_class_memory);
14646 
14647 %}
14648 
14649 instruct MoveL2D_reg_stack(stackSlotD dst, iRegL src) %{
14650 
14651   match(Set dst (MoveL2D src));
14652 
14653   effect(DEF dst, USE src);
14654 
14655   ins_cost(INSN_COST);
14656 
14657   format %{ "str $src, $dst\t# MoveL2D_reg_stack" %}
14658 
14659   ins_encode %{
14660     __ str($src$$Register, Address(sp, $dst$$disp));
14661   %}
14662 
14663   ins_pipe(istore_reg_reg);
14664 
14665 %}
14666 
14667 instruct MoveF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14668 
14669   match(Set dst (MoveF2I src));
14670 
14671   effect(DEF dst, USE src);
14672 
14673   ins_cost(INSN_COST);
14674 
14675   format %{ "fmovs $dst, $src\t# MoveF2I_reg_reg" %}
14676 
14677   ins_encode %{
14678     __ fmovs($dst$$Register, as_FloatRegister($src$$reg));
14679   %}
14680 
14681   ins_pipe(fp_f2i);
14682 
14683 %}
14684 
14685 instruct MoveI2F_reg_reg(vRegF dst, iRegI src) %{
14686 
14687   match(Set dst (MoveI2F src));
14688 
14689   effect(DEF dst, USE src);
14690 
14691   ins_cost(INSN_COST);
14692 
14693   format %{ "fmovs $dst, $src\t# MoveI2F_reg_reg" %}
14694 
14695   ins_encode %{
14696     __ fmovs(as_FloatRegister($dst$$reg), $src$$Register);
14697   %}
14698 
14699   ins_pipe(fp_i2f);
14700 
14701 %}
14702 
14703 instruct MoveD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14704 
14705   match(Set dst (MoveD2L src));
14706 
14707   effect(DEF dst, USE src);
14708 
14709   ins_cost(INSN_COST);
14710 
14711   format %{ "fmovd $dst, $src\t# MoveD2L_reg_reg" %}
14712 
14713   ins_encode %{
14714     __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
14715   %}
14716 
14717   ins_pipe(fp_d2l);
14718 
14719 %}
14720 
14721 instruct MoveL2D_reg_reg(vRegD dst, iRegL src) %{
14722 
14723   match(Set dst (MoveL2D src));
14724 
14725   effect(DEF dst, USE src);
14726 
14727   ins_cost(INSN_COST);
14728 
14729   format %{ "fmovd $dst, $src\t# MoveL2D_reg_reg" %}
14730 
14731   ins_encode %{
14732     __ fmovd(as_FloatRegister($dst$$reg), $src$$Register);
14733   %}
14734 
14735   ins_pipe(fp_l2d);
14736 
14737 %}
14738 
14739 // ============================================================================
14740 // clearing of an array
14741 
14742 instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, Universe dummy, rFlagsReg cr)
14743 %{
14744   match(Set dummy (ClearArray cnt base));
14745   effect(USE_KILL cnt, USE_KILL base, KILL cr);
14746 
14747   ins_cost(4 * INSN_COST);
14748   format %{ "ClearArray $cnt, $base" %}
14749 
14750   ins_encode %{
14751     address tpc = __ zero_words($base$$Register, $cnt$$Register);
14752     if (tpc == nullptr) {
14753       ciEnv::current()->record_failure("CodeCache is full");
14754       return;
14755     }
14756   %}
14757 
14758   ins_pipe(pipe_class_memory);
14759 %}
14760 
14761 instruct clearArray_imm_reg(immL cnt, iRegP_R10 base, iRegL_R11 temp, Universe dummy, rFlagsReg cr)
14762 %{
14763   predicate((uint64_t)n->in(2)->get_long()
14764             < (uint64_t)(BlockZeroingLowLimit >> LogBytesPerWord));
14765   match(Set dummy (ClearArray cnt base));
14766   effect(TEMP temp, USE_KILL base, KILL cr);
14767 
14768   ins_cost(4 * INSN_COST);
14769   format %{ "ClearArray $cnt, $base" %}
14770 
14771   ins_encode %{
14772     address tpc = __ zero_words($base$$Register, (uint64_t)$cnt$$constant);
14773     if (tpc == nullptr) {
14774       ciEnv::current()->record_failure("CodeCache is full");
14775       return;
14776     }
14777   %}
14778 
14779   ins_pipe(pipe_class_memory);
14780 %}
14781 
14782 // ============================================================================
14783 // Overflow Math Instructions
14784 
14785 instruct overflowAddI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14786 %{
14787   match(Set cr (OverflowAddI op1 op2));
14788 
14789   format %{ "cmnw  $op1, $op2\t# overflow check int" %}
14790   ins_cost(INSN_COST);
14791   ins_encode %{
14792     __ cmnw($op1$$Register, $op2$$Register);
14793   %}
14794 
14795   ins_pipe(icmp_reg_reg);
14796 %}
14797 
14798 instruct overflowAddI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
14799 %{
14800   match(Set cr (OverflowAddI op1 op2));
14801 
14802   format %{ "cmnw  $op1, $op2\t# overflow check int" %}
14803   ins_cost(INSN_COST);
14804   ins_encode %{
14805     __ cmnw($op1$$Register, $op2$$constant);
14806   %}
14807 
14808   ins_pipe(icmp_reg_imm);
14809 %}
14810 
14811 instruct overflowAddL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14812 %{
14813   match(Set cr (OverflowAddL op1 op2));
14814 
14815   format %{ "cmn   $op1, $op2\t# overflow check long" %}
14816   ins_cost(INSN_COST);
14817   ins_encode %{
14818     __ cmn($op1$$Register, $op2$$Register);
14819   %}
14820 
14821   ins_pipe(icmp_reg_reg);
14822 %}
14823 
14824 instruct overflowAddL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
14825 %{
14826   match(Set cr (OverflowAddL op1 op2));
14827 
14828   format %{ "adds  zr, $op1, $op2\t# overflow check long" %}
14829   ins_cost(INSN_COST);
14830   ins_encode %{
14831     __ adds(zr, $op1$$Register, $op2$$constant);
14832   %}
14833 
14834   ins_pipe(icmp_reg_imm);
14835 %}
14836 
14837 instruct overflowSubI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14838 %{
14839   match(Set cr (OverflowSubI op1 op2));
14840 
14841   format %{ "cmpw  $op1, $op2\t# overflow check int" %}
14842   ins_cost(INSN_COST);
14843   ins_encode %{
14844     __ cmpw($op1$$Register, $op2$$Register);
14845   %}
14846 
14847   ins_pipe(icmp_reg_reg);
14848 %}
14849 
14850 instruct overflowSubI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
14851 %{
14852   match(Set cr (OverflowSubI op1 op2));
14853 
14854   format %{ "cmpw  $op1, $op2\t# overflow check int" %}
14855   ins_cost(INSN_COST);
14856   ins_encode %{
14857     __ cmpw($op1$$Register, $op2$$constant);
14858   %}
14859 
14860   ins_pipe(icmp_reg_imm);
14861 %}
14862 
14863 instruct overflowSubL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14864 %{
14865   match(Set cr (OverflowSubL op1 op2));
14866 
14867   format %{ "cmp   $op1, $op2\t# overflow check long" %}
14868   ins_cost(INSN_COST);
14869   ins_encode %{
14870     __ cmp($op1$$Register, $op2$$Register);
14871   %}
14872 
14873   ins_pipe(icmp_reg_reg);
14874 %}
14875 
14876 instruct overflowSubL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
14877 %{
14878   match(Set cr (OverflowSubL op1 op2));
14879 
14880   format %{ "cmp   $op1, $op2\t# overflow check long" %}
14881   ins_cost(INSN_COST);
14882   ins_encode %{
14883     __ subs(zr, $op1$$Register, $op2$$constant);
14884   %}
14885 
14886   ins_pipe(icmp_reg_imm);
14887 %}
14888 
14889 instruct overflowNegI_reg(rFlagsReg cr, immI0 zero, iRegIorL2I op1)
14890 %{
14891   match(Set cr (OverflowSubI zero op1));
14892 
14893   format %{ "cmpw  zr, $op1\t# overflow check int" %}
14894   ins_cost(INSN_COST);
14895   ins_encode %{
14896     __ cmpw(zr, $op1$$Register);
14897   %}
14898 
14899   ins_pipe(icmp_reg_imm);
14900 %}
14901 
14902 instruct overflowNegL_reg(rFlagsReg cr, immI0 zero, iRegL op1)
14903 %{
14904   match(Set cr (OverflowSubL zero op1));
14905 
14906   format %{ "cmp   zr, $op1\t# overflow check long" %}
14907   ins_cost(INSN_COST);
14908   ins_encode %{
14909     __ cmp(zr, $op1$$Register);
14910   %}
14911 
14912   ins_pipe(icmp_reg_imm);
14913 %}
14914 
14915 instruct overflowMulI_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14916 %{
14917   match(Set cr (OverflowMulI op1 op2));
14918 
14919   format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
14920             "cmp   rscratch1, rscratch1, sxtw\n\t"
14921             "movw  rscratch1, #0x80000000\n\t"
14922             "cselw rscratch1, rscratch1, zr, NE\n\t"
14923             "cmpw  rscratch1, #1" %}
14924   ins_cost(5 * INSN_COST);
14925   ins_encode %{
14926     __ smull(rscratch1, $op1$$Register, $op2$$Register);
14927     __ subs(zr, rscratch1, rscratch1, ext::sxtw);      // NE => overflow
14928     __ movw(rscratch1, 0x80000000);                    // Develop 0 (EQ),
14929     __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
14930     __ cmpw(rscratch1, 1);                             // 0x80000000 - 1 => VS
14931   %}
14932 
14933   ins_pipe(pipe_slow);
14934 %}
14935 
14936 instruct overflowMulI_reg_branch(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, label labl, rFlagsReg cr)
14937 %{
14938   match(If cmp (OverflowMulI op1 op2));
14939   predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
14940             || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
14941   effect(USE labl, KILL cr);
14942 
14943   format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
14944             "cmp   rscratch1, rscratch1, sxtw\n\t"
14945             "b$cmp   $labl" %}
14946   ins_cost(3 * INSN_COST); // Branch is rare so treat as INSN_COST
14947   ins_encode %{
14948     Label* L = $labl$$label;
14949     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14950     __ smull(rscratch1, $op1$$Register, $op2$$Register);
14951     __ subs(zr, rscratch1, rscratch1, ext::sxtw);      // NE => overflow
14952     __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
14953   %}
14954 
14955   ins_pipe(pipe_serial);
14956 %}
14957 
14958 instruct overflowMulL_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14959 %{
14960   match(Set cr (OverflowMulL op1 op2));
14961 
14962   format %{ "mul   rscratch1, $op1, $op2\t#overflow check long\n\t"
14963             "smulh rscratch2, $op1, $op2\n\t"
14964             "cmp   rscratch2, rscratch1, ASR #63\n\t"
14965             "movw  rscratch1, #0x80000000\n\t"
14966             "cselw rscratch1, rscratch1, zr, NE\n\t"
14967             "cmpw  rscratch1, #1" %}
14968   ins_cost(6 * INSN_COST);
14969   ins_encode %{
14970     __ mul(rscratch1, $op1$$Register, $op2$$Register);   // Result bits 0..63
14971     __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
14972     __ cmp(rscratch2, rscratch1, Assembler::ASR, 63);    // Top is pure sign ext
14973     __ movw(rscratch1, 0x80000000);                    // Develop 0 (EQ),
14974     __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
14975     __ cmpw(rscratch1, 1);                             // 0x80000000 - 1 => VS
14976   %}
14977 
14978   ins_pipe(pipe_slow);
14979 %}
14980 
14981 instruct overflowMulL_reg_branch(cmpOp cmp, iRegL op1, iRegL op2, label labl, rFlagsReg cr)
14982 %{
14983   match(If cmp (OverflowMulL op1 op2));
14984   predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
14985             || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
14986   effect(USE labl, KILL cr);
14987 
14988   format %{ "mul   rscratch1, $op1, $op2\t#overflow check long\n\t"
14989             "smulh rscratch2, $op1, $op2\n\t"
14990             "cmp   rscratch2, rscratch1, ASR #63\n\t"
14991             "b$cmp $labl" %}
14992   ins_cost(4 * INSN_COST); // Branch is rare so treat as INSN_COST
14993   ins_encode %{
14994     Label* L = $labl$$label;
14995     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14996     __ mul(rscratch1, $op1$$Register, $op2$$Register);   // Result bits 0..63
14997     __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
14998     __ cmp(rscratch2, rscratch1, Assembler::ASR, 63);    // Top is pure sign ext
14999     __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15000   %}
15001 
15002   ins_pipe(pipe_serial);
15003 %}
15004 
15005 // ============================================================================
15006 // Compare Instructions
15007 
15008 instruct compI_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
15009 %{
15010   match(Set cr (CmpI op1 op2));
15011 
15012   effect(DEF cr, USE op1, USE op2);
15013 
15014   ins_cost(INSN_COST);
15015   format %{ "cmpw  $op1, $op2" %}
15016 
15017   ins_encode(aarch64_enc_cmpw(op1, op2));
15018 
15019   ins_pipe(icmp_reg_reg);
15020 %}
15021 
15022 instruct compI_reg_immI0(rFlagsReg cr, iRegI op1, immI0 zero)
15023 %{
15024   match(Set cr (CmpI op1 zero));
15025 
15026   effect(DEF cr, USE op1);
15027 
15028   ins_cost(INSN_COST);
15029   format %{ "cmpw $op1, 0" %}
15030 
15031   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15032 
15033   ins_pipe(icmp_reg_imm);
15034 %}
15035 
15036 instruct compI_reg_immIAddSub(rFlagsReg cr, iRegI op1, immIAddSub op2)
15037 %{
15038   match(Set cr (CmpI op1 op2));
15039 
15040   effect(DEF cr, USE op1);
15041 
15042   ins_cost(INSN_COST);
15043   format %{ "cmpw  $op1, $op2" %}
15044 
15045   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15046 
15047   ins_pipe(icmp_reg_imm);
15048 %}
15049 
15050 instruct compI_reg_immI(rFlagsReg cr, iRegI op1, immI op2)
15051 %{
15052   match(Set cr (CmpI op1 op2));
15053 
15054   effect(DEF cr, USE op1);
15055 
15056   ins_cost(INSN_COST * 2);
15057   format %{ "cmpw  $op1, $op2" %}
15058 
15059   ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15060 
15061   ins_pipe(icmp_reg_imm);
15062 %}
15063 
15064 // Unsigned compare Instructions; really, same as signed compare
15065 // except it should only be used to feed an If or a CMovI which takes a
15066 // cmpOpU.
15067 
15068 instruct compU_reg_reg(rFlagsRegU cr, iRegI op1, iRegI op2)
15069 %{
15070   match(Set cr (CmpU op1 op2));
15071 
15072   effect(DEF cr, USE op1, USE op2);
15073 
15074   ins_cost(INSN_COST);
15075   format %{ "cmpw  $op1, $op2\t# unsigned" %}
15076 
15077   ins_encode(aarch64_enc_cmpw(op1, op2));
15078 
15079   ins_pipe(icmp_reg_reg);
15080 %}
15081 
15082 instruct compU_reg_immI0(rFlagsRegU cr, iRegI op1, immI0 zero)
15083 %{
15084   match(Set cr (CmpU op1 zero));
15085 
15086   effect(DEF cr, USE op1);
15087 
15088   ins_cost(INSN_COST);
15089   format %{ "cmpw $op1, #0\t# unsigned" %}
15090 
15091   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15092 
15093   ins_pipe(icmp_reg_imm);
15094 %}
15095 
15096 instruct compU_reg_immIAddSub(rFlagsRegU cr, iRegI op1, immIAddSub op2)
15097 %{
15098   match(Set cr (CmpU op1 op2));
15099 
15100   effect(DEF cr, USE op1);
15101 
15102   ins_cost(INSN_COST);
15103   format %{ "cmpw  $op1, $op2\t# unsigned" %}
15104 
15105   ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15106 
15107   ins_pipe(icmp_reg_imm);
15108 %}
15109 
15110 instruct compU_reg_immI(rFlagsRegU cr, iRegI op1, immI op2)
15111 %{
15112   match(Set cr (CmpU op1 op2));
15113 
15114   effect(DEF cr, USE op1);
15115 
15116   ins_cost(INSN_COST * 2);
15117   format %{ "cmpw  $op1, $op2\t# unsigned" %}
15118 
15119   ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15120 
15121   ins_pipe(icmp_reg_imm);
15122 %}
15123 
15124 instruct compL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15125 %{
15126   match(Set cr (CmpL op1 op2));
15127 
15128   effect(DEF cr, USE op1, USE op2);
15129 
15130   ins_cost(INSN_COST);
15131   format %{ "cmp  $op1, $op2" %}
15132 
15133   ins_encode(aarch64_enc_cmp(op1, op2));
15134 
15135   ins_pipe(icmp_reg_reg);
15136 %}
15137 
15138 instruct compL_reg_immL0(rFlagsReg cr, iRegL op1, immL0 zero)
15139 %{
15140   match(Set cr (CmpL op1 zero));
15141 
15142   effect(DEF cr, USE op1);
15143 
15144   ins_cost(INSN_COST);
15145   format %{ "tst  $op1" %}
15146 
15147   ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15148 
15149   ins_pipe(icmp_reg_imm);
15150 %}
15151 
15152 instruct compL_reg_immLAddSub(rFlagsReg cr, iRegL op1, immLAddSub op2)
15153 %{
15154   match(Set cr (CmpL op1 op2));
15155 
15156   effect(DEF cr, USE op1);
15157 
15158   ins_cost(INSN_COST);
15159   format %{ "cmp  $op1, $op2" %}
15160 
15161   ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15162 
15163   ins_pipe(icmp_reg_imm);
15164 %}
15165 
15166 instruct compL_reg_immL(rFlagsReg cr, iRegL op1, immL op2)
15167 %{
15168   match(Set cr (CmpL op1 op2));
15169 
15170   effect(DEF cr, USE op1);
15171 
15172   ins_cost(INSN_COST * 2);
15173   format %{ "cmp  $op1, $op2" %}
15174 
15175   ins_encode(aarch64_enc_cmp_imm(op1, op2));
15176 
15177   ins_pipe(icmp_reg_imm);
15178 %}
15179 
15180 instruct compUL_reg_reg(rFlagsRegU cr, iRegL op1, iRegL op2)
15181 %{
15182   match(Set cr (CmpUL op1 op2));
15183 
15184   effect(DEF cr, USE op1, USE op2);
15185 
15186   ins_cost(INSN_COST);
15187   format %{ "cmp  $op1, $op2" %}
15188 
15189   ins_encode(aarch64_enc_cmp(op1, op2));
15190 
15191   ins_pipe(icmp_reg_reg);
15192 %}
15193 
15194 instruct compUL_reg_immL0(rFlagsRegU cr, iRegL op1, immL0 zero)
15195 %{
15196   match(Set cr (CmpUL op1 zero));
15197 
15198   effect(DEF cr, USE op1);
15199 
15200   ins_cost(INSN_COST);
15201   format %{ "tst  $op1" %}
15202 
15203   ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15204 
15205   ins_pipe(icmp_reg_imm);
15206 %}
15207 
15208 instruct compUL_reg_immLAddSub(rFlagsRegU cr, iRegL op1, immLAddSub op2)
15209 %{
15210   match(Set cr (CmpUL op1 op2));
15211 
15212   effect(DEF cr, USE op1);
15213 
15214   ins_cost(INSN_COST);
15215   format %{ "cmp  $op1, $op2" %}
15216 
15217   ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15218 
15219   ins_pipe(icmp_reg_imm);
15220 %}
15221 
15222 instruct compUL_reg_immL(rFlagsRegU cr, iRegL op1, immL op2)
15223 %{
15224   match(Set cr (CmpUL op1 op2));
15225 
15226   effect(DEF cr, USE op1);
15227 
15228   ins_cost(INSN_COST * 2);
15229   format %{ "cmp  $op1, $op2" %}
15230 
15231   ins_encode(aarch64_enc_cmp_imm(op1, op2));
15232 
15233   ins_pipe(icmp_reg_imm);
15234 %}
15235 
15236 instruct compP_reg_reg(rFlagsRegU cr, iRegP op1, iRegP op2)
15237 %{
15238   match(Set cr (CmpP op1 op2));
15239 
15240   effect(DEF cr, USE op1, USE op2);
15241 
15242   ins_cost(INSN_COST);
15243   format %{ "cmp  $op1, $op2\t // ptr" %}
15244 
15245   ins_encode(aarch64_enc_cmpp(op1, op2));
15246 
15247   ins_pipe(icmp_reg_reg);
15248 %}
15249 
15250 instruct compN_reg_reg(rFlagsRegU cr, iRegN op1, iRegN op2)
15251 %{
15252   match(Set cr (CmpN op1 op2));
15253 
15254   effect(DEF cr, USE op1, USE op2);
15255 
15256   ins_cost(INSN_COST);
15257   format %{ "cmp  $op1, $op2\t // compressed ptr" %}
15258 
15259   ins_encode(aarch64_enc_cmpn(op1, op2));
15260 
15261   ins_pipe(icmp_reg_reg);
15262 %}
15263 
15264 instruct testP_reg(rFlagsRegU cr, iRegP op1, immP0 zero)
15265 %{
15266   match(Set cr (CmpP op1 zero));
15267 
15268   effect(DEF cr, USE op1, USE zero);
15269 
15270   ins_cost(INSN_COST);
15271   format %{ "cmp  $op1, 0\t // ptr" %}
15272 
15273   ins_encode(aarch64_enc_testp(op1));
15274 
15275   ins_pipe(icmp_reg_imm);
15276 %}
15277 
15278 instruct testN_reg(rFlagsRegU cr, iRegN op1, immN0 zero)
15279 %{
15280   match(Set cr (CmpN op1 zero));
15281 
15282   effect(DEF cr, USE op1, USE zero);
15283 
15284   ins_cost(INSN_COST);
15285   format %{ "cmp  $op1, 0\t // compressed ptr" %}
15286 
15287   ins_encode(aarch64_enc_testn(op1));
15288 
15289   ins_pipe(icmp_reg_imm);
15290 %}
15291 
15292 // FP comparisons
15293 //
15294 // n.b. CmpF/CmpD set a normal flags reg which then gets compared
15295 // using normal cmpOp. See declaration of rFlagsReg for details.
15296 
15297 instruct compF_reg_reg(rFlagsReg cr, vRegF src1, vRegF src2)
15298 %{
15299   match(Set cr (CmpF src1 src2));
15300 
15301   ins_cost(3 * INSN_COST);
15302   format %{ "fcmps $src1, $src2" %}
15303 
15304   ins_encode %{
15305     __ fcmps(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15306   %}
15307 
15308   ins_pipe(pipe_class_compare);
15309 %}
15310 
15311 instruct compF_reg_zero(rFlagsReg cr, vRegF src1, immF0 src2)
15312 %{
15313   match(Set cr (CmpF src1 src2));
15314 
15315   ins_cost(3 * INSN_COST);
15316   format %{ "fcmps $src1, 0.0" %}
15317 
15318   ins_encode %{
15319     __ fcmps(as_FloatRegister($src1$$reg), 0.0);
15320   %}
15321 
15322   ins_pipe(pipe_class_compare);
15323 %}
15324 // FROM HERE
15325 
15326 instruct compD_reg_reg(rFlagsReg cr, vRegD src1, vRegD src2)
15327 %{
15328   match(Set cr (CmpD src1 src2));
15329 
15330   ins_cost(3 * INSN_COST);
15331   format %{ "fcmpd $src1, $src2" %}
15332 
15333   ins_encode %{
15334     __ fcmpd(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15335   %}
15336 
15337   ins_pipe(pipe_class_compare);
15338 %}
15339 
15340 instruct compD_reg_zero(rFlagsReg cr, vRegD src1, immD0 src2)
15341 %{
15342   match(Set cr (CmpD src1 src2));
15343 
15344   ins_cost(3 * INSN_COST);
15345   format %{ "fcmpd $src1, 0.0" %}
15346 
15347   ins_encode %{
15348     __ fcmpd(as_FloatRegister($src1$$reg), 0.0);
15349   %}
15350 
15351   ins_pipe(pipe_class_compare);
15352 %}
15353 
15354 instruct compF3_reg_reg(iRegINoSp dst, vRegF src1, vRegF src2, rFlagsReg cr)
15355 %{
15356   match(Set dst (CmpF3 src1 src2));
15357   effect(KILL cr);
15358 
15359   ins_cost(5 * INSN_COST);
15360   format %{ "fcmps $src1, $src2\n\t"
15361             "csinvw($dst, zr, zr, eq\n\t"
15362             "csnegw($dst, $dst, $dst, lt)"
15363   %}
15364 
15365   ins_encode %{
15366     Label done;
15367     FloatRegister s1 = as_FloatRegister($src1$$reg);
15368     FloatRegister s2 = as_FloatRegister($src2$$reg);
15369     Register d = as_Register($dst$$reg);
15370     __ fcmps(s1, s2);
15371     // installs 0 if EQ else -1
15372     __ csinvw(d, zr, zr, Assembler::EQ);
15373     // keeps -1 if less or unordered else installs 1
15374     __ csnegw(d, d, d, Assembler::LT);
15375     __ bind(done);
15376   %}
15377 
15378   ins_pipe(pipe_class_default);
15379 
15380 %}
15381 
15382 instruct compD3_reg_reg(iRegINoSp dst, vRegD src1, vRegD src2, rFlagsReg cr)
15383 %{
15384   match(Set dst (CmpD3 src1 src2));
15385   effect(KILL cr);
15386 
15387   ins_cost(5 * INSN_COST);
15388   format %{ "fcmpd $src1, $src2\n\t"
15389             "csinvw($dst, zr, zr, eq\n\t"
15390             "csnegw($dst, $dst, $dst, lt)"
15391   %}
15392 
15393   ins_encode %{
15394     Label done;
15395     FloatRegister s1 = as_FloatRegister($src1$$reg);
15396     FloatRegister s2 = as_FloatRegister($src2$$reg);
15397     Register d = as_Register($dst$$reg);
15398     __ fcmpd(s1, s2);
15399     // installs 0 if EQ else -1
15400     __ csinvw(d, zr, zr, Assembler::EQ);
15401     // keeps -1 if less or unordered else installs 1
15402     __ csnegw(d, d, d, Assembler::LT);
15403     __ bind(done);
15404   %}
15405   ins_pipe(pipe_class_default);
15406 
15407 %}
15408 
15409 instruct compF3_reg_immF0(iRegINoSp dst, vRegF src1, immF0 zero, rFlagsReg cr)
15410 %{
15411   match(Set dst (CmpF3 src1 zero));
15412   effect(KILL cr);
15413 
15414   ins_cost(5 * INSN_COST);
15415   format %{ "fcmps $src1, 0.0\n\t"
15416             "csinvw($dst, zr, zr, eq\n\t"
15417             "csnegw($dst, $dst, $dst, lt)"
15418   %}
15419 
15420   ins_encode %{
15421     Label done;
15422     FloatRegister s1 = as_FloatRegister($src1$$reg);
15423     Register d = as_Register($dst$$reg);
15424     __ fcmps(s1, 0.0);
15425     // installs 0 if EQ else -1
15426     __ csinvw(d, zr, zr, Assembler::EQ);
15427     // keeps -1 if less or unordered else installs 1
15428     __ csnegw(d, d, d, Assembler::LT);
15429     __ bind(done);
15430   %}
15431 
15432   ins_pipe(pipe_class_default);
15433 
15434 %}
15435 
15436 instruct compD3_reg_immD0(iRegINoSp dst, vRegD src1, immD0 zero, rFlagsReg cr)
15437 %{
15438   match(Set dst (CmpD3 src1 zero));
15439   effect(KILL cr);
15440 
15441   ins_cost(5 * INSN_COST);
15442   format %{ "fcmpd $src1, 0.0\n\t"
15443             "csinvw($dst, zr, zr, eq\n\t"
15444             "csnegw($dst, $dst, $dst, lt)"
15445   %}
15446 
15447   ins_encode %{
15448     Label done;
15449     FloatRegister s1 = as_FloatRegister($src1$$reg);
15450     Register d = as_Register($dst$$reg);
15451     __ fcmpd(s1, 0.0);
15452     // installs 0 if EQ else -1
15453     __ csinvw(d, zr, zr, Assembler::EQ);
15454     // keeps -1 if less or unordered else installs 1
15455     __ csnegw(d, d, d, Assembler::LT);
15456     __ bind(done);
15457   %}
15458   ins_pipe(pipe_class_default);
15459 
15460 %}
15461 
15462 instruct cmpLTMask_reg_reg(iRegINoSp dst, iRegIorL2I p, iRegIorL2I q, rFlagsReg cr)
15463 %{
15464   match(Set dst (CmpLTMask p q));
15465   effect(KILL cr);
15466 
15467   ins_cost(3 * INSN_COST);
15468 
15469   format %{ "cmpw $p, $q\t# cmpLTMask\n\t"
15470             "csetw $dst, lt\n\t"
15471             "subw $dst, zr, $dst"
15472   %}
15473 
15474   ins_encode %{
15475     __ cmpw(as_Register($p$$reg), as_Register($q$$reg));
15476     __ csetw(as_Register($dst$$reg), Assembler::LT);
15477     __ subw(as_Register($dst$$reg), zr, as_Register($dst$$reg));
15478   %}
15479 
15480   ins_pipe(ialu_reg_reg);
15481 %}
15482 
15483 instruct cmpLTMask_reg_zero(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr)
15484 %{
15485   match(Set dst (CmpLTMask src zero));
15486   effect(KILL cr);
15487 
15488   ins_cost(INSN_COST);
15489 
15490   format %{ "asrw $dst, $src, #31\t# cmpLTMask0" %}
15491 
15492   ins_encode %{
15493     __ asrw(as_Register($dst$$reg), as_Register($src$$reg), 31);
15494   %}
15495 
15496   ins_pipe(ialu_reg_shift);
15497 %}
15498 
15499 // ============================================================================
15500 // Max and Min
15501 
15502 // Like compI_reg_reg or compI_reg_immI0 but without match rule and second zero parameter.
15503 
15504 instruct compI_reg_imm0(rFlagsReg cr, iRegI src)
15505 %{
15506   effect(DEF cr, USE src);
15507   ins_cost(INSN_COST);
15508   format %{ "cmpw $src, 0" %}
15509 
15510   ins_encode %{
15511     __ cmpw($src$$Register, 0);
15512   %}
15513   ins_pipe(icmp_reg_imm);
15514 %}
15515 
15516 instruct minI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
15517 %{
15518   match(Set dst (MinI src1 src2));
15519   ins_cost(INSN_COST * 3);
15520 
15521   expand %{
15522     rFlagsReg cr;
15523     compI_reg_reg(cr, src1, src2);
15524     cmovI_reg_reg_lt(dst, src1, src2, cr);
15525   %}
15526 %}
15527 
15528 instruct maxI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
15529 %{
15530   match(Set dst (MaxI src1 src2));
15531   ins_cost(INSN_COST * 3);
15532 
15533   expand %{
15534     rFlagsReg cr;
15535     compI_reg_reg(cr, src1, src2);
15536     cmovI_reg_reg_gt(dst, src1, src2, cr);
15537   %}
15538 %}
15539 
15540 
15541 // ============================================================================
15542 // Branch Instructions
15543 
15544 // Direct Branch.
15545 instruct branch(label lbl)
15546 %{
15547   match(Goto);
15548 
15549   effect(USE lbl);
15550 
15551   ins_cost(BRANCH_COST);
15552   format %{ "b  $lbl" %}
15553 
15554   ins_encode(aarch64_enc_b(lbl));
15555 
15556   ins_pipe(pipe_branch);
15557 %}
15558 
15559 // Conditional Near Branch
15560 instruct branchCon(cmpOp cmp, rFlagsReg cr, label lbl)
15561 %{
15562   // Same match rule as `branchConFar'.
15563   match(If cmp cr);
15564 
15565   effect(USE lbl);
15566 
15567   ins_cost(BRANCH_COST);
15568   // If set to 1 this indicates that the current instruction is a
15569   // short variant of a long branch. This avoids using this
15570   // instruction in first-pass matching. It will then only be used in
15571   // the `Shorten_branches' pass.
15572   // ins_short_branch(1);
15573   format %{ "b$cmp  $lbl" %}
15574 
15575   ins_encode(aarch64_enc_br_con(cmp, lbl));
15576 
15577   ins_pipe(pipe_branch_cond);
15578 %}
15579 
15580 // Conditional Near Branch Unsigned
15581 instruct branchConU(cmpOpU cmp, rFlagsRegU cr, label lbl)
15582 %{
15583   // Same match rule as `branchConFar'.
15584   match(If cmp cr);
15585 
15586   effect(USE lbl);
15587 
15588   ins_cost(BRANCH_COST);
15589   // If set to 1 this indicates that the current instruction is a
15590   // short variant of a long branch. This avoids using this
15591   // instruction in first-pass matching. It will then only be used in
15592   // the `Shorten_branches' pass.
15593   // ins_short_branch(1);
15594   format %{ "b$cmp  $lbl\t# unsigned" %}
15595 
15596   ins_encode(aarch64_enc_br_conU(cmp, lbl));
15597 
15598   ins_pipe(pipe_branch_cond);
15599 %}
15600 
15601 // Make use of CBZ and CBNZ.  These instructions, as well as being
15602 // shorter than (cmp; branch), have the additional benefit of not
15603 // killing the flags.
15604 
15605 instruct cmpI_imm0_branch(cmpOpEqNe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsReg cr) %{
15606   match(If cmp (CmpI op1 op2));
15607   effect(USE labl);
15608 
15609   ins_cost(BRANCH_COST);
15610   format %{ "cbw$cmp   $op1, $labl" %}
15611   ins_encode %{
15612     Label* L = $labl$$label;
15613     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15614     if (cond == Assembler::EQ)
15615       __ cbzw($op1$$Register, *L);
15616     else
15617       __ cbnzw($op1$$Register, *L);
15618   %}
15619   ins_pipe(pipe_cmp_branch);
15620 %}
15621 
15622 instruct cmpL_imm0_branch(cmpOpEqNe cmp, iRegL op1, immL0 op2, label labl, rFlagsReg cr) %{
15623   match(If cmp (CmpL op1 op2));
15624   effect(USE labl);
15625 
15626   ins_cost(BRANCH_COST);
15627   format %{ "cb$cmp   $op1, $labl" %}
15628   ins_encode %{
15629     Label* L = $labl$$label;
15630     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15631     if (cond == Assembler::EQ)
15632       __ cbz($op1$$Register, *L);
15633     else
15634       __ cbnz($op1$$Register, *L);
15635   %}
15636   ins_pipe(pipe_cmp_branch);
15637 %}
15638 
15639 instruct cmpP_imm0_branch(cmpOpEqNe cmp, iRegP op1, immP0 op2, label labl, rFlagsReg cr) %{
15640   match(If cmp (CmpP op1 op2));
15641   effect(USE labl);
15642 
15643   ins_cost(BRANCH_COST);
15644   format %{ "cb$cmp   $op1, $labl" %}
15645   ins_encode %{
15646     Label* L = $labl$$label;
15647     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15648     if (cond == Assembler::EQ)
15649       __ cbz($op1$$Register, *L);
15650     else
15651       __ cbnz($op1$$Register, *L);
15652   %}
15653   ins_pipe(pipe_cmp_branch);
15654 %}
15655 
15656 instruct cmpN_imm0_branch(cmpOpEqNe cmp, iRegN op1, immN0 op2, label labl, rFlagsReg cr) %{
15657   match(If cmp (CmpN op1 op2));
15658   effect(USE labl);
15659 
15660   ins_cost(BRANCH_COST);
15661   format %{ "cbw$cmp   $op1, $labl" %}
15662   ins_encode %{
15663     Label* L = $labl$$label;
15664     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15665     if (cond == Assembler::EQ)
15666       __ cbzw($op1$$Register, *L);
15667     else
15668       __ cbnzw($op1$$Register, *L);
15669   %}
15670   ins_pipe(pipe_cmp_branch);
15671 %}
15672 
15673 instruct cmpP_narrowOop_imm0_branch(cmpOpEqNe cmp, iRegN oop, immP0 zero, label labl, rFlagsReg cr) %{
15674   match(If cmp (CmpP (DecodeN oop) zero));
15675   effect(USE labl);
15676 
15677   ins_cost(BRANCH_COST);
15678   format %{ "cb$cmp   $oop, $labl" %}
15679   ins_encode %{
15680     Label* L = $labl$$label;
15681     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15682     if (cond == Assembler::EQ)
15683       __ cbzw($oop$$Register, *L);
15684     else
15685       __ cbnzw($oop$$Register, *L);
15686   %}
15687   ins_pipe(pipe_cmp_branch);
15688 %}
15689 
15690 instruct cmpUI_imm0_branch(cmpOpUEqNeLeGt cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15691   match(If cmp (CmpU op1 op2));
15692   effect(USE labl);
15693 
15694   ins_cost(BRANCH_COST);
15695   format %{ "cbw$cmp   $op1, $labl" %}
15696   ins_encode %{
15697     Label* L = $labl$$label;
15698     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15699     if (cond == Assembler::EQ || cond == Assembler::LS) {
15700       __ cbzw($op1$$Register, *L);
15701     } else {
15702       assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
15703       __ cbnzw($op1$$Register, *L);
15704     }
15705   %}
15706   ins_pipe(pipe_cmp_branch);
15707 %}
15708 
15709 instruct cmpUL_imm0_branch(cmpOpUEqNeLeGt cmp, iRegL op1, immL0 op2, label labl) %{
15710   match(If cmp (CmpUL op1 op2));
15711   effect(USE labl);
15712 
15713   ins_cost(BRANCH_COST);
15714   format %{ "cb$cmp   $op1, $labl" %}
15715   ins_encode %{
15716     Label* L = $labl$$label;
15717     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15718     if (cond == Assembler::EQ || cond == Assembler::LS) {
15719       __ cbz($op1$$Register, *L);
15720     } else {
15721       assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
15722       __ cbnz($op1$$Register, *L);
15723     }
15724   %}
15725   ins_pipe(pipe_cmp_branch);
15726 %}
15727 
15728 // Test bit and Branch
15729 
15730 // Patterns for short (< 32KiB) variants
15731 instruct cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
15732   match(If cmp (CmpL op1 op2));
15733   effect(USE labl);
15734 
15735   ins_cost(BRANCH_COST);
15736   format %{ "cb$cmp   $op1, $labl # long" %}
15737   ins_encode %{
15738     Label* L = $labl$$label;
15739     Assembler::Condition cond =
15740       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15741     __ tbr(cond, $op1$$Register, 63, *L);
15742   %}
15743   ins_pipe(pipe_cmp_branch);
15744   ins_short_branch(1);
15745 %}
15746 
15747 instruct cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15748   match(If cmp (CmpI op1 op2));
15749   effect(USE labl);
15750 
15751   ins_cost(BRANCH_COST);
15752   format %{ "cb$cmp   $op1, $labl # int" %}
15753   ins_encode %{
15754     Label* L = $labl$$label;
15755     Assembler::Condition cond =
15756       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15757     __ tbr(cond, $op1$$Register, 31, *L);
15758   %}
15759   ins_pipe(pipe_cmp_branch);
15760   ins_short_branch(1);
15761 %}
15762 
15763 instruct cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
15764   match(If cmp (CmpL (AndL op1 op2) op3));
15765   predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
15766   effect(USE labl);
15767 
15768   ins_cost(BRANCH_COST);
15769   format %{ "tb$cmp   $op1, $op2, $labl" %}
15770   ins_encode %{
15771     Label* L = $labl$$label;
15772     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15773     int bit = exact_log2_long($op2$$constant);
15774     __ tbr(cond, $op1$$Register, bit, *L);
15775   %}
15776   ins_pipe(pipe_cmp_branch);
15777   ins_short_branch(1);
15778 %}
15779 
15780 instruct cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
15781   match(If cmp (CmpI (AndI op1 op2) op3));
15782   predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
15783   effect(USE labl);
15784 
15785   ins_cost(BRANCH_COST);
15786   format %{ "tb$cmp   $op1, $op2, $labl" %}
15787   ins_encode %{
15788     Label* L = $labl$$label;
15789     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15790     int bit = exact_log2((juint)$op2$$constant);
15791     __ tbr(cond, $op1$$Register, bit, *L);
15792   %}
15793   ins_pipe(pipe_cmp_branch);
15794   ins_short_branch(1);
15795 %}
15796 
15797 // And far variants
15798 instruct far_cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
15799   match(If cmp (CmpL op1 op2));
15800   effect(USE labl);
15801 
15802   ins_cost(BRANCH_COST);
15803   format %{ "cb$cmp   $op1, $labl # long" %}
15804   ins_encode %{
15805     Label* L = $labl$$label;
15806     Assembler::Condition cond =
15807       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15808     __ tbr(cond, $op1$$Register, 63, *L, /*far*/true);
15809   %}
15810   ins_pipe(pipe_cmp_branch);
15811 %}
15812 
15813 instruct far_cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15814   match(If cmp (CmpI op1 op2));
15815   effect(USE labl);
15816 
15817   ins_cost(BRANCH_COST);
15818   format %{ "cb$cmp   $op1, $labl # int" %}
15819   ins_encode %{
15820     Label* L = $labl$$label;
15821     Assembler::Condition cond =
15822       ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15823     __ tbr(cond, $op1$$Register, 31, *L, /*far*/true);
15824   %}
15825   ins_pipe(pipe_cmp_branch);
15826 %}
15827 
15828 instruct far_cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
15829   match(If cmp (CmpL (AndL op1 op2) op3));
15830   predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
15831   effect(USE labl);
15832 
15833   ins_cost(BRANCH_COST);
15834   format %{ "tb$cmp   $op1, $op2, $labl" %}
15835   ins_encode %{
15836     Label* L = $labl$$label;
15837     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15838     int bit = exact_log2_long($op2$$constant);
15839     __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
15840   %}
15841   ins_pipe(pipe_cmp_branch);
15842 %}
15843 
15844 instruct far_cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
15845   match(If cmp (CmpI (AndI op1 op2) op3));
15846   predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
15847   effect(USE labl);
15848 
15849   ins_cost(BRANCH_COST);
15850   format %{ "tb$cmp   $op1, $op2, $labl" %}
15851   ins_encode %{
15852     Label* L = $labl$$label;
15853     Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15854     int bit = exact_log2((juint)$op2$$constant);
15855     __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
15856   %}
15857   ins_pipe(pipe_cmp_branch);
15858 %}
15859 
15860 // Test bits
15861 
15862 instruct cmpL_and(cmpOp cmp, iRegL op1, immL op2, immL0 op3, rFlagsReg cr) %{
15863   match(Set cr (CmpL (AndL op1 op2) op3));
15864   predicate(Assembler::operand_valid_for_logical_immediate
15865             (/*is_32*/false, n->in(1)->in(2)->get_long()));
15866 
15867   ins_cost(INSN_COST);
15868   format %{ "tst $op1, $op2 # long" %}
15869   ins_encode %{
15870     __ tst($op1$$Register, $op2$$constant);
15871   %}
15872   ins_pipe(ialu_reg_reg);
15873 %}
15874 
15875 instruct cmpI_and(cmpOp cmp, iRegIorL2I op1, immI op2, immI0 op3, rFlagsReg cr) %{
15876   match(Set cr (CmpI (AndI op1 op2) op3));
15877   predicate(Assembler::operand_valid_for_logical_immediate
15878             (/*is_32*/true, n->in(1)->in(2)->get_int()));
15879 
15880   ins_cost(INSN_COST);
15881   format %{ "tst $op1, $op2 # int" %}
15882   ins_encode %{
15883     __ tstw($op1$$Register, $op2$$constant);
15884   %}
15885   ins_pipe(ialu_reg_reg);
15886 %}
15887 
15888 instruct cmpL_and_reg(cmpOp cmp, iRegL op1, iRegL op2, immL0 op3, rFlagsReg cr) %{
15889   match(Set cr (CmpL (AndL op1 op2) op3));
15890 
15891   ins_cost(INSN_COST);
15892   format %{ "tst $op1, $op2 # long" %}
15893   ins_encode %{
15894     __ tst($op1$$Register, $op2$$Register);
15895   %}
15896   ins_pipe(ialu_reg_reg);
15897 %}
15898 
15899 instruct cmpI_and_reg(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, immI0 op3, rFlagsReg cr) %{
15900   match(Set cr (CmpI (AndI op1 op2) op3));
15901 
15902   ins_cost(INSN_COST);
15903   format %{ "tstw $op1, $op2 # int" %}
15904   ins_encode %{
15905     __ tstw($op1$$Register, $op2$$Register);
15906   %}
15907   ins_pipe(ialu_reg_reg);
15908 %}
15909 
15910 
15911 // Conditional Far Branch
15912 // Conditional Far Branch Unsigned
15913 // TODO: fixme
15914 
15915 // counted loop end branch near
15916 instruct branchLoopEnd(cmpOp cmp, rFlagsReg cr, label lbl)
15917 %{
15918   match(CountedLoopEnd cmp cr);
15919 
15920   effect(USE lbl);
15921 
15922   ins_cost(BRANCH_COST);
15923   // short variant.
15924   // ins_short_branch(1);
15925   format %{ "b$cmp $lbl \t// counted loop end" %}
15926 
15927   ins_encode(aarch64_enc_br_con(cmp, lbl));
15928 
15929   ins_pipe(pipe_branch);
15930 %}
15931 
15932 // counted loop end branch far
15933 // TODO: fixme
15934 
15935 // ============================================================================
15936 // inlined locking and unlocking
15937 
15938 instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
15939 %{
15940   predicate(LockingMode != LM_LIGHTWEIGHT);
15941   match(Set cr (FastLock object box));
15942   effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
15943 
15944   ins_cost(5 * INSN_COST);
15945   format %{ "fastlock $object,$box\t! kills $tmp,$tmp2,$tmp3" %}
15946 
15947   ins_encode %{
15948     __ fast_lock($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
15949   %}
15950 
15951   ins_pipe(pipe_serial);
15952 %}
15953 
15954 instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
15955 %{
15956   predicate(LockingMode != LM_LIGHTWEIGHT);
15957   match(Set cr (FastUnlock object box));
15958   effect(TEMP tmp, TEMP tmp2);
15959 
15960   ins_cost(5 * INSN_COST);
15961   format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2" %}
15962 
15963   ins_encode %{
15964     __ fast_unlock($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register);
15965   %}
15966 
15967   ins_pipe(pipe_serial);
15968 %}
15969 
15970 instruct cmpFastLockLightweight(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
15971 %{
15972   predicate(LockingMode == LM_LIGHTWEIGHT);
15973   match(Set cr (FastLock object box));
15974   effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
15975 
15976   ins_cost(5 * INSN_COST);
15977   format %{ "fastlock $object,$box\t! kills $tmp,$tmp2,$tmp3" %}
15978 
15979   ins_encode %{
15980     __ fast_lock_lightweight($object$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
15981   %}
15982 
15983   ins_pipe(pipe_serial);
15984 %}
15985 
15986 instruct cmpFastUnlockLightweight(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
15987 %{
15988   predicate(LockingMode == LM_LIGHTWEIGHT);
15989   match(Set cr (FastUnlock object box));
15990   effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
15991 
15992   ins_cost(5 * INSN_COST);
15993   format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2, $tmp3" %}
15994 
15995   ins_encode %{
15996     __ fast_unlock_lightweight($object$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
15997   %}
15998 
15999   ins_pipe(pipe_serial);
16000 %}
16001 
16002 // ============================================================================
16003 // Safepoint Instructions
16004 
16005 // TODO
16006 // provide a near and far version of this code
16007 
16008 instruct safePoint(rFlagsReg cr, iRegP poll)
16009 %{
16010   match(SafePoint poll);
16011   effect(KILL cr);
16012 
16013   format %{
16014     "ldrw zr, [$poll]\t# Safepoint: poll for GC"
16015   %}
16016   ins_encode %{
16017     __ read_polling_page(as_Register($poll$$reg), relocInfo::poll_type);
16018   %}
16019   ins_pipe(pipe_serial); // ins_pipe(iload_reg_mem);
16020 %}
16021 
16022 
16023 // ============================================================================
16024 // Procedure Call/Return Instructions
16025 
16026 // Call Java Static Instruction
16027 
16028 instruct CallStaticJavaDirect(method meth)
16029 %{
16030   match(CallStaticJava);
16031 
16032   effect(USE meth);
16033 
16034   ins_cost(CALL_COST);
16035 
16036   format %{ "call,static $meth \t// ==> " %}
16037 
16038   ins_encode(aarch64_enc_java_static_call(meth),
16039              aarch64_enc_call_epilog);
16040 
16041   ins_pipe(pipe_class_call);
16042 %}
16043 
16044 // TO HERE
16045 
16046 // Call Java Dynamic Instruction
16047 instruct CallDynamicJavaDirect(method meth)
16048 %{
16049   match(CallDynamicJava);
16050 
16051   effect(USE meth);
16052 
16053   ins_cost(CALL_COST);
16054 
16055   format %{ "CALL,dynamic $meth \t// ==> " %}
16056 
16057   ins_encode(aarch64_enc_java_dynamic_call(meth),
16058              aarch64_enc_call_epilog);
16059 
16060   ins_pipe(pipe_class_call);
16061 %}
16062 
16063 // Call Runtime Instruction
16064 
16065 instruct CallRuntimeDirect(method meth)
16066 %{
16067   match(CallRuntime);
16068 
16069   effect(USE meth);
16070 
16071   ins_cost(CALL_COST);
16072 
16073   format %{ "CALL, runtime $meth" %}
16074 
16075   ins_encode( aarch64_enc_java_to_runtime(meth) );
16076 
16077   ins_pipe(pipe_class_call);
16078 %}
16079 
16080 // Call Runtime Instruction
16081 
16082 instruct CallLeafDirect(method meth)
16083 %{
16084   match(CallLeaf);
16085 
16086   effect(USE meth);
16087 
16088   ins_cost(CALL_COST);
16089 
16090   format %{ "CALL, runtime leaf $meth" %}
16091 
16092   ins_encode( aarch64_enc_java_to_runtime(meth) );
16093 
16094   ins_pipe(pipe_class_call);
16095 %}
16096 
16097 // Call Runtime Instruction
16098 
16099 instruct CallLeafNoFPDirect(method meth)
16100 %{
16101   match(CallLeafNoFP);
16102 
16103   effect(USE meth);
16104 
16105   ins_cost(CALL_COST);
16106 
16107   format %{ "CALL, runtime leaf nofp $meth" %}
16108 
16109   ins_encode( aarch64_enc_java_to_runtime(meth) );
16110 
16111   ins_pipe(pipe_class_call);
16112 %}
16113 
16114 // Tail Call; Jump from runtime stub to Java code.
16115 // Also known as an 'interprocedural jump'.
16116 // Target of jump will eventually return to caller.
16117 // TailJump below removes the return address.
16118 // Don't use rfp for 'jump_target' because a MachEpilogNode has already been
16119 // emitted just above the TailCall which has reset rfp to the caller state.
16120 instruct TailCalljmpInd(iRegPNoSpNoRfp jump_target, inline_cache_RegP method_ptr)
16121 %{
16122   match(TailCall jump_target method_ptr);
16123 
16124   ins_cost(CALL_COST);
16125 
16126   format %{ "br $jump_target\t# $method_ptr holds method" %}
16127 
16128   ins_encode(aarch64_enc_tail_call(jump_target));
16129 
16130   ins_pipe(pipe_class_call);
16131 %}
16132 
16133 instruct TailjmpInd(iRegPNoSpNoRfp jump_target, iRegP_R0 ex_oop)
16134 %{
16135   match(TailJump jump_target ex_oop);
16136 
16137   ins_cost(CALL_COST);
16138 
16139   format %{ "br $jump_target\t# $ex_oop holds exception oop" %}
16140 
16141   ins_encode(aarch64_enc_tail_jmp(jump_target));
16142 
16143   ins_pipe(pipe_class_call);
16144 %}
16145 
16146 // Create exception oop: created by stack-crawling runtime code.
16147 // Created exception is now available to this handler, and is setup
16148 // just prior to jumping to this handler. No code emitted.
16149 // TODO check
16150 // should ex_oop be in r0? intel uses rax, ppc cannot use r0 so uses rarg1
16151 instruct CreateException(iRegP_R0 ex_oop)
16152 %{
16153   match(Set ex_oop (CreateEx));
16154 
16155   format %{ " -- \t// exception oop; no code emitted" %}
16156 
16157   size(0);
16158 
16159   ins_encode( /*empty*/ );
16160 
16161   ins_pipe(pipe_class_empty);
16162 %}
16163 
16164 // Rethrow exception: The exception oop will come in the first
16165 // argument position. Then JUMP (not call) to the rethrow stub code.
16166 instruct RethrowException() %{
16167   match(Rethrow);
16168   ins_cost(CALL_COST);
16169 
16170   format %{ "b rethrow_stub" %}
16171 
16172   ins_encode( aarch64_enc_rethrow() );
16173 
16174   ins_pipe(pipe_class_call);
16175 %}
16176 
16177 
16178 // Return Instruction
16179 // epilog node loads ret address into lr as part of frame pop
16180 instruct Ret()
16181 %{
16182   match(Return);
16183 
16184   format %{ "ret\t// return register" %}
16185 
16186   ins_encode( aarch64_enc_ret() );
16187 
16188   ins_pipe(pipe_branch);
16189 %}
16190 
16191 // Die now.
16192 instruct ShouldNotReachHere() %{
16193   match(Halt);
16194 
16195   ins_cost(CALL_COST);
16196   format %{ "ShouldNotReachHere" %}
16197 
16198   ins_encode %{
16199     if (is_reachable()) {
16200       __ stop(_halt_reason);
16201     }
16202   %}
16203 
16204   ins_pipe(pipe_class_default);
16205 %}
16206 
16207 // ============================================================================
16208 // Partial Subtype Check
16209 //
16210 // superklass array for an instance of the superklass.  Set a hidden
16211 // internal cache on a hit (cache is checked with exposed code in
16212 // gen_subtype_check()).  Return NZ for a miss or zero for a hit.  The
16213 // encoding ALSO sets flags.
16214 
16215 instruct partialSubtypeCheck(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, rFlagsReg cr)
16216 %{
16217   match(Set result (PartialSubtypeCheck sub super));
16218   effect(KILL cr, KILL temp);
16219 
16220   ins_cost(1100);  // slightly larger than the next version
16221   format %{ "partialSubtypeCheck $result, $sub, $super" %}
16222 
16223   ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
16224 
16225   opcode(0x1); // Force zero of result reg on hit
16226 
16227   ins_pipe(pipe_class_memory);
16228 %}
16229 
16230 instruct partialSubtypeCheckConstSuper(iRegP_R4 sub, iRegP_R0 super_reg, immP super_con, vRegD_V0 vtemp, iRegP_R5 result,
16231                                        iRegP_R1 tempR1, iRegP_R2 tempR2, iRegP_R3 tempR3,
16232                                        rFlagsReg cr)
16233 %{
16234   match(Set result (PartialSubtypeCheck sub (Binary super_reg super_con)));
16235   predicate(UseSecondarySupersTable);
16236   effect(KILL cr, TEMP tempR1, TEMP tempR2, TEMP tempR3, TEMP vtemp);
16237 
16238   ins_cost(700);  // smaller than the next version
16239   format %{ "partialSubtypeCheck $result, $sub, $super_reg, $super_con" %}
16240 
16241   ins_encode %{
16242     bool success = false;
16243     u1 super_klass_slot = ((Klass*)$super_con$$constant)->hash_slot();
16244     if (InlineSecondarySupersTest) {
16245       success = __ lookup_secondary_supers_table($sub$$Register, $super_reg$$Register,
16246                                                  $tempR1$$Register, $tempR2$$Register, $tempR3$$Register,
16247                                                  $vtemp$$FloatRegister,
16248                                                  $result$$Register,
16249                                                  super_klass_slot);
16250     } else {
16251       address call = __ trampoline_call(RuntimeAddress(StubRoutines::lookup_secondary_supers_table_stub(super_klass_slot)));
16252       success = (call != nullptr);
16253     }
16254     if (!success) {
16255       ciEnv::current()->record_failure("CodeCache is full");
16256       return;
16257     }
16258   %}
16259 
16260   ins_pipe(pipe_class_memory);
16261 %}
16262 
16263 instruct partialSubtypeCheckVsZero(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, immP0 zero, rFlagsReg cr)
16264 %{
16265   match(Set cr (CmpP (PartialSubtypeCheck sub super) zero));
16266   effect(KILL temp, KILL result);
16267 
16268   ins_cost(1100);  // slightly larger than the next version
16269   format %{ "partialSubtypeCheck $result, $sub, $super == 0" %}
16270 
16271   ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
16272 
16273   opcode(0x0); // Don't zero result reg on hit
16274 
16275   ins_pipe(pipe_class_memory);
16276 %}
16277 
16278 // Intrisics for String.compareTo()
16279 
16280 instruct string_compareU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16281                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16282 %{
16283   predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
16284   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16285   effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16286 
16287   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1" %}
16288   ins_encode %{
16289     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16290     __ string_compare($str1$$Register, $str2$$Register,
16291                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16292                       $tmp1$$Register, $tmp2$$Register,
16293                       fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::UU);
16294   %}
16295   ins_pipe(pipe_class_memory);
16296 %}
16297 
16298 instruct string_compareL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16299                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16300 %{
16301   predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
16302   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16303   effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16304 
16305   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1" %}
16306   ins_encode %{
16307     __ string_compare($str1$$Register, $str2$$Register,
16308                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16309                       $tmp1$$Register, $tmp2$$Register,
16310                       fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::LL);
16311   %}
16312   ins_pipe(pipe_class_memory);
16313 %}
16314 
16315 instruct string_compareUL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16316                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16317                         vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16318 %{
16319   predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
16320   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16321   effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16322          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16323 
16324   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16325   ins_encode %{
16326     __ string_compare($str1$$Register, $str2$$Register,
16327                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16328                       $tmp1$$Register, $tmp2$$Register,
16329                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16330                       $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::UL);
16331   %}
16332   ins_pipe(pipe_class_memory);
16333 %}
16334 
16335 instruct string_compareLU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16336                         iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16337                         vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16338 %{
16339   predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
16340   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16341   effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16342          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16343 
16344   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16345   ins_encode %{
16346     __ string_compare($str1$$Register, $str2$$Register,
16347                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16348                       $tmp1$$Register, $tmp2$$Register,
16349                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16350                       $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::LU);
16351   %}
16352   ins_pipe(pipe_class_memory);
16353 %}
16354 
16355 // Note that Z registers alias the corresponding NEON registers, we declare the vector operands of
16356 // these string_compare variants as NEON register type for convenience so that the prototype of
16357 // string_compare can be shared with all variants.
16358 
16359 instruct string_compareLL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16360                               iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16361                               vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16362                               pRegGov_P1 pgtmp2, rFlagsReg cr)
16363 %{
16364   predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
16365   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16366   effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16367          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16368 
16369   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # USE sve" %}
16370   ins_encode %{
16371     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16372     __ string_compare($str1$$Register, $str2$$Register,
16373                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16374                       $tmp1$$Register, $tmp2$$Register,
16375                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16376                       as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16377                       StrIntrinsicNode::LL);
16378   %}
16379   ins_pipe(pipe_class_memory);
16380 %}
16381 
16382 instruct string_compareLU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16383                               iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16384                               vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16385                               pRegGov_P1 pgtmp2, rFlagsReg cr)
16386 %{
16387   predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
16388   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16389   effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16390          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16391 
16392   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # USE sve" %}
16393   ins_encode %{
16394     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16395     __ string_compare($str1$$Register, $str2$$Register,
16396                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16397                       $tmp1$$Register, $tmp2$$Register,
16398                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16399                       as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16400                       StrIntrinsicNode::LU);
16401   %}
16402   ins_pipe(pipe_class_memory);
16403 %}
16404 
16405 instruct string_compareUL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16406                               iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16407                               vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16408                               pRegGov_P1 pgtmp2, rFlagsReg cr)
16409 %{
16410   predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
16411   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16412   effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16413          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16414 
16415   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # USE sve" %}
16416   ins_encode %{
16417     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16418     __ string_compare($str1$$Register, $str2$$Register,
16419                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16420                       $tmp1$$Register, $tmp2$$Register,
16421                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16422                       as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16423                       StrIntrinsicNode::UL);
16424   %}
16425   ins_pipe(pipe_class_memory);
16426 %}
16427 
16428 instruct string_compareUU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16429                               iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16430                               vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16431                               pRegGov_P1 pgtmp2, rFlagsReg cr)
16432 %{
16433   predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
16434   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16435   effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16436          USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16437 
16438   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # USE sve" %}
16439   ins_encode %{
16440     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16441     __ string_compare($str1$$Register, $str2$$Register,
16442                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
16443                       $tmp1$$Register, $tmp2$$Register,
16444                       $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16445                       as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16446                       StrIntrinsicNode::UU);
16447   %}
16448   ins_pipe(pipe_class_memory);
16449 %}
16450 
16451 instruct string_indexofUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16452                           iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16453                           iRegINoSp tmp3, iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
16454                           vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
16455 %{
16456   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16457   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16458   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16459          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
16460          TEMP vtmp0, TEMP vtmp1, KILL cr);
16461   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UU) "
16462             "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
16463 
16464   ins_encode %{
16465     __ string_indexof($str1$$Register, $str2$$Register,
16466                       $cnt1$$Register, $cnt2$$Register,
16467                       $tmp1$$Register, $tmp2$$Register,
16468                       $tmp3$$Register, $tmp4$$Register,
16469                       $tmp5$$Register, $tmp6$$Register,
16470                       -1, $result$$Register, StrIntrinsicNode::UU);
16471   %}
16472   ins_pipe(pipe_class_memory);
16473 %}
16474 
16475 instruct string_indexofLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16476                           iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
16477                           iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
16478                           vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
16479 %{
16480   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16481   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16482   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16483          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
16484          TEMP vtmp0, TEMP vtmp1, KILL cr);
16485   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (LL) "
16486             "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
16487 
16488   ins_encode %{
16489     __ string_indexof($str1$$Register, $str2$$Register,
16490                       $cnt1$$Register, $cnt2$$Register,
16491                       $tmp1$$Register, $tmp2$$Register,
16492                       $tmp3$$Register, $tmp4$$Register,
16493                       $tmp5$$Register, $tmp6$$Register,
16494                       -1, $result$$Register, StrIntrinsicNode::LL);
16495   %}
16496   ins_pipe(pipe_class_memory);
16497 %}
16498 
16499 instruct string_indexofUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16500                           iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,iRegINoSp tmp3,
16501                           iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
16502                           vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
16503 %{
16504   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16505   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16506   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16507          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5,
16508          TEMP tmp6, TEMP vtmp0, TEMP vtmp1, KILL cr);
16509   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UL) "
16510             "# KILL $str1 cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
16511 
16512   ins_encode %{
16513     __ string_indexof($str1$$Register, $str2$$Register,
16514                       $cnt1$$Register, $cnt2$$Register,
16515                       $tmp1$$Register, $tmp2$$Register,
16516                       $tmp3$$Register, $tmp4$$Register,
16517                       $tmp5$$Register, $tmp6$$Register,
16518                       -1, $result$$Register, StrIntrinsicNode::UL);
16519   %}
16520   ins_pipe(pipe_class_memory);
16521 %}
16522 
16523 instruct string_indexof_conUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16524                               immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16525                               iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16526 %{
16527   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16528   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16529   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16530          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16531   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UU) "
16532             "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16533 
16534   ins_encode %{
16535     int icnt2 = (int)$int_cnt2$$constant;
16536     __ string_indexof($str1$$Register, $str2$$Register,
16537                       $cnt1$$Register, zr,
16538                       $tmp1$$Register, $tmp2$$Register,
16539                       $tmp3$$Register, $tmp4$$Register, zr, zr,
16540                       icnt2, $result$$Register, StrIntrinsicNode::UU);
16541   %}
16542   ins_pipe(pipe_class_memory);
16543 %}
16544 
16545 instruct string_indexof_conLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16546                               immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16547                               iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16548 %{
16549   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16550   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16551   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16552          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16553   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (LL) "
16554             "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16555 
16556   ins_encode %{
16557     int icnt2 = (int)$int_cnt2$$constant;
16558     __ string_indexof($str1$$Register, $str2$$Register,
16559                       $cnt1$$Register, zr,
16560                       $tmp1$$Register, $tmp2$$Register,
16561                       $tmp3$$Register, $tmp4$$Register, zr, zr,
16562                       icnt2, $result$$Register, StrIntrinsicNode::LL);
16563   %}
16564   ins_pipe(pipe_class_memory);
16565 %}
16566 
16567 instruct string_indexof_conUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16568                               immI_1 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16569                               iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16570 %{
16571   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16572   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16573   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16574          TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16575   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UL) "
16576             "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16577 
16578   ins_encode %{
16579     int icnt2 = (int)$int_cnt2$$constant;
16580     __ string_indexof($str1$$Register, $str2$$Register,
16581                       $cnt1$$Register, zr,
16582                       $tmp1$$Register, $tmp2$$Register,
16583                       $tmp3$$Register, $tmp4$$Register, zr, zr,
16584                       icnt2, $result$$Register, StrIntrinsicNode::UL);
16585   %}
16586   ins_pipe(pipe_class_memory);
16587 %}
16588 
16589 instruct string_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16590                              iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16591                              iRegINoSp tmp3, rFlagsReg cr)
16592 %{
16593   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16594   predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
16595   effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16596          TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16597 
16598   format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16599 
16600   ins_encode %{
16601     __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16602                            $result$$Register, $tmp1$$Register, $tmp2$$Register,
16603                            $tmp3$$Register);
16604   %}
16605   ins_pipe(pipe_class_memory);
16606 %}
16607 
16608 instruct stringL_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16609                               iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16610                               iRegINoSp tmp3, rFlagsReg cr)
16611 %{
16612   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16613   predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
16614   effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16615          TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16616 
16617   format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16618 
16619   ins_encode %{
16620     __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16621                             $result$$Register, $tmp1$$Register, $tmp2$$Register,
16622                             $tmp3$$Register);
16623   %}
16624   ins_pipe(pipe_class_memory);
16625 %}
16626 
16627 instruct stringL_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16628                                   iRegI_R0 result, vecA ztmp1, vecA ztmp2,
16629                                   pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
16630   predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L);
16631   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16632   effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
16633   format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
16634   ins_encode %{
16635     __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
16636                                $result$$Register, $ztmp1$$FloatRegister,
16637                                $ztmp2$$FloatRegister, $pgtmp$$PRegister,
16638                                $ptmp$$PRegister, true /* isL */);
16639   %}
16640   ins_pipe(pipe_class_memory);
16641 %}
16642 
16643 instruct stringU_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16644                                   iRegI_R0 result, vecA ztmp1, vecA ztmp2,
16645                                   pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
16646   predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U);
16647   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16648   effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
16649   format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
16650   ins_encode %{
16651     __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
16652                                $result$$Register, $ztmp1$$FloatRegister,
16653                                $ztmp2$$FloatRegister, $pgtmp$$PRegister,
16654                                $ptmp$$PRegister, false /* isL */);
16655   %}
16656   ins_pipe(pipe_class_memory);
16657 %}
16658 
16659 instruct string_equalsL(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
16660                         iRegI_R0 result, rFlagsReg cr)
16661 %{
16662   predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::LL);
16663   match(Set result (StrEquals (Binary str1 str2) cnt));
16664   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
16665 
16666   format %{ "String Equals $str1,$str2,$cnt -> $result" %}
16667   ins_encode %{
16668     // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16669     __ string_equals($str1$$Register, $str2$$Register,
16670                      $result$$Register, $cnt$$Register);
16671   %}
16672   ins_pipe(pipe_class_memory);
16673 %}
16674 
16675 instruct array_equalsB(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16676                        iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16677                        vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16678                        vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16679                        iRegP_R10 tmp, rFlagsReg cr)
16680 %{
16681   predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
16682   match(Set result (AryEq ary1 ary2));
16683   effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
16684          TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16685          TEMP vtmp6, TEMP vtmp7, KILL cr);
16686 
16687   format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
16688   ins_encode %{
16689     address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
16690                                    $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
16691                                    $result$$Register, $tmp$$Register, 1);
16692     if (tpc == nullptr) {
16693       ciEnv::current()->record_failure("CodeCache is full");
16694       return;
16695     }
16696   %}
16697   ins_pipe(pipe_class_memory);
16698 %}
16699 
16700 instruct array_equalsC(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16701                        iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16702                        vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16703                        vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16704                        iRegP_R10 tmp, rFlagsReg cr)
16705 %{
16706   predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
16707   match(Set result (AryEq ary1 ary2));
16708   effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
16709          TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16710          TEMP vtmp6, TEMP vtmp7, KILL cr);
16711 
16712   format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
16713   ins_encode %{
16714     address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
16715                                    $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
16716                                    $result$$Register, $tmp$$Register, 2);
16717     if (tpc == nullptr) {
16718       ciEnv::current()->record_failure("CodeCache is full");
16719       return;
16720     }
16721   %}
16722   ins_pipe(pipe_class_memory);
16723 %}
16724 
16725 instruct count_positives(iRegP_R1 ary1, iRegI_R2 len, iRegI_R0 result, rFlagsReg cr)
16726 %{
16727   match(Set result (CountPositives ary1 len));
16728   effect(USE_KILL ary1, USE_KILL len, KILL cr);
16729   format %{ "count positives byte[] $ary1,$len -> $result" %}
16730   ins_encode %{
16731     address tpc = __ count_positives($ary1$$Register, $len$$Register, $result$$Register);
16732     if (tpc == nullptr) {
16733       ciEnv::current()->record_failure("CodeCache is full");
16734       return;
16735     }
16736   %}
16737   ins_pipe( pipe_slow );
16738 %}
16739 
16740 // fast char[] to byte[] compression
16741 instruct string_compress(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16742                          vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16743                          vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16744                          iRegI_R0 result, rFlagsReg cr)
16745 %{
16746   match(Set result (StrCompressedCopy src (Binary dst len)));
16747   effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16748          USE_KILL src, USE_KILL dst, USE len, KILL cr);
16749 
16750   format %{ "String Compress $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16751   ins_encode %{
16752     __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
16753                            $result$$Register, $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16754                            $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
16755                            $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
16756   %}
16757   ins_pipe(pipe_slow);
16758 %}
16759 
16760 // fast byte[] to char[] inflation
16761 instruct string_inflate(Universe dummy, iRegP_R0 src, iRegP_R1 dst, iRegI_R2 len, iRegP_R3 tmp,
16762                         vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16763                         vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, rFlagsReg cr)
16764 %{
16765   match(Set dummy (StrInflatedCopy src (Binary dst len)));
16766   effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3,
16767          TEMP vtmp4, TEMP vtmp5, TEMP vtmp6, TEMP tmp,
16768          USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
16769 
16770   format %{ "String Inflate $src,$dst # KILL $tmp $src $dst $len V0-V6 cr" %}
16771   ins_encode %{
16772     address tpc = __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
16773                                         $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16774                                         $vtmp2$$FloatRegister, $tmp$$Register);
16775     if (tpc == nullptr) {
16776       ciEnv::current()->record_failure("CodeCache is full");
16777       return;
16778     }
16779   %}
16780   ins_pipe(pipe_class_memory);
16781 %}
16782 
16783 // encode char[] to byte[] in ISO_8859_1
16784 instruct encode_iso_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16785                           vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16786                           vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16787                           iRegI_R0 result, rFlagsReg cr)
16788 %{
16789   predicate(!((EncodeISOArrayNode*)n)->is_ascii());
16790   match(Set result (EncodeISOArray src (Binary dst len)));
16791   effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
16792          KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
16793 
16794   format %{ "Encode ISO array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16795   ins_encode %{
16796     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
16797                         $result$$Register, false,
16798                         $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16799                         $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
16800                         $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
16801   %}
16802   ins_pipe(pipe_class_memory);
16803 %}
16804 
16805 instruct encode_ascii_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16806                             vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16807                             vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16808                             iRegI_R0 result, rFlagsReg cr)
16809 %{
16810   predicate(((EncodeISOArrayNode*)n)->is_ascii());
16811   match(Set result (EncodeISOArray src (Binary dst len)));
16812   effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
16813          KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
16814 
16815   format %{ "Encode ASCII array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16816   ins_encode %{
16817     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
16818                         $result$$Register, true,
16819                         $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16820                         $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
16821                         $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
16822   %}
16823   ins_pipe(pipe_class_memory);
16824 %}
16825 
16826 //----------------------------- CompressBits/ExpandBits ------------------------
16827 
16828 instruct compressBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
16829                            vRegF tdst, vRegF tsrc, vRegF tmask) %{
16830   match(Set dst (CompressBits src mask));
16831   effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16832   format %{ "mov    $tsrc, $src\n\t"
16833             "mov    $tmask, $mask\n\t"
16834             "bext   $tdst, $tsrc, $tmask\n\t"
16835             "mov    $dst, $tdst"
16836           %}
16837   ins_encode %{
16838     __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
16839     __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
16840     __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16841     __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16842   %}
16843   ins_pipe(pipe_slow);
16844 %}
16845 
16846 instruct compressBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
16847                            vRegF tdst, vRegF tsrc, vRegF tmask) %{
16848   match(Set dst (CompressBits (LoadI mem) mask));
16849   effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16850   format %{ "ldrs   $tsrc, $mem\n\t"
16851             "ldrs   $tmask, $mask\n\t"
16852             "bext   $tdst, $tsrc, $tmask\n\t"
16853             "mov    $dst, $tdst"
16854           %}
16855   ins_encode %{
16856     loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
16857               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
16858     __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
16859     __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16860     __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16861   %}
16862   ins_pipe(pipe_slow);
16863 %}
16864 
16865 instruct compressBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
16866                            vRegD tdst, vRegD tsrc, vRegD tmask) %{
16867   match(Set dst (CompressBits src mask));
16868   effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16869   format %{ "mov    $tsrc, $src\n\t"
16870             "mov    $tmask, $mask\n\t"
16871             "bext   $tdst, $tsrc, $tmask\n\t"
16872             "mov    $dst, $tdst"
16873           %}
16874   ins_encode %{
16875     __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
16876     __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
16877     __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16878     __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16879   %}
16880   ins_pipe(pipe_slow);
16881 %}
16882 
16883 instruct compressBitsL_memcon(iRegLNoSp dst, memory8 mem, immL mask,
16884                            vRegF tdst, vRegF tsrc, vRegF tmask) %{
16885   match(Set dst (CompressBits (LoadL mem) mask));
16886   effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16887   format %{ "ldrd   $tsrc, $mem\n\t"
16888             "ldrd   $tmask, $mask\n\t"
16889             "bext   $tdst, $tsrc, $tmask\n\t"
16890             "mov    $dst, $tdst"
16891           %}
16892   ins_encode %{
16893     loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
16894               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
16895     __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
16896     __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16897     __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16898   %}
16899   ins_pipe(pipe_slow);
16900 %}
16901 
16902 instruct expandBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
16903                          vRegF tdst, vRegF tsrc, vRegF tmask) %{
16904   match(Set dst (ExpandBits src mask));
16905   effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16906   format %{ "mov    $tsrc, $src\n\t"
16907             "mov    $tmask, $mask\n\t"
16908             "bdep   $tdst, $tsrc, $tmask\n\t"
16909             "mov    $dst, $tdst"
16910           %}
16911   ins_encode %{
16912     __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
16913     __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
16914     __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16915     __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16916   %}
16917   ins_pipe(pipe_slow);
16918 %}
16919 
16920 instruct expandBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
16921                          vRegF tdst, vRegF tsrc, vRegF tmask) %{
16922   match(Set dst (ExpandBits (LoadI mem) mask));
16923   effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16924   format %{ "ldrs   $tsrc, $mem\n\t"
16925             "ldrs   $tmask, $mask\n\t"
16926             "bdep   $tdst, $tsrc, $tmask\n\t"
16927             "mov    $dst, $tdst"
16928           %}
16929   ins_encode %{
16930     loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
16931               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
16932     __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
16933     __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16934     __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16935   %}
16936   ins_pipe(pipe_slow);
16937 %}
16938 
16939 instruct expandBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
16940                          vRegD tdst, vRegD tsrc, vRegD tmask) %{
16941   match(Set dst (ExpandBits src mask));
16942   effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16943   format %{ "mov    $tsrc, $src\n\t"
16944             "mov    $tmask, $mask\n\t"
16945             "bdep   $tdst, $tsrc, $tmask\n\t"
16946             "mov    $dst, $tdst"
16947           %}
16948   ins_encode %{
16949     __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
16950     __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
16951     __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16952     __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16953   %}
16954   ins_pipe(pipe_slow);
16955 %}
16956 
16957 
16958 instruct expandBitsL_memcon(iRegINoSp dst, memory8 mem, immL mask,
16959                          vRegF tdst, vRegF tsrc, vRegF tmask) %{
16960   match(Set dst (ExpandBits (LoadL mem) mask));
16961   effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16962   format %{ "ldrd   $tsrc, $mem\n\t"
16963             "ldrd   $tmask, $mask\n\t"
16964             "bdep   $tdst, $tsrc, $tmask\n\t"
16965             "mov    $dst, $tdst"
16966           %}
16967   ins_encode %{
16968     loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
16969               as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
16970     __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
16971     __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16972     __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16973   %}
16974   ins_pipe(pipe_slow);
16975 %}
16976 
16977 // ============================================================================
16978 // This name is KNOWN by the ADLC and cannot be changed.
16979 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
16980 // for this guy.
16981 instruct tlsLoadP(thread_RegP dst)
16982 %{
16983   match(Set dst (ThreadLocal));
16984 
16985   ins_cost(0);
16986 
16987   format %{ " -- \t// $dst=Thread::current(), empty" %}
16988 
16989   size(0);
16990 
16991   ins_encode( /*empty*/ );
16992 
16993   ins_pipe(pipe_class_empty);
16994 %}
16995 
16996 //----------PEEPHOLE RULES-----------------------------------------------------
16997 // These must follow all instruction definitions as they use the names
16998 // defined in the instructions definitions.
16999 //
17000 // peepmatch ( root_instr_name [preceding_instruction]* );
17001 //
17002 // peepconstraint %{
17003 // (instruction_number.operand_name relational_op instruction_number.operand_name
17004 //  [, ...] );
17005 // // instruction numbers are zero-based using left to right order in peepmatch
17006 //
17007 // peepreplace ( instr_name  ( [instruction_number.operand_name]* ) );
17008 // // provide an instruction_number.operand_name for each operand that appears
17009 // // in the replacement instruction's match rule
17010 //
17011 // ---------VM FLAGS---------------------------------------------------------
17012 //
17013 // All peephole optimizations can be turned off using -XX:-OptoPeephole
17014 //
17015 // Each peephole rule is given an identifying number starting with zero and
17016 // increasing by one in the order seen by the parser.  An individual peephole
17017 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
17018 // on the command-line.
17019 //
17020 // ---------CURRENT LIMITATIONS----------------------------------------------
17021 //
17022 // Only match adjacent instructions in same basic block
17023 // Only equality constraints
17024 // Only constraints between operands, not (0.dest_reg == RAX_enc)
17025 // Only one replacement instruction
17026 //
17027 // ---------EXAMPLE----------------------------------------------------------
17028 //
17029 // // pertinent parts of existing instructions in architecture description
17030 // instruct movI(iRegINoSp dst, iRegI src)
17031 // %{
17032 //   match(Set dst (CopyI src));
17033 // %}
17034 //
17035 // instruct incI_iReg(iRegINoSp dst, immI1 src, rFlagsReg cr)
17036 // %{
17037 //   match(Set dst (AddI dst src));
17038 //   effect(KILL cr);
17039 // %}
17040 //
17041 // // Change (inc mov) to lea
17042 // peephole %{
17043 //   // increment preceded by register-register move
17044 //   peepmatch ( incI_iReg movI );
17045 //   // require that the destination register of the increment
17046 //   // match the destination register of the move
17047 //   peepconstraint ( 0.dst == 1.dst );
17048 //   // construct a replacement instruction that sets
17049 //   // the destination to ( move's source register + one )
17050 //   peepreplace ( leaI_iReg_immI( 0.dst 1.src 0.src ) );
17051 // %}
17052 //
17053 
17054 // Implementation no longer uses movX instructions since
17055 // machine-independent system no longer uses CopyX nodes.
17056 //
17057 // peephole
17058 // %{
17059 //   peepmatch (incI_iReg movI);
17060 //   peepconstraint (0.dst == 1.dst);
17061 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17062 // %}
17063 
17064 // peephole
17065 // %{
17066 //   peepmatch (decI_iReg movI);
17067 //   peepconstraint (0.dst == 1.dst);
17068 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17069 // %}
17070 
17071 // peephole
17072 // %{
17073 //   peepmatch (addI_iReg_imm movI);
17074 //   peepconstraint (0.dst == 1.dst);
17075 //   peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17076 // %}
17077 
17078 // peephole
17079 // %{
17080 //   peepmatch (incL_iReg movL);
17081 //   peepconstraint (0.dst == 1.dst);
17082 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17083 // %}
17084 
17085 // peephole
17086 // %{
17087 //   peepmatch (decL_iReg movL);
17088 //   peepconstraint (0.dst == 1.dst);
17089 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17090 // %}
17091 
17092 // peephole
17093 // %{
17094 //   peepmatch (addL_iReg_imm movL);
17095 //   peepconstraint (0.dst == 1.dst);
17096 //   peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17097 // %}
17098 
17099 // peephole
17100 // %{
17101 //   peepmatch (addP_iReg_imm movP);
17102 //   peepconstraint (0.dst == 1.dst);
17103 //   peepreplace (leaP_iReg_imm(0.dst 1.src 0.src));
17104 // %}
17105 
17106 // // Change load of spilled value to only a spill
17107 // instruct storeI(memory mem, iRegI src)
17108 // %{
17109 //   match(Set mem (StoreI mem src));
17110 // %}
17111 //
17112 // instruct loadI(iRegINoSp dst, memory mem)
17113 // %{
17114 //   match(Set dst (LoadI mem));
17115 // %}
17116 //
17117 
17118 //----------SMARTSPILL RULES---------------------------------------------------
17119 // These must follow all instruction definitions as they use the names
17120 // defined in the instructions definitions.
17121 
17122 // Local Variables:
17123 // mode: c++
17124 // End: