1 //
    2 // Copyright (c) 2003, 2025, Oracle and/or its affiliates. All rights reserved.
    3 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
    4 //
    5 // This code is free software; you can redistribute it and/or modify it
    6 // under the terms of the GNU General Public License version 2 only, as
    7 // published by the Free Software Foundation.
    8 //
    9 // This code is distributed in the hope that it will be useful, but WITHOUT
   10 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
   11 // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
   12 // version 2 for more details (a copy is included in the LICENSE file that
   13 // accompanied this code).
   14 //
   15 // You should have received a copy of the GNU General Public License version
   16 // 2 along with this work; if not, write to the Free Software Foundation,
   17 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
   18 //
   19 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
   20 // or visit www.oracle.com if you need additional information or have any
   21 // questions.
   22 //
   23 //
   24 
   25 // AMD64 Architecture Description File
   26 
   27 //----------REGISTER DEFINITION BLOCK------------------------------------------
   28 // This information is used by the matcher and the register allocator to
   29 // describe individual registers and classes of registers within the target
   30 // architecture.
   31 
   32 register %{
   33 //----------Architecture Description Register Definitions----------------------
   34 // General Registers
   35 // "reg_def"  name ( register save type, C convention save type,
   36 //                   ideal register type, encoding );
   37 // Register Save Types:
   38 //
   39 // NS  = No-Save:       The register allocator assumes that these registers
   40 //                      can be used without saving upon entry to the method, &
   41 //                      that they do not need to be saved at call sites.
   42 //
   43 // SOC = Save-On-Call:  The register allocator assumes that these registers
   44 //                      can be used without saving upon entry to the method,
   45 //                      but that they must be saved at call sites.
   46 //
   47 // SOE = Save-On-Entry: The register allocator assumes that these registers
   48 //                      must be saved before using them upon entry to the
   49 //                      method, but they do not need to be saved at call
   50 //                      sites.
   51 //
   52 // AS  = Always-Save:   The register allocator assumes that these registers
   53 //                      must be saved before using them upon entry to the
   54 //                      method, & that they must be saved at call sites.
   55 //
   56 // Ideal Register Type is used to determine how to save & restore a
   57 // register.  Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
   58 // spilled with LoadP/StoreP.  If the register supports both, use Op_RegI.
   59 //
   60 // The encoding number is the actual bit-pattern placed into the opcodes.
   61 
   62 // General Registers
   63 // R8-R15 must be encoded with REX.  (RSP, RBP, RSI, RDI need REX when
   64 // used as byte registers)
   65 
   66 // Previously set RBX, RSI, and RDI as save-on-entry for java code
   67 // Turn off SOE in java-code due to frequent use of uncommon-traps.
   68 // Now that allocator is better, turn on RSI and RDI as SOE registers.
   69 
   70 reg_def RAX  (SOC, SOC, Op_RegI,  0, rax->as_VMReg());
   71 reg_def RAX_H(SOC, SOC, Op_RegI,  0, rax->as_VMReg()->next());
   72 
   73 reg_def RCX  (SOC, SOC, Op_RegI,  1, rcx->as_VMReg());
   74 reg_def RCX_H(SOC, SOC, Op_RegI,  1, rcx->as_VMReg()->next());
   75 
   76 reg_def RDX  (SOC, SOC, Op_RegI,  2, rdx->as_VMReg());
   77 reg_def RDX_H(SOC, SOC, Op_RegI,  2, rdx->as_VMReg()->next());
   78 
   79 reg_def RBX  (SOC, SOE, Op_RegI,  3, rbx->as_VMReg());
   80 reg_def RBX_H(SOC, SOE, Op_RegI,  3, rbx->as_VMReg()->next());
   81 
   82 reg_def RSP  (NS,  NS,  Op_RegI,  4, rsp->as_VMReg());
   83 reg_def RSP_H(NS,  NS,  Op_RegI,  4, rsp->as_VMReg()->next());
   84 
   85 // now that adapter frames are gone RBP is always saved and restored by the prolog/epilog code
   86 reg_def RBP  (NS, SOE, Op_RegI,  5, rbp->as_VMReg());
   87 reg_def RBP_H(NS, SOE, Op_RegI,  5, rbp->as_VMReg()->next());
   88 
   89 #ifdef _WIN64
   90 
   91 reg_def RSI  (SOC, SOE, Op_RegI,  6, rsi->as_VMReg());
   92 reg_def RSI_H(SOC, SOE, Op_RegI,  6, rsi->as_VMReg()->next());
   93 
   94 reg_def RDI  (SOC, SOE, Op_RegI,  7, rdi->as_VMReg());
   95 reg_def RDI_H(SOC, SOE, Op_RegI,  7, rdi->as_VMReg()->next());
   96 
   97 #else
   98 
   99 reg_def RSI  (SOC, SOC, Op_RegI,  6, rsi->as_VMReg());
  100 reg_def RSI_H(SOC, SOC, Op_RegI,  6, rsi->as_VMReg()->next());
  101 
  102 reg_def RDI  (SOC, SOC, Op_RegI,  7, rdi->as_VMReg());
  103 reg_def RDI_H(SOC, SOC, Op_RegI,  7, rdi->as_VMReg()->next());
  104 
  105 #endif
  106 
  107 reg_def R8   (SOC, SOC, Op_RegI,  8, r8->as_VMReg());
  108 reg_def R8_H (SOC, SOC, Op_RegI,  8, r8->as_VMReg()->next());
  109 
  110 reg_def R9   (SOC, SOC, Op_RegI,  9, r9->as_VMReg());
  111 reg_def R9_H (SOC, SOC, Op_RegI,  9, r9->as_VMReg()->next());
  112 
  113 reg_def R10  (SOC, SOC, Op_RegI, 10, r10->as_VMReg());
  114 reg_def R10_H(SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
  115 
  116 reg_def R11  (SOC, SOC, Op_RegI, 11, r11->as_VMReg());
  117 reg_def R11_H(SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
  118 
  119 reg_def R12  (SOC, SOE, Op_RegI, 12, r12->as_VMReg());
  120 reg_def R12_H(SOC, SOE, Op_RegI, 12, r12->as_VMReg()->next());
  121 
  122 reg_def R13  (SOC, SOE, Op_RegI, 13, r13->as_VMReg());
  123 reg_def R13_H(SOC, SOE, Op_RegI, 13, r13->as_VMReg()->next());
  124 
  125 reg_def R14  (SOC, SOE, Op_RegI, 14, r14->as_VMReg());
  126 reg_def R14_H(SOC, SOE, Op_RegI, 14, r14->as_VMReg()->next());
  127 
  128 reg_def R15  (SOC, SOE, Op_RegI, 15, r15->as_VMReg());
  129 reg_def R15_H(SOC, SOE, Op_RegI, 15, r15->as_VMReg()->next());
  130 
  131 reg_def R16  (SOC, SOC, Op_RegI, 16, r16->as_VMReg());
  132 reg_def R16_H(SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
  133 
  134 reg_def R17  (SOC, SOC, Op_RegI, 17, r17->as_VMReg());
  135 reg_def R17_H(SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
  136 
  137 reg_def R18  (SOC, SOC, Op_RegI, 18, r18->as_VMReg());
  138 reg_def R18_H(SOC, SOC, Op_RegI, 18, r18->as_VMReg()->next());
  139 
  140 reg_def R19  (SOC, SOC, Op_RegI, 19, r19->as_VMReg());
  141 reg_def R19_H(SOC, SOC, Op_RegI, 19, r19->as_VMReg()->next());
  142 
  143 reg_def R20  (SOC, SOC, Op_RegI, 20, r20->as_VMReg());
  144 reg_def R20_H(SOC, SOC, Op_RegI, 20, r20->as_VMReg()->next());
  145 
  146 reg_def R21  (SOC, SOC, Op_RegI, 21, r21->as_VMReg());
  147 reg_def R21_H(SOC, SOC, Op_RegI, 21, r21->as_VMReg()->next());
  148 
  149 reg_def R22  (SOC, SOC, Op_RegI, 22, r22->as_VMReg());
  150 reg_def R22_H(SOC, SOC, Op_RegI, 22, r22->as_VMReg()->next());
  151 
  152 reg_def R23  (SOC, SOC, Op_RegI, 23, r23->as_VMReg());
  153 reg_def R23_H(SOC, SOC, Op_RegI, 23, r23->as_VMReg()->next());
  154 
  155 reg_def R24  (SOC, SOC, Op_RegI, 24, r24->as_VMReg());
  156 reg_def R24_H(SOC, SOC, Op_RegI, 24, r24->as_VMReg()->next());
  157 
  158 reg_def R25  (SOC, SOC, Op_RegI, 25, r25->as_VMReg());
  159 reg_def R25_H(SOC, SOC, Op_RegI, 25, r25->as_VMReg()->next());
  160 
  161 reg_def R26  (SOC, SOC, Op_RegI, 26, r26->as_VMReg());
  162 reg_def R26_H(SOC, SOC, Op_RegI, 26, r26->as_VMReg()->next());
  163 
  164 reg_def R27  (SOC, SOC, Op_RegI, 27, r27->as_VMReg());
  165 reg_def R27_H(SOC, SOC, Op_RegI, 27, r27->as_VMReg()->next());
  166 
  167 reg_def R28  (SOC, SOC, Op_RegI, 28, r28->as_VMReg());
  168 reg_def R28_H(SOC, SOC, Op_RegI, 28, r28->as_VMReg()->next());
  169 
  170 reg_def R29  (SOC, SOC, Op_RegI, 29, r29->as_VMReg());
  171 reg_def R29_H(SOC, SOC, Op_RegI, 29, r29->as_VMReg()->next());
  172 
  173 reg_def R30  (SOC, SOC, Op_RegI, 30, r30->as_VMReg());
  174 reg_def R30_H(SOC, SOC, Op_RegI, 30, r30->as_VMReg()->next());
  175 
  176 reg_def R31  (SOC, SOC, Op_RegI, 31, r31->as_VMReg());
  177 reg_def R31_H(SOC, SOC, Op_RegI, 31, r31->as_VMReg()->next());
  178 
  179 // Floating Point Registers
  180 
  181 // Specify priority of register selection within phases of register
  182 // allocation.  Highest priority is first.  A useful heuristic is to
  183 // give registers a low priority when they are required by machine
  184 // instructions, like EAX and EDX on I486, and choose no-save registers
  185 // before save-on-call, & save-on-call before save-on-entry.  Registers
  186 // which participate in fixed calling sequences should come last.
  187 // Registers which are used as pairs must fall on an even boundary.
  188 
  189 alloc_class chunk0(R10,         R10_H,
  190                    R11,         R11_H,
  191                    R8,          R8_H,
  192                    R9,          R9_H,
  193                    R12,         R12_H,
  194                    RCX,         RCX_H,
  195                    RBX,         RBX_H,
  196                    RDI,         RDI_H,
  197                    RDX,         RDX_H,
  198                    RSI,         RSI_H,
  199                    RAX,         RAX_H,
  200                    RBP,         RBP_H,
  201                    R13,         R13_H,
  202                    R14,         R14_H,
  203                    R15,         R15_H,
  204                    R16,         R16_H,
  205                    R17,         R17_H,
  206                    R18,         R18_H,
  207                    R19,         R19_H,
  208                    R20,         R20_H,
  209                    R21,         R21_H,
  210                    R22,         R22_H,
  211                    R23,         R23_H,
  212                    R24,         R24_H,
  213                    R25,         R25_H,
  214                    R26,         R26_H,
  215                    R27,         R27_H,
  216                    R28,         R28_H,
  217                    R29,         R29_H,
  218                    R30,         R30_H,
  219                    R31,         R31_H,
  220                    RSP,         RSP_H);
  221 
  222 
  223 //----------Architecture Description Register Classes--------------------------
  224 // Several register classes are automatically defined based upon information in
  225 // this architecture description.
  226 // 1) reg_class inline_cache_reg           ( /* as def'd in frame section */ )
  227 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
  228 //
  229 
  230 // Empty register class.
  231 reg_class no_reg();
  232 
  233 // Class for all pointer/long registers including APX extended GPRs.
  234 reg_class all_reg(RAX, RAX_H,
  235                   RDX, RDX_H,
  236                   RBP, RBP_H,
  237                   RDI, RDI_H,
  238                   RSI, RSI_H,
  239                   RCX, RCX_H,
  240                   RBX, RBX_H,
  241                   RSP, RSP_H,
  242                   R8,  R8_H,
  243                   R9,  R9_H,
  244                   R10, R10_H,
  245                   R11, R11_H,
  246                   R12, R12_H,
  247                   R13, R13_H,
  248                   R14, R14_H,
  249                   R15, R15_H,
  250                   R16, R16_H,
  251                   R17, R17_H,
  252                   R18, R18_H,
  253                   R19, R19_H,
  254                   R20, R20_H,
  255                   R21, R21_H,
  256                   R22, R22_H,
  257                   R23, R23_H,
  258                   R24, R24_H,
  259                   R25, R25_H,
  260                   R26, R26_H,
  261                   R27, R27_H,
  262                   R28, R28_H,
  263                   R29, R29_H,
  264                   R30, R30_H,
  265                   R31, R31_H);
  266 
  267 // Class for all int registers including APX extended GPRs.
  268 reg_class all_int_reg(RAX
  269                       RDX,
  270                       RBP,
  271                       RDI,
  272                       RSI,
  273                       RCX,
  274                       RBX,
  275                       R8,
  276                       R9,
  277                       R10,
  278                       R11,
  279                       R12,
  280                       R13,
  281                       R14,
  282                       R16,
  283                       R17,
  284                       R18,
  285                       R19,
  286                       R20,
  287                       R21,
  288                       R22,
  289                       R23,
  290                       R24,
  291                       R25,
  292                       R26,
  293                       R27,
  294                       R28,
  295                       R29,
  296                       R30,
  297                       R31);
  298 
  299 // Class for all pointer registers
  300 reg_class any_reg %{
  301   return _ANY_REG_mask;
  302 %}
  303 
  304 // Class for all pointer registers (excluding RSP)
  305 reg_class ptr_reg %{
  306   return _PTR_REG_mask;
  307 %}
  308 
  309 // Class for all pointer registers (excluding RSP and RBP)
  310 reg_class ptr_reg_no_rbp %{
  311   return _PTR_REG_NO_RBP_mask;
  312 %}
  313 
  314 // Class for all pointer registers (excluding RAX and RSP)
  315 reg_class ptr_no_rax_reg %{
  316   return _PTR_NO_RAX_REG_mask;
  317 %}
  318 
  319 // Class for all pointer registers (excluding RAX, RBX, and RSP)
  320 reg_class ptr_no_rax_rbx_reg %{
  321   return _PTR_NO_RAX_RBX_REG_mask;
  322 %}
  323 
  324 // Class for all long registers (excluding RSP)
  325 reg_class long_reg %{
  326   return _LONG_REG_mask;
  327 %}
  328 
  329 // Class for all long registers (excluding RAX, RDX and RSP)
  330 reg_class long_no_rax_rdx_reg %{
  331   return _LONG_NO_RAX_RDX_REG_mask;
  332 %}
  333 
  334 // Class for all long registers (excluding RCX and RSP)
  335 reg_class long_no_rcx_reg %{
  336   return _LONG_NO_RCX_REG_mask;
  337 %}
  338 
  339 // Class for all long registers (excluding RBP and R13)
  340 reg_class long_no_rbp_r13_reg %{
  341   return _LONG_NO_RBP_R13_REG_mask;
  342 %}
  343 
  344 // Class for all int registers (excluding RSP)
  345 reg_class int_reg %{
  346   return _INT_REG_mask;
  347 %}
  348 
  349 // Class for all int registers (excluding RAX, RDX, and RSP)
  350 reg_class int_no_rax_rdx_reg %{
  351   return _INT_NO_RAX_RDX_REG_mask;
  352 %}
  353 
  354 // Class for all int registers (excluding RCX and RSP)
  355 reg_class int_no_rcx_reg %{
  356   return _INT_NO_RCX_REG_mask;
  357 %}
  358 
  359 // Class for all int registers (excluding RBP and R13)
  360 reg_class int_no_rbp_r13_reg %{
  361   return _INT_NO_RBP_R13_REG_mask;
  362 %}
  363 
  364 // Singleton class for RAX pointer register
  365 reg_class ptr_rax_reg(RAX, RAX_H);
  366 
  367 // Singleton class for RBX pointer register
  368 reg_class ptr_rbx_reg(RBX, RBX_H);
  369 
  370 // Singleton class for RSI pointer register
  371 reg_class ptr_rsi_reg(RSI, RSI_H);
  372 
  373 // Singleton class for RBP pointer register
  374 reg_class ptr_rbp_reg(RBP, RBP_H);
  375 
  376 // Singleton class for RDI pointer register
  377 reg_class ptr_rdi_reg(RDI, RDI_H);
  378 
  379 // Singleton class for stack pointer
  380 reg_class ptr_rsp_reg(RSP, RSP_H);
  381 
  382 // Singleton class for TLS pointer
  383 reg_class ptr_r15_reg(R15, R15_H);
  384 
  385 // Singleton class for RAX long register
  386 reg_class long_rax_reg(RAX, RAX_H);
  387 
  388 // Singleton class for RCX long register
  389 reg_class long_rcx_reg(RCX, RCX_H);
  390 
  391 // Singleton class for RDX long register
  392 reg_class long_rdx_reg(RDX, RDX_H);
  393 
  394 // Singleton class for R11 long register
  395 reg_class long_r11_reg(R11, R11_H);
  396 
  397 // Singleton class for RAX int register
  398 reg_class int_rax_reg(RAX);
  399 
  400 // Singleton class for RBX int register
  401 reg_class int_rbx_reg(RBX);
  402 
  403 // Singleton class for RCX int register
  404 reg_class int_rcx_reg(RCX);
  405 
  406 // Singleton class for RDX int register
  407 reg_class int_rdx_reg(RDX);
  408 
  409 // Singleton class for RDI int register
  410 reg_class int_rdi_reg(RDI);
  411 
  412 // Singleton class for instruction pointer
  413 // reg_class ip_reg(RIP);
  414 
  415 %}
  416 
  417 //----------SOURCE BLOCK-------------------------------------------------------
  418 // This is a block of C++ code which provides values, functions, and
  419 // definitions necessary in the rest of the architecture description
  420 
  421 source_hpp %{
  422 
  423 #include "peephole_x86_64.hpp"
  424 
  425 bool castLL_is_imm32(const Node* n);
  426 
  427 %}
  428 
  429 source %{
  430 
  431 bool castLL_is_imm32(const Node* n) {
  432   assert(n->is_CastLL(), "must be a CastLL");
  433   const TypeLong* t = n->bottom_type()->is_long();
  434   return (t->_lo == min_jlong || Assembler::is_simm32(t->_lo)) && (t->_hi == max_jlong || Assembler::is_simm32(t->_hi));
  435 }
  436 
  437 %}
  438 
  439 // Register masks
  440 source_hpp %{
  441 
  442 extern RegMask _ANY_REG_mask;
  443 extern RegMask _PTR_REG_mask;
  444 extern RegMask _PTR_REG_NO_RBP_mask;
  445 extern RegMask _PTR_NO_RAX_REG_mask;
  446 extern RegMask _PTR_NO_RAX_RBX_REG_mask;
  447 extern RegMask _LONG_REG_mask;
  448 extern RegMask _LONG_NO_RAX_RDX_REG_mask;
  449 extern RegMask _LONG_NO_RCX_REG_mask;
  450 extern RegMask _LONG_NO_RBP_R13_REG_mask;
  451 extern RegMask _INT_REG_mask;
  452 extern RegMask _INT_NO_RAX_RDX_REG_mask;
  453 extern RegMask _INT_NO_RCX_REG_mask;
  454 extern RegMask _INT_NO_RBP_R13_REG_mask;
  455 extern RegMask _FLOAT_REG_mask;
  456 
  457 extern RegMask _STACK_OR_PTR_REG_mask;
  458 extern RegMask _STACK_OR_LONG_REG_mask;
  459 extern RegMask _STACK_OR_INT_REG_mask;
  460 
  461 inline const RegMask& STACK_OR_PTR_REG_mask()  { return _STACK_OR_PTR_REG_mask;  }
  462 inline const RegMask& STACK_OR_LONG_REG_mask() { return _STACK_OR_LONG_REG_mask; }
  463 inline const RegMask& STACK_OR_INT_REG_mask()  { return _STACK_OR_INT_REG_mask;  }
  464 
  465 %}
  466 
  467 source %{
  468 #define   RELOC_IMM64    Assembler::imm_operand
  469 #define   RELOC_DISP32   Assembler::disp32_operand
  470 
  471 #define __ masm->
  472 
  473 RegMask _ANY_REG_mask;
  474 RegMask _PTR_REG_mask;
  475 RegMask _PTR_REG_NO_RBP_mask;
  476 RegMask _PTR_NO_RAX_REG_mask;
  477 RegMask _PTR_NO_RAX_RBX_REG_mask;
  478 RegMask _LONG_REG_mask;
  479 RegMask _LONG_NO_RAX_RDX_REG_mask;
  480 RegMask _LONG_NO_RCX_REG_mask;
  481 RegMask _LONG_NO_RBP_R13_REG_mask;
  482 RegMask _INT_REG_mask;
  483 RegMask _INT_NO_RAX_RDX_REG_mask;
  484 RegMask _INT_NO_RCX_REG_mask;
  485 RegMask _INT_NO_RBP_R13_REG_mask;
  486 RegMask _FLOAT_REG_mask;
  487 RegMask _STACK_OR_PTR_REG_mask;
  488 RegMask _STACK_OR_LONG_REG_mask;
  489 RegMask _STACK_OR_INT_REG_mask;
  490 
  491 static bool need_r12_heapbase() {
  492   return UseCompressedOops;
  493 }
  494 
  495 void reg_mask_init() {
  496   constexpr Register egprs[] = {r16, r17, r18, r19, r20, r21, r22, r23, r24, r25, r26, r27, r28, r29, r30, r31};
  497 
  498   // _ALL_REG_mask is generated by adlc from the all_reg register class below.
  499   // We derive a number of subsets from it.
  500   _ANY_REG_mask = _ALL_REG_mask;
  501 
  502   if (PreserveFramePointer) {
  503     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  504     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()->next()));
  505   }
  506   if (need_r12_heapbase()) {
  507     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(r12->as_VMReg()));
  508     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(r12->as_VMReg()->next()));
  509   }
  510 
  511   _PTR_REG_mask = _ANY_REG_mask;
  512   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(rsp->as_VMReg()));
  513   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(rsp->as_VMReg()->next()));
  514   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(r15->as_VMReg()));
  515   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(r15->as_VMReg()->next()));
  516   if (!UseAPX) {
  517     for (uint i = 0; i < sizeof(egprs)/sizeof(Register); i++) {
  518       _PTR_REG_mask.Remove(OptoReg::as_OptoReg(egprs[i]->as_VMReg()));
  519       _PTR_REG_mask.Remove(OptoReg::as_OptoReg(egprs[i]->as_VMReg()->next()));
  520     }
  521   }
  522 
  523   _STACK_OR_PTR_REG_mask = _PTR_REG_mask;
  524   _STACK_OR_PTR_REG_mask.OR(STACK_OR_STACK_SLOTS_mask());
  525 
  526   _PTR_REG_NO_RBP_mask = _PTR_REG_mask;
  527   _PTR_REG_NO_RBP_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  528   _PTR_REG_NO_RBP_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()->next()));
  529 
  530   _PTR_NO_RAX_REG_mask = _PTR_REG_mask;
  531   _PTR_NO_RAX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()));
  532   _PTR_NO_RAX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()->next()));
  533 
  534   _PTR_NO_RAX_RBX_REG_mask = _PTR_NO_RAX_REG_mask;
  535   _PTR_NO_RAX_RBX_REG_mask.Remove(OptoReg::as_OptoReg(rbx->as_VMReg()));
  536   _PTR_NO_RAX_RBX_REG_mask.Remove(OptoReg::as_OptoReg(rbx->as_VMReg()->next()));
  537 
  538 
  539   _LONG_REG_mask = _PTR_REG_mask;
  540   _STACK_OR_LONG_REG_mask = _LONG_REG_mask;
  541   _STACK_OR_LONG_REG_mask.OR(STACK_OR_STACK_SLOTS_mask());
  542 
  543   _LONG_NO_RAX_RDX_REG_mask = _LONG_REG_mask;
  544   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()));
  545   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()->next()));
  546   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rdx->as_VMReg()));
  547   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rdx->as_VMReg()->next()));
  548 
  549   _LONG_NO_RCX_REG_mask = _LONG_REG_mask;
  550   _LONG_NO_RCX_REG_mask.Remove(OptoReg::as_OptoReg(rcx->as_VMReg()));
  551   _LONG_NO_RCX_REG_mask.Remove(OptoReg::as_OptoReg(rcx->as_VMReg()->next()));
  552 
  553   _LONG_NO_RBP_R13_REG_mask = _LONG_REG_mask;
  554   _LONG_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  555   _LONG_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()->next()));
  556   _LONG_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(r13->as_VMReg()));
  557   _LONG_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(r13->as_VMReg()->next()));
  558 
  559   _INT_REG_mask = _ALL_INT_REG_mask;
  560   if (!UseAPX) {
  561     for (uint i = 0; i < sizeof(egprs)/sizeof(Register); i++) {
  562       _INT_REG_mask.Remove(OptoReg::as_OptoReg(egprs[i]->as_VMReg()));
  563     }
  564   }
  565 
  566   if (PreserveFramePointer) {
  567     _INT_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  568   }
  569   if (need_r12_heapbase()) {
  570     _INT_REG_mask.Remove(OptoReg::as_OptoReg(r12->as_VMReg()));
  571   }
  572 
  573   _STACK_OR_INT_REG_mask = _INT_REG_mask;
  574   _STACK_OR_INT_REG_mask.OR(STACK_OR_STACK_SLOTS_mask());
  575 
  576   _INT_NO_RAX_RDX_REG_mask = _INT_REG_mask;
  577   _INT_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()));
  578   _INT_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rdx->as_VMReg()));
  579 
  580   _INT_NO_RCX_REG_mask = _INT_REG_mask;
  581   _INT_NO_RCX_REG_mask.Remove(OptoReg::as_OptoReg(rcx->as_VMReg()));
  582 
  583   _INT_NO_RBP_R13_REG_mask = _INT_REG_mask;
  584   _INT_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  585   _INT_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(r13->as_VMReg()));
  586 
  587   // _FLOAT_REG_LEGACY_mask/_FLOAT_REG_EVEX_mask is generated by adlc
  588   // from the float_reg_legacy/float_reg_evex register class.
  589   _FLOAT_REG_mask = VM_Version::supports_evex() ? _FLOAT_REG_EVEX_mask : _FLOAT_REG_LEGACY_mask;
  590 }
  591 
  592 static bool generate_vzeroupper(Compile* C) {
  593   return (VM_Version::supports_vzeroupper() && (C->max_vector_size() > 16 || C->clear_upper_avx() == true)) ? true: false;  // Generate vzeroupper
  594 }
  595 
  596 static int clear_avx_size() {
  597   return generate_vzeroupper(Compile::current()) ? 3: 0;  // vzeroupper
  598 }
  599 
  600 // !!!!! Special hack to get all types of calls to specify the byte offset
  601 //       from the start of the call to the point where the return address
  602 //       will point.
  603 int MachCallStaticJavaNode::ret_addr_offset()
  604 {
  605   int offset = 5; // 5 bytes from start of call to where return address points
  606   offset += clear_avx_size();
  607   return offset;
  608 }
  609 
  610 int MachCallDynamicJavaNode::ret_addr_offset()
  611 {
  612   int offset = 15; // 15 bytes from start of call to where return address points
  613   offset += clear_avx_size();
  614   return offset;
  615 }
  616 
  617 int MachCallRuntimeNode::ret_addr_offset() {
  618   int offset = 13; // movq r10,#addr; callq (r10)
  619   if (this->ideal_Opcode() != Op_CallLeafVector) {
  620     offset += clear_avx_size();
  621   }
  622   return offset;
  623 }
  624 //
  625 // Compute padding required for nodes which need alignment
  626 //
  627 
  628 // The address of the call instruction needs to be 4-byte aligned to
  629 // ensure that it does not span a cache line so that it can be patched.
  630 int CallStaticJavaDirectNode::compute_padding(int current_offset) const
  631 {
  632   current_offset += clear_avx_size(); // skip vzeroupper
  633   current_offset += 1; // skip call opcode byte
  634   return align_up(current_offset, alignment_required()) - current_offset;
  635 }
  636 
  637 // The address of the call instruction needs to be 4-byte aligned to
  638 // ensure that it does not span a cache line so that it can be patched.
  639 int CallDynamicJavaDirectNode::compute_padding(int current_offset) const
  640 {
  641   current_offset += clear_avx_size(); // skip vzeroupper
  642   current_offset += 11; // skip movq instruction + call opcode byte
  643   return align_up(current_offset, alignment_required()) - current_offset;
  644 }
  645 
  646 // This could be in MacroAssembler but it's fairly C2 specific
  647 static void emit_cmpfp_fixup(MacroAssembler* masm) {
  648   Label exit;
  649   __ jccb(Assembler::noParity, exit);
  650   __ pushf();
  651   //
  652   // comiss/ucomiss instructions set ZF,PF,CF flags and
  653   // zero OF,AF,SF for NaN values.
  654   // Fixup flags by zeroing ZF,PF so that compare of NaN
  655   // values returns 'less than' result (CF is set).
  656   // Leave the rest of flags unchanged.
  657   //
  658   //    7 6 5 4 3 2 1 0
  659   //   |S|Z|r|A|r|P|r|C|  (r - reserved bit)
  660   //    0 0 1 0 1 0 1 1   (0x2B)
  661   //
  662   __ andq(Address(rsp, 0), 0xffffff2b);
  663   __ popf();
  664   __ bind(exit);
  665 }
  666 
  667 static void emit_cmpfp3(MacroAssembler* masm, Register dst) {
  668   Label done;
  669   __ movl(dst, -1);
  670   __ jcc(Assembler::parity, done);
  671   __ jcc(Assembler::below, done);
  672   __ setcc(Assembler::notEqual, dst);
  673   __ bind(done);
  674 }
  675 
  676 // Math.min()    # Math.max()
  677 // --------------------------
  678 // ucomis[s/d]   #
  679 // ja   -> b     # a
  680 // jp   -> NaN   # NaN
  681 // jb   -> a     # b
  682 // je            #
  683 // |-jz -> a | b # a & b
  684 // |    -> a     #
  685 static void emit_fp_min_max(MacroAssembler* masm, XMMRegister dst,
  686                             XMMRegister a, XMMRegister b,
  687                             XMMRegister xmmt, Register rt,
  688                             bool min, bool single) {
  689 
  690   Label nan, zero, below, above, done;
  691 
  692   if (single)
  693     __ ucomiss(a, b);
  694   else
  695     __ ucomisd(a, b);
  696 
  697   if (dst->encoding() != (min ? b : a)->encoding())
  698     __ jccb(Assembler::above, above); // CF=0 & ZF=0
  699   else
  700     __ jccb(Assembler::above, done);
  701 
  702   __ jccb(Assembler::parity, nan);  // PF=1
  703   __ jccb(Assembler::below, below); // CF=1
  704 
  705   // equal
  706   __ vpxor(xmmt, xmmt, xmmt, Assembler::AVX_128bit);
  707   if (single) {
  708     __ ucomiss(a, xmmt);
  709     __ jccb(Assembler::equal, zero);
  710 
  711     __ movflt(dst, a);
  712     __ jmp(done);
  713   }
  714   else {
  715     __ ucomisd(a, xmmt);
  716     __ jccb(Assembler::equal, zero);
  717 
  718     __ movdbl(dst, a);
  719     __ jmp(done);
  720   }
  721 
  722   __ bind(zero);
  723   if (min)
  724     __ vpor(dst, a, b, Assembler::AVX_128bit);
  725   else
  726     __ vpand(dst, a, b, Assembler::AVX_128bit);
  727 
  728   __ jmp(done);
  729 
  730   __ bind(above);
  731   if (single)
  732     __ movflt(dst, min ? b : a);
  733   else
  734     __ movdbl(dst, min ? b : a);
  735 
  736   __ jmp(done);
  737 
  738   __ bind(nan);
  739   if (single) {
  740     __ movl(rt, 0x7fc00000); // Float.NaN
  741     __ movdl(dst, rt);
  742   }
  743   else {
  744     __ mov64(rt, 0x7ff8000000000000L); // Double.NaN
  745     __ movdq(dst, rt);
  746   }
  747   __ jmp(done);
  748 
  749   __ bind(below);
  750   if (single)
  751     __ movflt(dst, min ? a : b);
  752   else
  753     __ movdbl(dst, min ? a : b);
  754 
  755   __ bind(done);
  756 }
  757 
  758 //=============================================================================
  759 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
  760 
  761 int ConstantTable::calculate_table_base_offset() const {
  762   return 0;  // absolute addressing, no offset
  763 }
  764 
  765 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
  766 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
  767   ShouldNotReachHere();
  768 }
  769 
  770 void MachConstantBaseNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const {
  771   // Empty encoding
  772 }
  773 
  774 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
  775   return 0;
  776 }
  777 
  778 #ifndef PRODUCT
  779 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
  780   st->print("# MachConstantBaseNode (empty encoding)");
  781 }
  782 #endif
  783 
  784 
  785 //=============================================================================
  786 #ifndef PRODUCT
  787 void MachPrologNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
  788   Compile* C = ra_->C;
  789 
  790   int framesize = C->output()->frame_size_in_bytes();
  791   int bangsize = C->output()->bang_size_in_bytes();
  792   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
  793   // Remove wordSize for return addr which is already pushed.
  794   framesize -= wordSize;
  795 
  796   if (C->output()->need_stack_bang(bangsize)) {
  797     framesize -= wordSize;
  798     st->print("# stack bang (%d bytes)", bangsize);
  799     st->print("\n\t");
  800     st->print("pushq   rbp\t# Save rbp");
  801     if (PreserveFramePointer) {
  802         st->print("\n\t");
  803         st->print("movq    rbp, rsp\t# Save the caller's SP into rbp");
  804     }
  805     if (framesize) {
  806       st->print("\n\t");
  807       st->print("subq    rsp, #%d\t# Create frame",framesize);
  808     }
  809   } else {
  810     st->print("subq    rsp, #%d\t# Create frame",framesize);
  811     st->print("\n\t");
  812     framesize -= wordSize;
  813     st->print("movq    [rsp + #%d], rbp\t# Save rbp",framesize);
  814     if (PreserveFramePointer) {
  815       st->print("\n\t");
  816       st->print("movq    rbp, rsp\t# Save the caller's SP into rbp");
  817       if (framesize > 0) {
  818         st->print("\n\t");
  819         st->print("addq    rbp, #%d", framesize);
  820       }
  821     }
  822   }
  823 
  824   if (VerifyStackAtCalls) {
  825     st->print("\n\t");
  826     framesize -= wordSize;
  827     st->print("movq    [rsp + #%d], 0xbadb100d\t# Majik cookie for stack depth check",framesize);
  828 #ifdef ASSERT
  829     st->print("\n\t");
  830     st->print("# stack alignment check");
  831 #endif
  832   }
  833   if (C->stub_function() != nullptr) {
  834     st->print("\n\t");
  835     st->print("cmpl    [r15_thread + #disarmed_guard_value_offset], #disarmed_guard_value\t");
  836     st->print("\n\t");
  837     st->print("je      fast_entry\t");
  838     st->print("\n\t");
  839     st->print("call    #nmethod_entry_barrier_stub\t");
  840     st->print("\n\tfast_entry:");
  841   }
  842   st->cr();
  843 }
  844 #endif
  845 
  846 void MachPrologNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
  847   Compile* C = ra_->C;
  848 
  849   int framesize = C->output()->frame_size_in_bytes();
  850   int bangsize = C->output()->bang_size_in_bytes();
  851 
  852   if (C->clinit_barrier_on_entry()) {
  853     assert(VM_Version::supports_fast_class_init_checks(), "sanity");
  854     assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
  855 
  856     Label L_skip_barrier;
  857     Register klass = rscratch1;
  858 
  859     __ mov_metadata(klass, C->method()->holder()->constant_encoding());
  860     __ clinit_barrier(klass, &L_skip_barrier /*L_fast_path*/);
  861 
  862     __ jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub())); // slow path
  863 
  864     __ bind(L_skip_barrier);
  865   }
  866 
  867   __ verified_entry(framesize, C->output()->need_stack_bang(bangsize)?bangsize:0, false, C->stub_function() != nullptr);
  868 
  869   C->output()->set_frame_complete(__ offset());
  870 
  871   if (C->has_mach_constant_base_node()) {
  872     // NOTE: We set the table base offset here because users might be
  873     // emitted before MachConstantBaseNode.
  874     ConstantTable& constant_table = C->output()->constant_table();
  875     constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
  876   }
  877 }
  878 
  879 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
  880 {
  881   return MachNode::size(ra_); // too many variables; just compute it
  882                               // the hard way
  883 }
  884 
  885 int MachPrologNode::reloc() const
  886 {
  887   return 0; // a large enough number
  888 }
  889 
  890 //=============================================================================
  891 #ifndef PRODUCT
  892 void MachEpilogNode::format(PhaseRegAlloc* ra_, outputStream* st) const
  893 {
  894   Compile* C = ra_->C;
  895   if (generate_vzeroupper(C)) {
  896     st->print("vzeroupper");
  897     st->cr(); st->print("\t");
  898   }
  899 
  900   int framesize = C->output()->frame_size_in_bytes();
  901   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
  902   // Remove word for return adr already pushed
  903   // and RBP
  904   framesize -= 2*wordSize;
  905 
  906   if (framesize) {
  907     st->print_cr("addq    rsp, %d\t# Destroy frame", framesize);
  908     st->print("\t");
  909   }
  910 
  911   st->print_cr("popq    rbp");
  912   if (do_polling() && C->is_method_compilation()) {
  913     st->print("\t");
  914     st->print_cr("cmpq    rsp, poll_offset[r15_thread] \n\t"
  915                  "ja      #safepoint_stub\t"
  916                  "# Safepoint: poll for GC");
  917   }
  918 }
  919 #endif
  920 
  921 void MachEpilogNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
  922 {
  923   Compile* C = ra_->C;
  924 
  925   if (generate_vzeroupper(C)) {
  926     // Clear upper bits of YMM registers when current compiled code uses
  927     // wide vectors to avoid AVX <-> SSE transition penalty during call.
  928     __ vzeroupper();
  929   }
  930 
  931   int framesize = C->output()->frame_size_in_bytes();
  932   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
  933   // Remove word for return adr already pushed
  934   // and RBP
  935   framesize -= 2*wordSize;
  936 
  937   // Note that VerifyStackAtCalls' Majik cookie does not change the frame size popped here
  938 
  939   if (framesize) {
  940     __ addq(rsp, framesize);
  941   }
  942 
  943   __ popq(rbp);
  944 
  945   if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
  946     __ reserved_stack_check();
  947   }
  948 
  949   if (do_polling() && C->is_method_compilation()) {
  950     Label dummy_label;
  951     Label* code_stub = &dummy_label;
  952     if (!C->output()->in_scratch_emit_size()) {
  953       C2SafepointPollStub* stub = new (C->comp_arena()) C2SafepointPollStub(__ offset());
  954       C->output()->add_stub(stub);
  955       code_stub = &stub->entry();
  956     }
  957     __ relocate(relocInfo::poll_return_type);
  958     __ safepoint_poll(*code_stub, true /* at_return */, true /* in_nmethod */);
  959   }
  960 }
  961 
  962 uint MachEpilogNode::size(PhaseRegAlloc* ra_) const
  963 {
  964   return MachNode::size(ra_); // too many variables; just compute it
  965                               // the hard way
  966 }
  967 
  968 int MachEpilogNode::reloc() const
  969 {
  970   return 2; // a large enough number
  971 }
  972 
  973 const Pipeline* MachEpilogNode::pipeline() const
  974 {
  975   return MachNode::pipeline_class();
  976 }
  977 
  978 //=============================================================================
  979 
  980 enum RC {
  981   rc_bad,
  982   rc_int,
  983   rc_kreg,
  984   rc_float,
  985   rc_stack
  986 };
  987 
  988 static enum RC rc_class(OptoReg::Name reg)
  989 {
  990   if( !OptoReg::is_valid(reg)  ) return rc_bad;
  991 
  992   if (OptoReg::is_stack(reg)) return rc_stack;
  993 
  994   VMReg r = OptoReg::as_VMReg(reg);
  995 
  996   if (r->is_Register()) return rc_int;
  997 
  998   if (r->is_KRegister()) return rc_kreg;
  999 
 1000   assert(r->is_XMMRegister(), "must be");
 1001   return rc_float;
 1002 }
 1003 
 1004 // Next two methods are shared by 32- and 64-bit VM. They are defined in x86.ad.
 1005 static void vec_mov_helper(C2_MacroAssembler *masm, int src_lo, int dst_lo,
 1006                           int src_hi, int dst_hi, uint ireg, outputStream* st);
 1007 
 1008 void vec_spill_helper(C2_MacroAssembler *masm, bool is_load,
 1009                      int stack_offset, int reg, uint ireg, outputStream* st);
 1010 
 1011 static void vec_stack_to_stack_helper(C2_MacroAssembler *masm, int src_offset,
 1012                                       int dst_offset, uint ireg, outputStream* st) {
 1013   if (masm) {
 1014     switch (ireg) {
 1015     case Op_VecS:
 1016       __ movq(Address(rsp, -8), rax);
 1017       __ movl(rax, Address(rsp, src_offset));
 1018       __ movl(Address(rsp, dst_offset), rax);
 1019       __ movq(rax, Address(rsp, -8));
 1020       break;
 1021     case Op_VecD:
 1022       __ pushq(Address(rsp, src_offset));
 1023       __ popq (Address(rsp, dst_offset));
 1024       break;
 1025     case Op_VecX:
 1026       __ pushq(Address(rsp, src_offset));
 1027       __ popq (Address(rsp, dst_offset));
 1028       __ pushq(Address(rsp, src_offset+8));
 1029       __ popq (Address(rsp, dst_offset+8));
 1030       break;
 1031     case Op_VecY:
 1032       __ vmovdqu(Address(rsp, -32), xmm0);
 1033       __ vmovdqu(xmm0, Address(rsp, src_offset));
 1034       __ vmovdqu(Address(rsp, dst_offset), xmm0);
 1035       __ vmovdqu(xmm0, Address(rsp, -32));
 1036       break;
 1037     case Op_VecZ:
 1038       __ evmovdquq(Address(rsp, -64), xmm0, 2);
 1039       __ evmovdquq(xmm0, Address(rsp, src_offset), 2);
 1040       __ evmovdquq(Address(rsp, dst_offset), xmm0, 2);
 1041       __ evmovdquq(xmm0, Address(rsp, -64), 2);
 1042       break;
 1043     default:
 1044       ShouldNotReachHere();
 1045     }
 1046 #ifndef PRODUCT
 1047   } else {
 1048     switch (ireg) {
 1049     case Op_VecS:
 1050       st->print("movq    [rsp - #8], rax\t# 32-bit mem-mem spill\n\t"
 1051                 "movl    rax, [rsp + #%d]\n\t"
 1052                 "movl    [rsp + #%d], rax\n\t"
 1053                 "movq    rax, [rsp - #8]",
 1054                 src_offset, dst_offset);
 1055       break;
 1056     case Op_VecD:
 1057       st->print("pushq   [rsp + #%d]\t# 64-bit mem-mem spill\n\t"
 1058                 "popq    [rsp + #%d]",
 1059                 src_offset, dst_offset);
 1060       break;
 1061      case Op_VecX:
 1062       st->print("pushq   [rsp + #%d]\t# 128-bit mem-mem spill\n\t"
 1063                 "popq    [rsp + #%d]\n\t"
 1064                 "pushq   [rsp + #%d]\n\t"
 1065                 "popq    [rsp + #%d]",
 1066                 src_offset, dst_offset, src_offset+8, dst_offset+8);
 1067       break;
 1068     case Op_VecY:
 1069       st->print("vmovdqu [rsp - #32], xmm0\t# 256-bit mem-mem spill\n\t"
 1070                 "vmovdqu xmm0, [rsp + #%d]\n\t"
 1071                 "vmovdqu [rsp + #%d], xmm0\n\t"
 1072                 "vmovdqu xmm0, [rsp - #32]",
 1073                 src_offset, dst_offset);
 1074       break;
 1075     case Op_VecZ:
 1076       st->print("vmovdqu [rsp - #64], xmm0\t# 512-bit mem-mem spill\n\t"
 1077                 "vmovdqu xmm0, [rsp + #%d]\n\t"
 1078                 "vmovdqu [rsp + #%d], xmm0\n\t"
 1079                 "vmovdqu xmm0, [rsp - #64]",
 1080                 src_offset, dst_offset);
 1081       break;
 1082     default:
 1083       ShouldNotReachHere();
 1084     }
 1085 #endif
 1086   }
 1087 }
 1088 
 1089 uint MachSpillCopyNode::implementation(C2_MacroAssembler* masm,
 1090                                        PhaseRegAlloc* ra_,
 1091                                        bool do_size,
 1092                                        outputStream* st) const {
 1093   assert(masm != nullptr || st  != nullptr, "sanity");
 1094   // Get registers to move
 1095   OptoReg::Name src_second = ra_->get_reg_second(in(1));
 1096   OptoReg::Name src_first = ra_->get_reg_first(in(1));
 1097   OptoReg::Name dst_second = ra_->get_reg_second(this);
 1098   OptoReg::Name dst_first = ra_->get_reg_first(this);
 1099 
 1100   enum RC src_second_rc = rc_class(src_second);
 1101   enum RC src_first_rc = rc_class(src_first);
 1102   enum RC dst_second_rc = rc_class(dst_second);
 1103   enum RC dst_first_rc = rc_class(dst_first);
 1104 
 1105   assert(OptoReg::is_valid(src_first) && OptoReg::is_valid(dst_first),
 1106          "must move at least 1 register" );
 1107 
 1108   if (src_first == dst_first && src_second == dst_second) {
 1109     // Self copy, no move
 1110     return 0;
 1111   }
 1112   if (bottom_type()->isa_vect() != nullptr && bottom_type()->isa_vectmask() == nullptr) {
 1113     uint ireg = ideal_reg();
 1114     assert((src_first_rc != rc_int && dst_first_rc != rc_int), "sanity");
 1115     assert((ireg == Op_VecS || ireg == Op_VecD || ireg == Op_VecX || ireg == Op_VecY || ireg == Op_VecZ ), "sanity");
 1116     if( src_first_rc == rc_stack && dst_first_rc == rc_stack ) {
 1117       // mem -> mem
 1118       int src_offset = ra_->reg2offset(src_first);
 1119       int dst_offset = ra_->reg2offset(dst_first);
 1120       vec_stack_to_stack_helper(masm, src_offset, dst_offset, ireg, st);
 1121     } else if (src_first_rc == rc_float && dst_first_rc == rc_float ) {
 1122       vec_mov_helper(masm, src_first, dst_first, src_second, dst_second, ireg, st);
 1123     } else if (src_first_rc == rc_float && dst_first_rc == rc_stack ) {
 1124       int stack_offset = ra_->reg2offset(dst_first);
 1125       vec_spill_helper(masm, false, stack_offset, src_first, ireg, st);
 1126     } else if (src_first_rc == rc_stack && dst_first_rc == rc_float ) {
 1127       int stack_offset = ra_->reg2offset(src_first);
 1128       vec_spill_helper(masm, true,  stack_offset, dst_first, ireg, st);
 1129     } else {
 1130       ShouldNotReachHere();
 1131     }
 1132     return 0;
 1133   }
 1134   if (src_first_rc == rc_stack) {
 1135     // mem ->
 1136     if (dst_first_rc == rc_stack) {
 1137       // mem -> mem
 1138       assert(src_second != dst_first, "overlap");
 1139       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1140           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1141         // 64-bit
 1142         int src_offset = ra_->reg2offset(src_first);
 1143         int dst_offset = ra_->reg2offset(dst_first);
 1144         if (masm) {
 1145           __ pushq(Address(rsp, src_offset));
 1146           __ popq (Address(rsp, dst_offset));
 1147 #ifndef PRODUCT
 1148         } else {
 1149           st->print("pushq   [rsp + #%d]\t# 64-bit mem-mem spill\n\t"
 1150                     "popq    [rsp + #%d]",
 1151                      src_offset, dst_offset);
 1152 #endif
 1153         }
 1154       } else {
 1155         // 32-bit
 1156         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1157         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1158         // No pushl/popl, so:
 1159         int src_offset = ra_->reg2offset(src_first);
 1160         int dst_offset = ra_->reg2offset(dst_first);
 1161         if (masm) {
 1162           __ movq(Address(rsp, -8), rax);
 1163           __ movl(rax, Address(rsp, src_offset));
 1164           __ movl(Address(rsp, dst_offset), rax);
 1165           __ movq(rax, Address(rsp, -8));
 1166 #ifndef PRODUCT
 1167         } else {
 1168           st->print("movq    [rsp - #8], rax\t# 32-bit mem-mem spill\n\t"
 1169                     "movl    rax, [rsp + #%d]\n\t"
 1170                     "movl    [rsp + #%d], rax\n\t"
 1171                     "movq    rax, [rsp - #8]",
 1172                      src_offset, dst_offset);
 1173 #endif
 1174         }
 1175       }
 1176       return 0;
 1177     } else if (dst_first_rc == rc_int) {
 1178       // mem -> gpr
 1179       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1180           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1181         // 64-bit
 1182         int offset = ra_->reg2offset(src_first);
 1183         if (masm) {
 1184           __ movq(as_Register(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1185 #ifndef PRODUCT
 1186         } else {
 1187           st->print("movq    %s, [rsp + #%d]\t# spill",
 1188                      Matcher::regName[dst_first],
 1189                      offset);
 1190 #endif
 1191         }
 1192       } else {
 1193         // 32-bit
 1194         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1195         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1196         int offset = ra_->reg2offset(src_first);
 1197         if (masm) {
 1198           __ movl(as_Register(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1199 #ifndef PRODUCT
 1200         } else {
 1201           st->print("movl    %s, [rsp + #%d]\t# spill",
 1202                      Matcher::regName[dst_first],
 1203                      offset);
 1204 #endif
 1205         }
 1206       }
 1207       return 0;
 1208     } else if (dst_first_rc == rc_float) {
 1209       // mem-> xmm
 1210       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1211           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1212         // 64-bit
 1213         int offset = ra_->reg2offset(src_first);
 1214         if (masm) {
 1215           __ movdbl( as_XMMRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1216 #ifndef PRODUCT
 1217         } else {
 1218           st->print("%s  %s, [rsp + #%d]\t# spill",
 1219                      UseXmmLoadAndClearUpper ? "movsd " : "movlpd",
 1220                      Matcher::regName[dst_first],
 1221                      offset);
 1222 #endif
 1223         }
 1224       } else {
 1225         // 32-bit
 1226         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1227         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1228         int offset = ra_->reg2offset(src_first);
 1229         if (masm) {
 1230           __ movflt( as_XMMRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1231 #ifndef PRODUCT
 1232         } else {
 1233           st->print("movss   %s, [rsp + #%d]\t# spill",
 1234                      Matcher::regName[dst_first],
 1235                      offset);
 1236 #endif
 1237         }
 1238       }
 1239       return 0;
 1240     } else if (dst_first_rc == rc_kreg) {
 1241       // mem -> kreg
 1242       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1243           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1244         // 64-bit
 1245         int offset = ra_->reg2offset(src_first);
 1246         if (masm) {
 1247           __ kmov(as_KRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1248 #ifndef PRODUCT
 1249         } else {
 1250           st->print("kmovq   %s, [rsp + #%d]\t# spill",
 1251                      Matcher::regName[dst_first],
 1252                      offset);
 1253 #endif
 1254         }
 1255       }
 1256       return 0;
 1257     }
 1258   } else if (src_first_rc == rc_int) {
 1259     // gpr ->
 1260     if (dst_first_rc == rc_stack) {
 1261       // gpr -> mem
 1262       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1263           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1264         // 64-bit
 1265         int offset = ra_->reg2offset(dst_first);
 1266         if (masm) {
 1267           __ movq(Address(rsp, offset), as_Register(Matcher::_regEncode[src_first]));
 1268 #ifndef PRODUCT
 1269         } else {
 1270           st->print("movq    [rsp + #%d], %s\t# spill",
 1271                      offset,
 1272                      Matcher::regName[src_first]);
 1273 #endif
 1274         }
 1275       } else {
 1276         // 32-bit
 1277         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1278         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1279         int offset = ra_->reg2offset(dst_first);
 1280         if (masm) {
 1281           __ movl(Address(rsp, offset), as_Register(Matcher::_regEncode[src_first]));
 1282 #ifndef PRODUCT
 1283         } else {
 1284           st->print("movl    [rsp + #%d], %s\t# spill",
 1285                      offset,
 1286                      Matcher::regName[src_first]);
 1287 #endif
 1288         }
 1289       }
 1290       return 0;
 1291     } else if (dst_first_rc == rc_int) {
 1292       // gpr -> gpr
 1293       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1294           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1295         // 64-bit
 1296         if (masm) {
 1297           __ movq(as_Register(Matcher::_regEncode[dst_first]),
 1298                   as_Register(Matcher::_regEncode[src_first]));
 1299 #ifndef PRODUCT
 1300         } else {
 1301           st->print("movq    %s, %s\t# spill",
 1302                      Matcher::regName[dst_first],
 1303                      Matcher::regName[src_first]);
 1304 #endif
 1305         }
 1306         return 0;
 1307       } else {
 1308         // 32-bit
 1309         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1310         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1311         if (masm) {
 1312           __ movl(as_Register(Matcher::_regEncode[dst_first]),
 1313                   as_Register(Matcher::_regEncode[src_first]));
 1314 #ifndef PRODUCT
 1315         } else {
 1316           st->print("movl    %s, %s\t# spill",
 1317                      Matcher::regName[dst_first],
 1318                      Matcher::regName[src_first]);
 1319 #endif
 1320         }
 1321         return 0;
 1322       }
 1323     } else if (dst_first_rc == rc_float) {
 1324       // gpr -> xmm
 1325       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1326           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1327         // 64-bit
 1328         if (masm) {
 1329           __ movdq( as_XMMRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
 1330 #ifndef PRODUCT
 1331         } else {
 1332           st->print("movdq   %s, %s\t# spill",
 1333                      Matcher::regName[dst_first],
 1334                      Matcher::regName[src_first]);
 1335 #endif
 1336         }
 1337       } else {
 1338         // 32-bit
 1339         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1340         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1341         if (masm) {
 1342           __ movdl( as_XMMRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
 1343 #ifndef PRODUCT
 1344         } else {
 1345           st->print("movdl   %s, %s\t# spill",
 1346                      Matcher::regName[dst_first],
 1347                      Matcher::regName[src_first]);
 1348 #endif
 1349         }
 1350       }
 1351       return 0;
 1352     } else if (dst_first_rc == rc_kreg) {
 1353       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1354           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1355         // 64-bit
 1356         if (masm) {
 1357           __ kmov(as_KRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
 1358   #ifndef PRODUCT
 1359         } else {
 1360            st->print("kmovq   %s, %s\t# spill",
 1361                        Matcher::regName[dst_first],
 1362                        Matcher::regName[src_first]);
 1363   #endif
 1364         }
 1365       }
 1366       Unimplemented();
 1367       return 0;
 1368     }
 1369   } else if (src_first_rc == rc_float) {
 1370     // xmm ->
 1371     if (dst_first_rc == rc_stack) {
 1372       // xmm -> mem
 1373       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1374           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1375         // 64-bit
 1376         int offset = ra_->reg2offset(dst_first);
 1377         if (masm) {
 1378           __ movdbl( Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[src_first]));
 1379 #ifndef PRODUCT
 1380         } else {
 1381           st->print("movsd   [rsp + #%d], %s\t# spill",
 1382                      offset,
 1383                      Matcher::regName[src_first]);
 1384 #endif
 1385         }
 1386       } else {
 1387         // 32-bit
 1388         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1389         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1390         int offset = ra_->reg2offset(dst_first);
 1391         if (masm) {
 1392           __ movflt(Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[src_first]));
 1393 #ifndef PRODUCT
 1394         } else {
 1395           st->print("movss   [rsp + #%d], %s\t# spill",
 1396                      offset,
 1397                      Matcher::regName[src_first]);
 1398 #endif
 1399         }
 1400       }
 1401       return 0;
 1402     } else if (dst_first_rc == rc_int) {
 1403       // xmm -> gpr
 1404       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1405           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1406         // 64-bit
 1407         if (masm) {
 1408           __ movdq( as_Register(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
 1409 #ifndef PRODUCT
 1410         } else {
 1411           st->print("movdq   %s, %s\t# spill",
 1412                      Matcher::regName[dst_first],
 1413                      Matcher::regName[src_first]);
 1414 #endif
 1415         }
 1416       } else {
 1417         // 32-bit
 1418         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1419         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1420         if (masm) {
 1421           __ movdl( as_Register(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
 1422 #ifndef PRODUCT
 1423         } else {
 1424           st->print("movdl   %s, %s\t# spill",
 1425                      Matcher::regName[dst_first],
 1426                      Matcher::regName[src_first]);
 1427 #endif
 1428         }
 1429       }
 1430       return 0;
 1431     } else if (dst_first_rc == rc_float) {
 1432       // xmm -> xmm
 1433       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1434           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1435         // 64-bit
 1436         if (masm) {
 1437           __ movdbl( as_XMMRegister(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
 1438 #ifndef PRODUCT
 1439         } else {
 1440           st->print("%s  %s, %s\t# spill",
 1441                      UseXmmRegToRegMoveAll ? "movapd" : "movsd ",
 1442                      Matcher::regName[dst_first],
 1443                      Matcher::regName[src_first]);
 1444 #endif
 1445         }
 1446       } else {
 1447         // 32-bit
 1448         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1449         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1450         if (masm) {
 1451           __ movflt( as_XMMRegister(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
 1452 #ifndef PRODUCT
 1453         } else {
 1454           st->print("%s  %s, %s\t# spill",
 1455                      UseXmmRegToRegMoveAll ? "movaps" : "movss ",
 1456                      Matcher::regName[dst_first],
 1457                      Matcher::regName[src_first]);
 1458 #endif
 1459         }
 1460       }
 1461       return 0;
 1462     } else if (dst_first_rc == rc_kreg) {
 1463       assert(false, "Illegal spilling");
 1464       return 0;
 1465     }
 1466   } else if (src_first_rc == rc_kreg) {
 1467     if (dst_first_rc == rc_stack) {
 1468       // mem -> kreg
 1469       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1470           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1471         // 64-bit
 1472         int offset = ra_->reg2offset(dst_first);
 1473         if (masm) {
 1474           __ kmov(Address(rsp, offset), as_KRegister(Matcher::_regEncode[src_first]));
 1475 #ifndef PRODUCT
 1476         } else {
 1477           st->print("kmovq   [rsp + #%d] , %s\t# spill",
 1478                      offset,
 1479                      Matcher::regName[src_first]);
 1480 #endif
 1481         }
 1482       }
 1483       return 0;
 1484     } else if (dst_first_rc == rc_int) {
 1485       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1486           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1487         // 64-bit
 1488         if (masm) {
 1489           __ kmov(as_Register(Matcher::_regEncode[dst_first]), as_KRegister(Matcher::_regEncode[src_first]));
 1490 #ifndef PRODUCT
 1491         } else {
 1492          st->print("kmovq   %s, %s\t# spill",
 1493                      Matcher::regName[dst_first],
 1494                      Matcher::regName[src_first]);
 1495 #endif
 1496         }
 1497       }
 1498       Unimplemented();
 1499       return 0;
 1500     } else if (dst_first_rc == rc_kreg) {
 1501       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1502           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1503         // 64-bit
 1504         if (masm) {
 1505           __ kmov(as_KRegister(Matcher::_regEncode[dst_first]), as_KRegister(Matcher::_regEncode[src_first]));
 1506 #ifndef PRODUCT
 1507         } else {
 1508          st->print("kmovq   %s, %s\t# spill",
 1509                      Matcher::regName[dst_first],
 1510                      Matcher::regName[src_first]);
 1511 #endif
 1512         }
 1513       }
 1514       return 0;
 1515     } else if (dst_first_rc == rc_float) {
 1516       assert(false, "Illegal spill");
 1517       return 0;
 1518     }
 1519   }
 1520 
 1521   assert(0," foo ");
 1522   Unimplemented();
 1523   return 0;
 1524 }
 1525 
 1526 #ifndef PRODUCT
 1527 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream* st) const {
 1528   implementation(nullptr, ra_, false, st);
 1529 }
 1530 #endif
 1531 
 1532 void MachSpillCopyNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
 1533   implementation(masm, ra_, false, nullptr);
 1534 }
 1535 
 1536 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
 1537   return MachNode::size(ra_);
 1538 }
 1539 
 1540 //=============================================================================
 1541 #ifndef PRODUCT
 1542 void BoxLockNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 1543 {
 1544   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 1545   int reg = ra_->get_reg_first(this);
 1546   st->print("leaq    %s, [rsp + #%d]\t# box lock",
 1547             Matcher::regName[reg], offset);
 1548 }
 1549 #endif
 1550 
 1551 void BoxLockNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
 1552 {
 1553   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 1554   int reg = ra_->get_encode(this);
 1555 
 1556   __ lea(as_Register(reg), Address(rsp, offset));
 1557 }
 1558 
 1559 uint BoxLockNode::size(PhaseRegAlloc *ra_) const
 1560 {
 1561   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 1562   if (ra_->get_encode(this) > 15) {
 1563     return (offset < 0x80) ? 6 : 9; // REX2
 1564   } else {
 1565     return (offset < 0x80) ? 5 : 8; // REX
 1566   }
 1567 }
 1568 
 1569 //=============================================================================
 1570 #ifndef PRODUCT
 1571 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 1572 {
 1573   if (UseCompressedClassPointers) {
 1574     st->print_cr("movl    rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
 1575     st->print_cr("\tcmpl    rscratch1, [rax + CompiledICData::speculated_klass_offset()]\t # Inline cache check");
 1576   } else {
 1577     st->print_cr("movq    rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
 1578     st->print_cr("\tcmpq    rscratch1, [rax + CompiledICData::speculated_klass_offset()]\t # Inline cache check");
 1579   }
 1580   st->print_cr("\tjne     SharedRuntime::_ic_miss_stub");
 1581 }
 1582 #endif
 1583 
 1584 void MachUEPNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
 1585 {
 1586   __ ic_check(InteriorEntryAlignment);
 1587 }
 1588 
 1589 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
 1590 {
 1591   return MachNode::size(ra_); // too many variables; just compute it
 1592                               // the hard way
 1593 }
 1594 
 1595 
 1596 //=============================================================================
 1597 
 1598 bool Matcher::supports_vector_calling_convention(void) {
 1599   return EnableVectorSupport;
 1600 }
 1601 
 1602 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
 1603   assert(EnableVectorSupport, "sanity");
 1604   int lo = XMM0_num;
 1605   int hi = XMM0b_num;
 1606   if (ideal_reg == Op_VecX) hi = XMM0d_num;
 1607   else if (ideal_reg == Op_VecY) hi = XMM0h_num;
 1608   else if (ideal_reg == Op_VecZ) hi = XMM0p_num;
 1609   return OptoRegPair(hi, lo);
 1610 }
 1611 
 1612 // Is this branch offset short enough that a short branch can be used?
 1613 //
 1614 // NOTE: If the platform does not provide any short branch variants, then
 1615 //       this method should return false for offset 0.
 1616 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
 1617   // The passed offset is relative to address of the branch.
 1618   // On 86 a branch displacement is calculated relative to address
 1619   // of a next instruction.
 1620   offset -= br_size;
 1621 
 1622   // the short version of jmpConUCF2 contains multiple branches,
 1623   // making the reach slightly less
 1624   if (rule == jmpConUCF2_rule)
 1625     return (-126 <= offset && offset <= 125);
 1626   return (-128 <= offset && offset <= 127);
 1627 }
 1628 
 1629 // Return whether or not this register is ever used as an argument.
 1630 // This function is used on startup to build the trampoline stubs in
 1631 // generateOptoStub.  Registers not mentioned will be killed by the VM
 1632 // call in the trampoline, and arguments in those registers not be
 1633 // available to the callee.
 1634 bool Matcher::can_be_java_arg(int reg)
 1635 {
 1636   return
 1637     reg ==  RDI_num || reg == RDI_H_num ||
 1638     reg ==  RSI_num || reg == RSI_H_num ||
 1639     reg ==  RDX_num || reg == RDX_H_num ||
 1640     reg ==  RCX_num || reg == RCX_H_num ||
 1641     reg ==   R8_num || reg ==  R8_H_num ||
 1642     reg ==   R9_num || reg ==  R9_H_num ||
 1643     reg ==  R12_num || reg == R12_H_num ||
 1644     reg == XMM0_num || reg == XMM0b_num ||
 1645     reg == XMM1_num || reg == XMM1b_num ||
 1646     reg == XMM2_num || reg == XMM2b_num ||
 1647     reg == XMM3_num || reg == XMM3b_num ||
 1648     reg == XMM4_num || reg == XMM4b_num ||
 1649     reg == XMM5_num || reg == XMM5b_num ||
 1650     reg == XMM6_num || reg == XMM6b_num ||
 1651     reg == XMM7_num || reg == XMM7b_num;
 1652 }
 1653 
 1654 bool Matcher::is_spillable_arg(int reg)
 1655 {
 1656   return can_be_java_arg(reg);
 1657 }
 1658 
 1659 uint Matcher::int_pressure_limit()
 1660 {
 1661   return (INTPRESSURE == -1) ? _INT_REG_mask.Size() : INTPRESSURE;
 1662 }
 1663 
 1664 uint Matcher::float_pressure_limit()
 1665 {
 1666   // After experiment around with different values, the following default threshold
 1667   // works best for LCM's register pressure scheduling on x64.
 1668   uint dec_count  = VM_Version::supports_evex() ? 4 : 2;
 1669   uint default_float_pressure_threshold = _FLOAT_REG_mask.Size() - dec_count;
 1670   return (FLOATPRESSURE == -1) ? default_float_pressure_threshold : FLOATPRESSURE;
 1671 }
 1672 
 1673 bool Matcher::use_asm_for_ldiv_by_con( jlong divisor ) {
 1674   // In 64 bit mode a code which use multiply when
 1675   // devisor is constant is faster than hardware
 1676   // DIV instruction (it uses MulHiL).
 1677   return false;
 1678 }
 1679 
 1680 // Register for DIVI projection of divmodI
 1681 RegMask Matcher::divI_proj_mask() {
 1682   return INT_RAX_REG_mask();
 1683 }
 1684 
 1685 // Register for MODI projection of divmodI
 1686 RegMask Matcher::modI_proj_mask() {
 1687   return INT_RDX_REG_mask();
 1688 }
 1689 
 1690 // Register for DIVL projection of divmodL
 1691 RegMask Matcher::divL_proj_mask() {
 1692   return LONG_RAX_REG_mask();
 1693 }
 1694 
 1695 // Register for MODL projection of divmodL
 1696 RegMask Matcher::modL_proj_mask() {
 1697   return LONG_RDX_REG_mask();
 1698 }
 1699 
 1700 // Register for saving SP into on method handle invokes. Not used on x86_64.
 1701 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
 1702     return NO_REG_mask();
 1703 }
 1704 
 1705 %}
 1706 
 1707 //----------ENCODING BLOCK-----------------------------------------------------
 1708 // This block specifies the encoding classes used by the compiler to
 1709 // output byte streams.  Encoding classes are parameterized macros
 1710 // used by Machine Instruction Nodes in order to generate the bit
 1711 // encoding of the instruction.  Operands specify their base encoding
 1712 // interface with the interface keyword.  There are currently
 1713 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
 1714 // COND_INTER.  REG_INTER causes an operand to generate a function
 1715 // which returns its register number when queried.  CONST_INTER causes
 1716 // an operand to generate a function which returns the value of the
 1717 // constant when queried.  MEMORY_INTER causes an operand to generate
 1718 // four functions which return the Base Register, the Index Register,
 1719 // the Scale Value, and the Offset Value of the operand when queried.
 1720 // COND_INTER causes an operand to generate six functions which return
 1721 // the encoding code (ie - encoding bits for the instruction)
 1722 // associated with each basic boolean condition for a conditional
 1723 // instruction.
 1724 //
 1725 // Instructions specify two basic values for encoding.  Again, a
 1726 // function is available to check if the constant displacement is an
 1727 // oop. They use the ins_encode keyword to specify their encoding
 1728 // classes (which must be a sequence of enc_class names, and their
 1729 // parameters, specified in the encoding block), and they use the
 1730 // opcode keyword to specify, in order, their primary, secondary, and
 1731 // tertiary opcode.  Only the opcode sections which a particular
 1732 // instruction needs for encoding need to be specified.
 1733 encode %{
 1734   enc_class cdql_enc(no_rax_rdx_RegI div)
 1735   %{
 1736     // Full implementation of Java idiv and irem; checks for
 1737     // special case as described in JVM spec., p.243 & p.271.
 1738     //
 1739     //         normal case                           special case
 1740     //
 1741     // input : rax: dividend                         min_int
 1742     //         reg: divisor                          -1
 1743     //
 1744     // output: rax: quotient  (= rax idiv reg)       min_int
 1745     //         rdx: remainder (= rax irem reg)       0
 1746     //
 1747     //  Code sequnce:
 1748     //
 1749     //    0:   3d 00 00 00 80          cmp    $0x80000000,%eax
 1750     //    5:   75 07/08                jne    e <normal>
 1751     //    7:   33 d2                   xor    %edx,%edx
 1752     //  [div >= 8 -> offset + 1]
 1753     //  [REX_B]
 1754     //    9:   83 f9 ff                cmp    $0xffffffffffffffff,$div
 1755     //    c:   74 03/04                je     11 <done>
 1756     // 000000000000000e <normal>:
 1757     //    e:   99                      cltd
 1758     //  [div >= 8 -> offset + 1]
 1759     //  [REX_B]
 1760     //    f:   f7 f9                   idiv   $div
 1761     // 0000000000000011 <done>:
 1762     Label normal;
 1763     Label done;
 1764 
 1765     // cmp    $0x80000000,%eax
 1766     __ cmpl(as_Register(RAX_enc), 0x80000000);
 1767 
 1768     // jne    e <normal>
 1769     __ jccb(Assembler::notEqual, normal);
 1770 
 1771     // xor    %edx,%edx
 1772     __ xorl(as_Register(RDX_enc), as_Register(RDX_enc));
 1773 
 1774     // cmp    $0xffffffffffffffff,%ecx
 1775     __ cmpl($div$$Register, -1);
 1776 
 1777     // je     11 <done>
 1778     __ jccb(Assembler::equal, done);
 1779 
 1780     // <normal>
 1781     // cltd
 1782     __ bind(normal);
 1783     __ cdql();
 1784 
 1785     // idivl
 1786     // <done>
 1787     __ idivl($div$$Register);
 1788     __ bind(done);
 1789   %}
 1790 
 1791   enc_class cdqq_enc(no_rax_rdx_RegL div)
 1792   %{
 1793     // Full implementation of Java ldiv and lrem; checks for
 1794     // special case as described in JVM spec., p.243 & p.271.
 1795     //
 1796     //         normal case                           special case
 1797     //
 1798     // input : rax: dividend                         min_long
 1799     //         reg: divisor                          -1
 1800     //
 1801     // output: rax: quotient  (= rax idiv reg)       min_long
 1802     //         rdx: remainder (= rax irem reg)       0
 1803     //
 1804     //  Code sequnce:
 1805     //
 1806     //    0:   48 ba 00 00 00 00 00    mov    $0x8000000000000000,%rdx
 1807     //    7:   00 00 80
 1808     //    a:   48 39 d0                cmp    %rdx,%rax
 1809     //    d:   75 08                   jne    17 <normal>
 1810     //    f:   33 d2                   xor    %edx,%edx
 1811     //   11:   48 83 f9 ff             cmp    $0xffffffffffffffff,$div
 1812     //   15:   74 05                   je     1c <done>
 1813     // 0000000000000017 <normal>:
 1814     //   17:   48 99                   cqto
 1815     //   19:   48 f7 f9                idiv   $div
 1816     // 000000000000001c <done>:
 1817     Label normal;
 1818     Label done;
 1819 
 1820     // mov    $0x8000000000000000,%rdx
 1821     __ mov64(as_Register(RDX_enc), 0x8000000000000000);
 1822 
 1823     // cmp    %rdx,%rax
 1824     __ cmpq(as_Register(RAX_enc), as_Register(RDX_enc));
 1825 
 1826     // jne    17 <normal>
 1827     __ jccb(Assembler::notEqual, normal);
 1828 
 1829     // xor    %edx,%edx
 1830     __ xorl(as_Register(RDX_enc), as_Register(RDX_enc));
 1831 
 1832     // cmp    $0xffffffffffffffff,$div
 1833     __ cmpq($div$$Register, -1);
 1834 
 1835     // je     1e <done>
 1836     __ jccb(Assembler::equal, done);
 1837 
 1838     // <normal>
 1839     // cqto
 1840     __ bind(normal);
 1841     __ cdqq();
 1842 
 1843     // idivq (note: must be emitted by the user of this rule)
 1844     // <done>
 1845     __ idivq($div$$Register);
 1846     __ bind(done);
 1847   %}
 1848 
 1849   enc_class clear_avx %{
 1850     DEBUG_ONLY(int off0 = __ offset());
 1851     if (generate_vzeroupper(Compile::current())) {
 1852       // Clear upper bits of YMM registers to avoid AVX <-> SSE transition penalty
 1853       // Clear upper bits of YMM registers when current compiled code uses
 1854       // wide vectors to avoid AVX <-> SSE transition penalty during call.
 1855       __ vzeroupper();
 1856     }
 1857     DEBUG_ONLY(int off1 = __ offset());
 1858     assert(off1 - off0 == clear_avx_size(), "correct size prediction");
 1859   %}
 1860 
 1861   enc_class Java_To_Runtime(method meth) %{
 1862     __ lea(r10, RuntimeAddress((address)$meth$$method));
 1863     __ call(r10);
 1864     __ post_call_nop();
 1865   %}
 1866 
 1867   enc_class Java_Static_Call(method meth)
 1868   %{
 1869     // JAVA STATIC CALL
 1870     // CALL to fixup routine.  Fixup routine uses ScopeDesc info to
 1871     // determine who we intended to call.
 1872     if (!_method) {
 1873       __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, $meth$$method)));
 1874     } else if (_method->intrinsic_id() == vmIntrinsicID::_ensureMaterializedForStackWalk) {
 1875       // The NOP here is purely to ensure that eliding a call to
 1876       // JVM_EnsureMaterializedForStackWalk doesn't change the code size.
 1877       __ addr_nop_5();
 1878       __ block_comment("call JVM_EnsureMaterializedForStackWalk (elided)");
 1879     } else {
 1880       int method_index = resolved_method_index(masm);
 1881       RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
 1882                                                   : static_call_Relocation::spec(method_index);
 1883       address mark = __ pc();
 1884       int call_offset = __ offset();
 1885       __ call(AddressLiteral(CAST_FROM_FN_PTR(address, $meth$$method), rspec));
 1886       if (CodeBuffer::supports_shared_stubs() && _method->can_be_statically_bound()) {
 1887         // Calls of the same statically bound method can share
 1888         // a stub to the interpreter.
 1889         __ code()->shared_stub_to_interp_for(_method, call_offset);
 1890       } else {
 1891         // Emit stubs for static call.
 1892         address stub = CompiledDirectCall::emit_to_interp_stub(masm, mark);
 1893         __ clear_inst_mark();
 1894         if (stub == nullptr) {
 1895           ciEnv::current()->record_failure("CodeCache is full");
 1896           return;
 1897         }
 1898       }
 1899     }
 1900     __ post_call_nop();
 1901   %}
 1902 
 1903   enc_class Java_Dynamic_Call(method meth) %{
 1904     __ ic_call((address)$meth$$method, resolved_method_index(masm));
 1905     __ post_call_nop();
 1906   %}
 1907 
 1908 %}
 1909 
 1910 
 1911 
 1912 //----------FRAME--------------------------------------------------------------
 1913 // Definition of frame structure and management information.
 1914 //
 1915 //  S T A C K   L A Y O U T    Allocators stack-slot number
 1916 //                             |   (to get allocators register number
 1917 //  G  Owned by    |        |  v    add OptoReg::stack0())
 1918 //  r   CALLER     |        |
 1919 //  o     |        +--------+      pad to even-align allocators stack-slot
 1920 //  w     V        |  pad0  |        numbers; owned by CALLER
 1921 //  t   -----------+--------+----> Matcher::_in_arg_limit, unaligned
 1922 //  h     ^        |   in   |  5
 1923 //        |        |  args  |  4   Holes in incoming args owned by SELF
 1924 //  |     |        |        |  3
 1925 //  |     |        +--------+
 1926 //  V     |        | old out|      Empty on Intel, window on Sparc
 1927 //        |    old |preserve|      Must be even aligned.
 1928 //        |     SP-+--------+----> Matcher::_old_SP, even aligned
 1929 //        |        |   in   |  3   area for Intel ret address
 1930 //     Owned by    |preserve|      Empty on Sparc.
 1931 //       SELF      +--------+
 1932 //        |        |  pad2  |  2   pad to align old SP
 1933 //        |        +--------+  1
 1934 //        |        | locks  |  0
 1935 //        |        +--------+----> OptoReg::stack0(), even aligned
 1936 //        |        |  pad1  | 11   pad to align new SP
 1937 //        |        +--------+
 1938 //        |        |        | 10
 1939 //        |        | spills |  9   spills
 1940 //        V        |        |  8   (pad0 slot for callee)
 1941 //      -----------+--------+----> Matcher::_out_arg_limit, unaligned
 1942 //        ^        |  out   |  7
 1943 //        |        |  args  |  6   Holes in outgoing args owned by CALLEE
 1944 //     Owned by    +--------+
 1945 //      CALLEE     | new out|  6   Empty on Intel, window on Sparc
 1946 //        |    new |preserve|      Must be even-aligned.
 1947 //        |     SP-+--------+----> Matcher::_new_SP, even aligned
 1948 //        |        |        |
 1949 //
 1950 // Note 1: Only region 8-11 is determined by the allocator.  Region 0-5 is
 1951 //         known from SELF's arguments and the Java calling convention.
 1952 //         Region 6-7 is determined per call site.
 1953 // Note 2: If the calling convention leaves holes in the incoming argument
 1954 //         area, those holes are owned by SELF.  Holes in the outgoing area
 1955 //         are owned by the CALLEE.  Holes should not be necessary in the
 1956 //         incoming area, as the Java calling convention is completely under
 1957 //         the control of the AD file.  Doubles can be sorted and packed to
 1958 //         avoid holes.  Holes in the outgoing arguments may be necessary for
 1959 //         varargs C calling conventions.
 1960 // Note 3: Region 0-3 is even aligned, with pad2 as needed.  Region 3-5 is
 1961 //         even aligned with pad0 as needed.
 1962 //         Region 6 is even aligned.  Region 6-7 is NOT even aligned;
 1963 //         region 6-11 is even aligned; it may be padded out more so that
 1964 //         the region from SP to FP meets the minimum stack alignment.
 1965 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
 1966 //         alignment.  Region 11, pad1, may be dynamically extended so that
 1967 //         SP meets the minimum alignment.
 1968 
 1969 frame
 1970 %{
 1971   // These three registers define part of the calling convention
 1972   // between compiled code and the interpreter.
 1973   inline_cache_reg(RAX);                // Inline Cache Register
 1974 
 1975   // Optional: name the operand used by cisc-spilling to access
 1976   // [stack_pointer + offset]
 1977   cisc_spilling_operand_name(indOffset32);
 1978 
 1979   // Number of stack slots consumed by locking an object
 1980   sync_stack_slots(2);
 1981 
 1982   // Compiled code's Frame Pointer
 1983   frame_pointer(RSP);
 1984 
 1985   // Interpreter stores its frame pointer in a register which is
 1986   // stored to the stack by I2CAdaptors.
 1987   // I2CAdaptors convert from interpreted java to compiled java.
 1988   interpreter_frame_pointer(RBP);
 1989 
 1990   // Stack alignment requirement
 1991   stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
 1992 
 1993   // Number of outgoing stack slots killed above the out_preserve_stack_slots
 1994   // for calls to C.  Supports the var-args backing area for register parms.
 1995   varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
 1996 
 1997   // The after-PROLOG location of the return address.  Location of
 1998   // return address specifies a type (REG or STACK) and a number
 1999   // representing the register number (i.e. - use a register name) or
 2000   // stack slot.
 2001   // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
 2002   // Otherwise, it is above the locks and verification slot and alignment word
 2003   return_addr(STACK - 2 +
 2004               align_up((Compile::current()->in_preserve_stack_slots() +
 2005                         Compile::current()->fixed_slots()),
 2006                        stack_alignment_in_slots()));
 2007 
 2008   // Location of compiled Java return values.  Same as C for now.
 2009   return_value
 2010   %{
 2011     assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
 2012            "only return normal values");
 2013 
 2014     static const int lo[Op_RegL + 1] = {
 2015       0,
 2016       0,
 2017       RAX_num,  // Op_RegN
 2018       RAX_num,  // Op_RegI
 2019       RAX_num,  // Op_RegP
 2020       XMM0_num, // Op_RegF
 2021       XMM0_num, // Op_RegD
 2022       RAX_num   // Op_RegL
 2023     };
 2024     static const int hi[Op_RegL + 1] = {
 2025       0,
 2026       0,
 2027       OptoReg::Bad, // Op_RegN
 2028       OptoReg::Bad, // Op_RegI
 2029       RAX_H_num,    // Op_RegP
 2030       OptoReg::Bad, // Op_RegF
 2031       XMM0b_num,    // Op_RegD
 2032       RAX_H_num     // Op_RegL
 2033     };
 2034     // Excluded flags and vector registers.
 2035     assert(ARRAY_SIZE(hi) == _last_machine_leaf - 8, "missing type");
 2036     return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
 2037   %}
 2038 %}
 2039 
 2040 //----------ATTRIBUTES---------------------------------------------------------
 2041 //----------Operand Attributes-------------------------------------------------
 2042 op_attrib op_cost(0);        // Required cost attribute
 2043 
 2044 //----------Instruction Attributes---------------------------------------------
 2045 ins_attrib ins_cost(100);       // Required cost attribute
 2046 ins_attrib ins_size(8);         // Required size attribute (in bits)
 2047 ins_attrib ins_short_branch(0); // Required flag: is this instruction
 2048                                 // a non-matching short branch variant
 2049                                 // of some long branch?
 2050 ins_attrib ins_alignment(1);    // Required alignment attribute (must
 2051                                 // be a power of 2) specifies the
 2052                                 // alignment that some part of the
 2053                                 // instruction (not necessarily the
 2054                                 // start) requires.  If > 1, a
 2055                                 // compute_padding() function must be
 2056                                 // provided for the instruction
 2057 
 2058 // Whether this node is expanded during code emission into a sequence of
 2059 // instructions and the first instruction can perform an implicit null check.
 2060 ins_attrib ins_is_late_expanded_null_check_candidate(false);
 2061 
 2062 //----------OPERANDS-----------------------------------------------------------
 2063 // Operand definitions must precede instruction definitions for correct parsing
 2064 // in the ADLC because operands constitute user defined types which are used in
 2065 // instruction definitions.
 2066 
 2067 //----------Simple Operands----------------------------------------------------
 2068 // Immediate Operands
 2069 // Integer Immediate
 2070 operand immI()
 2071 %{
 2072   match(ConI);
 2073 
 2074   op_cost(10);
 2075   format %{ %}
 2076   interface(CONST_INTER);
 2077 %}
 2078 
 2079 // Constant for test vs zero
 2080 operand immI_0()
 2081 %{
 2082   predicate(n->get_int() == 0);
 2083   match(ConI);
 2084 
 2085   op_cost(0);
 2086   format %{ %}
 2087   interface(CONST_INTER);
 2088 %}
 2089 
 2090 // Constant for increment
 2091 operand immI_1()
 2092 %{
 2093   predicate(n->get_int() == 1);
 2094   match(ConI);
 2095 
 2096   op_cost(0);
 2097   format %{ %}
 2098   interface(CONST_INTER);
 2099 %}
 2100 
 2101 // Constant for decrement
 2102 operand immI_M1()
 2103 %{
 2104   predicate(n->get_int() == -1);
 2105   match(ConI);
 2106 
 2107   op_cost(0);
 2108   format %{ %}
 2109   interface(CONST_INTER);
 2110 %}
 2111 
 2112 operand immI_2()
 2113 %{
 2114   predicate(n->get_int() == 2);
 2115   match(ConI);
 2116 
 2117   op_cost(0);
 2118   format %{ %}
 2119   interface(CONST_INTER);
 2120 %}
 2121 
 2122 operand immI_4()
 2123 %{
 2124   predicate(n->get_int() == 4);
 2125   match(ConI);
 2126 
 2127   op_cost(0);
 2128   format %{ %}
 2129   interface(CONST_INTER);
 2130 %}
 2131 
 2132 operand immI_8()
 2133 %{
 2134   predicate(n->get_int() == 8);
 2135   match(ConI);
 2136 
 2137   op_cost(0);
 2138   format %{ %}
 2139   interface(CONST_INTER);
 2140 %}
 2141 
 2142 // Valid scale values for addressing modes
 2143 operand immI2()
 2144 %{
 2145   predicate(0 <= n->get_int() && (n->get_int() <= 3));
 2146   match(ConI);
 2147 
 2148   format %{ %}
 2149   interface(CONST_INTER);
 2150 %}
 2151 
 2152 operand immU7()
 2153 %{
 2154   predicate((0 <= n->get_int()) && (n->get_int() <= 0x7F));
 2155   match(ConI);
 2156 
 2157   op_cost(5);
 2158   format %{ %}
 2159   interface(CONST_INTER);
 2160 %}
 2161 
 2162 operand immI8()
 2163 %{
 2164   predicate((-0x80 <= n->get_int()) && (n->get_int() < 0x80));
 2165   match(ConI);
 2166 
 2167   op_cost(5);
 2168   format %{ %}
 2169   interface(CONST_INTER);
 2170 %}
 2171 
 2172 operand immU8()
 2173 %{
 2174   predicate((0 <= n->get_int()) && (n->get_int() <= 255));
 2175   match(ConI);
 2176 
 2177   op_cost(5);
 2178   format %{ %}
 2179   interface(CONST_INTER);
 2180 %}
 2181 
 2182 operand immI16()
 2183 %{
 2184   predicate((-32768 <= n->get_int()) && (n->get_int() <= 32767));
 2185   match(ConI);
 2186 
 2187   op_cost(10);
 2188   format %{ %}
 2189   interface(CONST_INTER);
 2190 %}
 2191 
 2192 // Int Immediate non-negative
 2193 operand immU31()
 2194 %{
 2195   predicate(n->get_int() >= 0);
 2196   match(ConI);
 2197 
 2198   op_cost(0);
 2199   format %{ %}
 2200   interface(CONST_INTER);
 2201 %}
 2202 
 2203 // Pointer Immediate
 2204 operand immP()
 2205 %{
 2206   match(ConP);
 2207 
 2208   op_cost(10);
 2209   format %{ %}
 2210   interface(CONST_INTER);
 2211 %}
 2212 
 2213 // Null Pointer Immediate
 2214 operand immP0()
 2215 %{
 2216   predicate(n->get_ptr() == 0);
 2217   match(ConP);
 2218 
 2219   op_cost(5);
 2220   format %{ %}
 2221   interface(CONST_INTER);
 2222 %}
 2223 
 2224 // Pointer Immediate
 2225 operand immN() %{
 2226   match(ConN);
 2227 
 2228   op_cost(10);
 2229   format %{ %}
 2230   interface(CONST_INTER);
 2231 %}
 2232 
 2233 operand immNKlass() %{
 2234   match(ConNKlass);
 2235 
 2236   op_cost(10);
 2237   format %{ %}
 2238   interface(CONST_INTER);
 2239 %}
 2240 
 2241 // Null Pointer Immediate
 2242 operand immN0() %{
 2243   predicate(n->get_narrowcon() == 0);
 2244   match(ConN);
 2245 
 2246   op_cost(5);
 2247   format %{ %}
 2248   interface(CONST_INTER);
 2249 %}
 2250 
 2251 operand immP31()
 2252 %{
 2253   predicate(n->as_Type()->type()->reloc() == relocInfo::none
 2254             && (n->get_ptr() >> 31) == 0);
 2255   match(ConP);
 2256 
 2257   op_cost(5);
 2258   format %{ %}
 2259   interface(CONST_INTER);
 2260 %}
 2261 
 2262 
 2263 // Long Immediate
 2264 operand immL()
 2265 %{
 2266   match(ConL);
 2267 
 2268   op_cost(20);
 2269   format %{ %}
 2270   interface(CONST_INTER);
 2271 %}
 2272 
 2273 // Long Immediate 8-bit
 2274 operand immL8()
 2275 %{
 2276   predicate(-0x80L <= n->get_long() && n->get_long() < 0x80L);
 2277   match(ConL);
 2278 
 2279   op_cost(5);
 2280   format %{ %}
 2281   interface(CONST_INTER);
 2282 %}
 2283 
 2284 // Long Immediate 32-bit unsigned
 2285 operand immUL32()
 2286 %{
 2287   predicate(n->get_long() == (unsigned int) (n->get_long()));
 2288   match(ConL);
 2289 
 2290   op_cost(10);
 2291   format %{ %}
 2292   interface(CONST_INTER);
 2293 %}
 2294 
 2295 // Long Immediate 32-bit signed
 2296 operand immL32()
 2297 %{
 2298   predicate(n->get_long() == (int) (n->get_long()));
 2299   match(ConL);
 2300 
 2301   op_cost(15);
 2302   format %{ %}
 2303   interface(CONST_INTER);
 2304 %}
 2305 
 2306 operand immL_Pow2()
 2307 %{
 2308   predicate(is_power_of_2((julong)n->get_long()));
 2309   match(ConL);
 2310 
 2311   op_cost(15);
 2312   format %{ %}
 2313   interface(CONST_INTER);
 2314 %}
 2315 
 2316 operand immL_NotPow2()
 2317 %{
 2318   predicate(is_power_of_2((julong)~n->get_long()));
 2319   match(ConL);
 2320 
 2321   op_cost(15);
 2322   format %{ %}
 2323   interface(CONST_INTER);
 2324 %}
 2325 
 2326 // Long Immediate zero
 2327 operand immL0()
 2328 %{
 2329   predicate(n->get_long() == 0L);
 2330   match(ConL);
 2331 
 2332   op_cost(10);
 2333   format %{ %}
 2334   interface(CONST_INTER);
 2335 %}
 2336 
 2337 // Constant for increment
 2338 operand immL1()
 2339 %{
 2340   predicate(n->get_long() == 1);
 2341   match(ConL);
 2342 
 2343   format %{ %}
 2344   interface(CONST_INTER);
 2345 %}
 2346 
 2347 // Constant for decrement
 2348 operand immL_M1()
 2349 %{
 2350   predicate(n->get_long() == -1);
 2351   match(ConL);
 2352 
 2353   format %{ %}
 2354   interface(CONST_INTER);
 2355 %}
 2356 
 2357 // Long Immediate: low 32-bit mask
 2358 operand immL_32bits()
 2359 %{
 2360   predicate(n->get_long() == 0xFFFFFFFFL);
 2361   match(ConL);
 2362   op_cost(20);
 2363 
 2364   format %{ %}
 2365   interface(CONST_INTER);
 2366 %}
 2367 
 2368 // Int Immediate: 2^n-1, positive
 2369 operand immI_Pow2M1()
 2370 %{
 2371   predicate((n->get_int() > 0)
 2372             && is_power_of_2((juint)n->get_int() + 1));
 2373   match(ConI);
 2374 
 2375   op_cost(20);
 2376   format %{ %}
 2377   interface(CONST_INTER);
 2378 %}
 2379 
 2380 // Float Immediate zero
 2381 operand immF0()
 2382 %{
 2383   predicate(jint_cast(n->getf()) == 0);
 2384   match(ConF);
 2385 
 2386   op_cost(5);
 2387   format %{ %}
 2388   interface(CONST_INTER);
 2389 %}
 2390 
 2391 // Float Immediate
 2392 operand immF()
 2393 %{
 2394   match(ConF);
 2395 
 2396   op_cost(15);
 2397   format %{ %}
 2398   interface(CONST_INTER);
 2399 %}
 2400 
 2401 // Half Float Immediate
 2402 operand immH()
 2403 %{
 2404   match(ConH);
 2405 
 2406   op_cost(15);
 2407   format %{ %}
 2408   interface(CONST_INTER);
 2409 %}
 2410 
 2411 // Double Immediate zero
 2412 operand immD0()
 2413 %{
 2414   predicate(jlong_cast(n->getd()) == 0);
 2415   match(ConD);
 2416 
 2417   op_cost(5);
 2418   format %{ %}
 2419   interface(CONST_INTER);
 2420 %}
 2421 
 2422 // Double Immediate
 2423 operand immD()
 2424 %{
 2425   match(ConD);
 2426 
 2427   op_cost(15);
 2428   format %{ %}
 2429   interface(CONST_INTER);
 2430 %}
 2431 
 2432 // Immediates for special shifts (sign extend)
 2433 
 2434 // Constants for increment
 2435 operand immI_16()
 2436 %{
 2437   predicate(n->get_int() == 16);
 2438   match(ConI);
 2439 
 2440   format %{ %}
 2441   interface(CONST_INTER);
 2442 %}
 2443 
 2444 operand immI_24()
 2445 %{
 2446   predicate(n->get_int() == 24);
 2447   match(ConI);
 2448 
 2449   format %{ %}
 2450   interface(CONST_INTER);
 2451 %}
 2452 
 2453 // Constant for byte-wide masking
 2454 operand immI_255()
 2455 %{
 2456   predicate(n->get_int() == 255);
 2457   match(ConI);
 2458 
 2459   format %{ %}
 2460   interface(CONST_INTER);
 2461 %}
 2462 
 2463 // Constant for short-wide masking
 2464 operand immI_65535()
 2465 %{
 2466   predicate(n->get_int() == 65535);
 2467   match(ConI);
 2468 
 2469   format %{ %}
 2470   interface(CONST_INTER);
 2471 %}
 2472 
 2473 // Constant for byte-wide masking
 2474 operand immL_255()
 2475 %{
 2476   predicate(n->get_long() == 255);
 2477   match(ConL);
 2478 
 2479   format %{ %}
 2480   interface(CONST_INTER);
 2481 %}
 2482 
 2483 // Constant for short-wide masking
 2484 operand immL_65535()
 2485 %{
 2486   predicate(n->get_long() == 65535);
 2487   match(ConL);
 2488 
 2489   format %{ %}
 2490   interface(CONST_INTER);
 2491 %}
 2492 
 2493 operand kReg()
 2494 %{
 2495   constraint(ALLOC_IN_RC(vectmask_reg));
 2496   match(RegVectMask);
 2497   format %{%}
 2498   interface(REG_INTER);
 2499 %}
 2500 
 2501 // Register Operands
 2502 // Integer Register
 2503 operand rRegI()
 2504 %{
 2505   constraint(ALLOC_IN_RC(int_reg));
 2506   match(RegI);
 2507 
 2508   match(rax_RegI);
 2509   match(rbx_RegI);
 2510   match(rcx_RegI);
 2511   match(rdx_RegI);
 2512   match(rdi_RegI);
 2513 
 2514   format %{ %}
 2515   interface(REG_INTER);
 2516 %}
 2517 
 2518 // Special Registers
 2519 operand rax_RegI()
 2520 %{
 2521   constraint(ALLOC_IN_RC(int_rax_reg));
 2522   match(RegI);
 2523   match(rRegI);
 2524 
 2525   format %{ "RAX" %}
 2526   interface(REG_INTER);
 2527 %}
 2528 
 2529 // Special Registers
 2530 operand rbx_RegI()
 2531 %{
 2532   constraint(ALLOC_IN_RC(int_rbx_reg));
 2533   match(RegI);
 2534   match(rRegI);
 2535 
 2536   format %{ "RBX" %}
 2537   interface(REG_INTER);
 2538 %}
 2539 
 2540 operand rcx_RegI()
 2541 %{
 2542   constraint(ALLOC_IN_RC(int_rcx_reg));
 2543   match(RegI);
 2544   match(rRegI);
 2545 
 2546   format %{ "RCX" %}
 2547   interface(REG_INTER);
 2548 %}
 2549 
 2550 operand rdx_RegI()
 2551 %{
 2552   constraint(ALLOC_IN_RC(int_rdx_reg));
 2553   match(RegI);
 2554   match(rRegI);
 2555 
 2556   format %{ "RDX" %}
 2557   interface(REG_INTER);
 2558 %}
 2559 
 2560 operand rdi_RegI()
 2561 %{
 2562   constraint(ALLOC_IN_RC(int_rdi_reg));
 2563   match(RegI);
 2564   match(rRegI);
 2565 
 2566   format %{ "RDI" %}
 2567   interface(REG_INTER);
 2568 %}
 2569 
 2570 operand no_rax_rdx_RegI()
 2571 %{
 2572   constraint(ALLOC_IN_RC(int_no_rax_rdx_reg));
 2573   match(RegI);
 2574   match(rbx_RegI);
 2575   match(rcx_RegI);
 2576   match(rdi_RegI);
 2577 
 2578   format %{ %}
 2579   interface(REG_INTER);
 2580 %}
 2581 
 2582 operand no_rbp_r13_RegI()
 2583 %{
 2584   constraint(ALLOC_IN_RC(int_no_rbp_r13_reg));
 2585   match(RegI);
 2586   match(rRegI);
 2587   match(rax_RegI);
 2588   match(rbx_RegI);
 2589   match(rcx_RegI);
 2590   match(rdx_RegI);
 2591   match(rdi_RegI);
 2592 
 2593   format %{ %}
 2594   interface(REG_INTER);
 2595 %}
 2596 
 2597 // Pointer Register
 2598 operand any_RegP()
 2599 %{
 2600   constraint(ALLOC_IN_RC(any_reg));
 2601   match(RegP);
 2602   match(rax_RegP);
 2603   match(rbx_RegP);
 2604   match(rdi_RegP);
 2605   match(rsi_RegP);
 2606   match(rbp_RegP);
 2607   match(r15_RegP);
 2608   match(rRegP);
 2609 
 2610   format %{ %}
 2611   interface(REG_INTER);
 2612 %}
 2613 
 2614 operand rRegP()
 2615 %{
 2616   constraint(ALLOC_IN_RC(ptr_reg));
 2617   match(RegP);
 2618   match(rax_RegP);
 2619   match(rbx_RegP);
 2620   match(rdi_RegP);
 2621   match(rsi_RegP);
 2622   match(rbp_RegP);  // See Q&A below about
 2623   match(r15_RegP);  // r15_RegP and rbp_RegP.
 2624 
 2625   format %{ %}
 2626   interface(REG_INTER);
 2627 %}
 2628 
 2629 operand rRegN() %{
 2630   constraint(ALLOC_IN_RC(int_reg));
 2631   match(RegN);
 2632 
 2633   format %{ %}
 2634   interface(REG_INTER);
 2635 %}
 2636 
 2637 // Question: Why is r15_RegP (the read-only TLS register) a match for rRegP?
 2638 // Answer: Operand match rules govern the DFA as it processes instruction inputs.
 2639 // It's fine for an instruction input that expects rRegP to match a r15_RegP.
 2640 // The output of an instruction is controlled by the allocator, which respects
 2641 // register class masks, not match rules.  Unless an instruction mentions
 2642 // r15_RegP or any_RegP explicitly as its output, r15 will not be considered
 2643 // by the allocator as an input.
 2644 // The same logic applies to rbp_RegP being a match for rRegP: If PreserveFramePointer==true,
 2645 // the RBP is used as a proper frame pointer and is not included in ptr_reg. As a
 2646 // result, RBP is not included in the output of the instruction either.
 2647 
 2648 // This operand is not allowed to use RBP even if
 2649 // RBP is not used to hold the frame pointer.
 2650 operand no_rbp_RegP()
 2651 %{
 2652   constraint(ALLOC_IN_RC(ptr_reg_no_rbp));
 2653   match(RegP);
 2654   match(rbx_RegP);
 2655   match(rsi_RegP);
 2656   match(rdi_RegP);
 2657 
 2658   format %{ %}
 2659   interface(REG_INTER);
 2660 %}
 2661 
 2662 // Special Registers
 2663 // Return a pointer value
 2664 operand rax_RegP()
 2665 %{
 2666   constraint(ALLOC_IN_RC(ptr_rax_reg));
 2667   match(RegP);
 2668   match(rRegP);
 2669 
 2670   format %{ %}
 2671   interface(REG_INTER);
 2672 %}
 2673 
 2674 // Special Registers
 2675 // Return a compressed pointer value
 2676 operand rax_RegN()
 2677 %{
 2678   constraint(ALLOC_IN_RC(int_rax_reg));
 2679   match(RegN);
 2680   match(rRegN);
 2681 
 2682   format %{ %}
 2683   interface(REG_INTER);
 2684 %}
 2685 
 2686 // Used in AtomicAdd
 2687 operand rbx_RegP()
 2688 %{
 2689   constraint(ALLOC_IN_RC(ptr_rbx_reg));
 2690   match(RegP);
 2691   match(rRegP);
 2692 
 2693   format %{ %}
 2694   interface(REG_INTER);
 2695 %}
 2696 
 2697 operand rsi_RegP()
 2698 %{
 2699   constraint(ALLOC_IN_RC(ptr_rsi_reg));
 2700   match(RegP);
 2701   match(rRegP);
 2702 
 2703   format %{ %}
 2704   interface(REG_INTER);
 2705 %}
 2706 
 2707 operand rbp_RegP()
 2708 %{
 2709   constraint(ALLOC_IN_RC(ptr_rbp_reg));
 2710   match(RegP);
 2711   match(rRegP);
 2712 
 2713   format %{ %}
 2714   interface(REG_INTER);
 2715 %}
 2716 
 2717 // Used in rep stosq
 2718 operand rdi_RegP()
 2719 %{
 2720   constraint(ALLOC_IN_RC(ptr_rdi_reg));
 2721   match(RegP);
 2722   match(rRegP);
 2723 
 2724   format %{ %}
 2725   interface(REG_INTER);
 2726 %}
 2727 
 2728 operand r15_RegP()
 2729 %{
 2730   constraint(ALLOC_IN_RC(ptr_r15_reg));
 2731   match(RegP);
 2732   match(rRegP);
 2733 
 2734   format %{ %}
 2735   interface(REG_INTER);
 2736 %}
 2737 
 2738 operand rRegL()
 2739 %{
 2740   constraint(ALLOC_IN_RC(long_reg));
 2741   match(RegL);
 2742   match(rax_RegL);
 2743   match(rdx_RegL);
 2744 
 2745   format %{ %}
 2746   interface(REG_INTER);
 2747 %}
 2748 
 2749 // Special Registers
 2750 operand no_rax_rdx_RegL()
 2751 %{
 2752   constraint(ALLOC_IN_RC(long_no_rax_rdx_reg));
 2753   match(RegL);
 2754   match(rRegL);
 2755 
 2756   format %{ %}
 2757   interface(REG_INTER);
 2758 %}
 2759 
 2760 operand rax_RegL()
 2761 %{
 2762   constraint(ALLOC_IN_RC(long_rax_reg));
 2763   match(RegL);
 2764   match(rRegL);
 2765 
 2766   format %{ "RAX" %}
 2767   interface(REG_INTER);
 2768 %}
 2769 
 2770 operand rcx_RegL()
 2771 %{
 2772   constraint(ALLOC_IN_RC(long_rcx_reg));
 2773   match(RegL);
 2774   match(rRegL);
 2775 
 2776   format %{ %}
 2777   interface(REG_INTER);
 2778 %}
 2779 
 2780 operand rdx_RegL()
 2781 %{
 2782   constraint(ALLOC_IN_RC(long_rdx_reg));
 2783   match(RegL);
 2784   match(rRegL);
 2785 
 2786   format %{ %}
 2787   interface(REG_INTER);
 2788 %}
 2789 
 2790 operand r11_RegL()
 2791 %{
 2792   constraint(ALLOC_IN_RC(long_r11_reg));
 2793   match(RegL);
 2794   match(rRegL);
 2795 
 2796   format %{ %}
 2797   interface(REG_INTER);
 2798 %}
 2799 
 2800 operand no_rbp_r13_RegL()
 2801 %{
 2802   constraint(ALLOC_IN_RC(long_no_rbp_r13_reg));
 2803   match(RegL);
 2804   match(rRegL);
 2805   match(rax_RegL);
 2806   match(rcx_RegL);
 2807   match(rdx_RegL);
 2808 
 2809   format %{ %}
 2810   interface(REG_INTER);
 2811 %}
 2812 
 2813 // Flags register, used as output of compare instructions
 2814 operand rFlagsReg()
 2815 %{
 2816   constraint(ALLOC_IN_RC(int_flags));
 2817   match(RegFlags);
 2818 
 2819   format %{ "RFLAGS" %}
 2820   interface(REG_INTER);
 2821 %}
 2822 
 2823 // Flags register, used as output of FLOATING POINT compare instructions
 2824 operand rFlagsRegU()
 2825 %{
 2826   constraint(ALLOC_IN_RC(int_flags));
 2827   match(RegFlags);
 2828 
 2829   format %{ "RFLAGS_U" %}
 2830   interface(REG_INTER);
 2831 %}
 2832 
 2833 operand rFlagsRegUCF() %{
 2834   constraint(ALLOC_IN_RC(int_flags));
 2835   match(RegFlags);
 2836   predicate(false);
 2837 
 2838   format %{ "RFLAGS_U_CF" %}
 2839   interface(REG_INTER);
 2840 %}
 2841 
 2842 // Float register operands
 2843 operand regF() %{
 2844    constraint(ALLOC_IN_RC(float_reg));
 2845    match(RegF);
 2846 
 2847    format %{ %}
 2848    interface(REG_INTER);
 2849 %}
 2850 
 2851 // Float register operands
 2852 operand legRegF() %{
 2853    constraint(ALLOC_IN_RC(float_reg_legacy));
 2854    match(RegF);
 2855 
 2856    format %{ %}
 2857    interface(REG_INTER);
 2858 %}
 2859 
 2860 // Float register operands
 2861 operand vlRegF() %{
 2862    constraint(ALLOC_IN_RC(float_reg_vl));
 2863    match(RegF);
 2864 
 2865    format %{ %}
 2866    interface(REG_INTER);
 2867 %}
 2868 
 2869 // Double register operands
 2870 operand regD() %{
 2871    constraint(ALLOC_IN_RC(double_reg));
 2872    match(RegD);
 2873 
 2874    format %{ %}
 2875    interface(REG_INTER);
 2876 %}
 2877 
 2878 // Double register operands
 2879 operand legRegD() %{
 2880    constraint(ALLOC_IN_RC(double_reg_legacy));
 2881    match(RegD);
 2882 
 2883    format %{ %}
 2884    interface(REG_INTER);
 2885 %}
 2886 
 2887 // Double register operands
 2888 operand vlRegD() %{
 2889    constraint(ALLOC_IN_RC(double_reg_vl));
 2890    match(RegD);
 2891 
 2892    format %{ %}
 2893    interface(REG_INTER);
 2894 %}
 2895 
 2896 //----------Memory Operands----------------------------------------------------
 2897 // Direct Memory Operand
 2898 // operand direct(immP addr)
 2899 // %{
 2900 //   match(addr);
 2901 
 2902 //   format %{ "[$addr]" %}
 2903 //   interface(MEMORY_INTER) %{
 2904 //     base(0xFFFFFFFF);
 2905 //     index(0x4);
 2906 //     scale(0x0);
 2907 //     disp($addr);
 2908 //   %}
 2909 // %}
 2910 
 2911 // Indirect Memory Operand
 2912 operand indirect(any_RegP reg)
 2913 %{
 2914   constraint(ALLOC_IN_RC(ptr_reg));
 2915   match(reg);
 2916 
 2917   format %{ "[$reg]" %}
 2918   interface(MEMORY_INTER) %{
 2919     base($reg);
 2920     index(0x4);
 2921     scale(0x0);
 2922     disp(0x0);
 2923   %}
 2924 %}
 2925 
 2926 // Indirect Memory Plus Short Offset Operand
 2927 operand indOffset8(any_RegP reg, immL8 off)
 2928 %{
 2929   constraint(ALLOC_IN_RC(ptr_reg));
 2930   match(AddP reg off);
 2931 
 2932   format %{ "[$reg + $off (8-bit)]" %}
 2933   interface(MEMORY_INTER) %{
 2934     base($reg);
 2935     index(0x4);
 2936     scale(0x0);
 2937     disp($off);
 2938   %}
 2939 %}
 2940 
 2941 // Indirect Memory Plus Long Offset Operand
 2942 operand indOffset32(any_RegP reg, immL32 off)
 2943 %{
 2944   constraint(ALLOC_IN_RC(ptr_reg));
 2945   match(AddP reg off);
 2946 
 2947   format %{ "[$reg + $off (32-bit)]" %}
 2948   interface(MEMORY_INTER) %{
 2949     base($reg);
 2950     index(0x4);
 2951     scale(0x0);
 2952     disp($off);
 2953   %}
 2954 %}
 2955 
 2956 // Indirect Memory Plus Index Register Plus Offset Operand
 2957 operand indIndexOffset(any_RegP reg, rRegL lreg, immL32 off)
 2958 %{
 2959   constraint(ALLOC_IN_RC(ptr_reg));
 2960   match(AddP (AddP reg lreg) off);
 2961 
 2962   op_cost(10);
 2963   format %{"[$reg + $off + $lreg]" %}
 2964   interface(MEMORY_INTER) %{
 2965     base($reg);
 2966     index($lreg);
 2967     scale(0x0);
 2968     disp($off);
 2969   %}
 2970 %}
 2971 
 2972 // Indirect Memory Plus Index Register Plus Offset Operand
 2973 operand indIndex(any_RegP reg, rRegL lreg)
 2974 %{
 2975   constraint(ALLOC_IN_RC(ptr_reg));
 2976   match(AddP reg lreg);
 2977 
 2978   op_cost(10);
 2979   format %{"[$reg + $lreg]" %}
 2980   interface(MEMORY_INTER) %{
 2981     base($reg);
 2982     index($lreg);
 2983     scale(0x0);
 2984     disp(0x0);
 2985   %}
 2986 %}
 2987 
 2988 // Indirect Memory Times Scale Plus Index Register
 2989 operand indIndexScale(any_RegP reg, rRegL lreg, immI2 scale)
 2990 %{
 2991   constraint(ALLOC_IN_RC(ptr_reg));
 2992   match(AddP reg (LShiftL lreg scale));
 2993 
 2994   op_cost(10);
 2995   format %{"[$reg + $lreg << $scale]" %}
 2996   interface(MEMORY_INTER) %{
 2997     base($reg);
 2998     index($lreg);
 2999     scale($scale);
 3000     disp(0x0);
 3001   %}
 3002 %}
 3003 
 3004 operand indPosIndexScale(any_RegP reg, rRegI idx, immI2 scale)
 3005 %{
 3006   constraint(ALLOC_IN_RC(ptr_reg));
 3007   predicate(n->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
 3008   match(AddP reg (LShiftL (ConvI2L idx) scale));
 3009 
 3010   op_cost(10);
 3011   format %{"[$reg + pos $idx << $scale]" %}
 3012   interface(MEMORY_INTER) %{
 3013     base($reg);
 3014     index($idx);
 3015     scale($scale);
 3016     disp(0x0);
 3017   %}
 3018 %}
 3019 
 3020 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
 3021 operand indIndexScaleOffset(any_RegP reg, immL32 off, rRegL lreg, immI2 scale)
 3022 %{
 3023   constraint(ALLOC_IN_RC(ptr_reg));
 3024   match(AddP (AddP reg (LShiftL lreg scale)) off);
 3025 
 3026   op_cost(10);
 3027   format %{"[$reg + $off + $lreg << $scale]" %}
 3028   interface(MEMORY_INTER) %{
 3029     base($reg);
 3030     index($lreg);
 3031     scale($scale);
 3032     disp($off);
 3033   %}
 3034 %}
 3035 
 3036 // Indirect Memory Plus Positive Index Register Plus Offset Operand
 3037 operand indPosIndexOffset(any_RegP reg, immL32 off, rRegI idx)
 3038 %{
 3039   constraint(ALLOC_IN_RC(ptr_reg));
 3040   predicate(n->in(2)->in(3)->as_Type()->type()->is_long()->_lo >= 0);
 3041   match(AddP (AddP reg (ConvI2L idx)) off);
 3042 
 3043   op_cost(10);
 3044   format %{"[$reg + $off + $idx]" %}
 3045   interface(MEMORY_INTER) %{
 3046     base($reg);
 3047     index($idx);
 3048     scale(0x0);
 3049     disp($off);
 3050   %}
 3051 %}
 3052 
 3053 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
 3054 operand indPosIndexScaleOffset(any_RegP reg, immL32 off, rRegI idx, immI2 scale)
 3055 %{
 3056   constraint(ALLOC_IN_RC(ptr_reg));
 3057   predicate(n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
 3058   match(AddP (AddP reg (LShiftL (ConvI2L idx) scale)) off);
 3059 
 3060   op_cost(10);
 3061   format %{"[$reg + $off + $idx << $scale]" %}
 3062   interface(MEMORY_INTER) %{
 3063     base($reg);
 3064     index($idx);
 3065     scale($scale);
 3066     disp($off);
 3067   %}
 3068 %}
 3069 
 3070 // Indirect Narrow Oop Plus Offset Operand
 3071 // Note: x86 architecture doesn't support "scale * index + offset" without a base
 3072 // we can't free r12 even with CompressedOops::base() == nullptr.
 3073 operand indCompressedOopOffset(rRegN reg, immL32 off) %{
 3074   predicate(UseCompressedOops && (CompressedOops::shift() == Address::times_8));
 3075   constraint(ALLOC_IN_RC(ptr_reg));
 3076   match(AddP (DecodeN reg) off);
 3077 
 3078   op_cost(10);
 3079   format %{"[R12 + $reg << 3 + $off] (compressed oop addressing)" %}
 3080   interface(MEMORY_INTER) %{
 3081     base(0xc); // R12
 3082     index($reg);
 3083     scale(0x3);
 3084     disp($off);
 3085   %}
 3086 %}
 3087 
 3088 // Indirect Memory Operand
 3089 operand indirectNarrow(rRegN reg)
 3090 %{
 3091   predicate(CompressedOops::shift() == 0);
 3092   constraint(ALLOC_IN_RC(ptr_reg));
 3093   match(DecodeN reg);
 3094 
 3095   format %{ "[$reg]" %}
 3096   interface(MEMORY_INTER) %{
 3097     base($reg);
 3098     index(0x4);
 3099     scale(0x0);
 3100     disp(0x0);
 3101   %}
 3102 %}
 3103 
 3104 // Indirect Memory Plus Short Offset Operand
 3105 operand indOffset8Narrow(rRegN reg, immL8 off)
 3106 %{
 3107   predicate(CompressedOops::shift() == 0);
 3108   constraint(ALLOC_IN_RC(ptr_reg));
 3109   match(AddP (DecodeN reg) off);
 3110 
 3111   format %{ "[$reg + $off (8-bit)]" %}
 3112   interface(MEMORY_INTER) %{
 3113     base($reg);
 3114     index(0x4);
 3115     scale(0x0);
 3116     disp($off);
 3117   %}
 3118 %}
 3119 
 3120 // Indirect Memory Plus Long Offset Operand
 3121 operand indOffset32Narrow(rRegN reg, immL32 off)
 3122 %{
 3123   predicate(CompressedOops::shift() == 0);
 3124   constraint(ALLOC_IN_RC(ptr_reg));
 3125   match(AddP (DecodeN reg) off);
 3126 
 3127   format %{ "[$reg + $off (32-bit)]" %}
 3128   interface(MEMORY_INTER) %{
 3129     base($reg);
 3130     index(0x4);
 3131     scale(0x0);
 3132     disp($off);
 3133   %}
 3134 %}
 3135 
 3136 // Indirect Memory Plus Index Register Plus Offset Operand
 3137 operand indIndexOffsetNarrow(rRegN reg, rRegL lreg, immL32 off)
 3138 %{
 3139   predicate(CompressedOops::shift() == 0);
 3140   constraint(ALLOC_IN_RC(ptr_reg));
 3141   match(AddP (AddP (DecodeN reg) lreg) off);
 3142 
 3143   op_cost(10);
 3144   format %{"[$reg + $off + $lreg]" %}
 3145   interface(MEMORY_INTER) %{
 3146     base($reg);
 3147     index($lreg);
 3148     scale(0x0);
 3149     disp($off);
 3150   %}
 3151 %}
 3152 
 3153 // Indirect Memory Plus Index Register Plus Offset Operand
 3154 operand indIndexNarrow(rRegN reg, rRegL lreg)
 3155 %{
 3156   predicate(CompressedOops::shift() == 0);
 3157   constraint(ALLOC_IN_RC(ptr_reg));
 3158   match(AddP (DecodeN reg) lreg);
 3159 
 3160   op_cost(10);
 3161   format %{"[$reg + $lreg]" %}
 3162   interface(MEMORY_INTER) %{
 3163     base($reg);
 3164     index($lreg);
 3165     scale(0x0);
 3166     disp(0x0);
 3167   %}
 3168 %}
 3169 
 3170 // Indirect Memory Times Scale Plus Index Register
 3171 operand indIndexScaleNarrow(rRegN reg, rRegL lreg, immI2 scale)
 3172 %{
 3173   predicate(CompressedOops::shift() == 0);
 3174   constraint(ALLOC_IN_RC(ptr_reg));
 3175   match(AddP (DecodeN reg) (LShiftL lreg scale));
 3176 
 3177   op_cost(10);
 3178   format %{"[$reg + $lreg << $scale]" %}
 3179   interface(MEMORY_INTER) %{
 3180     base($reg);
 3181     index($lreg);
 3182     scale($scale);
 3183     disp(0x0);
 3184   %}
 3185 %}
 3186 
 3187 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
 3188 operand indIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegL lreg, immI2 scale)
 3189 %{
 3190   predicate(CompressedOops::shift() == 0);
 3191   constraint(ALLOC_IN_RC(ptr_reg));
 3192   match(AddP (AddP (DecodeN reg) (LShiftL lreg scale)) off);
 3193 
 3194   op_cost(10);
 3195   format %{"[$reg + $off + $lreg << $scale]" %}
 3196   interface(MEMORY_INTER) %{
 3197     base($reg);
 3198     index($lreg);
 3199     scale($scale);
 3200     disp($off);
 3201   %}
 3202 %}
 3203 
 3204 // Indirect Memory Times Plus Positive Index Register Plus Offset Operand
 3205 operand indPosIndexOffsetNarrow(rRegN reg, immL32 off, rRegI idx)
 3206 %{
 3207   constraint(ALLOC_IN_RC(ptr_reg));
 3208   predicate(CompressedOops::shift() == 0 && n->in(2)->in(3)->as_Type()->type()->is_long()->_lo >= 0);
 3209   match(AddP (AddP (DecodeN reg) (ConvI2L idx)) off);
 3210 
 3211   op_cost(10);
 3212   format %{"[$reg + $off + $idx]" %}
 3213   interface(MEMORY_INTER) %{
 3214     base($reg);
 3215     index($idx);
 3216     scale(0x0);
 3217     disp($off);
 3218   %}
 3219 %}
 3220 
 3221 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
 3222 operand indPosIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegI idx, immI2 scale)
 3223 %{
 3224   constraint(ALLOC_IN_RC(ptr_reg));
 3225   predicate(CompressedOops::shift() == 0 && n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
 3226   match(AddP (AddP (DecodeN reg) (LShiftL (ConvI2L idx) scale)) off);
 3227 
 3228   op_cost(10);
 3229   format %{"[$reg + $off + $idx << $scale]" %}
 3230   interface(MEMORY_INTER) %{
 3231     base($reg);
 3232     index($idx);
 3233     scale($scale);
 3234     disp($off);
 3235   %}
 3236 %}
 3237 
 3238 //----------Special Memory Operands--------------------------------------------
 3239 // Stack Slot Operand - This operand is used for loading and storing temporary
 3240 //                      values on the stack where a match requires a value to
 3241 //                      flow through memory.
 3242 operand stackSlotP(sRegP reg)
 3243 %{
 3244   constraint(ALLOC_IN_RC(stack_slots));
 3245   // No match rule because this operand is only generated in matching
 3246 
 3247   format %{ "[$reg]" %}
 3248   interface(MEMORY_INTER) %{
 3249     base(0x4);   // RSP
 3250     index(0x4);  // No Index
 3251     scale(0x0);  // No Scale
 3252     disp($reg);  // Stack Offset
 3253   %}
 3254 %}
 3255 
 3256 operand stackSlotI(sRegI reg)
 3257 %{
 3258   constraint(ALLOC_IN_RC(stack_slots));
 3259   // No match rule because this operand is only generated in matching
 3260 
 3261   format %{ "[$reg]" %}
 3262   interface(MEMORY_INTER) %{
 3263     base(0x4);   // RSP
 3264     index(0x4);  // No Index
 3265     scale(0x0);  // No Scale
 3266     disp($reg);  // Stack Offset
 3267   %}
 3268 %}
 3269 
 3270 operand stackSlotF(sRegF reg)
 3271 %{
 3272   constraint(ALLOC_IN_RC(stack_slots));
 3273   // No match rule because this operand is only generated in matching
 3274 
 3275   format %{ "[$reg]" %}
 3276   interface(MEMORY_INTER) %{
 3277     base(0x4);   // RSP
 3278     index(0x4);  // No Index
 3279     scale(0x0);  // No Scale
 3280     disp($reg);  // Stack Offset
 3281   %}
 3282 %}
 3283 
 3284 operand stackSlotD(sRegD reg)
 3285 %{
 3286   constraint(ALLOC_IN_RC(stack_slots));
 3287   // No match rule because this operand is only generated in matching
 3288 
 3289   format %{ "[$reg]" %}
 3290   interface(MEMORY_INTER) %{
 3291     base(0x4);   // RSP
 3292     index(0x4);  // No Index
 3293     scale(0x0);  // No Scale
 3294     disp($reg);  // Stack Offset
 3295   %}
 3296 %}
 3297 operand stackSlotL(sRegL reg)
 3298 %{
 3299   constraint(ALLOC_IN_RC(stack_slots));
 3300   // No match rule because this operand is only generated in matching
 3301 
 3302   format %{ "[$reg]" %}
 3303   interface(MEMORY_INTER) %{
 3304     base(0x4);   // RSP
 3305     index(0x4);  // No Index
 3306     scale(0x0);  // No Scale
 3307     disp($reg);  // Stack Offset
 3308   %}
 3309 %}
 3310 
 3311 //----------Conditional Branch Operands----------------------------------------
 3312 // Comparison Op  - This is the operation of the comparison, and is limited to
 3313 //                  the following set of codes:
 3314 //                  L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
 3315 //
 3316 // Other attributes of the comparison, such as unsignedness, are specified
 3317 // by the comparison instruction that sets a condition code flags register.
 3318 // That result is represented by a flags operand whose subtype is appropriate
 3319 // to the unsignedness (etc.) of the comparison.
 3320 //
 3321 // Later, the instruction which matches both the Comparison Op (a Bool) and
 3322 // the flags (produced by the Cmp) specifies the coding of the comparison op
 3323 // by matching a specific subtype of Bool operand below, such as cmpOpU.
 3324 
 3325 // Comparison Code
 3326 operand cmpOp()
 3327 %{
 3328   match(Bool);
 3329 
 3330   format %{ "" %}
 3331   interface(COND_INTER) %{
 3332     equal(0x4, "e");
 3333     not_equal(0x5, "ne");
 3334     less(0xC, "l");
 3335     greater_equal(0xD, "ge");
 3336     less_equal(0xE, "le");
 3337     greater(0xF, "g");
 3338     overflow(0x0, "o");
 3339     no_overflow(0x1, "no");
 3340   %}
 3341 %}
 3342 
 3343 // Comparison Code, unsigned compare.  Used by FP also, with
 3344 // C2 (unordered) turned into GT or LT already.  The other bits
 3345 // C0 and C3 are turned into Carry & Zero flags.
 3346 operand cmpOpU()
 3347 %{
 3348   match(Bool);
 3349 
 3350   format %{ "" %}
 3351   interface(COND_INTER) %{
 3352     equal(0x4, "e");
 3353     not_equal(0x5, "ne");
 3354     less(0x2, "b");
 3355     greater_equal(0x3, "ae");
 3356     less_equal(0x6, "be");
 3357     greater(0x7, "a");
 3358     overflow(0x0, "o");
 3359     no_overflow(0x1, "no");
 3360   %}
 3361 %}
 3362 
 3363 
 3364 // Floating comparisons that don't require any fixup for the unordered case,
 3365 // If both inputs of the comparison are the same, ZF is always set so we
 3366 // don't need to use cmpOpUCF2 for eq/ne
 3367 operand cmpOpUCF() %{
 3368   match(Bool);
 3369   predicate(n->as_Bool()->_test._test == BoolTest::lt ||
 3370             n->as_Bool()->_test._test == BoolTest::ge ||
 3371             n->as_Bool()->_test._test == BoolTest::le ||
 3372             n->as_Bool()->_test._test == BoolTest::gt ||
 3373             n->in(1)->in(1) == n->in(1)->in(2));
 3374   format %{ "" %}
 3375   interface(COND_INTER) %{
 3376     equal(0xb, "np");
 3377     not_equal(0xa, "p");
 3378     less(0x2, "b");
 3379     greater_equal(0x3, "ae");
 3380     less_equal(0x6, "be");
 3381     greater(0x7, "a");
 3382     overflow(0x0, "o");
 3383     no_overflow(0x1, "no");
 3384   %}
 3385 %}
 3386 
 3387 
 3388 // Floating comparisons that can be fixed up with extra conditional jumps
 3389 operand cmpOpUCF2() %{
 3390   match(Bool);
 3391   predicate((n->as_Bool()->_test._test == BoolTest::ne ||
 3392              n->as_Bool()->_test._test == BoolTest::eq) &&
 3393             n->in(1)->in(1) != n->in(1)->in(2));
 3394   format %{ "" %}
 3395   interface(COND_INTER) %{
 3396     equal(0x4, "e");
 3397     not_equal(0x5, "ne");
 3398     less(0x2, "b");
 3399     greater_equal(0x3, "ae");
 3400     less_equal(0x6, "be");
 3401     greater(0x7, "a");
 3402     overflow(0x0, "o");
 3403     no_overflow(0x1, "no");
 3404   %}
 3405 %}
 3406 
 3407 //----------OPERAND CLASSES----------------------------------------------------
 3408 // Operand Classes are groups of operands that are used as to simplify
 3409 // instruction definitions by not requiring the AD writer to specify separate
 3410 // instructions for every form of operand when the instruction accepts
 3411 // multiple operand types with the same basic encoding and format.  The classic
 3412 // case of this is memory operands.
 3413 
 3414 opclass memory(indirect, indOffset8, indOffset32, indIndexOffset, indIndex,
 3415                indIndexScale, indPosIndexScale, indIndexScaleOffset, indPosIndexOffset, indPosIndexScaleOffset,
 3416                indCompressedOopOffset,
 3417                indirectNarrow, indOffset8Narrow, indOffset32Narrow,
 3418                indIndexOffsetNarrow, indIndexNarrow, indIndexScaleNarrow,
 3419                indIndexScaleOffsetNarrow, indPosIndexOffsetNarrow, indPosIndexScaleOffsetNarrow);
 3420 
 3421 //----------PIPELINE-----------------------------------------------------------
 3422 // Rules which define the behavior of the target architectures pipeline.
 3423 pipeline %{
 3424 
 3425 //----------ATTRIBUTES---------------------------------------------------------
 3426 attributes %{
 3427   variable_size_instructions;        // Fixed size instructions
 3428   max_instructions_per_bundle = 3;   // Up to 3 instructions per bundle
 3429   instruction_unit_size = 1;         // An instruction is 1 bytes long
 3430   instruction_fetch_unit_size = 16;  // The processor fetches one line
 3431   instruction_fetch_units = 1;       // of 16 bytes
 3432 
 3433   // List of nop instructions
 3434   nops( MachNop );
 3435 %}
 3436 
 3437 //----------RESOURCES----------------------------------------------------------
 3438 // Resources are the functional units available to the machine
 3439 
 3440 // Generic P2/P3 pipeline
 3441 // 3 decoders, only D0 handles big operands; a "bundle" is the limit of
 3442 // 3 instructions decoded per cycle.
 3443 // 2 load/store ops per cycle, 1 branch, 1 FPU,
 3444 // 3 ALU op, only ALU0 handles mul instructions.
 3445 resources( D0, D1, D2, DECODE = D0 | D1 | D2,
 3446            MS0, MS1, MS2, MEM = MS0 | MS1 | MS2,
 3447            BR, FPU,
 3448            ALU0, ALU1, ALU2, ALU = ALU0 | ALU1 | ALU2);
 3449 
 3450 //----------PIPELINE DESCRIPTION-----------------------------------------------
 3451 // Pipeline Description specifies the stages in the machine's pipeline
 3452 
 3453 // Generic P2/P3 pipeline
 3454 pipe_desc(S0, S1, S2, S3, S4, S5);
 3455 
 3456 //----------PIPELINE CLASSES---------------------------------------------------
 3457 // Pipeline Classes describe the stages in which input and output are
 3458 // referenced by the hardware pipeline.
 3459 
 3460 // Naming convention: ialu or fpu
 3461 // Then: _reg
 3462 // Then: _reg if there is a 2nd register
 3463 // Then: _long if it's a pair of instructions implementing a long
 3464 // Then: _fat if it requires the big decoder
 3465 //   Or: _mem if it requires the big decoder and a memory unit.
 3466 
 3467 // Integer ALU reg operation
 3468 pipe_class ialu_reg(rRegI dst)
 3469 %{
 3470     single_instruction;
 3471     dst    : S4(write);
 3472     dst    : S3(read);
 3473     DECODE : S0;        // any decoder
 3474     ALU    : S3;        // any alu
 3475 %}
 3476 
 3477 // Long ALU reg operation
 3478 pipe_class ialu_reg_long(rRegL dst)
 3479 %{
 3480     instruction_count(2);
 3481     dst    : S4(write);
 3482     dst    : S3(read);
 3483     DECODE : S0(2);     // any 2 decoders
 3484     ALU    : S3(2);     // both alus
 3485 %}
 3486 
 3487 // Integer ALU reg operation using big decoder
 3488 pipe_class ialu_reg_fat(rRegI dst)
 3489 %{
 3490     single_instruction;
 3491     dst    : S4(write);
 3492     dst    : S3(read);
 3493     D0     : S0;        // big decoder only
 3494     ALU    : S3;        // any alu
 3495 %}
 3496 
 3497 // Integer ALU reg-reg operation
 3498 pipe_class ialu_reg_reg(rRegI dst, rRegI src)
 3499 %{
 3500     single_instruction;
 3501     dst    : S4(write);
 3502     src    : S3(read);
 3503     DECODE : S0;        // any decoder
 3504     ALU    : S3;        // any alu
 3505 %}
 3506 
 3507 // Integer ALU reg-reg operation
 3508 pipe_class ialu_reg_reg_fat(rRegI dst, memory src)
 3509 %{
 3510     single_instruction;
 3511     dst    : S4(write);
 3512     src    : S3(read);
 3513     D0     : S0;        // big decoder only
 3514     ALU    : S3;        // any alu
 3515 %}
 3516 
 3517 // Integer ALU reg-mem operation
 3518 pipe_class ialu_reg_mem(rRegI dst, memory mem)
 3519 %{
 3520     single_instruction;
 3521     dst    : S5(write);
 3522     mem    : S3(read);
 3523     D0     : S0;        // big decoder only
 3524     ALU    : S4;        // any alu
 3525     MEM    : S3;        // any mem
 3526 %}
 3527 
 3528 // Integer mem operation (prefetch)
 3529 pipe_class ialu_mem(memory mem)
 3530 %{
 3531     single_instruction;
 3532     mem    : S3(read);
 3533     D0     : S0;        // big decoder only
 3534     MEM    : S3;        // any mem
 3535 %}
 3536 
 3537 // Integer Store to Memory
 3538 pipe_class ialu_mem_reg(memory mem, rRegI src)
 3539 %{
 3540     single_instruction;
 3541     mem    : S3(read);
 3542     src    : S5(read);
 3543     D0     : S0;        // big decoder only
 3544     ALU    : S4;        // any alu
 3545     MEM    : S3;
 3546 %}
 3547 
 3548 // // Long Store to Memory
 3549 // pipe_class ialu_mem_long_reg(memory mem, rRegL src)
 3550 // %{
 3551 //     instruction_count(2);
 3552 //     mem    : S3(read);
 3553 //     src    : S5(read);
 3554 //     D0     : S0(2);          // big decoder only; twice
 3555 //     ALU    : S4(2);     // any 2 alus
 3556 //     MEM    : S3(2);  // Both mems
 3557 // %}
 3558 
 3559 // Integer Store to Memory
 3560 pipe_class ialu_mem_imm(memory mem)
 3561 %{
 3562     single_instruction;
 3563     mem    : S3(read);
 3564     D0     : S0;        // big decoder only
 3565     ALU    : S4;        // any alu
 3566     MEM    : S3;
 3567 %}
 3568 
 3569 // Integer ALU0 reg-reg operation
 3570 pipe_class ialu_reg_reg_alu0(rRegI dst, rRegI src)
 3571 %{
 3572     single_instruction;
 3573     dst    : S4(write);
 3574     src    : S3(read);
 3575     D0     : S0;        // Big decoder only
 3576     ALU0   : S3;        // only alu0
 3577 %}
 3578 
 3579 // Integer ALU0 reg-mem operation
 3580 pipe_class ialu_reg_mem_alu0(rRegI dst, memory mem)
 3581 %{
 3582     single_instruction;
 3583     dst    : S5(write);
 3584     mem    : S3(read);
 3585     D0     : S0;        // big decoder only
 3586     ALU0   : S4;        // ALU0 only
 3587     MEM    : S3;        // any mem
 3588 %}
 3589 
 3590 // Integer ALU reg-reg operation
 3591 pipe_class ialu_cr_reg_reg(rFlagsReg cr, rRegI src1, rRegI src2)
 3592 %{
 3593     single_instruction;
 3594     cr     : S4(write);
 3595     src1   : S3(read);
 3596     src2   : S3(read);
 3597     DECODE : S0;        // any decoder
 3598     ALU    : S3;        // any alu
 3599 %}
 3600 
 3601 // Integer ALU reg-imm operation
 3602 pipe_class ialu_cr_reg_imm(rFlagsReg cr, rRegI src1)
 3603 %{
 3604     single_instruction;
 3605     cr     : S4(write);
 3606     src1   : S3(read);
 3607     DECODE : S0;        // any decoder
 3608     ALU    : S3;        // any alu
 3609 %}
 3610 
 3611 // Integer ALU reg-mem operation
 3612 pipe_class ialu_cr_reg_mem(rFlagsReg cr, rRegI src1, memory src2)
 3613 %{
 3614     single_instruction;
 3615     cr     : S4(write);
 3616     src1   : S3(read);
 3617     src2   : S3(read);
 3618     D0     : S0;        // big decoder only
 3619     ALU    : S4;        // any alu
 3620     MEM    : S3;
 3621 %}
 3622 
 3623 // Conditional move reg-reg
 3624 pipe_class pipe_cmplt( rRegI p, rRegI q, rRegI y)
 3625 %{
 3626     instruction_count(4);
 3627     y      : S4(read);
 3628     q      : S3(read);
 3629     p      : S3(read);
 3630     DECODE : S0(4);     // any decoder
 3631 %}
 3632 
 3633 // Conditional move reg-reg
 3634 pipe_class pipe_cmov_reg( rRegI dst, rRegI src, rFlagsReg cr)
 3635 %{
 3636     single_instruction;
 3637     dst    : S4(write);
 3638     src    : S3(read);
 3639     cr     : S3(read);
 3640     DECODE : S0;        // any decoder
 3641 %}
 3642 
 3643 // Conditional move reg-mem
 3644 pipe_class pipe_cmov_mem( rFlagsReg cr, rRegI dst, memory src)
 3645 %{
 3646     single_instruction;
 3647     dst    : S4(write);
 3648     src    : S3(read);
 3649     cr     : S3(read);
 3650     DECODE : S0;        // any decoder
 3651     MEM    : S3;
 3652 %}
 3653 
 3654 // Conditional move reg-reg long
 3655 pipe_class pipe_cmov_reg_long( rFlagsReg cr, rRegL dst, rRegL src)
 3656 %{
 3657     single_instruction;
 3658     dst    : S4(write);
 3659     src    : S3(read);
 3660     cr     : S3(read);
 3661     DECODE : S0(2);     // any 2 decoders
 3662 %}
 3663 
 3664 // Float reg-reg operation
 3665 pipe_class fpu_reg(regD dst)
 3666 %{
 3667     instruction_count(2);
 3668     dst    : S3(read);
 3669     DECODE : S0(2);     // any 2 decoders
 3670     FPU    : S3;
 3671 %}
 3672 
 3673 // Float reg-reg operation
 3674 pipe_class fpu_reg_reg(regD dst, regD src)
 3675 %{
 3676     instruction_count(2);
 3677     dst    : S4(write);
 3678     src    : S3(read);
 3679     DECODE : S0(2);     // any 2 decoders
 3680     FPU    : S3;
 3681 %}
 3682 
 3683 // Float reg-reg operation
 3684 pipe_class fpu_reg_reg_reg(regD dst, regD src1, regD src2)
 3685 %{
 3686     instruction_count(3);
 3687     dst    : S4(write);
 3688     src1   : S3(read);
 3689     src2   : S3(read);
 3690     DECODE : S0(3);     // any 3 decoders
 3691     FPU    : S3(2);
 3692 %}
 3693 
 3694 // Float reg-reg operation
 3695 pipe_class fpu_reg_reg_reg_reg(regD dst, regD src1, regD src2, regD src3)
 3696 %{
 3697     instruction_count(4);
 3698     dst    : S4(write);
 3699     src1   : S3(read);
 3700     src2   : S3(read);
 3701     src3   : S3(read);
 3702     DECODE : S0(4);     // any 3 decoders
 3703     FPU    : S3(2);
 3704 %}
 3705 
 3706 // Float reg-reg operation
 3707 pipe_class fpu_reg_mem_reg_reg(regD dst, memory src1, regD src2, regD src3)
 3708 %{
 3709     instruction_count(4);
 3710     dst    : S4(write);
 3711     src1   : S3(read);
 3712     src2   : S3(read);
 3713     src3   : S3(read);
 3714     DECODE : S1(3);     // any 3 decoders
 3715     D0     : S0;        // Big decoder only
 3716     FPU    : S3(2);
 3717     MEM    : S3;
 3718 %}
 3719 
 3720 // Float reg-mem operation
 3721 pipe_class fpu_reg_mem(regD dst, memory mem)
 3722 %{
 3723     instruction_count(2);
 3724     dst    : S5(write);
 3725     mem    : S3(read);
 3726     D0     : S0;        // big decoder only
 3727     DECODE : S1;        // any decoder for FPU POP
 3728     FPU    : S4;
 3729     MEM    : S3;        // any mem
 3730 %}
 3731 
 3732 // Float reg-mem operation
 3733 pipe_class fpu_reg_reg_mem(regD dst, regD src1, memory mem)
 3734 %{
 3735     instruction_count(3);
 3736     dst    : S5(write);
 3737     src1   : S3(read);
 3738     mem    : S3(read);
 3739     D0     : S0;        // big decoder only
 3740     DECODE : S1(2);     // any decoder for FPU POP
 3741     FPU    : S4;
 3742     MEM    : S3;        // any mem
 3743 %}
 3744 
 3745 // Float mem-reg operation
 3746 pipe_class fpu_mem_reg(memory mem, regD src)
 3747 %{
 3748     instruction_count(2);
 3749     src    : S5(read);
 3750     mem    : S3(read);
 3751     DECODE : S0;        // any decoder for FPU PUSH
 3752     D0     : S1;        // big decoder only
 3753     FPU    : S4;
 3754     MEM    : S3;        // any mem
 3755 %}
 3756 
 3757 pipe_class fpu_mem_reg_reg(memory mem, regD src1, regD src2)
 3758 %{
 3759     instruction_count(3);
 3760     src1   : S3(read);
 3761     src2   : S3(read);
 3762     mem    : S3(read);
 3763     DECODE : S0(2);     // any decoder for FPU PUSH
 3764     D0     : S1;        // big decoder only
 3765     FPU    : S4;
 3766     MEM    : S3;        // any mem
 3767 %}
 3768 
 3769 pipe_class fpu_mem_reg_mem(memory mem, regD src1, memory src2)
 3770 %{
 3771     instruction_count(3);
 3772     src1   : S3(read);
 3773     src2   : S3(read);
 3774     mem    : S4(read);
 3775     DECODE : S0;        // any decoder for FPU PUSH
 3776     D0     : S0(2);     // big decoder only
 3777     FPU    : S4;
 3778     MEM    : S3(2);     // any mem
 3779 %}
 3780 
 3781 pipe_class fpu_mem_mem(memory dst, memory src1)
 3782 %{
 3783     instruction_count(2);
 3784     src1   : S3(read);
 3785     dst    : S4(read);
 3786     D0     : S0(2);     // big decoder only
 3787     MEM    : S3(2);     // any mem
 3788 %}
 3789 
 3790 pipe_class fpu_mem_mem_mem(memory dst, memory src1, memory src2)
 3791 %{
 3792     instruction_count(3);
 3793     src1   : S3(read);
 3794     src2   : S3(read);
 3795     dst    : S4(read);
 3796     D0     : S0(3);     // big decoder only
 3797     FPU    : S4;
 3798     MEM    : S3(3);     // any mem
 3799 %}
 3800 
 3801 pipe_class fpu_mem_reg_con(memory mem, regD src1)
 3802 %{
 3803     instruction_count(3);
 3804     src1   : S4(read);
 3805     mem    : S4(read);
 3806     DECODE : S0;        // any decoder for FPU PUSH
 3807     D0     : S0(2);     // big decoder only
 3808     FPU    : S4;
 3809     MEM    : S3(2);     // any mem
 3810 %}
 3811 
 3812 // Float load constant
 3813 pipe_class fpu_reg_con(regD dst)
 3814 %{
 3815     instruction_count(2);
 3816     dst    : S5(write);
 3817     D0     : S0;        // big decoder only for the load
 3818     DECODE : S1;        // any decoder for FPU POP
 3819     FPU    : S4;
 3820     MEM    : S3;        // any mem
 3821 %}
 3822 
 3823 // Float load constant
 3824 pipe_class fpu_reg_reg_con(regD dst, regD src)
 3825 %{
 3826     instruction_count(3);
 3827     dst    : S5(write);
 3828     src    : S3(read);
 3829     D0     : S0;        // big decoder only for the load
 3830     DECODE : S1(2);     // any decoder for FPU POP
 3831     FPU    : S4;
 3832     MEM    : S3;        // any mem
 3833 %}
 3834 
 3835 // UnConditional branch
 3836 pipe_class pipe_jmp(label labl)
 3837 %{
 3838     single_instruction;
 3839     BR   : S3;
 3840 %}
 3841 
 3842 // Conditional branch
 3843 pipe_class pipe_jcc(cmpOp cmp, rFlagsReg cr, label labl)
 3844 %{
 3845     single_instruction;
 3846     cr    : S1(read);
 3847     BR    : S3;
 3848 %}
 3849 
 3850 // Allocation idiom
 3851 pipe_class pipe_cmpxchg(rRegP dst, rRegP heap_ptr)
 3852 %{
 3853     instruction_count(1); force_serialization;
 3854     fixed_latency(6);
 3855     heap_ptr : S3(read);
 3856     DECODE   : S0(3);
 3857     D0       : S2;
 3858     MEM      : S3;
 3859     ALU      : S3(2);
 3860     dst      : S5(write);
 3861     BR       : S5;
 3862 %}
 3863 
 3864 // Generic big/slow expanded idiom
 3865 pipe_class pipe_slow()
 3866 %{
 3867     instruction_count(10); multiple_bundles; force_serialization;
 3868     fixed_latency(100);
 3869     D0  : S0(2);
 3870     MEM : S3(2);
 3871 %}
 3872 
 3873 // The real do-nothing guy
 3874 pipe_class empty()
 3875 %{
 3876     instruction_count(0);
 3877 %}
 3878 
 3879 // Define the class for the Nop node
 3880 define
 3881 %{
 3882    MachNop = empty;
 3883 %}
 3884 
 3885 %}
 3886 
 3887 //----------INSTRUCTIONS-------------------------------------------------------
 3888 //
 3889 // match      -- States which machine-independent subtree may be replaced
 3890 //               by this instruction.
 3891 // ins_cost   -- The estimated cost of this instruction is used by instruction
 3892 //               selection to identify a minimum cost tree of machine
 3893 //               instructions that matches a tree of machine-independent
 3894 //               instructions.
 3895 // format     -- A string providing the disassembly for this instruction.
 3896 //               The value of an instruction's operand may be inserted
 3897 //               by referring to it with a '$' prefix.
 3898 // opcode     -- Three instruction opcodes may be provided.  These are referred
 3899 //               to within an encode class as $primary, $secondary, and $tertiary
 3900 //               rrspectively.  The primary opcode is commonly used to
 3901 //               indicate the type of machine instruction, while secondary
 3902 //               and tertiary are often used for prefix options or addressing
 3903 //               modes.
 3904 // ins_encode -- A list of encode classes with parameters. The encode class
 3905 //               name must have been defined in an 'enc_class' specification
 3906 //               in the encode section of the architecture description.
 3907 
 3908 // Dummy reg-to-reg vector moves. Removed during post-selection cleanup.
 3909 // Load Float
 3910 instruct MoveF2VL(vlRegF dst, regF src) %{
 3911   match(Set dst src);
 3912   format %{ "movss $dst,$src\t! load float (4 bytes)" %}
 3913   ins_encode %{
 3914     ShouldNotReachHere();
 3915   %}
 3916   ins_pipe( fpu_reg_reg );
 3917 %}
 3918 
 3919 // Load Float
 3920 instruct MoveF2LEG(legRegF dst, regF src) %{
 3921   match(Set dst src);
 3922   format %{ "movss $dst,$src\t# if src != dst load float (4 bytes)" %}
 3923   ins_encode %{
 3924     ShouldNotReachHere();
 3925   %}
 3926   ins_pipe( fpu_reg_reg );
 3927 %}
 3928 
 3929 // Load Float
 3930 instruct MoveVL2F(regF dst, vlRegF src) %{
 3931   match(Set dst src);
 3932   format %{ "movss $dst,$src\t! load float (4 bytes)" %}
 3933   ins_encode %{
 3934     ShouldNotReachHere();
 3935   %}
 3936   ins_pipe( fpu_reg_reg );
 3937 %}
 3938 
 3939 // Load Float
 3940 instruct MoveLEG2F(regF dst, legRegF src) %{
 3941   match(Set dst src);
 3942   format %{ "movss $dst,$src\t# if src != dst load float (4 bytes)" %}
 3943   ins_encode %{
 3944     ShouldNotReachHere();
 3945   %}
 3946   ins_pipe( fpu_reg_reg );
 3947 %}
 3948 
 3949 // Load Double
 3950 instruct MoveD2VL(vlRegD dst, regD src) %{
 3951   match(Set dst src);
 3952   format %{ "movsd $dst,$src\t! load double (8 bytes)" %}
 3953   ins_encode %{
 3954     ShouldNotReachHere();
 3955   %}
 3956   ins_pipe( fpu_reg_reg );
 3957 %}
 3958 
 3959 // Load Double
 3960 instruct MoveD2LEG(legRegD dst, regD src) %{
 3961   match(Set dst src);
 3962   format %{ "movsd $dst,$src\t# if src != dst load double (8 bytes)" %}
 3963   ins_encode %{
 3964     ShouldNotReachHere();
 3965   %}
 3966   ins_pipe( fpu_reg_reg );
 3967 %}
 3968 
 3969 // Load Double
 3970 instruct MoveVL2D(regD dst, vlRegD src) %{
 3971   match(Set dst src);
 3972   format %{ "movsd $dst,$src\t! load double (8 bytes)" %}
 3973   ins_encode %{
 3974     ShouldNotReachHere();
 3975   %}
 3976   ins_pipe( fpu_reg_reg );
 3977 %}
 3978 
 3979 // Load Double
 3980 instruct MoveLEG2D(regD dst, legRegD src) %{
 3981   match(Set dst src);
 3982   format %{ "movsd $dst,$src\t# if src != dst load double (8 bytes)" %}
 3983   ins_encode %{
 3984     ShouldNotReachHere();
 3985   %}
 3986   ins_pipe( fpu_reg_reg );
 3987 %}
 3988 
 3989 //----------Load/Store/Move Instructions---------------------------------------
 3990 //----------Load Instructions--------------------------------------------------
 3991 
 3992 // Load Byte (8 bit signed)
 3993 instruct loadB(rRegI dst, memory mem)
 3994 %{
 3995   match(Set dst (LoadB mem));
 3996 
 3997   ins_cost(125);
 3998   format %{ "movsbl  $dst, $mem\t# byte" %}
 3999 
 4000   ins_encode %{
 4001     __ movsbl($dst$$Register, $mem$$Address);
 4002   %}
 4003 
 4004   ins_pipe(ialu_reg_mem);
 4005 %}
 4006 
 4007 // Load Byte (8 bit signed) into Long Register
 4008 instruct loadB2L(rRegL dst, memory mem)
 4009 %{
 4010   match(Set dst (ConvI2L (LoadB mem)));
 4011 
 4012   ins_cost(125);
 4013   format %{ "movsbq  $dst, $mem\t# byte -> long" %}
 4014 
 4015   ins_encode %{
 4016     __ movsbq($dst$$Register, $mem$$Address);
 4017   %}
 4018 
 4019   ins_pipe(ialu_reg_mem);
 4020 %}
 4021 
 4022 // Load Unsigned Byte (8 bit UNsigned)
 4023 instruct loadUB(rRegI dst, memory mem)
 4024 %{
 4025   match(Set dst (LoadUB mem));
 4026 
 4027   ins_cost(125);
 4028   format %{ "movzbl  $dst, $mem\t# ubyte" %}
 4029 
 4030   ins_encode %{
 4031     __ movzbl($dst$$Register, $mem$$Address);
 4032   %}
 4033 
 4034   ins_pipe(ialu_reg_mem);
 4035 %}
 4036 
 4037 // Load Unsigned Byte (8 bit UNsigned) into Long Register
 4038 instruct loadUB2L(rRegL dst, memory mem)
 4039 %{
 4040   match(Set dst (ConvI2L (LoadUB mem)));
 4041 
 4042   ins_cost(125);
 4043   format %{ "movzbq  $dst, $mem\t# ubyte -> long" %}
 4044 
 4045   ins_encode %{
 4046     __ movzbq($dst$$Register, $mem$$Address);
 4047   %}
 4048 
 4049   ins_pipe(ialu_reg_mem);
 4050 %}
 4051 
 4052 // Load Unsigned Byte (8 bit UNsigned) with 32-bit mask into Long Register
 4053 instruct loadUB2L_immI(rRegL dst, memory mem, immI mask, rFlagsReg cr) %{
 4054   match(Set dst (ConvI2L (AndI (LoadUB mem) mask)));
 4055   effect(KILL cr);
 4056 
 4057   format %{ "movzbq  $dst, $mem\t# ubyte & 32-bit mask -> long\n\t"
 4058             "andl    $dst, right_n_bits($mask, 8)" %}
 4059   ins_encode %{
 4060     Register Rdst = $dst$$Register;
 4061     __ movzbq(Rdst, $mem$$Address);
 4062     __ andl(Rdst, $mask$$constant & right_n_bits(8));
 4063   %}
 4064   ins_pipe(ialu_reg_mem);
 4065 %}
 4066 
 4067 // Load Short (16 bit signed)
 4068 instruct loadS(rRegI dst, memory mem)
 4069 %{
 4070   match(Set dst (LoadS mem));
 4071 
 4072   ins_cost(125);
 4073   format %{ "movswl $dst, $mem\t# short" %}
 4074 
 4075   ins_encode %{
 4076     __ movswl($dst$$Register, $mem$$Address);
 4077   %}
 4078 
 4079   ins_pipe(ialu_reg_mem);
 4080 %}
 4081 
 4082 // Load Short (16 bit signed) to Byte (8 bit signed)
 4083 instruct loadS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
 4084   match(Set dst (RShiftI (LShiftI (LoadS mem) twentyfour) twentyfour));
 4085 
 4086   ins_cost(125);
 4087   format %{ "movsbl $dst, $mem\t# short -> byte" %}
 4088   ins_encode %{
 4089     __ movsbl($dst$$Register, $mem$$Address);
 4090   %}
 4091   ins_pipe(ialu_reg_mem);
 4092 %}
 4093 
 4094 // Load Short (16 bit signed) into Long Register
 4095 instruct loadS2L(rRegL dst, memory mem)
 4096 %{
 4097   match(Set dst (ConvI2L (LoadS mem)));
 4098 
 4099   ins_cost(125);
 4100   format %{ "movswq $dst, $mem\t# short -> long" %}
 4101 
 4102   ins_encode %{
 4103     __ movswq($dst$$Register, $mem$$Address);
 4104   %}
 4105 
 4106   ins_pipe(ialu_reg_mem);
 4107 %}
 4108 
 4109 // Load Unsigned Short/Char (16 bit UNsigned)
 4110 instruct loadUS(rRegI dst, memory mem)
 4111 %{
 4112   match(Set dst (LoadUS mem));
 4113 
 4114   ins_cost(125);
 4115   format %{ "movzwl  $dst, $mem\t# ushort/char" %}
 4116 
 4117   ins_encode %{
 4118     __ movzwl($dst$$Register, $mem$$Address);
 4119   %}
 4120 
 4121   ins_pipe(ialu_reg_mem);
 4122 %}
 4123 
 4124 // Load Unsigned Short/Char (16 bit UNsigned) to Byte (8 bit signed)
 4125 instruct loadUS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
 4126   match(Set dst (RShiftI (LShiftI (LoadUS mem) twentyfour) twentyfour));
 4127 
 4128   ins_cost(125);
 4129   format %{ "movsbl $dst, $mem\t# ushort -> byte" %}
 4130   ins_encode %{
 4131     __ movsbl($dst$$Register, $mem$$Address);
 4132   %}
 4133   ins_pipe(ialu_reg_mem);
 4134 %}
 4135 
 4136 // Load Unsigned Short/Char (16 bit UNsigned) into Long Register
 4137 instruct loadUS2L(rRegL dst, memory mem)
 4138 %{
 4139   match(Set dst (ConvI2L (LoadUS mem)));
 4140 
 4141   ins_cost(125);
 4142   format %{ "movzwq  $dst, $mem\t# ushort/char -> long" %}
 4143 
 4144   ins_encode %{
 4145     __ movzwq($dst$$Register, $mem$$Address);
 4146   %}
 4147 
 4148   ins_pipe(ialu_reg_mem);
 4149 %}
 4150 
 4151 // Load Unsigned Short/Char (16 bit UNsigned) with mask 0xFF into Long Register
 4152 instruct loadUS2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
 4153   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
 4154 
 4155   format %{ "movzbq  $dst, $mem\t# ushort/char & 0xFF -> long" %}
 4156   ins_encode %{
 4157     __ movzbq($dst$$Register, $mem$$Address);
 4158   %}
 4159   ins_pipe(ialu_reg_mem);
 4160 %}
 4161 
 4162 // Load Unsigned Short/Char (16 bit UNsigned) with 32-bit mask into Long Register
 4163 instruct loadUS2L_immI(rRegL dst, memory mem, immI mask, rFlagsReg cr) %{
 4164   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
 4165   effect(KILL cr);
 4166 
 4167   format %{ "movzwq  $dst, $mem\t# ushort/char & 32-bit mask -> long\n\t"
 4168             "andl    $dst, right_n_bits($mask, 16)" %}
 4169   ins_encode %{
 4170     Register Rdst = $dst$$Register;
 4171     __ movzwq(Rdst, $mem$$Address);
 4172     __ andl(Rdst, $mask$$constant & right_n_bits(16));
 4173   %}
 4174   ins_pipe(ialu_reg_mem);
 4175 %}
 4176 
 4177 // Load Integer
 4178 instruct loadI(rRegI dst, memory mem)
 4179 %{
 4180   match(Set dst (LoadI mem));
 4181 
 4182   ins_cost(125);
 4183   format %{ "movl    $dst, $mem\t# int" %}
 4184 
 4185   ins_encode %{
 4186     __ movl($dst$$Register, $mem$$Address);
 4187   %}
 4188 
 4189   ins_pipe(ialu_reg_mem);
 4190 %}
 4191 
 4192 // Load Integer (32 bit signed) to Byte (8 bit signed)
 4193 instruct loadI2B(rRegI dst, memory mem, immI_24 twentyfour) %{
 4194   match(Set dst (RShiftI (LShiftI (LoadI mem) twentyfour) twentyfour));
 4195 
 4196   ins_cost(125);
 4197   format %{ "movsbl  $dst, $mem\t# int -> byte" %}
 4198   ins_encode %{
 4199     __ movsbl($dst$$Register, $mem$$Address);
 4200   %}
 4201   ins_pipe(ialu_reg_mem);
 4202 %}
 4203 
 4204 // Load Integer (32 bit signed) to Unsigned Byte (8 bit UNsigned)
 4205 instruct loadI2UB(rRegI dst, memory mem, immI_255 mask) %{
 4206   match(Set dst (AndI (LoadI mem) mask));
 4207 
 4208   ins_cost(125);
 4209   format %{ "movzbl  $dst, $mem\t# int -> ubyte" %}
 4210   ins_encode %{
 4211     __ movzbl($dst$$Register, $mem$$Address);
 4212   %}
 4213   ins_pipe(ialu_reg_mem);
 4214 %}
 4215 
 4216 // Load Integer (32 bit signed) to Short (16 bit signed)
 4217 instruct loadI2S(rRegI dst, memory mem, immI_16 sixteen) %{
 4218   match(Set dst (RShiftI (LShiftI (LoadI mem) sixteen) sixteen));
 4219 
 4220   ins_cost(125);
 4221   format %{ "movswl  $dst, $mem\t# int -> short" %}
 4222   ins_encode %{
 4223     __ movswl($dst$$Register, $mem$$Address);
 4224   %}
 4225   ins_pipe(ialu_reg_mem);
 4226 %}
 4227 
 4228 // Load Integer (32 bit signed) to Unsigned Short/Char (16 bit UNsigned)
 4229 instruct loadI2US(rRegI dst, memory mem, immI_65535 mask) %{
 4230   match(Set dst (AndI (LoadI mem) mask));
 4231 
 4232   ins_cost(125);
 4233   format %{ "movzwl  $dst, $mem\t# int -> ushort/char" %}
 4234   ins_encode %{
 4235     __ movzwl($dst$$Register, $mem$$Address);
 4236   %}
 4237   ins_pipe(ialu_reg_mem);
 4238 %}
 4239 
 4240 // Load Integer into Long Register
 4241 instruct loadI2L(rRegL dst, memory mem)
 4242 %{
 4243   match(Set dst (ConvI2L (LoadI mem)));
 4244 
 4245   ins_cost(125);
 4246   format %{ "movslq  $dst, $mem\t# int -> long" %}
 4247 
 4248   ins_encode %{
 4249     __ movslq($dst$$Register, $mem$$Address);
 4250   %}
 4251 
 4252   ins_pipe(ialu_reg_mem);
 4253 %}
 4254 
 4255 // Load Integer with mask 0xFF into Long Register
 4256 instruct loadI2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
 4257   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
 4258 
 4259   format %{ "movzbq  $dst, $mem\t# int & 0xFF -> long" %}
 4260   ins_encode %{
 4261     __ movzbq($dst$$Register, $mem$$Address);
 4262   %}
 4263   ins_pipe(ialu_reg_mem);
 4264 %}
 4265 
 4266 // Load Integer with mask 0xFFFF into Long Register
 4267 instruct loadI2L_immI_65535(rRegL dst, memory mem, immI_65535 mask) %{
 4268   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
 4269 
 4270   format %{ "movzwq  $dst, $mem\t# int & 0xFFFF -> long" %}
 4271   ins_encode %{
 4272     __ movzwq($dst$$Register, $mem$$Address);
 4273   %}
 4274   ins_pipe(ialu_reg_mem);
 4275 %}
 4276 
 4277 // Load Integer with a 31-bit mask into Long Register
 4278 instruct loadI2L_immU31(rRegL dst, memory mem, immU31 mask, rFlagsReg cr) %{
 4279   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
 4280   effect(KILL cr);
 4281 
 4282   format %{ "movl    $dst, $mem\t# int & 31-bit mask -> long\n\t"
 4283             "andl    $dst, $mask" %}
 4284   ins_encode %{
 4285     Register Rdst = $dst$$Register;
 4286     __ movl(Rdst, $mem$$Address);
 4287     __ andl(Rdst, $mask$$constant);
 4288   %}
 4289   ins_pipe(ialu_reg_mem);
 4290 %}
 4291 
 4292 // Load Unsigned Integer into Long Register
 4293 instruct loadUI2L(rRegL dst, memory mem, immL_32bits mask)
 4294 %{
 4295   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
 4296 
 4297   ins_cost(125);
 4298   format %{ "movl    $dst, $mem\t# uint -> long" %}
 4299 
 4300   ins_encode %{
 4301     __ movl($dst$$Register, $mem$$Address);
 4302   %}
 4303 
 4304   ins_pipe(ialu_reg_mem);
 4305 %}
 4306 
 4307 // Load Long
 4308 instruct loadL(rRegL dst, memory mem)
 4309 %{
 4310   match(Set dst (LoadL mem));
 4311 
 4312   ins_cost(125);
 4313   format %{ "movq    $dst, $mem\t# long" %}
 4314 
 4315   ins_encode %{
 4316     __ movq($dst$$Register, $mem$$Address);
 4317   %}
 4318 
 4319   ins_pipe(ialu_reg_mem); // XXX
 4320 %}
 4321 
 4322 // Load Range
 4323 instruct loadRange(rRegI dst, memory mem)
 4324 %{
 4325   match(Set dst (LoadRange mem));
 4326 
 4327   ins_cost(125); // XXX
 4328   format %{ "movl    $dst, $mem\t# range" %}
 4329   ins_encode %{
 4330     __ movl($dst$$Register, $mem$$Address);
 4331   %}
 4332   ins_pipe(ialu_reg_mem);
 4333 %}
 4334 
 4335 // Load Pointer
 4336 instruct loadP(rRegP dst, memory mem)
 4337 %{
 4338   match(Set dst (LoadP mem));
 4339   predicate(n->as_Load()->barrier_data() == 0);
 4340 
 4341   ins_cost(125); // XXX
 4342   format %{ "movq    $dst, $mem\t# ptr" %}
 4343   ins_encode %{
 4344     __ movq($dst$$Register, $mem$$Address);
 4345   %}
 4346   ins_pipe(ialu_reg_mem); // XXX
 4347 %}
 4348 
 4349 // Load Compressed Pointer
 4350 instruct loadN(rRegN dst, memory mem)
 4351 %{
 4352    predicate(n->as_Load()->barrier_data() == 0);
 4353    match(Set dst (LoadN mem));
 4354 
 4355    ins_cost(125); // XXX
 4356    format %{ "movl    $dst, $mem\t# compressed ptr" %}
 4357    ins_encode %{
 4358      __ movl($dst$$Register, $mem$$Address);
 4359    %}
 4360    ins_pipe(ialu_reg_mem); // XXX
 4361 %}
 4362 
 4363 
 4364 // Load Klass Pointer
 4365 instruct loadKlass(rRegP dst, memory mem)
 4366 %{
 4367   match(Set dst (LoadKlass mem));
 4368 
 4369   ins_cost(125); // XXX
 4370   format %{ "movq    $dst, $mem\t# class" %}
 4371   ins_encode %{
 4372     __ movq($dst$$Register, $mem$$Address);
 4373   %}
 4374   ins_pipe(ialu_reg_mem); // XXX
 4375 %}
 4376 
 4377 // Load narrow Klass Pointer
 4378 instruct loadNKlass(rRegN dst, memory mem)
 4379 %{
 4380   predicate(!UseCompactObjectHeaders);
 4381   match(Set dst (LoadNKlass mem));
 4382 
 4383   ins_cost(125); // XXX
 4384   format %{ "movl    $dst, $mem\t# compressed klass ptr" %}
 4385   ins_encode %{
 4386     __ movl($dst$$Register, $mem$$Address);
 4387   %}
 4388   ins_pipe(ialu_reg_mem); // XXX
 4389 %}
 4390 
 4391 instruct loadNKlassCompactHeaders(rRegN dst, memory mem, rFlagsReg cr)
 4392 %{
 4393   predicate(UseCompactObjectHeaders);
 4394   match(Set dst (LoadNKlass mem));
 4395   effect(KILL cr);
 4396   ins_cost(125);
 4397   format %{
 4398     "movl    $dst, $mem\t# compressed klass ptr, shifted\n\t"
 4399     "shrl    $dst, markWord::klass_shift_at_offset"
 4400   %}
 4401   ins_encode %{
 4402     if (UseAPX) {
 4403       __ eshrl($dst$$Register, $mem$$Address, markWord::klass_shift_at_offset, false);
 4404     }
 4405     else {
 4406       __ movl($dst$$Register, $mem$$Address);
 4407       __ shrl($dst$$Register, markWord::klass_shift_at_offset);
 4408     }
 4409   %}
 4410   ins_pipe(ialu_reg_mem);
 4411 %}
 4412 
 4413 // Load Float
 4414 instruct loadF(regF dst, memory mem)
 4415 %{
 4416   match(Set dst (LoadF mem));
 4417 
 4418   ins_cost(145); // XXX
 4419   format %{ "movss   $dst, $mem\t# float" %}
 4420   ins_encode %{
 4421     __ movflt($dst$$XMMRegister, $mem$$Address);
 4422   %}
 4423   ins_pipe(pipe_slow); // XXX
 4424 %}
 4425 
 4426 // Load Double
 4427 instruct loadD_partial(regD dst, memory mem)
 4428 %{
 4429   predicate(!UseXmmLoadAndClearUpper);
 4430   match(Set dst (LoadD mem));
 4431 
 4432   ins_cost(145); // XXX
 4433   format %{ "movlpd  $dst, $mem\t# double" %}
 4434   ins_encode %{
 4435     __ movdbl($dst$$XMMRegister, $mem$$Address);
 4436   %}
 4437   ins_pipe(pipe_slow); // XXX
 4438 %}
 4439 
 4440 instruct loadD(regD dst, memory mem)
 4441 %{
 4442   predicate(UseXmmLoadAndClearUpper);
 4443   match(Set dst (LoadD mem));
 4444 
 4445   ins_cost(145); // XXX
 4446   format %{ "movsd   $dst, $mem\t# double" %}
 4447   ins_encode %{
 4448     __ movdbl($dst$$XMMRegister, $mem$$Address);
 4449   %}
 4450   ins_pipe(pipe_slow); // XXX
 4451 %}
 4452 
 4453 // max = java.lang.Math.max(float a, float b)
 4454 instruct maxF_avx10_reg(regF dst, regF a, regF b) %{
 4455   predicate(VM_Version::supports_avx10_2());
 4456   match(Set dst (MaxF a b));
 4457   format %{ "maxF $dst, $a, $b" %}
 4458   ins_encode %{
 4459     __ eminmaxss($dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, AVX10_MINMAX_MAX_COMPARE_SIGN);
 4460   %}
 4461   ins_pipe( pipe_slow );
 4462 %}
 4463 
 4464 // max = java.lang.Math.max(float a, float b)
 4465 instruct maxF_reg(legRegF dst, legRegF a, legRegF b, legRegF tmp, legRegF atmp, legRegF btmp) %{
 4466   predicate(!VM_Version::supports_avx10_2() && UseAVX > 0 && !VLoopReductions::is_reduction(n));
 4467   match(Set dst (MaxF a b));
 4468   effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
 4469   format %{ "maxF $dst, $a, $b \t! using $tmp, $atmp and $btmp as TEMP" %}
 4470   ins_encode %{
 4471     __ vminmax_fp(Op_MaxV, T_FLOAT, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, Assembler::AVX_128bit);
 4472   %}
 4473   ins_pipe( pipe_slow );
 4474 %}
 4475 
 4476 instruct maxF_reduction_reg(legRegF dst, legRegF a, legRegF b, legRegF xtmp, rRegI rtmp, rFlagsReg cr) %{
 4477   predicate(!VM_Version::supports_avx10_2() && UseAVX > 0 && VLoopReductions::is_reduction(n));
 4478   match(Set dst (MaxF a b));
 4479   effect(USE a, USE b, TEMP xtmp, TEMP rtmp, KILL cr);
 4480 
 4481   format %{ "maxF_reduction $dst, $a, $b \t!using $xtmp and $rtmp as TEMP" %}
 4482   ins_encode %{
 4483     emit_fp_min_max(masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xtmp$$XMMRegister, $rtmp$$Register,
 4484                     false /*min*/, true /*single*/);
 4485   %}
 4486   ins_pipe( pipe_slow );
 4487 %}
 4488 
 4489 // max = java.lang.Math.max(double a, double b)
 4490 instruct maxD_avx10_reg(regD dst, regD a, regD b) %{
 4491   predicate(VM_Version::supports_avx10_2());
 4492   match(Set dst (MaxD a b));
 4493   format %{ "maxD $dst, $a, $b" %}
 4494   ins_encode %{
 4495     __ eminmaxsd($dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, AVX10_MINMAX_MAX_COMPARE_SIGN);
 4496   %}
 4497   ins_pipe( pipe_slow );
 4498 %}
 4499 
 4500 // max = java.lang.Math.max(double a, double b)
 4501 instruct maxD_reg(legRegD dst, legRegD a, legRegD b, legRegD tmp, legRegD atmp, legRegD btmp) %{
 4502   predicate(!VM_Version::supports_avx10_2() && UseAVX > 0 && !VLoopReductions::is_reduction(n));
 4503   match(Set dst (MaxD a b));
 4504   effect(USE a, USE b, TEMP atmp, TEMP btmp, TEMP tmp);
 4505   format %{ "maxD $dst, $a, $b \t! using $tmp, $atmp and $btmp as TEMP" %}
 4506   ins_encode %{
 4507     __ vminmax_fp(Op_MaxV, T_DOUBLE, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, Assembler::AVX_128bit);
 4508   %}
 4509   ins_pipe( pipe_slow );
 4510 %}
 4511 
 4512 instruct maxD_reduction_reg(legRegD dst, legRegD a, legRegD b, legRegD xtmp, rRegL rtmp, rFlagsReg cr) %{
 4513   predicate(!VM_Version::supports_avx10_2() && UseAVX > 0 && VLoopReductions::is_reduction(n));
 4514   match(Set dst (MaxD a b));
 4515   effect(USE a, USE b, TEMP xtmp, TEMP rtmp, KILL cr);
 4516 
 4517   format %{ "maxD_reduction $dst, $a, $b \t! using $xtmp and $rtmp as TEMP" %}
 4518   ins_encode %{
 4519     emit_fp_min_max(masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xtmp$$XMMRegister, $rtmp$$Register,
 4520                     false /*min*/, false /*single*/);
 4521   %}
 4522   ins_pipe( pipe_slow );
 4523 %}
 4524 
 4525 // max = java.lang.Math.min(float a, float b)
 4526 instruct minF_avx10_reg(regF dst, regF a, regF b) %{
 4527   predicate(VM_Version::supports_avx10_2());
 4528   match(Set dst (MinF a b));
 4529   format %{ "minF $dst, $a, $b" %}
 4530   ins_encode %{
 4531     __ eminmaxss($dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, AVX10_MINMAX_MIN_COMPARE_SIGN);
 4532   %}
 4533   ins_pipe( pipe_slow );
 4534 %}
 4535 
 4536 // min = java.lang.Math.min(float a, float b)
 4537 instruct minF_reg(legRegF dst, legRegF a, legRegF b, legRegF tmp, legRegF atmp, legRegF btmp) %{
 4538   predicate(!VM_Version::supports_avx10_2() && UseAVX > 0 && !VLoopReductions::is_reduction(n));
 4539   match(Set dst (MinF a b));
 4540   effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
 4541   format %{ "minF $dst, $a, $b \t! using $tmp, $atmp and $btmp as TEMP" %}
 4542   ins_encode %{
 4543     __ vminmax_fp(Op_MinV, T_FLOAT, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, Assembler::AVX_128bit);
 4544   %}
 4545   ins_pipe( pipe_slow );
 4546 %}
 4547 
 4548 instruct minF_reduction_reg(legRegF dst, legRegF a, legRegF b, legRegF xtmp, rRegI rtmp, rFlagsReg cr) %{
 4549   predicate(!VM_Version::supports_avx10_2() && UseAVX > 0 && VLoopReductions::is_reduction(n));
 4550   match(Set dst (MinF a b));
 4551   effect(USE a, USE b, TEMP xtmp, TEMP rtmp, KILL cr);
 4552 
 4553   format %{ "minF_reduction $dst, $a, $b \t! using $xtmp and $rtmp as TEMP" %}
 4554   ins_encode %{
 4555     emit_fp_min_max(masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xtmp$$XMMRegister, $rtmp$$Register,
 4556                     true /*min*/, true /*single*/);
 4557   %}
 4558   ins_pipe( pipe_slow );
 4559 %}
 4560 
 4561 // max = java.lang.Math.min(double a, double b)
 4562 instruct minD_avx10_reg(regD dst, regD a, regD b) %{
 4563   predicate(VM_Version::supports_avx10_2());
 4564   match(Set dst (MinD a b));
 4565   format %{ "minD $dst, $a, $b" %}
 4566   ins_encode %{
 4567     __ eminmaxsd($dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, AVX10_MINMAX_MIN_COMPARE_SIGN);
 4568   %}
 4569   ins_pipe( pipe_slow );
 4570 %}
 4571 
 4572 // min = java.lang.Math.min(double a, double b)
 4573 instruct minD_reg(legRegD dst, legRegD a, legRegD b, legRegD tmp, legRegD atmp, legRegD btmp) %{
 4574   predicate(!VM_Version::supports_avx10_2() && UseAVX > 0 && !VLoopReductions::is_reduction(n));
 4575   match(Set dst (MinD a b));
 4576   effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
 4577     format %{ "minD $dst, $a, $b \t! using $tmp, $atmp and $btmp as TEMP" %}
 4578   ins_encode %{
 4579     __ vminmax_fp(Op_MinV, T_DOUBLE, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, Assembler::AVX_128bit);
 4580   %}
 4581   ins_pipe( pipe_slow );
 4582 %}
 4583 
 4584 instruct minD_reduction_reg(legRegD dst, legRegD a, legRegD b, legRegD xtmp, rRegL rtmp, rFlagsReg cr) %{
 4585   predicate(!VM_Version::supports_avx10_2() && UseAVX > 0 && VLoopReductions::is_reduction(n));
 4586   match(Set dst (MinD a b));
 4587   effect(USE a, USE b, TEMP xtmp, TEMP rtmp, KILL cr);
 4588 
 4589   format %{ "maxD_reduction $dst, $a, $b \t! using $xtmp and $rtmp as TEMP" %}
 4590   ins_encode %{
 4591     emit_fp_min_max(masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xtmp$$XMMRegister, $rtmp$$Register,
 4592                     true /*min*/, false /*single*/);
 4593   %}
 4594   ins_pipe( pipe_slow );
 4595 %}
 4596 
 4597 // Load Effective Address
 4598 instruct leaP8(rRegP dst, indOffset8 mem)
 4599 %{
 4600   match(Set dst mem);
 4601 
 4602   ins_cost(110); // XXX
 4603   format %{ "leaq    $dst, $mem\t# ptr 8" %}
 4604   ins_encode %{
 4605     __ leaq($dst$$Register, $mem$$Address);
 4606   %}
 4607   ins_pipe(ialu_reg_reg_fat);
 4608 %}
 4609 
 4610 instruct leaP32(rRegP dst, indOffset32 mem)
 4611 %{
 4612   match(Set dst mem);
 4613 
 4614   ins_cost(110);
 4615   format %{ "leaq    $dst, $mem\t# ptr 32" %}
 4616   ins_encode %{
 4617     __ leaq($dst$$Register, $mem$$Address);
 4618   %}
 4619   ins_pipe(ialu_reg_reg_fat);
 4620 %}
 4621 
 4622 instruct leaPIdxOff(rRegP dst, indIndexOffset mem)
 4623 %{
 4624   match(Set dst mem);
 4625 
 4626   ins_cost(110);
 4627   format %{ "leaq    $dst, $mem\t# ptr idxoff" %}
 4628   ins_encode %{
 4629     __ leaq($dst$$Register, $mem$$Address);
 4630   %}
 4631   ins_pipe(ialu_reg_reg_fat);
 4632 %}
 4633 
 4634 instruct leaPIdxScale(rRegP dst, indIndexScale mem)
 4635 %{
 4636   match(Set dst mem);
 4637 
 4638   ins_cost(110);
 4639   format %{ "leaq    $dst, $mem\t# ptr idxscale" %}
 4640   ins_encode %{
 4641     __ leaq($dst$$Register, $mem$$Address);
 4642   %}
 4643   ins_pipe(ialu_reg_reg_fat);
 4644 %}
 4645 
 4646 instruct leaPPosIdxScale(rRegP dst, indPosIndexScale mem)
 4647 %{
 4648   match(Set dst mem);
 4649 
 4650   ins_cost(110);
 4651   format %{ "leaq    $dst, $mem\t# ptr idxscale" %}
 4652   ins_encode %{
 4653     __ leaq($dst$$Register, $mem$$Address);
 4654   %}
 4655   ins_pipe(ialu_reg_reg_fat);
 4656 %}
 4657 
 4658 instruct leaPIdxScaleOff(rRegP dst, indIndexScaleOffset mem)
 4659 %{
 4660   match(Set dst mem);
 4661 
 4662   ins_cost(110);
 4663   format %{ "leaq    $dst, $mem\t# ptr idxscaleoff" %}
 4664   ins_encode %{
 4665     __ leaq($dst$$Register, $mem$$Address);
 4666   %}
 4667   ins_pipe(ialu_reg_reg_fat);
 4668 %}
 4669 
 4670 instruct leaPPosIdxOff(rRegP dst, indPosIndexOffset mem)
 4671 %{
 4672   match(Set dst mem);
 4673 
 4674   ins_cost(110);
 4675   format %{ "leaq    $dst, $mem\t# ptr posidxoff" %}
 4676   ins_encode %{
 4677     __ leaq($dst$$Register, $mem$$Address);
 4678   %}
 4679   ins_pipe(ialu_reg_reg_fat);
 4680 %}
 4681 
 4682 instruct leaPPosIdxScaleOff(rRegP dst, indPosIndexScaleOffset mem)
 4683 %{
 4684   match(Set dst mem);
 4685 
 4686   ins_cost(110);
 4687   format %{ "leaq    $dst, $mem\t# ptr posidxscaleoff" %}
 4688   ins_encode %{
 4689     __ leaq($dst$$Register, $mem$$Address);
 4690   %}
 4691   ins_pipe(ialu_reg_reg_fat);
 4692 %}
 4693 
 4694 // Load Effective Address which uses Narrow (32-bits) oop
 4695 instruct leaPCompressedOopOffset(rRegP dst, indCompressedOopOffset mem)
 4696 %{
 4697   predicate(UseCompressedOops && (CompressedOops::shift() != 0));
 4698   match(Set dst mem);
 4699 
 4700   ins_cost(110);
 4701   format %{ "leaq    $dst, $mem\t# ptr compressedoopoff32" %}
 4702   ins_encode %{
 4703     __ leaq($dst$$Register, $mem$$Address);
 4704   %}
 4705   ins_pipe(ialu_reg_reg_fat);
 4706 %}
 4707 
 4708 instruct leaP8Narrow(rRegP dst, indOffset8Narrow mem)
 4709 %{
 4710   predicate(CompressedOops::shift() == 0);
 4711   match(Set dst mem);
 4712 
 4713   ins_cost(110); // XXX
 4714   format %{ "leaq    $dst, $mem\t# ptr off8narrow" %}
 4715   ins_encode %{
 4716     __ leaq($dst$$Register, $mem$$Address);
 4717   %}
 4718   ins_pipe(ialu_reg_reg_fat);
 4719 %}
 4720 
 4721 instruct leaP32Narrow(rRegP dst, indOffset32Narrow mem)
 4722 %{
 4723   predicate(CompressedOops::shift() == 0);
 4724   match(Set dst mem);
 4725 
 4726   ins_cost(110);
 4727   format %{ "leaq    $dst, $mem\t# ptr off32narrow" %}
 4728   ins_encode %{
 4729     __ leaq($dst$$Register, $mem$$Address);
 4730   %}
 4731   ins_pipe(ialu_reg_reg_fat);
 4732 %}
 4733 
 4734 instruct leaPIdxOffNarrow(rRegP dst, indIndexOffsetNarrow mem)
 4735 %{
 4736   predicate(CompressedOops::shift() == 0);
 4737   match(Set dst mem);
 4738 
 4739   ins_cost(110);
 4740   format %{ "leaq    $dst, $mem\t# ptr idxoffnarrow" %}
 4741   ins_encode %{
 4742     __ leaq($dst$$Register, $mem$$Address);
 4743   %}
 4744   ins_pipe(ialu_reg_reg_fat);
 4745 %}
 4746 
 4747 instruct leaPIdxScaleNarrow(rRegP dst, indIndexScaleNarrow mem)
 4748 %{
 4749   predicate(CompressedOops::shift() == 0);
 4750   match(Set dst mem);
 4751 
 4752   ins_cost(110);
 4753   format %{ "leaq    $dst, $mem\t# ptr idxscalenarrow" %}
 4754   ins_encode %{
 4755     __ leaq($dst$$Register, $mem$$Address);
 4756   %}
 4757   ins_pipe(ialu_reg_reg_fat);
 4758 %}
 4759 
 4760 instruct leaPIdxScaleOffNarrow(rRegP dst, indIndexScaleOffsetNarrow mem)
 4761 %{
 4762   predicate(CompressedOops::shift() == 0);
 4763   match(Set dst mem);
 4764 
 4765   ins_cost(110);
 4766   format %{ "leaq    $dst, $mem\t# ptr idxscaleoffnarrow" %}
 4767   ins_encode %{
 4768     __ leaq($dst$$Register, $mem$$Address);
 4769   %}
 4770   ins_pipe(ialu_reg_reg_fat);
 4771 %}
 4772 
 4773 instruct leaPPosIdxOffNarrow(rRegP dst, indPosIndexOffsetNarrow mem)
 4774 %{
 4775   predicate(CompressedOops::shift() == 0);
 4776   match(Set dst mem);
 4777 
 4778   ins_cost(110);
 4779   format %{ "leaq    $dst, $mem\t# ptr posidxoffnarrow" %}
 4780   ins_encode %{
 4781     __ leaq($dst$$Register, $mem$$Address);
 4782   %}
 4783   ins_pipe(ialu_reg_reg_fat);
 4784 %}
 4785 
 4786 instruct leaPPosIdxScaleOffNarrow(rRegP dst, indPosIndexScaleOffsetNarrow mem)
 4787 %{
 4788   predicate(CompressedOops::shift() == 0);
 4789   match(Set dst mem);
 4790 
 4791   ins_cost(110);
 4792   format %{ "leaq    $dst, $mem\t# ptr posidxscaleoffnarrow" %}
 4793   ins_encode %{
 4794     __ leaq($dst$$Register, $mem$$Address);
 4795   %}
 4796   ins_pipe(ialu_reg_reg_fat);
 4797 %}
 4798 
 4799 instruct loadConI(rRegI dst, immI src)
 4800 %{
 4801   match(Set dst src);
 4802 
 4803   format %{ "movl    $dst, $src\t# int" %}
 4804   ins_encode %{
 4805     __ movl($dst$$Register, $src$$constant);
 4806   %}
 4807   ins_pipe(ialu_reg_fat); // XXX
 4808 %}
 4809 
 4810 instruct loadConI0(rRegI dst, immI_0 src, rFlagsReg cr)
 4811 %{
 4812   match(Set dst src);
 4813   effect(KILL cr);
 4814 
 4815   ins_cost(50);
 4816   format %{ "xorl    $dst, $dst\t# int" %}
 4817   ins_encode %{
 4818     __ xorl($dst$$Register, $dst$$Register);
 4819   %}
 4820   ins_pipe(ialu_reg);
 4821 %}
 4822 
 4823 instruct loadConL(rRegL dst, immL src)
 4824 %{
 4825   match(Set dst src);
 4826 
 4827   ins_cost(150);
 4828   format %{ "movq    $dst, $src\t# long" %}
 4829   ins_encode %{
 4830     __ mov64($dst$$Register, $src$$constant);
 4831   %}
 4832   ins_pipe(ialu_reg);
 4833 %}
 4834 
 4835 instruct loadConL0(rRegL dst, immL0 src, rFlagsReg cr)
 4836 %{
 4837   match(Set dst src);
 4838   effect(KILL cr);
 4839 
 4840   ins_cost(50);
 4841   format %{ "xorl    $dst, $dst\t# long" %}
 4842   ins_encode %{
 4843     __ xorl($dst$$Register, $dst$$Register);
 4844   %}
 4845   ins_pipe(ialu_reg); // XXX
 4846 %}
 4847 
 4848 instruct loadConUL32(rRegL dst, immUL32 src)
 4849 %{
 4850   match(Set dst src);
 4851 
 4852   ins_cost(60);
 4853   format %{ "movl    $dst, $src\t# long (unsigned 32-bit)" %}
 4854   ins_encode %{
 4855     __ movl($dst$$Register, $src$$constant);
 4856   %}
 4857   ins_pipe(ialu_reg);
 4858 %}
 4859 
 4860 instruct loadConL32(rRegL dst, immL32 src)
 4861 %{
 4862   match(Set dst src);
 4863 
 4864   ins_cost(70);
 4865   format %{ "movq    $dst, $src\t# long (32-bit)" %}
 4866   ins_encode %{
 4867     __ movq($dst$$Register, $src$$constant);
 4868   %}
 4869   ins_pipe(ialu_reg);
 4870 %}
 4871 
 4872 instruct loadConP(rRegP dst, immP con) %{
 4873   match(Set dst con);
 4874 
 4875   format %{ "movq    $dst, $con\t# ptr" %}
 4876   ins_encode %{
 4877     __ mov64($dst$$Register, $con$$constant, $con->constant_reloc(), RELOC_IMM64);
 4878   %}
 4879   ins_pipe(ialu_reg_fat); // XXX
 4880 %}
 4881 
 4882 instruct loadConP0(rRegP dst, immP0 src, rFlagsReg cr)
 4883 %{
 4884   match(Set dst src);
 4885   effect(KILL cr);
 4886 
 4887   ins_cost(50);
 4888   format %{ "xorl    $dst, $dst\t# ptr" %}
 4889   ins_encode %{
 4890     __ xorl($dst$$Register, $dst$$Register);
 4891   %}
 4892   ins_pipe(ialu_reg);
 4893 %}
 4894 
 4895 instruct loadConP31(rRegP dst, immP31 src, rFlagsReg cr)
 4896 %{
 4897   match(Set dst src);
 4898   effect(KILL cr);
 4899 
 4900   ins_cost(60);
 4901   format %{ "movl    $dst, $src\t# ptr (positive 32-bit)" %}
 4902   ins_encode %{
 4903     __ movl($dst$$Register, $src$$constant);
 4904   %}
 4905   ins_pipe(ialu_reg);
 4906 %}
 4907 
 4908 instruct loadConF(regF dst, immF con) %{
 4909   match(Set dst con);
 4910   ins_cost(125);
 4911   format %{ "movss   $dst, [$constantaddress]\t# load from constant table: float=$con" %}
 4912   ins_encode %{
 4913     __ movflt($dst$$XMMRegister, $constantaddress($con));
 4914   %}
 4915   ins_pipe(pipe_slow);
 4916 %}
 4917 
 4918 instruct loadConH(regF dst, immH con) %{
 4919   match(Set dst con);
 4920   ins_cost(125);
 4921   format %{ "movss   $dst, [$constantaddress]\t# load from constant table: halffloat=$con" %}
 4922   ins_encode %{
 4923     __ movflt($dst$$XMMRegister, $constantaddress($con));
 4924   %}
 4925   ins_pipe(pipe_slow);
 4926 %}
 4927 
 4928 instruct loadConN0(rRegN dst, immN0 src, rFlagsReg cr) %{
 4929   match(Set dst src);
 4930   effect(KILL cr);
 4931   format %{ "xorq    $dst, $src\t# compressed null pointer" %}
 4932   ins_encode %{
 4933     __ xorq($dst$$Register, $dst$$Register);
 4934   %}
 4935   ins_pipe(ialu_reg);
 4936 %}
 4937 
 4938 instruct loadConN(rRegN dst, immN src) %{
 4939   match(Set dst src);
 4940 
 4941   ins_cost(125);
 4942   format %{ "movl    $dst, $src\t# compressed ptr" %}
 4943   ins_encode %{
 4944     address con = (address)$src$$constant;
 4945     if (con == nullptr) {
 4946       ShouldNotReachHere();
 4947     } else {
 4948       __ set_narrow_oop($dst$$Register, (jobject)$src$$constant);
 4949     }
 4950   %}
 4951   ins_pipe(ialu_reg_fat); // XXX
 4952 %}
 4953 
 4954 instruct loadConNKlass(rRegN dst, immNKlass src) %{
 4955   match(Set dst src);
 4956 
 4957   ins_cost(125);
 4958   format %{ "movl    $dst, $src\t# compressed klass ptr" %}
 4959   ins_encode %{
 4960     address con = (address)$src$$constant;
 4961     if (con == nullptr) {
 4962       ShouldNotReachHere();
 4963     } else {
 4964       __ set_narrow_klass($dst$$Register, (Klass*)$src$$constant);
 4965     }
 4966   %}
 4967   ins_pipe(ialu_reg_fat); // XXX
 4968 %}
 4969 
 4970 instruct loadConF0(regF dst, immF0 src)
 4971 %{
 4972   match(Set dst src);
 4973   ins_cost(100);
 4974 
 4975   format %{ "xorps   $dst, $dst\t# float 0.0" %}
 4976   ins_encode %{
 4977     __ xorps($dst$$XMMRegister, $dst$$XMMRegister);
 4978   %}
 4979   ins_pipe(pipe_slow);
 4980 %}
 4981 
 4982 // Use the same format since predicate() can not be used here.
 4983 instruct loadConD(regD dst, immD con) %{
 4984   match(Set dst con);
 4985   ins_cost(125);
 4986   format %{ "movsd   $dst, [$constantaddress]\t# load from constant table: double=$con" %}
 4987   ins_encode %{
 4988     __ movdbl($dst$$XMMRegister, $constantaddress($con));
 4989   %}
 4990   ins_pipe(pipe_slow);
 4991 %}
 4992 
 4993 instruct loadConD0(regD dst, immD0 src)
 4994 %{
 4995   match(Set dst src);
 4996   ins_cost(100);
 4997 
 4998   format %{ "xorpd   $dst, $dst\t# double 0.0" %}
 4999   ins_encode %{
 5000     __ xorpd($dst$$XMMRegister, $dst$$XMMRegister);
 5001   %}
 5002   ins_pipe(pipe_slow);
 5003 %}
 5004 
 5005 instruct loadSSI(rRegI dst, stackSlotI src)
 5006 %{
 5007   match(Set dst src);
 5008 
 5009   ins_cost(125);
 5010   format %{ "movl    $dst, $src\t# int stk" %}
 5011   ins_encode %{
 5012     __ movl($dst$$Register, $src$$Address);
 5013   %}
 5014   ins_pipe(ialu_reg_mem);
 5015 %}
 5016 
 5017 instruct loadSSL(rRegL dst, stackSlotL src)
 5018 %{
 5019   match(Set dst src);
 5020 
 5021   ins_cost(125);
 5022   format %{ "movq    $dst, $src\t# long stk" %}
 5023   ins_encode %{
 5024     __ movq($dst$$Register, $src$$Address);
 5025   %}
 5026   ins_pipe(ialu_reg_mem);
 5027 %}
 5028 
 5029 instruct loadSSP(rRegP dst, stackSlotP src)
 5030 %{
 5031   match(Set dst src);
 5032 
 5033   ins_cost(125);
 5034   format %{ "movq    $dst, $src\t# ptr stk" %}
 5035   ins_encode %{
 5036     __ movq($dst$$Register, $src$$Address);
 5037   %}
 5038   ins_pipe(ialu_reg_mem);
 5039 %}
 5040 
 5041 instruct loadSSF(regF dst, stackSlotF src)
 5042 %{
 5043   match(Set dst src);
 5044 
 5045   ins_cost(125);
 5046   format %{ "movss   $dst, $src\t# float stk" %}
 5047   ins_encode %{
 5048     __ movflt($dst$$XMMRegister, Address(rsp, $src$$disp));
 5049   %}
 5050   ins_pipe(pipe_slow); // XXX
 5051 %}
 5052 
 5053 // Use the same format since predicate() can not be used here.
 5054 instruct loadSSD(regD dst, stackSlotD src)
 5055 %{
 5056   match(Set dst src);
 5057 
 5058   ins_cost(125);
 5059   format %{ "movsd   $dst, $src\t# double stk" %}
 5060   ins_encode  %{
 5061     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
 5062   %}
 5063   ins_pipe(pipe_slow); // XXX
 5064 %}
 5065 
 5066 // Prefetch instructions for allocation.
 5067 // Must be safe to execute with invalid address (cannot fault).
 5068 
 5069 instruct prefetchAlloc( memory mem ) %{
 5070   predicate(AllocatePrefetchInstr==3);
 5071   match(PrefetchAllocation mem);
 5072   ins_cost(125);
 5073 
 5074   format %{ "PREFETCHW $mem\t# Prefetch allocation into level 1 cache and mark modified" %}
 5075   ins_encode %{
 5076     __ prefetchw($mem$$Address);
 5077   %}
 5078   ins_pipe(ialu_mem);
 5079 %}
 5080 
 5081 instruct prefetchAllocNTA( memory mem ) %{
 5082   predicate(AllocatePrefetchInstr==0);
 5083   match(PrefetchAllocation mem);
 5084   ins_cost(125);
 5085 
 5086   format %{ "PREFETCHNTA $mem\t# Prefetch allocation to non-temporal cache for write" %}
 5087   ins_encode %{
 5088     __ prefetchnta($mem$$Address);
 5089   %}
 5090   ins_pipe(ialu_mem);
 5091 %}
 5092 
 5093 instruct prefetchAllocT0( memory mem ) %{
 5094   predicate(AllocatePrefetchInstr==1);
 5095   match(PrefetchAllocation mem);
 5096   ins_cost(125);
 5097 
 5098   format %{ "PREFETCHT0 $mem\t# Prefetch allocation to level 1 and 2 caches for write" %}
 5099   ins_encode %{
 5100     __ prefetcht0($mem$$Address);
 5101   %}
 5102   ins_pipe(ialu_mem);
 5103 %}
 5104 
 5105 instruct prefetchAllocT2( memory mem ) %{
 5106   predicate(AllocatePrefetchInstr==2);
 5107   match(PrefetchAllocation mem);
 5108   ins_cost(125);
 5109 
 5110   format %{ "PREFETCHT2 $mem\t# Prefetch allocation to level 2 cache for write" %}
 5111   ins_encode %{
 5112     __ prefetcht2($mem$$Address);
 5113   %}
 5114   ins_pipe(ialu_mem);
 5115 %}
 5116 
 5117 //----------Store Instructions-------------------------------------------------
 5118 
 5119 // Store Byte
 5120 instruct storeB(memory mem, rRegI src)
 5121 %{
 5122   match(Set mem (StoreB mem src));
 5123 
 5124   ins_cost(125); // XXX
 5125   format %{ "movb    $mem, $src\t# byte" %}
 5126   ins_encode %{
 5127     __ movb($mem$$Address, $src$$Register);
 5128   %}
 5129   ins_pipe(ialu_mem_reg);
 5130 %}
 5131 
 5132 // Store Char/Short
 5133 instruct storeC(memory mem, rRegI src)
 5134 %{
 5135   match(Set mem (StoreC mem src));
 5136 
 5137   ins_cost(125); // XXX
 5138   format %{ "movw    $mem, $src\t# char/short" %}
 5139   ins_encode %{
 5140     __ movw($mem$$Address, $src$$Register);
 5141   %}
 5142   ins_pipe(ialu_mem_reg);
 5143 %}
 5144 
 5145 // Store Integer
 5146 instruct storeI(memory mem, rRegI src)
 5147 %{
 5148   match(Set mem (StoreI mem src));
 5149 
 5150   ins_cost(125); // XXX
 5151   format %{ "movl    $mem, $src\t# int" %}
 5152   ins_encode %{
 5153     __ movl($mem$$Address, $src$$Register);
 5154   %}
 5155   ins_pipe(ialu_mem_reg);
 5156 %}
 5157 
 5158 // Store Long
 5159 instruct storeL(memory mem, rRegL src)
 5160 %{
 5161   match(Set mem (StoreL mem src));
 5162 
 5163   ins_cost(125); // XXX
 5164   format %{ "movq    $mem, $src\t# long" %}
 5165   ins_encode %{
 5166     __ movq($mem$$Address, $src$$Register);
 5167   %}
 5168   ins_pipe(ialu_mem_reg); // XXX
 5169 %}
 5170 
 5171 // Store Pointer
 5172 instruct storeP(memory mem, any_RegP src)
 5173 %{
 5174   predicate(n->as_Store()->barrier_data() == 0);
 5175   match(Set mem (StoreP mem src));
 5176 
 5177   ins_cost(125); // XXX
 5178   format %{ "movq    $mem, $src\t# ptr" %}
 5179   ins_encode %{
 5180     __ movq($mem$$Address, $src$$Register);
 5181   %}
 5182   ins_pipe(ialu_mem_reg);
 5183 %}
 5184 
 5185 instruct storeImmP0(memory mem, immP0 zero)
 5186 %{
 5187   predicate(UseCompressedOops && (CompressedOops::base() == nullptr) && n->as_Store()->barrier_data() == 0);
 5188   match(Set mem (StoreP mem zero));
 5189 
 5190   ins_cost(125); // XXX
 5191   format %{ "movq    $mem, R12\t# ptr (R12_heapbase==0)" %}
 5192   ins_encode %{
 5193     __ movq($mem$$Address, r12);
 5194   %}
 5195   ins_pipe(ialu_mem_reg);
 5196 %}
 5197 
 5198 // Store Null Pointer, mark word, or other simple pointer constant.
 5199 instruct storeImmP(memory mem, immP31 src)
 5200 %{
 5201   predicate(n->as_Store()->barrier_data() == 0);
 5202   match(Set mem (StoreP mem src));
 5203 
 5204   ins_cost(150); // XXX
 5205   format %{ "movq    $mem, $src\t# ptr" %}
 5206   ins_encode %{
 5207     __ movq($mem$$Address, $src$$constant);
 5208   %}
 5209   ins_pipe(ialu_mem_imm);
 5210 %}
 5211 
 5212 // Store Compressed Pointer
 5213 instruct storeN(memory mem, rRegN src)
 5214 %{
 5215   predicate(n->as_Store()->barrier_data() == 0);
 5216   match(Set mem (StoreN mem src));
 5217 
 5218   ins_cost(125); // XXX
 5219   format %{ "movl    $mem, $src\t# compressed ptr" %}
 5220   ins_encode %{
 5221     __ movl($mem$$Address, $src$$Register);
 5222   %}
 5223   ins_pipe(ialu_mem_reg);
 5224 %}
 5225 
 5226 instruct storeNKlass(memory mem, rRegN src)
 5227 %{
 5228   match(Set mem (StoreNKlass mem src));
 5229 
 5230   ins_cost(125); // XXX
 5231   format %{ "movl    $mem, $src\t# compressed klass ptr" %}
 5232   ins_encode %{
 5233     __ movl($mem$$Address, $src$$Register);
 5234   %}
 5235   ins_pipe(ialu_mem_reg);
 5236 %}
 5237 
 5238 instruct storeImmN0(memory mem, immN0 zero)
 5239 %{
 5240   predicate(CompressedOops::base() == nullptr && n->as_Store()->barrier_data() == 0);
 5241   match(Set mem (StoreN mem zero));
 5242 
 5243   ins_cost(125); // XXX
 5244   format %{ "movl    $mem, R12\t# compressed ptr (R12_heapbase==0)" %}
 5245   ins_encode %{
 5246     __ movl($mem$$Address, r12);
 5247   %}
 5248   ins_pipe(ialu_mem_reg);
 5249 %}
 5250 
 5251 instruct storeImmN(memory mem, immN src)
 5252 %{
 5253   predicate(n->as_Store()->barrier_data() == 0);
 5254   match(Set mem (StoreN mem src));
 5255 
 5256   ins_cost(150); // XXX
 5257   format %{ "movl    $mem, $src\t# compressed ptr" %}
 5258   ins_encode %{
 5259     address con = (address)$src$$constant;
 5260     if (con == nullptr) {
 5261       __ movl($mem$$Address, 0);
 5262     } else {
 5263       __ set_narrow_oop($mem$$Address, (jobject)$src$$constant);
 5264     }
 5265   %}
 5266   ins_pipe(ialu_mem_imm);
 5267 %}
 5268 
 5269 instruct storeImmNKlass(memory mem, immNKlass src)
 5270 %{
 5271   match(Set mem (StoreNKlass mem src));
 5272 
 5273   ins_cost(150); // XXX
 5274   format %{ "movl    $mem, $src\t# compressed klass ptr" %}
 5275   ins_encode %{
 5276     __ set_narrow_klass($mem$$Address, (Klass*)$src$$constant);
 5277   %}
 5278   ins_pipe(ialu_mem_imm);
 5279 %}
 5280 
 5281 // Store Integer Immediate
 5282 instruct storeImmI0(memory mem, immI_0 zero)
 5283 %{
 5284   predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
 5285   match(Set mem (StoreI mem zero));
 5286 
 5287   ins_cost(125); // XXX
 5288   format %{ "movl    $mem, R12\t# int (R12_heapbase==0)" %}
 5289   ins_encode %{
 5290     __ movl($mem$$Address, r12);
 5291   %}
 5292   ins_pipe(ialu_mem_reg);
 5293 %}
 5294 
 5295 instruct storeImmI(memory mem, immI src)
 5296 %{
 5297   match(Set mem (StoreI mem src));
 5298 
 5299   ins_cost(150);
 5300   format %{ "movl    $mem, $src\t# int" %}
 5301   ins_encode %{
 5302     __ movl($mem$$Address, $src$$constant);
 5303   %}
 5304   ins_pipe(ialu_mem_imm);
 5305 %}
 5306 
 5307 // Store Long Immediate
 5308 instruct storeImmL0(memory mem, immL0 zero)
 5309 %{
 5310   predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
 5311   match(Set mem (StoreL mem zero));
 5312 
 5313   ins_cost(125); // XXX
 5314   format %{ "movq    $mem, R12\t# long (R12_heapbase==0)" %}
 5315   ins_encode %{
 5316     __ movq($mem$$Address, r12);
 5317   %}
 5318   ins_pipe(ialu_mem_reg);
 5319 %}
 5320 
 5321 instruct storeImmL(memory mem, immL32 src)
 5322 %{
 5323   match(Set mem (StoreL mem src));
 5324 
 5325   ins_cost(150);
 5326   format %{ "movq    $mem, $src\t# long" %}
 5327   ins_encode %{
 5328     __ movq($mem$$Address, $src$$constant);
 5329   %}
 5330   ins_pipe(ialu_mem_imm);
 5331 %}
 5332 
 5333 // Store Short/Char Immediate
 5334 instruct storeImmC0(memory mem, immI_0 zero)
 5335 %{
 5336   predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
 5337   match(Set mem (StoreC mem zero));
 5338 
 5339   ins_cost(125); // XXX
 5340   format %{ "movw    $mem, R12\t# short/char (R12_heapbase==0)" %}
 5341   ins_encode %{
 5342     __ movw($mem$$Address, r12);
 5343   %}
 5344   ins_pipe(ialu_mem_reg);
 5345 %}
 5346 
 5347 instruct storeImmI16(memory mem, immI16 src)
 5348 %{
 5349   predicate(UseStoreImmI16);
 5350   match(Set mem (StoreC mem src));
 5351 
 5352   ins_cost(150);
 5353   format %{ "movw    $mem, $src\t# short/char" %}
 5354   ins_encode %{
 5355     __ movw($mem$$Address, $src$$constant);
 5356   %}
 5357   ins_pipe(ialu_mem_imm);
 5358 %}
 5359 
 5360 // Store Byte Immediate
 5361 instruct storeImmB0(memory mem, immI_0 zero)
 5362 %{
 5363   predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
 5364   match(Set mem (StoreB mem zero));
 5365 
 5366   ins_cost(125); // XXX
 5367   format %{ "movb    $mem, R12\t# short/char (R12_heapbase==0)" %}
 5368   ins_encode %{
 5369     __ movb($mem$$Address, r12);
 5370   %}
 5371   ins_pipe(ialu_mem_reg);
 5372 %}
 5373 
 5374 instruct storeImmB(memory mem, immI8 src)
 5375 %{
 5376   match(Set mem (StoreB mem src));
 5377 
 5378   ins_cost(150); // XXX
 5379   format %{ "movb    $mem, $src\t# byte" %}
 5380   ins_encode %{
 5381     __ movb($mem$$Address, $src$$constant);
 5382   %}
 5383   ins_pipe(ialu_mem_imm);
 5384 %}
 5385 
 5386 // Store Float
 5387 instruct storeF(memory mem, regF src)
 5388 %{
 5389   match(Set mem (StoreF mem src));
 5390 
 5391   ins_cost(95); // XXX
 5392   format %{ "movss   $mem, $src\t# float" %}
 5393   ins_encode %{
 5394     __ movflt($mem$$Address, $src$$XMMRegister);
 5395   %}
 5396   ins_pipe(pipe_slow); // XXX
 5397 %}
 5398 
 5399 // Store immediate Float value (it is faster than store from XMM register)
 5400 instruct storeF0(memory mem, immF0 zero)
 5401 %{
 5402   predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
 5403   match(Set mem (StoreF mem zero));
 5404 
 5405   ins_cost(25); // XXX
 5406   format %{ "movl    $mem, R12\t# float 0. (R12_heapbase==0)" %}
 5407   ins_encode %{
 5408     __ movl($mem$$Address, r12);
 5409   %}
 5410   ins_pipe(ialu_mem_reg);
 5411 %}
 5412 
 5413 instruct storeF_imm(memory mem, immF src)
 5414 %{
 5415   match(Set mem (StoreF mem src));
 5416 
 5417   ins_cost(50);
 5418   format %{ "movl    $mem, $src\t# float" %}
 5419   ins_encode %{
 5420     __ movl($mem$$Address, jint_cast($src$$constant));
 5421   %}
 5422   ins_pipe(ialu_mem_imm);
 5423 %}
 5424 
 5425 // Store Double
 5426 instruct storeD(memory mem, regD src)
 5427 %{
 5428   match(Set mem (StoreD mem src));
 5429 
 5430   ins_cost(95); // XXX
 5431   format %{ "movsd   $mem, $src\t# double" %}
 5432   ins_encode %{
 5433     __ movdbl($mem$$Address, $src$$XMMRegister);
 5434   %}
 5435   ins_pipe(pipe_slow); // XXX
 5436 %}
 5437 
 5438 // Store immediate double 0.0 (it is faster than store from XMM register)
 5439 instruct storeD0_imm(memory mem, immD0 src)
 5440 %{
 5441   predicate(!UseCompressedOops || (CompressedOops::base() != nullptr));
 5442   match(Set mem (StoreD mem src));
 5443 
 5444   ins_cost(50);
 5445   format %{ "movq    $mem, $src\t# double 0." %}
 5446   ins_encode %{
 5447     __ movq($mem$$Address, $src$$constant);
 5448   %}
 5449   ins_pipe(ialu_mem_imm);
 5450 %}
 5451 
 5452 instruct storeD0(memory mem, immD0 zero)
 5453 %{
 5454   predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
 5455   match(Set mem (StoreD mem zero));
 5456 
 5457   ins_cost(25); // XXX
 5458   format %{ "movq    $mem, R12\t# double 0. (R12_heapbase==0)" %}
 5459   ins_encode %{
 5460     __ movq($mem$$Address, r12);
 5461   %}
 5462   ins_pipe(ialu_mem_reg);
 5463 %}
 5464 
 5465 instruct storeSSI(stackSlotI dst, rRegI src)
 5466 %{
 5467   match(Set dst src);
 5468 
 5469   ins_cost(100);
 5470   format %{ "movl    $dst, $src\t# int stk" %}
 5471   ins_encode %{
 5472     __ movl($dst$$Address, $src$$Register);
 5473   %}
 5474   ins_pipe( ialu_mem_reg );
 5475 %}
 5476 
 5477 instruct storeSSL(stackSlotL dst, rRegL src)
 5478 %{
 5479   match(Set dst src);
 5480 
 5481   ins_cost(100);
 5482   format %{ "movq    $dst, $src\t# long stk" %}
 5483   ins_encode %{
 5484     __ movq($dst$$Address, $src$$Register);
 5485   %}
 5486   ins_pipe(ialu_mem_reg);
 5487 %}
 5488 
 5489 instruct storeSSP(stackSlotP dst, rRegP src)
 5490 %{
 5491   match(Set dst src);
 5492 
 5493   ins_cost(100);
 5494   format %{ "movq    $dst, $src\t# ptr stk" %}
 5495   ins_encode %{
 5496     __ movq($dst$$Address, $src$$Register);
 5497   %}
 5498   ins_pipe(ialu_mem_reg);
 5499 %}
 5500 
 5501 instruct storeSSF(stackSlotF dst, regF src)
 5502 %{
 5503   match(Set dst src);
 5504 
 5505   ins_cost(95); // XXX
 5506   format %{ "movss   $dst, $src\t# float stk" %}
 5507   ins_encode %{
 5508     __ movflt(Address(rsp, $dst$$disp), $src$$XMMRegister);
 5509   %}
 5510   ins_pipe(pipe_slow); // XXX
 5511 %}
 5512 
 5513 instruct storeSSD(stackSlotD dst, regD src)
 5514 %{
 5515   match(Set dst src);
 5516 
 5517   ins_cost(95); // XXX
 5518   format %{ "movsd   $dst, $src\t# double stk" %}
 5519   ins_encode %{
 5520     __ movdbl(Address(rsp, $dst$$disp), $src$$XMMRegister);
 5521   %}
 5522   ins_pipe(pipe_slow); // XXX
 5523 %}
 5524 
 5525 instruct cacheWB(indirect addr)
 5526 %{
 5527   predicate(VM_Version::supports_data_cache_line_flush());
 5528   match(CacheWB addr);
 5529 
 5530   ins_cost(100);
 5531   format %{"cache wb $addr" %}
 5532   ins_encode %{
 5533     assert($addr->index_position() < 0, "should be");
 5534     assert($addr$$disp == 0, "should be");
 5535     __ cache_wb(Address($addr$$base$$Register, 0));
 5536   %}
 5537   ins_pipe(pipe_slow); // XXX
 5538 %}
 5539 
 5540 instruct cacheWBPreSync()
 5541 %{
 5542   predicate(VM_Version::supports_data_cache_line_flush());
 5543   match(CacheWBPreSync);
 5544 
 5545   ins_cost(100);
 5546   format %{"cache wb presync" %}
 5547   ins_encode %{
 5548     __ cache_wbsync(true);
 5549   %}
 5550   ins_pipe(pipe_slow); // XXX
 5551 %}
 5552 
 5553 instruct cacheWBPostSync()
 5554 %{
 5555   predicate(VM_Version::supports_data_cache_line_flush());
 5556   match(CacheWBPostSync);
 5557 
 5558   ins_cost(100);
 5559   format %{"cache wb postsync" %}
 5560   ins_encode %{
 5561     __ cache_wbsync(false);
 5562   %}
 5563   ins_pipe(pipe_slow); // XXX
 5564 %}
 5565 
 5566 //----------BSWAP Instructions-------------------------------------------------
 5567 instruct bytes_reverse_int(rRegI dst) %{
 5568   match(Set dst (ReverseBytesI dst));
 5569 
 5570   format %{ "bswapl  $dst" %}
 5571   ins_encode %{
 5572     __ bswapl($dst$$Register);
 5573   %}
 5574   ins_pipe( ialu_reg );
 5575 %}
 5576 
 5577 instruct bytes_reverse_long(rRegL dst) %{
 5578   match(Set dst (ReverseBytesL dst));
 5579 
 5580   format %{ "bswapq  $dst" %}
 5581   ins_encode %{
 5582     __ bswapq($dst$$Register);
 5583   %}
 5584   ins_pipe( ialu_reg);
 5585 %}
 5586 
 5587 instruct bytes_reverse_unsigned_short(rRegI dst, rFlagsReg cr) %{
 5588   match(Set dst (ReverseBytesUS dst));
 5589   effect(KILL cr);
 5590 
 5591   format %{ "bswapl  $dst\n\t"
 5592             "shrl    $dst,16\n\t" %}
 5593   ins_encode %{
 5594     __ bswapl($dst$$Register);
 5595     __ shrl($dst$$Register, 16);
 5596   %}
 5597   ins_pipe( ialu_reg );
 5598 %}
 5599 
 5600 instruct bytes_reverse_short(rRegI dst, rFlagsReg cr) %{
 5601   match(Set dst (ReverseBytesS dst));
 5602   effect(KILL cr);
 5603 
 5604   format %{ "bswapl  $dst\n\t"
 5605             "sar     $dst,16\n\t" %}
 5606   ins_encode %{
 5607     __ bswapl($dst$$Register);
 5608     __ sarl($dst$$Register, 16);
 5609   %}
 5610   ins_pipe( ialu_reg );
 5611 %}
 5612 
 5613 //---------- Zeros Count Instructions ------------------------------------------
 5614 
 5615 instruct countLeadingZerosI(rRegI dst, rRegI src, rFlagsReg cr) %{
 5616   predicate(UseCountLeadingZerosInstruction);
 5617   match(Set dst (CountLeadingZerosI src));
 5618   effect(KILL cr);
 5619 
 5620   format %{ "lzcntl  $dst, $src\t# count leading zeros (int)" %}
 5621   ins_encode %{
 5622     __ lzcntl($dst$$Register, $src$$Register);
 5623   %}
 5624   ins_pipe(ialu_reg);
 5625 %}
 5626 
 5627 instruct countLeadingZerosI_mem(rRegI dst, memory src, rFlagsReg cr) %{
 5628   predicate(UseCountLeadingZerosInstruction);
 5629   match(Set dst (CountLeadingZerosI (LoadI src)));
 5630   effect(KILL cr);
 5631   ins_cost(175);
 5632   format %{ "lzcntl  $dst, $src\t# count leading zeros (int)" %}
 5633   ins_encode %{
 5634     __ lzcntl($dst$$Register, $src$$Address);
 5635   %}
 5636   ins_pipe(ialu_reg_mem);
 5637 %}
 5638 
 5639 instruct countLeadingZerosI_bsr(rRegI dst, rRegI src, rFlagsReg cr) %{
 5640   predicate(!UseCountLeadingZerosInstruction);
 5641   match(Set dst (CountLeadingZerosI src));
 5642   effect(KILL cr);
 5643 
 5644   format %{ "bsrl    $dst, $src\t# count leading zeros (int)\n\t"
 5645             "jnz     skip\n\t"
 5646             "movl    $dst, -1\n"
 5647       "skip:\n\t"
 5648             "negl    $dst\n\t"
 5649             "addl    $dst, 31" %}
 5650   ins_encode %{
 5651     Register Rdst = $dst$$Register;
 5652     Register Rsrc = $src$$Register;
 5653     Label skip;
 5654     __ bsrl(Rdst, Rsrc);
 5655     __ jccb(Assembler::notZero, skip);
 5656     __ movl(Rdst, -1);
 5657     __ bind(skip);
 5658     __ negl(Rdst);
 5659     __ addl(Rdst, BitsPerInt - 1);
 5660   %}
 5661   ins_pipe(ialu_reg);
 5662 %}
 5663 
 5664 instruct countLeadingZerosL(rRegI dst, rRegL src, rFlagsReg cr) %{
 5665   predicate(UseCountLeadingZerosInstruction);
 5666   match(Set dst (CountLeadingZerosL src));
 5667   effect(KILL cr);
 5668 
 5669   format %{ "lzcntq  $dst, $src\t# count leading zeros (long)" %}
 5670   ins_encode %{
 5671     __ lzcntq($dst$$Register, $src$$Register);
 5672   %}
 5673   ins_pipe(ialu_reg);
 5674 %}
 5675 
 5676 instruct countLeadingZerosL_mem(rRegI dst, memory src, rFlagsReg cr) %{
 5677   predicate(UseCountLeadingZerosInstruction);
 5678   match(Set dst (CountLeadingZerosL (LoadL src)));
 5679   effect(KILL cr);
 5680   ins_cost(175);
 5681   format %{ "lzcntq  $dst, $src\t# count leading zeros (long)" %}
 5682   ins_encode %{
 5683     __ lzcntq($dst$$Register, $src$$Address);
 5684   %}
 5685   ins_pipe(ialu_reg_mem);
 5686 %}
 5687 
 5688 instruct countLeadingZerosL_bsr(rRegI dst, rRegL src, rFlagsReg cr) %{
 5689   predicate(!UseCountLeadingZerosInstruction);
 5690   match(Set dst (CountLeadingZerosL src));
 5691   effect(KILL cr);
 5692 
 5693   format %{ "bsrq    $dst, $src\t# count leading zeros (long)\n\t"
 5694             "jnz     skip\n\t"
 5695             "movl    $dst, -1\n"
 5696       "skip:\n\t"
 5697             "negl    $dst\n\t"
 5698             "addl    $dst, 63" %}
 5699   ins_encode %{
 5700     Register Rdst = $dst$$Register;
 5701     Register Rsrc = $src$$Register;
 5702     Label skip;
 5703     __ bsrq(Rdst, Rsrc);
 5704     __ jccb(Assembler::notZero, skip);
 5705     __ movl(Rdst, -1);
 5706     __ bind(skip);
 5707     __ negl(Rdst);
 5708     __ addl(Rdst, BitsPerLong - 1);
 5709   %}
 5710   ins_pipe(ialu_reg);
 5711 %}
 5712 
 5713 instruct countTrailingZerosI(rRegI dst, rRegI src, rFlagsReg cr) %{
 5714   predicate(UseCountTrailingZerosInstruction);
 5715   match(Set dst (CountTrailingZerosI src));
 5716   effect(KILL cr);
 5717 
 5718   format %{ "tzcntl    $dst, $src\t# count trailing zeros (int)" %}
 5719   ins_encode %{
 5720     __ tzcntl($dst$$Register, $src$$Register);
 5721   %}
 5722   ins_pipe(ialu_reg);
 5723 %}
 5724 
 5725 instruct countTrailingZerosI_mem(rRegI dst, memory src, rFlagsReg cr) %{
 5726   predicate(UseCountTrailingZerosInstruction);
 5727   match(Set dst (CountTrailingZerosI (LoadI src)));
 5728   effect(KILL cr);
 5729   ins_cost(175);
 5730   format %{ "tzcntl    $dst, $src\t# count trailing zeros (int)" %}
 5731   ins_encode %{
 5732     __ tzcntl($dst$$Register, $src$$Address);
 5733   %}
 5734   ins_pipe(ialu_reg_mem);
 5735 %}
 5736 
 5737 instruct countTrailingZerosI_bsf(rRegI dst, rRegI src, rFlagsReg cr) %{
 5738   predicate(!UseCountTrailingZerosInstruction);
 5739   match(Set dst (CountTrailingZerosI src));
 5740   effect(KILL cr);
 5741 
 5742   format %{ "bsfl    $dst, $src\t# count trailing zeros (int)\n\t"
 5743             "jnz     done\n\t"
 5744             "movl    $dst, 32\n"
 5745       "done:" %}
 5746   ins_encode %{
 5747     Register Rdst = $dst$$Register;
 5748     Label done;
 5749     __ bsfl(Rdst, $src$$Register);
 5750     __ jccb(Assembler::notZero, done);
 5751     __ movl(Rdst, BitsPerInt);
 5752     __ bind(done);
 5753   %}
 5754   ins_pipe(ialu_reg);
 5755 %}
 5756 
 5757 instruct countTrailingZerosL(rRegI dst, rRegL src, rFlagsReg cr) %{
 5758   predicate(UseCountTrailingZerosInstruction);
 5759   match(Set dst (CountTrailingZerosL src));
 5760   effect(KILL cr);
 5761 
 5762   format %{ "tzcntq    $dst, $src\t# count trailing zeros (long)" %}
 5763   ins_encode %{
 5764     __ tzcntq($dst$$Register, $src$$Register);
 5765   %}
 5766   ins_pipe(ialu_reg);
 5767 %}
 5768 
 5769 instruct countTrailingZerosL_mem(rRegI dst, memory src, rFlagsReg cr) %{
 5770   predicate(UseCountTrailingZerosInstruction);
 5771   match(Set dst (CountTrailingZerosL (LoadL src)));
 5772   effect(KILL cr);
 5773   ins_cost(175);
 5774   format %{ "tzcntq    $dst, $src\t# count trailing zeros (long)" %}
 5775   ins_encode %{
 5776     __ tzcntq($dst$$Register, $src$$Address);
 5777   %}
 5778   ins_pipe(ialu_reg_mem);
 5779 %}
 5780 
 5781 instruct countTrailingZerosL_bsf(rRegI dst, rRegL src, rFlagsReg cr) %{
 5782   predicate(!UseCountTrailingZerosInstruction);
 5783   match(Set dst (CountTrailingZerosL src));
 5784   effect(KILL cr);
 5785 
 5786   format %{ "bsfq    $dst, $src\t# count trailing zeros (long)\n\t"
 5787             "jnz     done\n\t"
 5788             "movl    $dst, 64\n"
 5789       "done:" %}
 5790   ins_encode %{
 5791     Register Rdst = $dst$$Register;
 5792     Label done;
 5793     __ bsfq(Rdst, $src$$Register);
 5794     __ jccb(Assembler::notZero, done);
 5795     __ movl(Rdst, BitsPerLong);
 5796     __ bind(done);
 5797   %}
 5798   ins_pipe(ialu_reg);
 5799 %}
 5800 
 5801 //--------------- Reverse Operation Instructions ----------------
 5802 instruct bytes_reversebit_int(rRegI dst, rRegI src, rRegI rtmp, rFlagsReg cr) %{
 5803   predicate(!VM_Version::supports_gfni());
 5804   match(Set dst (ReverseI src));
 5805   effect(TEMP dst, TEMP rtmp, KILL cr);
 5806   format %{ "reverse_int $dst $src\t! using $rtmp as TEMP" %}
 5807   ins_encode %{
 5808     __ reverseI($dst$$Register, $src$$Register, xnoreg, xnoreg, $rtmp$$Register);
 5809   %}
 5810   ins_pipe( ialu_reg );
 5811 %}
 5812 
 5813 instruct bytes_reversebit_int_gfni(rRegI dst, rRegI src, vlRegF xtmp1, vlRegF xtmp2, rRegL rtmp, rFlagsReg cr) %{
 5814   predicate(VM_Version::supports_gfni());
 5815   match(Set dst (ReverseI src));
 5816   effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP rtmp, KILL cr);
 5817   format %{ "reverse_int $dst $src\t! using $rtmp, $xtmp1 and $xtmp2 as TEMP" %}
 5818   ins_encode %{
 5819     __ reverseI($dst$$Register, $src$$Register, $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, $rtmp$$Register);
 5820   %}
 5821   ins_pipe( ialu_reg );
 5822 %}
 5823 
 5824 instruct bytes_reversebit_long(rRegL dst, rRegL src, rRegL rtmp1, rRegL rtmp2, rFlagsReg cr) %{
 5825   predicate(!VM_Version::supports_gfni());
 5826   match(Set dst (ReverseL src));
 5827   effect(TEMP dst, TEMP rtmp1, TEMP rtmp2, KILL cr);
 5828   format %{ "reverse_long $dst $src\t! using $rtmp1 and $rtmp2 as TEMP" %}
 5829   ins_encode %{
 5830     __ reverseL($dst$$Register, $src$$Register, xnoreg, xnoreg, $rtmp1$$Register, $rtmp2$$Register);
 5831   %}
 5832   ins_pipe( ialu_reg );
 5833 %}
 5834 
 5835 instruct bytes_reversebit_long_gfni(rRegL dst, rRegL src, vlRegD xtmp1, vlRegD xtmp2, rRegL rtmp, rFlagsReg cr) %{
 5836   predicate(VM_Version::supports_gfni());
 5837   match(Set dst (ReverseL src));
 5838   effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP rtmp, KILL cr);
 5839   format %{ "reverse_long $dst $src\t! using $rtmp, $xtmp1 and $xtmp2 as TEMP" %}
 5840   ins_encode %{
 5841     __ reverseL($dst$$Register, $src$$Register, $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, $rtmp$$Register, noreg);
 5842   %}
 5843   ins_pipe( ialu_reg );
 5844 %}
 5845 
 5846 //---------- Population Count Instructions -------------------------------------
 5847 
 5848 instruct popCountI(rRegI dst, rRegI src, rFlagsReg cr) %{
 5849   predicate(UsePopCountInstruction);
 5850   match(Set dst (PopCountI src));
 5851   effect(KILL cr);
 5852 
 5853   format %{ "popcnt  $dst, $src" %}
 5854   ins_encode %{
 5855     __ popcntl($dst$$Register, $src$$Register);
 5856   %}
 5857   ins_pipe(ialu_reg);
 5858 %}
 5859 
 5860 instruct popCountI_mem(rRegI dst, memory mem, rFlagsReg cr) %{
 5861   predicate(UsePopCountInstruction);
 5862   match(Set dst (PopCountI (LoadI mem)));
 5863   effect(KILL cr);
 5864 
 5865   format %{ "popcnt  $dst, $mem" %}
 5866   ins_encode %{
 5867     __ popcntl($dst$$Register, $mem$$Address);
 5868   %}
 5869   ins_pipe(ialu_reg);
 5870 %}
 5871 
 5872 // Note: Long.bitCount(long) returns an int.
 5873 instruct popCountL(rRegI dst, rRegL src, rFlagsReg cr) %{
 5874   predicate(UsePopCountInstruction);
 5875   match(Set dst (PopCountL src));
 5876   effect(KILL cr);
 5877 
 5878   format %{ "popcnt  $dst, $src" %}
 5879   ins_encode %{
 5880     __ popcntq($dst$$Register, $src$$Register);
 5881   %}
 5882   ins_pipe(ialu_reg);
 5883 %}
 5884 
 5885 // Note: Long.bitCount(long) returns an int.
 5886 instruct popCountL_mem(rRegI dst, memory mem, rFlagsReg cr) %{
 5887   predicate(UsePopCountInstruction);
 5888   match(Set dst (PopCountL (LoadL mem)));
 5889   effect(KILL cr);
 5890 
 5891   format %{ "popcnt  $dst, $mem" %}
 5892   ins_encode %{
 5893     __ popcntq($dst$$Register, $mem$$Address);
 5894   %}
 5895   ins_pipe(ialu_reg);
 5896 %}
 5897 
 5898 
 5899 //----------MemBar Instructions-----------------------------------------------
 5900 // Memory barrier flavors
 5901 
 5902 instruct membar_acquire()
 5903 %{
 5904   match(MemBarAcquire);
 5905   match(LoadFence);
 5906   ins_cost(0);
 5907 
 5908   size(0);
 5909   format %{ "MEMBAR-acquire ! (empty encoding)" %}
 5910   ins_encode();
 5911   ins_pipe(empty);
 5912 %}
 5913 
 5914 instruct membar_acquire_lock()
 5915 %{
 5916   match(MemBarAcquireLock);
 5917   ins_cost(0);
 5918 
 5919   size(0);
 5920   format %{ "MEMBAR-acquire (prior CMPXCHG in FastLock so empty encoding)" %}
 5921   ins_encode();
 5922   ins_pipe(empty);
 5923 %}
 5924 
 5925 instruct membar_release()
 5926 %{
 5927   match(MemBarRelease);
 5928   match(StoreFence);
 5929   ins_cost(0);
 5930 
 5931   size(0);
 5932   format %{ "MEMBAR-release ! (empty encoding)" %}
 5933   ins_encode();
 5934   ins_pipe(empty);
 5935 %}
 5936 
 5937 instruct membar_release_lock()
 5938 %{
 5939   match(MemBarReleaseLock);
 5940   ins_cost(0);
 5941 
 5942   size(0);
 5943   format %{ "MEMBAR-release (a FastUnlock follows so empty encoding)" %}
 5944   ins_encode();
 5945   ins_pipe(empty);
 5946 %}
 5947 
 5948 instruct membar_volatile(rFlagsReg cr) %{
 5949   match(MemBarVolatile);
 5950   effect(KILL cr);
 5951   ins_cost(400);
 5952 
 5953   format %{
 5954     $$template
 5955     $$emit$$"lock addl [rsp + #0], 0\t! membar_volatile"
 5956   %}
 5957   ins_encode %{
 5958     __ membar(Assembler::StoreLoad);
 5959   %}
 5960   ins_pipe(pipe_slow);
 5961 %}
 5962 
 5963 instruct unnecessary_membar_volatile()
 5964 %{
 5965   match(MemBarVolatile);
 5966   predicate(Matcher::post_store_load_barrier(n));
 5967   ins_cost(0);
 5968 
 5969   size(0);
 5970   format %{ "MEMBAR-volatile (unnecessary so empty encoding)" %}
 5971   ins_encode();
 5972   ins_pipe(empty);
 5973 %}
 5974 
 5975 instruct membar_storestore() %{
 5976   match(MemBarStoreStore);
 5977   match(StoreStoreFence);
 5978   ins_cost(0);
 5979 
 5980   size(0);
 5981   format %{ "MEMBAR-storestore (empty encoding)" %}
 5982   ins_encode( );
 5983   ins_pipe(empty);
 5984 %}
 5985 
 5986 //----------Move Instructions--------------------------------------------------
 5987 
 5988 instruct castX2P(rRegP dst, rRegL src)
 5989 %{
 5990   match(Set dst (CastX2P src));
 5991 
 5992   format %{ "movq    $dst, $src\t# long->ptr" %}
 5993   ins_encode %{
 5994     if ($dst$$reg != $src$$reg) {
 5995       __ movptr($dst$$Register, $src$$Register);
 5996     }
 5997   %}
 5998   ins_pipe(ialu_reg_reg); // XXX
 5999 %}
 6000 
 6001 instruct castP2X(rRegL dst, rRegP src)
 6002 %{
 6003   match(Set dst (CastP2X src));
 6004 
 6005   format %{ "movq    $dst, $src\t# ptr -> long" %}
 6006   ins_encode %{
 6007     if ($dst$$reg != $src$$reg) {
 6008       __ movptr($dst$$Register, $src$$Register);
 6009     }
 6010   %}
 6011   ins_pipe(ialu_reg_reg); // XXX
 6012 %}
 6013 
 6014 // Convert oop into int for vectors alignment masking
 6015 instruct convP2I(rRegI dst, rRegP src)
 6016 %{
 6017   match(Set dst (ConvL2I (CastP2X src)));
 6018 
 6019   format %{ "movl    $dst, $src\t# ptr -> int" %}
 6020   ins_encode %{
 6021     __ movl($dst$$Register, $src$$Register);
 6022   %}
 6023   ins_pipe(ialu_reg_reg); // XXX
 6024 %}
 6025 
 6026 // Convert compressed oop into int for vectors alignment masking
 6027 // in case of 32bit oops (heap < 4Gb).
 6028 instruct convN2I(rRegI dst, rRegN src)
 6029 %{
 6030   predicate(CompressedOops::shift() == 0);
 6031   match(Set dst (ConvL2I (CastP2X (DecodeN src))));
 6032 
 6033   format %{ "movl    $dst, $src\t# compressed ptr -> int" %}
 6034   ins_encode %{
 6035     __ movl($dst$$Register, $src$$Register);
 6036   %}
 6037   ins_pipe(ialu_reg_reg); // XXX
 6038 %}
 6039 
 6040 // Convert oop pointer into compressed form
 6041 instruct encodeHeapOop(rRegN dst, rRegP src, rFlagsReg cr) %{
 6042   predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
 6043   match(Set dst (EncodeP src));
 6044   effect(KILL cr);
 6045   format %{ "encode_heap_oop $dst,$src" %}
 6046   ins_encode %{
 6047     Register s = $src$$Register;
 6048     Register d = $dst$$Register;
 6049     if (s != d) {
 6050       __ movq(d, s);
 6051     }
 6052     __ encode_heap_oop(d);
 6053   %}
 6054   ins_pipe(ialu_reg_long);
 6055 %}
 6056 
 6057 instruct encodeHeapOop_not_null(rRegN dst, rRegP src, rFlagsReg cr) %{
 6058   predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
 6059   match(Set dst (EncodeP src));
 6060   effect(KILL cr);
 6061   format %{ "encode_heap_oop_not_null $dst,$src" %}
 6062   ins_encode %{
 6063     __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
 6064   %}
 6065   ins_pipe(ialu_reg_long);
 6066 %}
 6067 
 6068 instruct decodeHeapOop(rRegP dst, rRegN src, rFlagsReg cr) %{
 6069   predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
 6070             n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
 6071   match(Set dst (DecodeN src));
 6072   effect(KILL cr);
 6073   format %{ "decode_heap_oop $dst,$src" %}
 6074   ins_encode %{
 6075     Register s = $src$$Register;
 6076     Register d = $dst$$Register;
 6077     if (s != d) {
 6078       __ movq(d, s);
 6079     }
 6080     __ decode_heap_oop(d);
 6081   %}
 6082   ins_pipe(ialu_reg_long);
 6083 %}
 6084 
 6085 instruct decodeHeapOop_not_null(rRegP dst, rRegN src, rFlagsReg cr) %{
 6086   predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
 6087             n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
 6088   match(Set dst (DecodeN src));
 6089   effect(KILL cr);
 6090   format %{ "decode_heap_oop_not_null $dst,$src" %}
 6091   ins_encode %{
 6092     Register s = $src$$Register;
 6093     Register d = $dst$$Register;
 6094     if (s != d) {
 6095       __ decode_heap_oop_not_null(d, s);
 6096     } else {
 6097       __ decode_heap_oop_not_null(d);
 6098     }
 6099   %}
 6100   ins_pipe(ialu_reg_long);
 6101 %}
 6102 
 6103 instruct encodeKlass_not_null(rRegN dst, rRegP src, rFlagsReg cr) %{
 6104   match(Set dst (EncodePKlass src));
 6105   effect(TEMP dst, KILL cr);
 6106   format %{ "encode_and_move_klass_not_null $dst,$src" %}
 6107   ins_encode %{
 6108     __ encode_and_move_klass_not_null($dst$$Register, $src$$Register);
 6109   %}
 6110   ins_pipe(ialu_reg_long);
 6111 %}
 6112 
 6113 instruct decodeKlass_not_null(rRegP dst, rRegN src, rFlagsReg cr) %{
 6114   match(Set dst (DecodeNKlass src));
 6115   effect(TEMP dst, KILL cr);
 6116   format %{ "decode_and_move_klass_not_null $dst,$src" %}
 6117   ins_encode %{
 6118     __ decode_and_move_klass_not_null($dst$$Register, $src$$Register);
 6119   %}
 6120   ins_pipe(ialu_reg_long);
 6121 %}
 6122 
 6123 //----------Conditional Move---------------------------------------------------
 6124 // Jump
 6125 // dummy instruction for generating temp registers
 6126 instruct jumpXtnd_offset(rRegL switch_val, immI2 shift, rRegI dest) %{
 6127   match(Jump (LShiftL switch_val shift));
 6128   ins_cost(350);
 6129   predicate(false);
 6130   effect(TEMP dest);
 6131 
 6132   format %{ "leaq    $dest, [$constantaddress]\n\t"
 6133             "jmp     [$dest + $switch_val << $shift]\n\t" %}
 6134   ins_encode %{
 6135     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
 6136     // to do that and the compiler is using that register as one it can allocate.
 6137     // So we build it all by hand.
 6138     // Address index(noreg, switch_reg, (Address::ScaleFactor)$shift$$constant);
 6139     // ArrayAddress dispatch(table, index);
 6140     Address dispatch($dest$$Register, $switch_val$$Register, (Address::ScaleFactor) $shift$$constant);
 6141     __ lea($dest$$Register, $constantaddress);
 6142     __ jmp(dispatch);
 6143   %}
 6144   ins_pipe(pipe_jmp);
 6145 %}
 6146 
 6147 instruct jumpXtnd_addr(rRegL switch_val, immI2 shift, immL32 offset, rRegI dest) %{
 6148   match(Jump (AddL (LShiftL switch_val shift) offset));
 6149   ins_cost(350);
 6150   effect(TEMP dest);
 6151 
 6152   format %{ "leaq    $dest, [$constantaddress]\n\t"
 6153             "jmp     [$dest + $switch_val << $shift + $offset]\n\t" %}
 6154   ins_encode %{
 6155     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
 6156     // to do that and the compiler is using that register as one it can allocate.
 6157     // So we build it all by hand.
 6158     // Address index(noreg, switch_reg, (Address::ScaleFactor) $shift$$constant, (int) $offset$$constant);
 6159     // ArrayAddress dispatch(table, index);
 6160     Address dispatch($dest$$Register, $switch_val$$Register, (Address::ScaleFactor) $shift$$constant, (int) $offset$$constant);
 6161     __ lea($dest$$Register, $constantaddress);
 6162     __ jmp(dispatch);
 6163   %}
 6164   ins_pipe(pipe_jmp);
 6165 %}
 6166 
 6167 instruct jumpXtnd(rRegL switch_val, rRegI dest) %{
 6168   match(Jump switch_val);
 6169   ins_cost(350);
 6170   effect(TEMP dest);
 6171 
 6172   format %{ "leaq    $dest, [$constantaddress]\n\t"
 6173             "jmp     [$dest + $switch_val]\n\t" %}
 6174   ins_encode %{
 6175     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
 6176     // to do that and the compiler is using that register as one it can allocate.
 6177     // So we build it all by hand.
 6178     // Address index(noreg, switch_reg, Address::times_1);
 6179     // ArrayAddress dispatch(table, index);
 6180     Address dispatch($dest$$Register, $switch_val$$Register, Address::times_1);
 6181     __ lea($dest$$Register, $constantaddress);
 6182     __ jmp(dispatch);
 6183   %}
 6184   ins_pipe(pipe_jmp);
 6185 %}
 6186 
 6187 // Conditional move
 6188 instruct cmovI_imm_01(rRegI dst, immI_1 src, rFlagsReg cr, cmpOp cop)
 6189 %{
 6190   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_int() == 0);
 6191   match(Set dst (CMoveI (Binary cop cr) (Binary src dst)));
 6192 
 6193   ins_cost(100); // XXX
 6194   format %{ "setbn$cop $dst\t# signed, int" %}
 6195   ins_encode %{
 6196     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 6197     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 6198   %}
 6199   ins_pipe(ialu_reg);
 6200 %}
 6201 
 6202 instruct cmovI_reg(rRegI dst, rRegI src, rFlagsReg cr, cmpOp cop)
 6203 %{
 6204   predicate(!UseAPX);
 6205   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
 6206 
 6207   ins_cost(200); // XXX
 6208   format %{ "cmovl$cop $dst, $src\t# signed, int" %}
 6209   ins_encode %{
 6210     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 6211   %}
 6212   ins_pipe(pipe_cmov_reg);
 6213 %}
 6214 
 6215 instruct cmovI_reg_ndd(rRegI dst, rRegI src1, rRegI src2, rFlagsReg cr, cmpOp cop)
 6216 %{
 6217   predicate(UseAPX);
 6218   match(Set dst (CMoveI (Binary cop cr) (Binary src1 src2)));
 6219 
 6220   ins_cost(200);
 6221   format %{ "ecmovl$cop $dst, $src1, $src2\t# signed, int ndd" %}
 6222   ins_encode %{
 6223     __ ecmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
 6224   %}
 6225   ins_pipe(pipe_cmov_reg);
 6226 %}
 6227 
 6228 instruct cmovI_imm_01U(rRegI dst, immI_1 src, rFlagsRegU cr, cmpOpU cop)
 6229 %{
 6230   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_int() == 0);
 6231   match(Set dst (CMoveI (Binary cop cr) (Binary src dst)));
 6232 
 6233   ins_cost(100); // XXX
 6234   format %{ "setbn$cop $dst\t# unsigned, int" %}
 6235   ins_encode %{
 6236     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 6237     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 6238   %}
 6239   ins_pipe(ialu_reg);
 6240 %}
 6241 
 6242 instruct cmovI_regU(cmpOpU cop, rFlagsRegU cr, rRegI dst, rRegI src) %{
 6243   predicate(!UseAPX);
 6244   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
 6245 
 6246   ins_cost(200); // XXX
 6247   format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
 6248   ins_encode %{
 6249     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 6250   %}
 6251   ins_pipe(pipe_cmov_reg);
 6252 %}
 6253 
 6254 instruct cmovI_regU_ndd(rRegI dst, cmpOpU cop, rFlagsRegU cr, rRegI src1, rRegI src2) %{
 6255   predicate(UseAPX);
 6256   match(Set dst (CMoveI (Binary cop cr) (Binary src1 src2)));
 6257 
 6258   ins_cost(200);
 6259   format %{ "ecmovl$cop $dst, $src1, $src2\t# unsigned, int ndd" %}
 6260   ins_encode %{
 6261     __ ecmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
 6262   %}
 6263   ins_pipe(pipe_cmov_reg);
 6264 %}
 6265 
 6266 instruct cmovI_imm_01UCF(rRegI dst, immI_1 src, rFlagsRegUCF cr, cmpOpUCF cop)
 6267 %{
 6268   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_int() == 0);
 6269   match(Set dst (CMoveI (Binary cop cr) (Binary src dst)));
 6270 
 6271   ins_cost(100); // XXX
 6272   format %{ "setbn$cop $dst\t# unsigned, int" %}
 6273   ins_encode %{
 6274     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 6275     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 6276   %}
 6277   ins_pipe(ialu_reg);
 6278 %}
 6279 
 6280 instruct cmovI_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
 6281   predicate(!UseAPX);
 6282   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
 6283   ins_cost(200);
 6284   expand %{
 6285     cmovI_regU(cop, cr, dst, src);
 6286   %}
 6287 %}
 6288 
 6289 instruct cmovI_regUCF_ndd(rRegI dst, cmpOpUCF cop, rFlagsRegUCF cr, rRegI src1, rRegI src2) %{
 6290   predicate(UseAPX);
 6291   match(Set dst (CMoveI (Binary cop cr) (Binary src1 src2)));
 6292   ins_cost(200);
 6293   format %{ "ecmovl$cop $dst, $src1, $src2\t# unsigned, int ndd" %}
 6294   ins_encode %{
 6295     __ ecmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
 6296   %}
 6297   ins_pipe(pipe_cmov_reg);
 6298 %}
 6299 
 6300 instruct cmovI_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
 6301   predicate(!UseAPX && n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
 6302   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
 6303 
 6304   ins_cost(200); // XXX
 6305   format %{ "cmovpl  $dst, $src\n\t"
 6306             "cmovnel $dst, $src" %}
 6307   ins_encode %{
 6308     __ cmovl(Assembler::parity, $dst$$Register, $src$$Register);
 6309     __ cmovl(Assembler::notEqual, $dst$$Register, $src$$Register);
 6310   %}
 6311   ins_pipe(pipe_cmov_reg);
 6312 %}
 6313 
 6314 instruct cmovI_regUCF2_ne_ndd(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegI dst, rRegI src1, rRegI src2) %{
 6315   predicate(UseAPX && n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
 6316   match(Set dst (CMoveI (Binary cop cr) (Binary src1 src2)));
 6317   effect(TEMP dst);
 6318 
 6319   ins_cost(200);
 6320   format %{ "ecmovpl  $dst, $src1, $src2\n\t"
 6321             "cmovnel  $dst, $src2" %}
 6322   ins_encode %{
 6323     __ ecmovl(Assembler::parity, $dst$$Register, $src1$$Register, $src2$$Register);
 6324     __ cmovl(Assembler::notEqual, $dst$$Register, $src2$$Register);
 6325   %}
 6326   ins_pipe(pipe_cmov_reg);
 6327 %}
 6328 
 6329 // Since (x == y) == !(x != y), we can flip the sense of the test by flipping the
 6330 // inputs of the CMove
 6331 instruct cmovI_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
 6332   predicate(!UseAPX && n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
 6333   match(Set dst (CMoveI (Binary cop cr) (Binary src dst)));
 6334   effect(TEMP dst);
 6335 
 6336   ins_cost(200); // XXX
 6337   format %{ "cmovpl  $dst, $src\n\t"
 6338             "cmovnel $dst, $src" %}
 6339   ins_encode %{
 6340     __ cmovl(Assembler::parity, $dst$$Register, $src$$Register);
 6341     __ cmovl(Assembler::notEqual, $dst$$Register, $src$$Register);
 6342   %}
 6343   ins_pipe(pipe_cmov_reg);
 6344 %}
 6345 
 6346 // We need this special handling for only eq / neq comparison since NaN == NaN is false,
 6347 // and parity flag bit is set if any of the operand is a NaN.
 6348 instruct cmovI_regUCF2_eq_ndd(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegI dst, rRegI src1, rRegI src2) %{
 6349   predicate(UseAPX && n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
 6350   match(Set dst (CMoveI (Binary cop cr) (Binary src2 src1)));
 6351   effect(TEMP dst);
 6352 
 6353   ins_cost(200);
 6354   format %{ "ecmovpl  $dst, $src1, $src2\n\t"
 6355             "cmovnel  $dst, $src2" %}
 6356   ins_encode %{
 6357     __ ecmovl(Assembler::parity, $dst$$Register, $src1$$Register, $src2$$Register);
 6358     __ cmovl(Assembler::notEqual, $dst$$Register, $src2$$Register);
 6359   %}
 6360   ins_pipe(pipe_cmov_reg);
 6361 %}
 6362 
 6363 // Conditional move
 6364 instruct cmovI_mem(cmpOp cop, rFlagsReg cr, rRegI dst, memory src) %{
 6365   predicate(!UseAPX);
 6366   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
 6367 
 6368   ins_cost(250); // XXX
 6369   format %{ "cmovl$cop $dst, $src\t# signed, int" %}
 6370   ins_encode %{
 6371     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
 6372   %}
 6373   ins_pipe(pipe_cmov_mem);
 6374 %}
 6375 
 6376 // Conditional move
 6377 instruct cmovI_rReg_rReg_mem_ndd(rRegI dst, cmpOp cop, rFlagsReg cr, rRegI src1, memory src2)
 6378 %{
 6379   predicate(UseAPX);
 6380   match(Set dst (CMoveI (Binary cop cr) (Binary src1 (LoadI src2))));
 6381 
 6382   ins_cost(250);
 6383   format %{ "ecmovl$cop $dst, $src1, $src2\t# signed, int ndd" %}
 6384   ins_encode %{
 6385     __ ecmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Address);
 6386   %}
 6387   ins_pipe(pipe_cmov_mem);
 6388 %}
 6389 
 6390 // Conditional move
 6391 instruct cmovI_memU(cmpOpU cop, rFlagsRegU cr, rRegI dst, memory src)
 6392 %{
 6393   predicate(!UseAPX);
 6394   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
 6395 
 6396   ins_cost(250); // XXX
 6397   format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
 6398   ins_encode %{
 6399     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
 6400   %}
 6401   ins_pipe(pipe_cmov_mem);
 6402 %}
 6403 
 6404 instruct cmovI_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, memory src) %{
 6405   predicate(!UseAPX);
 6406   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
 6407   ins_cost(250);
 6408   expand %{
 6409     cmovI_memU(cop, cr, dst, src);
 6410   %}
 6411 %}
 6412 
 6413 instruct cmovI_rReg_rReg_memU_ndd(rRegI dst, cmpOpU cop, rFlagsRegU cr, rRegI src1, memory src2)
 6414 %{
 6415   predicate(UseAPX);
 6416   match(Set dst (CMoveI (Binary cop cr) (Binary src1 (LoadI src2))));
 6417 
 6418   ins_cost(250);
 6419   format %{ "ecmovl$cop $dst, $src1, $src2\t# unsigned, int ndd" %}
 6420   ins_encode %{
 6421     __ ecmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Address);
 6422   %}
 6423   ins_pipe(pipe_cmov_mem);
 6424 %}
 6425 
 6426 instruct cmovI_rReg_rReg_memUCF_ndd(rRegI dst, cmpOpUCF cop, rFlagsRegUCF cr, rRegI src1, memory src2)
 6427 %{
 6428   predicate(UseAPX);
 6429   match(Set dst (CMoveI (Binary cop cr) (Binary src1 (LoadI src2))));
 6430   ins_cost(250);
 6431   format %{ "ecmovl$cop $dst, $src1, $src2\t# unsigned, int ndd" %}
 6432   ins_encode %{
 6433     __ ecmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Address);
 6434   %}
 6435   ins_pipe(pipe_cmov_mem);
 6436 %}
 6437 
 6438 // Conditional move
 6439 instruct cmovN_reg(rRegN dst, rRegN src, rFlagsReg cr, cmpOp cop)
 6440 %{
 6441   predicate(!UseAPX);
 6442   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
 6443 
 6444   ins_cost(200); // XXX
 6445   format %{ "cmovl$cop $dst, $src\t# signed, compressed ptr" %}
 6446   ins_encode %{
 6447     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 6448   %}
 6449   ins_pipe(pipe_cmov_reg);
 6450 %}
 6451 
 6452 // Conditional move ndd
 6453 instruct cmovN_reg_ndd(rRegN dst, rRegN src1, rRegN src2, rFlagsReg cr, cmpOp cop)
 6454 %{
 6455   predicate(UseAPX);
 6456   match(Set dst (CMoveN (Binary cop cr) (Binary src1 src2)));
 6457 
 6458   ins_cost(200);
 6459   format %{ "ecmovl$cop $dst, $src1, $src2\t# signed, compressed ptr ndd" %}
 6460   ins_encode %{
 6461     __ ecmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
 6462   %}
 6463   ins_pipe(pipe_cmov_reg);
 6464 %}
 6465 
 6466 // Conditional move
 6467 instruct cmovN_regU(cmpOpU cop, rFlagsRegU cr, rRegN dst, rRegN src)
 6468 %{
 6469   predicate(!UseAPX);
 6470   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
 6471 
 6472   ins_cost(200); // XXX
 6473   format %{ "cmovl$cop $dst, $src\t# unsigned, compressed ptr" %}
 6474   ins_encode %{
 6475     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 6476   %}
 6477   ins_pipe(pipe_cmov_reg);
 6478 %}
 6479 
 6480 instruct cmovN_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegN dst, rRegN src) %{
 6481   predicate(!UseAPX);
 6482   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
 6483   ins_cost(200);
 6484   expand %{
 6485     cmovN_regU(cop, cr, dst, src);
 6486   %}
 6487 %}
 6488 
 6489 // Conditional move ndd
 6490 instruct cmovN_regU_ndd(rRegN dst, cmpOpU cop, rFlagsRegU cr, rRegN src1, rRegN src2)
 6491 %{
 6492   predicate(UseAPX);
 6493   match(Set dst (CMoveN (Binary cop cr) (Binary src1 src2)));
 6494 
 6495   ins_cost(200);
 6496   format %{ "ecmovl$cop $dst, $src1, $src2\t# unsigned, compressed ptr ndd" %}
 6497   ins_encode %{
 6498     __ ecmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
 6499   %}
 6500   ins_pipe(pipe_cmov_reg);
 6501 %}
 6502 
 6503 instruct cmovN_regUCF_ndd(rRegN dst, cmpOpUCF cop, rFlagsRegUCF cr, rRegN src1, rRegN src2) %{
 6504   predicate(UseAPX);
 6505   match(Set dst (CMoveN (Binary cop cr) (Binary src1 src2)));
 6506   ins_cost(200);
 6507   format %{ "ecmovl$cop $dst, $src1, $src2\t# unsigned, compressed ptr ndd" %}
 6508   ins_encode %{
 6509     __ ecmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
 6510   %}
 6511   ins_pipe(pipe_cmov_reg);
 6512 %}
 6513 
 6514 instruct cmovN_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegN dst, rRegN src) %{
 6515   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
 6516   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
 6517 
 6518   ins_cost(200); // XXX
 6519   format %{ "cmovpl  $dst, $src\n\t"
 6520             "cmovnel $dst, $src" %}
 6521   ins_encode %{
 6522     __ cmovl(Assembler::parity, $dst$$Register, $src$$Register);
 6523     __ cmovl(Assembler::notEqual, $dst$$Register, $src$$Register);
 6524   %}
 6525   ins_pipe(pipe_cmov_reg);
 6526 %}
 6527 
 6528 // Since (x == y) == !(x != y), we can flip the sense of the test by flipping the
 6529 // inputs of the CMove
 6530 instruct cmovN_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegN dst, rRegN src) %{
 6531   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
 6532   match(Set dst (CMoveN (Binary cop cr) (Binary src dst)));
 6533 
 6534   ins_cost(200); // XXX
 6535   format %{ "cmovpl  $dst, $src\n\t"
 6536             "cmovnel $dst, $src" %}
 6537   ins_encode %{
 6538     __ cmovl(Assembler::parity, $dst$$Register, $src$$Register);
 6539     __ cmovl(Assembler::notEqual, $dst$$Register, $src$$Register);
 6540   %}
 6541   ins_pipe(pipe_cmov_reg);
 6542 %}
 6543 
 6544 // Conditional move
 6545 instruct cmovP_reg(rRegP dst, rRegP src, rFlagsReg cr, cmpOp cop)
 6546 %{
 6547   predicate(!UseAPX);
 6548   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
 6549 
 6550   ins_cost(200); // XXX
 6551   format %{ "cmovq$cop $dst, $src\t# signed, ptr" %}
 6552   ins_encode %{
 6553     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 6554   %}
 6555   ins_pipe(pipe_cmov_reg);  // XXX
 6556 %}
 6557 
 6558 // Conditional move ndd
 6559 instruct cmovP_reg_ndd(rRegP dst, rRegP src1, rRegP src2, rFlagsReg cr, cmpOp cop)
 6560 %{
 6561   predicate(UseAPX);
 6562   match(Set dst (CMoveP (Binary cop cr) (Binary src1 src2)));
 6563 
 6564   ins_cost(200);
 6565   format %{ "ecmovq$cop $dst, $src1, $src2\t# signed, ptr ndd" %}
 6566   ins_encode %{
 6567     __ ecmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
 6568   %}
 6569   ins_pipe(pipe_cmov_reg);
 6570 %}
 6571 
 6572 // Conditional move
 6573 instruct cmovP_regU(cmpOpU cop, rFlagsRegU cr, rRegP dst, rRegP src)
 6574 %{
 6575   predicate(!UseAPX);
 6576   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
 6577 
 6578   ins_cost(200); // XXX
 6579   format %{ "cmovq$cop $dst, $src\t# unsigned, ptr" %}
 6580   ins_encode %{
 6581     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 6582   %}
 6583   ins_pipe(pipe_cmov_reg); // XXX
 6584 %}
 6585 
 6586 // Conditional move ndd
 6587 instruct cmovP_regU_ndd(rRegP dst, cmpOpU cop, rFlagsRegU cr, rRegP src1, rRegP src2)
 6588 %{
 6589   predicate(UseAPX);
 6590   match(Set dst (CMoveP (Binary cop cr) (Binary src1 src2)));
 6591 
 6592   ins_cost(200);
 6593   format %{ "ecmovq$cop $dst, $src1, $src2\t# unsigned, ptr ndd" %}
 6594   ins_encode %{
 6595     __ ecmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
 6596   %}
 6597   ins_pipe(pipe_cmov_reg);
 6598 %}
 6599 
 6600 instruct cmovP_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
 6601   predicate(!UseAPX);
 6602   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
 6603   ins_cost(200);
 6604   expand %{
 6605     cmovP_regU(cop, cr, dst, src);
 6606   %}
 6607 %}
 6608 
 6609 instruct cmovP_regUCF_ndd(rRegP dst, cmpOpUCF cop, rFlagsRegUCF cr, rRegP src1, rRegP src2) %{
 6610   predicate(UseAPX);
 6611   match(Set dst (CMoveP (Binary cop cr) (Binary src1 src2)));
 6612   ins_cost(200);
 6613   format %{ "ecmovq$cop $dst, $src1, $src2\t# unsigned, ptr ndd" %}
 6614   ins_encode %{
 6615     __ ecmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
 6616   %}
 6617   ins_pipe(pipe_cmov_reg);
 6618 %}
 6619 
 6620 instruct cmovP_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
 6621   predicate(!UseAPX && n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
 6622   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
 6623 
 6624   ins_cost(200); // XXX
 6625   format %{ "cmovpq  $dst, $src\n\t"
 6626             "cmovneq $dst, $src" %}
 6627   ins_encode %{
 6628     __ cmovq(Assembler::parity, $dst$$Register, $src$$Register);
 6629     __ cmovq(Assembler::notEqual, $dst$$Register, $src$$Register);
 6630   %}
 6631   ins_pipe(pipe_cmov_reg);
 6632 %}
 6633 
 6634 instruct cmovP_regUCF2_ne_ndd(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegP dst, rRegP src1, rRegP src2) %{
 6635   predicate(UseAPX && n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
 6636   match(Set dst (CMoveP (Binary cop cr) (Binary src1 src2)));
 6637   effect(TEMP dst);
 6638 
 6639   ins_cost(200);
 6640   format %{ "ecmovpq  $dst, $src1, $src2\n\t"
 6641             "cmovneq  $dst, $src2" %}
 6642   ins_encode %{
 6643     __ ecmovq(Assembler::parity, $dst$$Register, $src1$$Register, $src2$$Register);
 6644     __ cmovq(Assembler::notEqual, $dst$$Register, $src2$$Register);
 6645   %}
 6646   ins_pipe(pipe_cmov_reg);
 6647 %}
 6648 
 6649 // Since (x == y) == !(x != y), we can flip the sense of the test by flipping the
 6650 // inputs of the CMove
 6651 instruct cmovP_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
 6652   predicate(!UseAPX && n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
 6653   match(Set dst (CMoveP (Binary cop cr) (Binary src dst)));
 6654 
 6655   ins_cost(200); // XXX
 6656   format %{ "cmovpq  $dst, $src\n\t"
 6657             "cmovneq $dst, $src" %}
 6658   ins_encode %{
 6659     __ cmovq(Assembler::parity, $dst$$Register, $src$$Register);
 6660     __ cmovq(Assembler::notEqual, $dst$$Register, $src$$Register);
 6661   %}
 6662   ins_pipe(pipe_cmov_reg);
 6663 %}
 6664 
 6665 instruct cmovP_regUCF2_eq_ndd(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegP dst, rRegP src1, rRegP src2) %{
 6666   predicate(UseAPX && n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
 6667   match(Set dst (CMoveP (Binary cop cr) (Binary src2 src1)));
 6668   effect(TEMP dst);
 6669 
 6670   ins_cost(200);
 6671   format %{ "ecmovpq  $dst, $src1, $src2\n\t"
 6672             "cmovneq  $dst, $src2" %}
 6673   ins_encode %{
 6674     __ ecmovq(Assembler::parity, $dst$$Register, $src1$$Register, $src2$$Register);
 6675     __ cmovq(Assembler::notEqual, $dst$$Register, $src2$$Register);
 6676   %}
 6677   ins_pipe(pipe_cmov_reg);
 6678 %}
 6679 
 6680 instruct cmovL_imm_01(rRegL dst, immL1 src, rFlagsReg cr, cmpOp cop)
 6681 %{
 6682   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_long() == 0);
 6683   match(Set dst (CMoveL (Binary cop cr) (Binary src dst)));
 6684 
 6685   ins_cost(100); // XXX
 6686   format %{ "setbn$cop $dst\t# signed, long" %}
 6687   ins_encode %{
 6688     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 6689     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 6690   %}
 6691   ins_pipe(ialu_reg);
 6692 %}
 6693 
 6694 instruct cmovL_reg(cmpOp cop, rFlagsReg cr, rRegL dst, rRegL src)
 6695 %{
 6696   predicate(!UseAPX);
 6697   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
 6698 
 6699   ins_cost(200); // XXX
 6700   format %{ "cmovq$cop $dst, $src\t# signed, long" %}
 6701   ins_encode %{
 6702     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 6703   %}
 6704   ins_pipe(pipe_cmov_reg);  // XXX
 6705 %}
 6706 
 6707 instruct cmovL_reg_ndd(rRegL dst, cmpOp cop, rFlagsReg cr, rRegL src1, rRegL src2)
 6708 %{
 6709   predicate(UseAPX);
 6710   match(Set dst (CMoveL (Binary cop cr) (Binary src1 src2)));
 6711 
 6712   ins_cost(200);
 6713   format %{ "ecmovq$cop $dst, $src1, $src2\t# signed, long ndd" %}
 6714   ins_encode %{
 6715     __ ecmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
 6716   %}
 6717   ins_pipe(pipe_cmov_reg);
 6718 %}
 6719 
 6720 instruct cmovL_mem(cmpOp cop, rFlagsReg cr, rRegL dst, memory src)
 6721 %{
 6722   predicate(!UseAPX);
 6723   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
 6724 
 6725   ins_cost(200); // XXX
 6726   format %{ "cmovq$cop $dst, $src\t# signed, long" %}
 6727   ins_encode %{
 6728     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
 6729   %}
 6730   ins_pipe(pipe_cmov_mem);  // XXX
 6731 %}
 6732 
 6733 instruct cmovL_rReg_rReg_mem_ndd(rRegL dst, cmpOp cop, rFlagsReg cr, rRegL src1, memory src2)
 6734 %{
 6735   predicate(UseAPX);
 6736   match(Set dst (CMoveL (Binary cop cr) (Binary src1 (LoadL src2))));
 6737 
 6738   ins_cost(200);
 6739   format %{ "ecmovq$cop $dst, $src1, $src2\t# signed, long ndd" %}
 6740   ins_encode %{
 6741     __ ecmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Address);
 6742   %}
 6743   ins_pipe(pipe_cmov_mem);
 6744 %}
 6745 
 6746 instruct cmovL_imm_01U(rRegL dst, immL1 src, rFlagsRegU cr, cmpOpU cop)
 6747 %{
 6748   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_long() == 0);
 6749   match(Set dst (CMoveL (Binary cop cr) (Binary src dst)));
 6750 
 6751   ins_cost(100); // XXX
 6752   format %{ "setbn$cop $dst\t# unsigned, long" %}
 6753   ins_encode %{
 6754     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 6755     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 6756   %}
 6757   ins_pipe(ialu_reg);
 6758 %}
 6759 
 6760 instruct cmovL_regU(cmpOpU cop, rFlagsRegU cr, rRegL dst, rRegL src)
 6761 %{
 6762   predicate(!UseAPX);
 6763   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
 6764 
 6765   ins_cost(200); // XXX
 6766   format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
 6767   ins_encode %{
 6768     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 6769   %}
 6770   ins_pipe(pipe_cmov_reg); // XXX
 6771 %}
 6772 
 6773 instruct cmovL_regU_ndd(rRegL dst, cmpOpU cop, rFlagsRegU cr, rRegL src1, rRegL src2)
 6774 %{
 6775   predicate(UseAPX);
 6776   match(Set dst (CMoveL (Binary cop cr) (Binary src1 src2)));
 6777 
 6778   ins_cost(200);
 6779   format %{ "ecmovq$cop $dst, $src1, $src2\t# unsigned, long ndd" %}
 6780   ins_encode %{
 6781     __ ecmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
 6782   %}
 6783   ins_pipe(pipe_cmov_reg);
 6784 %}
 6785 
 6786 instruct cmovL_imm_01UCF(rRegL dst, immL1 src, rFlagsRegUCF cr, cmpOpUCF cop)
 6787 %{
 6788   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_long() == 0);
 6789   match(Set dst (CMoveL (Binary cop cr) (Binary src dst)));
 6790 
 6791   ins_cost(100); // XXX
 6792   format %{ "setbn$cop $dst\t# unsigned, long" %}
 6793   ins_encode %{
 6794     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 6795     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 6796   %}
 6797   ins_pipe(ialu_reg);
 6798 %}
 6799 
 6800 instruct cmovL_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
 6801   predicate(!UseAPX);
 6802   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
 6803   ins_cost(200);
 6804   expand %{
 6805     cmovL_regU(cop, cr, dst, src);
 6806   %}
 6807 %}
 6808 
 6809 instruct cmovL_regUCF_ndd(rRegL dst, cmpOpUCF cop, rFlagsRegUCF cr, rRegL src1, rRegL src2)
 6810 %{
 6811   predicate(UseAPX);
 6812   match(Set dst (CMoveL (Binary cop cr) (Binary src1 src2)));
 6813   ins_cost(200);
 6814   format %{ "ecmovq$cop $dst, $src1, $src2\t# unsigned, long ndd" %}
 6815   ins_encode %{
 6816     __ ecmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
 6817   %}
 6818   ins_pipe(pipe_cmov_reg);
 6819 %}
 6820 
 6821 instruct cmovL_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
 6822   predicate(!UseAPX && n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
 6823   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
 6824 
 6825   ins_cost(200); // XXX
 6826   format %{ "cmovpq  $dst, $src\n\t"
 6827             "cmovneq $dst, $src" %}
 6828   ins_encode %{
 6829     __ cmovq(Assembler::parity, $dst$$Register, $src$$Register);
 6830     __ cmovq(Assembler::notEqual, $dst$$Register, $src$$Register);
 6831   %}
 6832   ins_pipe(pipe_cmov_reg);
 6833 %}
 6834 
 6835 instruct cmovL_regUCF2_ne_ndd(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegL dst, rRegL src1, rRegL src2) %{
 6836   predicate(UseAPX && n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
 6837   match(Set dst (CMoveL (Binary cop cr) (Binary src1 src2)));
 6838   effect(TEMP dst);
 6839 
 6840   ins_cost(200);
 6841   format %{ "ecmovpq  $dst, $src1, $src2\n\t"
 6842             "cmovneq  $dst, $src2" %}
 6843   ins_encode %{
 6844     __ ecmovq(Assembler::parity, $dst$$Register, $src1$$Register, $src2$$Register);
 6845     __ cmovq(Assembler::notEqual, $dst$$Register, $src2$$Register);
 6846   %}
 6847   ins_pipe(pipe_cmov_reg);
 6848 %}
 6849 
 6850 // Since (x == y) == !(x != y), we can flip the sense of the test by flipping the
 6851 // inputs of the CMove
 6852 instruct cmovL_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
 6853   predicate(!UseAPX && n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
 6854   match(Set dst (CMoveL (Binary cop cr) (Binary src dst)));
 6855 
 6856   ins_cost(200); // XXX
 6857   format %{ "cmovpq  $dst, $src\n\t"
 6858             "cmovneq $dst, $src" %}
 6859   ins_encode %{
 6860     __ cmovq(Assembler::parity, $dst$$Register, $src$$Register);
 6861     __ cmovq(Assembler::notEqual, $dst$$Register, $src$$Register);
 6862   %}
 6863   ins_pipe(pipe_cmov_reg);
 6864 %}
 6865 
 6866 instruct cmovL_regUCF2_eq_ndd(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegL dst, rRegL src1, rRegL src2) %{
 6867   predicate(UseAPX && n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
 6868   match(Set dst (CMoveL (Binary cop cr) (Binary src2 src1)));
 6869   effect(TEMP dst);
 6870 
 6871   ins_cost(200);
 6872   format %{ "ecmovpq  $dst, $src1, $src2\n\t"
 6873             "cmovneq $dst, $src2" %}
 6874   ins_encode %{
 6875     __ ecmovq(Assembler::parity, $dst$$Register, $src1$$Register, $src2$$Register);
 6876     __ cmovq(Assembler::notEqual, $dst$$Register, $src2$$Register);
 6877   %}
 6878   ins_pipe(pipe_cmov_reg);
 6879 %}
 6880 
 6881 instruct cmovL_memU(cmpOpU cop, rFlagsRegU cr, rRegL dst, memory src)
 6882 %{
 6883   predicate(!UseAPX);
 6884   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
 6885 
 6886   ins_cost(200); // XXX
 6887   format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
 6888   ins_encode %{
 6889     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
 6890   %}
 6891   ins_pipe(pipe_cmov_mem); // XXX
 6892 %}
 6893 
 6894 instruct cmovL_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, memory src) %{
 6895   predicate(!UseAPX);
 6896   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
 6897   ins_cost(200);
 6898   expand %{
 6899     cmovL_memU(cop, cr, dst, src);
 6900   %}
 6901 %}
 6902 
 6903 instruct cmovL_rReg_rReg_memU_ndd(rRegL dst, cmpOpU cop, rFlagsRegU cr, rRegL src1, memory src2)
 6904 %{
 6905   predicate(UseAPX);
 6906   match(Set dst (CMoveL (Binary cop cr) (Binary src1 (LoadL src2))));
 6907 
 6908   ins_cost(200);
 6909   format %{ "ecmovq$cop $dst, $src1, $src2\t# unsigned, long ndd" %}
 6910   ins_encode %{
 6911     __ ecmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Address);
 6912   %}
 6913   ins_pipe(pipe_cmov_mem);
 6914 %}
 6915 
 6916 instruct cmovL_rReg_rReg_memUCF_ndd(rRegL dst, cmpOpUCF cop, rFlagsRegUCF cr, rRegL src1, memory src2)
 6917 %{
 6918   predicate(UseAPX);
 6919   match(Set dst (CMoveL (Binary cop cr) (Binary src1 (LoadL src2))));
 6920   ins_cost(200);
 6921   format %{ "ecmovq$cop $dst, $src1, $src2\t# unsigned, long ndd" %}
 6922   ins_encode %{
 6923     __ ecmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Address);
 6924   %}
 6925   ins_pipe(pipe_cmov_mem);
 6926 %}
 6927 
 6928 instruct cmovF_reg(cmpOp cop, rFlagsReg cr, regF dst, regF src)
 6929 %{
 6930   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
 6931 
 6932   ins_cost(200); // XXX
 6933   format %{ "jn$cop    skip\t# signed cmove float\n\t"
 6934             "movss     $dst, $src\n"
 6935     "skip:" %}
 6936   ins_encode %{
 6937     Label Lskip;
 6938     // Invert sense of branch from sense of CMOV
 6939     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
 6940     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
 6941     __ bind(Lskip);
 6942   %}
 6943   ins_pipe(pipe_slow);
 6944 %}
 6945 
 6946 instruct cmovF_regU(cmpOpU cop, rFlagsRegU cr, regF dst, regF src)
 6947 %{
 6948   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
 6949 
 6950   ins_cost(200); // XXX
 6951   format %{ "jn$cop    skip\t# unsigned cmove float\n\t"
 6952             "movss     $dst, $src\n"
 6953     "skip:" %}
 6954   ins_encode %{
 6955     Label Lskip;
 6956     // Invert sense of branch from sense of CMOV
 6957     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
 6958     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
 6959     __ bind(Lskip);
 6960   %}
 6961   ins_pipe(pipe_slow);
 6962 %}
 6963 
 6964 instruct cmovF_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regF dst, regF src) %{
 6965   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
 6966   ins_cost(200);
 6967   expand %{
 6968     cmovF_regU(cop, cr, dst, src);
 6969   %}
 6970 %}
 6971 
 6972 instruct cmovD_reg(cmpOp cop, rFlagsReg cr, regD dst, regD src)
 6973 %{
 6974   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
 6975 
 6976   ins_cost(200); // XXX
 6977   format %{ "jn$cop    skip\t# signed cmove double\n\t"
 6978             "movsd     $dst, $src\n"
 6979     "skip:" %}
 6980   ins_encode %{
 6981     Label Lskip;
 6982     // Invert sense of branch from sense of CMOV
 6983     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
 6984     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
 6985     __ bind(Lskip);
 6986   %}
 6987   ins_pipe(pipe_slow);
 6988 %}
 6989 
 6990 instruct cmovD_regU(cmpOpU cop, rFlagsRegU cr, regD dst, regD src)
 6991 %{
 6992   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
 6993 
 6994   ins_cost(200); // XXX
 6995   format %{ "jn$cop    skip\t# unsigned cmove double\n\t"
 6996             "movsd     $dst, $src\n"
 6997     "skip:" %}
 6998   ins_encode %{
 6999     Label Lskip;
 7000     // Invert sense of branch from sense of CMOV
 7001     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
 7002     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
 7003     __ bind(Lskip);
 7004   %}
 7005   ins_pipe(pipe_slow);
 7006 %}
 7007 
 7008 instruct cmovD_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regD dst, regD src) %{
 7009   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
 7010   ins_cost(200);
 7011   expand %{
 7012     cmovD_regU(cop, cr, dst, src);
 7013   %}
 7014 %}
 7015 
 7016 //----------Arithmetic Instructions--------------------------------------------
 7017 //----------Addition Instructions----------------------------------------------
 7018 
 7019 instruct addI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 7020 %{
 7021   predicate(!UseAPX);
 7022   match(Set dst (AddI dst src));
 7023   effect(KILL cr);
 7024   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7025   format %{ "addl    $dst, $src\t# int" %}
 7026   ins_encode %{
 7027     __ addl($dst$$Register, $src$$Register);
 7028   %}
 7029   ins_pipe(ialu_reg_reg);
 7030 %}
 7031 
 7032 instruct addI_rReg_ndd(rRegI dst, rRegI src1, rRegI src2, rFlagsReg cr)
 7033 %{
 7034   predicate(UseAPX);
 7035   match(Set dst (AddI src1 src2));
 7036   effect(KILL cr);
 7037   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7038 
 7039   format %{ "eaddl    $dst, $src1, $src2\t# int ndd" %}
 7040   ins_encode %{
 7041     __ eaddl($dst$$Register, $src1$$Register, $src2$$Register, false);
 7042   %}
 7043   ins_pipe(ialu_reg_reg);
 7044 %}
 7045 
 7046 instruct addI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
 7047 %{
 7048   predicate(!UseAPX);
 7049   match(Set dst (AddI dst src));
 7050   effect(KILL cr);
 7051   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7052 
 7053   format %{ "addl    $dst, $src\t# int" %}
 7054   ins_encode %{
 7055     __ addl($dst$$Register, $src$$constant);
 7056   %}
 7057   ins_pipe( ialu_reg );
 7058 %}
 7059 
 7060 instruct addI_rReg_rReg_imm_ndd(rRegI dst, rRegI src1, immI src2, rFlagsReg cr)
 7061 %{
 7062   predicate(UseAPX);
 7063   match(Set dst (AddI src1 src2));
 7064   effect(KILL cr);
 7065   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7066 
 7067   format %{ "eaddl    $dst, $src1, $src2\t# int ndd" %}
 7068   ins_encode %{
 7069     __ eaddl($dst$$Register, $src1$$Register, $src2$$constant, false);
 7070   %}
 7071   ins_pipe( ialu_reg );
 7072 %}
 7073 
 7074 instruct addI_rReg_mem_imm_ndd(rRegI dst, memory src1, immI src2, rFlagsReg cr)
 7075 %{
 7076   predicate(UseAPX);
 7077   match(Set dst (AddI (LoadI src1) src2));
 7078   effect(KILL cr);
 7079   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7080 
 7081   format %{ "eaddl    $dst, $src1, $src2\t# int ndd" %}
 7082   ins_encode %{
 7083     __ eaddl($dst$$Register, $src1$$Address, $src2$$constant, false);
 7084   %}
 7085   ins_pipe( ialu_reg );
 7086 %}
 7087 
 7088 instruct addI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
 7089 %{
 7090   predicate(!UseAPX);
 7091   match(Set dst (AddI dst (LoadI src)));
 7092   effect(KILL cr);
 7093   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7094 
 7095   ins_cost(150); // XXX
 7096   format %{ "addl    $dst, $src\t# int" %}
 7097   ins_encode %{
 7098     __ addl($dst$$Register, $src$$Address);
 7099   %}
 7100   ins_pipe(ialu_reg_mem);
 7101 %}
 7102 
 7103 instruct addI_rReg_rReg_mem_ndd(rRegI dst, rRegI src1, memory src2, rFlagsReg cr)
 7104 %{
 7105   predicate(UseAPX);
 7106   match(Set dst (AddI src1 (LoadI src2)));
 7107   effect(KILL cr);
 7108   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7109 
 7110   ins_cost(150);
 7111   format %{ "eaddl    $dst, $src1, $src2\t# int ndd" %}
 7112   ins_encode %{
 7113     __ eaddl($dst$$Register, $src1$$Register, $src2$$Address, false);
 7114   %}
 7115   ins_pipe(ialu_reg_mem);
 7116 %}
 7117 
 7118 instruct addI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 7119 %{
 7120   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
 7121   effect(KILL cr);
 7122   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7123 
 7124   ins_cost(150); // XXX
 7125   format %{ "addl    $dst, $src\t# int" %}
 7126   ins_encode %{
 7127     __ addl($dst$$Address, $src$$Register);
 7128   %}
 7129   ins_pipe(ialu_mem_reg);
 7130 %}
 7131 
 7132 instruct addI_mem_imm(memory dst, immI src, rFlagsReg cr)
 7133 %{
 7134   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
 7135   effect(KILL cr);
 7136   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7137 
 7138 
 7139   ins_cost(125); // XXX
 7140   format %{ "addl    $dst, $src\t# int" %}
 7141   ins_encode %{
 7142     __ addl($dst$$Address, $src$$constant);
 7143   %}
 7144   ins_pipe(ialu_mem_imm);
 7145 %}
 7146 
 7147 instruct incI_rReg(rRegI dst, immI_1 src, rFlagsReg cr)
 7148 %{
 7149   predicate(!UseAPX && UseIncDec);
 7150   match(Set dst (AddI dst src));
 7151   effect(KILL cr);
 7152 
 7153   format %{ "incl    $dst\t# int" %}
 7154   ins_encode %{
 7155     __ incrementl($dst$$Register);
 7156   %}
 7157   ins_pipe(ialu_reg);
 7158 %}
 7159 
 7160 instruct incI_rReg_ndd(rRegI dst, rRegI src, immI_1 val, rFlagsReg cr)
 7161 %{
 7162   predicate(UseAPX && UseIncDec);
 7163   match(Set dst (AddI src val));
 7164   effect(KILL cr);
 7165 
 7166   format %{ "eincl    $dst, $src\t# int ndd" %}
 7167   ins_encode %{
 7168     __ eincl($dst$$Register, $src$$Register, false);
 7169   %}
 7170   ins_pipe(ialu_reg);
 7171 %}
 7172 
 7173 instruct incI_rReg_mem_ndd(rRegI dst, memory src, immI_1 val, rFlagsReg cr)
 7174 %{
 7175   predicate(UseAPX && UseIncDec);
 7176   match(Set dst (AddI (LoadI src) val));
 7177   effect(KILL cr);
 7178 
 7179   format %{ "eincl    $dst, $src\t# int ndd" %}
 7180   ins_encode %{
 7181     __ eincl($dst$$Register, $src$$Address, false);
 7182   %}
 7183   ins_pipe(ialu_reg);
 7184 %}
 7185 
 7186 instruct incI_mem(memory dst, immI_1 src, rFlagsReg cr)
 7187 %{
 7188   predicate(UseIncDec);
 7189   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
 7190   effect(KILL cr);
 7191 
 7192   ins_cost(125); // XXX
 7193   format %{ "incl    $dst\t# int" %}
 7194   ins_encode %{
 7195     __ incrementl($dst$$Address);
 7196   %}
 7197   ins_pipe(ialu_mem_imm);
 7198 %}
 7199 
 7200 // XXX why does that use AddI
 7201 instruct decI_rReg(rRegI dst, immI_M1 src, rFlagsReg cr)
 7202 %{
 7203   predicate(!UseAPX && UseIncDec);
 7204   match(Set dst (AddI dst src));
 7205   effect(KILL cr);
 7206 
 7207   format %{ "decl    $dst\t# int" %}
 7208   ins_encode %{
 7209     __ decrementl($dst$$Register);
 7210   %}
 7211   ins_pipe(ialu_reg);
 7212 %}
 7213 
 7214 instruct decI_rReg_ndd(rRegI dst, rRegI src, immI_M1 val, rFlagsReg cr)
 7215 %{
 7216   predicate(UseAPX && UseIncDec);
 7217   match(Set dst (AddI src val));
 7218   effect(KILL cr);
 7219 
 7220   format %{ "edecl    $dst, $src\t# int ndd" %}
 7221   ins_encode %{
 7222     __ edecl($dst$$Register, $src$$Register, false);
 7223   %}
 7224   ins_pipe(ialu_reg);
 7225 %}
 7226 
 7227 instruct decI_rReg_mem_ndd(rRegI dst, memory src, immI_M1 val, rFlagsReg cr)
 7228 %{
 7229   predicate(UseAPX && UseIncDec);
 7230   match(Set dst (AddI (LoadI src) val));
 7231   effect(KILL cr);
 7232 
 7233   format %{ "edecl    $dst, $src\t# int ndd" %}
 7234   ins_encode %{
 7235     __ edecl($dst$$Register, $src$$Address, false);
 7236   %}
 7237   ins_pipe(ialu_reg);
 7238 %}
 7239 
 7240 // XXX why does that use AddI
 7241 instruct decI_mem(memory dst, immI_M1 src, rFlagsReg cr)
 7242 %{
 7243   predicate(UseIncDec);
 7244   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
 7245   effect(KILL cr);
 7246 
 7247   ins_cost(125); // XXX
 7248   format %{ "decl    $dst\t# int" %}
 7249   ins_encode %{
 7250     __ decrementl($dst$$Address);
 7251   %}
 7252   ins_pipe(ialu_mem_imm);
 7253 %}
 7254 
 7255 instruct leaI_rReg_immI2_immI(rRegI dst, rRegI index, immI2 scale, immI disp)
 7256 %{
 7257   predicate(VM_Version::supports_fast_2op_lea());
 7258   match(Set dst (AddI (LShiftI index scale) disp));
 7259 
 7260   format %{ "leal $dst, [$index << $scale + $disp]\t# int" %}
 7261   ins_encode %{
 7262     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 7263     __ leal($dst$$Register, Address(noreg, $index$$Register, scale, $disp$$constant));
 7264   %}
 7265   ins_pipe(ialu_reg_reg);
 7266 %}
 7267 
 7268 instruct leaI_rReg_rReg_immI(rRegI dst, rRegI base, rRegI index, immI disp)
 7269 %{
 7270   predicate(VM_Version::supports_fast_3op_lea());
 7271   match(Set dst (AddI (AddI base index) disp));
 7272 
 7273   format %{ "leal $dst, [$base + $index + $disp]\t# int" %}
 7274   ins_encode %{
 7275     __ leal($dst$$Register, Address($base$$Register, $index$$Register, Address::times_1, $disp$$constant));
 7276   %}
 7277   ins_pipe(ialu_reg_reg);
 7278 %}
 7279 
 7280 instruct leaI_rReg_rReg_immI2(rRegI dst, no_rbp_r13_RegI base, rRegI index, immI2 scale)
 7281 %{
 7282   predicate(VM_Version::supports_fast_2op_lea());
 7283   match(Set dst (AddI base (LShiftI index scale)));
 7284 
 7285   format %{ "leal $dst, [$base + $index << $scale]\t# int" %}
 7286   ins_encode %{
 7287     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 7288     __ leal($dst$$Register, Address($base$$Register, $index$$Register, scale));
 7289   %}
 7290   ins_pipe(ialu_reg_reg);
 7291 %}
 7292 
 7293 instruct leaI_rReg_rReg_immI2_immI(rRegI dst, rRegI base, rRegI index, immI2 scale, immI disp)
 7294 %{
 7295   predicate(VM_Version::supports_fast_3op_lea());
 7296   match(Set dst (AddI (AddI base (LShiftI index scale)) disp));
 7297 
 7298   format %{ "leal $dst, [$base + $index << $scale + $disp]\t# int" %}
 7299   ins_encode %{
 7300     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 7301     __ leal($dst$$Register, Address($base$$Register, $index$$Register, scale, $disp$$constant));
 7302   %}
 7303   ins_pipe(ialu_reg_reg);
 7304 %}
 7305 
 7306 instruct addL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 7307 %{
 7308   predicate(!UseAPX);
 7309   match(Set dst (AddL dst src));
 7310   effect(KILL cr);
 7311   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7312 
 7313   format %{ "addq    $dst, $src\t# long" %}
 7314   ins_encode %{
 7315     __ addq($dst$$Register, $src$$Register);
 7316   %}
 7317   ins_pipe(ialu_reg_reg);
 7318 %}
 7319 
 7320 instruct addL_rReg_ndd(rRegL dst, rRegL src1, rRegL src2, rFlagsReg cr)
 7321 %{
 7322   predicate(UseAPX);
 7323   match(Set dst (AddL src1 src2));
 7324   effect(KILL cr);
 7325   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7326 
 7327   format %{ "eaddq    $dst, $src1, $src2\t# long ndd" %}
 7328   ins_encode %{
 7329     __ eaddq($dst$$Register, $src1$$Register, $src2$$Register, false);
 7330   %}
 7331   ins_pipe(ialu_reg_reg);
 7332 %}
 7333 
 7334 instruct addL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
 7335 %{
 7336   predicate(!UseAPX);
 7337   match(Set dst (AddL dst src));
 7338   effect(KILL cr);
 7339   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7340 
 7341   format %{ "addq    $dst, $src\t# long" %}
 7342   ins_encode %{
 7343     __ addq($dst$$Register, $src$$constant);
 7344   %}
 7345   ins_pipe( ialu_reg );
 7346 %}
 7347 
 7348 instruct addL_rReg_rReg_imm_ndd(rRegL dst, rRegL src1, immL32 src2, rFlagsReg cr)
 7349 %{
 7350   predicate(UseAPX);
 7351   match(Set dst (AddL src1 src2));
 7352   effect(KILL cr);
 7353   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7354 
 7355   format %{ "eaddq    $dst, $src1, $src2\t# long ndd" %}
 7356   ins_encode %{
 7357     __ eaddq($dst$$Register, $src1$$Register, $src2$$constant, false);
 7358   %}
 7359   ins_pipe( ialu_reg );
 7360 %}
 7361 
 7362 instruct addL_rReg_mem_imm_ndd(rRegL dst, memory src1, immL32 src2, rFlagsReg cr)
 7363 %{
 7364   predicate(UseAPX);
 7365   match(Set dst (AddL (LoadL src1) src2));
 7366   effect(KILL cr);
 7367   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7368 
 7369   format %{ "eaddq    $dst, $src1, $src2\t# long ndd" %}
 7370   ins_encode %{
 7371     __ eaddq($dst$$Register, $src1$$Address, $src2$$constant, false);
 7372   %}
 7373   ins_pipe( ialu_reg );
 7374 %}
 7375 
 7376 instruct addL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
 7377 %{
 7378   predicate(!UseAPX);
 7379   match(Set dst (AddL dst (LoadL src)));
 7380   effect(KILL cr);
 7381   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7382 
 7383   ins_cost(150); // XXX
 7384   format %{ "addq    $dst, $src\t# long" %}
 7385   ins_encode %{
 7386     __ addq($dst$$Register, $src$$Address);
 7387   %}
 7388   ins_pipe(ialu_reg_mem);
 7389 %}
 7390 
 7391 instruct addL_rReg_rReg_mem_ndd(rRegL dst, rRegL src1, memory src2, rFlagsReg cr)
 7392 %{
 7393   predicate(UseAPX);
 7394   match(Set dst (AddL src1 (LoadL src2)));
 7395   effect(KILL cr);
 7396   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7397 
 7398   ins_cost(150);
 7399   format %{ "eaddq    $dst, $src1, $src2\t# long ndd" %}
 7400   ins_encode %{
 7401     __ eaddq($dst$$Register, $src1$$Register, $src2$$Address, false);
 7402   %}
 7403   ins_pipe(ialu_reg_mem);
 7404 %}
 7405 
 7406 instruct addL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
 7407 %{
 7408   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
 7409   effect(KILL cr);
 7410   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7411 
 7412   ins_cost(150); // XXX
 7413   format %{ "addq    $dst, $src\t# long" %}
 7414   ins_encode %{
 7415     __ addq($dst$$Address, $src$$Register);
 7416   %}
 7417   ins_pipe(ialu_mem_reg);
 7418 %}
 7419 
 7420 instruct addL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
 7421 %{
 7422   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
 7423   effect(KILL cr);
 7424   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7425 
 7426   ins_cost(125); // XXX
 7427   format %{ "addq    $dst, $src\t# long" %}
 7428   ins_encode %{
 7429     __ addq($dst$$Address, $src$$constant);
 7430   %}
 7431   ins_pipe(ialu_mem_imm);
 7432 %}
 7433 
 7434 instruct incL_rReg(rRegL dst, immL1 src, rFlagsReg cr)
 7435 %{
 7436   predicate(!UseAPX && UseIncDec);
 7437   match(Set dst (AddL dst src));
 7438   effect(KILL cr);
 7439 
 7440   format %{ "incq    $dst\t# long" %}
 7441   ins_encode %{
 7442     __ incrementq($dst$$Register);
 7443   %}
 7444   ins_pipe(ialu_reg);
 7445 %}
 7446 
 7447 instruct incL_rReg_ndd(rRegL dst, rRegI src, immL1 val, rFlagsReg cr)
 7448 %{
 7449   predicate(UseAPX && UseIncDec);
 7450   match(Set dst (AddL src val));
 7451   effect(KILL cr);
 7452 
 7453   format %{ "eincq    $dst, $src\t# long ndd" %}
 7454   ins_encode %{
 7455     __ eincq($dst$$Register, $src$$Register, false);
 7456   %}
 7457   ins_pipe(ialu_reg);
 7458 %}
 7459 
 7460 instruct incL_rReg_mem_ndd(rRegL dst, memory src, immL1 val, rFlagsReg cr)
 7461 %{
 7462   predicate(UseAPX && UseIncDec);
 7463   match(Set dst (AddL (LoadL src) val));
 7464   effect(KILL cr);
 7465 
 7466   format %{ "eincq    $dst, $src\t# long ndd" %}
 7467   ins_encode %{
 7468     __ eincq($dst$$Register, $src$$Address, false);
 7469   %}
 7470   ins_pipe(ialu_reg);
 7471 %}
 7472 
 7473 instruct incL_mem(memory dst, immL1 src, rFlagsReg cr)
 7474 %{
 7475   predicate(UseIncDec);
 7476   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
 7477   effect(KILL cr);
 7478 
 7479   ins_cost(125); // XXX
 7480   format %{ "incq    $dst\t# long" %}
 7481   ins_encode %{
 7482     __ incrementq($dst$$Address);
 7483   %}
 7484   ins_pipe(ialu_mem_imm);
 7485 %}
 7486 
 7487 // XXX why does that use AddL
 7488 instruct decL_rReg(rRegL dst, immL_M1 src, rFlagsReg cr)
 7489 %{
 7490   predicate(!UseAPX && UseIncDec);
 7491   match(Set dst (AddL dst src));
 7492   effect(KILL cr);
 7493 
 7494   format %{ "decq    $dst\t# long" %}
 7495   ins_encode %{
 7496     __ decrementq($dst$$Register);
 7497   %}
 7498   ins_pipe(ialu_reg);
 7499 %}
 7500 
 7501 instruct decL_rReg_ndd(rRegL dst, rRegL src, immL_M1 val, rFlagsReg cr)
 7502 %{
 7503   predicate(UseAPX && UseIncDec);
 7504   match(Set dst (AddL src val));
 7505   effect(KILL cr);
 7506 
 7507   format %{ "edecq    $dst, $src\t# long ndd" %}
 7508   ins_encode %{
 7509     __ edecq($dst$$Register, $src$$Register, false);
 7510   %}
 7511   ins_pipe(ialu_reg);
 7512 %}
 7513 
 7514 instruct decL_rReg_mem_ndd(rRegL dst, memory src, immL_M1 val, rFlagsReg cr)
 7515 %{
 7516   predicate(UseAPX && UseIncDec);
 7517   match(Set dst (AddL (LoadL src) val));
 7518   effect(KILL cr);
 7519 
 7520   format %{ "edecq    $dst, $src\t# long ndd" %}
 7521   ins_encode %{
 7522     __ edecq($dst$$Register, $src$$Address, false);
 7523   %}
 7524   ins_pipe(ialu_reg);
 7525 %}
 7526 
 7527 // XXX why does that use AddL
 7528 instruct decL_mem(memory dst, immL_M1 src, rFlagsReg cr)
 7529 %{
 7530   predicate(UseIncDec);
 7531   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
 7532   effect(KILL cr);
 7533 
 7534   ins_cost(125); // XXX
 7535   format %{ "decq    $dst\t# long" %}
 7536   ins_encode %{
 7537     __ decrementq($dst$$Address);
 7538   %}
 7539   ins_pipe(ialu_mem_imm);
 7540 %}
 7541 
 7542 instruct leaL_rReg_immI2_immL32(rRegL dst, rRegL index, immI2 scale, immL32 disp)
 7543 %{
 7544   predicate(VM_Version::supports_fast_2op_lea());
 7545   match(Set dst (AddL (LShiftL index scale) disp));
 7546 
 7547   format %{ "leaq $dst, [$index << $scale + $disp]\t# long" %}
 7548   ins_encode %{
 7549     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 7550     __ leaq($dst$$Register, Address(noreg, $index$$Register, scale, $disp$$constant));
 7551   %}
 7552   ins_pipe(ialu_reg_reg);
 7553 %}
 7554 
 7555 instruct leaL_rReg_rReg_immL32(rRegL dst, rRegL base, rRegL index, immL32 disp)
 7556 %{
 7557   predicate(VM_Version::supports_fast_3op_lea());
 7558   match(Set dst (AddL (AddL base index) disp));
 7559 
 7560   format %{ "leaq $dst, [$base + $index + $disp]\t# long" %}
 7561   ins_encode %{
 7562     __ leaq($dst$$Register, Address($base$$Register, $index$$Register, Address::times_1, $disp$$constant));
 7563   %}
 7564   ins_pipe(ialu_reg_reg);
 7565 %}
 7566 
 7567 instruct leaL_rReg_rReg_immI2(rRegL dst, no_rbp_r13_RegL base, rRegL index, immI2 scale)
 7568 %{
 7569   predicate(VM_Version::supports_fast_2op_lea());
 7570   match(Set dst (AddL base (LShiftL index scale)));
 7571 
 7572   format %{ "leaq $dst, [$base + $index << $scale]\t# long" %}
 7573   ins_encode %{
 7574     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 7575     __ leaq($dst$$Register, Address($base$$Register, $index$$Register, scale));
 7576   %}
 7577   ins_pipe(ialu_reg_reg);
 7578 %}
 7579 
 7580 instruct leaL_rReg_rReg_immI2_immL32(rRegL dst, rRegL base, rRegL index, immI2 scale, immL32 disp)
 7581 %{
 7582   predicate(VM_Version::supports_fast_3op_lea());
 7583   match(Set dst (AddL (AddL base (LShiftL index scale)) disp));
 7584 
 7585   format %{ "leaq $dst, [$base + $index << $scale + $disp]\t# long" %}
 7586   ins_encode %{
 7587     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 7588     __ leaq($dst$$Register, Address($base$$Register, $index$$Register, scale, $disp$$constant));
 7589   %}
 7590   ins_pipe(ialu_reg_reg);
 7591 %}
 7592 
 7593 instruct addP_rReg(rRegP dst, rRegL src, rFlagsReg cr)
 7594 %{
 7595   match(Set dst (AddP dst src));
 7596   effect(KILL cr);
 7597   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7598 
 7599   format %{ "addq    $dst, $src\t# ptr" %}
 7600   ins_encode %{
 7601     __ addq($dst$$Register, $src$$Register);
 7602   %}
 7603   ins_pipe(ialu_reg_reg);
 7604 %}
 7605 
 7606 instruct addP_rReg_imm(rRegP dst, immL32 src, rFlagsReg cr)
 7607 %{
 7608   match(Set dst (AddP dst src));
 7609   effect(KILL cr);
 7610   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7611 
 7612   format %{ "addq    $dst, $src\t# ptr" %}
 7613   ins_encode %{
 7614     __ addq($dst$$Register, $src$$constant);
 7615   %}
 7616   ins_pipe( ialu_reg );
 7617 %}
 7618 
 7619 // XXX addP mem ops ????
 7620 
 7621 instruct checkCastPP(rRegP dst)
 7622 %{
 7623   match(Set dst (CheckCastPP dst));
 7624 
 7625   size(0);
 7626   format %{ "# checkcastPP of $dst" %}
 7627   ins_encode(/* empty encoding */);
 7628   ins_pipe(empty);
 7629 %}
 7630 
 7631 instruct castPP(rRegP dst)
 7632 %{
 7633   match(Set dst (CastPP dst));
 7634 
 7635   size(0);
 7636   format %{ "# castPP of $dst" %}
 7637   ins_encode(/* empty encoding */);
 7638   ins_pipe(empty);
 7639 %}
 7640 
 7641 instruct castII(rRegI dst)
 7642 %{
 7643   predicate(VerifyConstraintCasts == 0);
 7644   match(Set dst (CastII dst));
 7645 
 7646   size(0);
 7647   format %{ "# castII of $dst" %}
 7648   ins_encode(/* empty encoding */);
 7649   ins_cost(0);
 7650   ins_pipe(empty);
 7651 %}
 7652 
 7653 instruct castII_checked(rRegI dst, rFlagsReg cr)
 7654 %{
 7655   predicate(VerifyConstraintCasts > 0);
 7656   match(Set dst (CastII dst));
 7657 
 7658   effect(KILL cr);
 7659   format %{ "# cast_checked_II $dst" %}
 7660   ins_encode %{
 7661     __ verify_int_in_range(_idx, bottom_type()->is_int(), $dst$$Register);
 7662   %}
 7663   ins_pipe(pipe_slow);
 7664 %}
 7665 
 7666 instruct castLL(rRegL dst)
 7667 %{
 7668   predicate(VerifyConstraintCasts == 0);
 7669   match(Set dst (CastLL dst));
 7670 
 7671   size(0);
 7672   format %{ "# castLL of $dst" %}
 7673   ins_encode(/* empty encoding */);
 7674   ins_cost(0);
 7675   ins_pipe(empty);
 7676 %}
 7677 
 7678 instruct castLL_checked_L32(rRegL dst, rFlagsReg cr)
 7679 %{
 7680   predicate(VerifyConstraintCasts > 0 && castLL_is_imm32(n));
 7681   match(Set dst (CastLL dst));
 7682 
 7683   effect(KILL cr);
 7684   format %{ "# cast_checked_LL $dst" %}
 7685   ins_encode %{
 7686     __ verify_long_in_range(_idx, bottom_type()->is_long(), $dst$$Register, noreg);
 7687   %}
 7688   ins_pipe(pipe_slow);
 7689 %}
 7690 
 7691 instruct castLL_checked(rRegL dst, rRegL tmp, rFlagsReg cr)
 7692 %{
 7693   predicate(VerifyConstraintCasts > 0 && !castLL_is_imm32(n));
 7694   match(Set dst (CastLL dst));
 7695 
 7696   effect(KILL cr, TEMP tmp);
 7697   format %{ "# cast_checked_LL $dst\tusing $tmp as TEMP" %}
 7698   ins_encode %{
 7699     __ verify_long_in_range(_idx, bottom_type()->is_long(), $dst$$Register, $tmp$$Register);
 7700   %}
 7701   ins_pipe(pipe_slow);
 7702 %}
 7703 
 7704 instruct castFF(regF dst)
 7705 %{
 7706   match(Set dst (CastFF dst));
 7707 
 7708   size(0);
 7709   format %{ "# castFF of $dst" %}
 7710   ins_encode(/* empty encoding */);
 7711   ins_cost(0);
 7712   ins_pipe(empty);
 7713 %}
 7714 
 7715 instruct castHH(regF dst)
 7716 %{
 7717   match(Set dst (CastHH dst));
 7718 
 7719   size(0);
 7720   format %{ "# castHH of $dst" %}
 7721   ins_encode(/* empty encoding */);
 7722   ins_cost(0);
 7723   ins_pipe(empty);
 7724 %}
 7725 
 7726 instruct castDD(regD dst)
 7727 %{
 7728   match(Set dst (CastDD dst));
 7729 
 7730   size(0);
 7731   format %{ "# castDD of $dst" %}
 7732   ins_encode(/* empty encoding */);
 7733   ins_cost(0);
 7734   ins_pipe(empty);
 7735 %}
 7736 
 7737 // XXX No flag versions for CompareAndSwap{P,I,L} because matcher can't match them
 7738 instruct compareAndSwapP(rRegI res,
 7739                          memory mem_ptr,
 7740                          rax_RegP oldval, rRegP newval,
 7741                          rFlagsReg cr)
 7742 %{
 7743   predicate(n->as_LoadStore()->barrier_data() == 0);
 7744   match(Set res (CompareAndSwapP mem_ptr (Binary oldval newval)));
 7745   match(Set res (WeakCompareAndSwapP mem_ptr (Binary oldval newval)));
 7746   effect(KILL cr, KILL oldval);
 7747 
 7748   format %{ "cmpxchgq $mem_ptr,$newval\t# "
 7749             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 7750             "setcc $res \t# emits sete + movzbl or setzue for APX" %}
 7751   ins_encode %{
 7752     __ lock();
 7753     __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
 7754     __ setcc(Assembler::equal, $res$$Register);
 7755   %}
 7756   ins_pipe( pipe_cmpxchg );
 7757 %}
 7758 
 7759 instruct compareAndSwapL(rRegI res,
 7760                          memory mem_ptr,
 7761                          rax_RegL oldval, rRegL newval,
 7762                          rFlagsReg cr)
 7763 %{
 7764   match(Set res (CompareAndSwapL mem_ptr (Binary oldval newval)));
 7765   match(Set res (WeakCompareAndSwapL mem_ptr (Binary oldval newval)));
 7766   effect(KILL cr, KILL oldval);
 7767 
 7768   format %{ "cmpxchgq $mem_ptr,$newval\t# "
 7769             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 7770             "setcc $res \t# emits sete + movzbl or setzue for APX" %}
 7771   ins_encode %{
 7772     __ lock();
 7773     __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
 7774     __ setcc(Assembler::equal, $res$$Register);
 7775   %}
 7776   ins_pipe( pipe_cmpxchg );
 7777 %}
 7778 
 7779 instruct compareAndSwapI(rRegI res,
 7780                          memory mem_ptr,
 7781                          rax_RegI oldval, rRegI newval,
 7782                          rFlagsReg cr)
 7783 %{
 7784   match(Set res (CompareAndSwapI mem_ptr (Binary oldval newval)));
 7785   match(Set res (WeakCompareAndSwapI mem_ptr (Binary oldval newval)));
 7786   effect(KILL cr, KILL oldval);
 7787 
 7788   format %{ "cmpxchgl $mem_ptr,$newval\t# "
 7789             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 7790             "setcc $res \t# emits sete + movzbl or setzue for APX" %}
 7791   ins_encode %{
 7792     __ lock();
 7793     __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
 7794     __ setcc(Assembler::equal, $res$$Register);
 7795   %}
 7796   ins_pipe( pipe_cmpxchg );
 7797 %}
 7798 
 7799 instruct compareAndSwapB(rRegI res,
 7800                          memory mem_ptr,
 7801                          rax_RegI oldval, rRegI newval,
 7802                          rFlagsReg cr)
 7803 %{
 7804   match(Set res (CompareAndSwapB mem_ptr (Binary oldval newval)));
 7805   match(Set res (WeakCompareAndSwapB mem_ptr (Binary oldval newval)));
 7806   effect(KILL cr, KILL oldval);
 7807 
 7808   format %{ "cmpxchgb $mem_ptr,$newval\t# "
 7809             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 7810             "setcc $res \t# emits sete + movzbl or setzue for APX" %}
 7811   ins_encode %{
 7812     __ lock();
 7813     __ cmpxchgb($newval$$Register, $mem_ptr$$Address);
 7814     __ setcc(Assembler::equal, $res$$Register);
 7815   %}
 7816   ins_pipe( pipe_cmpxchg );
 7817 %}
 7818 
 7819 instruct compareAndSwapS(rRegI res,
 7820                          memory mem_ptr,
 7821                          rax_RegI oldval, rRegI newval,
 7822                          rFlagsReg cr)
 7823 %{
 7824   match(Set res (CompareAndSwapS mem_ptr (Binary oldval newval)));
 7825   match(Set res (WeakCompareAndSwapS mem_ptr (Binary oldval newval)));
 7826   effect(KILL cr, KILL oldval);
 7827 
 7828   format %{ "cmpxchgw $mem_ptr,$newval\t# "
 7829             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 7830             "setcc $res \t# emits sete + movzbl or setzue for APX" %}
 7831   ins_encode %{
 7832     __ lock();
 7833     __ cmpxchgw($newval$$Register, $mem_ptr$$Address);
 7834     __ setcc(Assembler::equal, $res$$Register);
 7835   %}
 7836   ins_pipe( pipe_cmpxchg );
 7837 %}
 7838 
 7839 instruct compareAndSwapN(rRegI res,
 7840                           memory mem_ptr,
 7841                           rax_RegN oldval, rRegN newval,
 7842                           rFlagsReg cr) %{
 7843   predicate(n->as_LoadStore()->barrier_data() == 0);
 7844   match(Set res (CompareAndSwapN mem_ptr (Binary oldval newval)));
 7845   match(Set res (WeakCompareAndSwapN mem_ptr (Binary oldval newval)));
 7846   effect(KILL cr, KILL oldval);
 7847 
 7848   format %{ "cmpxchgl $mem_ptr,$newval\t# "
 7849             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 7850             "setcc $res \t# emits sete + movzbl or setzue for APX" %}
 7851   ins_encode %{
 7852     __ lock();
 7853     __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
 7854     __ setcc(Assembler::equal, $res$$Register);
 7855   %}
 7856   ins_pipe( pipe_cmpxchg );
 7857 %}
 7858 
 7859 instruct compareAndExchangeB(
 7860                          memory mem_ptr,
 7861                          rax_RegI oldval, rRegI newval,
 7862                          rFlagsReg cr)
 7863 %{
 7864   match(Set oldval (CompareAndExchangeB mem_ptr (Binary oldval newval)));
 7865   effect(KILL cr);
 7866 
 7867   format %{ "cmpxchgb $mem_ptr,$newval\t# "
 7868             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
 7869   ins_encode %{
 7870     __ lock();
 7871     __ cmpxchgb($newval$$Register, $mem_ptr$$Address);
 7872   %}
 7873   ins_pipe( pipe_cmpxchg );
 7874 %}
 7875 
 7876 instruct compareAndExchangeS(
 7877                          memory mem_ptr,
 7878                          rax_RegI oldval, rRegI newval,
 7879                          rFlagsReg cr)
 7880 %{
 7881   match(Set oldval (CompareAndExchangeS mem_ptr (Binary oldval newval)));
 7882   effect(KILL cr);
 7883 
 7884   format %{ "cmpxchgw $mem_ptr,$newval\t# "
 7885             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
 7886   ins_encode %{
 7887     __ lock();
 7888     __ cmpxchgw($newval$$Register, $mem_ptr$$Address);
 7889   %}
 7890   ins_pipe( pipe_cmpxchg );
 7891 %}
 7892 
 7893 instruct compareAndExchangeI(
 7894                          memory mem_ptr,
 7895                          rax_RegI oldval, rRegI newval,
 7896                          rFlagsReg cr)
 7897 %{
 7898   match(Set oldval (CompareAndExchangeI mem_ptr (Binary oldval newval)));
 7899   effect(KILL cr);
 7900 
 7901   format %{ "cmpxchgl $mem_ptr,$newval\t# "
 7902             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
 7903   ins_encode %{
 7904     __ lock();
 7905     __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
 7906   %}
 7907   ins_pipe( pipe_cmpxchg );
 7908 %}
 7909 
 7910 instruct compareAndExchangeL(
 7911                          memory mem_ptr,
 7912                          rax_RegL oldval, rRegL newval,
 7913                          rFlagsReg cr)
 7914 %{
 7915   match(Set oldval (CompareAndExchangeL mem_ptr (Binary oldval newval)));
 7916   effect(KILL cr);
 7917 
 7918   format %{ "cmpxchgq $mem_ptr,$newval\t# "
 7919             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
 7920   ins_encode %{
 7921     __ lock();
 7922     __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
 7923   %}
 7924   ins_pipe( pipe_cmpxchg );
 7925 %}
 7926 
 7927 instruct compareAndExchangeN(
 7928                           memory mem_ptr,
 7929                           rax_RegN oldval, rRegN newval,
 7930                           rFlagsReg cr) %{
 7931   predicate(n->as_LoadStore()->barrier_data() == 0);
 7932   match(Set oldval (CompareAndExchangeN mem_ptr (Binary oldval newval)));
 7933   effect(KILL cr);
 7934 
 7935   format %{ "cmpxchgl $mem_ptr,$newval\t# "
 7936             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %}
 7937   ins_encode %{
 7938     __ lock();
 7939     __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
 7940   %}
 7941   ins_pipe( pipe_cmpxchg );
 7942 %}
 7943 
 7944 instruct compareAndExchangeP(
 7945                          memory mem_ptr,
 7946                          rax_RegP oldval, rRegP newval,
 7947                          rFlagsReg cr)
 7948 %{
 7949   predicate(n->as_LoadStore()->barrier_data() == 0);
 7950   match(Set oldval (CompareAndExchangeP mem_ptr (Binary oldval newval)));
 7951   effect(KILL cr);
 7952 
 7953   format %{ "cmpxchgq $mem_ptr,$newval\t# "
 7954             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %}
 7955   ins_encode %{
 7956     __ lock();
 7957     __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
 7958   %}
 7959   ins_pipe( pipe_cmpxchg );
 7960 %}
 7961 
 7962 instruct xaddB_reg_no_res(memory mem, Universe dummy, rRegI add, rFlagsReg cr) %{
 7963   predicate(n->as_LoadStore()->result_not_used());
 7964   match(Set dummy (GetAndAddB mem add));
 7965   effect(KILL cr);
 7966   format %{ "addb_lock   $mem, $add" %}
 7967   ins_encode %{
 7968     __ lock();
 7969     __ addb($mem$$Address, $add$$Register);
 7970   %}
 7971   ins_pipe(pipe_cmpxchg);
 7972 %}
 7973 
 7974 instruct xaddB_imm_no_res(memory mem, Universe dummy, immI add, rFlagsReg cr) %{
 7975   predicate(n->as_LoadStore()->result_not_used());
 7976   match(Set dummy (GetAndAddB mem add));
 7977   effect(KILL cr);
 7978   format %{ "addb_lock   $mem, $add" %}
 7979   ins_encode %{
 7980     __ lock();
 7981     __ addb($mem$$Address, $add$$constant);
 7982   %}
 7983   ins_pipe(pipe_cmpxchg);
 7984 %}
 7985 
 7986 instruct xaddB(memory mem, rRegI newval, rFlagsReg cr) %{
 7987   predicate(!n->as_LoadStore()->result_not_used());
 7988   match(Set newval (GetAndAddB mem newval));
 7989   effect(KILL cr);
 7990   format %{ "xaddb_lock  $mem, $newval" %}
 7991   ins_encode %{
 7992     __ lock();
 7993     __ xaddb($mem$$Address, $newval$$Register);
 7994   %}
 7995   ins_pipe(pipe_cmpxchg);
 7996 %}
 7997 
 7998 instruct xaddS_reg_no_res(memory mem, Universe dummy, rRegI add, rFlagsReg cr) %{
 7999   predicate(n->as_LoadStore()->result_not_used());
 8000   match(Set dummy (GetAndAddS mem add));
 8001   effect(KILL cr);
 8002   format %{ "addw_lock   $mem, $add" %}
 8003   ins_encode %{
 8004     __ lock();
 8005     __ addw($mem$$Address, $add$$Register);
 8006   %}
 8007   ins_pipe(pipe_cmpxchg);
 8008 %}
 8009 
 8010 instruct xaddS_imm_no_res(memory mem, Universe dummy, immI add, rFlagsReg cr) %{
 8011   predicate(UseStoreImmI16 && n->as_LoadStore()->result_not_used());
 8012   match(Set dummy (GetAndAddS mem add));
 8013   effect(KILL cr);
 8014   format %{ "addw_lock   $mem, $add" %}
 8015   ins_encode %{
 8016     __ lock();
 8017     __ addw($mem$$Address, $add$$constant);
 8018   %}
 8019   ins_pipe(pipe_cmpxchg);
 8020 %}
 8021 
 8022 instruct xaddS(memory mem, rRegI newval, rFlagsReg cr) %{
 8023   predicate(!n->as_LoadStore()->result_not_used());
 8024   match(Set newval (GetAndAddS mem newval));
 8025   effect(KILL cr);
 8026   format %{ "xaddw_lock  $mem, $newval" %}
 8027   ins_encode %{
 8028     __ lock();
 8029     __ xaddw($mem$$Address, $newval$$Register);
 8030   %}
 8031   ins_pipe(pipe_cmpxchg);
 8032 %}
 8033 
 8034 instruct xaddI_reg_no_res(memory mem, Universe dummy, rRegI add, rFlagsReg cr) %{
 8035   predicate(n->as_LoadStore()->result_not_used());
 8036   match(Set dummy (GetAndAddI mem add));
 8037   effect(KILL cr);
 8038   format %{ "addl_lock   $mem, $add" %}
 8039   ins_encode %{
 8040     __ lock();
 8041     __ addl($mem$$Address, $add$$Register);
 8042   %}
 8043   ins_pipe(pipe_cmpxchg);
 8044 %}
 8045 
 8046 instruct xaddI_imm_no_res(memory mem, Universe dummy, immI add, rFlagsReg cr) %{
 8047   predicate(n->as_LoadStore()->result_not_used());
 8048   match(Set dummy (GetAndAddI mem add));
 8049   effect(KILL cr);
 8050   format %{ "addl_lock   $mem, $add" %}
 8051   ins_encode %{
 8052     __ lock();
 8053     __ addl($mem$$Address, $add$$constant);
 8054   %}
 8055   ins_pipe(pipe_cmpxchg);
 8056 %}
 8057 
 8058 instruct xaddI(memory mem, rRegI newval, rFlagsReg cr) %{
 8059   predicate(!n->as_LoadStore()->result_not_used());
 8060   match(Set newval (GetAndAddI mem newval));
 8061   effect(KILL cr);
 8062   format %{ "xaddl_lock  $mem, $newval" %}
 8063   ins_encode %{
 8064     __ lock();
 8065     __ xaddl($mem$$Address, $newval$$Register);
 8066   %}
 8067   ins_pipe(pipe_cmpxchg);
 8068 %}
 8069 
 8070 instruct xaddL_reg_no_res(memory mem, Universe dummy, rRegL add, rFlagsReg cr) %{
 8071   predicate(n->as_LoadStore()->result_not_used());
 8072   match(Set dummy (GetAndAddL mem add));
 8073   effect(KILL cr);
 8074   format %{ "addq_lock   $mem, $add" %}
 8075   ins_encode %{
 8076     __ lock();
 8077     __ addq($mem$$Address, $add$$Register);
 8078   %}
 8079   ins_pipe(pipe_cmpxchg);
 8080 %}
 8081 
 8082 instruct xaddL_imm_no_res(memory mem, Universe dummy, immL32 add, rFlagsReg cr) %{
 8083   predicate(n->as_LoadStore()->result_not_used());
 8084   match(Set dummy (GetAndAddL mem add));
 8085   effect(KILL cr);
 8086   format %{ "addq_lock   $mem, $add" %}
 8087   ins_encode %{
 8088     __ lock();
 8089     __ addq($mem$$Address, $add$$constant);
 8090   %}
 8091   ins_pipe(pipe_cmpxchg);
 8092 %}
 8093 
 8094 instruct xaddL(memory mem, rRegL newval, rFlagsReg cr) %{
 8095   predicate(!n->as_LoadStore()->result_not_used());
 8096   match(Set newval (GetAndAddL mem newval));
 8097   effect(KILL cr);
 8098   format %{ "xaddq_lock  $mem, $newval" %}
 8099   ins_encode %{
 8100     __ lock();
 8101     __ xaddq($mem$$Address, $newval$$Register);
 8102   %}
 8103   ins_pipe(pipe_cmpxchg);
 8104 %}
 8105 
 8106 instruct xchgB( memory mem, rRegI newval) %{
 8107   match(Set newval (GetAndSetB mem newval));
 8108   format %{ "XCHGB  $newval,[$mem]" %}
 8109   ins_encode %{
 8110     __ xchgb($newval$$Register, $mem$$Address);
 8111   %}
 8112   ins_pipe( pipe_cmpxchg );
 8113 %}
 8114 
 8115 instruct xchgS( memory mem, rRegI newval) %{
 8116   match(Set newval (GetAndSetS mem newval));
 8117   format %{ "XCHGW  $newval,[$mem]" %}
 8118   ins_encode %{
 8119     __ xchgw($newval$$Register, $mem$$Address);
 8120   %}
 8121   ins_pipe( pipe_cmpxchg );
 8122 %}
 8123 
 8124 instruct xchgI( memory mem, rRegI newval) %{
 8125   match(Set newval (GetAndSetI mem newval));
 8126   format %{ "XCHGL  $newval,[$mem]" %}
 8127   ins_encode %{
 8128     __ xchgl($newval$$Register, $mem$$Address);
 8129   %}
 8130   ins_pipe( pipe_cmpxchg );
 8131 %}
 8132 
 8133 instruct xchgL( memory mem, rRegL newval) %{
 8134   match(Set newval (GetAndSetL mem newval));
 8135   format %{ "XCHGL  $newval,[$mem]" %}
 8136   ins_encode %{
 8137     __ xchgq($newval$$Register, $mem$$Address);
 8138   %}
 8139   ins_pipe( pipe_cmpxchg );
 8140 %}
 8141 
 8142 instruct xchgP( memory mem, rRegP newval) %{
 8143   match(Set newval (GetAndSetP mem newval));
 8144   predicate(n->as_LoadStore()->barrier_data() == 0);
 8145   format %{ "XCHGQ  $newval,[$mem]" %}
 8146   ins_encode %{
 8147     __ xchgq($newval$$Register, $mem$$Address);
 8148   %}
 8149   ins_pipe( pipe_cmpxchg );
 8150 %}
 8151 
 8152 instruct xchgN( memory mem, rRegN newval) %{
 8153   predicate(n->as_LoadStore()->barrier_data() == 0);
 8154   match(Set newval (GetAndSetN mem newval));
 8155   format %{ "XCHGL  $newval,$mem]" %}
 8156   ins_encode %{
 8157     __ xchgl($newval$$Register, $mem$$Address);
 8158   %}
 8159   ins_pipe( pipe_cmpxchg );
 8160 %}
 8161 
 8162 //----------Abs Instructions-------------------------------------------
 8163 
 8164 // Integer Absolute Instructions
 8165 instruct absI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 8166 %{
 8167   match(Set dst (AbsI src));
 8168   effect(TEMP dst, KILL cr);
 8169   format %{ "xorl    $dst, $dst\t# abs int\n\t"
 8170             "subl    $dst, $src\n\t"
 8171             "cmovll  $dst, $src" %}
 8172   ins_encode %{
 8173     __ xorl($dst$$Register, $dst$$Register);
 8174     __ subl($dst$$Register, $src$$Register);
 8175     __ cmovl(Assembler::less, $dst$$Register, $src$$Register);
 8176   %}
 8177 
 8178   ins_pipe(ialu_reg_reg);
 8179 %}
 8180 
 8181 // Long Absolute Instructions
 8182 instruct absL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 8183 %{
 8184   match(Set dst (AbsL src));
 8185   effect(TEMP dst, KILL cr);
 8186   format %{ "xorl    $dst, $dst\t# abs long\n\t"
 8187             "subq    $dst, $src\n\t"
 8188             "cmovlq  $dst, $src" %}
 8189   ins_encode %{
 8190     __ xorl($dst$$Register, $dst$$Register);
 8191     __ subq($dst$$Register, $src$$Register);
 8192     __ cmovq(Assembler::less, $dst$$Register, $src$$Register);
 8193   %}
 8194 
 8195   ins_pipe(ialu_reg_reg);
 8196 %}
 8197 
 8198 //----------Subtraction Instructions-------------------------------------------
 8199 
 8200 // Integer Subtraction Instructions
 8201 instruct subI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 8202 %{
 8203   predicate(!UseAPX);
 8204   match(Set dst (SubI dst src));
 8205   effect(KILL cr);
 8206   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 8207 
 8208   format %{ "subl    $dst, $src\t# int" %}
 8209   ins_encode %{
 8210     __ subl($dst$$Register, $src$$Register);
 8211   %}
 8212   ins_pipe(ialu_reg_reg);
 8213 %}
 8214 
 8215 instruct subI_rReg_ndd(rRegI dst, rRegI src1, rRegI src2, rFlagsReg cr)
 8216 %{
 8217   predicate(UseAPX);
 8218   match(Set dst (SubI src1 src2));
 8219   effect(KILL cr);
 8220   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 8221 
 8222   format %{ "esubl    $dst, $src1, $src2\t# int ndd" %}
 8223   ins_encode %{
 8224     __ esubl($dst$$Register, $src1$$Register, $src2$$Register, false);
 8225   %}
 8226   ins_pipe(ialu_reg_reg);
 8227 %}
 8228 
 8229 instruct subI_rReg_rReg_imm_ndd(rRegI dst, rRegI src1, immI src2, rFlagsReg cr)
 8230 %{
 8231   predicate(UseAPX);
 8232   match(Set dst (SubI src1 src2));
 8233   effect(KILL cr);
 8234   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 8235 
 8236   format %{ "esubl    $dst, $src1, $src2\t# int ndd" %}
 8237   ins_encode %{
 8238     __ esubl($dst$$Register, $src1$$Register, $src2$$constant, false);
 8239   %}
 8240   ins_pipe(ialu_reg_reg);
 8241 %}
 8242 
 8243 instruct subI_rReg_mem_imm_ndd(rRegI dst, memory src1, immI src2, rFlagsReg cr)
 8244 %{
 8245   predicate(UseAPX);
 8246   match(Set dst (SubI (LoadI src1) src2));
 8247   effect(KILL cr);
 8248   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 8249 
 8250   format %{ "esubl    $dst, $src1, $src2\t# int ndd" %}
 8251   ins_encode %{
 8252     __ esubl($dst$$Register, $src1$$Address, $src2$$constant, false);
 8253   %}
 8254   ins_pipe(ialu_reg_reg);
 8255 %}
 8256 
 8257 instruct subI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
 8258 %{
 8259   predicate(!UseAPX);
 8260   match(Set dst (SubI dst (LoadI src)));
 8261   effect(KILL cr);
 8262   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 8263 
 8264   ins_cost(150);
 8265   format %{ "subl    $dst, $src\t# int" %}
 8266   ins_encode %{
 8267     __ subl($dst$$Register, $src$$Address);
 8268   %}
 8269   ins_pipe(ialu_reg_mem);
 8270 %}
 8271 
 8272 instruct subI_rReg_rReg_mem_ndd(rRegI dst, rRegI src1, memory src2, rFlagsReg cr)
 8273 %{
 8274   predicate(UseAPX);
 8275   match(Set dst (SubI src1 (LoadI src2)));
 8276   effect(KILL cr);
 8277   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 8278 
 8279   ins_cost(150);
 8280   format %{ "esubl    $dst, $src1, $src2\t# int ndd" %}
 8281   ins_encode %{
 8282     __ esubl($dst$$Register, $src1$$Register, $src2$$Address, false);
 8283   %}
 8284   ins_pipe(ialu_reg_mem);
 8285 %}
 8286 
 8287 instruct subI_rReg_mem_rReg_ndd(rRegI dst, memory src1, rRegI src2, rFlagsReg cr)
 8288 %{
 8289   predicate(UseAPX);
 8290   match(Set dst (SubI (LoadI src1) src2));
 8291   effect(KILL cr);
 8292   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 8293 
 8294   ins_cost(150);
 8295   format %{ "esubl    $dst, $src1, $src2\t# int ndd" %}
 8296   ins_encode %{
 8297     __ esubl($dst$$Register, $src1$$Address, $src2$$Register, false);
 8298   %}
 8299   ins_pipe(ialu_reg_mem);
 8300 %}
 8301 
 8302 instruct subI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 8303 %{
 8304   match(Set dst (StoreI dst (SubI (LoadI dst) src)));
 8305   effect(KILL cr);
 8306   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 8307 
 8308   ins_cost(150);
 8309   format %{ "subl    $dst, $src\t# int" %}
 8310   ins_encode %{
 8311     __ subl($dst$$Address, $src$$Register);
 8312   %}
 8313   ins_pipe(ialu_mem_reg);
 8314 %}
 8315 
 8316 instruct subL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 8317 %{
 8318   predicate(!UseAPX);
 8319   match(Set dst (SubL dst src));
 8320   effect(KILL cr);
 8321   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 8322 
 8323   format %{ "subq    $dst, $src\t# long" %}
 8324   ins_encode %{
 8325     __ subq($dst$$Register, $src$$Register);
 8326   %}
 8327   ins_pipe(ialu_reg_reg);
 8328 %}
 8329 
 8330 instruct subL_rReg_ndd(rRegL dst, rRegL src1, rRegL src2, rFlagsReg cr)
 8331 %{
 8332   predicate(UseAPX);
 8333   match(Set dst (SubL src1 src2));
 8334   effect(KILL cr);
 8335   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 8336 
 8337   format %{ "esubq    $dst, $src1, $src2\t# long ndd" %}
 8338   ins_encode %{
 8339     __ esubq($dst$$Register, $src1$$Register, $src2$$Register, false);
 8340   %}
 8341   ins_pipe(ialu_reg_reg);
 8342 %}
 8343 
 8344 instruct subL_rReg_rReg_imm_ndd(rRegL dst, rRegL src1, immL32 src2, rFlagsReg cr)
 8345 %{
 8346   predicate(UseAPX);
 8347   match(Set dst (SubL src1 src2));
 8348   effect(KILL cr);
 8349   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 8350 
 8351   format %{ "esubq    $dst, $src1, $src2\t# long ndd" %}
 8352   ins_encode %{
 8353     __ esubq($dst$$Register, $src1$$Register, $src2$$constant, false);
 8354   %}
 8355   ins_pipe(ialu_reg_reg);
 8356 %}
 8357 
 8358 instruct subL_rReg_mem_imm_ndd(rRegL dst, memory src1, immL32 src2, rFlagsReg cr)
 8359 %{
 8360   predicate(UseAPX);
 8361   match(Set dst (SubL (LoadL src1) src2));
 8362   effect(KILL cr);
 8363   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 8364 
 8365   format %{ "esubq    $dst, $src1, $src2\t# long ndd" %}
 8366   ins_encode %{
 8367     __ esubq($dst$$Register, $src1$$Address, $src2$$constant, false);
 8368   %}
 8369   ins_pipe(ialu_reg_reg);
 8370 %}
 8371 
 8372 instruct subL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
 8373 %{
 8374   predicate(!UseAPX);
 8375   match(Set dst (SubL dst (LoadL src)));
 8376   effect(KILL cr);
 8377   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 8378 
 8379   ins_cost(150);
 8380   format %{ "subq    $dst, $src\t# long" %}
 8381   ins_encode %{
 8382     __ subq($dst$$Register, $src$$Address);
 8383   %}
 8384   ins_pipe(ialu_reg_mem);
 8385 %}
 8386 
 8387 instruct subL_rReg_rReg_mem_ndd(rRegL dst, rRegL src1, memory src2, rFlagsReg cr)
 8388 %{
 8389   predicate(UseAPX);
 8390   match(Set dst (SubL src1 (LoadL src2)));
 8391   effect(KILL cr);
 8392   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 8393 
 8394   ins_cost(150);
 8395   format %{ "esubq    $dst, $src1, $src2\t# long ndd" %}
 8396   ins_encode %{
 8397     __ esubq($dst$$Register, $src1$$Register, $src2$$Address, false);
 8398   %}
 8399   ins_pipe(ialu_reg_mem);
 8400 %}
 8401 
 8402 instruct subL_rReg_mem_rReg_ndd(rRegL dst, memory src1, rRegL src2, rFlagsReg cr)
 8403 %{
 8404   predicate(UseAPX);
 8405   match(Set dst (SubL (LoadL src1) src2));
 8406   effect(KILL cr);
 8407   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 8408 
 8409   ins_cost(150);
 8410   format %{ "esubq    $dst, $src1, $src2\t# long ndd" %}
 8411   ins_encode %{
 8412     __ esubq($dst$$Register, $src1$$Address, $src2$$Register, false);
 8413   %}
 8414   ins_pipe(ialu_reg_mem);
 8415 %}
 8416 
 8417 instruct subL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
 8418 %{
 8419   match(Set dst (StoreL dst (SubL (LoadL dst) src)));
 8420   effect(KILL cr);
 8421   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 8422 
 8423   ins_cost(150);
 8424   format %{ "subq    $dst, $src\t# long" %}
 8425   ins_encode %{
 8426     __ subq($dst$$Address, $src$$Register);
 8427   %}
 8428   ins_pipe(ialu_mem_reg);
 8429 %}
 8430 
 8431 // Subtract from a pointer
 8432 // XXX hmpf???
 8433 instruct subP_rReg(rRegP dst, rRegI src, immI_0 zero, rFlagsReg cr)
 8434 %{
 8435   match(Set dst (AddP dst (SubI zero src)));
 8436   effect(KILL cr);
 8437 
 8438   format %{ "subq    $dst, $src\t# ptr - int" %}
 8439   ins_encode %{
 8440     __ subq($dst$$Register, $src$$Register);
 8441   %}
 8442   ins_pipe(ialu_reg_reg);
 8443 %}
 8444 
 8445 instruct negI_rReg(rRegI dst, immI_0 zero, rFlagsReg cr)
 8446 %{
 8447   predicate(!UseAPX);
 8448   match(Set dst (SubI zero dst));
 8449   effect(KILL cr);
 8450   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
 8451 
 8452   format %{ "negl    $dst\t# int" %}
 8453   ins_encode %{
 8454     __ negl($dst$$Register);
 8455   %}
 8456   ins_pipe(ialu_reg);
 8457 %}
 8458 
 8459 instruct negI_rReg_ndd(rRegI dst, rRegI src, immI_0 zero, rFlagsReg cr)
 8460 %{
 8461   predicate(UseAPX);
 8462   match(Set dst (SubI zero src));
 8463   effect(KILL cr);
 8464   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
 8465 
 8466   format %{ "enegl    $dst, $src\t# int ndd" %}
 8467   ins_encode %{
 8468     __ enegl($dst$$Register, $src$$Register, false);
 8469   %}
 8470   ins_pipe(ialu_reg);
 8471 %}
 8472 
 8473 instruct negI_rReg_2(rRegI dst, rFlagsReg cr)
 8474 %{
 8475   predicate(!UseAPX);
 8476   match(Set dst (NegI dst));
 8477   effect(KILL cr);
 8478   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
 8479 
 8480   format %{ "negl    $dst\t# int" %}
 8481   ins_encode %{
 8482     __ negl($dst$$Register);
 8483   %}
 8484   ins_pipe(ialu_reg);
 8485 %}
 8486 
 8487 instruct negI_rReg_2_ndd(rRegI dst, rRegI src, rFlagsReg cr)
 8488 %{
 8489   predicate(UseAPX);
 8490   match(Set dst (NegI src));
 8491   effect(KILL cr);
 8492   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
 8493 
 8494   format %{ "enegl    $dst, $src\t# int ndd" %}
 8495   ins_encode %{
 8496     __ enegl($dst$$Register, $src$$Register, false);
 8497   %}
 8498   ins_pipe(ialu_reg);
 8499 %}
 8500 
 8501 instruct negI_mem(memory dst, immI_0 zero, rFlagsReg cr)
 8502 %{
 8503   match(Set dst (StoreI dst (SubI zero (LoadI dst))));
 8504   effect(KILL cr);
 8505   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
 8506 
 8507   format %{ "negl    $dst\t# int" %}
 8508   ins_encode %{
 8509     __ negl($dst$$Address);
 8510   %}
 8511   ins_pipe(ialu_reg);
 8512 %}
 8513 
 8514 instruct negL_rReg(rRegL dst, immL0 zero, rFlagsReg cr)
 8515 %{
 8516   predicate(!UseAPX);
 8517   match(Set dst (SubL zero dst));
 8518   effect(KILL cr);
 8519   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
 8520 
 8521   format %{ "negq    $dst\t# long" %}
 8522   ins_encode %{
 8523     __ negq($dst$$Register);
 8524   %}
 8525   ins_pipe(ialu_reg);
 8526 %}
 8527 
 8528 instruct negL_rReg_ndd(rRegL dst, rRegL src, immL0 zero, rFlagsReg cr)
 8529 %{
 8530   predicate(UseAPX);
 8531   match(Set dst (SubL zero src));
 8532   effect(KILL cr);
 8533   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
 8534 
 8535   format %{ "enegq    $dst, $src\t# long ndd" %}
 8536   ins_encode %{
 8537     __ enegq($dst$$Register, $src$$Register, false);
 8538   %}
 8539   ins_pipe(ialu_reg);
 8540 %}
 8541 
 8542 instruct negL_rReg_2(rRegL dst, rFlagsReg cr)
 8543 %{
 8544   predicate(!UseAPX);
 8545   match(Set dst (NegL dst));
 8546   effect(KILL cr);
 8547   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
 8548 
 8549   format %{ "negq    $dst\t# int" %}
 8550   ins_encode %{
 8551     __ negq($dst$$Register);
 8552   %}
 8553   ins_pipe(ialu_reg);
 8554 %}
 8555 
 8556 instruct negL_rReg_2_ndd(rRegL dst, rRegL src, rFlagsReg cr)
 8557 %{
 8558   predicate(UseAPX);
 8559   match(Set dst (NegL src));
 8560   effect(KILL cr);
 8561   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
 8562 
 8563   format %{ "enegq    $dst, $src\t# long ndd" %}
 8564   ins_encode %{
 8565     __ enegq($dst$$Register, $src$$Register, false);
 8566   %}
 8567   ins_pipe(ialu_reg);
 8568 %}
 8569 
 8570 instruct negL_mem(memory dst, immL0 zero, rFlagsReg cr)
 8571 %{
 8572   match(Set dst (StoreL dst (SubL zero (LoadL dst))));
 8573   effect(KILL cr);
 8574   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
 8575 
 8576   format %{ "negq    $dst\t# long" %}
 8577   ins_encode %{
 8578     __ negq($dst$$Address);
 8579   %}
 8580   ins_pipe(ialu_reg);
 8581 %}
 8582 
 8583 //----------Multiplication/Division Instructions-------------------------------
 8584 // Integer Multiplication Instructions
 8585 // Multiply Register
 8586 
 8587 instruct mulI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 8588 %{
 8589   predicate(!UseAPX);
 8590   match(Set dst (MulI dst src));
 8591   effect(KILL cr);
 8592 
 8593   ins_cost(300);
 8594   format %{ "imull   $dst, $src\t# int" %}
 8595   ins_encode %{
 8596     __ imull($dst$$Register, $src$$Register);
 8597   %}
 8598   ins_pipe(ialu_reg_reg_alu0);
 8599 %}
 8600 
 8601 instruct mulI_rReg_ndd(rRegI dst, rRegI src1, rRegI src2, rFlagsReg cr)
 8602 %{
 8603   predicate(UseAPX);
 8604   match(Set dst (MulI src1 src2));
 8605   effect(KILL cr);
 8606 
 8607   ins_cost(300);
 8608   format %{ "eimull   $dst, $src1, $src2\t# int ndd" %}
 8609   ins_encode %{
 8610     __ eimull($dst$$Register, $src1$$Register, $src2$$Register, false);
 8611   %}
 8612   ins_pipe(ialu_reg_reg_alu0);
 8613 %}
 8614 
 8615 instruct mulI_rReg_imm(rRegI dst, rRegI src, immI imm, rFlagsReg cr)
 8616 %{
 8617   match(Set dst (MulI src imm));
 8618   effect(KILL cr);
 8619 
 8620   ins_cost(300);
 8621   format %{ "imull   $dst, $src, $imm\t# int" %}
 8622   ins_encode %{
 8623     __ imull($dst$$Register, $src$$Register, $imm$$constant);
 8624   %}
 8625   ins_pipe(ialu_reg_reg_alu0);
 8626 %}
 8627 
 8628 instruct mulI_mem(rRegI dst, memory src, rFlagsReg cr)
 8629 %{
 8630   predicate(!UseAPX);
 8631   match(Set dst (MulI dst (LoadI src)));
 8632   effect(KILL cr);
 8633 
 8634   ins_cost(350);
 8635   format %{ "imull   $dst, $src\t# int" %}
 8636   ins_encode %{
 8637     __ imull($dst$$Register, $src$$Address);
 8638   %}
 8639   ins_pipe(ialu_reg_mem_alu0);
 8640 %}
 8641 
 8642 instruct mulI_rReg_rReg_mem_ndd(rRegI dst, rRegI src1, memory src2, rFlagsReg cr)
 8643 %{
 8644   predicate(UseAPX);
 8645   match(Set dst (MulI src1 (LoadI src2)));
 8646   effect(KILL cr);
 8647 
 8648   ins_cost(350);
 8649   format %{ "eimull   $dst, $src1, $src2\t# int ndd" %}
 8650   ins_encode %{
 8651     __ eimull($dst$$Register, $src1$$Register, $src2$$Address, false);
 8652   %}
 8653   ins_pipe(ialu_reg_mem_alu0);
 8654 %}
 8655 
 8656 instruct mulI_mem_imm(rRegI dst, memory src, immI imm, rFlagsReg cr)
 8657 %{
 8658   match(Set dst (MulI (LoadI src) imm));
 8659   effect(KILL cr);
 8660 
 8661   ins_cost(300);
 8662   format %{ "imull   $dst, $src, $imm\t# int" %}
 8663   ins_encode %{
 8664     __ imull($dst$$Register, $src$$Address, $imm$$constant);
 8665   %}
 8666   ins_pipe(ialu_reg_mem_alu0);
 8667 %}
 8668 
 8669 instruct mulAddS2I_rReg(rRegI dst, rRegI src1, rRegI src2, rRegI src3, rFlagsReg cr)
 8670 %{
 8671   match(Set dst (MulAddS2I (Binary dst src1) (Binary src2 src3)));
 8672   effect(KILL cr, KILL src2);
 8673 
 8674   expand %{ mulI_rReg(dst, src1, cr);
 8675            mulI_rReg(src2, src3, cr);
 8676            addI_rReg(dst, src2, cr); %}
 8677 %}
 8678 
 8679 instruct mulL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 8680 %{
 8681   predicate(!UseAPX);
 8682   match(Set dst (MulL dst src));
 8683   effect(KILL cr);
 8684 
 8685   ins_cost(300);
 8686   format %{ "imulq   $dst, $src\t# long" %}
 8687   ins_encode %{
 8688     __ imulq($dst$$Register, $src$$Register);
 8689   %}
 8690   ins_pipe(ialu_reg_reg_alu0);
 8691 %}
 8692 
 8693 instruct mulL_rReg_ndd(rRegL dst, rRegL src1, rRegL src2, rFlagsReg cr)
 8694 %{
 8695   predicate(UseAPX);
 8696   match(Set dst (MulL src1 src2));
 8697   effect(KILL cr);
 8698 
 8699   ins_cost(300);
 8700   format %{ "eimulq   $dst, $src1, $src2\t# long ndd" %}
 8701   ins_encode %{
 8702     __ eimulq($dst$$Register, $src1$$Register, $src2$$Register, false);
 8703   %}
 8704   ins_pipe(ialu_reg_reg_alu0);
 8705 %}
 8706 
 8707 instruct mulL_rReg_imm(rRegL dst, rRegL src, immL32 imm, rFlagsReg cr)
 8708 %{
 8709   match(Set dst (MulL src imm));
 8710   effect(KILL cr);
 8711 
 8712   ins_cost(300);
 8713   format %{ "imulq   $dst, $src, $imm\t# long" %}
 8714   ins_encode %{
 8715     __ imulq($dst$$Register, $src$$Register, $imm$$constant);
 8716   %}
 8717   ins_pipe(ialu_reg_reg_alu0);
 8718 %}
 8719 
 8720 instruct mulL_mem(rRegL dst, memory src, rFlagsReg cr)
 8721 %{
 8722   predicate(!UseAPX);
 8723   match(Set dst (MulL dst (LoadL src)));
 8724   effect(KILL cr);
 8725 
 8726   ins_cost(350);
 8727   format %{ "imulq   $dst, $src\t# long" %}
 8728   ins_encode %{
 8729     __ imulq($dst$$Register, $src$$Address);
 8730   %}
 8731   ins_pipe(ialu_reg_mem_alu0);
 8732 %}
 8733 
 8734 instruct mulL_rReg_rReg_mem_ndd(rRegL dst, rRegL src1, memory src2, rFlagsReg cr)
 8735 %{
 8736   predicate(UseAPX);
 8737   match(Set dst (MulL src1 (LoadL src2)));
 8738   effect(KILL cr);
 8739 
 8740   ins_cost(350);
 8741   format %{ "eimulq   $dst, $src1, $src2 \t# long" %}
 8742   ins_encode %{
 8743     __ eimulq($dst$$Register, $src1$$Register, $src2$$Address, false);
 8744   %}
 8745   ins_pipe(ialu_reg_mem_alu0);
 8746 %}
 8747 
 8748 instruct mulL_mem_imm(rRegL dst, memory src, immL32 imm, rFlagsReg cr)
 8749 %{
 8750   match(Set dst (MulL (LoadL src) imm));
 8751   effect(KILL cr);
 8752 
 8753   ins_cost(300);
 8754   format %{ "imulq   $dst, $src, $imm\t# long" %}
 8755   ins_encode %{
 8756     __ imulq($dst$$Register, $src$$Address, $imm$$constant);
 8757   %}
 8758   ins_pipe(ialu_reg_mem_alu0);
 8759 %}
 8760 
 8761 instruct mulHiL_rReg(rdx_RegL dst, rRegL src, rax_RegL rax, rFlagsReg cr)
 8762 %{
 8763   match(Set dst (MulHiL src rax));
 8764   effect(USE_KILL rax, KILL cr);
 8765 
 8766   ins_cost(300);
 8767   format %{ "imulq   RDX:RAX, RAX, $src\t# mulhi" %}
 8768   ins_encode %{
 8769     __ imulq($src$$Register);
 8770   %}
 8771   ins_pipe(ialu_reg_reg_alu0);
 8772 %}
 8773 
 8774 instruct umulHiL_rReg(rdx_RegL dst, rRegL src, rax_RegL rax, rFlagsReg cr)
 8775 %{
 8776   match(Set dst (UMulHiL src rax));
 8777   effect(USE_KILL rax, KILL cr);
 8778 
 8779   ins_cost(300);
 8780   format %{ "mulq   RDX:RAX, RAX, $src\t# umulhi" %}
 8781   ins_encode %{
 8782     __ mulq($src$$Register);
 8783   %}
 8784   ins_pipe(ialu_reg_reg_alu0);
 8785 %}
 8786 
 8787 instruct divI_rReg(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div,
 8788                    rFlagsReg cr)
 8789 %{
 8790   match(Set rax (DivI rax div));
 8791   effect(KILL rdx, KILL cr);
 8792 
 8793   ins_cost(30*100+10*100); // XXX
 8794   format %{ "cmpl    rax, 0x80000000\t# idiv\n\t"
 8795             "jne,s   normal\n\t"
 8796             "xorl    rdx, rdx\n\t"
 8797             "cmpl    $div, -1\n\t"
 8798             "je,s    done\n"
 8799     "normal: cdql\n\t"
 8800             "idivl   $div\n"
 8801     "done:"        %}
 8802   ins_encode(cdql_enc(div));
 8803   ins_pipe(ialu_reg_reg_alu0);
 8804 %}
 8805 
 8806 instruct divL_rReg(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div,
 8807                    rFlagsReg cr)
 8808 %{
 8809   match(Set rax (DivL rax div));
 8810   effect(KILL rdx, KILL cr);
 8811 
 8812   ins_cost(30*100+10*100); // XXX
 8813   format %{ "movq    rdx, 0x8000000000000000\t# ldiv\n\t"
 8814             "cmpq    rax, rdx\n\t"
 8815             "jne,s   normal\n\t"
 8816             "xorl    rdx, rdx\n\t"
 8817             "cmpq    $div, -1\n\t"
 8818             "je,s    done\n"
 8819     "normal: cdqq\n\t"
 8820             "idivq   $div\n"
 8821     "done:"        %}
 8822   ins_encode(cdqq_enc(div));
 8823   ins_pipe(ialu_reg_reg_alu0);
 8824 %}
 8825 
 8826 instruct udivI_rReg(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div, rFlagsReg cr)
 8827 %{
 8828   match(Set rax (UDivI rax div));
 8829   effect(KILL rdx, KILL cr);
 8830 
 8831   ins_cost(300);
 8832   format %{ "udivl $rax,$rax,$div\t# UDivI\n" %}
 8833   ins_encode %{
 8834     __ udivI($rax$$Register, $div$$Register, $rdx$$Register);
 8835   %}
 8836   ins_pipe(ialu_reg_reg_alu0);
 8837 %}
 8838 
 8839 instruct udivL_rReg(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div, rFlagsReg cr)
 8840 %{
 8841   match(Set rax (UDivL rax div));
 8842   effect(KILL rdx, KILL cr);
 8843 
 8844   ins_cost(300);
 8845   format %{ "udivq $rax,$rax,$div\t# UDivL\n" %}
 8846   ins_encode %{
 8847      __ udivL($rax$$Register, $div$$Register, $rdx$$Register);
 8848   %}
 8849   ins_pipe(ialu_reg_reg_alu0);
 8850 %}
 8851 
 8852 // Integer DIVMOD with Register, both quotient and mod results
 8853 instruct divModI_rReg_divmod(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div,
 8854                              rFlagsReg cr)
 8855 %{
 8856   match(DivModI rax div);
 8857   effect(KILL cr);
 8858 
 8859   ins_cost(30*100+10*100); // XXX
 8860   format %{ "cmpl    rax, 0x80000000\t# idiv\n\t"
 8861             "jne,s   normal\n\t"
 8862             "xorl    rdx, rdx\n\t"
 8863             "cmpl    $div, -1\n\t"
 8864             "je,s    done\n"
 8865     "normal: cdql\n\t"
 8866             "idivl   $div\n"
 8867     "done:"        %}
 8868   ins_encode(cdql_enc(div));
 8869   ins_pipe(pipe_slow);
 8870 %}
 8871 
 8872 // Long DIVMOD with Register, both quotient and mod results
 8873 instruct divModL_rReg_divmod(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div,
 8874                              rFlagsReg cr)
 8875 %{
 8876   match(DivModL rax div);
 8877   effect(KILL cr);
 8878 
 8879   ins_cost(30*100+10*100); // XXX
 8880   format %{ "movq    rdx, 0x8000000000000000\t# ldiv\n\t"
 8881             "cmpq    rax, rdx\n\t"
 8882             "jne,s   normal\n\t"
 8883             "xorl    rdx, rdx\n\t"
 8884             "cmpq    $div, -1\n\t"
 8885             "je,s    done\n"
 8886     "normal: cdqq\n\t"
 8887             "idivq   $div\n"
 8888     "done:"        %}
 8889   ins_encode(cdqq_enc(div));
 8890   ins_pipe(pipe_slow);
 8891 %}
 8892 
 8893 // Unsigned integer DIVMOD with Register, both quotient and mod results
 8894 instruct udivModI_rReg_divmod(rax_RegI rax, no_rax_rdx_RegI tmp, rdx_RegI rdx,
 8895                               no_rax_rdx_RegI div, rFlagsReg cr)
 8896 %{
 8897   match(UDivModI rax div);
 8898   effect(TEMP tmp, KILL cr);
 8899 
 8900   ins_cost(300);
 8901   format %{ "udivl $rax,$rax,$div\t# begin UDivModI\n\t"
 8902             "umodl $rdx,$rax,$div\t! using $tmp as TEMP # end UDivModI\n"
 8903           %}
 8904   ins_encode %{
 8905     __ udivmodI($rax$$Register, $div$$Register, $rdx$$Register, $tmp$$Register);
 8906   %}
 8907   ins_pipe(pipe_slow);
 8908 %}
 8909 
 8910 // Unsigned long DIVMOD with Register, both quotient and mod results
 8911 instruct udivModL_rReg_divmod(rax_RegL rax, no_rax_rdx_RegL tmp, rdx_RegL rdx,
 8912                               no_rax_rdx_RegL div, rFlagsReg cr)
 8913 %{
 8914   match(UDivModL rax div);
 8915   effect(TEMP tmp, KILL cr);
 8916 
 8917   ins_cost(300);
 8918   format %{ "udivq $rax,$rax,$div\t# begin UDivModL\n\t"
 8919             "umodq $rdx,$rax,$div\t! using $tmp as TEMP # end UDivModL\n"
 8920           %}
 8921   ins_encode %{
 8922     __ udivmodL($rax$$Register, $div$$Register, $rdx$$Register, $tmp$$Register);
 8923   %}
 8924   ins_pipe(pipe_slow);
 8925 %}
 8926 
 8927 instruct modI_rReg(rdx_RegI rdx, rax_RegI rax, no_rax_rdx_RegI div,
 8928                    rFlagsReg cr)
 8929 %{
 8930   match(Set rdx (ModI rax div));
 8931   effect(KILL rax, KILL cr);
 8932 
 8933   ins_cost(300); // XXX
 8934   format %{ "cmpl    rax, 0x80000000\t# irem\n\t"
 8935             "jne,s   normal\n\t"
 8936             "xorl    rdx, rdx\n\t"
 8937             "cmpl    $div, -1\n\t"
 8938             "je,s    done\n"
 8939     "normal: cdql\n\t"
 8940             "idivl   $div\n"
 8941     "done:"        %}
 8942   ins_encode(cdql_enc(div));
 8943   ins_pipe(ialu_reg_reg_alu0);
 8944 %}
 8945 
 8946 instruct modL_rReg(rdx_RegL rdx, rax_RegL rax, no_rax_rdx_RegL div,
 8947                    rFlagsReg cr)
 8948 %{
 8949   match(Set rdx (ModL rax div));
 8950   effect(KILL rax, KILL cr);
 8951 
 8952   ins_cost(300); // XXX
 8953   format %{ "movq    rdx, 0x8000000000000000\t# lrem\n\t"
 8954             "cmpq    rax, rdx\n\t"
 8955             "jne,s   normal\n\t"
 8956             "xorl    rdx, rdx\n\t"
 8957             "cmpq    $div, -1\n\t"
 8958             "je,s    done\n"
 8959     "normal: cdqq\n\t"
 8960             "idivq   $div\n"
 8961     "done:"        %}
 8962   ins_encode(cdqq_enc(div));
 8963   ins_pipe(ialu_reg_reg_alu0);
 8964 %}
 8965 
 8966 instruct umodI_rReg(rdx_RegI rdx, rax_RegI rax, no_rax_rdx_RegI div, rFlagsReg cr)
 8967 %{
 8968   match(Set rdx (UModI rax div));
 8969   effect(KILL rax, KILL cr);
 8970 
 8971   ins_cost(300);
 8972   format %{ "umodl $rdx,$rax,$div\t# UModI\n" %}
 8973   ins_encode %{
 8974     __ umodI($rax$$Register, $div$$Register, $rdx$$Register);
 8975   %}
 8976   ins_pipe(ialu_reg_reg_alu0);
 8977 %}
 8978 
 8979 instruct umodL_rReg(rdx_RegL rdx, rax_RegL rax, no_rax_rdx_RegL div, rFlagsReg cr)
 8980 %{
 8981   match(Set rdx (UModL rax div));
 8982   effect(KILL rax, KILL cr);
 8983 
 8984   ins_cost(300);
 8985   format %{ "umodq $rdx,$rax,$div\t# UModL\n" %}
 8986   ins_encode %{
 8987     __ umodL($rax$$Register, $div$$Register, $rdx$$Register);
 8988   %}
 8989   ins_pipe(ialu_reg_reg_alu0);
 8990 %}
 8991 
 8992 // Integer Shift Instructions
 8993 // Shift Left by one, two, three
 8994 instruct salI_rReg_immI2(rRegI dst, immI2 shift, rFlagsReg cr)
 8995 %{
 8996   predicate(!UseAPX);
 8997   match(Set dst (LShiftI dst shift));
 8998   effect(KILL cr);
 8999 
 9000   format %{ "sall    $dst, $shift" %}
 9001   ins_encode %{
 9002     __ sall($dst$$Register, $shift$$constant);
 9003   %}
 9004   ins_pipe(ialu_reg);
 9005 %}
 9006 
 9007 // Shift Left by one, two, three
 9008 instruct salI_rReg_immI2_ndd(rRegI dst, rRegI src, immI2 shift, rFlagsReg cr)
 9009 %{
 9010   predicate(UseAPX);
 9011   match(Set dst (LShiftI src shift));
 9012   effect(KILL cr);
 9013 
 9014   format %{ "esall    $dst, $src, $shift\t# int(ndd)" %}
 9015   ins_encode %{
 9016     __ esall($dst$$Register, $src$$Register, $shift$$constant, false);
 9017   %}
 9018   ins_pipe(ialu_reg);
 9019 %}
 9020 
 9021 // Shift Left by 8-bit immediate
 9022 instruct salI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
 9023 %{
 9024   predicate(!UseAPX);
 9025   match(Set dst (LShiftI dst shift));
 9026   effect(KILL cr);
 9027 
 9028   format %{ "sall    $dst, $shift" %}
 9029   ins_encode %{
 9030     __ sall($dst$$Register, $shift$$constant);
 9031   %}
 9032   ins_pipe(ialu_reg);
 9033 %}
 9034 
 9035 // Shift Left by 8-bit immediate
 9036 instruct salI_rReg_imm_ndd(rRegI dst, rRegI src, immI8 shift, rFlagsReg cr)
 9037 %{
 9038   predicate(UseAPX);
 9039   match(Set dst (LShiftI src shift));
 9040   effect(KILL cr);
 9041 
 9042   format %{ "esall    $dst, $src, $shift\t# int (ndd)" %}
 9043   ins_encode %{
 9044     __ esall($dst$$Register, $src$$Register, $shift$$constant, false);
 9045   %}
 9046   ins_pipe(ialu_reg);
 9047 %}
 9048 
 9049 instruct salI_rReg_mem_imm_ndd(rRegI dst, memory src, immI8 shift, rFlagsReg cr)
 9050 %{
 9051   predicate(UseAPX);
 9052   match(Set dst (LShiftI (LoadI src) shift));
 9053   effect(KILL cr);
 9054 
 9055   format %{ "esall    $dst, $src, $shift\t# int (ndd)" %}
 9056   ins_encode %{
 9057     __ esall($dst$$Register, $src$$Address, $shift$$constant, false);
 9058   %}
 9059   ins_pipe(ialu_reg);
 9060 %}
 9061 
 9062 // Shift Left by 8-bit immediate
 9063 instruct salI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 9064 %{
 9065   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
 9066   effect(KILL cr);
 9067 
 9068   format %{ "sall    $dst, $shift" %}
 9069   ins_encode %{
 9070     __ sall($dst$$Address, $shift$$constant);
 9071   %}
 9072   ins_pipe(ialu_mem_imm);
 9073 %}
 9074 
 9075 // Shift Left by variable
 9076 instruct salI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 9077 %{
 9078   predicate(!VM_Version::supports_bmi2());
 9079   match(Set dst (LShiftI dst shift));
 9080   effect(KILL cr);
 9081 
 9082   format %{ "sall    $dst, $shift" %}
 9083   ins_encode %{
 9084     __ sall($dst$$Register);
 9085   %}
 9086   ins_pipe(ialu_reg_reg);
 9087 %}
 9088 
 9089 // Shift Left by variable
 9090 instruct salI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 9091 %{
 9092   predicate(!VM_Version::supports_bmi2());
 9093   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
 9094   effect(KILL cr);
 9095 
 9096   format %{ "sall    $dst, $shift" %}
 9097   ins_encode %{
 9098     __ sall($dst$$Address);
 9099   %}
 9100   ins_pipe(ialu_mem_reg);
 9101 %}
 9102 
 9103 instruct salI_rReg_rReg(rRegI dst, rRegI src, rRegI shift)
 9104 %{
 9105   predicate(VM_Version::supports_bmi2());
 9106   match(Set dst (LShiftI src shift));
 9107 
 9108   format %{ "shlxl   $dst, $src, $shift" %}
 9109   ins_encode %{
 9110     __ shlxl($dst$$Register, $src$$Register, $shift$$Register);
 9111   %}
 9112   ins_pipe(ialu_reg_reg);
 9113 %}
 9114 
 9115 instruct salI_mem_rReg(rRegI dst, memory src, rRegI shift)
 9116 %{
 9117   predicate(VM_Version::supports_bmi2());
 9118   match(Set dst (LShiftI (LoadI src) shift));
 9119   ins_cost(175);
 9120   format %{ "shlxl   $dst, $src, $shift" %}
 9121   ins_encode %{
 9122     __ shlxl($dst$$Register, $src$$Address, $shift$$Register);
 9123   %}
 9124   ins_pipe(ialu_reg_mem);
 9125 %}
 9126 
 9127 // Arithmetic Shift Right by 8-bit immediate
 9128 instruct sarI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
 9129 %{
 9130   predicate(!UseAPX);
 9131   match(Set dst (RShiftI dst shift));
 9132   effect(KILL cr);
 9133 
 9134   format %{ "sarl    $dst, $shift" %}
 9135   ins_encode %{
 9136     __ sarl($dst$$Register, $shift$$constant);
 9137   %}
 9138   ins_pipe(ialu_mem_imm);
 9139 %}
 9140 
 9141 // Arithmetic Shift Right by 8-bit immediate
 9142 instruct sarI_rReg_imm_ndd(rRegI dst, rRegI src, immI8 shift, rFlagsReg cr)
 9143 %{
 9144   predicate(UseAPX);
 9145   match(Set dst (RShiftI src shift));
 9146   effect(KILL cr);
 9147 
 9148   format %{ "esarl    $dst, $src, $shift\t# int (ndd)" %}
 9149   ins_encode %{
 9150     __ esarl($dst$$Register, $src$$Register, $shift$$constant, false);
 9151   %}
 9152   ins_pipe(ialu_mem_imm);
 9153 %}
 9154 
 9155 instruct sarI_rReg_mem_imm_ndd(rRegI dst, memory src, immI8 shift, rFlagsReg cr)
 9156 %{
 9157   predicate(UseAPX);
 9158   match(Set dst (RShiftI (LoadI src) shift));
 9159   effect(KILL cr);
 9160 
 9161   format %{ "esarl    $dst, $src, $shift\t# int (ndd)" %}
 9162   ins_encode %{
 9163     __ esarl($dst$$Register, $src$$Address, $shift$$constant, false);
 9164   %}
 9165   ins_pipe(ialu_mem_imm);
 9166 %}
 9167 
 9168 // Arithmetic Shift Right by 8-bit immediate
 9169 instruct sarI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 9170 %{
 9171   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
 9172   effect(KILL cr);
 9173 
 9174   format %{ "sarl    $dst, $shift" %}
 9175   ins_encode %{
 9176     __ sarl($dst$$Address, $shift$$constant);
 9177   %}
 9178   ins_pipe(ialu_mem_imm);
 9179 %}
 9180 
 9181 // Arithmetic Shift Right by variable
 9182 instruct sarI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 9183 %{
 9184   predicate(!VM_Version::supports_bmi2());
 9185   match(Set dst (RShiftI dst shift));
 9186   effect(KILL cr);
 9187 
 9188   format %{ "sarl    $dst, $shift" %}
 9189   ins_encode %{
 9190     __ sarl($dst$$Register);
 9191   %}
 9192   ins_pipe(ialu_reg_reg);
 9193 %}
 9194 
 9195 // Arithmetic Shift Right by variable
 9196 instruct sarI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 9197 %{
 9198   predicate(!VM_Version::supports_bmi2());
 9199   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
 9200   effect(KILL cr);
 9201 
 9202   format %{ "sarl    $dst, $shift" %}
 9203   ins_encode %{
 9204     __ sarl($dst$$Address);
 9205   %}
 9206   ins_pipe(ialu_mem_reg);
 9207 %}
 9208 
 9209 instruct sarI_rReg_rReg(rRegI dst, rRegI src, rRegI shift)
 9210 %{
 9211   predicate(VM_Version::supports_bmi2());
 9212   match(Set dst (RShiftI src shift));
 9213 
 9214   format %{ "sarxl   $dst, $src, $shift" %}
 9215   ins_encode %{
 9216     __ sarxl($dst$$Register, $src$$Register, $shift$$Register);
 9217   %}
 9218   ins_pipe(ialu_reg_reg);
 9219 %}
 9220 
 9221 instruct sarI_mem_rReg(rRegI dst, memory src, rRegI shift)
 9222 %{
 9223   predicate(VM_Version::supports_bmi2());
 9224   match(Set dst (RShiftI (LoadI src) shift));
 9225   ins_cost(175);
 9226   format %{ "sarxl   $dst, $src, $shift" %}
 9227   ins_encode %{
 9228     __ sarxl($dst$$Register, $src$$Address, $shift$$Register);
 9229   %}
 9230   ins_pipe(ialu_reg_mem);
 9231 %}
 9232 
 9233 // Logical Shift Right by 8-bit immediate
 9234 instruct shrI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
 9235 %{
 9236   predicate(!UseAPX);
 9237   match(Set dst (URShiftI dst shift));
 9238   effect(KILL cr);
 9239 
 9240   format %{ "shrl    $dst, $shift" %}
 9241   ins_encode %{
 9242     __ shrl($dst$$Register, $shift$$constant);
 9243   %}
 9244   ins_pipe(ialu_reg);
 9245 %}
 9246 
 9247 // Logical Shift Right by 8-bit immediate
 9248 instruct shrI_rReg_imm_ndd(rRegI dst, rRegI src, immI8 shift, rFlagsReg cr)
 9249 %{
 9250   predicate(UseAPX);
 9251   match(Set dst (URShiftI src shift));
 9252   effect(KILL cr);
 9253 
 9254   format %{ "eshrl    $dst, $src, $shift\t # int (ndd)" %}
 9255   ins_encode %{
 9256     __ eshrl($dst$$Register, $src$$Register, $shift$$constant, false);
 9257   %}
 9258   ins_pipe(ialu_reg);
 9259 %}
 9260 
 9261 instruct shrI_rReg_mem_imm_ndd(rRegI dst, memory src, immI8 shift, rFlagsReg cr)
 9262 %{
 9263   predicate(UseAPX);
 9264   match(Set dst (URShiftI (LoadI src) shift));
 9265   effect(KILL cr);
 9266 
 9267   format %{ "eshrl    $dst, $src, $shift\t # int (ndd)" %}
 9268   ins_encode %{
 9269     __ eshrl($dst$$Register, $src$$Address, $shift$$constant, false);
 9270   %}
 9271   ins_pipe(ialu_reg);
 9272 %}
 9273 
 9274 // Logical Shift Right by 8-bit immediate
 9275 instruct shrI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 9276 %{
 9277   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
 9278   effect(KILL cr);
 9279 
 9280   format %{ "shrl    $dst, $shift" %}
 9281   ins_encode %{
 9282     __ shrl($dst$$Address, $shift$$constant);
 9283   %}
 9284   ins_pipe(ialu_mem_imm);
 9285 %}
 9286 
 9287 // Logical Shift Right by variable
 9288 instruct shrI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 9289 %{
 9290   predicate(!VM_Version::supports_bmi2());
 9291   match(Set dst (URShiftI dst shift));
 9292   effect(KILL cr);
 9293 
 9294   format %{ "shrl    $dst, $shift" %}
 9295   ins_encode %{
 9296     __ shrl($dst$$Register);
 9297   %}
 9298   ins_pipe(ialu_reg_reg);
 9299 %}
 9300 
 9301 // Logical Shift Right by variable
 9302 instruct shrI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 9303 %{
 9304   predicate(!VM_Version::supports_bmi2());
 9305   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
 9306   effect(KILL cr);
 9307 
 9308   format %{ "shrl    $dst, $shift" %}
 9309   ins_encode %{
 9310     __ shrl($dst$$Address);
 9311   %}
 9312   ins_pipe(ialu_mem_reg);
 9313 %}
 9314 
 9315 instruct shrI_rReg_rReg(rRegI dst, rRegI src, rRegI shift)
 9316 %{
 9317   predicate(VM_Version::supports_bmi2());
 9318   match(Set dst (URShiftI src shift));
 9319 
 9320   format %{ "shrxl   $dst, $src, $shift" %}
 9321   ins_encode %{
 9322     __ shrxl($dst$$Register, $src$$Register, $shift$$Register);
 9323   %}
 9324   ins_pipe(ialu_reg_reg);
 9325 %}
 9326 
 9327 instruct shrI_mem_rReg(rRegI dst, memory src, rRegI shift)
 9328 %{
 9329   predicate(VM_Version::supports_bmi2());
 9330   match(Set dst (URShiftI (LoadI src) shift));
 9331   ins_cost(175);
 9332   format %{ "shrxl   $dst, $src, $shift" %}
 9333   ins_encode %{
 9334     __ shrxl($dst$$Register, $src$$Address, $shift$$Register);
 9335   %}
 9336   ins_pipe(ialu_reg_mem);
 9337 %}
 9338 
 9339 // Long Shift Instructions
 9340 // Shift Left by one, two, three
 9341 instruct salL_rReg_immI2(rRegL dst, immI2 shift, rFlagsReg cr)
 9342 %{
 9343   predicate(!UseAPX);
 9344   match(Set dst (LShiftL dst shift));
 9345   effect(KILL cr);
 9346 
 9347   format %{ "salq    $dst, $shift" %}
 9348   ins_encode %{
 9349     __ salq($dst$$Register, $shift$$constant);
 9350   %}
 9351   ins_pipe(ialu_reg);
 9352 %}
 9353 
 9354 // Shift Left by one, two, three
 9355 instruct salL_rReg_immI2_ndd(rRegL dst, rRegL src, immI2 shift, rFlagsReg cr)
 9356 %{
 9357   predicate(UseAPX);
 9358   match(Set dst (LShiftL src shift));
 9359   effect(KILL cr);
 9360 
 9361   format %{ "esalq    $dst, $src, $shift\t# long (ndd)" %}
 9362   ins_encode %{
 9363     __ esalq($dst$$Register, $src$$Register, $shift$$constant, false);
 9364   %}
 9365   ins_pipe(ialu_reg);
 9366 %}
 9367 
 9368 // Shift Left by 8-bit immediate
 9369 instruct salL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
 9370 %{
 9371   predicate(!UseAPX);
 9372   match(Set dst (LShiftL dst shift));
 9373   effect(KILL cr);
 9374 
 9375   format %{ "salq    $dst, $shift" %}
 9376   ins_encode %{
 9377     __ salq($dst$$Register, $shift$$constant);
 9378   %}
 9379   ins_pipe(ialu_reg);
 9380 %}
 9381 
 9382 // Shift Left by 8-bit immediate
 9383 instruct salL_rReg_imm_ndd(rRegL dst, rRegL src, immI8 shift, rFlagsReg cr)
 9384 %{
 9385   predicate(UseAPX);
 9386   match(Set dst (LShiftL src shift));
 9387   effect(KILL cr);
 9388 
 9389   format %{ "esalq    $dst, $src, $shift\t# long (ndd)" %}
 9390   ins_encode %{
 9391     __ esalq($dst$$Register, $src$$Register, $shift$$constant, false);
 9392   %}
 9393   ins_pipe(ialu_reg);
 9394 %}
 9395 
 9396 instruct salL_rReg_mem_imm_ndd(rRegL dst, memory src, immI8 shift, rFlagsReg cr)
 9397 %{
 9398   predicate(UseAPX);
 9399   match(Set dst (LShiftL (LoadL src) shift));
 9400   effect(KILL cr);
 9401 
 9402   format %{ "esalq    $dst, $src, $shift\t# long (ndd)" %}
 9403   ins_encode %{
 9404     __ esalq($dst$$Register, $src$$Address, $shift$$constant, false);
 9405   %}
 9406   ins_pipe(ialu_reg);
 9407 %}
 9408 
 9409 // Shift Left by 8-bit immediate
 9410 instruct salL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 9411 %{
 9412   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
 9413   effect(KILL cr);
 9414 
 9415   format %{ "salq    $dst, $shift" %}
 9416   ins_encode %{
 9417     __ salq($dst$$Address, $shift$$constant);
 9418   %}
 9419   ins_pipe(ialu_mem_imm);
 9420 %}
 9421 
 9422 // Shift Left by variable
 9423 instruct salL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 9424 %{
 9425   predicate(!VM_Version::supports_bmi2());
 9426   match(Set dst (LShiftL dst shift));
 9427   effect(KILL cr);
 9428 
 9429   format %{ "salq    $dst, $shift" %}
 9430   ins_encode %{
 9431     __ salq($dst$$Register);
 9432   %}
 9433   ins_pipe(ialu_reg_reg);
 9434 %}
 9435 
 9436 // Shift Left by variable
 9437 instruct salL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 9438 %{
 9439   predicate(!VM_Version::supports_bmi2());
 9440   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
 9441   effect(KILL cr);
 9442 
 9443   format %{ "salq    $dst, $shift" %}
 9444   ins_encode %{
 9445     __ salq($dst$$Address);
 9446   %}
 9447   ins_pipe(ialu_mem_reg);
 9448 %}
 9449 
 9450 instruct salL_rReg_rReg(rRegL dst, rRegL src, rRegI shift)
 9451 %{
 9452   predicate(VM_Version::supports_bmi2());
 9453   match(Set dst (LShiftL src shift));
 9454 
 9455   format %{ "shlxq   $dst, $src, $shift" %}
 9456   ins_encode %{
 9457     __ shlxq($dst$$Register, $src$$Register, $shift$$Register);
 9458   %}
 9459   ins_pipe(ialu_reg_reg);
 9460 %}
 9461 
 9462 instruct salL_mem_rReg(rRegL dst, memory src, rRegI shift)
 9463 %{
 9464   predicate(VM_Version::supports_bmi2());
 9465   match(Set dst (LShiftL (LoadL src) shift));
 9466   ins_cost(175);
 9467   format %{ "shlxq   $dst, $src, $shift" %}
 9468   ins_encode %{
 9469     __ shlxq($dst$$Register, $src$$Address, $shift$$Register);
 9470   %}
 9471   ins_pipe(ialu_reg_mem);
 9472 %}
 9473 
 9474 // Arithmetic Shift Right by 8-bit immediate
 9475 instruct sarL_rReg_imm(rRegL dst, immI shift, rFlagsReg cr)
 9476 %{
 9477   predicate(!UseAPX);
 9478   match(Set dst (RShiftL dst shift));
 9479   effect(KILL cr);
 9480 
 9481   format %{ "sarq    $dst, $shift" %}
 9482   ins_encode %{
 9483     __ sarq($dst$$Register, (unsigned char)($shift$$constant & 0x3F));
 9484   %}
 9485   ins_pipe(ialu_mem_imm);
 9486 %}
 9487 
 9488 // Arithmetic Shift Right by 8-bit immediate
 9489 instruct sarL_rReg_imm_ndd(rRegL dst, rRegL src, immI shift, rFlagsReg cr)
 9490 %{
 9491   predicate(UseAPX);
 9492   match(Set dst (RShiftL src shift));
 9493   effect(KILL cr);
 9494 
 9495   format %{ "esarq    $dst, $src, $shift\t# long (ndd)" %}
 9496   ins_encode %{
 9497     __ esarq($dst$$Register, $src$$Register, (unsigned char)($shift$$constant & 0x3F), false);
 9498   %}
 9499   ins_pipe(ialu_mem_imm);
 9500 %}
 9501 
 9502 instruct sarL_rReg_mem_imm_ndd(rRegL dst, memory src, immI shift, rFlagsReg cr)
 9503 %{
 9504   predicate(UseAPX);
 9505   match(Set dst (RShiftL (LoadL src) shift));
 9506   effect(KILL cr);
 9507 
 9508   format %{ "esarq    $dst, $src, $shift\t# long (ndd)" %}
 9509   ins_encode %{
 9510     __ esarq($dst$$Register, $src$$Address, (unsigned char)($shift$$constant & 0x3F), false);
 9511   %}
 9512   ins_pipe(ialu_mem_imm);
 9513 %}
 9514 
 9515 // Arithmetic Shift Right by 8-bit immediate
 9516 instruct sarL_mem_imm(memory dst, immI shift, rFlagsReg cr)
 9517 %{
 9518   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
 9519   effect(KILL cr);
 9520 
 9521   format %{ "sarq    $dst, $shift" %}
 9522   ins_encode %{
 9523     __ sarq($dst$$Address, (unsigned char)($shift$$constant & 0x3F));
 9524   %}
 9525   ins_pipe(ialu_mem_imm);
 9526 %}
 9527 
 9528 // Arithmetic Shift Right by variable
 9529 instruct sarL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 9530 %{
 9531   predicate(!VM_Version::supports_bmi2());
 9532   match(Set dst (RShiftL dst shift));
 9533   effect(KILL cr);
 9534 
 9535   format %{ "sarq    $dst, $shift" %}
 9536   ins_encode %{
 9537     __ sarq($dst$$Register);
 9538   %}
 9539   ins_pipe(ialu_reg_reg);
 9540 %}
 9541 
 9542 // Arithmetic Shift Right by variable
 9543 instruct sarL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 9544 %{
 9545   predicate(!VM_Version::supports_bmi2());
 9546   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
 9547   effect(KILL cr);
 9548 
 9549   format %{ "sarq    $dst, $shift" %}
 9550   ins_encode %{
 9551     __ sarq($dst$$Address);
 9552   %}
 9553   ins_pipe(ialu_mem_reg);
 9554 %}
 9555 
 9556 instruct sarL_rReg_rReg(rRegL dst, rRegL src, rRegI shift)
 9557 %{
 9558   predicate(VM_Version::supports_bmi2());
 9559   match(Set dst (RShiftL src shift));
 9560 
 9561   format %{ "sarxq   $dst, $src, $shift" %}
 9562   ins_encode %{
 9563     __ sarxq($dst$$Register, $src$$Register, $shift$$Register);
 9564   %}
 9565   ins_pipe(ialu_reg_reg);
 9566 %}
 9567 
 9568 instruct sarL_mem_rReg(rRegL dst, memory src, rRegI shift)
 9569 %{
 9570   predicate(VM_Version::supports_bmi2());
 9571   match(Set dst (RShiftL (LoadL src) shift));
 9572   ins_cost(175);
 9573   format %{ "sarxq   $dst, $src, $shift" %}
 9574   ins_encode %{
 9575     __ sarxq($dst$$Register, $src$$Address, $shift$$Register);
 9576   %}
 9577   ins_pipe(ialu_reg_mem);
 9578 %}
 9579 
 9580 // Logical Shift Right by 8-bit immediate
 9581 instruct shrL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
 9582 %{
 9583   predicate(!UseAPX);
 9584   match(Set dst (URShiftL dst shift));
 9585   effect(KILL cr);
 9586 
 9587   format %{ "shrq    $dst, $shift" %}
 9588   ins_encode %{
 9589     __ shrq($dst$$Register, $shift$$constant);
 9590   %}
 9591   ins_pipe(ialu_reg);
 9592 %}
 9593 
 9594 // Logical Shift Right by 8-bit immediate
 9595 instruct shrL_rReg_imm_ndd(rRegL dst, rRegL src, immI8 shift, rFlagsReg cr)
 9596 %{
 9597   predicate(UseAPX);
 9598   match(Set dst (URShiftL src shift));
 9599   effect(KILL cr);
 9600 
 9601   format %{ "eshrq    $dst, $src, $shift\t# long (ndd)" %}
 9602   ins_encode %{
 9603     __ eshrq($dst$$Register, $src$$Register, $shift$$constant, false);
 9604   %}
 9605   ins_pipe(ialu_reg);
 9606 %}
 9607 
 9608 instruct shrL_rReg_mem_imm_ndd(rRegL dst, memory src, immI8 shift, rFlagsReg cr)
 9609 %{
 9610   predicate(UseAPX);
 9611   match(Set dst (URShiftL (LoadL src) shift));
 9612   effect(KILL cr);
 9613 
 9614   format %{ "eshrq    $dst, $src, $shift\t# long (ndd)" %}
 9615   ins_encode %{
 9616     __ eshrq($dst$$Register, $src$$Address, $shift$$constant, false);
 9617   %}
 9618   ins_pipe(ialu_reg);
 9619 %}
 9620 
 9621 // Logical Shift Right by 8-bit immediate
 9622 instruct shrL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 9623 %{
 9624   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
 9625   effect(KILL cr);
 9626 
 9627   format %{ "shrq    $dst, $shift" %}
 9628   ins_encode %{
 9629     __ shrq($dst$$Address, $shift$$constant);
 9630   %}
 9631   ins_pipe(ialu_mem_imm);
 9632 %}
 9633 
 9634 // Logical Shift Right by variable
 9635 instruct shrL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 9636 %{
 9637   predicate(!VM_Version::supports_bmi2());
 9638   match(Set dst (URShiftL dst shift));
 9639   effect(KILL cr);
 9640 
 9641   format %{ "shrq    $dst, $shift" %}
 9642   ins_encode %{
 9643     __ shrq($dst$$Register);
 9644   %}
 9645   ins_pipe(ialu_reg_reg);
 9646 %}
 9647 
 9648 // Logical Shift Right by variable
 9649 instruct shrL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 9650 %{
 9651   predicate(!VM_Version::supports_bmi2());
 9652   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
 9653   effect(KILL cr);
 9654 
 9655   format %{ "shrq    $dst, $shift" %}
 9656   ins_encode %{
 9657     __ shrq($dst$$Address);
 9658   %}
 9659   ins_pipe(ialu_mem_reg);
 9660 %}
 9661 
 9662 instruct shrL_rReg_rReg(rRegL dst, rRegL src, rRegI shift)
 9663 %{
 9664   predicate(VM_Version::supports_bmi2());
 9665   match(Set dst (URShiftL src shift));
 9666 
 9667   format %{ "shrxq   $dst, $src, $shift" %}
 9668   ins_encode %{
 9669     __ shrxq($dst$$Register, $src$$Register, $shift$$Register);
 9670   %}
 9671   ins_pipe(ialu_reg_reg);
 9672 %}
 9673 
 9674 instruct shrL_mem_rReg(rRegL dst, memory src, rRegI shift)
 9675 %{
 9676   predicate(VM_Version::supports_bmi2());
 9677   match(Set dst (URShiftL (LoadL src) shift));
 9678   ins_cost(175);
 9679   format %{ "shrxq   $dst, $src, $shift" %}
 9680   ins_encode %{
 9681     __ shrxq($dst$$Register, $src$$Address, $shift$$Register);
 9682   %}
 9683   ins_pipe(ialu_reg_mem);
 9684 %}
 9685 
 9686 // Logical Shift Right by 24, followed by Arithmetic Shift Left by 24.
 9687 // This idiom is used by the compiler for the i2b bytecode.
 9688 instruct i2b(rRegI dst, rRegI src, immI_24 twentyfour)
 9689 %{
 9690   match(Set dst (RShiftI (LShiftI src twentyfour) twentyfour));
 9691 
 9692   format %{ "movsbl  $dst, $src\t# i2b" %}
 9693   ins_encode %{
 9694     __ movsbl($dst$$Register, $src$$Register);
 9695   %}
 9696   ins_pipe(ialu_reg_reg);
 9697 %}
 9698 
 9699 // Logical Shift Right by 16, followed by Arithmetic Shift Left by 16.
 9700 // This idiom is used by the compiler the i2s bytecode.
 9701 instruct i2s(rRegI dst, rRegI src, immI_16 sixteen)
 9702 %{
 9703   match(Set dst (RShiftI (LShiftI src sixteen) sixteen));
 9704 
 9705   format %{ "movswl  $dst, $src\t# i2s" %}
 9706   ins_encode %{
 9707     __ movswl($dst$$Register, $src$$Register);
 9708   %}
 9709   ins_pipe(ialu_reg_reg);
 9710 %}
 9711 
 9712 // ROL/ROR instructions
 9713 
 9714 // Rotate left by constant.
 9715 instruct rolI_immI8_legacy(rRegI dst, immI8 shift, rFlagsReg cr)
 9716 %{
 9717   predicate(!VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 9718   match(Set dst (RotateLeft dst shift));
 9719   effect(KILL cr);
 9720   format %{ "roll    $dst, $shift" %}
 9721   ins_encode %{
 9722     __ roll($dst$$Register, $shift$$constant);
 9723   %}
 9724   ins_pipe(ialu_reg);
 9725 %}
 9726 
 9727 instruct rolI_immI8(rRegI dst, rRegI src, immI8 shift)
 9728 %{
 9729   predicate(!UseAPX && VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 9730   match(Set dst (RotateLeft src shift));
 9731   format %{ "rolxl   $dst, $src, $shift" %}
 9732   ins_encode %{
 9733     int shift = 32 - ($shift$$constant & 31);
 9734     __ rorxl($dst$$Register, $src$$Register, shift);
 9735   %}
 9736   ins_pipe(ialu_reg_reg);
 9737 %}
 9738 
 9739 instruct rolI_mem_immI8(rRegI dst, memory src, immI8 shift)
 9740 %{
 9741   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 9742   match(Set dst (RotateLeft (LoadI src) shift));
 9743   ins_cost(175);
 9744   format %{ "rolxl   $dst, $src, $shift" %}
 9745   ins_encode %{
 9746     int shift = 32 - ($shift$$constant & 31);
 9747     __ rorxl($dst$$Register, $src$$Address, shift);
 9748   %}
 9749   ins_pipe(ialu_reg_mem);
 9750 %}
 9751 
 9752 // Rotate Left by variable
 9753 instruct rolI_rReg_Var(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 9754 %{
 9755   predicate(!UseAPX && n->bottom_type()->basic_type() == T_INT);
 9756   match(Set dst (RotateLeft dst shift));
 9757   effect(KILL cr);
 9758   format %{ "roll    $dst, $shift" %}
 9759   ins_encode %{
 9760     __ roll($dst$$Register);
 9761   %}
 9762   ins_pipe(ialu_reg_reg);
 9763 %}
 9764 
 9765 // Rotate Left by variable
 9766 instruct rolI_rReg_Var_ndd(rRegI dst, rRegI src, rcx_RegI shift, rFlagsReg cr)
 9767 %{
 9768   predicate(UseAPX && n->bottom_type()->basic_type() == T_INT);
 9769   match(Set dst (RotateLeft src shift));
 9770   effect(KILL cr);
 9771 
 9772   format %{ "eroll    $dst, $src, $shift\t# rotate left (int ndd)" %}
 9773   ins_encode %{
 9774     __ eroll($dst$$Register, $src$$Register, false);
 9775   %}
 9776   ins_pipe(ialu_reg_reg);
 9777 %}
 9778 
 9779 // Rotate Right by constant.
 9780 instruct rorI_immI8_legacy(rRegI dst, immI8 shift, rFlagsReg cr)
 9781 %{
 9782   predicate(!VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 9783   match(Set dst (RotateRight dst shift));
 9784   effect(KILL cr);
 9785   format %{ "rorl    $dst, $shift" %}
 9786   ins_encode %{
 9787     __ rorl($dst$$Register, $shift$$constant);
 9788   %}
 9789   ins_pipe(ialu_reg);
 9790 %}
 9791 
 9792 // Rotate Right by constant.
 9793 instruct rorI_immI8(rRegI dst, rRegI src, immI8 shift)
 9794 %{
 9795   predicate(!UseAPX && VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 9796   match(Set dst (RotateRight src shift));
 9797   format %{ "rorxl   $dst, $src, $shift" %}
 9798   ins_encode %{
 9799     __ rorxl($dst$$Register, $src$$Register, $shift$$constant);
 9800   %}
 9801   ins_pipe(ialu_reg_reg);
 9802 %}
 9803 
 9804 instruct rorI_mem_immI8(rRegI dst, memory src, immI8 shift)
 9805 %{
 9806   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 9807   match(Set dst (RotateRight (LoadI src) shift));
 9808   ins_cost(175);
 9809   format %{ "rorxl   $dst, $src, $shift" %}
 9810   ins_encode %{
 9811     __ rorxl($dst$$Register, $src$$Address, $shift$$constant);
 9812   %}
 9813   ins_pipe(ialu_reg_mem);
 9814 %}
 9815 
 9816 // Rotate Right by variable
 9817 instruct rorI_rReg_Var(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 9818 %{
 9819   predicate(!UseAPX && n->bottom_type()->basic_type() == T_INT);
 9820   match(Set dst (RotateRight dst shift));
 9821   effect(KILL cr);
 9822   format %{ "rorl    $dst, $shift" %}
 9823   ins_encode %{
 9824     __ rorl($dst$$Register);
 9825   %}
 9826   ins_pipe(ialu_reg_reg);
 9827 %}
 9828 
 9829 // Rotate Right by variable
 9830 instruct rorI_rReg_Var_ndd(rRegI dst, rRegI src, rcx_RegI shift, rFlagsReg cr)
 9831 %{
 9832   predicate(UseAPX && n->bottom_type()->basic_type() == T_INT);
 9833   match(Set dst (RotateRight src shift));
 9834   effect(KILL cr);
 9835 
 9836   format %{ "erorl    $dst, $src, $shift\t# rotate right(int ndd)" %}
 9837   ins_encode %{
 9838     __ erorl($dst$$Register, $src$$Register, false);
 9839   %}
 9840   ins_pipe(ialu_reg_reg);
 9841 %}
 9842 
 9843 // Rotate Left by constant.
 9844 instruct rolL_immI8_legacy(rRegL dst, immI8 shift, rFlagsReg cr)
 9845 %{
 9846   predicate(!VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 9847   match(Set dst (RotateLeft dst shift));
 9848   effect(KILL cr);
 9849   format %{ "rolq    $dst, $shift" %}
 9850   ins_encode %{
 9851     __ rolq($dst$$Register, $shift$$constant);
 9852   %}
 9853   ins_pipe(ialu_reg);
 9854 %}
 9855 
 9856 instruct rolL_immI8(rRegL dst, rRegL src, immI8 shift)
 9857 %{
 9858   predicate(!UseAPX && VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 9859   match(Set dst (RotateLeft src shift));
 9860   format %{ "rolxq   $dst, $src, $shift" %}
 9861   ins_encode %{
 9862     int shift = 64 - ($shift$$constant & 63);
 9863     __ rorxq($dst$$Register, $src$$Register, shift);
 9864   %}
 9865   ins_pipe(ialu_reg_reg);
 9866 %}
 9867 
 9868 instruct rolL_mem_immI8(rRegL dst, memory src, immI8 shift)
 9869 %{
 9870   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 9871   match(Set dst (RotateLeft (LoadL src) shift));
 9872   ins_cost(175);
 9873   format %{ "rolxq   $dst, $src, $shift" %}
 9874   ins_encode %{
 9875     int shift = 64 - ($shift$$constant & 63);
 9876     __ rorxq($dst$$Register, $src$$Address, shift);
 9877   %}
 9878   ins_pipe(ialu_reg_mem);
 9879 %}
 9880 
 9881 // Rotate Left by variable
 9882 instruct rolL_rReg_Var(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 9883 %{
 9884   predicate(!UseAPX && n->bottom_type()->basic_type() == T_LONG);
 9885   match(Set dst (RotateLeft dst shift));
 9886   effect(KILL cr);
 9887   format %{ "rolq    $dst, $shift" %}
 9888   ins_encode %{
 9889     __ rolq($dst$$Register);
 9890   %}
 9891   ins_pipe(ialu_reg_reg);
 9892 %}
 9893 
 9894 // Rotate Left by variable
 9895 instruct rolL_rReg_Var_ndd(rRegL dst, rRegL src, rcx_RegI shift, rFlagsReg cr)
 9896 %{
 9897   predicate(UseAPX && n->bottom_type()->basic_type() == T_LONG);
 9898   match(Set dst (RotateLeft src shift));
 9899   effect(KILL cr);
 9900 
 9901   format %{ "erolq    $dst, $src, $shift\t# rotate left(long ndd)" %}
 9902   ins_encode %{
 9903     __ erolq($dst$$Register, $src$$Register, false);
 9904   %}
 9905   ins_pipe(ialu_reg_reg);
 9906 %}
 9907 
 9908 // Rotate Right by constant.
 9909 instruct rorL_immI8_legacy(rRegL dst, immI8 shift, rFlagsReg cr)
 9910 %{
 9911   predicate(!VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 9912   match(Set dst (RotateRight dst shift));
 9913   effect(KILL cr);
 9914   format %{ "rorq    $dst, $shift" %}
 9915   ins_encode %{
 9916     __ rorq($dst$$Register, $shift$$constant);
 9917   %}
 9918   ins_pipe(ialu_reg);
 9919 %}
 9920 
 9921 // Rotate Right by constant
 9922 instruct rorL_immI8(rRegL dst, rRegL src, immI8 shift)
 9923 %{
 9924   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 9925   match(Set dst (RotateRight src shift));
 9926   format %{ "rorxq   $dst, $src, $shift" %}
 9927   ins_encode %{
 9928     __ rorxq($dst$$Register, $src$$Register, $shift$$constant);
 9929   %}
 9930   ins_pipe(ialu_reg_reg);
 9931 %}
 9932 
 9933 instruct rorL_mem_immI8(rRegL dst, memory src, immI8 shift)
 9934 %{
 9935   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 9936   match(Set dst (RotateRight (LoadL src) shift));
 9937   ins_cost(175);
 9938   format %{ "rorxq   $dst, $src, $shift" %}
 9939   ins_encode %{
 9940     __ rorxq($dst$$Register, $src$$Address, $shift$$constant);
 9941   %}
 9942   ins_pipe(ialu_reg_mem);
 9943 %}
 9944 
 9945 // Rotate Right by variable
 9946 instruct rorL_rReg_Var(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 9947 %{
 9948   predicate(!UseAPX && n->bottom_type()->basic_type() == T_LONG);
 9949   match(Set dst (RotateRight dst shift));
 9950   effect(KILL cr);
 9951   format %{ "rorq    $dst, $shift" %}
 9952   ins_encode %{
 9953     __ rorq($dst$$Register);
 9954   %}
 9955   ins_pipe(ialu_reg_reg);
 9956 %}
 9957 
 9958 // Rotate Right by variable
 9959 instruct rorL_rReg_Var_ndd(rRegL dst, rRegL src, rcx_RegI shift, rFlagsReg cr)
 9960 %{
 9961   predicate(UseAPX && n->bottom_type()->basic_type() == T_LONG);
 9962   match(Set dst (RotateRight src shift));
 9963   effect(KILL cr);
 9964 
 9965   format %{ "erorq    $dst, $src, $shift\t# rotate right(long ndd)" %}
 9966   ins_encode %{
 9967     __ erorq($dst$$Register, $src$$Register, false);
 9968   %}
 9969   ins_pipe(ialu_reg_reg);
 9970 %}
 9971 
 9972 //----------------------------- CompressBits/ExpandBits ------------------------
 9973 
 9974 instruct compressBitsL_reg(rRegL dst, rRegL src, rRegL mask) %{
 9975   predicate(n->bottom_type()->isa_long());
 9976   match(Set dst (CompressBits src mask));
 9977   format %{ "pextq  $dst, $src, $mask\t! parallel bit extract" %}
 9978   ins_encode %{
 9979     __ pextq($dst$$Register, $src$$Register, $mask$$Register);
 9980   %}
 9981   ins_pipe( pipe_slow );
 9982 %}
 9983 
 9984 instruct expandBitsL_reg(rRegL dst, rRegL src, rRegL mask) %{
 9985   predicate(n->bottom_type()->isa_long());
 9986   match(Set dst (ExpandBits src mask));
 9987   format %{ "pdepq  $dst, $src, $mask\t! parallel bit deposit" %}
 9988   ins_encode %{
 9989     __ pdepq($dst$$Register, $src$$Register, $mask$$Register);
 9990   %}
 9991   ins_pipe( pipe_slow );
 9992 %}
 9993 
 9994 instruct compressBitsL_mem(rRegL dst, rRegL src, memory mask) %{
 9995   predicate(n->bottom_type()->isa_long());
 9996   match(Set dst (CompressBits src (LoadL mask)));
 9997   format %{ "pextq  $dst, $src, $mask\t! parallel bit extract" %}
 9998   ins_encode %{
 9999     __ pextq($dst$$Register, $src$$Register, $mask$$Address);
10000   %}
10001   ins_pipe( pipe_slow );
10002 %}
10003 
10004 instruct expandBitsL_mem(rRegL dst, rRegL src, memory mask) %{
10005   predicate(n->bottom_type()->isa_long());
10006   match(Set dst (ExpandBits src (LoadL mask)));
10007   format %{ "pdepq  $dst, $src, $mask\t! parallel bit deposit" %}
10008   ins_encode %{
10009     __ pdepq($dst$$Register, $src$$Register, $mask$$Address);
10010   %}
10011   ins_pipe( pipe_slow );
10012 %}
10013 
10014 
10015 // Logical Instructions
10016 
10017 // Integer Logical Instructions
10018 
10019 // And Instructions
10020 // And Register with Register
10021 instruct andI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
10022 %{
10023   predicate(!UseAPX);
10024   match(Set dst (AndI dst src));
10025   effect(KILL cr);
10026   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10027 
10028   format %{ "andl    $dst, $src\t# int" %}
10029   ins_encode %{
10030     __ andl($dst$$Register, $src$$Register);
10031   %}
10032   ins_pipe(ialu_reg_reg);
10033 %}
10034 
10035 // And Register with Register using New Data Destination (NDD)
10036 instruct andI_rReg_ndd(rRegI dst, rRegI src1, rRegI src2, rFlagsReg cr)
10037 %{
10038   predicate(UseAPX);
10039   match(Set dst (AndI src1 src2));
10040   effect(KILL cr);
10041   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10042 
10043   format %{ "eandl     $dst, $src1, $src2\t# int ndd" %}
10044   ins_encode %{
10045     __ eandl($dst$$Register, $src1$$Register, $src2$$Register, false);
10046 
10047   %}
10048   ins_pipe(ialu_reg_reg);
10049 %}
10050 
10051 // And Register with Immediate 255
10052 instruct andI_rReg_imm255(rRegI dst, rRegI src, immI_255 mask)
10053 %{
10054   match(Set dst (AndI src mask));
10055 
10056   format %{ "movzbl  $dst, $src\t# int & 0xFF" %}
10057   ins_encode %{
10058     __ movzbl($dst$$Register, $src$$Register);
10059   %}
10060   ins_pipe(ialu_reg);
10061 %}
10062 
10063 // And Register with Immediate 255 and promote to long
10064 instruct andI2L_rReg_imm255(rRegL dst, rRegI src, immI_255 mask)
10065 %{
10066   match(Set dst (ConvI2L (AndI src mask)));
10067 
10068   format %{ "movzbl  $dst, $src\t# int & 0xFF -> long" %}
10069   ins_encode %{
10070     __ movzbl($dst$$Register, $src$$Register);
10071   %}
10072   ins_pipe(ialu_reg);
10073 %}
10074 
10075 // And Register with Immediate 65535
10076 instruct andI_rReg_imm65535(rRegI dst, rRegI src, immI_65535 mask)
10077 %{
10078   match(Set dst (AndI src mask));
10079 
10080   format %{ "movzwl  $dst, $src\t# int & 0xFFFF" %}
10081   ins_encode %{
10082     __ movzwl($dst$$Register, $src$$Register);
10083   %}
10084   ins_pipe(ialu_reg);
10085 %}
10086 
10087 // And Register with Immediate 65535 and promote to long
10088 instruct andI2L_rReg_imm65535(rRegL dst, rRegI src, immI_65535 mask)
10089 %{
10090   match(Set dst (ConvI2L (AndI src mask)));
10091 
10092   format %{ "movzwl  $dst, $src\t# int & 0xFFFF -> long" %}
10093   ins_encode %{
10094     __ movzwl($dst$$Register, $src$$Register);
10095   %}
10096   ins_pipe(ialu_reg);
10097 %}
10098 
10099 // Can skip int2long conversions after AND with small bitmask
10100 instruct convI2LAndI_reg_immIbitmask(rRegL dst, rRegI src,  immI_Pow2M1 mask, rRegI tmp, rFlagsReg cr)
10101 %{
10102   predicate(VM_Version::supports_bmi2());
10103   ins_cost(125);
10104   effect(TEMP tmp, KILL cr);
10105   match(Set dst (ConvI2L (AndI src mask)));
10106   format %{ "bzhiq $dst, $src, $mask \t# using $tmp as TEMP, int &  immI_Pow2M1 -> long" %}
10107   ins_encode %{
10108     __ movl($tmp$$Register, exact_log2($mask$$constant + 1));
10109     __ bzhiq($dst$$Register, $src$$Register, $tmp$$Register);
10110   %}
10111   ins_pipe(ialu_reg_reg);
10112 %}
10113 
10114 // And Register with Immediate
10115 instruct andI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
10116 %{
10117   predicate(!UseAPX);
10118   match(Set dst (AndI dst src));
10119   effect(KILL cr);
10120   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10121 
10122   format %{ "andl    $dst, $src\t# int" %}
10123   ins_encode %{
10124     __ andl($dst$$Register, $src$$constant);
10125   %}
10126   ins_pipe(ialu_reg);
10127 %}
10128 
10129 instruct andI_rReg_rReg_imm_ndd(rRegI dst, rRegI src1, immI src2, rFlagsReg cr)
10130 %{
10131   predicate(UseAPX);
10132   match(Set dst (AndI src1 src2));
10133   effect(KILL cr);
10134   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10135 
10136   format %{ "eandl    $dst, $src1, $src2\t# int ndd" %}
10137   ins_encode %{
10138     __ eandl($dst$$Register, $src1$$Register, $src2$$constant, false);
10139   %}
10140   ins_pipe(ialu_reg);
10141 %}
10142 
10143 instruct andI_rReg_mem_imm_ndd(rRegI dst, memory src1, immI src2, rFlagsReg cr)
10144 %{
10145   predicate(UseAPX);
10146   match(Set dst (AndI (LoadI src1) src2));
10147   effect(KILL cr);
10148   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10149 
10150   format %{ "eandl    $dst, $src1, $src2\t# int ndd" %}
10151   ins_encode %{
10152     __ eandl($dst$$Register, $src1$$Address, $src2$$constant, false);
10153   %}
10154   ins_pipe(ialu_reg);
10155 %}
10156 
10157 // And Register with Memory
10158 instruct andI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
10159 %{
10160   predicate(!UseAPX);
10161   match(Set dst (AndI dst (LoadI src)));
10162   effect(KILL cr);
10163   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10164 
10165   ins_cost(150);
10166   format %{ "andl    $dst, $src\t# int" %}
10167   ins_encode %{
10168     __ andl($dst$$Register, $src$$Address);
10169   %}
10170   ins_pipe(ialu_reg_mem);
10171 %}
10172 
10173 instruct andI_rReg_rReg_mem_ndd(rRegI dst, rRegI src1, memory src2, rFlagsReg cr)
10174 %{
10175   predicate(UseAPX);
10176   match(Set dst (AndI src1 (LoadI src2)));
10177   effect(KILL cr);
10178   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10179 
10180   ins_cost(150);
10181   format %{ "eandl    $dst, $src1, $src2\t# int ndd" %}
10182   ins_encode %{
10183     __ eandl($dst$$Register, $src1$$Register, $src2$$Address, false);
10184   %}
10185   ins_pipe(ialu_reg_mem);
10186 %}
10187 
10188 // And Memory with Register
10189 instruct andB_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
10190 %{
10191   match(Set dst (StoreB dst (AndI (LoadB dst) src)));
10192   effect(KILL cr);
10193   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10194 
10195   ins_cost(150);
10196   format %{ "andb    $dst, $src\t# byte" %}
10197   ins_encode %{
10198     __ andb($dst$$Address, $src$$Register);
10199   %}
10200   ins_pipe(ialu_mem_reg);
10201 %}
10202 
10203 instruct andI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
10204 %{
10205   match(Set dst (StoreI dst (AndI (LoadI dst) src)));
10206   effect(KILL cr);
10207   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10208 
10209   ins_cost(150);
10210   format %{ "andl    $dst, $src\t# int" %}
10211   ins_encode %{
10212     __ andl($dst$$Address, $src$$Register);
10213   %}
10214   ins_pipe(ialu_mem_reg);
10215 %}
10216 
10217 // And Memory with Immediate
10218 instruct andI_mem_imm(memory dst, immI src, rFlagsReg cr)
10219 %{
10220   match(Set dst (StoreI dst (AndI (LoadI dst) src)));
10221   effect(KILL cr);
10222   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10223 
10224   ins_cost(125);
10225   format %{ "andl    $dst, $src\t# int" %}
10226   ins_encode %{
10227     __ andl($dst$$Address, $src$$constant);
10228   %}
10229   ins_pipe(ialu_mem_imm);
10230 %}
10231 
10232 // BMI1 instructions
10233 instruct andnI_rReg_rReg_mem(rRegI dst, rRegI src1, memory src2, immI_M1 minus_1, rFlagsReg cr) %{
10234   match(Set dst (AndI (XorI src1 minus_1) (LoadI src2)));
10235   predicate(UseBMI1Instructions);
10236   effect(KILL cr);
10237   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10238 
10239   ins_cost(125);
10240   format %{ "andnl  $dst, $src1, $src2" %}
10241 
10242   ins_encode %{
10243     __ andnl($dst$$Register, $src1$$Register, $src2$$Address);
10244   %}
10245   ins_pipe(ialu_reg_mem);
10246 %}
10247 
10248 instruct andnI_rReg_rReg_rReg(rRegI dst, rRegI src1, rRegI src2, immI_M1 minus_1, rFlagsReg cr) %{
10249   match(Set dst (AndI (XorI src1 minus_1) src2));
10250   predicate(UseBMI1Instructions);
10251   effect(KILL cr);
10252   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10253 
10254   format %{ "andnl  $dst, $src1, $src2" %}
10255 
10256   ins_encode %{
10257     __ andnl($dst$$Register, $src1$$Register, $src2$$Register);
10258   %}
10259   ins_pipe(ialu_reg);
10260 %}
10261 
10262 instruct blsiI_rReg_rReg(rRegI dst, rRegI src, immI_0 imm_zero, rFlagsReg cr) %{
10263   match(Set dst (AndI (SubI imm_zero src) src));
10264   predicate(UseBMI1Instructions);
10265   effect(KILL cr);
10266   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
10267 
10268   format %{ "blsil  $dst, $src" %}
10269 
10270   ins_encode %{
10271     __ blsil($dst$$Register, $src$$Register);
10272   %}
10273   ins_pipe(ialu_reg);
10274 %}
10275 
10276 instruct blsiI_rReg_mem(rRegI dst, memory src, immI_0 imm_zero, rFlagsReg cr) %{
10277   match(Set dst (AndI (SubI imm_zero (LoadI src) ) (LoadI src) ));
10278   predicate(UseBMI1Instructions);
10279   effect(KILL cr);
10280   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
10281 
10282   ins_cost(125);
10283   format %{ "blsil  $dst, $src" %}
10284 
10285   ins_encode %{
10286     __ blsil($dst$$Register, $src$$Address);
10287   %}
10288   ins_pipe(ialu_reg_mem);
10289 %}
10290 
10291 instruct blsmskI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, rFlagsReg cr)
10292 %{
10293   match(Set dst (XorI (AddI (LoadI src) minus_1) (LoadI src) ) );
10294   predicate(UseBMI1Instructions);
10295   effect(KILL cr);
10296   flag(PD::Flag_sets_sign_flag, PD::Flag_clears_zero_flag, PD::Flag_clears_overflow_flag);
10297 
10298   ins_cost(125);
10299   format %{ "blsmskl $dst, $src" %}
10300 
10301   ins_encode %{
10302     __ blsmskl($dst$$Register, $src$$Address);
10303   %}
10304   ins_pipe(ialu_reg_mem);
10305 %}
10306 
10307 instruct blsmskI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, rFlagsReg cr)
10308 %{
10309   match(Set dst (XorI (AddI src minus_1) src));
10310   predicate(UseBMI1Instructions);
10311   effect(KILL cr);
10312   flag(PD::Flag_sets_sign_flag, PD::Flag_clears_zero_flag, PD::Flag_clears_overflow_flag);
10313 
10314   format %{ "blsmskl $dst, $src" %}
10315 
10316   ins_encode %{
10317     __ blsmskl($dst$$Register, $src$$Register);
10318   %}
10319 
10320   ins_pipe(ialu_reg);
10321 %}
10322 
10323 instruct blsrI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, rFlagsReg cr)
10324 %{
10325   match(Set dst (AndI (AddI src minus_1) src) );
10326   predicate(UseBMI1Instructions);
10327   effect(KILL cr);
10328   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
10329 
10330   format %{ "blsrl  $dst, $src" %}
10331 
10332   ins_encode %{
10333     __ blsrl($dst$$Register, $src$$Register);
10334   %}
10335 
10336   ins_pipe(ialu_reg_mem);
10337 %}
10338 
10339 instruct blsrI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, rFlagsReg cr)
10340 %{
10341   match(Set dst (AndI (AddI (LoadI src) minus_1) (LoadI src) ) );
10342   predicate(UseBMI1Instructions);
10343   effect(KILL cr);
10344   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
10345 
10346   ins_cost(125);
10347   format %{ "blsrl  $dst, $src" %}
10348 
10349   ins_encode %{
10350     __ blsrl($dst$$Register, $src$$Address);
10351   %}
10352 
10353   ins_pipe(ialu_reg);
10354 %}
10355 
10356 // Or Instructions
10357 // Or Register with Register
10358 instruct orI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
10359 %{
10360   predicate(!UseAPX);
10361   match(Set dst (OrI dst src));
10362   effect(KILL cr);
10363   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10364 
10365   format %{ "orl     $dst, $src\t# int" %}
10366   ins_encode %{
10367     __ orl($dst$$Register, $src$$Register);
10368   %}
10369   ins_pipe(ialu_reg_reg);
10370 %}
10371 
10372 // Or Register with Register using New Data Destination (NDD)
10373 instruct orI_rReg_ndd(rRegI dst, rRegI src1, rRegI src2, rFlagsReg cr)
10374 %{
10375   predicate(UseAPX);
10376   match(Set dst (OrI src1 src2));
10377   effect(KILL cr);
10378   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10379 
10380   format %{ "eorl     $dst, $src1, $src2\t# int ndd" %}
10381   ins_encode %{
10382     __ eorl($dst$$Register, $src1$$Register, $src2$$Register, false);
10383   %}
10384   ins_pipe(ialu_reg_reg);
10385 %}
10386 
10387 // Or Register with Immediate
10388 instruct orI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
10389 %{
10390   predicate(!UseAPX);
10391   match(Set dst (OrI dst src));
10392   effect(KILL cr);
10393   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10394 
10395   format %{ "orl     $dst, $src\t# int" %}
10396   ins_encode %{
10397     __ orl($dst$$Register, $src$$constant);
10398   %}
10399   ins_pipe(ialu_reg);
10400 %}
10401 
10402 instruct orI_rReg_rReg_imm_ndd(rRegI dst, rRegI src1, immI src2, rFlagsReg cr)
10403 %{
10404   predicate(UseAPX);
10405   match(Set dst (OrI src1 src2));
10406   effect(KILL cr);
10407   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10408 
10409   format %{ "eorl     $dst, $src1, $src2\t# int ndd" %}
10410   ins_encode %{
10411     __ eorl($dst$$Register, $src1$$Register, $src2$$constant, false);
10412   %}
10413   ins_pipe(ialu_reg);
10414 %}
10415 
10416 instruct orI_rReg_imm_rReg_ndd(rRegI dst, immI src1, rRegI src2, rFlagsReg cr)
10417 %{
10418   predicate(UseAPX);
10419   match(Set dst (OrI src1 src2));
10420   effect(KILL cr);
10421   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10422 
10423   format %{ "eorl     $dst, $src2, $src1\t# int ndd" %}
10424   ins_encode %{
10425     __ eorl($dst$$Register, $src2$$Register, $src1$$constant, false);
10426   %}
10427   ins_pipe(ialu_reg);
10428 %}
10429 
10430 instruct orI_rReg_mem_imm_ndd(rRegI dst, memory src1, immI src2, rFlagsReg cr)
10431 %{
10432   predicate(UseAPX);
10433   match(Set dst (OrI (LoadI src1) src2));
10434   effect(KILL cr);
10435   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10436 
10437   format %{ "eorl     $dst, $src1, $src2\t# int ndd" %}
10438   ins_encode %{
10439     __ eorl($dst$$Register, $src1$$Address, $src2$$constant, false);
10440   %}
10441   ins_pipe(ialu_reg);
10442 %}
10443 
10444 // Or Register with Memory
10445 instruct orI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
10446 %{
10447   predicate(!UseAPX);
10448   match(Set dst (OrI dst (LoadI src)));
10449   effect(KILL cr);
10450   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10451 
10452   ins_cost(150);
10453   format %{ "orl     $dst, $src\t# int" %}
10454   ins_encode %{
10455     __ orl($dst$$Register, $src$$Address);
10456   %}
10457   ins_pipe(ialu_reg_mem);
10458 %}
10459 
10460 instruct orI_rReg_rReg_mem_ndd(rRegI dst, rRegI src1, memory src2, rFlagsReg cr)
10461 %{
10462   predicate(UseAPX);
10463   match(Set dst (OrI src1 (LoadI src2)));
10464   effect(KILL cr);
10465   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10466 
10467   ins_cost(150);
10468   format %{ "eorl     $dst, $src1, $src2\t# int ndd" %}
10469   ins_encode %{
10470     __ eorl($dst$$Register, $src1$$Register, $src2$$Address, false);
10471   %}
10472   ins_pipe(ialu_reg_mem);
10473 %}
10474 
10475 // Or Memory with Register
10476 instruct orB_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
10477 %{
10478   match(Set dst (StoreB dst (OrI (LoadB dst) src)));
10479   effect(KILL cr);
10480   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10481 
10482   ins_cost(150);
10483   format %{ "orb    $dst, $src\t# byte" %}
10484   ins_encode %{
10485     __ orb($dst$$Address, $src$$Register);
10486   %}
10487   ins_pipe(ialu_mem_reg);
10488 %}
10489 
10490 instruct orI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
10491 %{
10492   match(Set dst (StoreI dst (OrI (LoadI dst) src)));
10493   effect(KILL cr);
10494   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10495 
10496   ins_cost(150);
10497   format %{ "orl     $dst, $src\t# int" %}
10498   ins_encode %{
10499     __ orl($dst$$Address, $src$$Register);
10500   %}
10501   ins_pipe(ialu_mem_reg);
10502 %}
10503 
10504 // Or Memory with Immediate
10505 instruct orI_mem_imm(memory dst, immI src, rFlagsReg cr)
10506 %{
10507   match(Set dst (StoreI dst (OrI (LoadI dst) src)));
10508   effect(KILL cr);
10509   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10510 
10511   ins_cost(125);
10512   format %{ "orl     $dst, $src\t# int" %}
10513   ins_encode %{
10514     __ orl($dst$$Address, $src$$constant);
10515   %}
10516   ins_pipe(ialu_mem_imm);
10517 %}
10518 
10519 // Xor Instructions
10520 // Xor Register with Register
10521 instruct xorI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
10522 %{
10523   predicate(!UseAPX);
10524   match(Set dst (XorI dst src));
10525   effect(KILL cr);
10526   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10527 
10528   format %{ "xorl    $dst, $src\t# int" %}
10529   ins_encode %{
10530     __ xorl($dst$$Register, $src$$Register);
10531   %}
10532   ins_pipe(ialu_reg_reg);
10533 %}
10534 
10535 // Xor Register with Register using New Data Destination (NDD)
10536 instruct xorI_rReg_ndd(rRegI dst, rRegI src1, rRegI src2, rFlagsReg cr)
10537 %{
10538   predicate(UseAPX);
10539   match(Set dst (XorI src1 src2));
10540   effect(KILL cr);
10541   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10542 
10543   format %{ "exorl    $dst, $src1, $src2\t# int ndd" %}
10544   ins_encode %{
10545     __ exorl($dst$$Register, $src1$$Register, $src2$$Register, false);
10546   %}
10547   ins_pipe(ialu_reg_reg);
10548 %}
10549 
10550 // Xor Register with Immediate -1
10551 instruct xorI_rReg_im1(rRegI dst, immI_M1 imm)
10552 %{
10553   predicate(!UseAPX);
10554   match(Set dst (XorI dst imm));
10555 
10556   format %{ "notl    $dst" %}
10557   ins_encode %{
10558      __ notl($dst$$Register);
10559   %}
10560   ins_pipe(ialu_reg);
10561 %}
10562 
10563 instruct xorI_rReg_im1_ndd(rRegI dst, rRegI src, immI_M1 imm)
10564 %{
10565   match(Set dst (XorI src imm));
10566   predicate(UseAPX);
10567 
10568   format %{ "enotl    $dst, $src" %}
10569   ins_encode %{
10570      __ enotl($dst$$Register, $src$$Register);
10571   %}
10572   ins_pipe(ialu_reg);
10573 %}
10574 
10575 // Xor Register with Immediate
10576 instruct xorI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
10577 %{
10578   // Strict predicate check to make selection of xorI_rReg_im1 cost agnostic if immI src is -1.
10579   predicate(!UseAPX && n->in(2)->bottom_type()->is_int()->get_con() != -1);
10580   match(Set dst (XorI dst src));
10581   effect(KILL cr);
10582   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10583 
10584   format %{ "xorl    $dst, $src\t# int" %}
10585   ins_encode %{
10586     __ xorl($dst$$Register, $src$$constant);
10587   %}
10588   ins_pipe(ialu_reg);
10589 %}
10590 
10591 instruct xorI_rReg_rReg_imm_ndd(rRegI dst, rRegI src1, immI src2, rFlagsReg cr)
10592 %{
10593   // Strict predicate check to make selection of xorI_rReg_im1_ndd cost agnostic if immI src2 is -1.
10594   predicate(UseAPX && n->in(2)->bottom_type()->is_int()->get_con() != -1);
10595   match(Set dst (XorI src1 src2));
10596   effect(KILL cr);
10597   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10598 
10599   format %{ "exorl    $dst, $src1, $src2\t# int ndd" %}
10600   ins_encode %{
10601     __ exorl($dst$$Register, $src1$$Register, $src2$$constant, false);
10602   %}
10603   ins_pipe(ialu_reg);
10604 %}
10605 
10606 // Xor Memory with Immediate
10607 instruct xorI_rReg_mem_imm_ndd(rRegI dst, memory src1, immI src2, rFlagsReg cr)
10608 %{
10609   predicate(UseAPX);
10610   match(Set dst (XorI (LoadI src1) src2));
10611   effect(KILL cr);
10612   ins_cost(150);
10613   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10614 
10615   format %{ "exorl    $dst, $src1, $src2\t# int ndd" %}
10616   ins_encode %{
10617     __ exorl($dst$$Register, $src1$$Address, $src2$$constant, false);
10618   %}
10619   ins_pipe(ialu_reg);
10620 %}
10621 
10622 // Xor Register with Memory
10623 instruct xorI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
10624 %{
10625   predicate(!UseAPX);
10626   match(Set dst (XorI dst (LoadI src)));
10627   effect(KILL cr);
10628   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10629 
10630   ins_cost(150);
10631   format %{ "xorl    $dst, $src\t# int" %}
10632   ins_encode %{
10633     __ xorl($dst$$Register, $src$$Address);
10634   %}
10635   ins_pipe(ialu_reg_mem);
10636 %}
10637 
10638 instruct xorI_rReg_rReg_mem_ndd(rRegI dst, rRegI src1, memory src2, rFlagsReg cr)
10639 %{
10640   predicate(UseAPX);
10641   match(Set dst (XorI src1 (LoadI src2)));
10642   effect(KILL cr);
10643   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10644 
10645   ins_cost(150);
10646   format %{ "exorl    $dst, $src1, $src2\t# int ndd" %}
10647   ins_encode %{
10648     __ exorl($dst$$Register, $src1$$Register, $src2$$Address, false);
10649   %}
10650   ins_pipe(ialu_reg_mem);
10651 %}
10652 
10653 // Xor Memory with Register
10654 instruct xorB_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
10655 %{
10656   match(Set dst (StoreB dst (XorI (LoadB dst) src)));
10657   effect(KILL cr);
10658   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10659 
10660   ins_cost(150);
10661   format %{ "xorb    $dst, $src\t# byte" %}
10662   ins_encode %{
10663     __ xorb($dst$$Address, $src$$Register);
10664   %}
10665   ins_pipe(ialu_mem_reg);
10666 %}
10667 
10668 instruct xorI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
10669 %{
10670   match(Set dst (StoreI dst (XorI (LoadI dst) src)));
10671   effect(KILL cr);
10672   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10673 
10674   ins_cost(150);
10675   format %{ "xorl    $dst, $src\t# int" %}
10676   ins_encode %{
10677     __ xorl($dst$$Address, $src$$Register);
10678   %}
10679   ins_pipe(ialu_mem_reg);
10680 %}
10681 
10682 // Xor Memory with Immediate
10683 instruct xorI_mem_imm(memory dst, immI src, rFlagsReg cr)
10684 %{
10685   match(Set dst (StoreI dst (XorI (LoadI dst) src)));
10686   effect(KILL cr);
10687   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10688 
10689   ins_cost(125);
10690   format %{ "xorl    $dst, $src\t# int" %}
10691   ins_encode %{
10692     __ xorl($dst$$Address, $src$$constant);
10693   %}
10694   ins_pipe(ialu_mem_imm);
10695 %}
10696 
10697 
10698 // Long Logical Instructions
10699 
10700 // And Instructions
10701 // And Register with Register
10702 instruct andL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
10703 %{
10704   predicate(!UseAPX);
10705   match(Set dst (AndL dst src));
10706   effect(KILL cr);
10707   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10708 
10709   format %{ "andq    $dst, $src\t# long" %}
10710   ins_encode %{
10711     __ andq($dst$$Register, $src$$Register);
10712   %}
10713   ins_pipe(ialu_reg_reg);
10714 %}
10715 
10716 // And Register with Register using New Data Destination (NDD)
10717 instruct andL_rReg_ndd(rRegL dst, rRegL src1, rRegL src2, rFlagsReg cr)
10718 %{
10719   predicate(UseAPX);
10720   match(Set dst (AndL src1 src2));
10721   effect(KILL cr);
10722   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10723 
10724   format %{ "eandq     $dst, $src1, $src2\t# long ndd" %}
10725   ins_encode %{
10726     __ eandq($dst$$Register, $src1$$Register, $src2$$Register, false);
10727 
10728   %}
10729   ins_pipe(ialu_reg_reg);
10730 %}
10731 
10732 // And Register with Immediate 255
10733 instruct andL_rReg_imm255(rRegL dst, rRegL src, immL_255 mask)
10734 %{
10735   match(Set dst (AndL src mask));
10736 
10737   format %{ "movzbl  $dst, $src\t# long & 0xFF" %}
10738   ins_encode %{
10739     // movzbl zeroes out the upper 32-bit and does not need REX.W
10740     __ movzbl($dst$$Register, $src$$Register);
10741   %}
10742   ins_pipe(ialu_reg);
10743 %}
10744 
10745 // And Register with Immediate 65535
10746 instruct andL_rReg_imm65535(rRegL dst, rRegL src, immL_65535 mask)
10747 %{
10748   match(Set dst (AndL src mask));
10749 
10750   format %{ "movzwl  $dst, $src\t# long & 0xFFFF" %}
10751   ins_encode %{
10752     // movzwl zeroes out the upper 32-bit and does not need REX.W
10753     __ movzwl($dst$$Register, $src$$Register);
10754   %}
10755   ins_pipe(ialu_reg);
10756 %}
10757 
10758 // And Register with Immediate
10759 instruct andL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
10760 %{
10761   predicate(!UseAPX);
10762   match(Set dst (AndL dst src));
10763   effect(KILL cr);
10764   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10765 
10766   format %{ "andq    $dst, $src\t# long" %}
10767   ins_encode %{
10768     __ andq($dst$$Register, $src$$constant);
10769   %}
10770   ins_pipe(ialu_reg);
10771 %}
10772 
10773 instruct andL_rReg_rReg_imm_ndd(rRegL dst, rRegL src1, immL32 src2, rFlagsReg cr)
10774 %{
10775   predicate(UseAPX);
10776   match(Set dst (AndL src1 src2));
10777   effect(KILL cr);
10778   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10779 
10780   format %{ "eandq    $dst, $src1, $src2\t# long ndd" %}
10781   ins_encode %{
10782     __ eandq($dst$$Register, $src1$$Register, $src2$$constant, false);
10783   %}
10784   ins_pipe(ialu_reg);
10785 %}
10786 
10787 instruct andL_rReg_mem_imm_ndd(rRegL dst, memory src1, immL32 src2, rFlagsReg cr)
10788 %{
10789   predicate(UseAPX);
10790   match(Set dst (AndL (LoadL src1) src2));
10791   effect(KILL cr);
10792   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10793 
10794   format %{ "eandq    $dst, $src1, $src2\t# long ndd" %}
10795   ins_encode %{
10796     __ eandq($dst$$Register, $src1$$Address, $src2$$constant, false);
10797   %}
10798   ins_pipe(ialu_reg);
10799 %}
10800 
10801 // And Register with Memory
10802 instruct andL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
10803 %{
10804   predicate(!UseAPX);
10805   match(Set dst (AndL dst (LoadL src)));
10806   effect(KILL cr);
10807   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10808 
10809   ins_cost(150);
10810   format %{ "andq    $dst, $src\t# long" %}
10811   ins_encode %{
10812     __ andq($dst$$Register, $src$$Address);
10813   %}
10814   ins_pipe(ialu_reg_mem);
10815 %}
10816 
10817 instruct andL_rReg_rReg_mem_ndd(rRegL dst, rRegL src1, memory src2, rFlagsReg cr)
10818 %{
10819   predicate(UseAPX);
10820   match(Set dst (AndL src1 (LoadL src2)));
10821   effect(KILL cr);
10822   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10823 
10824   ins_cost(150);
10825   format %{ "eandq    $dst, $src1, $src2\t# long ndd" %}
10826   ins_encode %{
10827     __ eandq($dst$$Register, $src1$$Register, $src2$$Address, false);
10828   %}
10829   ins_pipe(ialu_reg_mem);
10830 %}
10831 
10832 // And Memory with Register
10833 instruct andL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
10834 %{
10835   match(Set dst (StoreL dst (AndL (LoadL dst) src)));
10836   effect(KILL cr);
10837   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10838 
10839   ins_cost(150);
10840   format %{ "andq    $dst, $src\t# long" %}
10841   ins_encode %{
10842     __ andq($dst$$Address, $src$$Register);
10843   %}
10844   ins_pipe(ialu_mem_reg);
10845 %}
10846 
10847 // And Memory with Immediate
10848 instruct andL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
10849 %{
10850   match(Set dst (StoreL dst (AndL (LoadL dst) src)));
10851   effect(KILL cr);
10852   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10853 
10854   ins_cost(125);
10855   format %{ "andq    $dst, $src\t# long" %}
10856   ins_encode %{
10857     __ andq($dst$$Address, $src$$constant);
10858   %}
10859   ins_pipe(ialu_mem_imm);
10860 %}
10861 
10862 instruct btrL_mem_imm(memory dst, immL_NotPow2 con, rFlagsReg cr)
10863 %{
10864   // con should be a pure 64-bit immediate given that not(con) is a power of 2
10865   // because AND/OR works well enough for 8/32-bit values.
10866   predicate(log2i_graceful(~n->in(3)->in(2)->get_long()) > 30);
10867 
10868   match(Set dst (StoreL dst (AndL (LoadL dst) con)));
10869   effect(KILL cr);
10870 
10871   ins_cost(125);
10872   format %{ "btrq    $dst, log2(not($con))\t# long" %}
10873   ins_encode %{
10874     __ btrq($dst$$Address, log2i_exact((julong)~$con$$constant));
10875   %}
10876   ins_pipe(ialu_mem_imm);
10877 %}
10878 
10879 // BMI1 instructions
10880 instruct andnL_rReg_rReg_mem(rRegL dst, rRegL src1, memory src2, immL_M1 minus_1, rFlagsReg cr) %{
10881   match(Set dst (AndL (XorL src1 minus_1) (LoadL src2)));
10882   predicate(UseBMI1Instructions);
10883   effect(KILL cr);
10884   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10885 
10886   ins_cost(125);
10887   format %{ "andnq  $dst, $src1, $src2" %}
10888 
10889   ins_encode %{
10890     __ andnq($dst$$Register, $src1$$Register, $src2$$Address);
10891   %}
10892   ins_pipe(ialu_reg_mem);
10893 %}
10894 
10895 instruct andnL_rReg_rReg_rReg(rRegL dst, rRegL src1, rRegL src2, immL_M1 minus_1, rFlagsReg cr) %{
10896   match(Set dst (AndL (XorL src1 minus_1) src2));
10897   predicate(UseBMI1Instructions);
10898   effect(KILL cr);
10899   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10900 
10901   format %{ "andnq  $dst, $src1, $src2" %}
10902 
10903   ins_encode %{
10904   __ andnq($dst$$Register, $src1$$Register, $src2$$Register);
10905   %}
10906   ins_pipe(ialu_reg_mem);
10907 %}
10908 
10909 instruct blsiL_rReg_rReg(rRegL dst, rRegL src, immL0 imm_zero, rFlagsReg cr) %{
10910   match(Set dst (AndL (SubL imm_zero src) src));
10911   predicate(UseBMI1Instructions);
10912   effect(KILL cr);
10913   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
10914 
10915   format %{ "blsiq  $dst, $src" %}
10916 
10917   ins_encode %{
10918     __ blsiq($dst$$Register, $src$$Register);
10919   %}
10920   ins_pipe(ialu_reg);
10921 %}
10922 
10923 instruct blsiL_rReg_mem(rRegL dst, memory src, immL0 imm_zero, rFlagsReg cr) %{
10924   match(Set dst (AndL (SubL imm_zero (LoadL src) ) (LoadL src) ));
10925   predicate(UseBMI1Instructions);
10926   effect(KILL cr);
10927   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
10928 
10929   ins_cost(125);
10930   format %{ "blsiq  $dst, $src" %}
10931 
10932   ins_encode %{
10933     __ blsiq($dst$$Register, $src$$Address);
10934   %}
10935   ins_pipe(ialu_reg_mem);
10936 %}
10937 
10938 instruct blsmskL_rReg_mem(rRegL dst, memory src, immL_M1 minus_1, rFlagsReg cr)
10939 %{
10940   match(Set dst (XorL (AddL (LoadL src) minus_1) (LoadL src) ) );
10941   predicate(UseBMI1Instructions);
10942   effect(KILL cr);
10943   flag(PD::Flag_sets_sign_flag, PD::Flag_clears_zero_flag, PD::Flag_clears_overflow_flag);
10944 
10945   ins_cost(125);
10946   format %{ "blsmskq $dst, $src" %}
10947 
10948   ins_encode %{
10949     __ blsmskq($dst$$Register, $src$$Address);
10950   %}
10951   ins_pipe(ialu_reg_mem);
10952 %}
10953 
10954 instruct blsmskL_rReg_rReg(rRegL dst, rRegL src, immL_M1 minus_1, rFlagsReg cr)
10955 %{
10956   match(Set dst (XorL (AddL src minus_1) src));
10957   predicate(UseBMI1Instructions);
10958   effect(KILL cr);
10959   flag(PD::Flag_sets_sign_flag, PD::Flag_clears_zero_flag, PD::Flag_clears_overflow_flag);
10960 
10961   format %{ "blsmskq $dst, $src" %}
10962 
10963   ins_encode %{
10964     __ blsmskq($dst$$Register, $src$$Register);
10965   %}
10966 
10967   ins_pipe(ialu_reg);
10968 %}
10969 
10970 instruct blsrL_rReg_rReg(rRegL dst, rRegL src, immL_M1 minus_1, rFlagsReg cr)
10971 %{
10972   match(Set dst (AndL (AddL src minus_1) src) );
10973   predicate(UseBMI1Instructions);
10974   effect(KILL cr);
10975   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
10976 
10977   format %{ "blsrq  $dst, $src" %}
10978 
10979   ins_encode %{
10980     __ blsrq($dst$$Register, $src$$Register);
10981   %}
10982 
10983   ins_pipe(ialu_reg);
10984 %}
10985 
10986 instruct blsrL_rReg_mem(rRegL dst, memory src, immL_M1 minus_1, rFlagsReg cr)
10987 %{
10988   match(Set dst (AndL (AddL (LoadL src) minus_1) (LoadL src)) );
10989   predicate(UseBMI1Instructions);
10990   effect(KILL cr);
10991   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
10992 
10993   ins_cost(125);
10994   format %{ "blsrq  $dst, $src" %}
10995 
10996   ins_encode %{
10997     __ blsrq($dst$$Register, $src$$Address);
10998   %}
10999 
11000   ins_pipe(ialu_reg);
11001 %}
11002 
11003 // Or Instructions
11004 // Or Register with Register
11005 instruct orL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
11006 %{
11007   predicate(!UseAPX);
11008   match(Set dst (OrL dst src));
11009   effect(KILL cr);
11010   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11011 
11012   format %{ "orq     $dst, $src\t# long" %}
11013   ins_encode %{
11014     __ orq($dst$$Register, $src$$Register);
11015   %}
11016   ins_pipe(ialu_reg_reg);
11017 %}
11018 
11019 // Or Register with Register using New Data Destination (NDD)
11020 instruct orL_rReg_ndd(rRegL dst, rRegL src1, rRegL src2, rFlagsReg cr)
11021 %{
11022   predicate(UseAPX);
11023   match(Set dst (OrL src1 src2));
11024   effect(KILL cr);
11025   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11026 
11027   format %{ "eorq     $dst, $src1, $src2\t# long ndd" %}
11028   ins_encode %{
11029     __ eorq($dst$$Register, $src1$$Register, $src2$$Register, false);
11030 
11031   %}
11032   ins_pipe(ialu_reg_reg);
11033 %}
11034 
11035 // Use any_RegP to match R15 (TLS register) without spilling.
11036 instruct orL_rReg_castP2X(rRegL dst, any_RegP src, rFlagsReg cr) %{
11037   match(Set dst (OrL dst (CastP2X src)));
11038   effect(KILL cr);
11039   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11040 
11041   format %{ "orq     $dst, $src\t# long" %}
11042   ins_encode %{
11043     __ orq($dst$$Register, $src$$Register);
11044   %}
11045   ins_pipe(ialu_reg_reg);
11046 %}
11047 
11048 instruct orL_rReg_castP2X_ndd(rRegL dst, any_RegP src1, any_RegP src2, rFlagsReg cr) %{
11049   match(Set dst (OrL src1 (CastP2X src2)));
11050   effect(KILL cr);
11051   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11052 
11053   format %{ "eorq     $dst, $src1, $src2\t# long ndd" %}
11054   ins_encode %{
11055     __ eorq($dst$$Register, $src1$$Register, $src2$$Register, false);
11056   %}
11057   ins_pipe(ialu_reg_reg);
11058 %}
11059 
11060 // Or Register with Immediate
11061 instruct orL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
11062 %{
11063   predicate(!UseAPX);
11064   match(Set dst (OrL dst src));
11065   effect(KILL cr);
11066   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11067 
11068   format %{ "orq     $dst, $src\t# long" %}
11069   ins_encode %{
11070     __ orq($dst$$Register, $src$$constant);
11071   %}
11072   ins_pipe(ialu_reg);
11073 %}
11074 
11075 instruct orL_rReg_rReg_imm_ndd(rRegL dst, rRegL src1, immL32 src2, rFlagsReg cr)
11076 %{
11077   predicate(UseAPX);
11078   match(Set dst (OrL src1 src2));
11079   effect(KILL cr);
11080   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11081 
11082   format %{ "eorq     $dst, $src1, $src2\t# long ndd" %}
11083   ins_encode %{
11084     __ eorq($dst$$Register, $src1$$Register, $src2$$constant, false);
11085   %}
11086   ins_pipe(ialu_reg);
11087 %}
11088 
11089 instruct orL_rReg_imm_rReg_ndd(rRegL dst, immL32 src1, rRegL src2, rFlagsReg cr)
11090 %{
11091   predicate(UseAPX);
11092   match(Set dst (OrL src1 src2));
11093   effect(KILL cr);
11094   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11095 
11096   format %{ "eorq     $dst, $src2, $src1\t# long ndd" %}
11097   ins_encode %{
11098     __ eorq($dst$$Register, $src2$$Register, $src1$$constant, false);
11099   %}
11100   ins_pipe(ialu_reg);
11101 %}
11102 
11103 // Or Memory with Immediate
11104 instruct orL_rReg_mem_imm_ndd(rRegL dst, memory src1, immL32 src2, rFlagsReg cr)
11105 %{
11106   predicate(UseAPX);
11107   match(Set dst (OrL (LoadL src1) src2));
11108   effect(KILL cr);
11109   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11110 
11111   format %{ "eorq     $dst, $src1, $src2\t# long ndd" %}
11112   ins_encode %{
11113     __ eorq($dst$$Register, $src1$$Address, $src2$$constant, false);
11114   %}
11115   ins_pipe(ialu_reg);
11116 %}
11117 
11118 // Or Register with Memory
11119 instruct orL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
11120 %{
11121   predicate(!UseAPX);
11122   match(Set dst (OrL dst (LoadL src)));
11123   effect(KILL cr);
11124   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11125 
11126   ins_cost(150);
11127   format %{ "orq     $dst, $src\t# long" %}
11128   ins_encode %{
11129     __ orq($dst$$Register, $src$$Address);
11130   %}
11131   ins_pipe(ialu_reg_mem);
11132 %}
11133 
11134 instruct orL_rReg_rReg_mem_ndd(rRegL dst, rRegL src1, memory src2, rFlagsReg cr)
11135 %{
11136   predicate(UseAPX);
11137   match(Set dst (OrL src1 (LoadL src2)));
11138   effect(KILL cr);
11139   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11140 
11141   ins_cost(150);
11142   format %{ "eorq     $dst, $src1, $src2\t# long ndd" %}
11143   ins_encode %{
11144     __ eorq($dst$$Register, $src1$$Register, $src2$$Address, false);
11145   %}
11146   ins_pipe(ialu_reg_mem);
11147 %}
11148 
11149 // Or Memory with Register
11150 instruct orL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
11151 %{
11152   match(Set dst (StoreL dst (OrL (LoadL dst) src)));
11153   effect(KILL cr);
11154   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11155 
11156   ins_cost(150);
11157   format %{ "orq     $dst, $src\t# long" %}
11158   ins_encode %{
11159     __ orq($dst$$Address, $src$$Register);
11160   %}
11161   ins_pipe(ialu_mem_reg);
11162 %}
11163 
11164 // Or Memory with Immediate
11165 instruct orL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
11166 %{
11167   match(Set dst (StoreL dst (OrL (LoadL dst) src)));
11168   effect(KILL cr);
11169   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11170 
11171   ins_cost(125);
11172   format %{ "orq     $dst, $src\t# long" %}
11173   ins_encode %{
11174     __ orq($dst$$Address, $src$$constant);
11175   %}
11176   ins_pipe(ialu_mem_imm);
11177 %}
11178 
11179 instruct btsL_mem_imm(memory dst, immL_Pow2 con, rFlagsReg cr)
11180 %{
11181   // con should be a pure 64-bit power of 2 immediate
11182   // because AND/OR works well enough for 8/32-bit values.
11183   predicate(log2i_graceful(n->in(3)->in(2)->get_long()) > 31);
11184 
11185   match(Set dst (StoreL dst (OrL (LoadL dst) con)));
11186   effect(KILL cr);
11187 
11188   ins_cost(125);
11189   format %{ "btsq    $dst, log2($con)\t# long" %}
11190   ins_encode %{
11191     __ btsq($dst$$Address, log2i_exact((julong)$con$$constant));
11192   %}
11193   ins_pipe(ialu_mem_imm);
11194 %}
11195 
11196 // Xor Instructions
11197 // Xor Register with Register
11198 instruct xorL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
11199 %{
11200   predicate(!UseAPX);
11201   match(Set dst (XorL dst src));
11202   effect(KILL cr);
11203   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11204 
11205   format %{ "xorq    $dst, $src\t# long" %}
11206   ins_encode %{
11207     __ xorq($dst$$Register, $src$$Register);
11208   %}
11209   ins_pipe(ialu_reg_reg);
11210 %}
11211 
11212 // Xor Register with Register using New Data Destination (NDD)
11213 instruct xorL_rReg_ndd(rRegL dst, rRegL src1, rRegL src2, rFlagsReg cr)
11214 %{
11215   predicate(UseAPX);
11216   match(Set dst (XorL src1 src2));
11217   effect(KILL cr);
11218   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11219 
11220   format %{ "exorq    $dst, $src1, $src2\t# long ndd" %}
11221   ins_encode %{
11222     __ exorq($dst$$Register, $src1$$Register, $src2$$Register, false);
11223   %}
11224   ins_pipe(ialu_reg_reg);
11225 %}
11226 
11227 // Xor Register with Immediate -1
11228 instruct xorL_rReg_im1(rRegL dst, immL_M1 imm)
11229 %{
11230   predicate(!UseAPX);
11231   match(Set dst (XorL dst imm));
11232 
11233   format %{ "notq   $dst" %}
11234   ins_encode %{
11235      __ notq($dst$$Register);
11236   %}
11237   ins_pipe(ialu_reg);
11238 %}
11239 
11240 instruct xorL_rReg_im1_ndd(rRegL dst,rRegL src, immL_M1 imm)
11241 %{
11242   predicate(UseAPX);
11243   match(Set dst (XorL src imm));
11244 
11245   format %{ "enotq   $dst, $src" %}
11246   ins_encode %{
11247     __ enotq($dst$$Register, $src$$Register);
11248   %}
11249   ins_pipe(ialu_reg);
11250 %}
11251 
11252 // Xor Register with Immediate
11253 instruct xorL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
11254 %{
11255   // Strict predicate check to make selection of xorL_rReg_im1 cost agnostic if immL32 src is -1.
11256   predicate(!UseAPX && n->in(2)->bottom_type()->is_long()->get_con() != -1L);
11257   match(Set dst (XorL dst src));
11258   effect(KILL cr);
11259   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11260 
11261   format %{ "xorq    $dst, $src\t# long" %}
11262   ins_encode %{
11263     __ xorq($dst$$Register, $src$$constant);
11264   %}
11265   ins_pipe(ialu_reg);
11266 %}
11267 
11268 instruct xorL_rReg_rReg_imm(rRegL dst, rRegL src1, immL32 src2, rFlagsReg cr)
11269 %{
11270   // Strict predicate check to make selection of xorL_rReg_im1_ndd cost agnostic if immL32 src2 is -1.
11271   predicate(UseAPX && n->in(2)->bottom_type()->is_long()->get_con() != -1L);
11272   match(Set dst (XorL src1 src2));
11273   effect(KILL cr);
11274   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11275 
11276   format %{ "exorq    $dst, $src1, $src2\t# long ndd" %}
11277   ins_encode %{
11278     __ exorq($dst$$Register, $src1$$Register, $src2$$constant, false);
11279   %}
11280   ins_pipe(ialu_reg);
11281 %}
11282 
11283 // Xor Memory with Immediate
11284 instruct xorL_rReg_mem_imm(rRegL dst, memory src1, immL32 src2, rFlagsReg cr)
11285 %{
11286   predicate(UseAPX);
11287   match(Set dst (XorL (LoadL src1) src2));
11288   effect(KILL cr);
11289   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11290   ins_cost(150);
11291 
11292   format %{ "exorq    $dst, $src1, $src2\t# long ndd" %}
11293   ins_encode %{
11294     __ exorq($dst$$Register, $src1$$Address, $src2$$constant, false);
11295   %}
11296   ins_pipe(ialu_reg);
11297 %}
11298 
11299 // Xor Register with Memory
11300 instruct xorL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
11301 %{
11302   predicate(!UseAPX);
11303   match(Set dst (XorL dst (LoadL src)));
11304   effect(KILL cr);
11305   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11306 
11307   ins_cost(150);
11308   format %{ "xorq    $dst, $src\t# long" %}
11309   ins_encode %{
11310     __ xorq($dst$$Register, $src$$Address);
11311   %}
11312   ins_pipe(ialu_reg_mem);
11313 %}
11314 
11315 instruct xorL_rReg_rReg_mem_ndd(rRegL dst, rRegL src1, memory src2, rFlagsReg cr)
11316 %{
11317   predicate(UseAPX);
11318   match(Set dst (XorL src1 (LoadL src2)));
11319   effect(KILL cr);
11320   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11321 
11322   ins_cost(150);
11323   format %{ "exorq    $dst, $src1, $src2\t# long ndd" %}
11324   ins_encode %{
11325     __ exorq($dst$$Register, $src1$$Register, $src2$$Address, false);
11326   %}
11327   ins_pipe(ialu_reg_mem);
11328 %}
11329 
11330 // Xor Memory with Register
11331 instruct xorL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
11332 %{
11333   match(Set dst (StoreL dst (XorL (LoadL dst) src)));
11334   effect(KILL cr);
11335   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11336 
11337   ins_cost(150);
11338   format %{ "xorq    $dst, $src\t# long" %}
11339   ins_encode %{
11340     __ xorq($dst$$Address, $src$$Register);
11341   %}
11342   ins_pipe(ialu_mem_reg);
11343 %}
11344 
11345 // Xor Memory with Immediate
11346 instruct xorL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
11347 %{
11348   match(Set dst (StoreL dst (XorL (LoadL dst) src)));
11349   effect(KILL cr);
11350   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11351 
11352   ins_cost(125);
11353   format %{ "xorq    $dst, $src\t# long" %}
11354   ins_encode %{
11355     __ xorq($dst$$Address, $src$$constant);
11356   %}
11357   ins_pipe(ialu_mem_imm);
11358 %}
11359 
11360 instruct cmpLTMask(rRegI dst, rRegI p, rRegI q, rFlagsReg cr)
11361 %{
11362   match(Set dst (CmpLTMask p q));
11363   effect(KILL cr);
11364 
11365   ins_cost(400);
11366   format %{ "cmpl    $p, $q\t# cmpLTMask\n\t"
11367             "setcc   $dst \t# emits setlt + movzbl or setzul for APX"
11368             "negl    $dst" %}
11369   ins_encode %{
11370     __ cmpl($p$$Register, $q$$Register);
11371     __ setcc(Assembler::less, $dst$$Register);
11372     __ negl($dst$$Register);
11373   %}
11374   ins_pipe(pipe_slow);
11375 %}
11376 
11377 instruct cmpLTMask0(rRegI dst, immI_0 zero, rFlagsReg cr)
11378 %{
11379   match(Set dst (CmpLTMask dst zero));
11380   effect(KILL cr);
11381 
11382   ins_cost(100);
11383   format %{ "sarl    $dst, #31\t# cmpLTMask0" %}
11384   ins_encode %{
11385     __ sarl($dst$$Register, 31);
11386   %}
11387   ins_pipe(ialu_reg);
11388 %}
11389 
11390 /* Better to save a register than avoid a branch */
11391 instruct cadd_cmpLTMask(rRegI p, rRegI q, rRegI y, rFlagsReg cr)
11392 %{
11393   match(Set p (AddI (AndI (CmpLTMask p q) y) (SubI p q)));
11394   effect(KILL cr);
11395   ins_cost(300);
11396   format %{ "subl    $p,$q\t# cadd_cmpLTMask\n\t"
11397             "jge     done\n\t"
11398             "addl    $p,$y\n"
11399             "done:   " %}
11400   ins_encode %{
11401     Register Rp = $p$$Register;
11402     Register Rq = $q$$Register;
11403     Register Ry = $y$$Register;
11404     Label done;
11405     __ subl(Rp, Rq);
11406     __ jccb(Assembler::greaterEqual, done);
11407     __ addl(Rp, Ry);
11408     __ bind(done);
11409   %}
11410   ins_pipe(pipe_cmplt);
11411 %}
11412 
11413 /* Better to save a register than avoid a branch */
11414 instruct and_cmpLTMask(rRegI p, rRegI q, rRegI y, rFlagsReg cr)
11415 %{
11416   match(Set y (AndI (CmpLTMask p q) y));
11417   effect(KILL cr);
11418 
11419   ins_cost(300);
11420 
11421   format %{ "cmpl    $p, $q\t# and_cmpLTMask\n\t"
11422             "jlt     done\n\t"
11423             "xorl    $y, $y\n"
11424             "done:   " %}
11425   ins_encode %{
11426     Register Rp = $p$$Register;
11427     Register Rq = $q$$Register;
11428     Register Ry = $y$$Register;
11429     Label done;
11430     __ cmpl(Rp, Rq);
11431     __ jccb(Assembler::less, done);
11432     __ xorl(Ry, Ry);
11433     __ bind(done);
11434   %}
11435   ins_pipe(pipe_cmplt);
11436 %}
11437 
11438 
11439 //---------- FP Instructions------------------------------------------------
11440 
11441 // Really expensive, avoid
11442 instruct cmpF_cc_reg(rFlagsRegU cr, regF src1, regF src2)
11443 %{
11444   match(Set cr (CmpF src1 src2));
11445 
11446   ins_cost(500);
11447   format %{ "ucomiss $src1, $src2\n\t"
11448             "jnp,s   exit\n\t"
11449             "pushfq\t# saw NaN, set CF\n\t"
11450             "andq    [rsp], #0xffffff2b\n\t"
11451             "popfq\n"
11452     "exit:" %}
11453   ins_encode %{
11454     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
11455     emit_cmpfp_fixup(masm);
11456   %}
11457   ins_pipe(pipe_slow);
11458 %}
11459 
11460 instruct cmpF_cc_reg_CF(rFlagsRegUCF cr, regF src1, regF src2) %{
11461   match(Set cr (CmpF src1 src2));
11462 
11463   ins_cost(100);
11464   format %{ "ucomiss $src1, $src2" %}
11465   ins_encode %{
11466     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
11467   %}
11468   ins_pipe(pipe_slow);
11469 %}
11470 
11471 instruct cmpF_cc_memCF(rFlagsRegUCF cr, regF src1, memory src2) %{
11472   match(Set cr (CmpF src1 (LoadF src2)));
11473 
11474   ins_cost(100);
11475   format %{ "ucomiss $src1, $src2" %}
11476   ins_encode %{
11477     __ ucomiss($src1$$XMMRegister, $src2$$Address);
11478   %}
11479   ins_pipe(pipe_slow);
11480 %}
11481 
11482 instruct cmpF_cc_immCF(rFlagsRegUCF cr, regF src, immF con) %{
11483   match(Set cr (CmpF src con));
11484   ins_cost(100);
11485   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con" %}
11486   ins_encode %{
11487     __ ucomiss($src$$XMMRegister, $constantaddress($con));
11488   %}
11489   ins_pipe(pipe_slow);
11490 %}
11491 
11492 // Really expensive, avoid
11493 instruct cmpD_cc_reg(rFlagsRegU cr, regD src1, regD src2)
11494 %{
11495   match(Set cr (CmpD src1 src2));
11496 
11497   ins_cost(500);
11498   format %{ "ucomisd $src1, $src2\n\t"
11499             "jnp,s   exit\n\t"
11500             "pushfq\t# saw NaN, set CF\n\t"
11501             "andq    [rsp], #0xffffff2b\n\t"
11502             "popfq\n"
11503     "exit:" %}
11504   ins_encode %{
11505     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
11506     emit_cmpfp_fixup(masm);
11507   %}
11508   ins_pipe(pipe_slow);
11509 %}
11510 
11511 instruct cmpD_cc_reg_CF(rFlagsRegUCF cr, regD src1, regD src2) %{
11512   match(Set cr (CmpD src1 src2));
11513 
11514   ins_cost(100);
11515   format %{ "ucomisd $src1, $src2 test" %}
11516   ins_encode %{
11517     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
11518   %}
11519   ins_pipe(pipe_slow);
11520 %}
11521 
11522 instruct cmpD_cc_memCF(rFlagsRegUCF cr, regD src1, memory src2) %{
11523   match(Set cr (CmpD src1 (LoadD src2)));
11524 
11525   ins_cost(100);
11526   format %{ "ucomisd $src1, $src2" %}
11527   ins_encode %{
11528     __ ucomisd($src1$$XMMRegister, $src2$$Address);
11529   %}
11530   ins_pipe(pipe_slow);
11531 %}
11532 
11533 instruct cmpD_cc_immCF(rFlagsRegUCF cr, regD src, immD con) %{
11534   match(Set cr (CmpD src con));
11535   ins_cost(100);
11536   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con" %}
11537   ins_encode %{
11538     __ ucomisd($src$$XMMRegister, $constantaddress($con));
11539   %}
11540   ins_pipe(pipe_slow);
11541 %}
11542 
11543 // Compare into -1,0,1
11544 instruct cmpF_reg(rRegI dst, regF src1, regF src2, rFlagsReg cr)
11545 %{
11546   match(Set dst (CmpF3 src1 src2));
11547   effect(KILL cr);
11548 
11549   ins_cost(275);
11550   format %{ "ucomiss $src1, $src2\n\t"
11551             "movl    $dst, #-1\n\t"
11552             "jp,s    done\n\t"
11553             "jb,s    done\n\t"
11554             "setne   $dst\n\t"
11555             "movzbl  $dst, $dst\n"
11556     "done:" %}
11557   ins_encode %{
11558     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
11559     emit_cmpfp3(masm, $dst$$Register);
11560   %}
11561   ins_pipe(pipe_slow);
11562 %}
11563 
11564 // Compare into -1,0,1
11565 instruct cmpF_mem(rRegI dst, regF src1, memory src2, rFlagsReg cr)
11566 %{
11567   match(Set dst (CmpF3 src1 (LoadF src2)));
11568   effect(KILL cr);
11569 
11570   ins_cost(275);
11571   format %{ "ucomiss $src1, $src2\n\t"
11572             "movl    $dst, #-1\n\t"
11573             "jp,s    done\n\t"
11574             "jb,s    done\n\t"
11575             "setne   $dst\n\t"
11576             "movzbl  $dst, $dst\n"
11577     "done:" %}
11578   ins_encode %{
11579     __ ucomiss($src1$$XMMRegister, $src2$$Address);
11580     emit_cmpfp3(masm, $dst$$Register);
11581   %}
11582   ins_pipe(pipe_slow);
11583 %}
11584 
11585 // Compare into -1,0,1
11586 instruct cmpF_imm(rRegI dst, regF src, immF con, rFlagsReg cr) %{
11587   match(Set dst (CmpF3 src con));
11588   effect(KILL cr);
11589 
11590   ins_cost(275);
11591   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con\n\t"
11592             "movl    $dst, #-1\n\t"
11593             "jp,s    done\n\t"
11594             "jb,s    done\n\t"
11595             "setne   $dst\n\t"
11596             "movzbl  $dst, $dst\n"
11597     "done:" %}
11598   ins_encode %{
11599     __ ucomiss($src$$XMMRegister, $constantaddress($con));
11600     emit_cmpfp3(masm, $dst$$Register);
11601   %}
11602   ins_pipe(pipe_slow);
11603 %}
11604 
11605 // Compare into -1,0,1
11606 instruct cmpD_reg(rRegI dst, regD src1, regD src2, rFlagsReg cr)
11607 %{
11608   match(Set dst (CmpD3 src1 src2));
11609   effect(KILL cr);
11610 
11611   ins_cost(275);
11612   format %{ "ucomisd $src1, $src2\n\t"
11613             "movl    $dst, #-1\n\t"
11614             "jp,s    done\n\t"
11615             "jb,s    done\n\t"
11616             "setne   $dst\n\t"
11617             "movzbl  $dst, $dst\n"
11618     "done:" %}
11619   ins_encode %{
11620     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
11621     emit_cmpfp3(masm, $dst$$Register);
11622   %}
11623   ins_pipe(pipe_slow);
11624 %}
11625 
11626 // Compare into -1,0,1
11627 instruct cmpD_mem(rRegI dst, regD src1, memory src2, rFlagsReg cr)
11628 %{
11629   match(Set dst (CmpD3 src1 (LoadD src2)));
11630   effect(KILL cr);
11631 
11632   ins_cost(275);
11633   format %{ "ucomisd $src1, $src2\n\t"
11634             "movl    $dst, #-1\n\t"
11635             "jp,s    done\n\t"
11636             "jb,s    done\n\t"
11637             "setne   $dst\n\t"
11638             "movzbl  $dst, $dst\n"
11639     "done:" %}
11640   ins_encode %{
11641     __ ucomisd($src1$$XMMRegister, $src2$$Address);
11642     emit_cmpfp3(masm, $dst$$Register);
11643   %}
11644   ins_pipe(pipe_slow);
11645 %}
11646 
11647 // Compare into -1,0,1
11648 instruct cmpD_imm(rRegI dst, regD src, immD con, rFlagsReg cr) %{
11649   match(Set dst (CmpD3 src con));
11650   effect(KILL cr);
11651 
11652   ins_cost(275);
11653   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con\n\t"
11654             "movl    $dst, #-1\n\t"
11655             "jp,s    done\n\t"
11656             "jb,s    done\n\t"
11657             "setne   $dst\n\t"
11658             "movzbl  $dst, $dst\n"
11659     "done:" %}
11660   ins_encode %{
11661     __ ucomisd($src$$XMMRegister, $constantaddress($con));
11662     emit_cmpfp3(masm, $dst$$Register);
11663   %}
11664   ins_pipe(pipe_slow);
11665 %}
11666 
11667 //----------Arithmetic Conversion Instructions---------------------------------
11668 
11669 instruct convF2D_reg_reg(regD dst, regF src)
11670 %{
11671   match(Set dst (ConvF2D src));
11672 
11673   format %{ "cvtss2sd $dst, $src" %}
11674   ins_encode %{
11675     __ cvtss2sd ($dst$$XMMRegister, $src$$XMMRegister);
11676   %}
11677   ins_pipe(pipe_slow); // XXX
11678 %}
11679 
11680 instruct convF2D_reg_mem(regD dst, memory src)
11681 %{
11682   predicate(UseAVX == 0);
11683   match(Set dst (ConvF2D (LoadF src)));
11684 
11685   format %{ "cvtss2sd $dst, $src" %}
11686   ins_encode %{
11687     __ cvtss2sd ($dst$$XMMRegister, $src$$Address);
11688   %}
11689   ins_pipe(pipe_slow); // XXX
11690 %}
11691 
11692 instruct convD2F_reg_reg(regF dst, regD src)
11693 %{
11694   match(Set dst (ConvD2F src));
11695 
11696   format %{ "cvtsd2ss $dst, $src" %}
11697   ins_encode %{
11698     __ cvtsd2ss ($dst$$XMMRegister, $src$$XMMRegister);
11699   %}
11700   ins_pipe(pipe_slow); // XXX
11701 %}
11702 
11703 instruct convD2F_reg_mem(regF dst, memory src)
11704 %{
11705   predicate(UseAVX == 0);
11706   match(Set dst (ConvD2F (LoadD src)));
11707 
11708   format %{ "cvtsd2ss $dst, $src" %}
11709   ins_encode %{
11710     __ cvtsd2ss ($dst$$XMMRegister, $src$$Address);
11711   %}
11712   ins_pipe(pipe_slow); // XXX
11713 %}
11714 
11715 // XXX do mem variants
11716 instruct convF2I_reg_reg(rRegI dst, regF src, rFlagsReg cr)
11717 %{
11718   match(Set dst (ConvF2I src));
11719   effect(KILL cr);
11720   format %{ "convert_f2i $dst, $src" %}
11721   ins_encode %{
11722     __ convertF2I(T_INT, T_FLOAT, $dst$$Register, $src$$XMMRegister);
11723   %}
11724   ins_pipe(pipe_slow);
11725 %}
11726 
11727 instruct convF2L_reg_reg(rRegL dst, regF src, rFlagsReg cr)
11728 %{
11729   match(Set dst (ConvF2L src));
11730   effect(KILL cr);
11731   format %{ "convert_f2l $dst, $src"%}
11732   ins_encode %{
11733     __ convertF2I(T_LONG, T_FLOAT, $dst$$Register, $src$$XMMRegister);
11734   %}
11735   ins_pipe(pipe_slow);
11736 %}
11737 
11738 instruct convD2I_reg_reg(rRegI dst, regD src, rFlagsReg cr)
11739 %{
11740   match(Set dst (ConvD2I src));
11741   effect(KILL cr);
11742   format %{ "convert_d2i $dst, $src"%}
11743   ins_encode %{
11744     __ convertF2I(T_INT, T_DOUBLE, $dst$$Register, $src$$XMMRegister);
11745   %}
11746   ins_pipe(pipe_slow);
11747 %}
11748 
11749 instruct convD2L_reg_reg(rRegL dst, regD src, rFlagsReg cr)
11750 %{
11751   match(Set dst (ConvD2L src));
11752   effect(KILL cr);
11753   format %{ "convert_d2l $dst, $src"%}
11754   ins_encode %{
11755     __ convertF2I(T_LONG, T_DOUBLE, $dst$$Register, $src$$XMMRegister);
11756   %}
11757   ins_pipe(pipe_slow);
11758 %}
11759 
11760 instruct round_double_reg(rRegL dst, regD src, rRegL rtmp, rcx_RegL rcx, rFlagsReg cr)
11761 %{
11762   match(Set dst (RoundD src));
11763   effect(TEMP dst, TEMP rtmp, TEMP rcx, KILL cr);
11764   format %{ "round_double $dst,$src \t! using $rtmp and $rcx as TEMP"%}
11765   ins_encode %{
11766     __ round_double($dst$$Register, $src$$XMMRegister, $rtmp$$Register, $rcx$$Register);
11767   %}
11768   ins_pipe(pipe_slow);
11769 %}
11770 
11771 instruct round_float_reg(rRegI dst, regF src, rRegL rtmp, rcx_RegL rcx, rFlagsReg cr)
11772 %{
11773   match(Set dst (RoundF src));
11774   effect(TEMP dst, TEMP rtmp, TEMP rcx, KILL cr);
11775   format %{ "round_float $dst,$src" %}
11776   ins_encode %{
11777     __ round_float($dst$$Register, $src$$XMMRegister, $rtmp$$Register, $rcx$$Register);
11778   %}
11779   ins_pipe(pipe_slow);
11780 %}
11781 
11782 instruct convI2F_reg_reg(vlRegF dst, rRegI src)
11783 %{
11784   predicate(!UseXmmI2F);
11785   match(Set dst (ConvI2F src));
11786 
11787   format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
11788   ins_encode %{
11789     if (UseAVX > 0) {
11790       __ pxor($dst$$XMMRegister, $dst$$XMMRegister);
11791     }
11792     __ cvtsi2ssl ($dst$$XMMRegister, $src$$Register);
11793   %}
11794   ins_pipe(pipe_slow); // XXX
11795 %}
11796 
11797 instruct convI2F_reg_mem(regF dst, memory src)
11798 %{
11799   predicate(UseAVX == 0);
11800   match(Set dst (ConvI2F (LoadI src)));
11801 
11802   format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
11803   ins_encode %{
11804     __ cvtsi2ssl ($dst$$XMMRegister, $src$$Address);
11805   %}
11806   ins_pipe(pipe_slow); // XXX
11807 %}
11808 
11809 instruct convI2D_reg_reg(vlRegD dst, rRegI src)
11810 %{
11811   predicate(!UseXmmI2D);
11812   match(Set dst (ConvI2D src));
11813 
11814   format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
11815   ins_encode %{
11816     if (UseAVX > 0) {
11817       __ pxor($dst$$XMMRegister, $dst$$XMMRegister);
11818     }
11819     __ cvtsi2sdl ($dst$$XMMRegister, $src$$Register);
11820   %}
11821   ins_pipe(pipe_slow); // XXX
11822 %}
11823 
11824 instruct convI2D_reg_mem(regD dst, memory src)
11825 %{
11826   predicate(UseAVX == 0);
11827   match(Set dst (ConvI2D (LoadI src)));
11828 
11829   format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
11830   ins_encode %{
11831     __ cvtsi2sdl ($dst$$XMMRegister, $src$$Address);
11832   %}
11833   ins_pipe(pipe_slow); // XXX
11834 %}
11835 
11836 instruct convXI2F_reg(regF dst, rRegI src)
11837 %{
11838   predicate(UseXmmI2F);
11839   match(Set dst (ConvI2F src));
11840 
11841   format %{ "movdl $dst, $src\n\t"
11842             "cvtdq2psl $dst, $dst\t# i2f" %}
11843   ins_encode %{
11844     __ movdl($dst$$XMMRegister, $src$$Register);
11845     __ cvtdq2ps($dst$$XMMRegister, $dst$$XMMRegister);
11846   %}
11847   ins_pipe(pipe_slow); // XXX
11848 %}
11849 
11850 instruct convXI2D_reg(regD dst, rRegI src)
11851 %{
11852   predicate(UseXmmI2D);
11853   match(Set dst (ConvI2D src));
11854 
11855   format %{ "movdl $dst, $src\n\t"
11856             "cvtdq2pdl $dst, $dst\t# i2d" %}
11857   ins_encode %{
11858     __ movdl($dst$$XMMRegister, $src$$Register);
11859     __ cvtdq2pd($dst$$XMMRegister, $dst$$XMMRegister);
11860   %}
11861   ins_pipe(pipe_slow); // XXX
11862 %}
11863 
11864 instruct convL2F_reg_reg(vlRegF dst, rRegL src)
11865 %{
11866   match(Set dst (ConvL2F src));
11867 
11868   format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
11869   ins_encode %{
11870     if (UseAVX > 0) {
11871       __ pxor($dst$$XMMRegister, $dst$$XMMRegister);
11872     }
11873     __ cvtsi2ssq ($dst$$XMMRegister, $src$$Register);
11874   %}
11875   ins_pipe(pipe_slow); // XXX
11876 %}
11877 
11878 instruct convL2F_reg_mem(regF dst, memory src)
11879 %{
11880   predicate(UseAVX == 0);
11881   match(Set dst (ConvL2F (LoadL src)));
11882 
11883   format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
11884   ins_encode %{
11885     __ cvtsi2ssq ($dst$$XMMRegister, $src$$Address);
11886   %}
11887   ins_pipe(pipe_slow); // XXX
11888 %}
11889 
11890 instruct convL2D_reg_reg(vlRegD dst, rRegL src)
11891 %{
11892   match(Set dst (ConvL2D src));
11893 
11894   format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
11895   ins_encode %{
11896     if (UseAVX > 0) {
11897       __ pxor($dst$$XMMRegister, $dst$$XMMRegister);
11898     }
11899     __ cvtsi2sdq ($dst$$XMMRegister, $src$$Register);
11900   %}
11901   ins_pipe(pipe_slow); // XXX
11902 %}
11903 
11904 instruct convL2D_reg_mem(regD dst, memory src)
11905 %{
11906   predicate(UseAVX == 0);
11907   match(Set dst (ConvL2D (LoadL src)));
11908 
11909   format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
11910   ins_encode %{
11911     __ cvtsi2sdq ($dst$$XMMRegister, $src$$Address);
11912   %}
11913   ins_pipe(pipe_slow); // XXX
11914 %}
11915 
11916 instruct convI2L_reg_reg(rRegL dst, rRegI src)
11917 %{
11918   match(Set dst (ConvI2L src));
11919 
11920   ins_cost(125);
11921   format %{ "movslq  $dst, $src\t# i2l" %}
11922   ins_encode %{
11923     __ movslq($dst$$Register, $src$$Register);
11924   %}
11925   ins_pipe(ialu_reg_reg);
11926 %}
11927 
11928 // Zero-extend convert int to long
11929 instruct convI2L_reg_reg_zex(rRegL dst, rRegI src, immL_32bits mask)
11930 %{
11931   match(Set dst (AndL (ConvI2L src) mask));
11932 
11933   format %{ "movl    $dst, $src\t# i2l zero-extend\n\t" %}
11934   ins_encode %{
11935     if ($dst$$reg != $src$$reg) {
11936       __ movl($dst$$Register, $src$$Register);
11937     }
11938   %}
11939   ins_pipe(ialu_reg_reg);
11940 %}
11941 
11942 // Zero-extend convert int to long
11943 instruct convI2L_reg_mem_zex(rRegL dst, memory src, immL_32bits mask)
11944 %{
11945   match(Set dst (AndL (ConvI2L (LoadI src)) mask));
11946 
11947   format %{ "movl    $dst, $src\t# i2l zero-extend\n\t" %}
11948   ins_encode %{
11949     __ movl($dst$$Register, $src$$Address);
11950   %}
11951   ins_pipe(ialu_reg_mem);
11952 %}
11953 
11954 instruct zerox_long_reg_reg(rRegL dst, rRegL src, immL_32bits mask)
11955 %{
11956   match(Set dst (AndL src mask));
11957 
11958   format %{ "movl    $dst, $src\t# zero-extend long" %}
11959   ins_encode %{
11960     __ movl($dst$$Register, $src$$Register);
11961   %}
11962   ins_pipe(ialu_reg_reg);
11963 %}
11964 
11965 instruct convL2I_reg_reg(rRegI dst, rRegL src)
11966 %{
11967   match(Set dst (ConvL2I src));
11968 
11969   format %{ "movl    $dst, $src\t# l2i" %}
11970   ins_encode %{
11971     __ movl($dst$$Register, $src$$Register);
11972   %}
11973   ins_pipe(ialu_reg_reg);
11974 %}
11975 
11976 
11977 instruct MoveF2I_stack_reg(rRegI dst, stackSlotF src) %{
11978   match(Set dst (MoveF2I src));
11979   effect(DEF dst, USE src);
11980 
11981   ins_cost(125);
11982   format %{ "movl    $dst, $src\t# MoveF2I_stack_reg" %}
11983   ins_encode %{
11984     __ movl($dst$$Register, Address(rsp, $src$$disp));
11985   %}
11986   ins_pipe(ialu_reg_mem);
11987 %}
11988 
11989 instruct MoveI2F_stack_reg(regF dst, stackSlotI src) %{
11990   match(Set dst (MoveI2F src));
11991   effect(DEF dst, USE src);
11992 
11993   ins_cost(125);
11994   format %{ "movss   $dst, $src\t# MoveI2F_stack_reg" %}
11995   ins_encode %{
11996     __ movflt($dst$$XMMRegister, Address(rsp, $src$$disp));
11997   %}
11998   ins_pipe(pipe_slow);
11999 %}
12000 
12001 instruct MoveD2L_stack_reg(rRegL dst, stackSlotD src) %{
12002   match(Set dst (MoveD2L src));
12003   effect(DEF dst, USE src);
12004 
12005   ins_cost(125);
12006   format %{ "movq    $dst, $src\t# MoveD2L_stack_reg" %}
12007   ins_encode %{
12008     __ movq($dst$$Register, Address(rsp, $src$$disp));
12009   %}
12010   ins_pipe(ialu_reg_mem);
12011 %}
12012 
12013 instruct MoveL2D_stack_reg_partial(regD dst, stackSlotL src) %{
12014   predicate(!UseXmmLoadAndClearUpper);
12015   match(Set dst (MoveL2D src));
12016   effect(DEF dst, USE src);
12017 
12018   ins_cost(125);
12019   format %{ "movlpd  $dst, $src\t# MoveL2D_stack_reg" %}
12020   ins_encode %{
12021     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
12022   %}
12023   ins_pipe(pipe_slow);
12024 %}
12025 
12026 instruct MoveL2D_stack_reg(regD dst, stackSlotL src) %{
12027   predicate(UseXmmLoadAndClearUpper);
12028   match(Set dst (MoveL2D src));
12029   effect(DEF dst, USE src);
12030 
12031   ins_cost(125);
12032   format %{ "movsd   $dst, $src\t# MoveL2D_stack_reg" %}
12033   ins_encode %{
12034     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
12035   %}
12036   ins_pipe(pipe_slow);
12037 %}
12038 
12039 
12040 instruct MoveF2I_reg_stack(stackSlotI dst, regF src) %{
12041   match(Set dst (MoveF2I src));
12042   effect(DEF dst, USE src);
12043 
12044   ins_cost(95); // XXX
12045   format %{ "movss   $dst, $src\t# MoveF2I_reg_stack" %}
12046   ins_encode %{
12047     __ movflt(Address(rsp, $dst$$disp), $src$$XMMRegister);
12048   %}
12049   ins_pipe(pipe_slow);
12050 %}
12051 
12052 instruct MoveI2F_reg_stack(stackSlotF dst, rRegI src) %{
12053   match(Set dst (MoveI2F src));
12054   effect(DEF dst, USE src);
12055 
12056   ins_cost(100);
12057   format %{ "movl    $dst, $src\t# MoveI2F_reg_stack" %}
12058   ins_encode %{
12059     __ movl(Address(rsp, $dst$$disp), $src$$Register);
12060   %}
12061   ins_pipe( ialu_mem_reg );
12062 %}
12063 
12064 instruct MoveD2L_reg_stack(stackSlotL dst, regD src) %{
12065   match(Set dst (MoveD2L src));
12066   effect(DEF dst, USE src);
12067 
12068   ins_cost(95); // XXX
12069   format %{ "movsd   $dst, $src\t# MoveL2D_reg_stack" %}
12070   ins_encode %{
12071     __ movdbl(Address(rsp, $dst$$disp), $src$$XMMRegister);
12072   %}
12073   ins_pipe(pipe_slow);
12074 %}
12075 
12076 instruct MoveL2D_reg_stack(stackSlotD dst, rRegL src) %{
12077   match(Set dst (MoveL2D src));
12078   effect(DEF dst, USE src);
12079 
12080   ins_cost(100);
12081   format %{ "movq    $dst, $src\t# MoveL2D_reg_stack" %}
12082   ins_encode %{
12083     __ movq(Address(rsp, $dst$$disp), $src$$Register);
12084   %}
12085   ins_pipe(ialu_mem_reg);
12086 %}
12087 
12088 instruct MoveF2I_reg_reg(rRegI dst, regF src) %{
12089   match(Set dst (MoveF2I src));
12090   effect(DEF dst, USE src);
12091   ins_cost(85);
12092   format %{ "movd    $dst,$src\t# MoveF2I" %}
12093   ins_encode %{
12094     __ movdl($dst$$Register, $src$$XMMRegister);
12095   %}
12096   ins_pipe( pipe_slow );
12097 %}
12098 
12099 instruct MoveD2L_reg_reg(rRegL dst, regD src) %{
12100   match(Set dst (MoveD2L src));
12101   effect(DEF dst, USE src);
12102   ins_cost(85);
12103   format %{ "movd    $dst,$src\t# MoveD2L" %}
12104   ins_encode %{
12105     __ movdq($dst$$Register, $src$$XMMRegister);
12106   %}
12107   ins_pipe( pipe_slow );
12108 %}
12109 
12110 instruct MoveI2F_reg_reg(regF dst, rRegI src) %{
12111   match(Set dst (MoveI2F src));
12112   effect(DEF dst, USE src);
12113   ins_cost(100);
12114   format %{ "movd    $dst,$src\t# MoveI2F" %}
12115   ins_encode %{
12116     __ movdl($dst$$XMMRegister, $src$$Register);
12117   %}
12118   ins_pipe( pipe_slow );
12119 %}
12120 
12121 instruct MoveL2D_reg_reg(regD dst, rRegL src) %{
12122   match(Set dst (MoveL2D src));
12123   effect(DEF dst, USE src);
12124   ins_cost(100);
12125   format %{ "movd    $dst,$src\t# MoveL2D" %}
12126   ins_encode %{
12127      __ movdq($dst$$XMMRegister, $src$$Register);
12128   %}
12129   ins_pipe( pipe_slow );
12130 %}
12131 
12132 // Fast clearing of an array
12133 // Small non-constant lenght ClearArray for non-AVX512 targets.
12134 instruct rep_stos(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegI zero,
12135                   Universe dummy, rFlagsReg cr)
12136 %{
12137   predicate(!((ClearArrayNode*)n)->is_large() && (UseAVX <= 2));
12138   match(Set dummy (ClearArray cnt base));
12139   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, KILL zero, KILL cr);
12140 
12141   format %{ $$template
12142     $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
12143     $$emit$$"cmp     InitArrayShortSize,rcx\n\t"
12144     $$emit$$"jg      LARGE\n\t"
12145     $$emit$$"dec     rcx\n\t"
12146     $$emit$$"js      DONE\t# Zero length\n\t"
12147     $$emit$$"mov     rax,(rdi,rcx,8)\t# LOOP\n\t"
12148     $$emit$$"dec     rcx\n\t"
12149     $$emit$$"jge     LOOP\n\t"
12150     $$emit$$"jmp     DONE\n\t"
12151     $$emit$$"# LARGE:\n\t"
12152     if (UseFastStosb) {
12153        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
12154        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--\n\t"
12155     } else if (UseXMMForObjInit) {
12156        $$emit$$"mov     rdi,rax\n\t"
12157        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
12158        $$emit$$"jmpq    L_zero_64_bytes\n\t"
12159        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
12160        $$emit$$"vmovdqu ymm0,(rax)\n\t"
12161        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
12162        $$emit$$"add     0x40,rax\n\t"
12163        $$emit$$"# L_zero_64_bytes:\n\t"
12164        $$emit$$"sub     0x8,rcx\n\t"
12165        $$emit$$"jge     L_loop\n\t"
12166        $$emit$$"add     0x4,rcx\n\t"
12167        $$emit$$"jl      L_tail\n\t"
12168        $$emit$$"vmovdqu ymm0,(rax)\n\t"
12169        $$emit$$"add     0x20,rax\n\t"
12170        $$emit$$"sub     0x4,rcx\n\t"
12171        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
12172        $$emit$$"add     0x4,rcx\n\t"
12173        $$emit$$"jle     L_end\n\t"
12174        $$emit$$"dec     rcx\n\t"
12175        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
12176        $$emit$$"vmovq   xmm0,(rax)\n\t"
12177        $$emit$$"add     0x8,rax\n\t"
12178        $$emit$$"dec     rcx\n\t"
12179        $$emit$$"jge     L_sloop\n\t"
12180        $$emit$$"# L_end:\n\t"
12181     } else {
12182        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--\n\t"
12183     }
12184     $$emit$$"# DONE"
12185   %}
12186   ins_encode %{
12187     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
12188                  $tmp$$XMMRegister, false, knoreg);
12189   %}
12190   ins_pipe(pipe_slow);
12191 %}
12192 
12193 // Small non-constant length ClearArray for AVX512 targets.
12194 instruct rep_stos_evex(rcx_RegL cnt, rdi_RegP base, legRegD tmp, kReg ktmp, rax_RegI zero,
12195                        Universe dummy, rFlagsReg cr)
12196 %{
12197   predicate(!((ClearArrayNode*)n)->is_large() && (UseAVX > 2));
12198   match(Set dummy (ClearArray cnt base));
12199   ins_cost(125);
12200   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, TEMP ktmp, KILL zero, KILL cr);
12201 
12202   format %{ $$template
12203     $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
12204     $$emit$$"cmp     InitArrayShortSize,rcx\n\t"
12205     $$emit$$"jg      LARGE\n\t"
12206     $$emit$$"dec     rcx\n\t"
12207     $$emit$$"js      DONE\t# Zero length\n\t"
12208     $$emit$$"mov     rax,(rdi,rcx,8)\t# LOOP\n\t"
12209     $$emit$$"dec     rcx\n\t"
12210     $$emit$$"jge     LOOP\n\t"
12211     $$emit$$"jmp     DONE\n\t"
12212     $$emit$$"# LARGE:\n\t"
12213     if (UseFastStosb) {
12214        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
12215        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--\n\t"
12216     } else if (UseXMMForObjInit) {
12217        $$emit$$"mov     rdi,rax\n\t"
12218        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
12219        $$emit$$"jmpq    L_zero_64_bytes\n\t"
12220        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
12221        $$emit$$"vmovdqu ymm0,(rax)\n\t"
12222        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
12223        $$emit$$"add     0x40,rax\n\t"
12224        $$emit$$"# L_zero_64_bytes:\n\t"
12225        $$emit$$"sub     0x8,rcx\n\t"
12226        $$emit$$"jge     L_loop\n\t"
12227        $$emit$$"add     0x4,rcx\n\t"
12228        $$emit$$"jl      L_tail\n\t"
12229        $$emit$$"vmovdqu ymm0,(rax)\n\t"
12230        $$emit$$"add     0x20,rax\n\t"
12231        $$emit$$"sub     0x4,rcx\n\t"
12232        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
12233        $$emit$$"add     0x4,rcx\n\t"
12234        $$emit$$"jle     L_end\n\t"
12235        $$emit$$"dec     rcx\n\t"
12236        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
12237        $$emit$$"vmovq   xmm0,(rax)\n\t"
12238        $$emit$$"add     0x8,rax\n\t"
12239        $$emit$$"dec     rcx\n\t"
12240        $$emit$$"jge     L_sloop\n\t"
12241        $$emit$$"# L_end:\n\t"
12242     } else {
12243        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--\n\t"
12244     }
12245     $$emit$$"# DONE"
12246   %}
12247   ins_encode %{
12248     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
12249                  $tmp$$XMMRegister, false, $ktmp$$KRegister);
12250   %}
12251   ins_pipe(pipe_slow);
12252 %}
12253 
12254 // Large non-constant length ClearArray for non-AVX512 targets.
12255 instruct rep_stos_large(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegI zero,
12256                         Universe dummy, rFlagsReg cr)
12257 %{
12258   predicate((UseAVX <=2) && ((ClearArrayNode*)n)->is_large());
12259   match(Set dummy (ClearArray cnt base));
12260   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, KILL zero, KILL cr);
12261 
12262   format %{ $$template
12263     if (UseFastStosb) {
12264        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
12265        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
12266        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--"
12267     } else if (UseXMMForObjInit) {
12268        $$emit$$"mov     rdi,rax\t# ClearArray:\n\t"
12269        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
12270        $$emit$$"jmpq    L_zero_64_bytes\n\t"
12271        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
12272        $$emit$$"vmovdqu ymm0,(rax)\n\t"
12273        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
12274        $$emit$$"add     0x40,rax\n\t"
12275        $$emit$$"# L_zero_64_bytes:\n\t"
12276        $$emit$$"sub     0x8,rcx\n\t"
12277        $$emit$$"jge     L_loop\n\t"
12278        $$emit$$"add     0x4,rcx\n\t"
12279        $$emit$$"jl      L_tail\n\t"
12280        $$emit$$"vmovdqu ymm0,(rax)\n\t"
12281        $$emit$$"add     0x20,rax\n\t"
12282        $$emit$$"sub     0x4,rcx\n\t"
12283        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
12284        $$emit$$"add     0x4,rcx\n\t"
12285        $$emit$$"jle     L_end\n\t"
12286        $$emit$$"dec     rcx\n\t"
12287        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
12288        $$emit$$"vmovq   xmm0,(rax)\n\t"
12289        $$emit$$"add     0x8,rax\n\t"
12290        $$emit$$"dec     rcx\n\t"
12291        $$emit$$"jge     L_sloop\n\t"
12292        $$emit$$"# L_end:\n\t"
12293     } else {
12294        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
12295        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--"
12296     }
12297   %}
12298   ins_encode %{
12299     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
12300                  $tmp$$XMMRegister, true, knoreg);
12301   %}
12302   ins_pipe(pipe_slow);
12303 %}
12304 
12305 // Large non-constant length ClearArray for AVX512 targets.
12306 instruct rep_stos_large_evex(rcx_RegL cnt, rdi_RegP base, legRegD tmp, kReg ktmp, rax_RegI zero,
12307                              Universe dummy, rFlagsReg cr)
12308 %{
12309   predicate((UseAVX > 2) && ((ClearArrayNode*)n)->is_large());
12310   match(Set dummy (ClearArray cnt base));
12311   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, TEMP ktmp, KILL zero, KILL cr);
12312 
12313   format %{ $$template
12314     if (UseFastStosb) {
12315        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
12316        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
12317        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--"
12318     } else if (UseXMMForObjInit) {
12319        $$emit$$"mov     rdi,rax\t# ClearArray:\n\t"
12320        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
12321        $$emit$$"jmpq    L_zero_64_bytes\n\t"
12322        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
12323        $$emit$$"vmovdqu ymm0,(rax)\n\t"
12324        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
12325        $$emit$$"add     0x40,rax\n\t"
12326        $$emit$$"# L_zero_64_bytes:\n\t"
12327        $$emit$$"sub     0x8,rcx\n\t"
12328        $$emit$$"jge     L_loop\n\t"
12329        $$emit$$"add     0x4,rcx\n\t"
12330        $$emit$$"jl      L_tail\n\t"
12331        $$emit$$"vmovdqu ymm0,(rax)\n\t"
12332        $$emit$$"add     0x20,rax\n\t"
12333        $$emit$$"sub     0x4,rcx\n\t"
12334        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
12335        $$emit$$"add     0x4,rcx\n\t"
12336        $$emit$$"jle     L_end\n\t"
12337        $$emit$$"dec     rcx\n\t"
12338        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
12339        $$emit$$"vmovq   xmm0,(rax)\n\t"
12340        $$emit$$"add     0x8,rax\n\t"
12341        $$emit$$"dec     rcx\n\t"
12342        $$emit$$"jge     L_sloop\n\t"
12343        $$emit$$"# L_end:\n\t"
12344     } else {
12345        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
12346        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--"
12347     }
12348   %}
12349   ins_encode %{
12350     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
12351                  $tmp$$XMMRegister, true, $ktmp$$KRegister);
12352   %}
12353   ins_pipe(pipe_slow);
12354 %}
12355 
12356 // Small constant length ClearArray for AVX512 targets.
12357 instruct rep_stos_im(immL cnt, rRegP base, regD tmp, rRegI zero, kReg ktmp, Universe dummy, rFlagsReg cr)
12358 %{
12359   predicate(!((ClearArrayNode*)n)->is_large() && (MaxVectorSize >= 32) && VM_Version::supports_avx512vl());
12360   match(Set dummy (ClearArray cnt base));
12361   ins_cost(100);
12362   effect(TEMP tmp, TEMP zero, TEMP ktmp, KILL cr);
12363   format %{ "clear_mem_imm $base , $cnt  \n\t" %}
12364   ins_encode %{
12365    __ clear_mem($base$$Register, $cnt$$constant, $zero$$Register, $tmp$$XMMRegister, $ktmp$$KRegister);
12366   %}
12367   ins_pipe(pipe_slow);
12368 %}
12369 
12370 instruct string_compareL(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
12371                          rax_RegI result, legRegD tmp1, rFlagsReg cr)
12372 %{
12373   predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL);
12374   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
12375   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
12376 
12377   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
12378   ins_encode %{
12379     __ string_compare($str1$$Register, $str2$$Register,
12380                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
12381                       $tmp1$$XMMRegister, StrIntrinsicNode::LL, knoreg);
12382   %}
12383   ins_pipe( pipe_slow );
12384 %}
12385 
12386 instruct string_compareL_evex(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
12387                               rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
12388 %{
12389   predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL);
12390   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
12391   effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
12392 
12393   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
12394   ins_encode %{
12395     __ string_compare($str1$$Register, $str2$$Register,
12396                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
12397                       $tmp1$$XMMRegister, StrIntrinsicNode::LL, $ktmp$$KRegister);
12398   %}
12399   ins_pipe( pipe_slow );
12400 %}
12401 
12402 instruct string_compareU(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
12403                          rax_RegI result, legRegD tmp1, rFlagsReg cr)
12404 %{
12405   predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU);
12406   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
12407   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
12408 
12409   format %{ "String Compare char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
12410   ins_encode %{
12411     __ string_compare($str1$$Register, $str2$$Register,
12412                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
12413                       $tmp1$$XMMRegister, StrIntrinsicNode::UU, knoreg);
12414   %}
12415   ins_pipe( pipe_slow );
12416 %}
12417 
12418 instruct string_compareU_evex(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
12419                               rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
12420 %{
12421   predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU);
12422   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
12423   effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
12424 
12425   format %{ "String Compare char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
12426   ins_encode %{
12427     __ string_compare($str1$$Register, $str2$$Register,
12428                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
12429                       $tmp1$$XMMRegister, StrIntrinsicNode::UU, $ktmp$$KRegister);
12430   %}
12431   ins_pipe( pipe_slow );
12432 %}
12433 
12434 instruct string_compareLU(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
12435                           rax_RegI result, legRegD tmp1, rFlagsReg cr)
12436 %{
12437   predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU);
12438   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
12439   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
12440 
12441   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
12442   ins_encode %{
12443     __ string_compare($str1$$Register, $str2$$Register,
12444                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
12445                       $tmp1$$XMMRegister, StrIntrinsicNode::LU, knoreg);
12446   %}
12447   ins_pipe( pipe_slow );
12448 %}
12449 
12450 instruct string_compareLU_evex(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
12451                                rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
12452 %{
12453   predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU);
12454   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
12455   effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
12456 
12457   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
12458   ins_encode %{
12459     __ string_compare($str1$$Register, $str2$$Register,
12460                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
12461                       $tmp1$$XMMRegister, StrIntrinsicNode::LU, $ktmp$$KRegister);
12462   %}
12463   ins_pipe( pipe_slow );
12464 %}
12465 
12466 instruct string_compareUL(rsi_RegP str1, rdx_RegI cnt1, rdi_RegP str2, rcx_RegI cnt2,
12467                           rax_RegI result, legRegD tmp1, rFlagsReg cr)
12468 %{
12469   predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL);
12470   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
12471   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
12472 
12473   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
12474   ins_encode %{
12475     __ string_compare($str2$$Register, $str1$$Register,
12476                       $cnt2$$Register, $cnt1$$Register, $result$$Register,
12477                       $tmp1$$XMMRegister, StrIntrinsicNode::UL, knoreg);
12478   %}
12479   ins_pipe( pipe_slow );
12480 %}
12481 
12482 instruct string_compareUL_evex(rsi_RegP str1, rdx_RegI cnt1, rdi_RegP str2, rcx_RegI cnt2,
12483                                rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
12484 %{
12485   predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL);
12486   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
12487   effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
12488 
12489   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
12490   ins_encode %{
12491     __ string_compare($str2$$Register, $str1$$Register,
12492                       $cnt2$$Register, $cnt1$$Register, $result$$Register,
12493                       $tmp1$$XMMRegister, StrIntrinsicNode::UL, $ktmp$$KRegister);
12494   %}
12495   ins_pipe( pipe_slow );
12496 %}
12497 
12498 // fast search of substring with known size.
12499 instruct string_indexof_conL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
12500                              rbx_RegI result, legRegD tmp_vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
12501 %{
12502   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL));
12503   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
12504   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
12505 
12506   format %{ "String IndexOf byte[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $tmp_vec, $cnt1, $cnt2, $tmp" %}
12507   ins_encode %{
12508     int icnt2 = (int)$int_cnt2$$constant;
12509     if (icnt2 >= 16) {
12510       // IndexOf for constant substrings with size >= 16 elements
12511       // which don't need to be loaded through stack.
12512       __ string_indexofC8($str1$$Register, $str2$$Register,
12513                           $cnt1$$Register, $cnt2$$Register,
12514                           icnt2, $result$$Register,
12515                           $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
12516     } else {
12517       // Small strings are loaded through stack if they cross page boundary.
12518       __ string_indexof($str1$$Register, $str2$$Register,
12519                         $cnt1$$Register, $cnt2$$Register,
12520                         icnt2, $result$$Register,
12521                         $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
12522     }
12523   %}
12524   ins_pipe( pipe_slow );
12525 %}
12526 
12527 // fast search of substring with known size.
12528 instruct string_indexof_conU(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
12529                              rbx_RegI result, legRegD tmp_vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
12530 %{
12531   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU));
12532   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
12533   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
12534 
12535   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $tmp_vec, $cnt1, $cnt2, $tmp" %}
12536   ins_encode %{
12537     int icnt2 = (int)$int_cnt2$$constant;
12538     if (icnt2 >= 8) {
12539       // IndexOf for constant substrings with size >= 8 elements
12540       // which don't need to be loaded through stack.
12541       __ string_indexofC8($str1$$Register, $str2$$Register,
12542                           $cnt1$$Register, $cnt2$$Register,
12543                           icnt2, $result$$Register,
12544                           $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
12545     } else {
12546       // Small strings are loaded through stack if they cross page boundary.
12547       __ string_indexof($str1$$Register, $str2$$Register,
12548                         $cnt1$$Register, $cnt2$$Register,
12549                         icnt2, $result$$Register,
12550                         $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
12551     }
12552   %}
12553   ins_pipe( pipe_slow );
12554 %}
12555 
12556 // fast search of substring with known size.
12557 instruct string_indexof_conUL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
12558                               rbx_RegI result, legRegD tmp_vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
12559 %{
12560   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL));
12561   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
12562   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
12563 
12564   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $tmp_vec, $cnt1, $cnt2, $tmp" %}
12565   ins_encode %{
12566     int icnt2 = (int)$int_cnt2$$constant;
12567     if (icnt2 >= 8) {
12568       // IndexOf for constant substrings with size >= 8 elements
12569       // which don't need to be loaded through stack.
12570       __ string_indexofC8($str1$$Register, $str2$$Register,
12571                           $cnt1$$Register, $cnt2$$Register,
12572                           icnt2, $result$$Register,
12573                           $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
12574     } else {
12575       // Small strings are loaded through stack if they cross page boundary.
12576       __ string_indexof($str1$$Register, $str2$$Register,
12577                         $cnt1$$Register, $cnt2$$Register,
12578                         icnt2, $result$$Register,
12579                         $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
12580     }
12581   %}
12582   ins_pipe( pipe_slow );
12583 %}
12584 
12585 instruct string_indexofL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
12586                          rbx_RegI result, legRegD tmp_vec, rcx_RegI tmp, rFlagsReg cr)
12587 %{
12588   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL));
12589   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
12590   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
12591 
12592   format %{ "String IndexOf byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
12593   ins_encode %{
12594     __ string_indexof($str1$$Register, $str2$$Register,
12595                       $cnt1$$Register, $cnt2$$Register,
12596                       (-1), $result$$Register,
12597                       $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
12598   %}
12599   ins_pipe( pipe_slow );
12600 %}
12601 
12602 instruct string_indexofU(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
12603                          rbx_RegI result, legRegD tmp_vec, rcx_RegI tmp, rFlagsReg cr)
12604 %{
12605   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU));
12606   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
12607   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
12608 
12609   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
12610   ins_encode %{
12611     __ string_indexof($str1$$Register, $str2$$Register,
12612                       $cnt1$$Register, $cnt2$$Register,
12613                       (-1), $result$$Register,
12614                       $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
12615   %}
12616   ins_pipe( pipe_slow );
12617 %}
12618 
12619 instruct string_indexofUL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
12620                           rbx_RegI result, legRegD tmp_vec, rcx_RegI tmp, rFlagsReg cr)
12621 %{
12622   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL));
12623   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
12624   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
12625 
12626   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
12627   ins_encode %{
12628     __ string_indexof($str1$$Register, $str2$$Register,
12629                       $cnt1$$Register, $cnt2$$Register,
12630                       (-1), $result$$Register,
12631                       $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
12632   %}
12633   ins_pipe( pipe_slow );
12634 %}
12635 
12636 instruct string_indexof_char(rdi_RegP str1, rdx_RegI cnt1, rax_RegI ch,
12637                               rbx_RegI result, legRegD tmp_vec1, legRegD tmp_vec2, legRegD tmp_vec3, rcx_RegI tmp, rFlagsReg cr)
12638 %{
12639   predicate(UseSSE42Intrinsics && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
12640   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
12641   effect(TEMP tmp_vec1, TEMP tmp_vec2, TEMP tmp_vec3, USE_KILL str1, USE_KILL cnt1, USE_KILL ch, TEMP tmp, KILL cr);
12642   format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result   // KILL all" %}
12643   ins_encode %{
12644     __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register, $result$$Register,
12645                            $tmp_vec1$$XMMRegister, $tmp_vec2$$XMMRegister, $tmp_vec3$$XMMRegister, $tmp$$Register);
12646   %}
12647   ins_pipe( pipe_slow );
12648 %}
12649 
12650 instruct stringL_indexof_char(rdi_RegP str1, rdx_RegI cnt1, rax_RegI ch,
12651                               rbx_RegI result, legRegD tmp_vec1, legRegD tmp_vec2, legRegD tmp_vec3, rcx_RegI tmp, rFlagsReg cr)
12652 %{
12653   predicate(UseSSE42Intrinsics && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
12654   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
12655   effect(TEMP tmp_vec1, TEMP tmp_vec2, TEMP tmp_vec3, USE_KILL str1, USE_KILL cnt1, USE_KILL ch, TEMP tmp, KILL cr);
12656   format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result   // KILL all" %}
12657   ins_encode %{
12658     __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register, $result$$Register,
12659                            $tmp_vec1$$XMMRegister, $tmp_vec2$$XMMRegister, $tmp_vec3$$XMMRegister, $tmp$$Register);
12660   %}
12661   ins_pipe( pipe_slow );
12662 %}
12663 
12664 // fast string equals
12665 instruct string_equals(rdi_RegP str1, rsi_RegP str2, rcx_RegI cnt, rax_RegI result,
12666                        legRegD tmp1, legRegD tmp2, rbx_RegI tmp3, rFlagsReg cr)
12667 %{
12668   predicate(!VM_Version::supports_avx512vlbw());
12669   match(Set result (StrEquals (Binary str1 str2) cnt));
12670   effect(TEMP tmp1, TEMP tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL tmp3, KILL cr);
12671 
12672   format %{ "String Equals $str1,$str2,$cnt -> $result    // KILL $tmp1, $tmp2, $tmp3" %}
12673   ins_encode %{
12674     __ arrays_equals(false, $str1$$Register, $str2$$Register,
12675                      $cnt$$Register, $result$$Register, $tmp3$$Register,
12676                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, knoreg);
12677   %}
12678   ins_pipe( pipe_slow );
12679 %}
12680 
12681 instruct string_equals_evex(rdi_RegP str1, rsi_RegP str2, rcx_RegI cnt, rax_RegI result,
12682                            legRegD tmp1, legRegD tmp2, kReg ktmp, rbx_RegI tmp3, rFlagsReg cr)
12683 %{
12684   predicate(VM_Version::supports_avx512vlbw());
12685   match(Set result (StrEquals (Binary str1 str2) cnt));
12686   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL tmp3, KILL cr);
12687 
12688   format %{ "String Equals $str1,$str2,$cnt -> $result    // KILL $tmp1, $tmp2, $tmp3" %}
12689   ins_encode %{
12690     __ arrays_equals(false, $str1$$Register, $str2$$Register,
12691                      $cnt$$Register, $result$$Register, $tmp3$$Register,
12692                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, $ktmp$$KRegister);
12693   %}
12694   ins_pipe( pipe_slow );
12695 %}
12696 
12697 // fast array equals
12698 instruct array_equalsB(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
12699                        legRegD tmp1, legRegD tmp2, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
12700 %{
12701   predicate(!VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
12702   match(Set result (AryEq ary1 ary2));
12703   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
12704 
12705   format %{ "Array Equals byte[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
12706   ins_encode %{
12707     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
12708                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
12709                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, knoreg);
12710   %}
12711   ins_pipe( pipe_slow );
12712 %}
12713 
12714 instruct array_equalsB_evex(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
12715                             legRegD tmp1, legRegD tmp2, kReg ktmp, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
12716 %{
12717   predicate(VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
12718   match(Set result (AryEq ary1 ary2));
12719   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
12720 
12721   format %{ "Array Equals byte[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
12722   ins_encode %{
12723     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
12724                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
12725                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, $ktmp$$KRegister);
12726   %}
12727   ins_pipe( pipe_slow );
12728 %}
12729 
12730 instruct array_equalsC(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
12731                        legRegD tmp1, legRegD tmp2, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
12732 %{
12733   predicate(!VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
12734   match(Set result (AryEq ary1 ary2));
12735   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
12736 
12737   format %{ "Array Equals char[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
12738   ins_encode %{
12739     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
12740                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
12741                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, true /* char */, knoreg);
12742   %}
12743   ins_pipe( pipe_slow );
12744 %}
12745 
12746 instruct array_equalsC_evex(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
12747                             legRegD tmp1, legRegD tmp2, kReg ktmp, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
12748 %{
12749   predicate(VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
12750   match(Set result (AryEq ary1 ary2));
12751   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
12752 
12753   format %{ "Array Equals char[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
12754   ins_encode %{
12755     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
12756                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
12757                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, true /* char */, $ktmp$$KRegister);
12758   %}
12759   ins_pipe( pipe_slow );
12760 %}
12761 
12762 instruct arrays_hashcode(rdi_RegP ary1, rdx_RegI cnt1, rbx_RegI result, immU8 basic_type,
12763                          legRegD tmp_vec1, legRegD tmp_vec2, legRegD tmp_vec3, legRegD tmp_vec4,
12764                          legRegD tmp_vec5, legRegD tmp_vec6, legRegD tmp_vec7, legRegD tmp_vec8,
12765                          legRegD tmp_vec9, legRegD tmp_vec10, legRegD tmp_vec11, legRegD tmp_vec12,
12766                          legRegD tmp_vec13, rRegI tmp1, rRegI tmp2, rRegI tmp3, rFlagsReg cr)
12767 %{
12768   predicate(UseAVX >= 2);
12769   match(Set result (VectorizedHashCode (Binary ary1 cnt1) (Binary result basic_type)));
12770   effect(TEMP tmp_vec1, TEMP tmp_vec2, TEMP tmp_vec3, TEMP tmp_vec4, TEMP tmp_vec5, TEMP tmp_vec6,
12771          TEMP tmp_vec7, TEMP tmp_vec8, TEMP tmp_vec9, TEMP tmp_vec10, TEMP tmp_vec11, TEMP tmp_vec12,
12772          TEMP tmp_vec13, TEMP tmp1, TEMP tmp2, TEMP tmp3, USE_KILL ary1, USE_KILL cnt1,
12773          USE basic_type, KILL cr);
12774 
12775   format %{ "Array HashCode array[] $ary1,$cnt1,$result,$basic_type -> $result   // KILL all" %}
12776   ins_encode %{
12777     __ arrays_hashcode($ary1$$Register, $cnt1$$Register, $result$$Register,
12778                        $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
12779                        $tmp_vec1$$XMMRegister, $tmp_vec2$$XMMRegister, $tmp_vec3$$XMMRegister,
12780                        $tmp_vec4$$XMMRegister, $tmp_vec5$$XMMRegister, $tmp_vec6$$XMMRegister,
12781                        $tmp_vec7$$XMMRegister, $tmp_vec8$$XMMRegister, $tmp_vec9$$XMMRegister,
12782                        $tmp_vec10$$XMMRegister, $tmp_vec11$$XMMRegister, $tmp_vec12$$XMMRegister,
12783                        $tmp_vec13$$XMMRegister, (BasicType)$basic_type$$constant);
12784   %}
12785   ins_pipe( pipe_slow );
12786 %}
12787 
12788 instruct count_positives(rsi_RegP ary1, rcx_RegI len, rax_RegI result,
12789                          legRegD tmp1, legRegD tmp2, rbx_RegI tmp3, rFlagsReg cr,)
12790 %{
12791   predicate(!VM_Version::supports_avx512vlbw() || !VM_Version::supports_bmi2());
12792   match(Set result (CountPositives ary1 len));
12793   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL len, KILL tmp3, KILL cr);
12794 
12795   format %{ "countPositives byte[] $ary1,$len -> $result   // KILL $tmp1, $tmp2, $tmp3" %}
12796   ins_encode %{
12797     __ count_positives($ary1$$Register, $len$$Register,
12798                        $result$$Register, $tmp3$$Register,
12799                        $tmp1$$XMMRegister, $tmp2$$XMMRegister, knoreg, knoreg);
12800   %}
12801   ins_pipe( pipe_slow );
12802 %}
12803 
12804 instruct count_positives_evex(rsi_RegP ary1, rcx_RegI len, rax_RegI result,
12805                               legRegD tmp1, legRegD tmp2, kReg ktmp1, kReg ktmp2, rbx_RegI tmp3, rFlagsReg cr,)
12806 %{
12807   predicate(VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2());
12808   match(Set result (CountPositives ary1 len));
12809   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp1, TEMP ktmp2, USE_KILL ary1, USE_KILL len, KILL tmp3, KILL cr);
12810 
12811   format %{ "countPositives byte[] $ary1,$len -> $result   // KILL $tmp1, $tmp2, $tmp3" %}
12812   ins_encode %{
12813     __ count_positives($ary1$$Register, $len$$Register,
12814                        $result$$Register, $tmp3$$Register,
12815                        $tmp1$$XMMRegister, $tmp2$$XMMRegister, $ktmp1$$KRegister, $ktmp2$$KRegister);
12816   %}
12817   ins_pipe( pipe_slow );
12818 %}
12819 
12820 // fast char[] to byte[] compression
12821 instruct string_compress(rsi_RegP src, rdi_RegP dst, rdx_RegI len, legRegD tmp1, legRegD tmp2, legRegD tmp3,
12822                          legRegD tmp4, rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
12823   predicate(!VM_Version::supports_avx512vlbw() || !VM_Version::supports_bmi2());
12824   match(Set result (StrCompressedCopy src (Binary dst len)));
12825   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst,
12826          USE_KILL len, KILL tmp5, KILL cr);
12827 
12828   format %{ "String Compress $src,$dst -> $result    // KILL RAX, RCX, RDX" %}
12829   ins_encode %{
12830     __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
12831                            $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
12832                            $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register,
12833                            knoreg, knoreg);
12834   %}
12835   ins_pipe( pipe_slow );
12836 %}
12837 
12838 instruct string_compress_evex(rsi_RegP src, rdi_RegP dst, rdx_RegI len, legRegD tmp1, legRegD tmp2, legRegD tmp3,
12839                               legRegD tmp4, kReg ktmp1, kReg ktmp2, rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
12840   predicate(VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2());
12841   match(Set result (StrCompressedCopy src (Binary dst len)));
12842   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP ktmp1, TEMP ktmp2, USE_KILL src, USE_KILL dst,
12843          USE_KILL len, KILL tmp5, KILL cr);
12844 
12845   format %{ "String Compress $src,$dst -> $result    // KILL RAX, RCX, RDX" %}
12846   ins_encode %{
12847     __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
12848                            $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
12849                            $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register,
12850                            $ktmp1$$KRegister, $ktmp2$$KRegister);
12851   %}
12852   ins_pipe( pipe_slow );
12853 %}
12854 // fast byte[] to char[] inflation
12855 instruct string_inflate(Universe dummy, rsi_RegP src, rdi_RegP dst, rdx_RegI len,
12856                         legRegD tmp1, rcx_RegI tmp2, rFlagsReg cr) %{
12857   predicate(!VM_Version::supports_avx512vlbw() || !VM_Version::supports_bmi2());
12858   match(Set dummy (StrInflatedCopy src (Binary dst len)));
12859   effect(TEMP tmp1, TEMP tmp2, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
12860 
12861   format %{ "String Inflate $src,$dst    // KILL $tmp1, $tmp2" %}
12862   ins_encode %{
12863     __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
12864                           $tmp1$$XMMRegister, $tmp2$$Register, knoreg);
12865   %}
12866   ins_pipe( pipe_slow );
12867 %}
12868 
12869 instruct string_inflate_evex(Universe dummy, rsi_RegP src, rdi_RegP dst, rdx_RegI len,
12870                              legRegD tmp1, kReg ktmp, rcx_RegI tmp2, rFlagsReg cr) %{
12871   predicate(VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2());
12872   match(Set dummy (StrInflatedCopy src (Binary dst len)));
12873   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
12874 
12875   format %{ "String Inflate $src,$dst    // KILL $tmp1, $tmp2" %}
12876   ins_encode %{
12877     __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
12878                           $tmp1$$XMMRegister, $tmp2$$Register, $ktmp$$KRegister);
12879   %}
12880   ins_pipe( pipe_slow );
12881 %}
12882 
12883 // encode char[] to byte[] in ISO_8859_1
12884 instruct encode_iso_array(rsi_RegP src, rdi_RegP dst, rdx_RegI len,
12885                           legRegD tmp1, legRegD tmp2, legRegD tmp3, legRegD tmp4,
12886                           rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
12887   predicate(!((EncodeISOArrayNode*)n)->is_ascii());
12888   match(Set result (EncodeISOArray src (Binary dst len)));
12889   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);
12890 
12891   format %{ "Encode iso array $src,$dst,$len -> $result    // KILL RCX, RDX, $tmp1, $tmp2, $tmp3, $tmp4, RSI, RDI " %}
12892   ins_encode %{
12893     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
12894                         $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
12895                         $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register, false);
12896   %}
12897   ins_pipe( pipe_slow );
12898 %}
12899 
12900 // encode char[] to byte[] in ASCII
12901 instruct encode_ascii_array(rsi_RegP src, rdi_RegP dst, rdx_RegI len,
12902                             legRegD tmp1, legRegD tmp2, legRegD tmp3, legRegD tmp4,
12903                             rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
12904   predicate(((EncodeISOArrayNode*)n)->is_ascii());
12905   match(Set result (EncodeISOArray src (Binary dst len)));
12906   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);
12907 
12908   format %{ "Encode ascii array $src,$dst,$len -> $result    // KILL RCX, RDX, $tmp1, $tmp2, $tmp3, $tmp4, RSI, RDI " %}
12909   ins_encode %{
12910     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
12911                         $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
12912                         $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register, true);
12913   %}
12914   ins_pipe( pipe_slow );
12915 %}
12916 
12917 //----------Overflow Math Instructions-----------------------------------------
12918 
12919 instruct overflowAddI_rReg(rFlagsReg cr, rax_RegI op1, rRegI op2)
12920 %{
12921   match(Set cr (OverflowAddI op1 op2));
12922   effect(DEF cr, USE_KILL op1, USE op2);
12923 
12924   format %{ "addl    $op1, $op2\t# overflow check int" %}
12925 
12926   ins_encode %{
12927     __ addl($op1$$Register, $op2$$Register);
12928   %}
12929   ins_pipe(ialu_reg_reg);
12930 %}
12931 
12932 instruct overflowAddI_rReg_imm(rFlagsReg cr, rax_RegI op1, immI op2)
12933 %{
12934   match(Set cr (OverflowAddI op1 op2));
12935   effect(DEF cr, USE_KILL op1, USE op2);
12936 
12937   format %{ "addl    $op1, $op2\t# overflow check int" %}
12938 
12939   ins_encode %{
12940     __ addl($op1$$Register, $op2$$constant);
12941   %}
12942   ins_pipe(ialu_reg_reg);
12943 %}
12944 
12945 instruct overflowAddL_rReg(rFlagsReg cr, rax_RegL op1, rRegL op2)
12946 %{
12947   match(Set cr (OverflowAddL op1 op2));
12948   effect(DEF cr, USE_KILL op1, USE op2);
12949 
12950   format %{ "addq    $op1, $op2\t# overflow check long" %}
12951   ins_encode %{
12952     __ addq($op1$$Register, $op2$$Register);
12953   %}
12954   ins_pipe(ialu_reg_reg);
12955 %}
12956 
12957 instruct overflowAddL_rReg_imm(rFlagsReg cr, rax_RegL op1, immL32 op2)
12958 %{
12959   match(Set cr (OverflowAddL op1 op2));
12960   effect(DEF cr, USE_KILL op1, USE op2);
12961 
12962   format %{ "addq    $op1, $op2\t# overflow check long" %}
12963   ins_encode %{
12964     __ addq($op1$$Register, $op2$$constant);
12965   %}
12966   ins_pipe(ialu_reg_reg);
12967 %}
12968 
12969 instruct overflowSubI_rReg(rFlagsReg cr, rRegI op1, rRegI op2)
12970 %{
12971   match(Set cr (OverflowSubI op1 op2));
12972 
12973   format %{ "cmpl    $op1, $op2\t# overflow check int" %}
12974   ins_encode %{
12975     __ cmpl($op1$$Register, $op2$$Register);
12976   %}
12977   ins_pipe(ialu_reg_reg);
12978 %}
12979 
12980 instruct overflowSubI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2)
12981 %{
12982   match(Set cr (OverflowSubI op1 op2));
12983 
12984   format %{ "cmpl    $op1, $op2\t# overflow check int" %}
12985   ins_encode %{
12986     __ cmpl($op1$$Register, $op2$$constant);
12987   %}
12988   ins_pipe(ialu_reg_reg);
12989 %}
12990 
12991 instruct overflowSubL_rReg(rFlagsReg cr, rRegL op1, rRegL op2)
12992 %{
12993   match(Set cr (OverflowSubL op1 op2));
12994 
12995   format %{ "cmpq    $op1, $op2\t# overflow check long" %}
12996   ins_encode %{
12997     __ cmpq($op1$$Register, $op2$$Register);
12998   %}
12999   ins_pipe(ialu_reg_reg);
13000 %}
13001 
13002 instruct overflowSubL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2)
13003 %{
13004   match(Set cr (OverflowSubL op1 op2));
13005 
13006   format %{ "cmpq    $op1, $op2\t# overflow check long" %}
13007   ins_encode %{
13008     __ cmpq($op1$$Register, $op2$$constant);
13009   %}
13010   ins_pipe(ialu_reg_reg);
13011 %}
13012 
13013 instruct overflowNegI_rReg(rFlagsReg cr, immI_0 zero, rax_RegI op2)
13014 %{
13015   match(Set cr (OverflowSubI zero op2));
13016   effect(DEF cr, USE_KILL op2);
13017 
13018   format %{ "negl    $op2\t# overflow check int" %}
13019   ins_encode %{
13020     __ negl($op2$$Register);
13021   %}
13022   ins_pipe(ialu_reg_reg);
13023 %}
13024 
13025 instruct overflowNegL_rReg(rFlagsReg cr, immL0 zero, rax_RegL op2)
13026 %{
13027   match(Set cr (OverflowSubL zero op2));
13028   effect(DEF cr, USE_KILL op2);
13029 
13030   format %{ "negq    $op2\t# overflow check long" %}
13031   ins_encode %{
13032     __ negq($op2$$Register);
13033   %}
13034   ins_pipe(ialu_reg_reg);
13035 %}
13036 
13037 instruct overflowMulI_rReg(rFlagsReg cr, rax_RegI op1, rRegI op2)
13038 %{
13039   match(Set cr (OverflowMulI op1 op2));
13040   effect(DEF cr, USE_KILL op1, USE op2);
13041 
13042   format %{ "imull    $op1, $op2\t# overflow check int" %}
13043   ins_encode %{
13044     __ imull($op1$$Register, $op2$$Register);
13045   %}
13046   ins_pipe(ialu_reg_reg_alu0);
13047 %}
13048 
13049 instruct overflowMulI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2, rRegI tmp)
13050 %{
13051   match(Set cr (OverflowMulI op1 op2));
13052   effect(DEF cr, TEMP tmp, USE op1, USE op2);
13053 
13054   format %{ "imull    $tmp, $op1, $op2\t# overflow check int" %}
13055   ins_encode %{
13056     __ imull($tmp$$Register, $op1$$Register, $op2$$constant);
13057   %}
13058   ins_pipe(ialu_reg_reg_alu0);
13059 %}
13060 
13061 instruct overflowMulL_rReg(rFlagsReg cr, rax_RegL op1, rRegL op2)
13062 %{
13063   match(Set cr (OverflowMulL op1 op2));
13064   effect(DEF cr, USE_KILL op1, USE op2);
13065 
13066   format %{ "imulq    $op1, $op2\t# overflow check long" %}
13067   ins_encode %{
13068     __ imulq($op1$$Register, $op2$$Register);
13069   %}
13070   ins_pipe(ialu_reg_reg_alu0);
13071 %}
13072 
13073 instruct overflowMulL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2, rRegL tmp)
13074 %{
13075   match(Set cr (OverflowMulL op1 op2));
13076   effect(DEF cr, TEMP tmp, USE op1, USE op2);
13077 
13078   format %{ "imulq    $tmp, $op1, $op2\t# overflow check long" %}
13079   ins_encode %{
13080     __ imulq($tmp$$Register, $op1$$Register, $op2$$constant);
13081   %}
13082   ins_pipe(ialu_reg_reg_alu0);
13083 %}
13084 
13085 
13086 //----------Control Flow Instructions------------------------------------------
13087 // Signed compare Instructions
13088 
13089 // XXX more variants!!
13090 instruct compI_rReg(rFlagsReg cr, rRegI op1, rRegI op2)
13091 %{
13092   match(Set cr (CmpI op1 op2));
13093   effect(DEF cr, USE op1, USE op2);
13094 
13095   format %{ "cmpl    $op1, $op2" %}
13096   ins_encode %{
13097     __ cmpl($op1$$Register, $op2$$Register);
13098   %}
13099   ins_pipe(ialu_cr_reg_reg);
13100 %}
13101 
13102 instruct compI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2)
13103 %{
13104   match(Set cr (CmpI op1 op2));
13105 
13106   format %{ "cmpl    $op1, $op2" %}
13107   ins_encode %{
13108     __ cmpl($op1$$Register, $op2$$constant);
13109   %}
13110   ins_pipe(ialu_cr_reg_imm);
13111 %}
13112 
13113 instruct compI_rReg_mem(rFlagsReg cr, rRegI op1, memory op2)
13114 %{
13115   match(Set cr (CmpI op1 (LoadI op2)));
13116 
13117   ins_cost(500); // XXX
13118   format %{ "cmpl    $op1, $op2" %}
13119   ins_encode %{
13120     __ cmpl($op1$$Register, $op2$$Address);
13121   %}
13122   ins_pipe(ialu_cr_reg_mem);
13123 %}
13124 
13125 instruct testI_reg(rFlagsReg cr, rRegI src, immI_0 zero)
13126 %{
13127   match(Set cr (CmpI src zero));
13128 
13129   format %{ "testl   $src, $src" %}
13130   ins_encode %{
13131     __ testl($src$$Register, $src$$Register);
13132   %}
13133   ins_pipe(ialu_cr_reg_imm);
13134 %}
13135 
13136 instruct testI_reg_imm(rFlagsReg cr, rRegI src, immI con, immI_0 zero)
13137 %{
13138   match(Set cr (CmpI (AndI src con) zero));
13139 
13140   format %{ "testl   $src, $con" %}
13141   ins_encode %{
13142     __ testl($src$$Register, $con$$constant);
13143   %}
13144   ins_pipe(ialu_cr_reg_imm);
13145 %}
13146 
13147 instruct testI_reg_reg(rFlagsReg cr, rRegI src1, rRegI src2, immI_0 zero)
13148 %{
13149   match(Set cr (CmpI (AndI src1 src2) zero));
13150 
13151   format %{ "testl   $src1, $src2" %}
13152   ins_encode %{
13153     __ testl($src1$$Register, $src2$$Register);
13154   %}
13155   ins_pipe(ialu_cr_reg_imm);
13156 %}
13157 
13158 instruct testI_reg_mem(rFlagsReg cr, rRegI src, memory mem, immI_0 zero)
13159 %{
13160   match(Set cr (CmpI (AndI src (LoadI mem)) zero));
13161 
13162   format %{ "testl   $src, $mem" %}
13163   ins_encode %{
13164     __ testl($src$$Register, $mem$$Address);
13165   %}
13166   ins_pipe(ialu_cr_reg_mem);
13167 %}
13168 
13169 // Unsigned compare Instructions; really, same as signed except they
13170 // produce an rFlagsRegU instead of rFlagsReg.
13171 instruct compU_rReg(rFlagsRegU cr, rRegI op1, rRegI op2)
13172 %{
13173   match(Set cr (CmpU op1 op2));
13174 
13175   format %{ "cmpl    $op1, $op2\t# unsigned" %}
13176   ins_encode %{
13177     __ cmpl($op1$$Register, $op2$$Register);
13178   %}
13179   ins_pipe(ialu_cr_reg_reg);
13180 %}
13181 
13182 instruct compU_rReg_imm(rFlagsRegU cr, rRegI op1, immI op2)
13183 %{
13184   match(Set cr (CmpU op1 op2));
13185 
13186   format %{ "cmpl    $op1, $op2\t# unsigned" %}
13187   ins_encode %{
13188     __ cmpl($op1$$Register, $op2$$constant);
13189   %}
13190   ins_pipe(ialu_cr_reg_imm);
13191 %}
13192 
13193 instruct compU_rReg_mem(rFlagsRegU cr, rRegI op1, memory op2)
13194 %{
13195   match(Set cr (CmpU op1 (LoadI op2)));
13196 
13197   ins_cost(500); // XXX
13198   format %{ "cmpl    $op1, $op2\t# unsigned" %}
13199   ins_encode %{
13200     __ cmpl($op1$$Register, $op2$$Address);
13201   %}
13202   ins_pipe(ialu_cr_reg_mem);
13203 %}
13204 
13205 instruct testU_reg(rFlagsRegU cr, rRegI src, immI_0 zero)
13206 %{
13207   match(Set cr (CmpU src zero));
13208 
13209   format %{ "testl   $src, $src\t# unsigned" %}
13210   ins_encode %{
13211     __ testl($src$$Register, $src$$Register);
13212   %}
13213   ins_pipe(ialu_cr_reg_imm);
13214 %}
13215 
13216 instruct compP_rReg(rFlagsRegU cr, rRegP op1, rRegP op2)
13217 %{
13218   match(Set cr (CmpP op1 op2));
13219 
13220   format %{ "cmpq    $op1, $op2\t# ptr" %}
13221   ins_encode %{
13222     __ cmpq($op1$$Register, $op2$$Register);
13223   %}
13224   ins_pipe(ialu_cr_reg_reg);
13225 %}
13226 
13227 instruct compP_rReg_mem(rFlagsRegU cr, rRegP op1, memory op2)
13228 %{
13229   match(Set cr (CmpP op1 (LoadP op2)));
13230   predicate(n->in(2)->as_Load()->barrier_data() == 0);
13231 
13232   ins_cost(500); // XXX
13233   format %{ "cmpq    $op1, $op2\t# ptr" %}
13234   ins_encode %{
13235     __ cmpq($op1$$Register, $op2$$Address);
13236   %}
13237   ins_pipe(ialu_cr_reg_mem);
13238 %}
13239 
13240 // XXX this is generalized by compP_rReg_mem???
13241 // Compare raw pointer (used in out-of-heap check).
13242 // Only works because non-oop pointers must be raw pointers
13243 // and raw pointers have no anti-dependencies.
13244 instruct compP_mem_rReg(rFlagsRegU cr, rRegP op1, memory op2)
13245 %{
13246   predicate(n->in(2)->in(2)->bottom_type()->reloc() == relocInfo::none &&
13247             n->in(2)->as_Load()->barrier_data() == 0);
13248   match(Set cr (CmpP op1 (LoadP op2)));
13249 
13250   format %{ "cmpq    $op1, $op2\t# raw ptr" %}
13251   ins_encode %{
13252     __ cmpq($op1$$Register, $op2$$Address);
13253   %}
13254   ins_pipe(ialu_cr_reg_mem);
13255 %}
13256 
13257 // This will generate a signed flags result. This should be OK since
13258 // any compare to a zero should be eq/neq.
13259 instruct testP_reg(rFlagsReg cr, rRegP src, immP0 zero)
13260 %{
13261   match(Set cr (CmpP src zero));
13262 
13263   format %{ "testq   $src, $src\t# ptr" %}
13264   ins_encode %{
13265     __ testq($src$$Register, $src$$Register);
13266   %}
13267   ins_pipe(ialu_cr_reg_imm);
13268 %}
13269 
13270 // This will generate a signed flags result. This should be OK since
13271 // any compare to a zero should be eq/neq.
13272 instruct testP_mem(rFlagsReg cr, memory op, immP0 zero)
13273 %{
13274   predicate((!UseCompressedOops || (CompressedOops::base() != nullptr)) &&
13275             n->in(1)->as_Load()->barrier_data() == 0);
13276   match(Set cr (CmpP (LoadP op) zero));
13277 
13278   ins_cost(500); // XXX
13279   format %{ "testq   $op, 0xffffffffffffffff\t# ptr" %}
13280   ins_encode %{
13281     __ testq($op$$Address, 0xFFFFFFFF);
13282   %}
13283   ins_pipe(ialu_cr_reg_imm);
13284 %}
13285 
13286 instruct testP_mem_reg0(rFlagsReg cr, memory mem, immP0 zero)
13287 %{
13288   predicate(UseCompressedOops && (CompressedOops::base() == nullptr) &&
13289             n->in(1)->as_Load()->barrier_data() == 0);
13290   match(Set cr (CmpP (LoadP mem) zero));
13291 
13292   format %{ "cmpq    R12, $mem\t# ptr (R12_heapbase==0)" %}
13293   ins_encode %{
13294     __ cmpq(r12, $mem$$Address);
13295   %}
13296   ins_pipe(ialu_cr_reg_mem);
13297 %}
13298 
13299 instruct compN_rReg(rFlagsRegU cr, rRegN op1, rRegN op2)
13300 %{
13301   match(Set cr (CmpN op1 op2));
13302 
13303   format %{ "cmpl    $op1, $op2\t# compressed ptr" %}
13304   ins_encode %{ __ cmpl($op1$$Register, $op2$$Register); %}
13305   ins_pipe(ialu_cr_reg_reg);
13306 %}
13307 
13308 instruct compN_rReg_mem(rFlagsRegU cr, rRegN src, memory mem)
13309 %{
13310   predicate(n->in(2)->as_Load()->barrier_data() == 0);
13311   match(Set cr (CmpN src (LoadN mem)));
13312 
13313   format %{ "cmpl    $src, $mem\t# compressed ptr" %}
13314   ins_encode %{
13315     __ cmpl($src$$Register, $mem$$Address);
13316   %}
13317   ins_pipe(ialu_cr_reg_mem);
13318 %}
13319 
13320 instruct compN_rReg_imm(rFlagsRegU cr, rRegN op1, immN op2) %{
13321   match(Set cr (CmpN op1 op2));
13322 
13323   format %{ "cmpl    $op1, $op2\t# compressed ptr" %}
13324   ins_encode %{
13325     __ cmp_narrow_oop($op1$$Register, (jobject)$op2$$constant);
13326   %}
13327   ins_pipe(ialu_cr_reg_imm);
13328 %}
13329 
13330 instruct compN_mem_imm(rFlagsRegU cr, memory mem, immN src)
13331 %{
13332   predicate(n->in(2)->as_Load()->barrier_data() == 0);
13333   match(Set cr (CmpN src (LoadN mem)));
13334 
13335   format %{ "cmpl    $mem, $src\t# compressed ptr" %}
13336   ins_encode %{
13337     __ cmp_narrow_oop($mem$$Address, (jobject)$src$$constant);
13338   %}
13339   ins_pipe(ialu_cr_reg_mem);
13340 %}
13341 
13342 instruct compN_rReg_imm_klass(rFlagsRegU cr, rRegN op1, immNKlass op2) %{
13343   match(Set cr (CmpN op1 op2));
13344 
13345   format %{ "cmpl    $op1, $op2\t# compressed klass ptr" %}
13346   ins_encode %{
13347     __ cmp_narrow_klass($op1$$Register, (Klass*)$op2$$constant);
13348   %}
13349   ins_pipe(ialu_cr_reg_imm);
13350 %}
13351 
13352 instruct compN_mem_imm_klass(rFlagsRegU cr, memory mem, immNKlass src)
13353 %{
13354   predicate(!UseCompactObjectHeaders);
13355   match(Set cr (CmpN src (LoadNKlass mem)));
13356 
13357   format %{ "cmpl    $mem, $src\t# compressed klass ptr" %}
13358   ins_encode %{
13359     __ cmp_narrow_klass($mem$$Address, (Klass*)$src$$constant);
13360   %}
13361   ins_pipe(ialu_cr_reg_mem);
13362 %}
13363 
13364 instruct testN_reg(rFlagsReg cr, rRegN src, immN0 zero) %{
13365   match(Set cr (CmpN src zero));
13366 
13367   format %{ "testl   $src, $src\t# compressed ptr" %}
13368   ins_encode %{ __ testl($src$$Register, $src$$Register); %}
13369   ins_pipe(ialu_cr_reg_imm);
13370 %}
13371 
13372 instruct testN_mem(rFlagsReg cr, memory mem, immN0 zero)
13373 %{
13374   predicate(CompressedOops::base() != nullptr &&
13375             n->in(1)->as_Load()->barrier_data() == 0);
13376   match(Set cr (CmpN (LoadN mem) zero));
13377 
13378   ins_cost(500); // XXX
13379   format %{ "testl   $mem, 0xffffffff\t# compressed ptr" %}
13380   ins_encode %{
13381     __ cmpl($mem$$Address, (int)0xFFFFFFFF);
13382   %}
13383   ins_pipe(ialu_cr_reg_mem);
13384 %}
13385 
13386 instruct testN_mem_reg0(rFlagsReg cr, memory mem, immN0 zero)
13387 %{
13388   predicate(CompressedOops::base() == nullptr &&
13389             n->in(1)->as_Load()->barrier_data() == 0);
13390   match(Set cr (CmpN (LoadN mem) zero));
13391 
13392   format %{ "cmpl    R12, $mem\t# compressed ptr (R12_heapbase==0)" %}
13393   ins_encode %{
13394     __ cmpl(r12, $mem$$Address);
13395   %}
13396   ins_pipe(ialu_cr_reg_mem);
13397 %}
13398 
13399 // Yanked all unsigned pointer compare operations.
13400 // Pointer compares are done with CmpP which is already unsigned.
13401 
13402 instruct compL_rReg(rFlagsReg cr, rRegL op1, rRegL op2)
13403 %{
13404   match(Set cr (CmpL op1 op2));
13405 
13406   format %{ "cmpq    $op1, $op2" %}
13407   ins_encode %{
13408     __ cmpq($op1$$Register, $op2$$Register);
13409   %}
13410   ins_pipe(ialu_cr_reg_reg);
13411 %}
13412 
13413 instruct compL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2)
13414 %{
13415   match(Set cr (CmpL op1 op2));
13416 
13417   format %{ "cmpq    $op1, $op2" %}
13418   ins_encode %{
13419     __ cmpq($op1$$Register, $op2$$constant);
13420   %}
13421   ins_pipe(ialu_cr_reg_imm);
13422 %}
13423 
13424 instruct compL_rReg_mem(rFlagsReg cr, rRegL op1, memory op2)
13425 %{
13426   match(Set cr (CmpL op1 (LoadL op2)));
13427 
13428   format %{ "cmpq    $op1, $op2" %}
13429   ins_encode %{
13430     __ cmpq($op1$$Register, $op2$$Address);
13431   %}
13432   ins_pipe(ialu_cr_reg_mem);
13433 %}
13434 
13435 instruct testL_reg(rFlagsReg cr, rRegL src, immL0 zero)
13436 %{
13437   match(Set cr (CmpL src zero));
13438 
13439   format %{ "testq   $src, $src" %}
13440   ins_encode %{
13441     __ testq($src$$Register, $src$$Register);
13442   %}
13443   ins_pipe(ialu_cr_reg_imm);
13444 %}
13445 
13446 instruct testL_reg_imm(rFlagsReg cr, rRegL src, immL32 con, immL0 zero)
13447 %{
13448   match(Set cr (CmpL (AndL src con) zero));
13449 
13450   format %{ "testq   $src, $con\t# long" %}
13451   ins_encode %{
13452     __ testq($src$$Register, $con$$constant);
13453   %}
13454   ins_pipe(ialu_cr_reg_imm);
13455 %}
13456 
13457 instruct testL_reg_reg(rFlagsReg cr, rRegL src1, rRegL src2, immL0 zero)
13458 %{
13459   match(Set cr (CmpL (AndL src1 src2) zero));
13460 
13461   format %{ "testq   $src1, $src2\t# long" %}
13462   ins_encode %{
13463     __ testq($src1$$Register, $src2$$Register);
13464   %}
13465   ins_pipe(ialu_cr_reg_imm);
13466 %}
13467 
13468 instruct testL_reg_mem(rFlagsReg cr, rRegL src, memory mem, immL0 zero)
13469 %{
13470   match(Set cr (CmpL (AndL src (LoadL mem)) zero));
13471 
13472   format %{ "testq   $src, $mem" %}
13473   ins_encode %{
13474     __ testq($src$$Register, $mem$$Address);
13475   %}
13476   ins_pipe(ialu_cr_reg_mem);
13477 %}
13478 
13479 instruct testL_reg_mem2(rFlagsReg cr, rRegP src, memory mem, immL0 zero)
13480 %{
13481   match(Set cr (CmpL (AndL (CastP2X src) (LoadL mem)) zero));
13482 
13483   format %{ "testq   $src, $mem" %}
13484   ins_encode %{
13485     __ testq($src$$Register, $mem$$Address);
13486   %}
13487   ins_pipe(ialu_cr_reg_mem);
13488 %}
13489 
13490 // Manifest a CmpU result in an integer register.  Very painful.
13491 // This is the test to avoid.
13492 instruct cmpU3_reg_reg(rRegI dst, rRegI src1, rRegI src2, rFlagsReg flags)
13493 %{
13494   match(Set dst (CmpU3 src1 src2));
13495   effect(KILL flags);
13496 
13497   ins_cost(275); // XXX
13498   format %{ "cmpl    $src1, $src2\t# CmpL3\n\t"
13499             "movl    $dst, -1\n\t"
13500             "jb,u    done\n\t"
13501             "setcc   $dst \t# emits setne + movzbl or setzune for APX"
13502     "done:" %}
13503   ins_encode %{
13504     Label done;
13505     __ cmpl($src1$$Register, $src2$$Register);
13506     __ movl($dst$$Register, -1);
13507     __ jccb(Assembler::below, done);
13508     __ setcc(Assembler::notZero, $dst$$Register);
13509     __ bind(done);
13510   %}
13511   ins_pipe(pipe_slow);
13512 %}
13513 
13514 // Manifest a CmpL result in an integer register.  Very painful.
13515 // This is the test to avoid.
13516 instruct cmpL3_reg_reg(rRegI dst, rRegL src1, rRegL src2, rFlagsReg flags)
13517 %{
13518   match(Set dst (CmpL3 src1 src2));
13519   effect(KILL flags);
13520 
13521   ins_cost(275); // XXX
13522   format %{ "cmpq    $src1, $src2\t# CmpL3\n\t"
13523             "movl    $dst, -1\n\t"
13524             "jl,s    done\n\t"
13525             "setcc   $dst \t# emits setne + movzbl or setzune for APX"
13526     "done:" %}
13527   ins_encode %{
13528     Label done;
13529     __ cmpq($src1$$Register, $src2$$Register);
13530     __ movl($dst$$Register, -1);
13531     __ jccb(Assembler::less, done);
13532     __ setcc(Assembler::notZero, $dst$$Register);
13533     __ bind(done);
13534   %}
13535   ins_pipe(pipe_slow);
13536 %}
13537 
13538 // Manifest a CmpUL result in an integer register.  Very painful.
13539 // This is the test to avoid.
13540 instruct cmpUL3_reg_reg(rRegI dst, rRegL src1, rRegL src2, rFlagsReg flags)
13541 %{
13542   match(Set dst (CmpUL3 src1 src2));
13543   effect(KILL flags);
13544 
13545   ins_cost(275); // XXX
13546   format %{ "cmpq    $src1, $src2\t# CmpL3\n\t"
13547             "movl    $dst, -1\n\t"
13548             "jb,u    done\n\t"
13549             "setcc   $dst \t# emits setne + movzbl or setzune for APX"
13550     "done:" %}
13551   ins_encode %{
13552     Label done;
13553     __ cmpq($src1$$Register, $src2$$Register);
13554     __ movl($dst$$Register, -1);
13555     __ jccb(Assembler::below, done);
13556     __ setcc(Assembler::notZero, $dst$$Register);
13557     __ bind(done);
13558   %}
13559   ins_pipe(pipe_slow);
13560 %}
13561 
13562 // Unsigned long compare Instructions; really, same as signed long except they
13563 // produce an rFlagsRegU instead of rFlagsReg.
13564 instruct compUL_rReg(rFlagsRegU cr, rRegL op1, rRegL op2)
13565 %{
13566   match(Set cr (CmpUL op1 op2));
13567 
13568   format %{ "cmpq    $op1, $op2\t# unsigned" %}
13569   ins_encode %{
13570     __ cmpq($op1$$Register, $op2$$Register);
13571   %}
13572   ins_pipe(ialu_cr_reg_reg);
13573 %}
13574 
13575 instruct compUL_rReg_imm(rFlagsRegU cr, rRegL op1, immL32 op2)
13576 %{
13577   match(Set cr (CmpUL op1 op2));
13578 
13579   format %{ "cmpq    $op1, $op2\t# unsigned" %}
13580   ins_encode %{
13581     __ cmpq($op1$$Register, $op2$$constant);
13582   %}
13583   ins_pipe(ialu_cr_reg_imm);
13584 %}
13585 
13586 instruct compUL_rReg_mem(rFlagsRegU cr, rRegL op1, memory op2)
13587 %{
13588   match(Set cr (CmpUL op1 (LoadL op2)));
13589 
13590   format %{ "cmpq    $op1, $op2\t# unsigned" %}
13591   ins_encode %{
13592     __ cmpq($op1$$Register, $op2$$Address);
13593   %}
13594   ins_pipe(ialu_cr_reg_mem);
13595 %}
13596 
13597 instruct testUL_reg(rFlagsRegU cr, rRegL src, immL0 zero)
13598 %{
13599   match(Set cr (CmpUL src zero));
13600 
13601   format %{ "testq   $src, $src\t# unsigned" %}
13602   ins_encode %{
13603     __ testq($src$$Register, $src$$Register);
13604   %}
13605   ins_pipe(ialu_cr_reg_imm);
13606 %}
13607 
13608 instruct compB_mem_imm(rFlagsReg cr, memory mem, immI8 imm)
13609 %{
13610   match(Set cr (CmpI (LoadB mem) imm));
13611 
13612   ins_cost(125);
13613   format %{ "cmpb    $mem, $imm" %}
13614   ins_encode %{ __ cmpb($mem$$Address, $imm$$constant); %}
13615   ins_pipe(ialu_cr_reg_mem);
13616 %}
13617 
13618 instruct testUB_mem_imm(rFlagsReg cr, memory mem, immU7 imm, immI_0 zero)
13619 %{
13620   match(Set cr (CmpI (AndI (LoadUB mem) imm) zero));
13621 
13622   ins_cost(125);
13623   format %{ "testb   $mem, $imm\t# ubyte" %}
13624   ins_encode %{ __ testb($mem$$Address, $imm$$constant); %}
13625   ins_pipe(ialu_cr_reg_mem);
13626 %}
13627 
13628 instruct testB_mem_imm(rFlagsReg cr, memory mem, immI8 imm, immI_0 zero)
13629 %{
13630   match(Set cr (CmpI (AndI (LoadB mem) imm) zero));
13631 
13632   ins_cost(125);
13633   format %{ "testb   $mem, $imm\t# byte" %}
13634   ins_encode %{ __ testb($mem$$Address, $imm$$constant); %}
13635   ins_pipe(ialu_cr_reg_mem);
13636 %}
13637 
13638 //----------Max and Min--------------------------------------------------------
13639 // Min Instructions
13640 
13641 instruct cmovI_reg_g(rRegI dst, rRegI src, rFlagsReg cr)
13642 %{
13643   predicate(!UseAPX);
13644   effect(USE_DEF dst, USE src, USE cr);
13645 
13646   format %{ "cmovlgt $dst, $src\t# min" %}
13647   ins_encode %{
13648     __ cmovl(Assembler::greater, $dst$$Register, $src$$Register);
13649   %}
13650   ins_pipe(pipe_cmov_reg);
13651 %}
13652 
13653 instruct cmovI_reg_g_ndd(rRegI dst, rRegI src1, rRegI src2, rFlagsReg cr)
13654 %{
13655   predicate(UseAPX);
13656   effect(DEF dst, USE src1, USE src2, USE cr);
13657 
13658   format %{ "ecmovlgt $dst, $src1, $src2\t# min ndd" %}
13659   ins_encode %{
13660     __ ecmovl(Assembler::greater, $dst$$Register, $src1$$Register, $src2$$Register);
13661   %}
13662   ins_pipe(pipe_cmov_reg);
13663 %}
13664 
13665 instruct minI_rReg(rRegI dst, rRegI src)
13666 %{
13667   predicate(!UseAPX);
13668   match(Set dst (MinI dst src));
13669 
13670   ins_cost(200);
13671   expand %{
13672     rFlagsReg cr;
13673     compI_rReg(cr, dst, src);
13674     cmovI_reg_g(dst, src, cr);
13675   %}
13676 %}
13677 
13678 instruct minI_rReg_ndd(rRegI dst, rRegI src1, rRegI src2)
13679 %{
13680   predicate(UseAPX);
13681   match(Set dst (MinI src1 src2));
13682   effect(DEF dst, USE src1, USE src2);
13683 
13684   ins_cost(200);
13685   expand %{
13686     rFlagsReg cr;
13687     compI_rReg(cr, src1, src2);
13688     cmovI_reg_g_ndd(dst, src1, src2, cr);
13689   %}
13690 %}
13691 
13692 instruct cmovI_reg_l(rRegI dst, rRegI src, rFlagsReg cr)
13693 %{
13694   predicate(!UseAPX);
13695   effect(USE_DEF dst, USE src, USE cr);
13696 
13697   format %{ "cmovllt $dst, $src\t# max" %}
13698   ins_encode %{
13699     __ cmovl(Assembler::less, $dst$$Register, $src$$Register);
13700   %}
13701   ins_pipe(pipe_cmov_reg);
13702 %}
13703 
13704 instruct cmovI_reg_l_ndd(rRegI dst, rRegI src1, rRegI src2, rFlagsReg cr)
13705 %{
13706   predicate(UseAPX);
13707   effect(DEF dst, USE src1, USE src2, USE cr);
13708 
13709   format %{ "ecmovllt $dst, $src1, $src2\t# max ndd" %}
13710   ins_encode %{
13711     __ ecmovl(Assembler::less, $dst$$Register, $src1$$Register, $src2$$Register);
13712   %}
13713   ins_pipe(pipe_cmov_reg);
13714 %}
13715 
13716 instruct maxI_rReg(rRegI dst, rRegI src)
13717 %{
13718   predicate(!UseAPX);
13719   match(Set dst (MaxI dst src));
13720 
13721   ins_cost(200);
13722   expand %{
13723     rFlagsReg cr;
13724     compI_rReg(cr, dst, src);
13725     cmovI_reg_l(dst, src, cr);
13726   %}
13727 %}
13728 
13729 instruct maxI_rReg_ndd(rRegI dst, rRegI src1, rRegI src2)
13730 %{
13731   predicate(UseAPX);
13732   match(Set dst (MaxI src1 src2));
13733   effect(DEF dst, USE src1, USE src2);
13734 
13735   ins_cost(200);
13736   expand %{
13737     rFlagsReg cr;
13738     compI_rReg(cr, src1, src2);
13739     cmovI_reg_l_ndd(dst, src1, src2, cr);
13740   %}
13741 %}
13742 
13743 // ============================================================================
13744 // Branch Instructions
13745 
13746 // Jump Direct - Label defines a relative address from JMP+1
13747 instruct jmpDir(label labl)
13748 %{
13749   match(Goto);
13750   effect(USE labl);
13751 
13752   ins_cost(300);
13753   format %{ "jmp     $labl" %}
13754   size(5);
13755   ins_encode %{
13756     Label* L = $labl$$label;
13757     __ jmp(*L, false); // Always long jump
13758   %}
13759   ins_pipe(pipe_jmp);
13760 %}
13761 
13762 // Jump Direct Conditional - Label defines a relative address from Jcc+1
13763 instruct jmpCon(cmpOp cop, rFlagsReg cr, label labl)
13764 %{
13765   match(If cop cr);
13766   effect(USE labl);
13767 
13768   ins_cost(300);
13769   format %{ "j$cop     $labl" %}
13770   size(6);
13771   ins_encode %{
13772     Label* L = $labl$$label;
13773     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
13774   %}
13775   ins_pipe(pipe_jcc);
13776 %}
13777 
13778 // Jump Direct Conditional - Label defines a relative address from Jcc+1
13779 instruct jmpLoopEnd(cmpOp cop, rFlagsReg cr, label labl)
13780 %{
13781   match(CountedLoopEnd cop cr);
13782   effect(USE labl);
13783 
13784   ins_cost(300);
13785   format %{ "j$cop     $labl\t# loop end" %}
13786   size(6);
13787   ins_encode %{
13788     Label* L = $labl$$label;
13789     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
13790   %}
13791   ins_pipe(pipe_jcc);
13792 %}
13793 
13794 // Jump Direct Conditional - using unsigned comparison
13795 instruct jmpConU(cmpOpU cop, rFlagsRegU cmp, label labl) %{
13796   match(If cop cmp);
13797   effect(USE labl);
13798 
13799   ins_cost(300);
13800   format %{ "j$cop,u   $labl" %}
13801   size(6);
13802   ins_encode %{
13803     Label* L = $labl$$label;
13804     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
13805   %}
13806   ins_pipe(pipe_jcc);
13807 %}
13808 
13809 instruct jmpConUCF(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
13810   match(If cop cmp);
13811   effect(USE labl);
13812 
13813   ins_cost(200);
13814   format %{ "j$cop,u   $labl" %}
13815   size(6);
13816   ins_encode %{
13817     Label* L = $labl$$label;
13818     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
13819   %}
13820   ins_pipe(pipe_jcc);
13821 %}
13822 
13823 instruct jmpConUCF2(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
13824   match(If cop cmp);
13825   effect(USE labl);
13826 
13827   ins_cost(200);
13828   format %{ $$template
13829     if ($cop$$cmpcode == Assembler::notEqual) {
13830       $$emit$$"jp,u    $labl\n\t"
13831       $$emit$$"j$cop,u   $labl"
13832     } else {
13833       $$emit$$"jp,u    done\n\t"
13834       $$emit$$"j$cop,u   $labl\n\t"
13835       $$emit$$"done:"
13836     }
13837   %}
13838   ins_encode %{
13839     Label* l = $labl$$label;
13840     if ($cop$$cmpcode == Assembler::notEqual) {
13841       __ jcc(Assembler::parity, *l, false);
13842       __ jcc(Assembler::notEqual, *l, false);
13843     } else if ($cop$$cmpcode == Assembler::equal) {
13844       Label done;
13845       __ jccb(Assembler::parity, done);
13846       __ jcc(Assembler::equal, *l, false);
13847       __ bind(done);
13848     } else {
13849        ShouldNotReachHere();
13850     }
13851   %}
13852   ins_pipe(pipe_jcc);
13853 %}
13854 
13855 // ============================================================================
13856 // The 2nd slow-half of a subtype check.  Scan the subklass's 2ndary
13857 // superklass array for an instance of the superklass.  Set a hidden
13858 // internal cache on a hit (cache is checked with exposed code in
13859 // gen_subtype_check()).  Return NZ for a miss or zero for a hit.  The
13860 // encoding ALSO sets flags.
13861 
13862 instruct partialSubtypeCheck(rdi_RegP result,
13863                              rsi_RegP sub, rax_RegP super, rcx_RegI rcx,
13864                              rFlagsReg cr)
13865 %{
13866   match(Set result (PartialSubtypeCheck sub super));
13867   predicate(!UseSecondarySupersTable);
13868   effect(KILL rcx, KILL cr);
13869 
13870   ins_cost(1100);  // slightly larger than the next version
13871   format %{ "movq    rdi, [$sub + in_bytes(Klass::secondary_supers_offset())]\n\t"
13872             "movl    rcx, [rdi + Array<Klass*>::length_offset_in_bytes()]\t# length to scan\n\t"
13873             "addq    rdi, Array<Klass*>::base_offset_in_bytes()\t# Skip to start of data; set NZ in case count is zero\n\t"
13874             "repne   scasq\t# Scan *rdi++ for a match with rax while rcx--\n\t"
13875             "jne,s   miss\t\t# Missed: rdi not-zero\n\t"
13876             "movq    [$sub + in_bytes(Klass::secondary_super_cache_offset())], $super\t# Hit: update cache\n\t"
13877             "xorq    $result, $result\t\t Hit: rdi zero\n\t"
13878     "miss:\t" %}
13879 
13880   ins_encode %{
13881     Label miss;
13882     // NB: Callers may assume that, when $result is a valid register,
13883     // check_klass_subtype_slow_path_linear sets it to a nonzero
13884     // value.
13885     __ check_klass_subtype_slow_path_linear($sub$$Register, $super$$Register,
13886                                             $rcx$$Register, $result$$Register,
13887                                             nullptr, &miss,
13888                                             /*set_cond_codes:*/ true);
13889     __ xorptr($result$$Register, $result$$Register);
13890     __ bind(miss);
13891   %}
13892 
13893   ins_pipe(pipe_slow);
13894 %}
13895 
13896 // ============================================================================
13897 // Two versions of hashtable-based partialSubtypeCheck, both used when
13898 // we need to search for a super class in the secondary supers array.
13899 // The first is used when we don't know _a priori_ the class being
13900 // searched for. The second, far more common, is used when we do know:
13901 // this is used for instanceof, checkcast, and any case where C2 can
13902 // determine it by constant propagation.
13903 
13904 instruct partialSubtypeCheckVarSuper(rsi_RegP sub, rax_RegP super, rdi_RegP result,
13905                                        rdx_RegL temp1, rcx_RegL temp2, rbx_RegP temp3, r11_RegL temp4,
13906                                        rFlagsReg cr)
13907 %{
13908   match(Set result (PartialSubtypeCheck sub super));
13909   predicate(UseSecondarySupersTable);
13910   effect(KILL cr, TEMP temp1, TEMP temp2, TEMP temp3, TEMP temp4);
13911 
13912   ins_cost(1000);
13913   format %{ "partialSubtypeCheck $result, $sub, $super" %}
13914 
13915   ins_encode %{
13916     __ lookup_secondary_supers_table_var($sub$$Register, $super$$Register, $temp1$$Register, $temp2$$Register,
13917 					 $temp3$$Register, $temp4$$Register, $result$$Register);
13918   %}
13919 
13920   ins_pipe(pipe_slow);
13921 %}
13922 
13923 instruct partialSubtypeCheckConstSuper(rsi_RegP sub, rax_RegP super_reg, immP super_con, rdi_RegP result,
13924                                        rdx_RegL temp1, rcx_RegL temp2, rbx_RegP temp3, r11_RegL temp4,
13925                                        rFlagsReg cr)
13926 %{
13927   match(Set result (PartialSubtypeCheck sub (Binary super_reg super_con)));
13928   predicate(UseSecondarySupersTable);
13929   effect(KILL cr, TEMP temp1, TEMP temp2, TEMP temp3, TEMP temp4);
13930 
13931   ins_cost(700);  // smaller than the next version
13932   format %{ "partialSubtypeCheck $result, $sub, $super_reg, $super_con" %}
13933 
13934   ins_encode %{
13935     u1 super_klass_slot = ((Klass*)$super_con$$constant)->hash_slot();
13936     if (InlineSecondarySupersTest) {
13937       __ lookup_secondary_supers_table_const($sub$$Register, $super_reg$$Register, $temp1$$Register, $temp2$$Register,
13938                                        $temp3$$Register, $temp4$$Register, $result$$Register,
13939                                        super_klass_slot);
13940     } else {
13941       __ call(RuntimeAddress(StubRoutines::lookup_secondary_supers_table_stub(super_klass_slot)));
13942     }
13943   %}
13944 
13945   ins_pipe(pipe_slow);
13946 %}
13947 
13948 // ============================================================================
13949 // Branch Instructions -- short offset versions
13950 //
13951 // These instructions are used to replace jumps of a long offset (the default
13952 // match) with jumps of a shorter offset.  These instructions are all tagged
13953 // with the ins_short_branch attribute, which causes the ADLC to suppress the
13954 // match rules in general matching.  Instead, the ADLC generates a conversion
13955 // method in the MachNode which can be used to do in-place replacement of the
13956 // long variant with the shorter variant.  The compiler will determine if a
13957 // branch can be taken by the is_short_branch_offset() predicate in the machine
13958 // specific code section of the file.
13959 
13960 // Jump Direct - Label defines a relative address from JMP+1
13961 instruct jmpDir_short(label labl) %{
13962   match(Goto);
13963   effect(USE labl);
13964 
13965   ins_cost(300);
13966   format %{ "jmp,s   $labl" %}
13967   size(2);
13968   ins_encode %{
13969     Label* L = $labl$$label;
13970     __ jmpb(*L);
13971   %}
13972   ins_pipe(pipe_jmp);
13973   ins_short_branch(1);
13974 %}
13975 
13976 // Jump Direct Conditional - Label defines a relative address from Jcc+1
13977 instruct jmpCon_short(cmpOp cop, rFlagsReg cr, label labl) %{
13978   match(If cop cr);
13979   effect(USE labl);
13980 
13981   ins_cost(300);
13982   format %{ "j$cop,s   $labl" %}
13983   size(2);
13984   ins_encode %{
13985     Label* L = $labl$$label;
13986     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
13987   %}
13988   ins_pipe(pipe_jcc);
13989   ins_short_branch(1);
13990 %}
13991 
13992 // Jump Direct Conditional - Label defines a relative address from Jcc+1
13993 instruct jmpLoopEnd_short(cmpOp cop, rFlagsReg cr, label labl) %{
13994   match(CountedLoopEnd cop cr);
13995   effect(USE labl);
13996 
13997   ins_cost(300);
13998   format %{ "j$cop,s   $labl\t# loop end" %}
13999   size(2);
14000   ins_encode %{
14001     Label* L = $labl$$label;
14002     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
14003   %}
14004   ins_pipe(pipe_jcc);
14005   ins_short_branch(1);
14006 %}
14007 
14008 // Jump Direct Conditional - using unsigned comparison
14009 instruct jmpConU_short(cmpOpU cop, rFlagsRegU cmp, label labl) %{
14010   match(If cop cmp);
14011   effect(USE labl);
14012 
14013   ins_cost(300);
14014   format %{ "j$cop,us  $labl" %}
14015   size(2);
14016   ins_encode %{
14017     Label* L = $labl$$label;
14018     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
14019   %}
14020   ins_pipe(pipe_jcc);
14021   ins_short_branch(1);
14022 %}
14023 
14024 instruct jmpConUCF_short(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
14025   match(If cop cmp);
14026   effect(USE labl);
14027 
14028   ins_cost(300);
14029   format %{ "j$cop,us  $labl" %}
14030   size(2);
14031   ins_encode %{
14032     Label* L = $labl$$label;
14033     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
14034   %}
14035   ins_pipe(pipe_jcc);
14036   ins_short_branch(1);
14037 %}
14038 
14039 instruct jmpConUCF2_short(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
14040   match(If cop cmp);
14041   effect(USE labl);
14042 
14043   ins_cost(300);
14044   format %{ $$template
14045     if ($cop$$cmpcode == Assembler::notEqual) {
14046       $$emit$$"jp,u,s  $labl\n\t"
14047       $$emit$$"j$cop,u,s  $labl"
14048     } else {
14049       $$emit$$"jp,u,s  done\n\t"
14050       $$emit$$"j$cop,u,s  $labl\n\t"
14051       $$emit$$"done:"
14052     }
14053   %}
14054   size(4);
14055   ins_encode %{
14056     Label* l = $labl$$label;
14057     if ($cop$$cmpcode == Assembler::notEqual) {
14058       __ jccb(Assembler::parity, *l);
14059       __ jccb(Assembler::notEqual, *l);
14060     } else if ($cop$$cmpcode == Assembler::equal) {
14061       Label done;
14062       __ jccb(Assembler::parity, done);
14063       __ jccb(Assembler::equal, *l);
14064       __ bind(done);
14065     } else {
14066        ShouldNotReachHere();
14067     }
14068   %}
14069   ins_pipe(pipe_jcc);
14070   ins_short_branch(1);
14071 %}
14072 
14073 // ============================================================================
14074 // inlined locking and unlocking
14075 
14076 instruct cmpFastLock(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI tmp, rRegP scr) %{
14077   predicate(LockingMode != LM_LIGHTWEIGHT);
14078   match(Set cr (FastLock object box));
14079   effect(TEMP tmp, TEMP scr, USE_KILL box);
14080   ins_cost(300);
14081   format %{ "fastlock $object,$box\t! kills $box,$tmp,$scr" %}
14082   ins_encode %{
14083     __ fast_lock($object$$Register, $box$$Register, $tmp$$Register,
14084                  $scr$$Register, noreg, noreg, r15_thread, nullptr);
14085   %}
14086   ins_pipe(pipe_slow);
14087 %}
14088 
14089 instruct cmpFastUnlock(rFlagsReg cr, rRegP object, rax_RegP box, rRegP tmp) %{
14090   predicate(LockingMode != LM_LIGHTWEIGHT);
14091   match(Set cr (FastUnlock object box));
14092   effect(TEMP tmp, USE_KILL box);
14093   ins_cost(300);
14094   format %{ "fastunlock $object,$box\t! kills $box,$tmp" %}
14095   ins_encode %{
14096     __ fast_unlock($object$$Register, $box$$Register, $tmp$$Register);
14097   %}
14098   ins_pipe(pipe_slow);
14099 %}
14100 
14101 instruct cmpFastLockLightweight(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI rax_reg, rRegP tmp) %{
14102   predicate(LockingMode == LM_LIGHTWEIGHT);
14103   match(Set cr (FastLock object box));
14104   effect(TEMP rax_reg, TEMP tmp, USE_KILL box);
14105   ins_cost(300);
14106   format %{ "fastlock $object,$box\t! kills $box,$rax_reg,$tmp" %}
14107   ins_encode %{
14108     __ fast_lock_lightweight($object$$Register, $box$$Register, $rax_reg$$Register, $tmp$$Register, r15_thread);
14109   %}
14110   ins_pipe(pipe_slow);
14111 %}
14112 
14113 instruct cmpFastUnlockLightweight(rFlagsReg cr, rRegP object, rax_RegP rax_reg, rRegP tmp) %{
14114   predicate(LockingMode == LM_LIGHTWEIGHT);
14115   match(Set cr (FastUnlock object rax_reg));
14116   effect(TEMP tmp, USE_KILL rax_reg);
14117   ins_cost(300);
14118   format %{ "fastunlock $object,$rax_reg\t! kills $rax_reg,$tmp" %}
14119   ins_encode %{
14120     __ fast_unlock_lightweight($object$$Register, $rax_reg$$Register, $tmp$$Register, r15_thread);
14121   %}
14122   ins_pipe(pipe_slow);
14123 %}
14124 
14125 
14126 // ============================================================================
14127 // Safepoint Instructions
14128 instruct safePoint_poll_tls(rFlagsReg cr, rRegP poll)
14129 %{
14130   match(SafePoint poll);
14131   effect(KILL cr, USE poll);
14132 
14133   format %{ "testl   rax, [$poll]\t"
14134             "# Safepoint: poll for GC" %}
14135   ins_cost(125);
14136   ins_encode %{
14137     __ relocate(relocInfo::poll_type);
14138     address pre_pc = __ pc();
14139     __ testl(rax, Address($poll$$Register, 0));
14140     assert(nativeInstruction_at(pre_pc)->is_safepoint_poll(), "must emit test %%eax [reg]");
14141   %}
14142   ins_pipe(ialu_reg_mem);
14143 %}
14144 
14145 instruct mask_all_evexL(kReg dst, rRegL src) %{
14146   match(Set dst (MaskAll src));
14147   format %{ "mask_all_evexL $dst, $src \t! mask all operation" %}
14148   ins_encode %{
14149     int mask_len = Matcher::vector_length(this);
14150     __ vector_maskall_operation($dst$$KRegister, $src$$Register, mask_len);
14151   %}
14152   ins_pipe( pipe_slow );
14153 %}
14154 
14155 instruct mask_all_evexI_GT32(kReg dst, rRegI src, rRegL tmp) %{
14156   predicate(Matcher::vector_length(n) > 32);
14157   match(Set dst (MaskAll src));
14158   effect(TEMP tmp);
14159   format %{ "mask_all_evexI_GT32 $dst, $src \t! using $tmp as TEMP" %}
14160   ins_encode %{
14161     int mask_len = Matcher::vector_length(this);
14162     __ movslq($tmp$$Register, $src$$Register);
14163     __ vector_maskall_operation($dst$$KRegister, $tmp$$Register, mask_len);
14164   %}
14165   ins_pipe( pipe_slow );
14166 %}
14167 
14168 // ============================================================================
14169 // Procedure Call/Return Instructions
14170 // Call Java Static Instruction
14171 // Note: If this code changes, the corresponding ret_addr_offset() and
14172 //       compute_padding() functions will have to be adjusted.
14173 instruct CallStaticJavaDirect(method meth) %{
14174   match(CallStaticJava);
14175   effect(USE meth);
14176 
14177   ins_cost(300);
14178   format %{ "call,static " %}
14179   opcode(0xE8); /* E8 cd */
14180   ins_encode(clear_avx, Java_Static_Call(meth), call_epilog);
14181   ins_pipe(pipe_slow);
14182   ins_alignment(4);
14183 %}
14184 
14185 // Call Java Dynamic Instruction
14186 // Note: If this code changes, the corresponding ret_addr_offset() and
14187 //       compute_padding() functions will have to be adjusted.
14188 instruct CallDynamicJavaDirect(method meth)
14189 %{
14190   match(CallDynamicJava);
14191   effect(USE meth);
14192 
14193   ins_cost(300);
14194   format %{ "movq    rax, #Universe::non_oop_word()\n\t"
14195             "call,dynamic " %}
14196   ins_encode(clear_avx, Java_Dynamic_Call(meth), call_epilog);
14197   ins_pipe(pipe_slow);
14198   ins_alignment(4);
14199 %}
14200 
14201 // Call Runtime Instruction
14202 instruct CallRuntimeDirect(method meth)
14203 %{
14204   match(CallRuntime);
14205   effect(USE meth);
14206 
14207   ins_cost(300);
14208   format %{ "call,runtime " %}
14209   ins_encode(clear_avx, Java_To_Runtime(meth));
14210   ins_pipe(pipe_slow);
14211 %}
14212 
14213 // Call runtime without safepoint
14214 instruct CallLeafDirect(method meth)
14215 %{
14216   match(CallLeaf);
14217   effect(USE meth);
14218 
14219   ins_cost(300);
14220   format %{ "call_leaf,runtime " %}
14221   ins_encode(clear_avx, Java_To_Runtime(meth));
14222   ins_pipe(pipe_slow);
14223 %}
14224 
14225 // Call runtime without safepoint and with vector arguments
14226 instruct CallLeafDirectVector(method meth)
14227 %{
14228   match(CallLeafVector);
14229   effect(USE meth);
14230 
14231   ins_cost(300);
14232   format %{ "call_leaf,vector " %}
14233   ins_encode(Java_To_Runtime(meth));
14234   ins_pipe(pipe_slow);
14235 %}
14236 
14237 // Call runtime without safepoint
14238 instruct CallLeafNoFPDirect(method meth)
14239 %{
14240   match(CallLeafNoFP);
14241   effect(USE meth);
14242 
14243   ins_cost(300);
14244   format %{ "call_leaf_nofp,runtime " %}
14245   ins_encode(clear_avx, Java_To_Runtime(meth));
14246   ins_pipe(pipe_slow);
14247 %}
14248 
14249 // Return Instruction
14250 // Remove the return address & jump to it.
14251 // Notice: We always emit a nop after a ret to make sure there is room
14252 // for safepoint patching
14253 instruct Ret()
14254 %{
14255   match(Return);
14256 
14257   format %{ "ret" %}
14258   ins_encode %{
14259     __ ret(0);
14260   %}
14261   ins_pipe(pipe_jmp);
14262 %}
14263 
14264 // Tail Call; Jump from runtime stub to Java code.
14265 // Also known as an 'interprocedural jump'.
14266 // Target of jump will eventually return to caller.
14267 // TailJump below removes the return address.
14268 // Don't use rbp for 'jump_target' because a MachEpilogNode has already been
14269 // emitted just above the TailCall which has reset rbp to the caller state.
14270 instruct TailCalljmpInd(no_rbp_RegP jump_target, rbx_RegP method_ptr)
14271 %{
14272   match(TailCall jump_target method_ptr);
14273 
14274   ins_cost(300);
14275   format %{ "jmp     $jump_target\t# rbx holds method" %}
14276   ins_encode %{
14277     __ jmp($jump_target$$Register);
14278   %}
14279   ins_pipe(pipe_jmp);
14280 %}
14281 
14282 // Tail Jump; remove the return address; jump to target.
14283 // TailCall above leaves the return address around.
14284 instruct tailjmpInd(no_rbp_RegP jump_target, rax_RegP ex_oop)
14285 %{
14286   match(TailJump jump_target ex_oop);
14287 
14288   ins_cost(300);
14289   format %{ "popq    rdx\t# pop return address\n\t"
14290             "jmp     $jump_target" %}
14291   ins_encode %{
14292     __ popq(as_Register(RDX_enc));
14293     __ jmp($jump_target$$Register);
14294   %}
14295   ins_pipe(pipe_jmp);
14296 %}
14297 
14298 // Forward exception.
14299 instruct ForwardExceptionjmp()
14300 %{
14301   match(ForwardException);
14302 
14303   format %{ "jmp     forward_exception_stub" %}
14304   ins_encode %{
14305     __ jump(RuntimeAddress(StubRoutines::forward_exception_entry()), noreg);
14306   %}
14307   ins_pipe(pipe_jmp);
14308 %}
14309 
14310 // Create exception oop: created by stack-crawling runtime code.
14311 // Created exception is now available to this handler, and is setup
14312 // just prior to jumping to this handler.  No code emitted.
14313 instruct CreateException(rax_RegP ex_oop)
14314 %{
14315   match(Set ex_oop (CreateEx));
14316 
14317   size(0);
14318   // use the following format syntax
14319   format %{ "# exception oop is in rax; no code emitted" %}
14320   ins_encode();
14321   ins_pipe(empty);
14322 %}
14323 
14324 // Rethrow exception:
14325 // The exception oop will come in the first argument position.
14326 // Then JUMP (not call) to the rethrow stub code.
14327 instruct RethrowException()
14328 %{
14329   match(Rethrow);
14330 
14331   // use the following format syntax
14332   format %{ "jmp     rethrow_stub" %}
14333   ins_encode %{
14334     __ jump(RuntimeAddress(OptoRuntime::rethrow_stub()), noreg);
14335   %}
14336   ins_pipe(pipe_jmp);
14337 %}
14338 
14339 // ============================================================================
14340 // This name is KNOWN by the ADLC and cannot be changed.
14341 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
14342 // for this guy.
14343 instruct tlsLoadP(r15_RegP dst) %{
14344   match(Set dst (ThreadLocal));
14345   effect(DEF dst);
14346 
14347   size(0);
14348   format %{ "# TLS is in R15" %}
14349   ins_encode( /*empty encoding*/ );
14350   ins_pipe(ialu_reg_reg);
14351 %}
14352 
14353 
14354 //----------PEEPHOLE RULES-----------------------------------------------------
14355 // These must follow all instruction definitions as they use the names
14356 // defined in the instructions definitions.
14357 //
14358 // peeppredicate ( rule_predicate );
14359 // // the predicate unless which the peephole rule will be ignored
14360 //
14361 // peepmatch ( root_instr_name [preceding_instruction]* );
14362 //
14363 // peepprocedure ( procedure_name );
14364 // // provide a procedure name to perform the optimization, the procedure should
14365 // // reside in the architecture dependent peephole file, the method has the
14366 // // signature of MachNode* (Block*, int, PhaseRegAlloc*, (MachNode*)(*)(), int...)
14367 // // with the arguments being the basic block, the current node index inside the
14368 // // block, the register allocator, the functions upon invoked return a new node
14369 // // defined in peepreplace, and the rules of the nodes appearing in the
14370 // // corresponding peepmatch, the function return true if successful, else
14371 // // return false
14372 //
14373 // peepconstraint %{
14374 // (instruction_number.operand_name relational_op instruction_number.operand_name
14375 //  [, ...] );
14376 // // instruction numbers are zero-based using left to right order in peepmatch
14377 //
14378 // peepreplace ( instr_name  ( [instruction_number.operand_name]* ) );
14379 // // provide an instruction_number.operand_name for each operand that appears
14380 // // in the replacement instruction's match rule
14381 //
14382 // ---------VM FLAGS---------------------------------------------------------
14383 //
14384 // All peephole optimizations can be turned off using -XX:-OptoPeephole
14385 //
14386 // Each peephole rule is given an identifying number starting with zero and
14387 // increasing by one in the order seen by the parser.  An individual peephole
14388 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
14389 // on the command-line.
14390 //
14391 // ---------CURRENT LIMITATIONS----------------------------------------------
14392 //
14393 // Only transformations inside a basic block (do we need more for peephole)
14394 //
14395 // ---------EXAMPLE----------------------------------------------------------
14396 //
14397 // // pertinent parts of existing instructions in architecture description
14398 // instruct movI(rRegI dst, rRegI src)
14399 // %{
14400 //   match(Set dst (CopyI src));
14401 // %}
14402 //
14403 // instruct incI_rReg(rRegI dst, immI_1 src, rFlagsReg cr)
14404 // %{
14405 //   match(Set dst (AddI dst src));
14406 //   effect(KILL cr);
14407 // %}
14408 //
14409 // instruct leaI_rReg_immI(rRegI dst, immI_1 src)
14410 // %{
14411 //   match(Set dst (AddI dst src));
14412 // %}
14413 //
14414 // 1. Simple replacement
14415 // - Only match adjacent instructions in same basic block
14416 // - Only equality constraints
14417 // - Only constraints between operands, not (0.dest_reg == RAX_enc)
14418 // - Only one replacement instruction
14419 //
14420 // // Change (inc mov) to lea
14421 // peephole %{
14422 //   // lea should only be emitted when beneficial
14423 //   peeppredicate( VM_Version::supports_fast_2op_lea() );
14424 //   // increment preceded by register-register move
14425 //   peepmatch ( incI_rReg movI );
14426 //   // require that the destination register of the increment
14427 //   // match the destination register of the move
14428 //   peepconstraint ( 0.dst == 1.dst );
14429 //   // construct a replacement instruction that sets
14430 //   // the destination to ( move's source register + one )
14431 //   peepreplace ( leaI_rReg_immI( 0.dst 1.src 0.src ) );
14432 // %}
14433 //
14434 // 2. Procedural replacement
14435 // - More flexible finding relevent nodes
14436 // - More flexible constraints
14437 // - More flexible transformations
14438 // - May utilise architecture-dependent API more effectively
14439 // - Currently only one replacement instruction due to adlc parsing capabilities
14440 //
14441 // // Change (inc mov) to lea
14442 // peephole %{
14443 //   // lea should only be emitted when beneficial
14444 //   peeppredicate( VM_Version::supports_fast_2op_lea() );
14445 //   // the rule numbers of these nodes inside are passed into the function below
14446 //   peepmatch ( incI_rReg movI );
14447 //   // the method that takes the responsibility of transformation
14448 //   peepprocedure ( inc_mov_to_lea );
14449 //   // the replacement is a leaI_rReg_immI, a lambda upon invoked creating this
14450 //   // node is passed into the function above
14451 //   peepreplace ( leaI_rReg_immI() );
14452 // %}
14453 
14454 // These instructions is not matched by the matcher but used by the peephole
14455 instruct leaI_rReg_rReg_peep(rRegI dst, rRegI src1, rRegI src2)
14456 %{
14457   predicate(false);
14458   match(Set dst (AddI src1 src2));
14459   format %{ "leal    $dst, [$src1 + $src2]" %}
14460   ins_encode %{
14461     Register dst = $dst$$Register;
14462     Register src1 = $src1$$Register;
14463     Register src2 = $src2$$Register;
14464     if (src1 != rbp && src1 != r13) {
14465       __ leal(dst, Address(src1, src2, Address::times_1));
14466     } else {
14467       assert(src2 != rbp && src2 != r13, "");
14468       __ leal(dst, Address(src2, src1, Address::times_1));
14469     }
14470   %}
14471   ins_pipe(ialu_reg_reg);
14472 %}
14473 
14474 instruct leaI_rReg_immI_peep(rRegI dst, rRegI src1, immI src2)
14475 %{
14476   predicate(false);
14477   match(Set dst (AddI src1 src2));
14478   format %{ "leal    $dst, [$src1 + $src2]" %}
14479   ins_encode %{
14480     __ leal($dst$$Register, Address($src1$$Register, $src2$$constant));
14481   %}
14482   ins_pipe(ialu_reg_reg);
14483 %}
14484 
14485 instruct leaI_rReg_immI2_peep(rRegI dst, rRegI src, immI2 shift)
14486 %{
14487   predicate(false);
14488   match(Set dst (LShiftI src shift));
14489   format %{ "leal    $dst, [$src << $shift]" %}
14490   ins_encode %{
14491     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($shift$$constant);
14492     Register src = $src$$Register;
14493     if (scale == Address::times_2 && src != rbp && src != r13) {
14494       __ leal($dst$$Register, Address(src, src, Address::times_1));
14495     } else {
14496       __ leal($dst$$Register, Address(noreg, src, scale));
14497     }
14498   %}
14499   ins_pipe(ialu_reg_reg);
14500 %}
14501 
14502 instruct leaL_rReg_rReg_peep(rRegL dst, rRegL src1, rRegL src2)
14503 %{
14504   predicate(false);
14505   match(Set dst (AddL src1 src2));
14506   format %{ "leaq    $dst, [$src1 + $src2]" %}
14507   ins_encode %{
14508     Register dst = $dst$$Register;
14509     Register src1 = $src1$$Register;
14510     Register src2 = $src2$$Register;
14511     if (src1 != rbp && src1 != r13) {
14512       __ leaq(dst, Address(src1, src2, Address::times_1));
14513     } else {
14514       assert(src2 != rbp && src2 != r13, "");
14515       __ leaq(dst, Address(src2, src1, Address::times_1));
14516     }
14517   %}
14518   ins_pipe(ialu_reg_reg);
14519 %}
14520 
14521 instruct leaL_rReg_immL32_peep(rRegL dst, rRegL src1, immL32 src2)
14522 %{
14523   predicate(false);
14524   match(Set dst (AddL src1 src2));
14525   format %{ "leaq    $dst, [$src1 + $src2]" %}
14526   ins_encode %{
14527     __ leaq($dst$$Register, Address($src1$$Register, $src2$$constant));
14528   %}
14529   ins_pipe(ialu_reg_reg);
14530 %}
14531 
14532 instruct leaL_rReg_immI2_peep(rRegL dst, rRegL src, immI2 shift)
14533 %{
14534   predicate(false);
14535   match(Set dst (LShiftL src shift));
14536   format %{ "leaq    $dst, [$src << $shift]" %}
14537   ins_encode %{
14538     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($shift$$constant);
14539     Register src = $src$$Register;
14540     if (scale == Address::times_2 && src != rbp && src != r13) {
14541       __ leaq($dst$$Register, Address(src, src, Address::times_1));
14542     } else {
14543       __ leaq($dst$$Register, Address(noreg, src, scale));
14544     }
14545   %}
14546   ins_pipe(ialu_reg_reg);
14547 %}
14548 
14549 // These peephole rules replace mov + I pairs (where I is one of {add, inc, dec,
14550 // sal}) with lea instructions. The {add, sal} rules are beneficial in
14551 // processors with at least partial ALU support for lea
14552 // (supports_fast_2op_lea()), whereas the {inc, dec} rules are only generally
14553 // beneficial for processors with full ALU support
14554 // (VM_Version::supports_fast_3op_lea()) and Intel Cascade Lake.
14555 
14556 peephole
14557 %{
14558   peeppredicate(VM_Version::supports_fast_2op_lea());
14559   peepmatch (addI_rReg);
14560   peepprocedure (lea_coalesce_reg);
14561   peepreplace (leaI_rReg_rReg_peep());
14562 %}
14563 
14564 peephole
14565 %{
14566   peeppredicate(VM_Version::supports_fast_2op_lea());
14567   peepmatch (addI_rReg_imm);
14568   peepprocedure (lea_coalesce_imm);
14569   peepreplace (leaI_rReg_immI_peep());
14570 %}
14571 
14572 peephole
14573 %{
14574   peeppredicate(VM_Version::supports_fast_3op_lea() ||
14575                 VM_Version::is_intel_cascade_lake());
14576   peepmatch (incI_rReg);
14577   peepprocedure (lea_coalesce_imm);
14578   peepreplace (leaI_rReg_immI_peep());
14579 %}
14580 
14581 peephole
14582 %{
14583   peeppredicate(VM_Version::supports_fast_3op_lea() ||
14584                 VM_Version::is_intel_cascade_lake());
14585   peepmatch (decI_rReg);
14586   peepprocedure (lea_coalesce_imm);
14587   peepreplace (leaI_rReg_immI_peep());
14588 %}
14589 
14590 peephole
14591 %{
14592   peeppredicate(VM_Version::supports_fast_2op_lea());
14593   peepmatch (salI_rReg_immI2);
14594   peepprocedure (lea_coalesce_imm);
14595   peepreplace (leaI_rReg_immI2_peep());
14596 %}
14597 
14598 peephole
14599 %{
14600   peeppredicate(VM_Version::supports_fast_2op_lea());
14601   peepmatch (addL_rReg);
14602   peepprocedure (lea_coalesce_reg);
14603   peepreplace (leaL_rReg_rReg_peep());
14604 %}
14605 
14606 peephole
14607 %{
14608   peeppredicate(VM_Version::supports_fast_2op_lea());
14609   peepmatch (addL_rReg_imm);
14610   peepprocedure (lea_coalesce_imm);
14611   peepreplace (leaL_rReg_immL32_peep());
14612 %}
14613 
14614 peephole
14615 %{
14616   peeppredicate(VM_Version::supports_fast_3op_lea() ||
14617                 VM_Version::is_intel_cascade_lake());
14618   peepmatch (incL_rReg);
14619   peepprocedure (lea_coalesce_imm);
14620   peepreplace (leaL_rReg_immL32_peep());
14621 %}
14622 
14623 peephole
14624 %{
14625   peeppredicate(VM_Version::supports_fast_3op_lea() ||
14626                 VM_Version::is_intel_cascade_lake());
14627   peepmatch (decL_rReg);
14628   peepprocedure (lea_coalesce_imm);
14629   peepreplace (leaL_rReg_immL32_peep());
14630 %}
14631 
14632 peephole
14633 %{
14634   peeppredicate(VM_Version::supports_fast_2op_lea());
14635   peepmatch (salL_rReg_immI2);
14636   peepprocedure (lea_coalesce_imm);
14637   peepreplace (leaL_rReg_immI2_peep());
14638 %}
14639 
14640 peephole
14641 %{
14642   peepmatch (leaPCompressedOopOffset);
14643   peepprocedure (lea_remove_redundant);
14644 %}
14645 
14646 peephole
14647 %{
14648   peepmatch (leaP8Narrow);
14649   peepprocedure (lea_remove_redundant);
14650 %}
14651 
14652 peephole
14653 %{
14654   peepmatch (leaP32Narrow);
14655   peepprocedure (lea_remove_redundant);
14656 %}
14657 
14658 // These peephole rules matches instructions which set flags and are followed by a testI/L_reg
14659 // The test instruction is redudanent in case the downstream instuctions (like JCC or CMOV) only use flags that are already set by the previous instruction
14660 
14661 //int variant
14662 peephole
14663 %{
14664   peepmatch (testI_reg);
14665   peepprocedure (test_may_remove);
14666 %}
14667 
14668 //long variant
14669 peephole
14670 %{
14671   peepmatch (testL_reg);
14672   peepprocedure (test_may_remove);
14673 %}
14674 
14675 
14676 //----------SMARTSPILL RULES---------------------------------------------------
14677 // These must follow all instruction definitions as they use the names
14678 // defined in the instructions definitions.