1 //
   2 // Copyright (c) 2003, 2018, Oracle and/or its affiliates. All rights reserved.
   3 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4 //
   5 // This code is free software; you can redistribute it and/or modify it
   6 // under the terms of the GNU General Public License version 2 only, as
   7 // published by the Free Software Foundation.
   8 //
   9 // This code is distributed in the hope that it will be useful, but WITHOUT
  10 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11 // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12 // version 2 for more details (a copy is included in the LICENSE file that
  13 // accompanied this code).
  14 //
  15 // You should have received a copy of the GNU General Public License version
  16 // 2 along with this work; if not, write to the Free Software Foundation,
  17 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18 //
  19 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20 // or visit www.oracle.com if you need additional information or have any
  21 // questions.
  22 //
  23 //
  24 
  25 // AMD64 Architecture Description File
  26 
  27 //----------REGISTER DEFINITION BLOCK------------------------------------------
  28 // This information is used by the matcher and the register allocator to
  29 // describe individual registers and classes of registers within the target
  30 // archtecture.
  31 
  32 register %{
  33 //----------Architecture Description Register Definitions----------------------
  34 // General Registers
  35 // "reg_def"  name ( register save type, C convention save type,
  36 //                   ideal register type, encoding );
  37 // Register Save Types:
  38 //
  39 // NS  = No-Save:       The register allocator assumes that these registers
  40 //                      can be used without saving upon entry to the method, &
  41 //                      that they do not need to be saved at call sites.
  42 //
  43 // SOC = Save-On-Call:  The register allocator assumes that these registers
  44 //                      can be used without saving upon entry to the method,
  45 //                      but that they must be saved at call sites.
  46 //
  47 // SOE = Save-On-Entry: The register allocator assumes that these registers
  48 //                      must be saved before using them upon entry to the
  49 //                      method, but they do not need to be saved at call
  50 //                      sites.
  51 //
  52 // AS  = Always-Save:   The register allocator assumes that these registers
  53 //                      must be saved before using them upon entry to the
  54 //                      method, & that they must be saved at call sites.
  55 //
  56 // Ideal Register Type is used to determine how to save & restore a
  57 // register.  Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
  58 // spilled with LoadP/StoreP.  If the register supports both, use Op_RegI.
  59 //
  60 // The encoding number is the actual bit-pattern placed into the opcodes.
  61 
  62 // General Registers
  63 // R8-R15 must be encoded with REX.  (RSP, RBP, RSI, RDI need REX when
  64 // used as byte registers)
  65 
  66 // Previously set RBX, RSI, and RDI as save-on-entry for java code
  67 // Turn off SOE in java-code due to frequent use of uncommon-traps.
  68 // Now that allocator is better, turn on RSI and RDI as SOE registers.
  69 
  70 reg_def RAX  (SOC, SOC, Op_RegI,  0, rax->as_VMReg());
  71 reg_def RAX_H(SOC, SOC, Op_RegI,  0, rax->as_VMReg()->next());
  72 
  73 reg_def RCX  (SOC, SOC, Op_RegI,  1, rcx->as_VMReg());
  74 reg_def RCX_H(SOC, SOC, Op_RegI,  1, rcx->as_VMReg()->next());
  75 
  76 reg_def RDX  (SOC, SOC, Op_RegI,  2, rdx->as_VMReg());
  77 reg_def RDX_H(SOC, SOC, Op_RegI,  2, rdx->as_VMReg()->next());
  78 
  79 reg_def RBX  (SOC, SOE, Op_RegI,  3, rbx->as_VMReg());
  80 reg_def RBX_H(SOC, SOE, Op_RegI,  3, rbx->as_VMReg()->next());
  81 
  82 reg_def RSP  (NS,  NS,  Op_RegI,  4, rsp->as_VMReg());
  83 reg_def RSP_H(NS,  NS,  Op_RegI,  4, rsp->as_VMReg()->next());
  84 
  85 // now that adapter frames are gone RBP is always saved and restored by the prolog/epilog code
  86 reg_def RBP  (NS, SOE, Op_RegI,  5, rbp->as_VMReg());
  87 reg_def RBP_H(NS, SOE, Op_RegI,  5, rbp->as_VMReg()->next());
  88 
  89 #ifdef _WIN64
  90 
  91 reg_def RSI  (SOC, SOE, Op_RegI,  6, rsi->as_VMReg());
  92 reg_def RSI_H(SOC, SOE, Op_RegI,  6, rsi->as_VMReg()->next());
  93 
  94 reg_def RDI  (SOC, SOE, Op_RegI,  7, rdi->as_VMReg());
  95 reg_def RDI_H(SOC, SOE, Op_RegI,  7, rdi->as_VMReg()->next());
  96 
  97 #else
  98 
  99 reg_def RSI  (SOC, SOC, Op_RegI,  6, rsi->as_VMReg());
 100 reg_def RSI_H(SOC, SOC, Op_RegI,  6, rsi->as_VMReg()->next());
 101 
 102 reg_def RDI  (SOC, SOC, Op_RegI,  7, rdi->as_VMReg());
 103 reg_def RDI_H(SOC, SOC, Op_RegI,  7, rdi->as_VMReg()->next());
 104 
 105 #endif
 106 
 107 reg_def R8   (SOC, SOC, Op_RegI,  8, r8->as_VMReg());
 108 reg_def R8_H (SOC, SOC, Op_RegI,  8, r8->as_VMReg()->next());
 109 
 110 reg_def R9   (SOC, SOC, Op_RegI,  9, r9->as_VMReg());
 111 reg_def R9_H (SOC, SOC, Op_RegI,  9, r9->as_VMReg()->next());
 112 
 113 reg_def R10  (SOC, SOC, Op_RegI, 10, r10->as_VMReg());
 114 reg_def R10_H(SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
 115 
 116 reg_def R11  (SOC, SOC, Op_RegI, 11, r11->as_VMReg());
 117 reg_def R11_H(SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
 118 
 119 reg_def R12  (SOC, SOE, Op_RegI, 12, r12->as_VMReg());
 120 reg_def R12_H(SOC, SOE, Op_RegI, 12, r12->as_VMReg()->next());
 121 
 122 reg_def R13  (SOC, SOE, Op_RegI, 13, r13->as_VMReg());
 123 reg_def R13_H(SOC, SOE, Op_RegI, 13, r13->as_VMReg()->next());
 124 
 125 reg_def R14  (SOC, SOE, Op_RegI, 14, r14->as_VMReg());
 126 reg_def R14_H(SOC, SOE, Op_RegI, 14, r14->as_VMReg()->next());
 127 
 128 reg_def R15  (SOC, SOE, Op_RegI, 15, r15->as_VMReg());
 129 reg_def R15_H(SOC, SOE, Op_RegI, 15, r15->as_VMReg()->next());
 130 
 131 
 132 // Floating Point Registers
 133 
 134 // Specify priority of register selection within phases of register
 135 // allocation.  Highest priority is first.  A useful heuristic is to
 136 // give registers a low priority when they are required by machine
 137 // instructions, like EAX and EDX on I486, and choose no-save registers
 138 // before save-on-call, & save-on-call before save-on-entry.  Registers
 139 // which participate in fixed calling sequences should come last.
 140 // Registers which are used as pairs must fall on an even boundary.
 141 
 142 alloc_class chunk0(R10,         R10_H,
 143                    R11,         R11_H,
 144                    R8,          R8_H,
 145                    R9,          R9_H,
 146                    R12,         R12_H,
 147                    RCX,         RCX_H,
 148                    RBX,         RBX_H,
 149                    RDI,         RDI_H,
 150                    RDX,         RDX_H,
 151                    RSI,         RSI_H,
 152                    RAX,         RAX_H,
 153                    RBP,         RBP_H,
 154                    R13,         R13_H,
 155                    R14,         R14_H,
 156                    R15,         R15_H,
 157                    RSP,         RSP_H);
 158 
 159 
 160 //----------Architecture Description Register Classes--------------------------
 161 // Several register classes are automatically defined based upon information in
 162 // this architecture description.
 163 // 1) reg_class inline_cache_reg           ( /* as def'd in frame section */ )
 164 // 2) reg_class compiler_method_oop_reg    ( /* as def'd in frame section */ )
 165 // 2) reg_class interpreter_method_oop_reg ( /* as def'd in frame section */ )
 166 // 3) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
 167 //
 168 
 169 // Empty register class.
 170 reg_class no_reg();
 171 
 172 // Class for all pointer registers (including RSP and RBP)
 173 reg_class any_reg_with_rbp(RAX, RAX_H,
 174                            RDX, RDX_H,
 175                            RBP, RBP_H,
 176                            RDI, RDI_H,
 177                            RSI, RSI_H,
 178                            RCX, RCX_H,
 179                            RBX, RBX_H,
 180                            RSP, RSP_H,
 181                            R8,  R8_H,
 182                            R9,  R9_H,
 183                            R10, R10_H,
 184                            R11, R11_H,
 185                            R12, R12_H,
 186                            R13, R13_H,
 187                            R14, R14_H,
 188                            R15, R15_H);
 189 
 190 // Class for all pointer registers (including RSP, but excluding RBP)
 191 reg_class any_reg_no_rbp(RAX, RAX_H,
 192                          RDX, RDX_H,
 193                          RDI, RDI_H,
 194                          RSI, RSI_H,
 195                          RCX, RCX_H,
 196                          RBX, RBX_H,
 197                          RSP, RSP_H,
 198                          R8,  R8_H,
 199                          R9,  R9_H,
 200                          R10, R10_H,
 201                          R11, R11_H,
 202                          R12, R12_H,
 203                          R13, R13_H,
 204                          R14, R14_H,
 205                          R15, R15_H);
 206 
 207 // Dynamic register class that selects at runtime between register classes
 208 // any_reg_no_rbp and any_reg_with_rbp (depending on the value of the flag PreserveFramePointer).
 209 // Equivalent to: return PreserveFramePointer ? any_reg_no_rbp : any_reg_with_rbp;
 210 reg_class_dynamic any_reg(any_reg_no_rbp, any_reg_with_rbp, %{ PreserveFramePointer %});
 211 
 212 // Class for all pointer registers (excluding RSP)
 213 reg_class ptr_reg_with_rbp(RAX, RAX_H,
 214                            RDX, RDX_H,
 215                            RBP, RBP_H,
 216                            RDI, RDI_H,
 217                            RSI, RSI_H,
 218                            RCX, RCX_H,
 219                            RBX, RBX_H,
 220                            R8,  R8_H,
 221                            R9,  R9_H,
 222                            R10, R10_H,
 223                            R11, R11_H,
 224                            R13, R13_H,
 225                            R14, R14_H);
 226 
 227 // Class for all pointer registers (excluding RSP and RBP)
 228 reg_class ptr_reg_no_rbp(RAX, RAX_H,
 229                          RDX, RDX_H,
 230                          RDI, RDI_H,
 231                          RSI, RSI_H,
 232                          RCX, RCX_H,
 233                          RBX, RBX_H,
 234                          R8,  R8_H,
 235                          R9,  R9_H,
 236                          R10, R10_H,
 237                          R11, R11_H,
 238                          R13, R13_H,
 239                          R14, R14_H);
 240 
 241 // Dynamic register class that selects between ptr_reg_no_rbp and ptr_reg_with_rbp.
 242 reg_class_dynamic ptr_reg(ptr_reg_no_rbp, ptr_reg_with_rbp, %{ PreserveFramePointer %});
 243 
 244 // Class for all pointer registers (excluding RAX and RSP)
 245 reg_class ptr_no_rax_reg_with_rbp(RDX, RDX_H,
 246                                   RBP, RBP_H,
 247                                   RDI, RDI_H,
 248                                   RSI, RSI_H,
 249                                   RCX, RCX_H,
 250                                   RBX, RBX_H,
 251                                   R8,  R8_H,
 252                                   R9,  R9_H,
 253                                   R10, R10_H,
 254                                   R11, R11_H,
 255                                   R13, R13_H,
 256                                   R14, R14_H);
 257 
 258 // Class for all pointer registers (excluding RAX, RSP, and RBP)
 259 reg_class ptr_no_rax_reg_no_rbp(RDX, RDX_H,
 260                                 RDI, RDI_H,
 261                                 RSI, RSI_H,
 262                                 RCX, RCX_H,
 263                                 RBX, RBX_H,
 264                                 R8,  R8_H,
 265                                 R9,  R9_H,
 266                                 R10, R10_H,
 267                                 R11, R11_H,
 268                                 R13, R13_H,
 269                                 R14, R14_H);
 270 
 271 // Dynamic register class that selects between ptr_no_rax_reg_no_rbp and ptr_no_rax_reg_with_rbp.
 272 reg_class_dynamic ptr_no_rax_reg(ptr_no_rax_reg_no_rbp, ptr_no_rax_reg_with_rbp, %{ PreserveFramePointer %});
 273 
 274 // Class for all pointer registers (excluding RAX, RBX, and RSP)
 275 reg_class ptr_no_rax_rbx_reg_with_rbp(RDX, RDX_H,
 276                                       RBP, RBP_H,
 277                                       RDI, RDI_H,
 278                                       RSI, RSI_H,
 279                                       RCX, RCX_H,
 280                                       R8,  R8_H,
 281                                       R9,  R9_H,
 282                                       R10, R10_H,
 283                                       R11, R11_H,
 284                                       R13, R13_H,
 285                                       R14, R14_H);
 286 
 287 // Class for all pointer registers (excluding RAX, RBX, RSP, and RBP)
 288 reg_class ptr_no_rax_rbx_reg_no_rbp(RDX, RDX_H,
 289                                     RDI, RDI_H,
 290                                     RSI, RSI_H,
 291                                     RCX, RCX_H,
 292                                     R8,  R8_H,
 293                                     R9,  R9_H,
 294                                     R10, R10_H,
 295                                     R11, R11_H,
 296                                     R13, R13_H,
 297                                     R14, R14_H);
 298 
 299 // Dynamic register class that selects between ptr_no_rax_rbx_reg_no_rbp and ptr_no_rax_rbx_reg_with_rbp.
 300 reg_class_dynamic ptr_no_rax_rbx_reg(ptr_no_rax_rbx_reg_no_rbp, ptr_no_rax_rbx_reg_with_rbp, %{ PreserveFramePointer %});
 301 
 302 // Singleton class for RAX pointer register
 303 reg_class ptr_rax_reg(RAX, RAX_H);
 304 
 305 // Singleton class for RBX pointer register
 306 reg_class ptr_rbx_reg(RBX, RBX_H);
 307 
 308 // Singleton class for RSI pointer register
 309 reg_class ptr_rsi_reg(RSI, RSI_H);
 310 
 311 // Singleton class for RDI pointer register
 312 reg_class ptr_rdi_reg(RDI, RDI_H);
 313 
 314 // Singleton class for stack pointer
 315 reg_class ptr_rsp_reg(RSP, RSP_H);
 316 
 317 // Singleton class for TLS pointer
 318 reg_class ptr_r15_reg(R15, R15_H);
 319 
 320 // The registers which can be used for
 321 // a thread local safepoint poll
 322 // * R12 is reserved for heap base
 323 // * R13 cannot be encoded for addressing without an offset byte
 324 // * R15 is reserved for the JavaThread
 325 reg_class ptr_rex_reg(R8,  R8_H,
 326                       R9,  R9_H,
 327                       R10, R10_H,
 328                       R11, R11_H,
 329                       R14, R14_H);
 330 
 331 
 332 // Class for all long registers (excluding RSP)
 333 reg_class long_reg_with_rbp(RAX, RAX_H,
 334                             RDX, RDX_H,
 335                             RBP, RBP_H,
 336                             RDI, RDI_H,
 337                             RSI, RSI_H,
 338                             RCX, RCX_H,
 339                             RBX, RBX_H,
 340                             R8,  R8_H,
 341                             R9,  R9_H,
 342                             R10, R10_H,
 343                             R11, R11_H,
 344                             R13, R13_H,
 345                             R14, R14_H);
 346 
 347 // Class for all long registers (excluding RSP and RBP)
 348 reg_class long_reg_no_rbp(RAX, RAX_H,
 349                           RDX, RDX_H,
 350                           RDI, RDI_H,
 351                           RSI, RSI_H,
 352                           RCX, RCX_H,
 353                           RBX, RBX_H,
 354                           R8,  R8_H,
 355                           R9,  R9_H,
 356                           R10, R10_H,
 357                           R11, R11_H,
 358                           R13, R13_H,
 359                           R14, R14_H);
 360 
 361 // Dynamic register class that selects between long_reg_no_rbp and long_reg_with_rbp.
 362 reg_class_dynamic long_reg(long_reg_no_rbp, long_reg_with_rbp, %{ PreserveFramePointer %});
 363 
 364 // Class for all long registers (excluding RAX, RDX and RSP)
 365 reg_class long_no_rax_rdx_reg_with_rbp(RBP, RBP_H,
 366                                        RDI, RDI_H,
 367                                        RSI, RSI_H,
 368                                        RCX, RCX_H,
 369                                        RBX, RBX_H,
 370                                        R8,  R8_H,
 371                                        R9,  R9_H,
 372                                        R10, R10_H,
 373                                        R11, R11_H,
 374                                        R13, R13_H,
 375                                        R14, R14_H);
 376 
 377 // Class for all long registers (excluding RAX, RDX, RSP, and RBP)
 378 reg_class long_no_rax_rdx_reg_no_rbp(RDI, RDI_H,
 379                                      RSI, RSI_H,
 380                                      RCX, RCX_H,
 381                                      RBX, RBX_H,
 382                                      R8,  R8_H,
 383                                      R9,  R9_H,
 384                                      R10, R10_H,
 385                                      R11, R11_H,
 386                                      R13, R13_H,
 387                                      R14, R14_H);
 388 
 389 // Dynamic register class that selects between long_no_rax_rdx_reg_no_rbp and long_no_rax_rdx_reg_with_rbp.
 390 reg_class_dynamic long_no_rax_rdx_reg(long_no_rax_rdx_reg_no_rbp, long_no_rax_rdx_reg_with_rbp, %{ PreserveFramePointer %});
 391 
 392 // Class for all long registers (excluding RCX and RSP)
 393 reg_class long_no_rcx_reg_with_rbp(RBP, RBP_H,
 394                                    RDI, RDI_H,
 395                                    RSI, RSI_H,
 396                                    RAX, RAX_H,
 397                                    RDX, RDX_H,
 398                                    RBX, RBX_H,
 399                                    R8,  R8_H,
 400                                    R9,  R9_H,
 401                                    R10, R10_H,
 402                                    R11, R11_H,
 403                                    R13, R13_H,
 404                                    R14, R14_H);
 405 
 406 // Class for all long registers (excluding RCX, RSP, and RBP)
 407 reg_class long_no_rcx_reg_no_rbp(RDI, RDI_H,
 408                                  RSI, RSI_H,
 409                                  RAX, RAX_H,
 410                                  RDX, RDX_H,
 411                                  RBX, RBX_H,
 412                                  R8,  R8_H,
 413                                  R9,  R9_H,
 414                                  R10, R10_H,
 415                                  R11, R11_H,
 416                                  R13, R13_H,
 417                                  R14, R14_H);
 418 
 419 // Dynamic register class that selects between long_no_rcx_reg_no_rbp and long_no_rcx_reg_with_rbp.
 420 reg_class_dynamic long_no_rcx_reg(long_no_rcx_reg_no_rbp, long_no_rcx_reg_with_rbp, %{ PreserveFramePointer %});
 421 
 422 // Singleton class for RAX long register
 423 reg_class long_rax_reg(RAX, RAX_H);
 424 
 425 // Singleton class for RCX long register
 426 reg_class long_rcx_reg(RCX, RCX_H);
 427 
 428 // Singleton class for RDX long register
 429 reg_class long_rdx_reg(RDX, RDX_H);
 430 
 431 // Class for all int registers (excluding RSP)
 432 reg_class int_reg_with_rbp(RAX,
 433                            RDX,
 434                            RBP,
 435                            RDI,
 436                            RSI,
 437                            RCX,
 438                            RBX,
 439                            R8,
 440                            R9,
 441                            R10,
 442                            R11,
 443                            R13,
 444                            R14);
 445 
 446 // Class for all int registers (excluding RSP and RBP)
 447 reg_class int_reg_no_rbp(RAX,
 448                          RDX,
 449                          RDI,
 450                          RSI,
 451                          RCX,
 452                          RBX,
 453                          R8,
 454                          R9,
 455                          R10,
 456                          R11,
 457                          R13,
 458                          R14);
 459 
 460 // Dynamic register class that selects between int_reg_no_rbp and int_reg_with_rbp.
 461 reg_class_dynamic int_reg(int_reg_no_rbp, int_reg_with_rbp, %{ PreserveFramePointer %});
 462 
 463 // Class for all int registers (excluding RCX and RSP)
 464 reg_class int_no_rcx_reg_with_rbp(RAX,
 465                                   RDX,
 466                                   RBP,
 467                                   RDI,
 468                                   RSI,
 469                                   RBX,
 470                                   R8,
 471                                   R9,
 472                                   R10,
 473                                   R11,
 474                                   R13,
 475                                   R14);
 476 
 477 // Class for all int registers (excluding RCX, RSP, and RBP)
 478 reg_class int_no_rcx_reg_no_rbp(RAX,
 479                                 RDX,
 480                                 RDI,
 481                                 RSI,
 482                                 RBX,
 483                                 R8,
 484                                 R9,
 485                                 R10,
 486                                 R11,
 487                                 R13,
 488                                 R14);
 489 
 490 // Dynamic register class that selects between int_no_rcx_reg_no_rbp and int_no_rcx_reg_with_rbp.
 491 reg_class_dynamic int_no_rcx_reg(int_no_rcx_reg_no_rbp, int_no_rcx_reg_with_rbp, %{ PreserveFramePointer %});
 492 
 493 // Class for all int registers (excluding RAX, RDX, and RSP)
 494 reg_class int_no_rax_rdx_reg_with_rbp(RBP,
 495                                       RDI,
 496                                       RSI,
 497                                       RCX,
 498                                       RBX,
 499                                       R8,
 500                                       R9,
 501                                       R10,
 502                                       R11,
 503                                       R13,
 504                                       R14);
 505 
 506 // Class for all int registers (excluding RAX, RDX, RSP, and RBP)
 507 reg_class int_no_rax_rdx_reg_no_rbp(RDI,
 508                                     RSI,
 509                                     RCX,
 510                                     RBX,
 511                                     R8,
 512                                     R9,
 513                                     R10,
 514                                     R11,
 515                                     R13,
 516                                     R14);
 517 
 518 // Dynamic register class that selects between int_no_rax_rdx_reg_no_rbp and int_no_rax_rdx_reg_with_rbp.
 519 reg_class_dynamic int_no_rax_rdx_reg(int_no_rax_rdx_reg_no_rbp, int_no_rax_rdx_reg_with_rbp, %{ PreserveFramePointer %});
 520 
 521 // Singleton class for RAX int register
 522 reg_class int_rax_reg(RAX);
 523 
 524 // Singleton class for RBX int register
 525 reg_class int_rbx_reg(RBX);
 526 
 527 // Singleton class for RCX int register
 528 reg_class int_rcx_reg(RCX);
 529 
 530 // Singleton class for RCX int register
 531 reg_class int_rdx_reg(RDX);
 532 
 533 // Singleton class for RCX int register
 534 reg_class int_rdi_reg(RDI);
 535 
 536 // Singleton class for instruction pointer
 537 // reg_class ip_reg(RIP);
 538 
 539 %}
 540 
 541 source_hpp %{
 542 #if INCLUDE_ZGC
 543 #include "gc/z/zBarrierSetAssembler.hpp"
 544 #endif
 545 #if INCLUDE_SHENANDOAHGC
 546 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
 547 #endif
 548 %}
 549 
 550 //----------SOURCE BLOCK-------------------------------------------------------
 551 // This is a block of C++ code which provides values, functions, and
 552 // definitions necessary in the rest of the architecture description
 553 source %{
 554 #define   RELOC_IMM64    Assembler::imm_operand
 555 #define   RELOC_DISP32   Assembler::disp32_operand
 556 
 557 #define __ _masm.
 558 
 559 static bool generate_vzeroupper(Compile* C) {
 560   return (VM_Version::supports_vzeroupper() && (C->max_vector_size() > 16 || C->clear_upper_avx() == true)) ? true: false;  // Generate vzeroupper
 561 }
 562 
 563 static int clear_avx_size() {
 564   return generate_vzeroupper(Compile::current()) ? 3: 0;  // vzeroupper
 565 }
 566 
 567 // !!!!! Special hack to get all types of calls to specify the byte offset
 568 //       from the start of the call to the point where the return address
 569 //       will point.
 570 int MachCallStaticJavaNode::ret_addr_offset()
 571 {
 572   int offset = 5; // 5 bytes from start of call to where return address points
 573   offset += clear_avx_size();
 574   return offset;
 575 }
 576 
 577 int MachCallDynamicJavaNode::ret_addr_offset()
 578 {
 579   int offset = 15; // 15 bytes from start of call to where return address points
 580   offset += clear_avx_size();
 581   return offset;
 582 }
 583 
 584 int MachCallRuntimeNode::ret_addr_offset() {
 585   int offset = 13; // movq r10,#addr; callq (r10)
 586   offset += clear_avx_size();
 587   return offset;
 588 }
 589 
 590 // Indicate if the safepoint node needs the polling page as an input,
 591 // it does if the polling page is more than disp32 away.
 592 bool SafePointNode::needs_polling_address_input()
 593 {
 594   return SafepointMechanism::uses_thread_local_poll() || Assembler::is_polling_page_far();
 595 }
 596 
 597 //
 598 // Compute padding required for nodes which need alignment
 599 //
 600 
 601 // The address of the call instruction needs to be 4-byte aligned to
 602 // ensure that it does not span a cache line so that it can be patched.
 603 int CallStaticJavaDirectNode::compute_padding(int current_offset) const
 604 {
 605   current_offset += clear_avx_size(); // skip vzeroupper
 606   current_offset += 1; // skip call opcode byte
 607   return align_up(current_offset, alignment_required()) - current_offset;
 608 }
 609 
 610 // The address of the call instruction needs to be 4-byte aligned to
 611 // ensure that it does not span a cache line so that it can be patched.
 612 int CallDynamicJavaDirectNode::compute_padding(int current_offset) const
 613 {
 614   current_offset += clear_avx_size(); // skip vzeroupper
 615   current_offset += 11; // skip movq instruction + call opcode byte
 616   return align_up(current_offset, alignment_required()) - current_offset;
 617 }
 618 
 619 // EMIT_RM()
 620 void emit_rm(CodeBuffer &cbuf, int f1, int f2, int f3) {
 621   unsigned char c = (unsigned char) ((f1 << 6) | (f2 << 3) | f3);
 622   cbuf.insts()->emit_int8(c);
 623 }
 624 
 625 // EMIT_CC()
 626 void emit_cc(CodeBuffer &cbuf, int f1, int f2) {
 627   unsigned char c = (unsigned char) (f1 | f2);
 628   cbuf.insts()->emit_int8(c);
 629 }
 630 
 631 // EMIT_OPCODE()
 632 void emit_opcode(CodeBuffer &cbuf, int code) {
 633   cbuf.insts()->emit_int8((unsigned char) code);
 634 }
 635 
 636 // EMIT_OPCODE() w/ relocation information
 637 void emit_opcode(CodeBuffer &cbuf,
 638                  int code, relocInfo::relocType reloc, int offset, int format)
 639 {
 640   cbuf.relocate(cbuf.insts_mark() + offset, reloc, format);
 641   emit_opcode(cbuf, code);
 642 }
 643 
 644 // EMIT_D8()
 645 void emit_d8(CodeBuffer &cbuf, int d8) {
 646   cbuf.insts()->emit_int8((unsigned char) d8);
 647 }
 648 
 649 // EMIT_D16()
 650 void emit_d16(CodeBuffer &cbuf, int d16) {
 651   cbuf.insts()->emit_int16(d16);
 652 }
 653 
 654 // EMIT_D32()
 655 void emit_d32(CodeBuffer &cbuf, int d32) {
 656   cbuf.insts()->emit_int32(d32);
 657 }
 658 
 659 // EMIT_D64()
 660 void emit_d64(CodeBuffer &cbuf, int64_t d64) {
 661   cbuf.insts()->emit_int64(d64);
 662 }
 663 
 664 // emit 32 bit value and construct relocation entry from relocInfo::relocType
 665 void emit_d32_reloc(CodeBuffer& cbuf,
 666                     int d32,
 667                     relocInfo::relocType reloc,
 668                     int format)
 669 {
 670   assert(reloc != relocInfo::external_word_type, "use 2-arg emit_d32_reloc");
 671   cbuf.relocate(cbuf.insts_mark(), reloc, format);
 672   cbuf.insts()->emit_int32(d32);
 673 }
 674 
 675 // emit 32 bit value and construct relocation entry from RelocationHolder
 676 void emit_d32_reloc(CodeBuffer& cbuf, int d32, RelocationHolder const& rspec, int format) {
 677 #ifdef ASSERT
 678   if (rspec.reloc()->type() == relocInfo::oop_type &&
 679       d32 != 0 && d32 != (intptr_t) Universe::non_oop_word()) {
 680     assert(Universe::heap()->is_in_reserved((address)(intptr_t)d32), "should be real oop");
 681     assert(oopDesc::is_oop(cast_to_oop((intptr_t)d32)) && (ScavengeRootsInCode || !Universe::heap()->is_scavengable(cast_to_oop((intptr_t)d32))), "cannot embed scavengable oops in code");
 682   }
 683 #endif
 684   cbuf.relocate(cbuf.insts_mark(), rspec, format);
 685   cbuf.insts()->emit_int32(d32);
 686 }
 687 
 688 void emit_d32_reloc(CodeBuffer& cbuf, address addr) {
 689   address next_ip = cbuf.insts_end() + 4;
 690   emit_d32_reloc(cbuf, (int) (addr - next_ip),
 691                  external_word_Relocation::spec(addr),
 692                  RELOC_DISP32);
 693 }
 694 
 695 
 696 // emit 64 bit value and construct relocation entry from relocInfo::relocType
 697 void emit_d64_reloc(CodeBuffer& cbuf, int64_t d64, relocInfo::relocType reloc, int format) {
 698   cbuf.relocate(cbuf.insts_mark(), reloc, format);
 699   cbuf.insts()->emit_int64(d64);
 700 }
 701 
 702 // emit 64 bit value and construct relocation entry from RelocationHolder
 703 void emit_d64_reloc(CodeBuffer& cbuf, int64_t d64, RelocationHolder const& rspec, int format) {
 704 #ifdef ASSERT
 705   if (rspec.reloc()->type() == relocInfo::oop_type &&
 706       d64 != 0 && d64 != (int64_t) Universe::non_oop_word()) {
 707     assert(Universe::heap()->is_in_reserved((address)d64), "should be real oop");
 708     assert(oopDesc::is_oop(cast_to_oop(d64)) && (ScavengeRootsInCode || !Universe::heap()->is_scavengable(cast_to_oop(d64))),
 709            "cannot embed scavengable oops in code");
 710   }
 711 #endif
 712   cbuf.relocate(cbuf.insts_mark(), rspec, format);
 713   cbuf.insts()->emit_int64(d64);
 714 }
 715 
 716 // Access stack slot for load or store
 717 void store_to_stackslot(CodeBuffer &cbuf, int opcode, int rm_field, int disp)
 718 {
 719   emit_opcode(cbuf, opcode);                  // (e.g., FILD   [RSP+src])
 720   if (-0x80 <= disp && disp < 0x80) {
 721     emit_rm(cbuf, 0x01, rm_field, RSP_enc);   // R/M byte
 722     emit_rm(cbuf, 0x00, RSP_enc, RSP_enc);    // SIB byte
 723     emit_d8(cbuf, disp);     // Displacement  // R/M byte
 724   } else {
 725     emit_rm(cbuf, 0x02, rm_field, RSP_enc);   // R/M byte
 726     emit_rm(cbuf, 0x00, RSP_enc, RSP_enc);    // SIB byte
 727     emit_d32(cbuf, disp);     // Displacement // R/M byte
 728   }
 729 }
 730 
 731    // rRegI ereg, memory mem) %{    // emit_reg_mem
 732 void encode_RegMem(CodeBuffer &cbuf,
 733                    int reg,
 734                    int base, int index, int scale, int disp, relocInfo::relocType disp_reloc)
 735 {
 736   assert(disp_reloc == relocInfo::none, "cannot have disp");
 737   int regenc = reg & 7;
 738   int baseenc = base & 7;
 739   int indexenc = index & 7;
 740 
 741   // There is no index & no scale, use form without SIB byte
 742   if (index == 0x4 && scale == 0 && base != RSP_enc && base != R12_enc) {
 743     // If no displacement, mode is 0x0; unless base is [RBP] or [R13]
 744     if (disp == 0 && base != RBP_enc && base != R13_enc) {
 745       emit_rm(cbuf, 0x0, regenc, baseenc); // *
 746     } else if (-0x80 <= disp && disp < 0x80 && disp_reloc == relocInfo::none) {
 747       // If 8-bit displacement, mode 0x1
 748       emit_rm(cbuf, 0x1, regenc, baseenc); // *
 749       emit_d8(cbuf, disp);
 750     } else {
 751       // If 32-bit displacement
 752       if (base == -1) { // Special flag for absolute address
 753         emit_rm(cbuf, 0x0, regenc, 0x5); // *
 754         if (disp_reloc != relocInfo::none) {
 755           emit_d32_reloc(cbuf, disp, relocInfo::oop_type, RELOC_DISP32);
 756         } else {
 757           emit_d32(cbuf, disp);
 758         }
 759       } else {
 760         // Normal base + offset
 761         emit_rm(cbuf, 0x2, regenc, baseenc); // *
 762         if (disp_reloc != relocInfo::none) {
 763           emit_d32_reloc(cbuf, disp, relocInfo::oop_type, RELOC_DISP32);
 764         } else {
 765           emit_d32(cbuf, disp);
 766         }
 767       }
 768     }
 769   } else {
 770     // Else, encode with the SIB byte
 771     // If no displacement, mode is 0x0; unless base is [RBP] or [R13]
 772     if (disp == 0 && base != RBP_enc && base != R13_enc) {
 773       // If no displacement
 774       emit_rm(cbuf, 0x0, regenc, 0x4); // *
 775       emit_rm(cbuf, scale, indexenc, baseenc);
 776     } else {
 777       if (-0x80 <= disp && disp < 0x80 && disp_reloc == relocInfo::none) {
 778         // If 8-bit displacement, mode 0x1
 779         emit_rm(cbuf, 0x1, regenc, 0x4); // *
 780         emit_rm(cbuf, scale, indexenc, baseenc);
 781         emit_d8(cbuf, disp);
 782       } else {
 783         // If 32-bit displacement
 784         if (base == 0x04 ) {
 785           emit_rm(cbuf, 0x2, regenc, 0x4);
 786           emit_rm(cbuf, scale, indexenc, 0x04); // XXX is this valid???
 787         } else {
 788           emit_rm(cbuf, 0x2, regenc, 0x4);
 789           emit_rm(cbuf, scale, indexenc, baseenc); // *
 790         }
 791         if (disp_reloc != relocInfo::none) {
 792           emit_d32_reloc(cbuf, disp, relocInfo::oop_type, RELOC_DISP32);
 793         } else {
 794           emit_d32(cbuf, disp);
 795         }
 796       }
 797     }
 798   }
 799 }
 800 
 801 // This could be in MacroAssembler but it's fairly C2 specific
 802 void emit_cmpfp_fixup(MacroAssembler& _masm) {
 803   Label exit;
 804   __ jccb(Assembler::noParity, exit);
 805   __ pushf();
 806   //
 807   // comiss/ucomiss instructions set ZF,PF,CF flags and
 808   // zero OF,AF,SF for NaN values.
 809   // Fixup flags by zeroing ZF,PF so that compare of NaN
 810   // values returns 'less than' result (CF is set).
 811   // Leave the rest of flags unchanged.
 812   //
 813   //    7 6 5 4 3 2 1 0
 814   //   |S|Z|r|A|r|P|r|C|  (r - reserved bit)
 815   //    0 0 1 0 1 0 1 1   (0x2B)
 816   //
 817   __ andq(Address(rsp, 0), 0xffffff2b);
 818   __ popf();
 819   __ bind(exit);
 820 }
 821 
 822 void emit_cmpfp3(MacroAssembler& _masm, Register dst) {
 823   Label done;
 824   __ movl(dst, -1);
 825   __ jcc(Assembler::parity, done);
 826   __ jcc(Assembler::below, done);
 827   __ setb(Assembler::notEqual, dst);
 828   __ movzbl(dst, dst);
 829   __ bind(done);
 830 }
 831 
 832 
 833 //=============================================================================
 834 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
 835 
 836 int Compile::ConstantTable::calculate_table_base_offset() const {
 837   return 0;  // absolute addressing, no offset
 838 }
 839 
 840 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
 841 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
 842   ShouldNotReachHere();
 843 }
 844 
 845 void MachConstantBaseNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const {
 846   // Empty encoding
 847 }
 848 
 849 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
 850   return 0;
 851 }
 852 
 853 #ifndef PRODUCT
 854 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
 855   st->print("# MachConstantBaseNode (empty encoding)");
 856 }
 857 #endif
 858 
 859 
 860 //=============================================================================
 861 #ifndef PRODUCT
 862 void MachPrologNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
 863   Compile* C = ra_->C;
 864 
 865   int framesize = C->frame_size_in_bytes();
 866   int bangsize = C->bang_size_in_bytes();
 867   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
 868   // Remove wordSize for return addr which is already pushed.
 869   framesize -= wordSize;
 870 
 871   if (C->need_stack_bang(bangsize)) {
 872     framesize -= wordSize;
 873     st->print("# stack bang (%d bytes)", bangsize);
 874     st->print("\n\t");
 875     st->print("pushq   rbp\t# Save rbp");
 876     if (PreserveFramePointer) {
 877         st->print("\n\t");
 878         st->print("movq    rbp, rsp\t# Save the caller's SP into rbp");
 879     }
 880     if (framesize) {
 881       st->print("\n\t");
 882       st->print("subq    rsp, #%d\t# Create frame",framesize);
 883     }
 884   } else {
 885     st->print("subq    rsp, #%d\t# Create frame",framesize);
 886     st->print("\n\t");
 887     framesize -= wordSize;
 888     st->print("movq    [rsp + #%d], rbp\t# Save rbp",framesize);
 889     if (PreserveFramePointer) {
 890       st->print("\n\t");
 891       st->print("movq    rbp, rsp\t# Save the caller's SP into rbp");
 892       if (framesize > 0) {
 893         st->print("\n\t");
 894         st->print("addq    rbp, #%d", framesize);
 895       }
 896     }
 897   }
 898 
 899   if (VerifyStackAtCalls) {
 900     st->print("\n\t");
 901     framesize -= wordSize;
 902     st->print("movq    [rsp + #%d], 0xbadb100d\t# Majik cookie for stack depth check",framesize);
 903 #ifdef ASSERT
 904     st->print("\n\t");
 905     st->print("# stack alignment check");
 906 #endif
 907   }
 908   st->cr();
 909 }
 910 #endif
 911 
 912 void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 913   Compile* C = ra_->C;
 914   MacroAssembler _masm(&cbuf);
 915 
 916   int framesize = C->frame_size_in_bytes();
 917   int bangsize = C->bang_size_in_bytes();
 918 
 919   __ verified_entry(framesize, C->need_stack_bang(bangsize)?bangsize:0, false);
 920 
 921   C->set_frame_complete(cbuf.insts_size());
 922 
 923   if (C->has_mach_constant_base_node()) {
 924     // NOTE: We set the table base offset here because users might be
 925     // emitted before MachConstantBaseNode.
 926     Compile::ConstantTable& constant_table = C->constant_table();
 927     constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
 928   }
 929 }
 930 
 931 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
 932 {
 933   return MachNode::size(ra_); // too many variables; just compute it
 934                               // the hard way
 935 }
 936 
 937 int MachPrologNode::reloc() const
 938 {
 939   return 0; // a large enough number
 940 }
 941 
 942 //=============================================================================
 943 #ifndef PRODUCT
 944 void MachEpilogNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 945 {
 946   Compile* C = ra_->C;
 947   if (generate_vzeroupper(C)) {
 948     st->print("vzeroupper");
 949     st->cr(); st->print("\t");
 950   }
 951 
 952   int framesize = C->frame_size_in_bytes();
 953   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
 954   // Remove word for return adr already pushed
 955   // and RBP
 956   framesize -= 2*wordSize;
 957 
 958   if (framesize) {
 959     st->print_cr("addq    rsp, %d\t# Destroy frame", framesize);
 960     st->print("\t");
 961   }
 962 
 963   st->print_cr("popq   rbp");
 964   if (do_polling() && C->is_method_compilation()) {
 965     st->print("\t");
 966     if (SafepointMechanism::uses_thread_local_poll()) {
 967       st->print_cr("movq   rscratch1, poll_offset[r15_thread] #polling_page_address\n\t"
 968                    "testl  rax, [rscratch1]\t"
 969                    "# Safepoint: poll for GC");
 970     } else if (Assembler::is_polling_page_far()) {
 971       st->print_cr("movq   rscratch1, #polling_page_address\n\t"
 972                    "testl  rax, [rscratch1]\t"
 973                    "# Safepoint: poll for GC");
 974     } else {
 975       st->print_cr("testl  rax, [rip + #offset_to_poll_page]\t"
 976                    "# Safepoint: poll for GC");
 977     }
 978   }
 979 }
 980 #endif
 981 
 982 void MachEpilogNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
 983 {
 984   Compile* C = ra_->C;
 985   MacroAssembler _masm(&cbuf);
 986 
 987   if (generate_vzeroupper(C)) {
 988     // Clear upper bits of YMM registers when current compiled code uses
 989     // wide vectors to avoid AVX <-> SSE transition penalty during call.
 990     __ vzeroupper();
 991   }
 992 
 993   int framesize = C->frame_size_in_bytes();
 994   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
 995   // Remove word for return adr already pushed
 996   // and RBP
 997   framesize -= 2*wordSize;
 998 
 999   // Note that VerifyStackAtCalls' Majik cookie does not change the frame size popped here
1000 
1001   if (framesize) {
1002     emit_opcode(cbuf, Assembler::REX_W);
1003     if (framesize < 0x80) {
1004       emit_opcode(cbuf, 0x83); // addq rsp, #framesize
1005       emit_rm(cbuf, 0x3, 0x00, RSP_enc);
1006       emit_d8(cbuf, framesize);
1007     } else {
1008       emit_opcode(cbuf, 0x81); // addq rsp, #framesize
1009       emit_rm(cbuf, 0x3, 0x00, RSP_enc);
1010       emit_d32(cbuf, framesize);
1011     }
1012   }
1013 
1014   // popq rbp
1015   emit_opcode(cbuf, 0x58 | RBP_enc);
1016 
1017   if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
1018     __ reserved_stack_check();
1019   }
1020 
1021   if (do_polling() && C->is_method_compilation()) {
1022     MacroAssembler _masm(&cbuf);
1023     if (SafepointMechanism::uses_thread_local_poll()) {
1024       __ movq(rscratch1, Address(r15_thread, Thread::polling_page_offset()));
1025       __ relocate(relocInfo::poll_return_type);
1026       __ testl(rax, Address(rscratch1, 0));
1027     } else {
1028       AddressLiteral polling_page(os::get_polling_page(), relocInfo::poll_return_type);
1029       if (Assembler::is_polling_page_far()) {
1030         __ lea(rscratch1, polling_page);
1031         __ relocate(relocInfo::poll_return_type);
1032         __ testl(rax, Address(rscratch1, 0));
1033       } else {
1034         __ testl(rax, polling_page);
1035       }
1036     }
1037   }
1038 }
1039 
1040 uint MachEpilogNode::size(PhaseRegAlloc* ra_) const
1041 {
1042   return MachNode::size(ra_); // too many variables; just compute it
1043                               // the hard way
1044 }
1045 
1046 int MachEpilogNode::reloc() const
1047 {
1048   return 2; // a large enough number
1049 }
1050 
1051 const Pipeline* MachEpilogNode::pipeline() const
1052 {
1053   return MachNode::pipeline_class();
1054 }
1055 
1056 int MachEpilogNode::safepoint_offset() const
1057 {
1058   return 0;
1059 }
1060 
1061 //=============================================================================
1062 
1063 enum RC {
1064   rc_bad,
1065   rc_int,
1066   rc_float,
1067   rc_stack
1068 };
1069 
1070 static enum RC rc_class(OptoReg::Name reg)
1071 {
1072   if( !OptoReg::is_valid(reg)  ) return rc_bad;
1073 
1074   if (OptoReg::is_stack(reg)) return rc_stack;
1075 
1076   VMReg r = OptoReg::as_VMReg(reg);
1077 
1078   if (r->is_Register()) return rc_int;
1079 
1080   assert(r->is_XMMRegister(), "must be");
1081   return rc_float;
1082 }
1083 
1084 // Next two methods are shared by 32- and 64-bit VM. They are defined in x86.ad.
1085 static int vec_mov_helper(CodeBuffer *cbuf, bool do_size, int src_lo, int dst_lo,
1086                           int src_hi, int dst_hi, uint ireg, outputStream* st);
1087 
1088 static int vec_spill_helper(CodeBuffer *cbuf, bool do_size, bool is_load,
1089                             int stack_offset, int reg, uint ireg, outputStream* st);
1090 
1091 static void vec_stack_to_stack_helper(CodeBuffer *cbuf, int src_offset,
1092                                       int dst_offset, uint ireg, outputStream* st) {
1093   if (cbuf) {
1094     MacroAssembler _masm(cbuf);
1095     switch (ireg) {
1096     case Op_VecS:
1097       __ movq(Address(rsp, -8), rax);
1098       __ movl(rax, Address(rsp, src_offset));
1099       __ movl(Address(rsp, dst_offset), rax);
1100       __ movq(rax, Address(rsp, -8));
1101       break;
1102     case Op_VecD:
1103       __ pushq(Address(rsp, src_offset));
1104       __ popq (Address(rsp, dst_offset));
1105       break;
1106     case Op_VecX:
1107       __ pushq(Address(rsp, src_offset));
1108       __ popq (Address(rsp, dst_offset));
1109       __ pushq(Address(rsp, src_offset+8));
1110       __ popq (Address(rsp, dst_offset+8));
1111       break;
1112     case Op_VecY:
1113       __ vmovdqu(Address(rsp, -32), xmm0);
1114       __ vmovdqu(xmm0, Address(rsp, src_offset));
1115       __ vmovdqu(Address(rsp, dst_offset), xmm0);
1116       __ vmovdqu(xmm0, Address(rsp, -32));
1117       break;
1118     case Op_VecZ:
1119       __ evmovdquq(Address(rsp, -64), xmm0, 2);
1120       __ evmovdquq(xmm0, Address(rsp, src_offset), 2);
1121       __ evmovdquq(Address(rsp, dst_offset), xmm0, 2);
1122       __ evmovdquq(xmm0, Address(rsp, -64), 2);
1123       break;
1124     default:
1125       ShouldNotReachHere();
1126     }
1127 #ifndef PRODUCT
1128   } else {
1129     switch (ireg) {
1130     case Op_VecS:
1131       st->print("movq    [rsp - #8], rax\t# 32-bit mem-mem spill\n\t"
1132                 "movl    rax, [rsp + #%d]\n\t"
1133                 "movl    [rsp + #%d], rax\n\t"
1134                 "movq    rax, [rsp - #8]",
1135                 src_offset, dst_offset);
1136       break;
1137     case Op_VecD:
1138       st->print("pushq   [rsp + #%d]\t# 64-bit mem-mem spill\n\t"
1139                 "popq    [rsp + #%d]",
1140                 src_offset, dst_offset);
1141       break;
1142      case Op_VecX:
1143       st->print("pushq   [rsp + #%d]\t# 128-bit mem-mem spill\n\t"
1144                 "popq    [rsp + #%d]\n\t"
1145                 "pushq   [rsp + #%d]\n\t"
1146                 "popq    [rsp + #%d]",
1147                 src_offset, dst_offset, src_offset+8, dst_offset+8);
1148       break;
1149     case Op_VecY:
1150       st->print("vmovdqu [rsp - #32], xmm0\t# 256-bit mem-mem spill\n\t"
1151                 "vmovdqu xmm0, [rsp + #%d]\n\t"
1152                 "vmovdqu [rsp + #%d], xmm0\n\t"
1153                 "vmovdqu xmm0, [rsp - #32]",
1154                 src_offset, dst_offset);
1155       break;
1156     case Op_VecZ:
1157       st->print("vmovdqu [rsp - #64], xmm0\t# 512-bit mem-mem spill\n\t"
1158                 "vmovdqu xmm0, [rsp + #%d]\n\t"
1159                 "vmovdqu [rsp + #%d], xmm0\n\t"
1160                 "vmovdqu xmm0, [rsp - #64]",
1161                 src_offset, dst_offset);
1162       break;
1163     default:
1164       ShouldNotReachHere();
1165     }
1166 #endif
1167   }
1168 }
1169 
1170 uint MachSpillCopyNode::implementation(CodeBuffer* cbuf,
1171                                        PhaseRegAlloc* ra_,
1172                                        bool do_size,
1173                                        outputStream* st) const {
1174   assert(cbuf != NULL || st  != NULL, "sanity");
1175   // Get registers to move
1176   OptoReg::Name src_second = ra_->get_reg_second(in(1));
1177   OptoReg::Name src_first = ra_->get_reg_first(in(1));
1178   OptoReg::Name dst_second = ra_->get_reg_second(this);
1179   OptoReg::Name dst_first = ra_->get_reg_first(this);
1180 
1181   enum RC src_second_rc = rc_class(src_second);
1182   enum RC src_first_rc = rc_class(src_first);
1183   enum RC dst_second_rc = rc_class(dst_second);
1184   enum RC dst_first_rc = rc_class(dst_first);
1185 
1186   assert(OptoReg::is_valid(src_first) && OptoReg::is_valid(dst_first),
1187          "must move at least 1 register" );
1188 
1189   if (src_first == dst_first && src_second == dst_second) {
1190     // Self copy, no move
1191     return 0;
1192   }
1193   if (bottom_type()->isa_vect() != NULL) {
1194     uint ireg = ideal_reg();
1195     assert((src_first_rc != rc_int && dst_first_rc != rc_int), "sanity");
1196     assert((ireg == Op_VecS || ireg == Op_VecD || ireg == Op_VecX || ireg == Op_VecY || ireg == Op_VecZ ), "sanity");
1197     if( src_first_rc == rc_stack && dst_first_rc == rc_stack ) {
1198       // mem -> mem
1199       int src_offset = ra_->reg2offset(src_first);
1200       int dst_offset = ra_->reg2offset(dst_first);
1201       vec_stack_to_stack_helper(cbuf, src_offset, dst_offset, ireg, st);
1202     } else if (src_first_rc == rc_float && dst_first_rc == rc_float ) {
1203       vec_mov_helper(cbuf, false, src_first, dst_first, src_second, dst_second, ireg, st);
1204     } else if (src_first_rc == rc_float && dst_first_rc == rc_stack ) {
1205       int stack_offset = ra_->reg2offset(dst_first);
1206       vec_spill_helper(cbuf, false, false, stack_offset, src_first, ireg, st);
1207     } else if (src_first_rc == rc_stack && dst_first_rc == rc_float ) {
1208       int stack_offset = ra_->reg2offset(src_first);
1209       vec_spill_helper(cbuf, false, true,  stack_offset, dst_first, ireg, st);
1210     } else {
1211       ShouldNotReachHere();
1212     }
1213     return 0;
1214   }
1215   if (src_first_rc == rc_stack) {
1216     // mem ->
1217     if (dst_first_rc == rc_stack) {
1218       // mem -> mem
1219       assert(src_second != dst_first, "overlap");
1220       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1221           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1222         // 64-bit
1223         int src_offset = ra_->reg2offset(src_first);
1224         int dst_offset = ra_->reg2offset(dst_first);
1225         if (cbuf) {
1226           MacroAssembler _masm(cbuf);
1227           __ pushq(Address(rsp, src_offset));
1228           __ popq (Address(rsp, dst_offset));
1229 #ifndef PRODUCT
1230         } else {
1231           st->print("pushq   [rsp + #%d]\t# 64-bit mem-mem spill\n\t"
1232                     "popq    [rsp + #%d]",
1233                      src_offset, dst_offset);
1234 #endif
1235         }
1236       } else {
1237         // 32-bit
1238         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1239         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1240         // No pushl/popl, so:
1241         int src_offset = ra_->reg2offset(src_first);
1242         int dst_offset = ra_->reg2offset(dst_first);
1243         if (cbuf) {
1244           MacroAssembler _masm(cbuf);
1245           __ movq(Address(rsp, -8), rax);
1246           __ movl(rax, Address(rsp, src_offset));
1247           __ movl(Address(rsp, dst_offset), rax);
1248           __ movq(rax, Address(rsp, -8));
1249 #ifndef PRODUCT
1250         } else {
1251           st->print("movq    [rsp - #8], rax\t# 32-bit mem-mem spill\n\t"
1252                     "movl    rax, [rsp + #%d]\n\t"
1253                     "movl    [rsp + #%d], rax\n\t"
1254                     "movq    rax, [rsp - #8]",
1255                      src_offset, dst_offset);
1256 #endif
1257         }
1258       }
1259       return 0;
1260     } else if (dst_first_rc == rc_int) {
1261       // mem -> gpr
1262       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1263           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1264         // 64-bit
1265         int offset = ra_->reg2offset(src_first);
1266         if (cbuf) {
1267           MacroAssembler _masm(cbuf);
1268           __ movq(as_Register(Matcher::_regEncode[dst_first]), Address(rsp, offset));
1269 #ifndef PRODUCT
1270         } else {
1271           st->print("movq    %s, [rsp + #%d]\t# spill",
1272                      Matcher::regName[dst_first],
1273                      offset);
1274 #endif
1275         }
1276       } else {
1277         // 32-bit
1278         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1279         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1280         int offset = ra_->reg2offset(src_first);
1281         if (cbuf) {
1282           MacroAssembler _masm(cbuf);
1283           __ movl(as_Register(Matcher::_regEncode[dst_first]), Address(rsp, offset));
1284 #ifndef PRODUCT
1285         } else {
1286           st->print("movl    %s, [rsp + #%d]\t# spill",
1287                      Matcher::regName[dst_first],
1288                      offset);
1289 #endif
1290         }
1291       }
1292       return 0;
1293     } else if (dst_first_rc == rc_float) {
1294       // mem-> xmm
1295       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1296           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1297         // 64-bit
1298         int offset = ra_->reg2offset(src_first);
1299         if (cbuf) {
1300           MacroAssembler _masm(cbuf);
1301           __ movdbl( as_XMMRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
1302 #ifndef PRODUCT
1303         } else {
1304           st->print("%s  %s, [rsp + #%d]\t# spill",
1305                      UseXmmLoadAndClearUpper ? "movsd " : "movlpd",
1306                      Matcher::regName[dst_first],
1307                      offset);
1308 #endif
1309         }
1310       } else {
1311         // 32-bit
1312         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1313         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1314         int offset = ra_->reg2offset(src_first);
1315         if (cbuf) {
1316           MacroAssembler _masm(cbuf);
1317           __ movflt( as_XMMRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
1318 #ifndef PRODUCT
1319         } else {
1320           st->print("movss   %s, [rsp + #%d]\t# spill",
1321                      Matcher::regName[dst_first],
1322                      offset);
1323 #endif
1324         }
1325       }
1326       return 0;
1327     }
1328   } else if (src_first_rc == rc_int) {
1329     // gpr ->
1330     if (dst_first_rc == rc_stack) {
1331       // gpr -> mem
1332       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1333           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1334         // 64-bit
1335         int offset = ra_->reg2offset(dst_first);
1336         if (cbuf) {
1337           MacroAssembler _masm(cbuf);
1338           __ movq(Address(rsp, offset), as_Register(Matcher::_regEncode[src_first]));
1339 #ifndef PRODUCT
1340         } else {
1341           st->print("movq    [rsp + #%d], %s\t# spill",
1342                      offset,
1343                      Matcher::regName[src_first]);
1344 #endif
1345         }
1346       } else {
1347         // 32-bit
1348         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1349         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1350         int offset = ra_->reg2offset(dst_first);
1351         if (cbuf) {
1352           MacroAssembler _masm(cbuf);
1353           __ movl(Address(rsp, offset), as_Register(Matcher::_regEncode[src_first]));
1354 #ifndef PRODUCT
1355         } else {
1356           st->print("movl    [rsp + #%d], %s\t# spill",
1357                      offset,
1358                      Matcher::regName[src_first]);
1359 #endif
1360         }
1361       }
1362       return 0;
1363     } else if (dst_first_rc == rc_int) {
1364       // gpr -> gpr
1365       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1366           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1367         // 64-bit
1368         if (cbuf) {
1369           MacroAssembler _masm(cbuf);
1370           __ movq(as_Register(Matcher::_regEncode[dst_first]),
1371                   as_Register(Matcher::_regEncode[src_first]));
1372 #ifndef PRODUCT
1373         } else {
1374           st->print("movq    %s, %s\t# spill",
1375                      Matcher::regName[dst_first],
1376                      Matcher::regName[src_first]);
1377 #endif
1378         }
1379         return 0;
1380       } else {
1381         // 32-bit
1382         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1383         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1384         if (cbuf) {
1385           MacroAssembler _masm(cbuf);
1386           __ movl(as_Register(Matcher::_regEncode[dst_first]),
1387                   as_Register(Matcher::_regEncode[src_first]));
1388 #ifndef PRODUCT
1389         } else {
1390           st->print("movl    %s, %s\t# spill",
1391                      Matcher::regName[dst_first],
1392                      Matcher::regName[src_first]);
1393 #endif
1394         }
1395         return 0;
1396       }
1397     } else if (dst_first_rc == rc_float) {
1398       // gpr -> xmm
1399       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1400           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1401         // 64-bit
1402         if (cbuf) {
1403           MacroAssembler _masm(cbuf);
1404           __ movdq( as_XMMRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
1405 #ifndef PRODUCT
1406         } else {
1407           st->print("movdq   %s, %s\t# spill",
1408                      Matcher::regName[dst_first],
1409                      Matcher::regName[src_first]);
1410 #endif
1411         }
1412       } else {
1413         // 32-bit
1414         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1415         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1416         if (cbuf) {
1417           MacroAssembler _masm(cbuf);
1418           __ movdl( as_XMMRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
1419 #ifndef PRODUCT
1420         } else {
1421           st->print("movdl   %s, %s\t# spill",
1422                      Matcher::regName[dst_first],
1423                      Matcher::regName[src_first]);
1424 #endif
1425         }
1426       }
1427       return 0;
1428     }
1429   } else if (src_first_rc == rc_float) {
1430     // xmm ->
1431     if (dst_first_rc == rc_stack) {
1432       // xmm -> mem
1433       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1434           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1435         // 64-bit
1436         int offset = ra_->reg2offset(dst_first);
1437         if (cbuf) {
1438           MacroAssembler _masm(cbuf);
1439           __ movdbl( Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[src_first]));
1440 #ifndef PRODUCT
1441         } else {
1442           st->print("movsd   [rsp + #%d], %s\t# spill",
1443                      offset,
1444                      Matcher::regName[src_first]);
1445 #endif
1446         }
1447       } else {
1448         // 32-bit
1449         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1450         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1451         int offset = ra_->reg2offset(dst_first);
1452         if (cbuf) {
1453           MacroAssembler _masm(cbuf);
1454           __ movflt(Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[src_first]));
1455 #ifndef PRODUCT
1456         } else {
1457           st->print("movss   [rsp + #%d], %s\t# spill",
1458                      offset,
1459                      Matcher::regName[src_first]);
1460 #endif
1461         }
1462       }
1463       return 0;
1464     } else if (dst_first_rc == rc_int) {
1465       // xmm -> gpr
1466       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1467           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1468         // 64-bit
1469         if (cbuf) {
1470           MacroAssembler _masm(cbuf);
1471           __ movdq( as_Register(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
1472 #ifndef PRODUCT
1473         } else {
1474           st->print("movdq   %s, %s\t# spill",
1475                      Matcher::regName[dst_first],
1476                      Matcher::regName[src_first]);
1477 #endif
1478         }
1479       } else {
1480         // 32-bit
1481         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1482         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1483         if (cbuf) {
1484           MacroAssembler _masm(cbuf);
1485           __ movdl( as_Register(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
1486 #ifndef PRODUCT
1487         } else {
1488           st->print("movdl   %s, %s\t# spill",
1489                      Matcher::regName[dst_first],
1490                      Matcher::regName[src_first]);
1491 #endif
1492         }
1493       }
1494       return 0;
1495     } else if (dst_first_rc == rc_float) {
1496       // xmm -> xmm
1497       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1498           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1499         // 64-bit
1500         if (cbuf) {
1501           MacroAssembler _masm(cbuf);
1502           __ movdbl( as_XMMRegister(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
1503 #ifndef PRODUCT
1504         } else {
1505           st->print("%s  %s, %s\t# spill",
1506                      UseXmmRegToRegMoveAll ? "movapd" : "movsd ",
1507                      Matcher::regName[dst_first],
1508                      Matcher::regName[src_first]);
1509 #endif
1510         }
1511       } else {
1512         // 32-bit
1513         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1514         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1515         if (cbuf) {
1516           MacroAssembler _masm(cbuf);
1517           __ movflt( as_XMMRegister(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
1518 #ifndef PRODUCT
1519         } else {
1520           st->print("%s  %s, %s\t# spill",
1521                      UseXmmRegToRegMoveAll ? "movaps" : "movss ",
1522                      Matcher::regName[dst_first],
1523                      Matcher::regName[src_first]);
1524 #endif
1525         }
1526       }
1527       return 0;
1528     }
1529   }
1530 
1531   assert(0," foo ");
1532   Unimplemented();
1533   return 0;
1534 }
1535 
1536 #ifndef PRODUCT
1537 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream* st) const {
1538   implementation(NULL, ra_, false, st);
1539 }
1540 #endif
1541 
1542 void MachSpillCopyNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1543   implementation(&cbuf, ra_, false, NULL);
1544 }
1545 
1546 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
1547   return MachNode::size(ra_);
1548 }
1549 
1550 //=============================================================================
1551 #ifndef PRODUCT
1552 void BoxLockNode::format(PhaseRegAlloc* ra_, outputStream* st) const
1553 {
1554   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
1555   int reg = ra_->get_reg_first(this);
1556   st->print("leaq    %s, [rsp + #%d]\t# box lock",
1557             Matcher::regName[reg], offset);
1558 }
1559 #endif
1560 
1561 void BoxLockNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
1562 {
1563   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
1564   int reg = ra_->get_encode(this);
1565   if (offset >= 0x80) {
1566     emit_opcode(cbuf, reg < 8 ? Assembler::REX_W : Assembler::REX_WR);
1567     emit_opcode(cbuf, 0x8D); // LEA  reg,[SP+offset]
1568     emit_rm(cbuf, 0x2, reg & 7, 0x04);
1569     emit_rm(cbuf, 0x0, 0x04, RSP_enc);
1570     emit_d32(cbuf, offset);
1571   } else {
1572     emit_opcode(cbuf, reg < 8 ? Assembler::REX_W : Assembler::REX_WR);
1573     emit_opcode(cbuf, 0x8D); // LEA  reg,[SP+offset]
1574     emit_rm(cbuf, 0x1, reg & 7, 0x04);
1575     emit_rm(cbuf, 0x0, 0x04, RSP_enc);
1576     emit_d8(cbuf, offset);
1577   }
1578 }
1579 
1580 uint BoxLockNode::size(PhaseRegAlloc *ra_) const
1581 {
1582   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
1583   return (offset < 0x80) ? 5 : 8; // REX
1584 }
1585 
1586 //=============================================================================
1587 #ifndef PRODUCT
1588 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
1589 {
1590   if (UseCompressedClassPointers) {
1591     st->print_cr("movl    rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
1592     st->print_cr("\tdecode_klass_not_null rscratch1, rscratch1");
1593     st->print_cr("\tcmpq    rax, rscratch1\t # Inline cache check");
1594   } else {
1595     st->print_cr("\tcmpq    rax, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t"
1596                  "# Inline cache check");
1597   }
1598   st->print_cr("\tjne     SharedRuntime::_ic_miss_stub");
1599   st->print_cr("\tnop\t# nops to align entry point");
1600 }
1601 #endif
1602 
1603 void MachUEPNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
1604 {
1605   MacroAssembler masm(&cbuf);
1606   uint insts_size = cbuf.insts_size();
1607   if (UseCompressedClassPointers) {
1608     masm.load_klass(rscratch1, j_rarg0);
1609     masm.cmpptr(rax, rscratch1);
1610   } else {
1611     masm.cmpptr(rax, Address(j_rarg0, oopDesc::klass_offset_in_bytes()));
1612   }
1613 
1614   masm.jump_cc(Assembler::notEqual, RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
1615 
1616   /* WARNING these NOPs are critical so that verified entry point is properly
1617      4 bytes aligned for patching by NativeJump::patch_verified_entry() */
1618   int nops_cnt = 4 - ((cbuf.insts_size() - insts_size) & 0x3);
1619   if (OptoBreakpoint) {
1620     // Leave space for int3
1621     nops_cnt -= 1;
1622   }
1623   nops_cnt &= 0x3; // Do not add nops if code is aligned.
1624   if (nops_cnt > 0)
1625     masm.nop(nops_cnt);
1626 }
1627 
1628 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
1629 {
1630   return MachNode::size(ra_); // too many variables; just compute it
1631                               // the hard way
1632 }
1633 
1634 
1635 //=============================================================================
1636 
1637 int Matcher::regnum_to_fpu_offset(int regnum)
1638 {
1639   return regnum - 32; // The FP registers are in the second chunk
1640 }
1641 
1642 // This is UltraSparc specific, true just means we have fast l2f conversion
1643 const bool Matcher::convL2FSupported(void) {
1644   return true;
1645 }
1646 
1647 // Is this branch offset short enough that a short branch can be used?
1648 //
1649 // NOTE: If the platform does not provide any short branch variants, then
1650 //       this method should return false for offset 0.
1651 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
1652   // The passed offset is relative to address of the branch.
1653   // On 86 a branch displacement is calculated relative to address
1654   // of a next instruction.
1655   offset -= br_size;
1656 
1657   // the short version of jmpConUCF2 contains multiple branches,
1658   // making the reach slightly less
1659   if (rule == jmpConUCF2_rule)
1660     return (-126 <= offset && offset <= 125);
1661   return (-128 <= offset && offset <= 127);
1662 }
1663 
1664 const bool Matcher::isSimpleConstant64(jlong value) {
1665   // Will one (StoreL ConL) be cheaper than two (StoreI ConI)?.
1666   //return value == (int) value;  // Cf. storeImmL and immL32.
1667 
1668   // Probably always true, even if a temp register is required.
1669   return true;
1670 }
1671 
1672 // The ecx parameter to rep stosq for the ClearArray node is in words.
1673 const bool Matcher::init_array_count_is_in_bytes = false;
1674 
1675 // No additional cost for CMOVL.
1676 const int Matcher::long_cmove_cost() { return 0; }
1677 
1678 // No CMOVF/CMOVD with SSE2
1679 const int Matcher::float_cmove_cost() { return ConditionalMoveLimit; }
1680 
1681 // Does the CPU require late expand (see block.cpp for description of late expand)?
1682 const bool Matcher::require_postalloc_expand = false;
1683 
1684 // Do we need to mask the count passed to shift instructions or does
1685 // the cpu only look at the lower 5/6 bits anyway?
1686 const bool Matcher::need_masked_shift_count = false;
1687 
1688 bool Matcher::narrow_oop_use_complex_address() {
1689   assert(UseCompressedOops, "only for compressed oops code");
1690   return (LogMinObjAlignmentInBytes <= 3);
1691 }
1692 
1693 bool Matcher::narrow_klass_use_complex_address() {
1694   assert(UseCompressedClassPointers, "only for compressed klass code");
1695   return (LogKlassAlignmentInBytes <= 3);
1696 }
1697 
1698 bool Matcher::const_oop_prefer_decode() {
1699   // Prefer ConN+DecodeN over ConP.
1700   return true;
1701 }
1702 
1703 bool Matcher::const_klass_prefer_decode() {
1704   // TODO: Either support matching DecodeNKlass (heap-based) in operand
1705   //       or condisider the following:
1706   // Prefer ConNKlass+DecodeNKlass over ConP in simple compressed klass mode.
1707   //return Universe::narrow_klass_base() == NULL;
1708   return true;
1709 }
1710 
1711 // Is it better to copy float constants, or load them directly from
1712 // memory?  Intel can load a float constant from a direct address,
1713 // requiring no extra registers.  Most RISCs will have to materialize
1714 // an address into a register first, so they would do better to copy
1715 // the constant from stack.
1716 const bool Matcher::rematerialize_float_constants = true; // XXX
1717 
1718 // If CPU can load and store mis-aligned doubles directly then no
1719 // fixup is needed.  Else we split the double into 2 integer pieces
1720 // and move it piece-by-piece.  Only happens when passing doubles into
1721 // C code as the Java calling convention forces doubles to be aligned.
1722 const bool Matcher::misaligned_doubles_ok = true;
1723 
1724 // No-op on amd64
1725 void Matcher::pd_implicit_null_fixup(MachNode *node, uint idx) {}
1726 
1727 // Advertise here if the CPU requires explicit rounding operations to
1728 // implement the UseStrictFP mode.
1729 const bool Matcher::strict_fp_requires_explicit_rounding = true;
1730 
1731 // Are floats conerted to double when stored to stack during deoptimization?
1732 // On x64 it is stored without convertion so we can use normal access.
1733 bool Matcher::float_in_double() { return false; }
1734 
1735 // Do ints take an entire long register or just half?
1736 const bool Matcher::int_in_long = true;
1737 
1738 // Return whether or not this register is ever used as an argument.
1739 // This function is used on startup to build the trampoline stubs in
1740 // generateOptoStub.  Registers not mentioned will be killed by the VM
1741 // call in the trampoline, and arguments in those registers not be
1742 // available to the callee.
1743 bool Matcher::can_be_java_arg(int reg)
1744 {
1745   return
1746     reg ==  RDI_num || reg == RDI_H_num ||
1747     reg ==  RSI_num || reg == RSI_H_num ||
1748     reg ==  RDX_num || reg == RDX_H_num ||
1749     reg ==  RCX_num || reg == RCX_H_num ||
1750     reg ==   R8_num || reg ==  R8_H_num ||
1751     reg ==   R9_num || reg ==  R9_H_num ||
1752     reg ==  R12_num || reg == R12_H_num ||
1753     reg == XMM0_num || reg == XMM0b_num ||
1754     reg == XMM1_num || reg == XMM1b_num ||
1755     reg == XMM2_num || reg == XMM2b_num ||
1756     reg == XMM3_num || reg == XMM3b_num ||
1757     reg == XMM4_num || reg == XMM4b_num ||
1758     reg == XMM5_num || reg == XMM5b_num ||
1759     reg == XMM6_num || reg == XMM6b_num ||
1760     reg == XMM7_num || reg == XMM7b_num;
1761 }
1762 
1763 bool Matcher::is_spillable_arg(int reg)
1764 {
1765   return can_be_java_arg(reg);
1766 }
1767 
1768 bool Matcher::use_asm_for_ldiv_by_con( jlong divisor ) {
1769   // In 64 bit mode a code which use multiply when
1770   // devisor is constant is faster than hardware
1771   // DIV instruction (it uses MulHiL).
1772   return false;
1773 }
1774 
1775 // Register for DIVI projection of divmodI
1776 RegMask Matcher::divI_proj_mask() {
1777   return INT_RAX_REG_mask();
1778 }
1779 
1780 // Register for MODI projection of divmodI
1781 RegMask Matcher::modI_proj_mask() {
1782   return INT_RDX_REG_mask();
1783 }
1784 
1785 // Register for DIVL projection of divmodL
1786 RegMask Matcher::divL_proj_mask() {
1787   return LONG_RAX_REG_mask();
1788 }
1789 
1790 // Register for MODL projection of divmodL
1791 RegMask Matcher::modL_proj_mask() {
1792   return LONG_RDX_REG_mask();
1793 }
1794 
1795 // Register for saving SP into on method handle invokes. Not used on x86_64.
1796 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
1797     return NO_REG_mask();
1798 }
1799 
1800 %}
1801 
1802 //----------ENCODING BLOCK-----------------------------------------------------
1803 // This block specifies the encoding classes used by the compiler to
1804 // output byte streams.  Encoding classes are parameterized macros
1805 // used by Machine Instruction Nodes in order to generate the bit
1806 // encoding of the instruction.  Operands specify their base encoding
1807 // interface with the interface keyword.  There are currently
1808 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
1809 // COND_INTER.  REG_INTER causes an operand to generate a function
1810 // which returns its register number when queried.  CONST_INTER causes
1811 // an operand to generate a function which returns the value of the
1812 // constant when queried.  MEMORY_INTER causes an operand to generate
1813 // four functions which return the Base Register, the Index Register,
1814 // the Scale Value, and the Offset Value of the operand when queried.
1815 // COND_INTER causes an operand to generate six functions which return
1816 // the encoding code (ie - encoding bits for the instruction)
1817 // associated with each basic boolean condition for a conditional
1818 // instruction.
1819 //
1820 // Instructions specify two basic values for encoding.  Again, a
1821 // function is available to check if the constant displacement is an
1822 // oop. They use the ins_encode keyword to specify their encoding
1823 // classes (which must be a sequence of enc_class names, and their
1824 // parameters, specified in the encoding block), and they use the
1825 // opcode keyword to specify, in order, their primary, secondary, and
1826 // tertiary opcode.  Only the opcode sections which a particular
1827 // instruction needs for encoding need to be specified.
1828 encode %{
1829   // Build emit functions for each basic byte or larger field in the
1830   // intel encoding scheme (opcode, rm, sib, immediate), and call them
1831   // from C++ code in the enc_class source block.  Emit functions will
1832   // live in the main source block for now.  In future, we can
1833   // generalize this by adding a syntax that specifies the sizes of
1834   // fields in an order, so that the adlc can build the emit functions
1835   // automagically
1836 
1837   // Emit primary opcode
1838   enc_class OpcP
1839   %{
1840     emit_opcode(cbuf, $primary);
1841   %}
1842 
1843   // Emit secondary opcode
1844   enc_class OpcS
1845   %{
1846     emit_opcode(cbuf, $secondary);
1847   %}
1848 
1849   // Emit tertiary opcode
1850   enc_class OpcT
1851   %{
1852     emit_opcode(cbuf, $tertiary);
1853   %}
1854 
1855   // Emit opcode directly
1856   enc_class Opcode(immI d8)
1857   %{
1858     emit_opcode(cbuf, $d8$$constant);
1859   %}
1860 
1861   // Emit size prefix
1862   enc_class SizePrefix
1863   %{
1864     emit_opcode(cbuf, 0x66);
1865   %}
1866 
1867   enc_class reg(rRegI reg)
1868   %{
1869     emit_rm(cbuf, 0x3, 0, $reg$$reg & 7);
1870   %}
1871 
1872   enc_class reg_reg(rRegI dst, rRegI src)
1873   %{
1874     emit_rm(cbuf, 0x3, $dst$$reg & 7, $src$$reg & 7);
1875   %}
1876 
1877   enc_class opc_reg_reg(immI opcode, rRegI dst, rRegI src)
1878   %{
1879     emit_opcode(cbuf, $opcode$$constant);
1880     emit_rm(cbuf, 0x3, $dst$$reg & 7, $src$$reg & 7);
1881   %}
1882 
1883   enc_class cdql_enc(no_rax_rdx_RegI div)
1884   %{
1885     // Full implementation of Java idiv and irem; checks for
1886     // special case as described in JVM spec., p.243 & p.271.
1887     //
1888     //         normal case                           special case
1889     //
1890     // input : rax: dividend                         min_int
1891     //         reg: divisor                          -1
1892     //
1893     // output: rax: quotient  (= rax idiv reg)       min_int
1894     //         rdx: remainder (= rax irem reg)       0
1895     //
1896     //  Code sequnce:
1897     //
1898     //    0:   3d 00 00 00 80          cmp    $0x80000000,%eax
1899     //    5:   75 07/08                jne    e <normal>
1900     //    7:   33 d2                   xor    %edx,%edx
1901     //  [div >= 8 -> offset + 1]
1902     //  [REX_B]
1903     //    9:   83 f9 ff                cmp    $0xffffffffffffffff,$div
1904     //    c:   74 03/04                je     11 <done>
1905     // 000000000000000e <normal>:
1906     //    e:   99                      cltd
1907     //  [div >= 8 -> offset + 1]
1908     //  [REX_B]
1909     //    f:   f7 f9                   idiv   $div
1910     // 0000000000000011 <done>:
1911 
1912     // cmp    $0x80000000,%eax
1913     emit_opcode(cbuf, 0x3d);
1914     emit_d8(cbuf, 0x00);
1915     emit_d8(cbuf, 0x00);
1916     emit_d8(cbuf, 0x00);
1917     emit_d8(cbuf, 0x80);
1918 
1919     // jne    e <normal>
1920     emit_opcode(cbuf, 0x75);
1921     emit_d8(cbuf, $div$$reg < 8 ? 0x07 : 0x08);
1922 
1923     // xor    %edx,%edx
1924     emit_opcode(cbuf, 0x33);
1925     emit_d8(cbuf, 0xD2);
1926 
1927     // cmp    $0xffffffffffffffff,%ecx
1928     if ($div$$reg >= 8) {
1929       emit_opcode(cbuf, Assembler::REX_B);
1930     }
1931     emit_opcode(cbuf, 0x83);
1932     emit_rm(cbuf, 0x3, 0x7, $div$$reg & 7);
1933     emit_d8(cbuf, 0xFF);
1934 
1935     // je     11 <done>
1936     emit_opcode(cbuf, 0x74);
1937     emit_d8(cbuf, $div$$reg < 8 ? 0x03 : 0x04);
1938 
1939     // <normal>
1940     // cltd
1941     emit_opcode(cbuf, 0x99);
1942 
1943     // idivl (note: must be emitted by the user of this rule)
1944     // <done>
1945   %}
1946 
1947   enc_class cdqq_enc(no_rax_rdx_RegL div)
1948   %{
1949     // Full implementation of Java ldiv and lrem; checks for
1950     // special case as described in JVM spec., p.243 & p.271.
1951     //
1952     //         normal case                           special case
1953     //
1954     // input : rax: dividend                         min_long
1955     //         reg: divisor                          -1
1956     //
1957     // output: rax: quotient  (= rax idiv reg)       min_long
1958     //         rdx: remainder (= rax irem reg)       0
1959     //
1960     //  Code sequnce:
1961     //
1962     //    0:   48 ba 00 00 00 00 00    mov    $0x8000000000000000,%rdx
1963     //    7:   00 00 80
1964     //    a:   48 39 d0                cmp    %rdx,%rax
1965     //    d:   75 08                   jne    17 <normal>
1966     //    f:   33 d2                   xor    %edx,%edx
1967     //   11:   48 83 f9 ff             cmp    $0xffffffffffffffff,$div
1968     //   15:   74 05                   je     1c <done>
1969     // 0000000000000017 <normal>:
1970     //   17:   48 99                   cqto
1971     //   19:   48 f7 f9                idiv   $div
1972     // 000000000000001c <done>:
1973 
1974     // mov    $0x8000000000000000,%rdx
1975     emit_opcode(cbuf, Assembler::REX_W);
1976     emit_opcode(cbuf, 0xBA);
1977     emit_d8(cbuf, 0x00);
1978     emit_d8(cbuf, 0x00);
1979     emit_d8(cbuf, 0x00);
1980     emit_d8(cbuf, 0x00);
1981     emit_d8(cbuf, 0x00);
1982     emit_d8(cbuf, 0x00);
1983     emit_d8(cbuf, 0x00);
1984     emit_d8(cbuf, 0x80);
1985 
1986     // cmp    %rdx,%rax
1987     emit_opcode(cbuf, Assembler::REX_W);
1988     emit_opcode(cbuf, 0x39);
1989     emit_d8(cbuf, 0xD0);
1990 
1991     // jne    17 <normal>
1992     emit_opcode(cbuf, 0x75);
1993     emit_d8(cbuf, 0x08);
1994 
1995     // xor    %edx,%edx
1996     emit_opcode(cbuf, 0x33);
1997     emit_d8(cbuf, 0xD2);
1998 
1999     // cmp    $0xffffffffffffffff,$div
2000     emit_opcode(cbuf, $div$$reg < 8 ? Assembler::REX_W : Assembler::REX_WB);
2001     emit_opcode(cbuf, 0x83);
2002     emit_rm(cbuf, 0x3, 0x7, $div$$reg & 7);
2003     emit_d8(cbuf, 0xFF);
2004 
2005     // je     1e <done>
2006     emit_opcode(cbuf, 0x74);
2007     emit_d8(cbuf, 0x05);
2008 
2009     // <normal>
2010     // cqto
2011     emit_opcode(cbuf, Assembler::REX_W);
2012     emit_opcode(cbuf, 0x99);
2013 
2014     // idivq (note: must be emitted by the user of this rule)
2015     // <done>
2016   %}
2017 
2018   // Opcde enc_class for 8/32 bit immediate instructions with sign-extension
2019   enc_class OpcSE(immI imm)
2020   %{
2021     // Emit primary opcode and set sign-extend bit
2022     // Check for 8-bit immediate, and set sign extend bit in opcode
2023     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
2024       emit_opcode(cbuf, $primary | 0x02);
2025     } else {
2026       // 32-bit immediate
2027       emit_opcode(cbuf, $primary);
2028     }
2029   %}
2030 
2031   enc_class OpcSErm(rRegI dst, immI imm)
2032   %{
2033     // OpcSEr/m
2034     int dstenc = $dst$$reg;
2035     if (dstenc >= 8) {
2036       emit_opcode(cbuf, Assembler::REX_B);
2037       dstenc -= 8;
2038     }
2039     // Emit primary opcode and set sign-extend bit
2040     // Check for 8-bit immediate, and set sign extend bit in opcode
2041     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
2042       emit_opcode(cbuf, $primary | 0x02);
2043     } else {
2044       // 32-bit immediate
2045       emit_opcode(cbuf, $primary);
2046     }
2047     // Emit r/m byte with secondary opcode, after primary opcode.
2048     emit_rm(cbuf, 0x3, $secondary, dstenc);
2049   %}
2050 
2051   enc_class OpcSErm_wide(rRegL dst, immI imm)
2052   %{
2053     // OpcSEr/m
2054     int dstenc = $dst$$reg;
2055     if (dstenc < 8) {
2056       emit_opcode(cbuf, Assembler::REX_W);
2057     } else {
2058       emit_opcode(cbuf, Assembler::REX_WB);
2059       dstenc -= 8;
2060     }
2061     // Emit primary opcode and set sign-extend bit
2062     // Check for 8-bit immediate, and set sign extend bit in opcode
2063     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
2064       emit_opcode(cbuf, $primary | 0x02);
2065     } else {
2066       // 32-bit immediate
2067       emit_opcode(cbuf, $primary);
2068     }
2069     // Emit r/m byte with secondary opcode, after primary opcode.
2070     emit_rm(cbuf, 0x3, $secondary, dstenc);
2071   %}
2072 
2073   enc_class Con8or32(immI imm)
2074   %{
2075     // Check for 8-bit immediate, and set sign extend bit in opcode
2076     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
2077       $$$emit8$imm$$constant;
2078     } else {
2079       // 32-bit immediate
2080       $$$emit32$imm$$constant;
2081     }
2082   %}
2083 
2084   enc_class opc2_reg(rRegI dst)
2085   %{
2086     // BSWAP
2087     emit_cc(cbuf, $secondary, $dst$$reg);
2088   %}
2089 
2090   enc_class opc3_reg(rRegI dst)
2091   %{
2092     // BSWAP
2093     emit_cc(cbuf, $tertiary, $dst$$reg);
2094   %}
2095 
2096   enc_class reg_opc(rRegI div)
2097   %{
2098     // INC, DEC, IDIV, IMOD, JMP indirect, ...
2099     emit_rm(cbuf, 0x3, $secondary, $div$$reg & 7);
2100   %}
2101 
2102   enc_class enc_cmov(cmpOp cop)
2103   %{
2104     // CMOV
2105     $$$emit8$primary;
2106     emit_cc(cbuf, $secondary, $cop$$cmpcode);
2107   %}
2108 
2109   enc_class enc_PartialSubtypeCheck()
2110   %{
2111     Register Rrdi = as_Register(RDI_enc); // result register
2112     Register Rrax = as_Register(RAX_enc); // super class
2113     Register Rrcx = as_Register(RCX_enc); // killed
2114     Register Rrsi = as_Register(RSI_enc); // sub class
2115     Label miss;
2116     const bool set_cond_codes = true;
2117 
2118     MacroAssembler _masm(&cbuf);
2119     __ check_klass_subtype_slow_path(Rrsi, Rrax, Rrcx, Rrdi,
2120                                      NULL, &miss,
2121                                      /*set_cond_codes:*/ true);
2122     if ($primary) {
2123       __ xorptr(Rrdi, Rrdi);
2124     }
2125     __ bind(miss);
2126   %}
2127 
2128   enc_class clear_avx %{
2129     debug_only(int off0 = cbuf.insts_size());
2130     if (generate_vzeroupper(Compile::current())) {
2131       // Clear upper bits of YMM registers to avoid AVX <-> SSE transition penalty
2132       // Clear upper bits of YMM registers when current compiled code uses
2133       // wide vectors to avoid AVX <-> SSE transition penalty during call.
2134       MacroAssembler _masm(&cbuf);
2135       __ vzeroupper();
2136     }
2137     debug_only(int off1 = cbuf.insts_size());
2138     assert(off1 - off0 == clear_avx_size(), "correct size prediction");
2139   %}
2140 
2141   enc_class Java_To_Runtime(method meth) %{
2142     // No relocation needed
2143     MacroAssembler _masm(&cbuf);
2144     __ mov64(r10, (int64_t) $meth$$method);
2145     __ call(r10);
2146   %}
2147 
2148   enc_class Java_To_Interpreter(method meth)
2149   %{
2150     // CALL Java_To_Interpreter
2151     // This is the instruction starting address for relocation info.
2152     cbuf.set_insts_mark();
2153     $$$emit8$primary;
2154     // CALL directly to the runtime
2155     emit_d32_reloc(cbuf,
2156                    (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
2157                    runtime_call_Relocation::spec(),
2158                    RELOC_DISP32);
2159   %}
2160 
2161   enc_class Java_Static_Call(method meth)
2162   %{
2163     // JAVA STATIC CALL
2164     // CALL to fixup routine.  Fixup routine uses ScopeDesc info to
2165     // determine who we intended to call.
2166     cbuf.set_insts_mark();
2167     $$$emit8$primary;
2168 
2169     if (!_method) {
2170       emit_d32_reloc(cbuf, (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
2171                      runtime_call_Relocation::spec(),
2172                      RELOC_DISP32);
2173     } else {
2174       int method_index = resolved_method_index(cbuf);
2175       RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
2176                                                   : static_call_Relocation::spec(method_index);
2177       emit_d32_reloc(cbuf, (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
2178                      rspec, RELOC_DISP32);
2179       // Emit stubs for static call.
2180       address mark = cbuf.insts_mark();
2181       address stub = CompiledStaticCall::emit_to_interp_stub(cbuf, mark);
2182       if (stub == NULL) {
2183         ciEnv::current()->record_failure("CodeCache is full");
2184         return;
2185       }
2186 #if INCLUDE_AOT
2187       CompiledStaticCall::emit_to_aot_stub(cbuf, mark);
2188 #endif
2189     }
2190   %}
2191 
2192   enc_class Java_Dynamic_Call(method meth) %{
2193     MacroAssembler _masm(&cbuf);
2194     __ ic_call((address)$meth$$method, resolved_method_index(cbuf));
2195   %}
2196 
2197   enc_class Java_Compiled_Call(method meth)
2198   %{
2199     // JAVA COMPILED CALL
2200     int disp = in_bytes(Method:: from_compiled_offset());
2201 
2202     // XXX XXX offset is 128 is 1.5 NON-PRODUCT !!!
2203     // assert(-0x80 <= disp && disp < 0x80, "compiled_code_offset isn't small");
2204 
2205     // callq *disp(%rax)
2206     cbuf.set_insts_mark();
2207     $$$emit8$primary;
2208     if (disp < 0x80) {
2209       emit_rm(cbuf, 0x01, $secondary, RAX_enc); // R/M byte
2210       emit_d8(cbuf, disp); // Displacement
2211     } else {
2212       emit_rm(cbuf, 0x02, $secondary, RAX_enc); // R/M byte
2213       emit_d32(cbuf, disp); // Displacement
2214     }
2215   %}
2216 
2217   enc_class reg_opc_imm(rRegI dst, immI8 shift)
2218   %{
2219     // SAL, SAR, SHR
2220     int dstenc = $dst$$reg;
2221     if (dstenc >= 8) {
2222       emit_opcode(cbuf, Assembler::REX_B);
2223       dstenc -= 8;
2224     }
2225     $$$emit8$primary;
2226     emit_rm(cbuf, 0x3, $secondary, dstenc);
2227     $$$emit8$shift$$constant;
2228   %}
2229 
2230   enc_class reg_opc_imm_wide(rRegL dst, immI8 shift)
2231   %{
2232     // SAL, SAR, SHR
2233     int dstenc = $dst$$reg;
2234     if (dstenc < 8) {
2235       emit_opcode(cbuf, Assembler::REX_W);
2236     } else {
2237       emit_opcode(cbuf, Assembler::REX_WB);
2238       dstenc -= 8;
2239     }
2240     $$$emit8$primary;
2241     emit_rm(cbuf, 0x3, $secondary, dstenc);
2242     $$$emit8$shift$$constant;
2243   %}
2244 
2245   enc_class load_immI(rRegI dst, immI src)
2246   %{
2247     int dstenc = $dst$$reg;
2248     if (dstenc >= 8) {
2249       emit_opcode(cbuf, Assembler::REX_B);
2250       dstenc -= 8;
2251     }
2252     emit_opcode(cbuf, 0xB8 | dstenc);
2253     $$$emit32$src$$constant;
2254   %}
2255 
2256   enc_class load_immL(rRegL dst, immL src)
2257   %{
2258     int dstenc = $dst$$reg;
2259     if (dstenc < 8) {
2260       emit_opcode(cbuf, Assembler::REX_W);
2261     } else {
2262       emit_opcode(cbuf, Assembler::REX_WB);
2263       dstenc -= 8;
2264     }
2265     emit_opcode(cbuf, 0xB8 | dstenc);
2266     emit_d64(cbuf, $src$$constant);
2267   %}
2268 
2269   enc_class load_immUL32(rRegL dst, immUL32 src)
2270   %{
2271     // same as load_immI, but this time we care about zeroes in the high word
2272     int dstenc = $dst$$reg;
2273     if (dstenc >= 8) {
2274       emit_opcode(cbuf, Assembler::REX_B);
2275       dstenc -= 8;
2276     }
2277     emit_opcode(cbuf, 0xB8 | dstenc);
2278     $$$emit32$src$$constant;
2279   %}
2280 
2281   enc_class load_immL32(rRegL dst, immL32 src)
2282   %{
2283     int dstenc = $dst$$reg;
2284     if (dstenc < 8) {
2285       emit_opcode(cbuf, Assembler::REX_W);
2286     } else {
2287       emit_opcode(cbuf, Assembler::REX_WB);
2288       dstenc -= 8;
2289     }
2290     emit_opcode(cbuf, 0xC7);
2291     emit_rm(cbuf, 0x03, 0x00, dstenc);
2292     $$$emit32$src$$constant;
2293   %}
2294 
2295   enc_class load_immP31(rRegP dst, immP32 src)
2296   %{
2297     // same as load_immI, but this time we care about zeroes in the high word
2298     int dstenc = $dst$$reg;
2299     if (dstenc >= 8) {
2300       emit_opcode(cbuf, Assembler::REX_B);
2301       dstenc -= 8;
2302     }
2303     emit_opcode(cbuf, 0xB8 | dstenc);
2304     $$$emit32$src$$constant;
2305   %}
2306 
2307   enc_class load_immP(rRegP dst, immP src)
2308   %{
2309     int dstenc = $dst$$reg;
2310     if (dstenc < 8) {
2311       emit_opcode(cbuf, Assembler::REX_W);
2312     } else {
2313       emit_opcode(cbuf, Assembler::REX_WB);
2314       dstenc -= 8;
2315     }
2316     emit_opcode(cbuf, 0xB8 | dstenc);
2317     // This next line should be generated from ADLC
2318     if ($src->constant_reloc() != relocInfo::none) {
2319       emit_d64_reloc(cbuf, $src$$constant, $src->constant_reloc(), RELOC_IMM64);
2320     } else {
2321       emit_d64(cbuf, $src$$constant);
2322     }
2323   %}
2324 
2325   enc_class Con32(immI src)
2326   %{
2327     // Output immediate
2328     $$$emit32$src$$constant;
2329   %}
2330 
2331   enc_class Con32F_as_bits(immF src)
2332   %{
2333     // Output Float immediate bits
2334     jfloat jf = $src$$constant;
2335     jint jf_as_bits = jint_cast(jf);
2336     emit_d32(cbuf, jf_as_bits);
2337   %}
2338 
2339   enc_class Con16(immI src)
2340   %{
2341     // Output immediate
2342     $$$emit16$src$$constant;
2343   %}
2344 
2345   // How is this different from Con32??? XXX
2346   enc_class Con_d32(immI src)
2347   %{
2348     emit_d32(cbuf,$src$$constant);
2349   %}
2350 
2351   enc_class conmemref (rRegP t1) %{    // Con32(storeImmI)
2352     // Output immediate memory reference
2353     emit_rm(cbuf, 0x00, $t1$$reg, 0x05 );
2354     emit_d32(cbuf, 0x00);
2355   %}
2356 
2357   enc_class lock_prefix()
2358   %{
2359     if (os::is_MP()) {
2360       emit_opcode(cbuf, 0xF0); // lock
2361     }
2362   %}
2363 
2364   enc_class REX_mem(memory mem)
2365   %{
2366     if ($mem$$base >= 8) {
2367       if ($mem$$index < 8) {
2368         emit_opcode(cbuf, Assembler::REX_B);
2369       } else {
2370         emit_opcode(cbuf, Assembler::REX_XB);
2371       }
2372     } else {
2373       if ($mem$$index >= 8) {
2374         emit_opcode(cbuf, Assembler::REX_X);
2375       }
2376     }
2377   %}
2378 
2379   enc_class REX_mem_wide(memory mem)
2380   %{
2381     if ($mem$$base >= 8) {
2382       if ($mem$$index < 8) {
2383         emit_opcode(cbuf, Assembler::REX_WB);
2384       } else {
2385         emit_opcode(cbuf, Assembler::REX_WXB);
2386       }
2387     } else {
2388       if ($mem$$index < 8) {
2389         emit_opcode(cbuf, Assembler::REX_W);
2390       } else {
2391         emit_opcode(cbuf, Assembler::REX_WX);
2392       }
2393     }
2394   %}
2395 
2396   // for byte regs
2397   enc_class REX_breg(rRegI reg)
2398   %{
2399     if ($reg$$reg >= 4) {
2400       emit_opcode(cbuf, $reg$$reg < 8 ? Assembler::REX : Assembler::REX_B);
2401     }
2402   %}
2403 
2404   // for byte regs
2405   enc_class REX_reg_breg(rRegI dst, rRegI src)
2406   %{
2407     if ($dst$$reg < 8) {
2408       if ($src$$reg >= 4) {
2409         emit_opcode(cbuf, $src$$reg < 8 ? Assembler::REX : Assembler::REX_B);
2410       }
2411     } else {
2412       if ($src$$reg < 8) {
2413         emit_opcode(cbuf, Assembler::REX_R);
2414       } else {
2415         emit_opcode(cbuf, Assembler::REX_RB);
2416       }
2417     }
2418   %}
2419 
2420   // for byte regs
2421   enc_class REX_breg_mem(rRegI reg, memory mem)
2422   %{
2423     if ($reg$$reg < 8) {
2424       if ($mem$$base < 8) {
2425         if ($mem$$index >= 8) {
2426           emit_opcode(cbuf, Assembler::REX_X);
2427         } else if ($reg$$reg >= 4) {
2428           emit_opcode(cbuf, Assembler::REX);
2429         }
2430       } else {
2431         if ($mem$$index < 8) {
2432           emit_opcode(cbuf, Assembler::REX_B);
2433         } else {
2434           emit_opcode(cbuf, Assembler::REX_XB);
2435         }
2436       }
2437     } else {
2438       if ($mem$$base < 8) {
2439         if ($mem$$index < 8) {
2440           emit_opcode(cbuf, Assembler::REX_R);
2441         } else {
2442           emit_opcode(cbuf, Assembler::REX_RX);
2443         }
2444       } else {
2445         if ($mem$$index < 8) {
2446           emit_opcode(cbuf, Assembler::REX_RB);
2447         } else {
2448           emit_opcode(cbuf, Assembler::REX_RXB);
2449         }
2450       }
2451     }
2452   %}
2453 
2454   enc_class REX_reg(rRegI reg)
2455   %{
2456     if ($reg$$reg >= 8) {
2457       emit_opcode(cbuf, Assembler::REX_B);
2458     }
2459   %}
2460 
2461   enc_class REX_reg_wide(rRegI reg)
2462   %{
2463     if ($reg$$reg < 8) {
2464       emit_opcode(cbuf, Assembler::REX_W);
2465     } else {
2466       emit_opcode(cbuf, Assembler::REX_WB);
2467     }
2468   %}
2469 
2470   enc_class REX_reg_reg(rRegI dst, rRegI src)
2471   %{
2472     if ($dst$$reg < 8) {
2473       if ($src$$reg >= 8) {
2474         emit_opcode(cbuf, Assembler::REX_B);
2475       }
2476     } else {
2477       if ($src$$reg < 8) {
2478         emit_opcode(cbuf, Assembler::REX_R);
2479       } else {
2480         emit_opcode(cbuf, Assembler::REX_RB);
2481       }
2482     }
2483   %}
2484 
2485   enc_class REX_reg_reg_wide(rRegI dst, rRegI src)
2486   %{
2487     if ($dst$$reg < 8) {
2488       if ($src$$reg < 8) {
2489         emit_opcode(cbuf, Assembler::REX_W);
2490       } else {
2491         emit_opcode(cbuf, Assembler::REX_WB);
2492       }
2493     } else {
2494       if ($src$$reg < 8) {
2495         emit_opcode(cbuf, Assembler::REX_WR);
2496       } else {
2497         emit_opcode(cbuf, Assembler::REX_WRB);
2498       }
2499     }
2500   %}
2501 
2502   enc_class REX_reg_mem(rRegI reg, memory mem)
2503   %{
2504     if ($reg$$reg < 8) {
2505       if ($mem$$base < 8) {
2506         if ($mem$$index >= 8) {
2507           emit_opcode(cbuf, Assembler::REX_X);
2508         }
2509       } else {
2510         if ($mem$$index < 8) {
2511           emit_opcode(cbuf, Assembler::REX_B);
2512         } else {
2513           emit_opcode(cbuf, Assembler::REX_XB);
2514         }
2515       }
2516     } else {
2517       if ($mem$$base < 8) {
2518         if ($mem$$index < 8) {
2519           emit_opcode(cbuf, Assembler::REX_R);
2520         } else {
2521           emit_opcode(cbuf, Assembler::REX_RX);
2522         }
2523       } else {
2524         if ($mem$$index < 8) {
2525           emit_opcode(cbuf, Assembler::REX_RB);
2526         } else {
2527           emit_opcode(cbuf, Assembler::REX_RXB);
2528         }
2529       }
2530     }
2531   %}
2532 
2533   enc_class REX_reg_mem_wide(rRegL reg, memory mem)
2534   %{
2535     if ($reg$$reg < 8) {
2536       if ($mem$$base < 8) {
2537         if ($mem$$index < 8) {
2538           emit_opcode(cbuf, Assembler::REX_W);
2539         } else {
2540           emit_opcode(cbuf, Assembler::REX_WX);
2541         }
2542       } else {
2543         if ($mem$$index < 8) {
2544           emit_opcode(cbuf, Assembler::REX_WB);
2545         } else {
2546           emit_opcode(cbuf, Assembler::REX_WXB);
2547         }
2548       }
2549     } else {
2550       if ($mem$$base < 8) {
2551         if ($mem$$index < 8) {
2552           emit_opcode(cbuf, Assembler::REX_WR);
2553         } else {
2554           emit_opcode(cbuf, Assembler::REX_WRX);
2555         }
2556       } else {
2557         if ($mem$$index < 8) {
2558           emit_opcode(cbuf, Assembler::REX_WRB);
2559         } else {
2560           emit_opcode(cbuf, Assembler::REX_WRXB);
2561         }
2562       }
2563     }
2564   %}
2565 
2566   enc_class reg_mem(rRegI ereg, memory mem)
2567   %{
2568     // High registers handle in encode_RegMem
2569     int reg = $ereg$$reg;
2570     int base = $mem$$base;
2571     int index = $mem$$index;
2572     int scale = $mem$$scale;
2573     int disp = $mem$$disp;
2574     relocInfo::relocType disp_reloc = $mem->disp_reloc();
2575 
2576     encode_RegMem(cbuf, reg, base, index, scale, disp, disp_reloc);
2577   %}
2578 
2579   enc_class RM_opc_mem(immI rm_opcode, memory mem)
2580   %{
2581     int rm_byte_opcode = $rm_opcode$$constant;
2582 
2583     // High registers handle in encode_RegMem
2584     int base = $mem$$base;
2585     int index = $mem$$index;
2586     int scale = $mem$$scale;
2587     int displace = $mem$$disp;
2588 
2589     relocInfo::relocType disp_reloc = $mem->disp_reloc();       // disp-as-oop when
2590                                             // working with static
2591                                             // globals
2592     encode_RegMem(cbuf, rm_byte_opcode, base, index, scale, displace,
2593                   disp_reloc);
2594   %}
2595 
2596   enc_class reg_lea(rRegI dst, rRegI src0, immI src1)
2597   %{
2598     int reg_encoding = $dst$$reg;
2599     int base         = $src0$$reg;      // 0xFFFFFFFF indicates no base
2600     int index        = 0x04;            // 0x04 indicates no index
2601     int scale        = 0x00;            // 0x00 indicates no scale
2602     int displace     = $src1$$constant; // 0x00 indicates no displacement
2603     relocInfo::relocType disp_reloc = relocInfo::none;
2604     encode_RegMem(cbuf, reg_encoding, base, index, scale, displace,
2605                   disp_reloc);
2606   %}
2607 
2608   enc_class neg_reg(rRegI dst)
2609   %{
2610     int dstenc = $dst$$reg;
2611     if (dstenc >= 8) {
2612       emit_opcode(cbuf, Assembler::REX_B);
2613       dstenc -= 8;
2614     }
2615     // NEG $dst
2616     emit_opcode(cbuf, 0xF7);
2617     emit_rm(cbuf, 0x3, 0x03, dstenc);
2618   %}
2619 
2620   enc_class neg_reg_wide(rRegI dst)
2621   %{
2622     int dstenc = $dst$$reg;
2623     if (dstenc < 8) {
2624       emit_opcode(cbuf, Assembler::REX_W);
2625     } else {
2626       emit_opcode(cbuf, Assembler::REX_WB);
2627       dstenc -= 8;
2628     }
2629     // NEG $dst
2630     emit_opcode(cbuf, 0xF7);
2631     emit_rm(cbuf, 0x3, 0x03, dstenc);
2632   %}
2633 
2634   enc_class setLT_reg(rRegI dst)
2635   %{
2636     int dstenc = $dst$$reg;
2637     if (dstenc >= 8) {
2638       emit_opcode(cbuf, Assembler::REX_B);
2639       dstenc -= 8;
2640     } else if (dstenc >= 4) {
2641       emit_opcode(cbuf, Assembler::REX);
2642     }
2643     // SETLT $dst
2644     emit_opcode(cbuf, 0x0F);
2645     emit_opcode(cbuf, 0x9C);
2646     emit_rm(cbuf, 0x3, 0x0, dstenc);
2647   %}
2648 
2649   enc_class setNZ_reg(rRegI dst)
2650   %{
2651     int dstenc = $dst$$reg;
2652     if (dstenc >= 8) {
2653       emit_opcode(cbuf, Assembler::REX_B);
2654       dstenc -= 8;
2655     } else if (dstenc >= 4) {
2656       emit_opcode(cbuf, Assembler::REX);
2657     }
2658     // SETNZ $dst
2659     emit_opcode(cbuf, 0x0F);
2660     emit_opcode(cbuf, 0x95);
2661     emit_rm(cbuf, 0x3, 0x0, dstenc);
2662   %}
2663 
2664 
2665   // Compare the lonogs and set -1, 0, or 1 into dst
2666   enc_class cmpl3_flag(rRegL src1, rRegL src2, rRegI dst)
2667   %{
2668     int src1enc = $src1$$reg;
2669     int src2enc = $src2$$reg;
2670     int dstenc = $dst$$reg;
2671 
2672     // cmpq $src1, $src2
2673     if (src1enc < 8) {
2674       if (src2enc < 8) {
2675         emit_opcode(cbuf, Assembler::REX_W);
2676       } else {
2677         emit_opcode(cbuf, Assembler::REX_WB);
2678       }
2679     } else {
2680       if (src2enc < 8) {
2681         emit_opcode(cbuf, Assembler::REX_WR);
2682       } else {
2683         emit_opcode(cbuf, Assembler::REX_WRB);
2684       }
2685     }
2686     emit_opcode(cbuf, 0x3B);
2687     emit_rm(cbuf, 0x3, src1enc & 7, src2enc & 7);
2688 
2689     // movl $dst, -1
2690     if (dstenc >= 8) {
2691       emit_opcode(cbuf, Assembler::REX_B);
2692     }
2693     emit_opcode(cbuf, 0xB8 | (dstenc & 7));
2694     emit_d32(cbuf, -1);
2695 
2696     // jl,s done
2697     emit_opcode(cbuf, 0x7C);
2698     emit_d8(cbuf, dstenc < 4 ? 0x06 : 0x08);
2699 
2700     // setne $dst
2701     if (dstenc >= 4) {
2702       emit_opcode(cbuf, dstenc < 8 ? Assembler::REX : Assembler::REX_B);
2703     }
2704     emit_opcode(cbuf, 0x0F);
2705     emit_opcode(cbuf, 0x95);
2706     emit_opcode(cbuf, 0xC0 | (dstenc & 7));
2707 
2708     // movzbl $dst, $dst
2709     if (dstenc >= 4) {
2710       emit_opcode(cbuf, dstenc < 8 ? Assembler::REX : Assembler::REX_RB);
2711     }
2712     emit_opcode(cbuf, 0x0F);
2713     emit_opcode(cbuf, 0xB6);
2714     emit_rm(cbuf, 0x3, dstenc & 7, dstenc & 7);
2715   %}
2716 
2717   enc_class Push_ResultXD(regD dst) %{
2718     MacroAssembler _masm(&cbuf);
2719     __ fstp_d(Address(rsp, 0));
2720     __ movdbl($dst$$XMMRegister, Address(rsp, 0));
2721     __ addptr(rsp, 8);
2722   %}
2723 
2724   enc_class Push_SrcXD(regD src) %{
2725     MacroAssembler _masm(&cbuf);
2726     __ subptr(rsp, 8);
2727     __ movdbl(Address(rsp, 0), $src$$XMMRegister);
2728     __ fld_d(Address(rsp, 0));
2729   %}
2730 
2731 
2732   enc_class enc_rethrow()
2733   %{
2734     cbuf.set_insts_mark();
2735     emit_opcode(cbuf, 0xE9); // jmp entry
2736     emit_d32_reloc(cbuf,
2737                    (int) (OptoRuntime::rethrow_stub() - cbuf.insts_end() - 4),
2738                    runtime_call_Relocation::spec(),
2739                    RELOC_DISP32);
2740   %}
2741 
2742 %}
2743 
2744 
2745 
2746 //----------FRAME--------------------------------------------------------------
2747 // Definition of frame structure and management information.
2748 //
2749 //  S T A C K   L A Y O U T    Allocators stack-slot number
2750 //                             |   (to get allocators register number
2751 //  G  Owned by    |        |  v    add OptoReg::stack0())
2752 //  r   CALLER     |        |
2753 //  o     |        +--------+      pad to even-align allocators stack-slot
2754 //  w     V        |  pad0  |        numbers; owned by CALLER
2755 //  t   -----------+--------+----> Matcher::_in_arg_limit, unaligned
2756 //  h     ^        |   in   |  5
2757 //        |        |  args  |  4   Holes in incoming args owned by SELF
2758 //  |     |        |        |  3
2759 //  |     |        +--------+
2760 //  V     |        | old out|      Empty on Intel, window on Sparc
2761 //        |    old |preserve|      Must be even aligned.
2762 //        |     SP-+--------+----> Matcher::_old_SP, even aligned
2763 //        |        |   in   |  3   area for Intel ret address
2764 //     Owned by    |preserve|      Empty on Sparc.
2765 //       SELF      +--------+
2766 //        |        |  pad2  |  2   pad to align old SP
2767 //        |        +--------+  1
2768 //        |        | locks  |  0
2769 //        |        +--------+----> OptoReg::stack0(), even aligned
2770 //        |        |  pad1  | 11   pad to align new SP
2771 //        |        +--------+
2772 //        |        |        | 10
2773 //        |        | spills |  9   spills
2774 //        V        |        |  8   (pad0 slot for callee)
2775 //      -----------+--------+----> Matcher::_out_arg_limit, unaligned
2776 //        ^        |  out   |  7
2777 //        |        |  args  |  6   Holes in outgoing args owned by CALLEE
2778 //     Owned by    +--------+
2779 //      CALLEE     | new out|  6   Empty on Intel, window on Sparc
2780 //        |    new |preserve|      Must be even-aligned.
2781 //        |     SP-+--------+----> Matcher::_new_SP, even aligned
2782 //        |        |        |
2783 //
2784 // Note 1: Only region 8-11 is determined by the allocator.  Region 0-5 is
2785 //         known from SELF's arguments and the Java calling convention.
2786 //         Region 6-7 is determined per call site.
2787 // Note 2: If the calling convention leaves holes in the incoming argument
2788 //         area, those holes are owned by SELF.  Holes in the outgoing area
2789 //         are owned by the CALLEE.  Holes should not be nessecary in the
2790 //         incoming area, as the Java calling convention is completely under
2791 //         the control of the AD file.  Doubles can be sorted and packed to
2792 //         avoid holes.  Holes in the outgoing arguments may be nessecary for
2793 //         varargs C calling conventions.
2794 // Note 3: Region 0-3 is even aligned, with pad2 as needed.  Region 3-5 is
2795 //         even aligned with pad0 as needed.
2796 //         Region 6 is even aligned.  Region 6-7 is NOT even aligned;
2797 //         region 6-11 is even aligned; it may be padded out more so that
2798 //         the region from SP to FP meets the minimum stack alignment.
2799 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
2800 //         alignment.  Region 11, pad1, may be dynamically extended so that
2801 //         SP meets the minimum alignment.
2802 
2803 frame
2804 %{
2805   // What direction does stack grow in (assumed to be same for C & Java)
2806   stack_direction(TOWARDS_LOW);
2807 
2808   // These three registers define part of the calling convention
2809   // between compiled code and the interpreter.
2810   inline_cache_reg(RAX);                // Inline Cache Register
2811   interpreter_method_oop_reg(RBX);      // Method Oop Register when
2812                                         // calling interpreter
2813 
2814   // Optional: name the operand used by cisc-spilling to access
2815   // [stack_pointer + offset]
2816   cisc_spilling_operand_name(indOffset32);
2817 
2818   // Number of stack slots consumed by locking an object
2819   sync_stack_slots(2);
2820 
2821   // Compiled code's Frame Pointer
2822   frame_pointer(RSP);
2823 
2824   // Interpreter stores its frame pointer in a register which is
2825   // stored to the stack by I2CAdaptors.
2826   // I2CAdaptors convert from interpreted java to compiled java.
2827   interpreter_frame_pointer(RBP);
2828 
2829   // Stack alignment requirement
2830   stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
2831 
2832   // Number of stack slots between incoming argument block and the start of
2833   // a new frame.  The PROLOG must add this many slots to the stack.  The
2834   // EPILOG must remove this many slots.  amd64 needs two slots for
2835   // return address.
2836   in_preserve_stack_slots(4 + 2 * VerifyStackAtCalls);
2837 
2838   // Number of outgoing stack slots killed above the out_preserve_stack_slots
2839   // for calls to C.  Supports the var-args backing area for register parms.
2840   varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
2841 
2842   // The after-PROLOG location of the return address.  Location of
2843   // return address specifies a type (REG or STACK) and a number
2844   // representing the register number (i.e. - use a register name) or
2845   // stack slot.
2846   // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
2847   // Otherwise, it is above the locks and verification slot and alignment word
2848   return_addr(STACK - 2 +
2849               align_up((Compile::current()->in_preserve_stack_slots() +
2850                         Compile::current()->fixed_slots()),
2851                        stack_alignment_in_slots()));
2852 
2853   // Body of function which returns an integer array locating
2854   // arguments either in registers or in stack slots.  Passed an array
2855   // of ideal registers called "sig" and a "length" count.  Stack-slot
2856   // offsets are based on outgoing arguments, i.e. a CALLER setting up
2857   // arguments for a CALLEE.  Incoming stack arguments are
2858   // automatically biased by the preserve_stack_slots field above.
2859 
2860   calling_convention
2861   %{
2862     // No difference between ingoing/outgoing just pass false
2863     SharedRuntime::java_calling_convention(sig_bt, regs, length, false);
2864   %}
2865 
2866   c_calling_convention
2867   %{
2868     // This is obviously always outgoing
2869     (void) SharedRuntime::c_calling_convention(sig_bt, regs, /*regs2=*/NULL, length);
2870   %}
2871 
2872   // Location of compiled Java return values.  Same as C for now.
2873   return_value
2874   %{
2875     assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
2876            "only return normal values");
2877 
2878     static const int lo[Op_RegL + 1] = {
2879       0,
2880       0,
2881       RAX_num,  // Op_RegN
2882       RAX_num,  // Op_RegI
2883       RAX_num,  // Op_RegP
2884       XMM0_num, // Op_RegF
2885       XMM0_num, // Op_RegD
2886       RAX_num   // Op_RegL
2887     };
2888     static const int hi[Op_RegL + 1] = {
2889       0,
2890       0,
2891       OptoReg::Bad, // Op_RegN
2892       OptoReg::Bad, // Op_RegI
2893       RAX_H_num,    // Op_RegP
2894       OptoReg::Bad, // Op_RegF
2895       XMM0b_num,    // Op_RegD
2896       RAX_H_num     // Op_RegL
2897     };
2898     // Excluded flags and vector registers.
2899     assert(ARRAY_SIZE(hi) == _last_machine_leaf - 6, "missing type");
2900     return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
2901   %}
2902 %}
2903 
2904 //----------ATTRIBUTES---------------------------------------------------------
2905 //----------Operand Attributes-------------------------------------------------
2906 op_attrib op_cost(0);        // Required cost attribute
2907 
2908 //----------Instruction Attributes---------------------------------------------
2909 ins_attrib ins_cost(100);       // Required cost attribute
2910 ins_attrib ins_size(8);         // Required size attribute (in bits)
2911 ins_attrib ins_short_branch(0); // Required flag: is this instruction
2912                                 // a non-matching short branch variant
2913                                 // of some long branch?
2914 ins_attrib ins_alignment(1);    // Required alignment attribute (must
2915                                 // be a power of 2) specifies the
2916                                 // alignment that some part of the
2917                                 // instruction (not necessarily the
2918                                 // start) requires.  If > 1, a
2919                                 // compute_padding() function must be
2920                                 // provided for the instruction
2921 
2922 //----------OPERANDS-----------------------------------------------------------
2923 // Operand definitions must precede instruction definitions for correct parsing
2924 // in the ADLC because operands constitute user defined types which are used in
2925 // instruction definitions.
2926 
2927 //----------Simple Operands----------------------------------------------------
2928 // Immediate Operands
2929 // Integer Immediate
2930 operand immI()
2931 %{
2932   match(ConI);
2933 
2934   op_cost(10);
2935   format %{ %}
2936   interface(CONST_INTER);
2937 %}
2938 
2939 // Constant for test vs zero
2940 operand immI0()
2941 %{
2942   predicate(n->get_int() == 0);
2943   match(ConI);
2944 
2945   op_cost(0);
2946   format %{ %}
2947   interface(CONST_INTER);
2948 %}
2949 
2950 // Constant for increment
2951 operand immI1()
2952 %{
2953   predicate(n->get_int() == 1);
2954   match(ConI);
2955 
2956   op_cost(0);
2957   format %{ %}
2958   interface(CONST_INTER);
2959 %}
2960 
2961 // Constant for decrement
2962 operand immI_M1()
2963 %{
2964   predicate(n->get_int() == -1);
2965   match(ConI);
2966 
2967   op_cost(0);
2968   format %{ %}
2969   interface(CONST_INTER);
2970 %}
2971 
2972 // Valid scale values for addressing modes
2973 operand immI2()
2974 %{
2975   predicate(0 <= n->get_int() && (n->get_int() <= 3));
2976   match(ConI);
2977 
2978   format %{ %}
2979   interface(CONST_INTER);
2980 %}
2981 
2982 operand immI8()
2983 %{
2984   predicate((-0x80 <= n->get_int()) && (n->get_int() < 0x80));
2985   match(ConI);
2986 
2987   op_cost(5);
2988   format %{ %}
2989   interface(CONST_INTER);
2990 %}
2991 
2992 operand immU8()
2993 %{
2994   predicate((0 <= n->get_int()) && (n->get_int() <= 255));
2995   match(ConI);
2996 
2997   op_cost(5);
2998   format %{ %}
2999   interface(CONST_INTER);
3000 %}
3001 
3002 operand immI16()
3003 %{
3004   predicate((-32768 <= n->get_int()) && (n->get_int() <= 32767));
3005   match(ConI);
3006 
3007   op_cost(10);
3008   format %{ %}
3009   interface(CONST_INTER);
3010 %}
3011 
3012 // Int Immediate non-negative
3013 operand immU31()
3014 %{
3015   predicate(n->get_int() >= 0);
3016   match(ConI);
3017 
3018   op_cost(0);
3019   format %{ %}
3020   interface(CONST_INTER);
3021 %}
3022 
3023 // Constant for long shifts
3024 operand immI_32()
3025 %{
3026   predicate( n->get_int() == 32 );
3027   match(ConI);
3028 
3029   op_cost(0);
3030   format %{ %}
3031   interface(CONST_INTER);
3032 %}
3033 
3034 // Constant for long shifts
3035 operand immI_64()
3036 %{
3037   predicate( n->get_int() == 64 );
3038   match(ConI);
3039 
3040   op_cost(0);
3041   format %{ %}
3042   interface(CONST_INTER);
3043 %}
3044 
3045 // Pointer Immediate
3046 operand immP()
3047 %{
3048   match(ConP);
3049 
3050   op_cost(10);
3051   format %{ %}
3052   interface(CONST_INTER);
3053 %}
3054 
3055 // NULL Pointer Immediate
3056 operand immP0()
3057 %{
3058   predicate(n->get_ptr() == 0);
3059   match(ConP);
3060 
3061   op_cost(5);
3062   format %{ %}
3063   interface(CONST_INTER);
3064 %}
3065 
3066 // Pointer Immediate
3067 operand immN() %{
3068   match(ConN);
3069 
3070   op_cost(10);
3071   format %{ %}
3072   interface(CONST_INTER);
3073 %}
3074 
3075 operand immNKlass() %{
3076   match(ConNKlass);
3077 
3078   op_cost(10);
3079   format %{ %}
3080   interface(CONST_INTER);
3081 %}
3082 
3083 // NULL Pointer Immediate
3084 operand immN0() %{
3085   predicate(n->get_narrowcon() == 0);
3086   match(ConN);
3087 
3088   op_cost(5);
3089   format %{ %}
3090   interface(CONST_INTER);
3091 %}
3092 
3093 operand immP31()
3094 %{
3095   predicate(n->as_Type()->type()->reloc() == relocInfo::none
3096             && (n->get_ptr() >> 31) == 0);
3097   match(ConP);
3098 
3099   op_cost(5);
3100   format %{ %}
3101   interface(CONST_INTER);
3102 %}
3103 
3104 
3105 // Long Immediate
3106 operand immL()
3107 %{
3108   match(ConL);
3109 
3110   op_cost(20);
3111   format %{ %}
3112   interface(CONST_INTER);
3113 %}
3114 
3115 // Long Immediate 8-bit
3116 operand immL8()
3117 %{
3118   predicate(-0x80L <= n->get_long() && n->get_long() < 0x80L);
3119   match(ConL);
3120 
3121   op_cost(5);
3122   format %{ %}
3123   interface(CONST_INTER);
3124 %}
3125 
3126 // Long Immediate 32-bit unsigned
3127 operand immUL32()
3128 %{
3129   predicate(n->get_long() == (unsigned int) (n->get_long()));
3130   match(ConL);
3131 
3132   op_cost(10);
3133   format %{ %}
3134   interface(CONST_INTER);
3135 %}
3136 
3137 // Long Immediate 32-bit signed
3138 operand immL32()
3139 %{
3140   predicate(n->get_long() == (int) (n->get_long()));
3141   match(ConL);
3142 
3143   op_cost(15);
3144   format %{ %}
3145   interface(CONST_INTER);
3146 %}
3147 
3148 // Long Immediate zero
3149 operand immL0()
3150 %{
3151   predicate(n->get_long() == 0L);
3152   match(ConL);
3153 
3154   op_cost(10);
3155   format %{ %}
3156   interface(CONST_INTER);
3157 %}
3158 
3159 // Constant for increment
3160 operand immL1()
3161 %{
3162   predicate(n->get_long() == 1);
3163   match(ConL);
3164 
3165   format %{ %}
3166   interface(CONST_INTER);
3167 %}
3168 
3169 // Constant for decrement
3170 operand immL_M1()
3171 %{
3172   predicate(n->get_long() == -1);
3173   match(ConL);
3174 
3175   format %{ %}
3176   interface(CONST_INTER);
3177 %}
3178 
3179 // Long Immediate: the value 10
3180 operand immL10()
3181 %{
3182   predicate(n->get_long() == 10);
3183   match(ConL);
3184 
3185   format %{ %}
3186   interface(CONST_INTER);
3187 %}
3188 
3189 // Long immediate from 0 to 127.
3190 // Used for a shorter form of long mul by 10.
3191 operand immL_127()
3192 %{
3193   predicate(0 <= n->get_long() && n->get_long() < 0x80);
3194   match(ConL);
3195 
3196   op_cost(10);
3197   format %{ %}
3198   interface(CONST_INTER);
3199 %}
3200 
3201 // Long Immediate: low 32-bit mask
3202 operand immL_32bits()
3203 %{
3204   predicate(n->get_long() == 0xFFFFFFFFL);
3205   match(ConL);
3206   op_cost(20);
3207 
3208   format %{ %}
3209   interface(CONST_INTER);
3210 %}
3211 
3212 // Float Immediate zero
3213 operand immF0()
3214 %{
3215   predicate(jint_cast(n->getf()) == 0);
3216   match(ConF);
3217 
3218   op_cost(5);
3219   format %{ %}
3220   interface(CONST_INTER);
3221 %}
3222 
3223 // Float Immediate
3224 operand immF()
3225 %{
3226   match(ConF);
3227 
3228   op_cost(15);
3229   format %{ %}
3230   interface(CONST_INTER);
3231 %}
3232 
3233 // Double Immediate zero
3234 operand immD0()
3235 %{
3236   predicate(jlong_cast(n->getd()) == 0);
3237   match(ConD);
3238 
3239   op_cost(5);
3240   format %{ %}
3241   interface(CONST_INTER);
3242 %}
3243 
3244 // Double Immediate
3245 operand immD()
3246 %{
3247   match(ConD);
3248 
3249   op_cost(15);
3250   format %{ %}
3251   interface(CONST_INTER);
3252 %}
3253 
3254 // Immediates for special shifts (sign extend)
3255 
3256 // Constants for increment
3257 operand immI_16()
3258 %{
3259   predicate(n->get_int() == 16);
3260   match(ConI);
3261 
3262   format %{ %}
3263   interface(CONST_INTER);
3264 %}
3265 
3266 operand immI_24()
3267 %{
3268   predicate(n->get_int() == 24);
3269   match(ConI);
3270 
3271   format %{ %}
3272   interface(CONST_INTER);
3273 %}
3274 
3275 // Constant for byte-wide masking
3276 operand immI_255()
3277 %{
3278   predicate(n->get_int() == 255);
3279   match(ConI);
3280 
3281   format %{ %}
3282   interface(CONST_INTER);
3283 %}
3284 
3285 // Constant for short-wide masking
3286 operand immI_65535()
3287 %{
3288   predicate(n->get_int() == 65535);
3289   match(ConI);
3290 
3291   format %{ %}
3292   interface(CONST_INTER);
3293 %}
3294 
3295 // Constant for byte-wide masking
3296 operand immL_255()
3297 %{
3298   predicate(n->get_long() == 255);
3299   match(ConL);
3300 
3301   format %{ %}
3302   interface(CONST_INTER);
3303 %}
3304 
3305 // Constant for short-wide masking
3306 operand immL_65535()
3307 %{
3308   predicate(n->get_long() == 65535);
3309   match(ConL);
3310 
3311   format %{ %}
3312   interface(CONST_INTER);
3313 %}
3314 
3315 // Register Operands
3316 // Integer Register
3317 operand rRegI()
3318 %{
3319   constraint(ALLOC_IN_RC(int_reg));
3320   match(RegI);
3321 
3322   match(rax_RegI);
3323   match(rbx_RegI);
3324   match(rcx_RegI);
3325   match(rdx_RegI);
3326   match(rdi_RegI);
3327 
3328   format %{ %}
3329   interface(REG_INTER);
3330 %}
3331 
3332 // Special Registers
3333 operand rax_RegI()
3334 %{
3335   constraint(ALLOC_IN_RC(int_rax_reg));
3336   match(RegI);
3337   match(rRegI);
3338 
3339   format %{ "RAX" %}
3340   interface(REG_INTER);
3341 %}
3342 
3343 // Special Registers
3344 operand rbx_RegI()
3345 %{
3346   constraint(ALLOC_IN_RC(int_rbx_reg));
3347   match(RegI);
3348   match(rRegI);
3349 
3350   format %{ "RBX" %}
3351   interface(REG_INTER);
3352 %}
3353 
3354 operand rcx_RegI()
3355 %{
3356   constraint(ALLOC_IN_RC(int_rcx_reg));
3357   match(RegI);
3358   match(rRegI);
3359 
3360   format %{ "RCX" %}
3361   interface(REG_INTER);
3362 %}
3363 
3364 operand rdx_RegI()
3365 %{
3366   constraint(ALLOC_IN_RC(int_rdx_reg));
3367   match(RegI);
3368   match(rRegI);
3369 
3370   format %{ "RDX" %}
3371   interface(REG_INTER);
3372 %}
3373 
3374 operand rdi_RegI()
3375 %{
3376   constraint(ALLOC_IN_RC(int_rdi_reg));
3377   match(RegI);
3378   match(rRegI);
3379 
3380   format %{ "RDI" %}
3381   interface(REG_INTER);
3382 %}
3383 
3384 operand no_rcx_RegI()
3385 %{
3386   constraint(ALLOC_IN_RC(int_no_rcx_reg));
3387   match(RegI);
3388   match(rax_RegI);
3389   match(rbx_RegI);
3390   match(rdx_RegI);
3391   match(rdi_RegI);
3392 
3393   format %{ %}
3394   interface(REG_INTER);
3395 %}
3396 
3397 operand no_rax_rdx_RegI()
3398 %{
3399   constraint(ALLOC_IN_RC(int_no_rax_rdx_reg));
3400   match(RegI);
3401   match(rbx_RegI);
3402   match(rcx_RegI);
3403   match(rdi_RegI);
3404 
3405   format %{ %}
3406   interface(REG_INTER);
3407 %}
3408 
3409 // Pointer Register
3410 operand any_RegP()
3411 %{
3412   constraint(ALLOC_IN_RC(any_reg));
3413   match(RegP);
3414   match(rax_RegP);
3415   match(rbx_RegP);
3416   match(rdi_RegP);
3417   match(rsi_RegP);
3418   match(rbp_RegP);
3419   match(r15_RegP);
3420   match(rRegP);
3421 
3422   format %{ %}
3423   interface(REG_INTER);
3424 %}
3425 
3426 operand rRegP()
3427 %{
3428   constraint(ALLOC_IN_RC(ptr_reg));
3429   match(RegP);
3430   match(rax_RegP);
3431   match(rbx_RegP);
3432   match(rdi_RegP);
3433   match(rsi_RegP);
3434   match(rbp_RegP);  // See Q&A below about
3435   match(r15_RegP);  // r15_RegP and rbp_RegP.
3436 
3437   format %{ %}
3438   interface(REG_INTER);
3439 %}
3440 
3441 operand rRegN() %{
3442   constraint(ALLOC_IN_RC(int_reg));
3443   match(RegN);
3444 
3445   format %{ %}
3446   interface(REG_INTER);
3447 %}
3448 
3449 // Question: Why is r15_RegP (the read-only TLS register) a match for rRegP?
3450 // Answer: Operand match rules govern the DFA as it processes instruction inputs.
3451 // It's fine for an instruction input that expects rRegP to match a r15_RegP.
3452 // The output of an instruction is controlled by the allocator, which respects
3453 // register class masks, not match rules.  Unless an instruction mentions
3454 // r15_RegP or any_RegP explicitly as its output, r15 will not be considered
3455 // by the allocator as an input.
3456 // The same logic applies to rbp_RegP being a match for rRegP: If PreserveFramePointer==true,
3457 // the RBP is used as a proper frame pointer and is not included in ptr_reg. As a
3458 // result, RBP is not included in the output of the instruction either.
3459 
3460 operand no_rax_RegP()
3461 %{
3462   constraint(ALLOC_IN_RC(ptr_no_rax_reg));
3463   match(RegP);
3464   match(rbx_RegP);
3465   match(rsi_RegP);
3466   match(rdi_RegP);
3467 
3468   format %{ %}
3469   interface(REG_INTER);
3470 %}
3471 
3472 // This operand is not allowed to use RBP even if
3473 // RBP is not used to hold the frame pointer.
3474 operand no_rbp_RegP()
3475 %{
3476   constraint(ALLOC_IN_RC(ptr_reg_no_rbp));
3477   match(RegP);
3478   match(rbx_RegP);
3479   match(rsi_RegP);
3480   match(rdi_RegP);
3481 
3482   format %{ %}
3483   interface(REG_INTER);
3484 %}
3485 
3486 operand no_rax_rbx_RegP()
3487 %{
3488   constraint(ALLOC_IN_RC(ptr_no_rax_rbx_reg));
3489   match(RegP);
3490   match(rsi_RegP);
3491   match(rdi_RegP);
3492 
3493   format %{ %}
3494   interface(REG_INTER);
3495 %}
3496 
3497 // Special Registers
3498 // Return a pointer value
3499 operand rax_RegP()
3500 %{
3501   constraint(ALLOC_IN_RC(ptr_rax_reg));
3502   match(RegP);
3503   match(rRegP);
3504 
3505   format %{ %}
3506   interface(REG_INTER);
3507 %}
3508 
3509 // Special Registers
3510 // Return a compressed pointer value
3511 operand rax_RegN()
3512 %{
3513   constraint(ALLOC_IN_RC(int_rax_reg));
3514   match(RegN);
3515   match(rRegN);
3516 
3517   format %{ %}
3518   interface(REG_INTER);
3519 %}
3520 
3521 // Used in AtomicAdd
3522 operand rbx_RegP()
3523 %{
3524   constraint(ALLOC_IN_RC(ptr_rbx_reg));
3525   match(RegP);
3526   match(rRegP);
3527 
3528   format %{ %}
3529   interface(REG_INTER);
3530 %}
3531 
3532 operand rsi_RegP()
3533 %{
3534   constraint(ALLOC_IN_RC(ptr_rsi_reg));
3535   match(RegP);
3536   match(rRegP);
3537 
3538   format %{ %}
3539   interface(REG_INTER);
3540 %}
3541 
3542 // Used in rep stosq
3543 operand rdi_RegP()
3544 %{
3545   constraint(ALLOC_IN_RC(ptr_rdi_reg));
3546   match(RegP);
3547   match(rRegP);
3548 
3549   format %{ %}
3550   interface(REG_INTER);
3551 %}
3552 
3553 operand r15_RegP()
3554 %{
3555   constraint(ALLOC_IN_RC(ptr_r15_reg));
3556   match(RegP);
3557   match(rRegP);
3558 
3559   format %{ %}
3560   interface(REG_INTER);
3561 %}
3562 
3563 operand rex_RegP()
3564 %{
3565   constraint(ALLOC_IN_RC(ptr_rex_reg));
3566   match(RegP);
3567   match(rRegP);
3568 
3569   format %{ %}
3570   interface(REG_INTER);
3571 %}
3572 
3573 operand rRegL()
3574 %{
3575   constraint(ALLOC_IN_RC(long_reg));
3576   match(RegL);
3577   match(rax_RegL);
3578   match(rdx_RegL);
3579 
3580   format %{ %}
3581   interface(REG_INTER);
3582 %}
3583 
3584 // Special Registers
3585 operand no_rax_rdx_RegL()
3586 %{
3587   constraint(ALLOC_IN_RC(long_no_rax_rdx_reg));
3588   match(RegL);
3589   match(rRegL);
3590 
3591   format %{ %}
3592   interface(REG_INTER);
3593 %}
3594 
3595 operand no_rax_RegL()
3596 %{
3597   constraint(ALLOC_IN_RC(long_no_rax_rdx_reg));
3598   match(RegL);
3599   match(rRegL);
3600   match(rdx_RegL);
3601 
3602   format %{ %}
3603   interface(REG_INTER);
3604 %}
3605 
3606 operand no_rcx_RegL()
3607 %{
3608   constraint(ALLOC_IN_RC(long_no_rcx_reg));
3609   match(RegL);
3610   match(rRegL);
3611 
3612   format %{ %}
3613   interface(REG_INTER);
3614 %}
3615 
3616 operand rax_RegL()
3617 %{
3618   constraint(ALLOC_IN_RC(long_rax_reg));
3619   match(RegL);
3620   match(rRegL);
3621 
3622   format %{ "RAX" %}
3623   interface(REG_INTER);
3624 %}
3625 
3626 operand rcx_RegL()
3627 %{
3628   constraint(ALLOC_IN_RC(long_rcx_reg));
3629   match(RegL);
3630   match(rRegL);
3631 
3632   format %{ %}
3633   interface(REG_INTER);
3634 %}
3635 
3636 operand rdx_RegL()
3637 %{
3638   constraint(ALLOC_IN_RC(long_rdx_reg));
3639   match(RegL);
3640   match(rRegL);
3641 
3642   format %{ %}
3643   interface(REG_INTER);
3644 %}
3645 
3646 // Flags register, used as output of compare instructions
3647 operand rFlagsReg()
3648 %{
3649   constraint(ALLOC_IN_RC(int_flags));
3650   match(RegFlags);
3651 
3652   format %{ "RFLAGS" %}
3653   interface(REG_INTER);
3654 %}
3655 
3656 // Flags register, used as output of FLOATING POINT compare instructions
3657 operand rFlagsRegU()
3658 %{
3659   constraint(ALLOC_IN_RC(int_flags));
3660   match(RegFlags);
3661 
3662   format %{ "RFLAGS_U" %}
3663   interface(REG_INTER);
3664 %}
3665 
3666 operand rFlagsRegUCF() %{
3667   constraint(ALLOC_IN_RC(int_flags));
3668   match(RegFlags);
3669   predicate(false);
3670 
3671   format %{ "RFLAGS_U_CF" %}
3672   interface(REG_INTER);
3673 %}
3674 
3675 // Float register operands
3676 operand regF() %{
3677    constraint(ALLOC_IN_RC(float_reg));
3678    match(RegF);
3679 
3680    format %{ %}
3681    interface(REG_INTER);
3682 %}
3683 
3684 // Float register operands
3685 operand vlRegF() %{
3686    constraint(ALLOC_IN_RC(float_reg_vl));
3687    match(RegF);
3688 
3689    format %{ %}
3690    interface(REG_INTER);
3691 %}
3692 
3693 // Double register operands
3694 operand regD() %{
3695    constraint(ALLOC_IN_RC(double_reg));
3696    match(RegD);
3697 
3698    format %{ %}
3699    interface(REG_INTER);
3700 %}
3701 
3702 // Double register operands
3703 operand vlRegD() %{
3704    constraint(ALLOC_IN_RC(double_reg_vl));
3705    match(RegD);
3706 
3707    format %{ %}
3708    interface(REG_INTER);
3709 %}
3710 
3711 // Vectors
3712 operand vecS() %{
3713   constraint(ALLOC_IN_RC(vectors_reg_vlbwdq));
3714   match(VecS);
3715 
3716   format %{ %}
3717   interface(REG_INTER);
3718 %}
3719 
3720 // Vectors
3721 operand legVecS() %{
3722   constraint(ALLOC_IN_RC(vectors_reg_legacy));
3723   match(VecS);
3724 
3725   format %{ %}
3726   interface(REG_INTER);
3727 %}
3728 
3729 operand vecD() %{
3730   constraint(ALLOC_IN_RC(vectord_reg_vlbwdq));
3731   match(VecD);
3732 
3733   format %{ %}
3734   interface(REG_INTER);
3735 %}
3736 
3737 operand legVecD() %{
3738   constraint(ALLOC_IN_RC(vectord_reg_legacy));
3739   match(VecD);
3740 
3741   format %{ %}
3742   interface(REG_INTER);
3743 %}
3744 
3745 operand vecX() %{
3746   constraint(ALLOC_IN_RC(vectorx_reg_vlbwdq));
3747   match(VecX);
3748 
3749   format %{ %}
3750   interface(REG_INTER);
3751 %}
3752 
3753 operand legVecX() %{
3754   constraint(ALLOC_IN_RC(vectorx_reg_legacy));
3755   match(VecX);
3756 
3757   format %{ %}
3758   interface(REG_INTER);
3759 %}
3760 
3761 operand vecY() %{
3762   constraint(ALLOC_IN_RC(vectory_reg_vlbwdq));
3763   match(VecY);
3764 
3765   format %{ %}
3766   interface(REG_INTER);
3767 %}
3768 
3769 operand legVecY() %{
3770   constraint(ALLOC_IN_RC(vectory_reg_legacy));
3771   match(VecY);
3772 
3773   format %{ %}
3774   interface(REG_INTER);
3775 %}
3776 
3777 //----------Memory Operands----------------------------------------------------
3778 // Direct Memory Operand
3779 // operand direct(immP addr)
3780 // %{
3781 //   match(addr);
3782 
3783 //   format %{ "[$addr]" %}
3784 //   interface(MEMORY_INTER) %{
3785 //     base(0xFFFFFFFF);
3786 //     index(0x4);
3787 //     scale(0x0);
3788 //     disp($addr);
3789 //   %}
3790 // %}
3791 
3792 // Indirect Memory Operand
3793 operand indirect(any_RegP reg)
3794 %{
3795   constraint(ALLOC_IN_RC(ptr_reg));
3796   match(reg);
3797 
3798   format %{ "[$reg]" %}
3799   interface(MEMORY_INTER) %{
3800     base($reg);
3801     index(0x4);
3802     scale(0x0);
3803     disp(0x0);
3804   %}
3805 %}
3806 
3807 // Indirect Memory Plus Short Offset Operand
3808 operand indOffset8(any_RegP reg, immL8 off)
3809 %{
3810   constraint(ALLOC_IN_RC(ptr_reg));
3811   match(AddP reg off);
3812 
3813   format %{ "[$reg + $off (8-bit)]" %}
3814   interface(MEMORY_INTER) %{
3815     base($reg);
3816     index(0x4);
3817     scale(0x0);
3818     disp($off);
3819   %}
3820 %}
3821 
3822 // Indirect Memory Plus Long Offset Operand
3823 operand indOffset32(any_RegP reg, immL32 off)
3824 %{
3825   constraint(ALLOC_IN_RC(ptr_reg));
3826   match(AddP reg off);
3827 
3828   format %{ "[$reg + $off (32-bit)]" %}
3829   interface(MEMORY_INTER) %{
3830     base($reg);
3831     index(0x4);
3832     scale(0x0);
3833     disp($off);
3834   %}
3835 %}
3836 
3837 // Indirect Memory Plus Index Register Plus Offset Operand
3838 operand indIndexOffset(any_RegP reg, rRegL lreg, immL32 off)
3839 %{
3840   constraint(ALLOC_IN_RC(ptr_reg));
3841   match(AddP (AddP reg lreg) off);
3842 
3843   op_cost(10);
3844   format %{"[$reg + $off + $lreg]" %}
3845   interface(MEMORY_INTER) %{
3846     base($reg);
3847     index($lreg);
3848     scale(0x0);
3849     disp($off);
3850   %}
3851 %}
3852 
3853 // Indirect Memory Plus Index Register Plus Offset Operand
3854 operand indIndex(any_RegP reg, rRegL lreg)
3855 %{
3856   constraint(ALLOC_IN_RC(ptr_reg));
3857   match(AddP reg lreg);
3858 
3859   op_cost(10);
3860   format %{"[$reg + $lreg]" %}
3861   interface(MEMORY_INTER) %{
3862     base($reg);
3863     index($lreg);
3864     scale(0x0);
3865     disp(0x0);
3866   %}
3867 %}
3868 
3869 // Indirect Memory Times Scale Plus Index Register
3870 operand indIndexScale(any_RegP reg, rRegL lreg, immI2 scale)
3871 %{
3872   constraint(ALLOC_IN_RC(ptr_reg));
3873   match(AddP reg (LShiftL lreg scale));
3874 
3875   op_cost(10);
3876   format %{"[$reg + $lreg << $scale]" %}
3877   interface(MEMORY_INTER) %{
3878     base($reg);
3879     index($lreg);
3880     scale($scale);
3881     disp(0x0);
3882   %}
3883 %}
3884 
3885 operand indPosIndexScale(any_RegP reg, rRegI idx, immI2 scale)
3886 %{
3887   constraint(ALLOC_IN_RC(ptr_reg));
3888   predicate(n->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
3889   match(AddP reg (LShiftL (ConvI2L idx) scale));
3890 
3891   op_cost(10);
3892   format %{"[$reg + pos $idx << $scale]" %}
3893   interface(MEMORY_INTER) %{
3894     base($reg);
3895     index($idx);
3896     scale($scale);
3897     disp(0x0);
3898   %}
3899 %}
3900 
3901 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
3902 operand indIndexScaleOffset(any_RegP reg, immL32 off, rRegL lreg, immI2 scale)
3903 %{
3904   constraint(ALLOC_IN_RC(ptr_reg));
3905   match(AddP (AddP reg (LShiftL lreg scale)) off);
3906 
3907   op_cost(10);
3908   format %{"[$reg + $off + $lreg << $scale]" %}
3909   interface(MEMORY_INTER) %{
3910     base($reg);
3911     index($lreg);
3912     scale($scale);
3913     disp($off);
3914   %}
3915 %}
3916 
3917 // Indirect Memory Plus Positive Index Register Plus Offset Operand
3918 operand indPosIndexOffset(any_RegP reg, immL32 off, rRegI idx)
3919 %{
3920   constraint(ALLOC_IN_RC(ptr_reg));
3921   predicate(n->in(2)->in(3)->as_Type()->type()->is_long()->_lo >= 0);
3922   match(AddP (AddP reg (ConvI2L idx)) off);
3923 
3924   op_cost(10);
3925   format %{"[$reg + $off + $idx]" %}
3926   interface(MEMORY_INTER) %{
3927     base($reg);
3928     index($idx);
3929     scale(0x0);
3930     disp($off);
3931   %}
3932 %}
3933 
3934 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
3935 operand indPosIndexScaleOffset(any_RegP reg, immL32 off, rRegI idx, immI2 scale)
3936 %{
3937   constraint(ALLOC_IN_RC(ptr_reg));
3938   predicate(n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
3939   match(AddP (AddP reg (LShiftL (ConvI2L idx) scale)) off);
3940 
3941   op_cost(10);
3942   format %{"[$reg + $off + $idx << $scale]" %}
3943   interface(MEMORY_INTER) %{
3944     base($reg);
3945     index($idx);
3946     scale($scale);
3947     disp($off);
3948   %}
3949 %}
3950 
3951 // Indirect Narrow Oop Plus Offset Operand
3952 // Note: x86 architecture doesn't support "scale * index + offset" without a base
3953 // we can't free r12 even with Universe::narrow_oop_base() == NULL.
3954 operand indCompressedOopOffset(rRegN reg, immL32 off) %{
3955   predicate(UseCompressedOops && (Universe::narrow_oop_shift() == Address::times_8));
3956   constraint(ALLOC_IN_RC(ptr_reg));
3957   match(AddP (DecodeN reg) off);
3958 
3959   op_cost(10);
3960   format %{"[R12 + $reg << 3 + $off] (compressed oop addressing)" %}
3961   interface(MEMORY_INTER) %{
3962     base(0xc); // R12
3963     index($reg);
3964     scale(0x3);
3965     disp($off);
3966   %}
3967 %}
3968 
3969 // Indirect Memory Operand
3970 operand indirectNarrow(rRegN reg)
3971 %{
3972   predicate(Universe::narrow_oop_shift() == 0);
3973   constraint(ALLOC_IN_RC(ptr_reg));
3974   match(DecodeN reg);
3975 
3976   format %{ "[$reg]" %}
3977   interface(MEMORY_INTER) %{
3978     base($reg);
3979     index(0x4);
3980     scale(0x0);
3981     disp(0x0);
3982   %}
3983 %}
3984 
3985 // Indirect Memory Plus Short Offset Operand
3986 operand indOffset8Narrow(rRegN reg, immL8 off)
3987 %{
3988   predicate(Universe::narrow_oop_shift() == 0);
3989   constraint(ALLOC_IN_RC(ptr_reg));
3990   match(AddP (DecodeN reg) off);
3991 
3992   format %{ "[$reg + $off (8-bit)]" %}
3993   interface(MEMORY_INTER) %{
3994     base($reg);
3995     index(0x4);
3996     scale(0x0);
3997     disp($off);
3998   %}
3999 %}
4000 
4001 // Indirect Memory Plus Long Offset Operand
4002 operand indOffset32Narrow(rRegN reg, immL32 off)
4003 %{
4004   predicate(Universe::narrow_oop_shift() == 0);
4005   constraint(ALLOC_IN_RC(ptr_reg));
4006   match(AddP (DecodeN reg) off);
4007 
4008   format %{ "[$reg + $off (32-bit)]" %}
4009   interface(MEMORY_INTER) %{
4010     base($reg);
4011     index(0x4);
4012     scale(0x0);
4013     disp($off);
4014   %}
4015 %}
4016 
4017 // Indirect Memory Plus Index Register Plus Offset Operand
4018 operand indIndexOffsetNarrow(rRegN reg, rRegL lreg, immL32 off)
4019 %{
4020   predicate(Universe::narrow_oop_shift() == 0);
4021   constraint(ALLOC_IN_RC(ptr_reg));
4022   match(AddP (AddP (DecodeN reg) lreg) off);
4023 
4024   op_cost(10);
4025   format %{"[$reg + $off + $lreg]" %}
4026   interface(MEMORY_INTER) %{
4027     base($reg);
4028     index($lreg);
4029     scale(0x0);
4030     disp($off);
4031   %}
4032 %}
4033 
4034 // Indirect Memory Plus Index Register Plus Offset Operand
4035 operand indIndexNarrow(rRegN reg, rRegL lreg)
4036 %{
4037   predicate(Universe::narrow_oop_shift() == 0);
4038   constraint(ALLOC_IN_RC(ptr_reg));
4039   match(AddP (DecodeN reg) lreg);
4040 
4041   op_cost(10);
4042   format %{"[$reg + $lreg]" %}
4043   interface(MEMORY_INTER) %{
4044     base($reg);
4045     index($lreg);
4046     scale(0x0);
4047     disp(0x0);
4048   %}
4049 %}
4050 
4051 // Indirect Memory Times Scale Plus Index Register
4052 operand indIndexScaleNarrow(rRegN reg, rRegL lreg, immI2 scale)
4053 %{
4054   predicate(Universe::narrow_oop_shift() == 0);
4055   constraint(ALLOC_IN_RC(ptr_reg));
4056   match(AddP (DecodeN reg) (LShiftL lreg scale));
4057 
4058   op_cost(10);
4059   format %{"[$reg + $lreg << $scale]" %}
4060   interface(MEMORY_INTER) %{
4061     base($reg);
4062     index($lreg);
4063     scale($scale);
4064     disp(0x0);
4065   %}
4066 %}
4067 
4068 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
4069 operand indIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegL lreg, immI2 scale)
4070 %{
4071   predicate(Universe::narrow_oop_shift() == 0);
4072   constraint(ALLOC_IN_RC(ptr_reg));
4073   match(AddP (AddP (DecodeN reg) (LShiftL lreg scale)) off);
4074 
4075   op_cost(10);
4076   format %{"[$reg + $off + $lreg << $scale]" %}
4077   interface(MEMORY_INTER) %{
4078     base($reg);
4079     index($lreg);
4080     scale($scale);
4081     disp($off);
4082   %}
4083 %}
4084 
4085 // Indirect Memory Times Plus Positive Index Register Plus Offset Operand
4086 operand indPosIndexOffsetNarrow(rRegN reg, immL32 off, rRegI idx)
4087 %{
4088   constraint(ALLOC_IN_RC(ptr_reg));
4089   predicate(Universe::narrow_oop_shift() == 0 && n->in(2)->in(3)->as_Type()->type()->is_long()->_lo >= 0);
4090   match(AddP (AddP (DecodeN reg) (ConvI2L idx)) off);
4091 
4092   op_cost(10);
4093   format %{"[$reg + $off + $idx]" %}
4094   interface(MEMORY_INTER) %{
4095     base($reg);
4096     index($idx);
4097     scale(0x0);
4098     disp($off);
4099   %}
4100 %}
4101 
4102 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
4103 operand indPosIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegI idx, immI2 scale)
4104 %{
4105   constraint(ALLOC_IN_RC(ptr_reg));
4106   predicate(Universe::narrow_oop_shift() == 0 && n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
4107   match(AddP (AddP (DecodeN reg) (LShiftL (ConvI2L idx) scale)) off);
4108 
4109   op_cost(10);
4110   format %{"[$reg + $off + $idx << $scale]" %}
4111   interface(MEMORY_INTER) %{
4112     base($reg);
4113     index($idx);
4114     scale($scale);
4115     disp($off);
4116   %}
4117 %}
4118 
4119 //----------Special Memory Operands--------------------------------------------
4120 // Stack Slot Operand - This operand is used for loading and storing temporary
4121 //                      values on the stack where a match requires a value to
4122 //                      flow through memory.
4123 operand stackSlotP(sRegP reg)
4124 %{
4125   constraint(ALLOC_IN_RC(stack_slots));
4126   // No match rule because this operand is only generated in matching
4127 
4128   format %{ "[$reg]" %}
4129   interface(MEMORY_INTER) %{
4130     base(0x4);   // RSP
4131     index(0x4);  // No Index
4132     scale(0x0);  // No Scale
4133     disp($reg);  // Stack Offset
4134   %}
4135 %}
4136 
4137 operand stackSlotI(sRegI reg)
4138 %{
4139   constraint(ALLOC_IN_RC(stack_slots));
4140   // No match rule because this operand is only generated in matching
4141 
4142   format %{ "[$reg]" %}
4143   interface(MEMORY_INTER) %{
4144     base(0x4);   // RSP
4145     index(0x4);  // No Index
4146     scale(0x0);  // No Scale
4147     disp($reg);  // Stack Offset
4148   %}
4149 %}
4150 
4151 operand stackSlotF(sRegF reg)
4152 %{
4153   constraint(ALLOC_IN_RC(stack_slots));
4154   // No match rule because this operand is only generated in matching
4155 
4156   format %{ "[$reg]" %}
4157   interface(MEMORY_INTER) %{
4158     base(0x4);   // RSP
4159     index(0x4);  // No Index
4160     scale(0x0);  // No Scale
4161     disp($reg);  // Stack Offset
4162   %}
4163 %}
4164 
4165 operand stackSlotD(sRegD reg)
4166 %{
4167   constraint(ALLOC_IN_RC(stack_slots));
4168   // No match rule because this operand is only generated in matching
4169 
4170   format %{ "[$reg]" %}
4171   interface(MEMORY_INTER) %{
4172     base(0x4);   // RSP
4173     index(0x4);  // No Index
4174     scale(0x0);  // No Scale
4175     disp($reg);  // Stack Offset
4176   %}
4177 %}
4178 operand stackSlotL(sRegL reg)
4179 %{
4180   constraint(ALLOC_IN_RC(stack_slots));
4181   // No match rule because this operand is only generated in matching
4182 
4183   format %{ "[$reg]" %}
4184   interface(MEMORY_INTER) %{
4185     base(0x4);   // RSP
4186     index(0x4);  // No Index
4187     scale(0x0);  // No Scale
4188     disp($reg);  // Stack Offset
4189   %}
4190 %}
4191 
4192 //----------Conditional Branch Operands----------------------------------------
4193 // Comparison Op  - This is the operation of the comparison, and is limited to
4194 //                  the following set of codes:
4195 //                  L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
4196 //
4197 // Other attributes of the comparison, such as unsignedness, are specified
4198 // by the comparison instruction that sets a condition code flags register.
4199 // That result is represented by a flags operand whose subtype is appropriate
4200 // to the unsignedness (etc.) of the comparison.
4201 //
4202 // Later, the instruction which matches both the Comparison Op (a Bool) and
4203 // the flags (produced by the Cmp) specifies the coding of the comparison op
4204 // by matching a specific subtype of Bool operand below, such as cmpOpU.
4205 
4206 // Comparision Code
4207 operand cmpOp()
4208 %{
4209   match(Bool);
4210 
4211   format %{ "" %}
4212   interface(COND_INTER) %{
4213     equal(0x4, "e");
4214     not_equal(0x5, "ne");
4215     less(0xC, "l");
4216     greater_equal(0xD, "ge");
4217     less_equal(0xE, "le");
4218     greater(0xF, "g");
4219     overflow(0x0, "o");
4220     no_overflow(0x1, "no");
4221   %}
4222 %}
4223 
4224 // Comparison Code, unsigned compare.  Used by FP also, with
4225 // C2 (unordered) turned into GT or LT already.  The other bits
4226 // C0 and C3 are turned into Carry & Zero flags.
4227 operand cmpOpU()
4228 %{
4229   match(Bool);
4230 
4231   format %{ "" %}
4232   interface(COND_INTER) %{
4233     equal(0x4, "e");
4234     not_equal(0x5, "ne");
4235     less(0x2, "b");
4236     greater_equal(0x3, "nb");
4237     less_equal(0x6, "be");
4238     greater(0x7, "nbe");
4239     overflow(0x0, "o");
4240     no_overflow(0x1, "no");
4241   %}
4242 %}
4243 
4244 
4245 // Floating comparisons that don't require any fixup for the unordered case
4246 operand cmpOpUCF() %{
4247   match(Bool);
4248   predicate(n->as_Bool()->_test._test == BoolTest::lt ||
4249             n->as_Bool()->_test._test == BoolTest::ge ||
4250             n->as_Bool()->_test._test == BoolTest::le ||
4251             n->as_Bool()->_test._test == BoolTest::gt);
4252   format %{ "" %}
4253   interface(COND_INTER) %{
4254     equal(0x4, "e");
4255     not_equal(0x5, "ne");
4256     less(0x2, "b");
4257     greater_equal(0x3, "nb");
4258     less_equal(0x6, "be");
4259     greater(0x7, "nbe");
4260     overflow(0x0, "o");
4261     no_overflow(0x1, "no");
4262   %}
4263 %}
4264 
4265 
4266 // Floating comparisons that can be fixed up with extra conditional jumps
4267 operand cmpOpUCF2() %{
4268   match(Bool);
4269   predicate(n->as_Bool()->_test._test == BoolTest::ne ||
4270             n->as_Bool()->_test._test == BoolTest::eq);
4271   format %{ "" %}
4272   interface(COND_INTER) %{
4273     equal(0x4, "e");
4274     not_equal(0x5, "ne");
4275     less(0x2, "b");
4276     greater_equal(0x3, "nb");
4277     less_equal(0x6, "be");
4278     greater(0x7, "nbe");
4279     overflow(0x0, "o");
4280     no_overflow(0x1, "no");
4281   %}
4282 %}
4283 
4284 // Operands for bound floating pointer register arguments
4285 operand rxmm0() %{
4286   constraint(ALLOC_IN_RC(xmm0_reg));  match(VecX);
4287   predicate((UseSSE > 0) && (UseAVX<= 2));  format%{%}  interface(REG_INTER);
4288 %}
4289 operand rxmm1() %{
4290   constraint(ALLOC_IN_RC(xmm1_reg));  match(VecX);
4291   predicate((UseSSE > 0) && (UseAVX <= 2));  format%{%}  interface(REG_INTER);
4292 %}
4293 operand rxmm2() %{
4294   constraint(ALLOC_IN_RC(xmm2_reg));  match(VecX);
4295   predicate((UseSSE > 0) && (UseAVX <= 2));  format%{%}  interface(REG_INTER);
4296 %}
4297 operand rxmm3() %{
4298   constraint(ALLOC_IN_RC(xmm3_reg));  match(VecX);
4299   predicate((UseSSE > 0) && (UseAVX <= 2));  format%{%}  interface(REG_INTER);
4300 %}
4301 operand rxmm4() %{
4302   constraint(ALLOC_IN_RC(xmm4_reg));  match(VecX);
4303   predicate((UseSSE > 0) && (UseAVX <= 2));  format%{%}  interface(REG_INTER);
4304 %}
4305 operand rxmm5() %{
4306   constraint(ALLOC_IN_RC(xmm5_reg));  match(VecX);
4307   predicate((UseSSE > 0) && (UseAVX <= 2));  format%{%}  interface(REG_INTER);
4308 %}
4309 operand rxmm6() %{
4310   constraint(ALLOC_IN_RC(xmm6_reg));  match(VecX);
4311   predicate((UseSSE > 0) && (UseAVX <= 2));  format%{%}  interface(REG_INTER);
4312 %}
4313 operand rxmm7() %{
4314   constraint(ALLOC_IN_RC(xmm7_reg));  match(VecX);
4315   predicate((UseSSE > 0) && (UseAVX <= 2));  format%{%}  interface(REG_INTER);
4316 %}
4317 operand rxmm8() %{
4318   constraint(ALLOC_IN_RC(xmm8_reg));  match(VecX);
4319   predicate((UseSSE > 0) && (UseAVX <= 2));  format%{%}  interface(REG_INTER);
4320 %}
4321 operand rxmm9() %{
4322   constraint(ALLOC_IN_RC(xmm9_reg));  match(VecX);
4323   predicate((UseSSE > 0) && (UseAVX <= 2));  format%{%}  interface(REG_INTER);
4324 %}
4325 operand rxmm10() %{
4326   constraint(ALLOC_IN_RC(xmm10_reg));  match(VecX);
4327   predicate((UseSSE > 0) && (UseAVX <= 2));  format%{%}  interface(REG_INTER);
4328 %}
4329 operand rxmm11() %{
4330   constraint(ALLOC_IN_RC(xmm11_reg));  match(VecX);
4331   predicate((UseSSE > 0) && (UseAVX <= 2));  format%{%}  interface(REG_INTER);
4332 %}
4333 operand rxmm12() %{
4334   constraint(ALLOC_IN_RC(xmm12_reg));  match(VecX);
4335   predicate((UseSSE > 0) && (UseAVX <= 2));  format%{%}  interface(REG_INTER);
4336 %}
4337 operand rxmm13() %{
4338   constraint(ALLOC_IN_RC(xmm13_reg));  match(VecX);
4339   predicate((UseSSE > 0) && (UseAVX <= 2));  format%{%}  interface(REG_INTER);
4340 %}
4341 operand rxmm14() %{
4342   constraint(ALLOC_IN_RC(xmm14_reg));  match(VecX);
4343   predicate((UseSSE > 0) && (UseAVX <= 2));  format%{%}  interface(REG_INTER);
4344 %}
4345 operand rxmm15() %{
4346   constraint(ALLOC_IN_RC(xmm15_reg));  match(VecX);
4347   predicate((UseSSE > 0) && (UseAVX <= 2));  format%{%}  interface(REG_INTER);
4348 %}
4349 operand rxmm16() %{
4350   constraint(ALLOC_IN_RC(xmm16_reg));  match(VecX);
4351   predicate(UseAVX == 3);  format%{%}  interface(REG_INTER);
4352 %}
4353 operand rxmm17() %{
4354   constraint(ALLOC_IN_RC(xmm17_reg));  match(VecX);
4355   predicate(UseAVX == 3);  format%{%}  interface(REG_INTER);
4356 %}
4357 operand rxmm18() %{
4358   constraint(ALLOC_IN_RC(xmm18_reg));  match(VecX);
4359   predicate(UseAVX == 3);  format%{%}  interface(REG_INTER);
4360 %}
4361 operand rxmm19() %{
4362   constraint(ALLOC_IN_RC(xmm19_reg));  match(VecX);
4363   predicate(UseAVX == 3);  format%{%}  interface(REG_INTER);
4364 %}
4365 operand rxmm20() %{
4366   constraint(ALLOC_IN_RC(xmm20_reg));  match(VecX);
4367   predicate(UseAVX == 3);  format%{%}  interface(REG_INTER);
4368 %}
4369 operand rxmm21() %{
4370   constraint(ALLOC_IN_RC(xmm21_reg));  match(VecX);
4371   predicate(UseAVX == 3);  format%{%}  interface(REG_INTER);
4372 %}
4373 operand rxmm22() %{
4374   constraint(ALLOC_IN_RC(xmm22_reg));  match(VecX);
4375   predicate(UseAVX == 3);  format%{%}  interface(REG_INTER);
4376 %}
4377 operand rxmm23() %{
4378   constraint(ALLOC_IN_RC(xmm23_reg));  match(VecX);
4379   predicate(UseAVX == 3);  format%{%}  interface(REG_INTER);
4380 %}
4381 operand rxmm24() %{
4382   constraint(ALLOC_IN_RC(xmm24_reg));  match(VecX);
4383   predicate(UseAVX == 3);  format%{%}  interface(REG_INTER);
4384 %}
4385 operand rxmm25() %{
4386   constraint(ALLOC_IN_RC(xmm25_reg));  match(VecX);
4387   predicate(UseAVX == 3);  format%{%}  interface(REG_INTER);
4388 %}
4389 operand rxmm26() %{
4390   constraint(ALLOC_IN_RC(xmm26_reg));  match(VecX);
4391   predicate(UseAVX == 3);  format%{%}  interface(REG_INTER);
4392 %}
4393 operand rxmm27() %{
4394   constraint(ALLOC_IN_RC(xmm27_reg));  match(VecX);
4395   predicate(UseAVX == 3);  format%{%}  interface(REG_INTER);
4396 %}
4397 operand rxmm28() %{
4398   constraint(ALLOC_IN_RC(xmm28_reg));  match(VecX);
4399   predicate(UseAVX == 3);  format%{%}  interface(REG_INTER);
4400 %}
4401 operand rxmm29() %{
4402   constraint(ALLOC_IN_RC(xmm29_reg));  match(VecX);
4403   predicate(UseAVX == 3);  format%{%}  interface(REG_INTER);
4404 %}
4405 operand rxmm30() %{
4406   constraint(ALLOC_IN_RC(xmm30_reg));  match(VecX);
4407   predicate(UseAVX == 3);  format%{%}  interface(REG_INTER);
4408 %}
4409 operand rxmm31() %{
4410   constraint(ALLOC_IN_RC(xmm31_reg));  match(VecX);
4411   predicate(UseAVX == 3);  format%{%}  interface(REG_INTER);
4412 %}
4413 
4414 //----------OPERAND CLASSES----------------------------------------------------
4415 // Operand Classes are groups of operands that are used as to simplify
4416 // instruction definitions by not requiring the AD writer to specify separate
4417 // instructions for every form of operand when the instruction accepts
4418 // multiple operand types with the same basic encoding and format.  The classic
4419 // case of this is memory operands.
4420 
4421 opclass memory(indirect, indOffset8, indOffset32, indIndexOffset, indIndex,
4422                indIndexScale, indPosIndexScale, indIndexScaleOffset, indPosIndexOffset, indPosIndexScaleOffset,
4423                indCompressedOopOffset,
4424                indirectNarrow, indOffset8Narrow, indOffset32Narrow,
4425                indIndexOffsetNarrow, indIndexNarrow, indIndexScaleNarrow,
4426                indIndexScaleOffsetNarrow, indPosIndexOffsetNarrow, indPosIndexScaleOffsetNarrow);
4427 
4428 //----------PIPELINE-----------------------------------------------------------
4429 // Rules which define the behavior of the target architectures pipeline.
4430 pipeline %{
4431 
4432 //----------ATTRIBUTES---------------------------------------------------------
4433 attributes %{
4434   variable_size_instructions;        // Fixed size instructions
4435   max_instructions_per_bundle = 3;   // Up to 3 instructions per bundle
4436   instruction_unit_size = 1;         // An instruction is 1 bytes long
4437   instruction_fetch_unit_size = 16;  // The processor fetches one line
4438   instruction_fetch_units = 1;       // of 16 bytes
4439 
4440   // List of nop instructions
4441   nops( MachNop );
4442 %}
4443 
4444 //----------RESOURCES----------------------------------------------------------
4445 // Resources are the functional units available to the machine
4446 
4447 // Generic P2/P3 pipeline
4448 // 3 decoders, only D0 handles big operands; a "bundle" is the limit of
4449 // 3 instructions decoded per cycle.
4450 // 2 load/store ops per cycle, 1 branch, 1 FPU,
4451 // 3 ALU op, only ALU0 handles mul instructions.
4452 resources( D0, D1, D2, DECODE = D0 | D1 | D2,
4453            MS0, MS1, MS2, MEM = MS0 | MS1 | MS2,
4454            BR, FPU,
4455            ALU0, ALU1, ALU2, ALU = ALU0 | ALU1 | ALU2);
4456 
4457 //----------PIPELINE DESCRIPTION-----------------------------------------------
4458 // Pipeline Description specifies the stages in the machine's pipeline
4459 
4460 // Generic P2/P3 pipeline
4461 pipe_desc(S0, S1, S2, S3, S4, S5);
4462 
4463 //----------PIPELINE CLASSES---------------------------------------------------
4464 // Pipeline Classes describe the stages in which input and output are
4465 // referenced by the hardware pipeline.
4466 
4467 // Naming convention: ialu or fpu
4468 // Then: _reg
4469 // Then: _reg if there is a 2nd register
4470 // Then: _long if it's a pair of instructions implementing a long
4471 // Then: _fat if it requires the big decoder
4472 //   Or: _mem if it requires the big decoder and a memory unit.
4473 
4474 // Integer ALU reg operation
4475 pipe_class ialu_reg(rRegI dst)
4476 %{
4477     single_instruction;
4478     dst    : S4(write);
4479     dst    : S3(read);
4480     DECODE : S0;        // any decoder
4481     ALU    : S3;        // any alu
4482 %}
4483 
4484 // Long ALU reg operation
4485 pipe_class ialu_reg_long(rRegL dst)
4486 %{
4487     instruction_count(2);
4488     dst    : S4(write);
4489     dst    : S3(read);
4490     DECODE : S0(2);     // any 2 decoders
4491     ALU    : S3(2);     // both alus
4492 %}
4493 
4494 // Integer ALU reg operation using big decoder
4495 pipe_class ialu_reg_fat(rRegI dst)
4496 %{
4497     single_instruction;
4498     dst    : S4(write);
4499     dst    : S3(read);
4500     D0     : S0;        // big decoder only
4501     ALU    : S3;        // any alu
4502 %}
4503 
4504 // Long ALU reg operation using big decoder
4505 pipe_class ialu_reg_long_fat(rRegL dst)
4506 %{
4507     instruction_count(2);
4508     dst    : S4(write);
4509     dst    : S3(read);
4510     D0     : S0(2);     // big decoder only; twice
4511     ALU    : S3(2);     // any 2 alus
4512 %}
4513 
4514 // Integer ALU reg-reg operation
4515 pipe_class ialu_reg_reg(rRegI dst, rRegI src)
4516 %{
4517     single_instruction;
4518     dst    : S4(write);
4519     src    : S3(read);
4520     DECODE : S0;        // any decoder
4521     ALU    : S3;        // any alu
4522 %}
4523 
4524 // Long ALU reg-reg operation
4525 pipe_class ialu_reg_reg_long(rRegL dst, rRegL src)
4526 %{
4527     instruction_count(2);
4528     dst    : S4(write);
4529     src    : S3(read);
4530     DECODE : S0(2);     // any 2 decoders
4531     ALU    : S3(2);     // both alus
4532 %}
4533 
4534 // Integer ALU reg-reg operation
4535 pipe_class ialu_reg_reg_fat(rRegI dst, memory src)
4536 %{
4537     single_instruction;
4538     dst    : S4(write);
4539     src    : S3(read);
4540     D0     : S0;        // big decoder only
4541     ALU    : S3;        // any alu
4542 %}
4543 
4544 // Long ALU reg-reg operation
4545 pipe_class ialu_reg_reg_long_fat(rRegL dst, rRegL src)
4546 %{
4547     instruction_count(2);
4548     dst    : S4(write);
4549     src    : S3(read);
4550     D0     : S0(2);     // big decoder only; twice
4551     ALU    : S3(2);     // both alus
4552 %}
4553 
4554 // Integer ALU reg-mem operation
4555 pipe_class ialu_reg_mem(rRegI dst, memory mem)
4556 %{
4557     single_instruction;
4558     dst    : S5(write);
4559     mem    : S3(read);
4560     D0     : S0;        // big decoder only
4561     ALU    : S4;        // any alu
4562     MEM    : S3;        // any mem
4563 %}
4564 
4565 // Integer mem operation (prefetch)
4566 pipe_class ialu_mem(memory mem)
4567 %{
4568     single_instruction;
4569     mem    : S3(read);
4570     D0     : S0;        // big decoder only
4571     MEM    : S3;        // any mem
4572 %}
4573 
4574 // Integer Store to Memory
4575 pipe_class ialu_mem_reg(memory mem, rRegI src)
4576 %{
4577     single_instruction;
4578     mem    : S3(read);
4579     src    : S5(read);
4580     D0     : S0;        // big decoder only
4581     ALU    : S4;        // any alu
4582     MEM    : S3;
4583 %}
4584 
4585 // // Long Store to Memory
4586 // pipe_class ialu_mem_long_reg(memory mem, rRegL src)
4587 // %{
4588 //     instruction_count(2);
4589 //     mem    : S3(read);
4590 //     src    : S5(read);
4591 //     D0     : S0(2);          // big decoder only; twice
4592 //     ALU    : S4(2);     // any 2 alus
4593 //     MEM    : S3(2);  // Both mems
4594 // %}
4595 
4596 // Integer Store to Memory
4597 pipe_class ialu_mem_imm(memory mem)
4598 %{
4599     single_instruction;
4600     mem    : S3(read);
4601     D0     : S0;        // big decoder only
4602     ALU    : S4;        // any alu
4603     MEM    : S3;
4604 %}
4605 
4606 // Integer ALU0 reg-reg operation
4607 pipe_class ialu_reg_reg_alu0(rRegI dst, rRegI src)
4608 %{
4609     single_instruction;
4610     dst    : S4(write);
4611     src    : S3(read);
4612     D0     : S0;        // Big decoder only
4613     ALU0   : S3;        // only alu0
4614 %}
4615 
4616 // Integer ALU0 reg-mem operation
4617 pipe_class ialu_reg_mem_alu0(rRegI dst, memory mem)
4618 %{
4619     single_instruction;
4620     dst    : S5(write);
4621     mem    : S3(read);
4622     D0     : S0;        // big decoder only
4623     ALU0   : S4;        // ALU0 only
4624     MEM    : S3;        // any mem
4625 %}
4626 
4627 // Integer ALU reg-reg operation
4628 pipe_class ialu_cr_reg_reg(rFlagsReg cr, rRegI src1, rRegI src2)
4629 %{
4630     single_instruction;
4631     cr     : S4(write);
4632     src1   : S3(read);
4633     src2   : S3(read);
4634     DECODE : S0;        // any decoder
4635     ALU    : S3;        // any alu
4636 %}
4637 
4638 // Integer ALU reg-imm operation
4639 pipe_class ialu_cr_reg_imm(rFlagsReg cr, rRegI src1)
4640 %{
4641     single_instruction;
4642     cr     : S4(write);
4643     src1   : S3(read);
4644     DECODE : S0;        // any decoder
4645     ALU    : S3;        // any alu
4646 %}
4647 
4648 // Integer ALU reg-mem operation
4649 pipe_class ialu_cr_reg_mem(rFlagsReg cr, rRegI src1, memory src2)
4650 %{
4651     single_instruction;
4652     cr     : S4(write);
4653     src1   : S3(read);
4654     src2   : S3(read);
4655     D0     : S0;        // big decoder only
4656     ALU    : S4;        // any alu
4657     MEM    : S3;
4658 %}
4659 
4660 // Conditional move reg-reg
4661 pipe_class pipe_cmplt( rRegI p, rRegI q, rRegI y)
4662 %{
4663     instruction_count(4);
4664     y      : S4(read);
4665     q      : S3(read);
4666     p      : S3(read);
4667     DECODE : S0(4);     // any decoder
4668 %}
4669 
4670 // Conditional move reg-reg
4671 pipe_class pipe_cmov_reg( rRegI dst, rRegI src, rFlagsReg cr)
4672 %{
4673     single_instruction;
4674     dst    : S4(write);
4675     src    : S3(read);
4676     cr     : S3(read);
4677     DECODE : S0;        // any decoder
4678 %}
4679 
4680 // Conditional move reg-mem
4681 pipe_class pipe_cmov_mem( rFlagsReg cr, rRegI dst, memory src)
4682 %{
4683     single_instruction;
4684     dst    : S4(write);
4685     src    : S3(read);
4686     cr     : S3(read);
4687     DECODE : S0;        // any decoder
4688     MEM    : S3;
4689 %}
4690 
4691 // Conditional move reg-reg long
4692 pipe_class pipe_cmov_reg_long( rFlagsReg cr, rRegL dst, rRegL src)
4693 %{
4694     single_instruction;
4695     dst    : S4(write);
4696     src    : S3(read);
4697     cr     : S3(read);
4698     DECODE : S0(2);     // any 2 decoders
4699 %}
4700 
4701 // XXX
4702 // // Conditional move double reg-reg
4703 // pipe_class pipe_cmovD_reg( rFlagsReg cr, regDPR1 dst, regD src)
4704 // %{
4705 //     single_instruction;
4706 //     dst    : S4(write);
4707 //     src    : S3(read);
4708 //     cr     : S3(read);
4709 //     DECODE : S0;     // any decoder
4710 // %}
4711 
4712 // Float reg-reg operation
4713 pipe_class fpu_reg(regD dst)
4714 %{
4715     instruction_count(2);
4716     dst    : S3(read);
4717     DECODE : S0(2);     // any 2 decoders
4718     FPU    : S3;
4719 %}
4720 
4721 // Float reg-reg operation
4722 pipe_class fpu_reg_reg(regD dst, regD src)
4723 %{
4724     instruction_count(2);
4725     dst    : S4(write);
4726     src    : S3(read);
4727     DECODE : S0(2);     // any 2 decoders
4728     FPU    : S3;
4729 %}
4730 
4731 // Float reg-reg operation
4732 pipe_class fpu_reg_reg_reg(regD dst, regD src1, regD src2)
4733 %{
4734     instruction_count(3);
4735     dst    : S4(write);
4736     src1   : S3(read);
4737     src2   : S3(read);
4738     DECODE : S0(3);     // any 3 decoders
4739     FPU    : S3(2);
4740 %}
4741 
4742 // Float reg-reg operation
4743 pipe_class fpu_reg_reg_reg_reg(regD dst, regD src1, regD src2, regD src3)
4744 %{
4745     instruction_count(4);
4746     dst    : S4(write);
4747     src1   : S3(read);
4748     src2   : S3(read);
4749     src3   : S3(read);
4750     DECODE : S0(4);     // any 3 decoders
4751     FPU    : S3(2);
4752 %}
4753 
4754 // Float reg-reg operation
4755 pipe_class fpu_reg_mem_reg_reg(regD dst, memory src1, regD src2, regD src3)
4756 %{
4757     instruction_count(4);
4758     dst    : S4(write);
4759     src1   : S3(read);
4760     src2   : S3(read);
4761     src3   : S3(read);
4762     DECODE : S1(3);     // any 3 decoders
4763     D0     : S0;        // Big decoder only
4764     FPU    : S3(2);
4765     MEM    : S3;
4766 %}
4767 
4768 // Float reg-mem operation
4769 pipe_class fpu_reg_mem(regD dst, memory mem)
4770 %{
4771     instruction_count(2);
4772     dst    : S5(write);
4773     mem    : S3(read);
4774     D0     : S0;        // big decoder only
4775     DECODE : S1;        // any decoder for FPU POP
4776     FPU    : S4;
4777     MEM    : S3;        // any mem
4778 %}
4779 
4780 // Float reg-mem operation
4781 pipe_class fpu_reg_reg_mem(regD dst, regD src1, memory mem)
4782 %{
4783     instruction_count(3);
4784     dst    : S5(write);
4785     src1   : S3(read);
4786     mem    : S3(read);
4787     D0     : S0;        // big decoder only
4788     DECODE : S1(2);     // any decoder for FPU POP
4789     FPU    : S4;
4790     MEM    : S3;        // any mem
4791 %}
4792 
4793 // Float mem-reg operation
4794 pipe_class fpu_mem_reg(memory mem, regD src)
4795 %{
4796     instruction_count(2);
4797     src    : S5(read);
4798     mem    : S3(read);
4799     DECODE : S0;        // any decoder for FPU PUSH
4800     D0     : S1;        // big decoder only
4801     FPU    : S4;
4802     MEM    : S3;        // any mem
4803 %}
4804 
4805 pipe_class fpu_mem_reg_reg(memory mem, regD src1, regD src2)
4806 %{
4807     instruction_count(3);
4808     src1   : S3(read);
4809     src2   : S3(read);
4810     mem    : S3(read);
4811     DECODE : S0(2);     // any decoder for FPU PUSH
4812     D0     : S1;        // big decoder only
4813     FPU    : S4;
4814     MEM    : S3;        // any mem
4815 %}
4816 
4817 pipe_class fpu_mem_reg_mem(memory mem, regD src1, memory src2)
4818 %{
4819     instruction_count(3);
4820     src1   : S3(read);
4821     src2   : S3(read);
4822     mem    : S4(read);
4823     DECODE : S0;        // any decoder for FPU PUSH
4824     D0     : S0(2);     // big decoder only
4825     FPU    : S4;
4826     MEM    : S3(2);     // any mem
4827 %}
4828 
4829 pipe_class fpu_mem_mem(memory dst, memory src1)
4830 %{
4831     instruction_count(2);
4832     src1   : S3(read);
4833     dst    : S4(read);
4834     D0     : S0(2);     // big decoder only
4835     MEM    : S3(2);     // any mem
4836 %}
4837 
4838 pipe_class fpu_mem_mem_mem(memory dst, memory src1, memory src2)
4839 %{
4840     instruction_count(3);
4841     src1   : S3(read);
4842     src2   : S3(read);
4843     dst    : S4(read);
4844     D0     : S0(3);     // big decoder only
4845     FPU    : S4;
4846     MEM    : S3(3);     // any mem
4847 %}
4848 
4849 pipe_class fpu_mem_reg_con(memory mem, regD src1)
4850 %{
4851     instruction_count(3);
4852     src1   : S4(read);
4853     mem    : S4(read);
4854     DECODE : S0;        // any decoder for FPU PUSH
4855     D0     : S0(2);     // big decoder only
4856     FPU    : S4;
4857     MEM    : S3(2);     // any mem
4858 %}
4859 
4860 // Float load constant
4861 pipe_class fpu_reg_con(regD dst)
4862 %{
4863     instruction_count(2);
4864     dst    : S5(write);
4865     D0     : S0;        // big decoder only for the load
4866     DECODE : S1;        // any decoder for FPU POP
4867     FPU    : S4;
4868     MEM    : S3;        // any mem
4869 %}
4870 
4871 // Float load constant
4872 pipe_class fpu_reg_reg_con(regD dst, regD src)
4873 %{
4874     instruction_count(3);
4875     dst    : S5(write);
4876     src    : S3(read);
4877     D0     : S0;        // big decoder only for the load
4878     DECODE : S1(2);     // any decoder for FPU POP
4879     FPU    : S4;
4880     MEM    : S3;        // any mem
4881 %}
4882 
4883 // UnConditional branch
4884 pipe_class pipe_jmp(label labl)
4885 %{
4886     single_instruction;
4887     BR   : S3;
4888 %}
4889 
4890 // Conditional branch
4891 pipe_class pipe_jcc(cmpOp cmp, rFlagsReg cr, label labl)
4892 %{
4893     single_instruction;
4894     cr    : S1(read);
4895     BR    : S3;
4896 %}
4897 
4898 // Allocation idiom
4899 pipe_class pipe_cmpxchg(rRegP dst, rRegP heap_ptr)
4900 %{
4901     instruction_count(1); force_serialization;
4902     fixed_latency(6);
4903     heap_ptr : S3(read);
4904     DECODE   : S0(3);
4905     D0       : S2;
4906     MEM      : S3;
4907     ALU      : S3(2);
4908     dst      : S5(write);
4909     BR       : S5;
4910 %}
4911 
4912 // Generic big/slow expanded idiom
4913 pipe_class pipe_slow()
4914 %{
4915     instruction_count(10); multiple_bundles; force_serialization;
4916     fixed_latency(100);
4917     D0  : S0(2);
4918     MEM : S3(2);
4919 %}
4920 
4921 // The real do-nothing guy
4922 pipe_class empty()
4923 %{
4924     instruction_count(0);
4925 %}
4926 
4927 // Define the class for the Nop node
4928 define
4929 %{
4930    MachNop = empty;
4931 %}
4932 
4933 %}
4934 
4935 //----------INSTRUCTIONS-------------------------------------------------------
4936 //
4937 // match      -- States which machine-independent subtree may be replaced
4938 //               by this instruction.
4939 // ins_cost   -- The estimated cost of this instruction is used by instruction
4940 //               selection to identify a minimum cost tree of machine
4941 //               instructions that matches a tree of machine-independent
4942 //               instructions.
4943 // format     -- A string providing the disassembly for this instruction.
4944 //               The value of an instruction's operand may be inserted
4945 //               by referring to it with a '$' prefix.
4946 // opcode     -- Three instruction opcodes may be provided.  These are referred
4947 //               to within an encode class as $primary, $secondary, and $tertiary
4948 //               rrspectively.  The primary opcode is commonly used to
4949 //               indicate the type of machine instruction, while secondary
4950 //               and tertiary are often used for prefix options or addressing
4951 //               modes.
4952 // ins_encode -- A list of encode classes with parameters. The encode class
4953 //               name must have been defined in an 'enc_class' specification
4954 //               in the encode section of the architecture description.
4955 
4956 
4957 //----------Load/Store/Move Instructions---------------------------------------
4958 //----------Load Instructions--------------------------------------------------
4959 
4960 // Load Byte (8 bit signed)
4961 instruct loadB(rRegI dst, memory mem)
4962 %{
4963   match(Set dst (LoadB mem));
4964 
4965   ins_cost(125);
4966   format %{ "movsbl  $dst, $mem\t# byte" %}
4967 
4968   ins_encode %{
4969     __ movsbl($dst$$Register, $mem$$Address);
4970   %}
4971 
4972   ins_pipe(ialu_reg_mem);
4973 %}
4974 
4975 // Load Byte (8 bit signed) into Long Register
4976 instruct loadB2L(rRegL dst, memory mem)
4977 %{
4978   match(Set dst (ConvI2L (LoadB mem)));
4979 
4980   ins_cost(125);
4981   format %{ "movsbq  $dst, $mem\t# byte -> long" %}
4982 
4983   ins_encode %{
4984     __ movsbq($dst$$Register, $mem$$Address);
4985   %}
4986 
4987   ins_pipe(ialu_reg_mem);
4988 %}
4989 
4990 // Load Unsigned Byte (8 bit UNsigned)
4991 instruct loadUB(rRegI dst, memory mem)
4992 %{
4993   match(Set dst (LoadUB mem));
4994 
4995   ins_cost(125);
4996   format %{ "movzbl  $dst, $mem\t# ubyte" %}
4997 
4998   ins_encode %{
4999     __ movzbl($dst$$Register, $mem$$Address);
5000   %}
5001 
5002   ins_pipe(ialu_reg_mem);
5003 %}
5004 
5005 // Load Unsigned Byte (8 bit UNsigned) into Long Register
5006 instruct loadUB2L(rRegL dst, memory mem)
5007 %{
5008   match(Set dst (ConvI2L (LoadUB mem)));
5009 
5010   ins_cost(125);
5011   format %{ "movzbq  $dst, $mem\t# ubyte -> long" %}
5012 
5013   ins_encode %{
5014     __ movzbq($dst$$Register, $mem$$Address);
5015   %}
5016 
5017   ins_pipe(ialu_reg_mem);
5018 %}
5019 
5020 // Load Unsigned Byte (8 bit UNsigned) with 32-bit mask into Long Register
5021 instruct loadUB2L_immI(rRegL dst, memory mem, immI mask, rFlagsReg cr) %{
5022   match(Set dst (ConvI2L (AndI (LoadUB mem) mask)));
5023   effect(KILL cr);
5024 
5025   format %{ "movzbq  $dst, $mem\t# ubyte & 32-bit mask -> long\n\t"
5026             "andl    $dst, right_n_bits($mask, 8)" %}
5027   ins_encode %{
5028     Register Rdst = $dst$$Register;
5029     __ movzbq(Rdst, $mem$$Address);
5030     __ andl(Rdst, $mask$$constant & right_n_bits(8));
5031   %}
5032   ins_pipe(ialu_reg_mem);
5033 %}
5034 
5035 // Load Short (16 bit signed)
5036 instruct loadS(rRegI dst, memory mem)
5037 %{
5038   match(Set dst (LoadS mem));
5039 
5040   ins_cost(125);
5041   format %{ "movswl $dst, $mem\t# short" %}
5042 
5043   ins_encode %{
5044     __ movswl($dst$$Register, $mem$$Address);
5045   %}
5046 
5047   ins_pipe(ialu_reg_mem);
5048 %}
5049 
5050 // Load Short (16 bit signed) to Byte (8 bit signed)
5051 instruct loadS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
5052   match(Set dst (RShiftI (LShiftI (LoadS mem) twentyfour) twentyfour));
5053 
5054   ins_cost(125);
5055   format %{ "movsbl $dst, $mem\t# short -> byte" %}
5056   ins_encode %{
5057     __ movsbl($dst$$Register, $mem$$Address);
5058   %}
5059   ins_pipe(ialu_reg_mem);
5060 %}
5061 
5062 // Load Short (16 bit signed) into Long Register
5063 instruct loadS2L(rRegL dst, memory mem)
5064 %{
5065   match(Set dst (ConvI2L (LoadS mem)));
5066 
5067   ins_cost(125);
5068   format %{ "movswq $dst, $mem\t# short -> long" %}
5069 
5070   ins_encode %{
5071     __ movswq($dst$$Register, $mem$$Address);
5072   %}
5073 
5074   ins_pipe(ialu_reg_mem);
5075 %}
5076 
5077 // Load Unsigned Short/Char (16 bit UNsigned)
5078 instruct loadUS(rRegI dst, memory mem)
5079 %{
5080   match(Set dst (LoadUS mem));
5081 
5082   ins_cost(125);
5083   format %{ "movzwl  $dst, $mem\t# ushort/char" %}
5084 
5085   ins_encode %{
5086     __ movzwl($dst$$Register, $mem$$Address);
5087   %}
5088 
5089   ins_pipe(ialu_reg_mem);
5090 %}
5091 
5092 // Load Unsigned Short/Char (16 bit UNsigned) to Byte (8 bit signed)
5093 instruct loadUS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
5094   match(Set dst (RShiftI (LShiftI (LoadUS mem) twentyfour) twentyfour));
5095 
5096   ins_cost(125);
5097   format %{ "movsbl $dst, $mem\t# ushort -> byte" %}
5098   ins_encode %{
5099     __ movsbl($dst$$Register, $mem$$Address);
5100   %}
5101   ins_pipe(ialu_reg_mem);
5102 %}
5103 
5104 // Load Unsigned Short/Char (16 bit UNsigned) into Long Register
5105 instruct loadUS2L(rRegL dst, memory mem)
5106 %{
5107   match(Set dst (ConvI2L (LoadUS mem)));
5108 
5109   ins_cost(125);
5110   format %{ "movzwq  $dst, $mem\t# ushort/char -> long" %}
5111 
5112   ins_encode %{
5113     __ movzwq($dst$$Register, $mem$$Address);
5114   %}
5115 
5116   ins_pipe(ialu_reg_mem);
5117 %}
5118 
5119 // Load Unsigned Short/Char (16 bit UNsigned) with mask 0xFF into Long Register
5120 instruct loadUS2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
5121   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
5122 
5123   format %{ "movzbq  $dst, $mem\t# ushort/char & 0xFF -> long" %}
5124   ins_encode %{
5125     __ movzbq($dst$$Register, $mem$$Address);
5126   %}
5127   ins_pipe(ialu_reg_mem);
5128 %}
5129 
5130 // Load Unsigned Short/Char (16 bit UNsigned) with 32-bit mask into Long Register
5131 instruct loadUS2L_immI(rRegL dst, memory mem, immI mask, rFlagsReg cr) %{
5132   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
5133   effect(KILL cr);
5134 
5135   format %{ "movzwq  $dst, $mem\t# ushort/char & 32-bit mask -> long\n\t"
5136             "andl    $dst, right_n_bits($mask, 16)" %}
5137   ins_encode %{
5138     Register Rdst = $dst$$Register;
5139     __ movzwq(Rdst, $mem$$Address);
5140     __ andl(Rdst, $mask$$constant & right_n_bits(16));
5141   %}
5142   ins_pipe(ialu_reg_mem);
5143 %}
5144 
5145 // Load Integer
5146 instruct loadI(rRegI dst, memory mem)
5147 %{
5148   match(Set dst (LoadI mem));
5149 
5150   ins_cost(125);
5151   format %{ "movl    $dst, $mem\t# int" %}
5152 
5153   ins_encode %{
5154     __ movl($dst$$Register, $mem$$Address);
5155   %}
5156 
5157   ins_pipe(ialu_reg_mem);
5158 %}
5159 
5160 // Load Integer (32 bit signed) to Byte (8 bit signed)
5161 instruct loadI2B(rRegI dst, memory mem, immI_24 twentyfour) %{
5162   match(Set dst (RShiftI (LShiftI (LoadI mem) twentyfour) twentyfour));
5163 
5164   ins_cost(125);
5165   format %{ "movsbl  $dst, $mem\t# int -> byte" %}
5166   ins_encode %{
5167     __ movsbl($dst$$Register, $mem$$Address);
5168   %}
5169   ins_pipe(ialu_reg_mem);
5170 %}
5171 
5172 // Load Integer (32 bit signed) to Unsigned Byte (8 bit UNsigned)
5173 instruct loadI2UB(rRegI dst, memory mem, immI_255 mask) %{
5174   match(Set dst (AndI (LoadI mem) mask));
5175 
5176   ins_cost(125);
5177   format %{ "movzbl  $dst, $mem\t# int -> ubyte" %}
5178   ins_encode %{
5179     __ movzbl($dst$$Register, $mem$$Address);
5180   %}
5181   ins_pipe(ialu_reg_mem);
5182 %}
5183 
5184 // Load Integer (32 bit signed) to Short (16 bit signed)
5185 instruct loadI2S(rRegI dst, memory mem, immI_16 sixteen) %{
5186   match(Set dst (RShiftI (LShiftI (LoadI mem) sixteen) sixteen));
5187 
5188   ins_cost(125);
5189   format %{ "movswl  $dst, $mem\t# int -> short" %}
5190   ins_encode %{
5191     __ movswl($dst$$Register, $mem$$Address);
5192   %}
5193   ins_pipe(ialu_reg_mem);
5194 %}
5195 
5196 // Load Integer (32 bit signed) to Unsigned Short/Char (16 bit UNsigned)
5197 instruct loadI2US(rRegI dst, memory mem, immI_65535 mask) %{
5198   match(Set dst (AndI (LoadI mem) mask));
5199 
5200   ins_cost(125);
5201   format %{ "movzwl  $dst, $mem\t# int -> ushort/char" %}
5202   ins_encode %{
5203     __ movzwl($dst$$Register, $mem$$Address);
5204   %}
5205   ins_pipe(ialu_reg_mem);
5206 %}
5207 
5208 // Load Integer into Long Register
5209 instruct loadI2L(rRegL dst, memory mem)
5210 %{
5211   match(Set dst (ConvI2L (LoadI mem)));
5212 
5213   ins_cost(125);
5214   format %{ "movslq  $dst, $mem\t# int -> long" %}
5215 
5216   ins_encode %{
5217     __ movslq($dst$$Register, $mem$$Address);
5218   %}
5219 
5220   ins_pipe(ialu_reg_mem);
5221 %}
5222 
5223 // Load Integer with mask 0xFF into Long Register
5224 instruct loadI2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
5225   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
5226 
5227   format %{ "movzbq  $dst, $mem\t# int & 0xFF -> long" %}
5228   ins_encode %{
5229     __ movzbq($dst$$Register, $mem$$Address);
5230   %}
5231   ins_pipe(ialu_reg_mem);
5232 %}
5233 
5234 // Load Integer with mask 0xFFFF into Long Register
5235 instruct loadI2L_immI_65535(rRegL dst, memory mem, immI_65535 mask) %{
5236   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
5237 
5238   format %{ "movzwq  $dst, $mem\t# int & 0xFFFF -> long" %}
5239   ins_encode %{
5240     __ movzwq($dst$$Register, $mem$$Address);
5241   %}
5242   ins_pipe(ialu_reg_mem);
5243 %}
5244 
5245 // Load Integer with a 31-bit mask into Long Register
5246 instruct loadI2L_immU31(rRegL dst, memory mem, immU31 mask, rFlagsReg cr) %{
5247   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
5248   effect(KILL cr);
5249 
5250   format %{ "movl    $dst, $mem\t# int & 31-bit mask -> long\n\t"
5251             "andl    $dst, $mask" %}
5252   ins_encode %{
5253     Register Rdst = $dst$$Register;
5254     __ movl(Rdst, $mem$$Address);
5255     __ andl(Rdst, $mask$$constant);
5256   %}
5257   ins_pipe(ialu_reg_mem);
5258 %}
5259 
5260 // Load Unsigned Integer into Long Register
5261 instruct loadUI2L(rRegL dst, memory mem, immL_32bits mask)
5262 %{
5263   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
5264 
5265   ins_cost(125);
5266   format %{ "movl    $dst, $mem\t# uint -> long" %}
5267 
5268   ins_encode %{
5269     __ movl($dst$$Register, $mem$$Address);
5270   %}
5271 
5272   ins_pipe(ialu_reg_mem);
5273 %}
5274 
5275 // Load Long
5276 instruct loadL(rRegL dst, memory mem)
5277 %{
5278   match(Set dst (LoadL mem));
5279 
5280   ins_cost(125);
5281   format %{ "movq    $dst, $mem\t# long" %}
5282 
5283   ins_encode %{
5284     __ movq($dst$$Register, $mem$$Address);
5285   %}
5286 
5287   ins_pipe(ialu_reg_mem); // XXX
5288 %}
5289 
5290 // Load Range
5291 instruct loadRange(rRegI dst, memory mem)
5292 %{
5293   match(Set dst (LoadRange mem));
5294 
5295   ins_cost(125); // XXX
5296   format %{ "movl    $dst, $mem\t# range" %}
5297   opcode(0x8B);
5298   ins_encode(REX_reg_mem(dst, mem), OpcP, reg_mem(dst, mem));
5299   ins_pipe(ialu_reg_mem);
5300 %}
5301 
5302 // Load Pointer
5303 instruct loadP(rRegP dst, memory mem)
5304 %{
5305   match(Set dst (LoadP mem));
5306 
5307   ins_cost(125); // XXX
5308   format %{ "movq    $dst, $mem\t# ptr" %}
5309   opcode(0x8B);
5310   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5311   ins_pipe(ialu_reg_mem); // XXX
5312 %}
5313 
5314 // Load Compressed Pointer
5315 instruct loadN(rRegN dst, memory mem)
5316 %{
5317    match(Set dst (LoadN mem));
5318 
5319    ins_cost(125); // XXX
5320    format %{ "movl    $dst, $mem\t# compressed ptr" %}
5321    ins_encode %{
5322      __ movl($dst$$Register, $mem$$Address);
5323    %}
5324    ins_pipe(ialu_reg_mem); // XXX
5325 %}
5326 
5327 
5328 // Load Klass Pointer
5329 instruct loadKlass(rRegP dst, memory mem)
5330 %{
5331   match(Set dst (LoadKlass mem));
5332 
5333   ins_cost(125); // XXX
5334   format %{ "movq    $dst, $mem\t# class" %}
5335   opcode(0x8B);
5336   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5337   ins_pipe(ialu_reg_mem); // XXX
5338 %}
5339 
5340 // Load narrow Klass Pointer
5341 instruct loadNKlass(rRegN dst, memory mem)
5342 %{
5343   match(Set dst (LoadNKlass mem));
5344 
5345   ins_cost(125); // XXX
5346   format %{ "movl    $dst, $mem\t# compressed klass ptr" %}
5347   ins_encode %{
5348     __ movl($dst$$Register, $mem$$Address);
5349   %}
5350   ins_pipe(ialu_reg_mem); // XXX
5351 %}
5352 
5353 // Load Float
5354 instruct loadF(regF dst, memory mem)
5355 %{
5356   match(Set dst (LoadF mem));
5357 
5358   ins_cost(145); // XXX
5359   format %{ "movss   $dst, $mem\t# float" %}
5360   ins_encode %{
5361     __ movflt($dst$$XMMRegister, $mem$$Address);
5362   %}
5363   ins_pipe(pipe_slow); // XXX
5364 %}
5365 
5366 // Load Float
5367 instruct MoveF2VL(vlRegF dst, regF src) %{
5368   match(Set dst src);
5369   format %{ "movss $dst,$src\t! load float (4 bytes)" %}
5370   ins_encode %{
5371     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
5372   %}
5373   ins_pipe( fpu_reg_reg );
5374 %}
5375 
5376 // Load Float
5377 instruct MoveVL2F(regF dst, vlRegF src) %{
5378   match(Set dst src);
5379   format %{ "movss $dst,$src\t! load float (4 bytes)" %}
5380   ins_encode %{
5381     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
5382   %}
5383   ins_pipe( fpu_reg_reg );
5384 %}
5385 
5386 // Load Double
5387 instruct loadD_partial(regD dst, memory mem)
5388 %{
5389   predicate(!UseXmmLoadAndClearUpper);
5390   match(Set dst (LoadD mem));
5391 
5392   ins_cost(145); // XXX
5393   format %{ "movlpd  $dst, $mem\t# double" %}
5394   ins_encode %{
5395     __ movdbl($dst$$XMMRegister, $mem$$Address);
5396   %}
5397   ins_pipe(pipe_slow); // XXX
5398 %}
5399 
5400 instruct loadD(regD dst, memory mem)
5401 %{
5402   predicate(UseXmmLoadAndClearUpper);
5403   match(Set dst (LoadD mem));
5404 
5405   ins_cost(145); // XXX
5406   format %{ "movsd   $dst, $mem\t# double" %}
5407   ins_encode %{
5408     __ movdbl($dst$$XMMRegister, $mem$$Address);
5409   %}
5410   ins_pipe(pipe_slow); // XXX
5411 %}
5412 
5413 // Load Double
5414 instruct MoveD2VL(vlRegD dst, regD src) %{
5415   match(Set dst src);
5416   format %{ "movsd $dst,$src\t! load double (8 bytes)" %}
5417   ins_encode %{
5418     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
5419   %}
5420   ins_pipe( fpu_reg_reg );
5421 %}
5422 
5423 // Load Double
5424 instruct MoveVL2D(regD dst, vlRegD src) %{
5425   match(Set dst src);
5426   format %{ "movsd $dst,$src\t! load double (8 bytes)" %}
5427   ins_encode %{
5428     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
5429   %}
5430   ins_pipe( fpu_reg_reg );
5431 %}
5432 
5433 // Load Effective Address
5434 instruct leaP8(rRegP dst, indOffset8 mem)
5435 %{
5436   match(Set dst mem);
5437 
5438   ins_cost(110); // XXX
5439   format %{ "leaq    $dst, $mem\t# ptr 8" %}
5440   opcode(0x8D);
5441   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5442   ins_pipe(ialu_reg_reg_fat);
5443 %}
5444 
5445 instruct leaP32(rRegP dst, indOffset32 mem)
5446 %{
5447   match(Set dst mem);
5448 
5449   ins_cost(110);
5450   format %{ "leaq    $dst, $mem\t# ptr 32" %}
5451   opcode(0x8D);
5452   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5453   ins_pipe(ialu_reg_reg_fat);
5454 %}
5455 
5456 // instruct leaPIdx(rRegP dst, indIndex mem)
5457 // %{
5458 //   match(Set dst mem);
5459 
5460 //   ins_cost(110);
5461 //   format %{ "leaq    $dst, $mem\t# ptr idx" %}
5462 //   opcode(0x8D);
5463 //   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5464 //   ins_pipe(ialu_reg_reg_fat);
5465 // %}
5466 
5467 instruct leaPIdxOff(rRegP dst, indIndexOffset mem)
5468 %{
5469   match(Set dst mem);
5470 
5471   ins_cost(110);
5472   format %{ "leaq    $dst, $mem\t# ptr idxoff" %}
5473   opcode(0x8D);
5474   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5475   ins_pipe(ialu_reg_reg_fat);
5476 %}
5477 
5478 instruct leaPIdxScale(rRegP dst, indIndexScale mem)
5479 %{
5480   match(Set dst mem);
5481 
5482   ins_cost(110);
5483   format %{ "leaq    $dst, $mem\t# ptr idxscale" %}
5484   opcode(0x8D);
5485   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5486   ins_pipe(ialu_reg_reg_fat);
5487 %}
5488 
5489 instruct leaPPosIdxScale(rRegP dst, indPosIndexScale mem)
5490 %{
5491   match(Set dst mem);
5492 
5493   ins_cost(110);
5494   format %{ "leaq    $dst, $mem\t# ptr idxscale" %}
5495   opcode(0x8D);
5496   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5497   ins_pipe(ialu_reg_reg_fat);
5498 %}
5499 
5500 instruct leaPIdxScaleOff(rRegP dst, indIndexScaleOffset mem)
5501 %{
5502   match(Set dst mem);
5503 
5504   ins_cost(110);
5505   format %{ "leaq    $dst, $mem\t# ptr idxscaleoff" %}
5506   opcode(0x8D);
5507   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5508   ins_pipe(ialu_reg_reg_fat);
5509 %}
5510 
5511 instruct leaPPosIdxOff(rRegP dst, indPosIndexOffset mem)
5512 %{
5513   match(Set dst mem);
5514 
5515   ins_cost(110);
5516   format %{ "leaq    $dst, $mem\t# ptr posidxoff" %}
5517   opcode(0x8D);
5518   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5519   ins_pipe(ialu_reg_reg_fat);
5520 %}
5521 
5522 instruct leaPPosIdxScaleOff(rRegP dst, indPosIndexScaleOffset mem)
5523 %{
5524   match(Set dst mem);
5525 
5526   ins_cost(110);
5527   format %{ "leaq    $dst, $mem\t# ptr posidxscaleoff" %}
5528   opcode(0x8D);
5529   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5530   ins_pipe(ialu_reg_reg_fat);
5531 %}
5532 
5533 // Load Effective Address which uses Narrow (32-bits) oop
5534 instruct leaPCompressedOopOffset(rRegP dst, indCompressedOopOffset mem)
5535 %{
5536   predicate(UseCompressedOops && (Universe::narrow_oop_shift() != 0));
5537   match(Set dst mem);
5538 
5539   ins_cost(110);
5540   format %{ "leaq    $dst, $mem\t# ptr compressedoopoff32" %}
5541   opcode(0x8D);
5542   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5543   ins_pipe(ialu_reg_reg_fat);
5544 %}
5545 
5546 instruct leaP8Narrow(rRegP dst, indOffset8Narrow mem)
5547 %{
5548   predicate(Universe::narrow_oop_shift() == 0);
5549   match(Set dst mem);
5550 
5551   ins_cost(110); // XXX
5552   format %{ "leaq    $dst, $mem\t# ptr off8narrow" %}
5553   opcode(0x8D);
5554   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5555   ins_pipe(ialu_reg_reg_fat);
5556 %}
5557 
5558 instruct leaP32Narrow(rRegP dst, indOffset32Narrow mem)
5559 %{
5560   predicate(Universe::narrow_oop_shift() == 0);
5561   match(Set dst mem);
5562 
5563   ins_cost(110);
5564   format %{ "leaq    $dst, $mem\t# ptr off32narrow" %}
5565   opcode(0x8D);
5566   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5567   ins_pipe(ialu_reg_reg_fat);
5568 %}
5569 
5570 instruct leaPIdxOffNarrow(rRegP dst, indIndexOffsetNarrow mem)
5571 %{
5572   predicate(Universe::narrow_oop_shift() == 0);
5573   match(Set dst mem);
5574 
5575   ins_cost(110);
5576   format %{ "leaq    $dst, $mem\t# ptr idxoffnarrow" %}
5577   opcode(0x8D);
5578   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5579   ins_pipe(ialu_reg_reg_fat);
5580 %}
5581 
5582 instruct leaPIdxScaleNarrow(rRegP dst, indIndexScaleNarrow mem)
5583 %{
5584   predicate(Universe::narrow_oop_shift() == 0);
5585   match(Set dst mem);
5586 
5587   ins_cost(110);
5588   format %{ "leaq    $dst, $mem\t# ptr idxscalenarrow" %}
5589   opcode(0x8D);
5590   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5591   ins_pipe(ialu_reg_reg_fat);
5592 %}
5593 
5594 instruct leaPIdxScaleOffNarrow(rRegP dst, indIndexScaleOffsetNarrow mem)
5595 %{
5596   predicate(Universe::narrow_oop_shift() == 0);
5597   match(Set dst mem);
5598 
5599   ins_cost(110);
5600   format %{ "leaq    $dst, $mem\t# ptr idxscaleoffnarrow" %}
5601   opcode(0x8D);
5602   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5603   ins_pipe(ialu_reg_reg_fat);
5604 %}
5605 
5606 instruct leaPPosIdxOffNarrow(rRegP dst, indPosIndexOffsetNarrow mem)
5607 %{
5608   predicate(Universe::narrow_oop_shift() == 0);
5609   match(Set dst mem);
5610 
5611   ins_cost(110);
5612   format %{ "leaq    $dst, $mem\t# ptr posidxoffnarrow" %}
5613   opcode(0x8D);
5614   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5615   ins_pipe(ialu_reg_reg_fat);
5616 %}
5617 
5618 instruct leaPPosIdxScaleOffNarrow(rRegP dst, indPosIndexScaleOffsetNarrow mem)
5619 %{
5620   predicate(Universe::narrow_oop_shift() == 0);
5621   match(Set dst mem);
5622 
5623   ins_cost(110);
5624   format %{ "leaq    $dst, $mem\t# ptr posidxscaleoffnarrow" %}
5625   opcode(0x8D);
5626   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5627   ins_pipe(ialu_reg_reg_fat);
5628 %}
5629 
5630 instruct loadConI(rRegI dst, immI src)
5631 %{
5632   match(Set dst src);
5633 
5634   format %{ "movl    $dst, $src\t# int" %}
5635   ins_encode(load_immI(dst, src));
5636   ins_pipe(ialu_reg_fat); // XXX
5637 %}
5638 
5639 instruct loadConI0(rRegI dst, immI0 src, rFlagsReg cr)
5640 %{
5641   match(Set dst src);
5642   effect(KILL cr);
5643 
5644   ins_cost(50);
5645   format %{ "xorl    $dst, $dst\t# int" %}
5646   opcode(0x33); /* + rd */
5647   ins_encode(REX_reg_reg(dst, dst), OpcP, reg_reg(dst, dst));
5648   ins_pipe(ialu_reg);
5649 %}
5650 
5651 instruct loadConL(rRegL dst, immL src)
5652 %{
5653   match(Set dst src);
5654 
5655   ins_cost(150);
5656   format %{ "movq    $dst, $src\t# long" %}
5657   ins_encode(load_immL(dst, src));
5658   ins_pipe(ialu_reg);
5659 %}
5660 
5661 instruct loadConL0(rRegL dst, immL0 src, rFlagsReg cr)
5662 %{
5663   match(Set dst src);
5664   effect(KILL cr);
5665 
5666   ins_cost(50);
5667   format %{ "xorl    $dst, $dst\t# long" %}
5668   opcode(0x33); /* + rd */
5669   ins_encode(REX_reg_reg(dst, dst), OpcP, reg_reg(dst, dst));
5670   ins_pipe(ialu_reg); // XXX
5671 %}
5672 
5673 instruct loadConUL32(rRegL dst, immUL32 src)
5674 %{
5675   match(Set dst src);
5676 
5677   ins_cost(60);
5678   format %{ "movl    $dst, $src\t# long (unsigned 32-bit)" %}
5679   ins_encode(load_immUL32(dst, src));
5680   ins_pipe(ialu_reg);
5681 %}
5682 
5683 instruct loadConL32(rRegL dst, immL32 src)
5684 %{
5685   match(Set dst src);
5686 
5687   ins_cost(70);
5688   format %{ "movq    $dst, $src\t# long (32-bit)" %}
5689   ins_encode(load_immL32(dst, src));
5690   ins_pipe(ialu_reg);
5691 %}
5692 
5693 instruct loadConP(rRegP dst, immP con) %{
5694   match(Set dst con);
5695 
5696   format %{ "movq    $dst, $con\t# ptr" %}
5697   ins_encode(load_immP(dst, con));
5698   ins_pipe(ialu_reg_fat); // XXX
5699 %}
5700 
5701 instruct loadConP0(rRegP dst, immP0 src, rFlagsReg cr)
5702 %{
5703   match(Set dst src);
5704   effect(KILL cr);
5705 
5706   ins_cost(50);
5707   format %{ "xorl    $dst, $dst\t# ptr" %}
5708   opcode(0x33); /* + rd */
5709   ins_encode(REX_reg_reg(dst, dst), OpcP, reg_reg(dst, dst));
5710   ins_pipe(ialu_reg);
5711 %}
5712 
5713 instruct loadConP31(rRegP dst, immP31 src, rFlagsReg cr)
5714 %{
5715   match(Set dst src);
5716   effect(KILL cr);
5717 
5718   ins_cost(60);
5719   format %{ "movl    $dst, $src\t# ptr (positive 32-bit)" %}
5720   ins_encode(load_immP31(dst, src));
5721   ins_pipe(ialu_reg);
5722 %}
5723 
5724 instruct loadConF(regF dst, immF con) %{
5725   match(Set dst con);
5726   ins_cost(125);
5727   format %{ "movss   $dst, [$constantaddress]\t# load from constant table: float=$con" %}
5728   ins_encode %{
5729     __ movflt($dst$$XMMRegister, $constantaddress($con));
5730   %}
5731   ins_pipe(pipe_slow);
5732 %}
5733 
5734 instruct loadConN0(rRegN dst, immN0 src, rFlagsReg cr) %{
5735   match(Set dst src);
5736   effect(KILL cr);
5737   format %{ "xorq    $dst, $src\t# compressed NULL ptr" %}
5738   ins_encode %{
5739     __ xorq($dst$$Register, $dst$$Register);
5740   %}
5741   ins_pipe(ialu_reg);
5742 %}
5743 
5744 instruct loadConN(rRegN dst, immN src) %{
5745   match(Set dst src);
5746 
5747   ins_cost(125);
5748   format %{ "movl    $dst, $src\t# compressed ptr" %}
5749   ins_encode %{
5750     address con = (address)$src$$constant;
5751     if (con == NULL) {
5752       ShouldNotReachHere();
5753     } else {
5754       __ set_narrow_oop($dst$$Register, (jobject)$src$$constant);
5755     }
5756   %}
5757   ins_pipe(ialu_reg_fat); // XXX
5758 %}
5759 
5760 instruct loadConNKlass(rRegN dst, immNKlass src) %{
5761   match(Set dst src);
5762 
5763   ins_cost(125);
5764   format %{ "movl    $dst, $src\t# compressed klass ptr" %}
5765   ins_encode %{
5766     address con = (address)$src$$constant;
5767     if (con == NULL) {
5768       ShouldNotReachHere();
5769     } else {
5770       __ set_narrow_klass($dst$$Register, (Klass*)$src$$constant);
5771     }
5772   %}
5773   ins_pipe(ialu_reg_fat); // XXX
5774 %}
5775 
5776 instruct loadConF0(regF dst, immF0 src)
5777 %{
5778   match(Set dst src);
5779   ins_cost(100);
5780 
5781   format %{ "xorps   $dst, $dst\t# float 0.0" %}
5782   ins_encode %{
5783     __ xorps($dst$$XMMRegister, $dst$$XMMRegister);
5784   %}
5785   ins_pipe(pipe_slow);
5786 %}
5787 
5788 // Use the same format since predicate() can not be used here.
5789 instruct loadConD(regD dst, immD con) %{
5790   match(Set dst con);
5791   ins_cost(125);
5792   format %{ "movsd   $dst, [$constantaddress]\t# load from constant table: double=$con" %}
5793   ins_encode %{
5794     __ movdbl($dst$$XMMRegister, $constantaddress($con));
5795   %}
5796   ins_pipe(pipe_slow);
5797 %}
5798 
5799 instruct loadConD0(regD dst, immD0 src)
5800 %{
5801   match(Set dst src);
5802   ins_cost(100);
5803 
5804   format %{ "xorpd   $dst, $dst\t# double 0.0" %}
5805   ins_encode %{
5806     __ xorpd ($dst$$XMMRegister, $dst$$XMMRegister);
5807   %}
5808   ins_pipe(pipe_slow);
5809 %}
5810 
5811 instruct loadSSI(rRegI dst, stackSlotI src)
5812 %{
5813   match(Set dst src);
5814 
5815   ins_cost(125);
5816   format %{ "movl    $dst, $src\t# int stk" %}
5817   opcode(0x8B);
5818   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
5819   ins_pipe(ialu_reg_mem);
5820 %}
5821 
5822 instruct loadSSL(rRegL dst, stackSlotL src)
5823 %{
5824   match(Set dst src);
5825 
5826   ins_cost(125);
5827   format %{ "movq    $dst, $src\t# long stk" %}
5828   opcode(0x8B);
5829   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
5830   ins_pipe(ialu_reg_mem);
5831 %}
5832 
5833 instruct loadSSP(rRegP dst, stackSlotP src)
5834 %{
5835   match(Set dst src);
5836 
5837   ins_cost(125);
5838   format %{ "movq    $dst, $src\t# ptr stk" %}
5839   opcode(0x8B);
5840   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
5841   ins_pipe(ialu_reg_mem);
5842 %}
5843 
5844 instruct loadSSF(regF dst, stackSlotF src)
5845 %{
5846   match(Set dst src);
5847 
5848   ins_cost(125);
5849   format %{ "movss   $dst, $src\t# float stk" %}
5850   ins_encode %{
5851     __ movflt($dst$$XMMRegister, Address(rsp, $src$$disp));
5852   %}
5853   ins_pipe(pipe_slow); // XXX
5854 %}
5855 
5856 // Use the same format since predicate() can not be used here.
5857 instruct loadSSD(regD dst, stackSlotD src)
5858 %{
5859   match(Set dst src);
5860 
5861   ins_cost(125);
5862   format %{ "movsd   $dst, $src\t# double stk" %}
5863   ins_encode  %{
5864     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
5865   %}
5866   ins_pipe(pipe_slow); // XXX
5867 %}
5868 
5869 // Prefetch instructions for allocation.
5870 // Must be safe to execute with invalid address (cannot fault).
5871 
5872 instruct prefetchAlloc( memory mem ) %{
5873   predicate(AllocatePrefetchInstr==3);
5874   match(PrefetchAllocation mem);
5875   ins_cost(125);
5876 
5877   format %{ "PREFETCHW $mem\t# Prefetch allocation into level 1 cache and mark modified" %}
5878   ins_encode %{
5879     __ prefetchw($mem$$Address);
5880   %}
5881   ins_pipe(ialu_mem);
5882 %}
5883 
5884 instruct prefetchAllocNTA( memory mem ) %{
5885   predicate(AllocatePrefetchInstr==0);
5886   match(PrefetchAllocation mem);
5887   ins_cost(125);
5888 
5889   format %{ "PREFETCHNTA $mem\t# Prefetch allocation to non-temporal cache for write" %}
5890   ins_encode %{
5891     __ prefetchnta($mem$$Address);
5892   %}
5893   ins_pipe(ialu_mem);
5894 %}
5895 
5896 instruct prefetchAllocT0( memory mem ) %{
5897   predicate(AllocatePrefetchInstr==1);
5898   match(PrefetchAllocation mem);
5899   ins_cost(125);
5900 
5901   format %{ "PREFETCHT0 $mem\t# Prefetch allocation to level 1 and 2 caches for write" %}
5902   ins_encode %{
5903     __ prefetcht0($mem$$Address);
5904   %}
5905   ins_pipe(ialu_mem);
5906 %}
5907 
5908 instruct prefetchAllocT2( memory mem ) %{
5909   predicate(AllocatePrefetchInstr==2);
5910   match(PrefetchAllocation mem);
5911   ins_cost(125);
5912 
5913   format %{ "PREFETCHT2 $mem\t# Prefetch allocation to level 2 cache for write" %}
5914   ins_encode %{
5915     __ prefetcht2($mem$$Address);
5916   %}
5917   ins_pipe(ialu_mem);
5918 %}
5919 
5920 //----------Store Instructions-------------------------------------------------
5921 
5922 // Store Byte
5923 instruct storeB(memory mem, rRegI src)
5924 %{
5925   match(Set mem (StoreB mem src));
5926 
5927   ins_cost(125); // XXX
5928   format %{ "movb    $mem, $src\t# byte" %}
5929   opcode(0x88);
5930   ins_encode(REX_breg_mem(src, mem), OpcP, reg_mem(src, mem));
5931   ins_pipe(ialu_mem_reg);
5932 %}
5933 
5934 // Store Char/Short
5935 instruct storeC(memory mem, rRegI src)
5936 %{
5937   match(Set mem (StoreC mem src));
5938 
5939   ins_cost(125); // XXX
5940   format %{ "movw    $mem, $src\t# char/short" %}
5941   opcode(0x89);
5942   ins_encode(SizePrefix, REX_reg_mem(src, mem), OpcP, reg_mem(src, mem));
5943   ins_pipe(ialu_mem_reg);
5944 %}
5945 
5946 // Store Integer
5947 instruct storeI(memory mem, rRegI src)
5948 %{
5949   match(Set mem (StoreI mem src));
5950 
5951   ins_cost(125); // XXX
5952   format %{ "movl    $mem, $src\t# int" %}
5953   opcode(0x89);
5954   ins_encode(REX_reg_mem(src, mem), OpcP, reg_mem(src, mem));
5955   ins_pipe(ialu_mem_reg);
5956 %}
5957 
5958 // Store Long
5959 instruct storeL(memory mem, rRegL src)
5960 %{
5961   match(Set mem (StoreL mem src));
5962 
5963   ins_cost(125); // XXX
5964   format %{ "movq    $mem, $src\t# long" %}
5965   opcode(0x89);
5966   ins_encode(REX_reg_mem_wide(src, mem), OpcP, reg_mem(src, mem));
5967   ins_pipe(ialu_mem_reg); // XXX
5968 %}
5969 
5970 // Store Pointer
5971 instruct storeP(memory mem, any_RegP src)
5972 %{
5973   match(Set mem (StoreP mem src));
5974 
5975   ins_cost(125); // XXX
5976   format %{ "movq    $mem, $src\t# ptr" %}
5977   opcode(0x89);
5978   ins_encode(REX_reg_mem_wide(src, mem), OpcP, reg_mem(src, mem));
5979   ins_pipe(ialu_mem_reg);
5980 %}
5981 
5982 instruct storeImmP0(memory mem, immP0 zero)
5983 %{
5984   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5985   match(Set mem (StoreP mem zero));
5986 
5987   ins_cost(125); // XXX
5988   format %{ "movq    $mem, R12\t# ptr (R12_heapbase==0)" %}
5989   ins_encode %{
5990     __ movq($mem$$Address, r12);
5991   %}
5992   ins_pipe(ialu_mem_reg);
5993 %}
5994 
5995 // Store NULL Pointer, mark word, or other simple pointer constant.
5996 instruct storeImmP(memory mem, immP31 src)
5997 %{
5998   match(Set mem (StoreP mem src));
5999 
6000   ins_cost(150); // XXX
6001   format %{ "movq    $mem, $src\t# ptr" %}
6002   opcode(0xC7); /* C7 /0 */
6003   ins_encode(REX_mem_wide(mem), OpcP, RM_opc_mem(0x00, mem), Con32(src));
6004   ins_pipe(ialu_mem_imm);
6005 %}
6006 
6007 // Store Compressed Pointer
6008 instruct storeN(memory mem, rRegN src)
6009 %{
6010   match(Set mem (StoreN mem src));
6011 
6012   ins_cost(125); // XXX
6013   format %{ "movl    $mem, $src\t# compressed ptr" %}
6014   ins_encode %{
6015     __ movl($mem$$Address, $src$$Register);
6016   %}
6017   ins_pipe(ialu_mem_reg);
6018 %}
6019 
6020 instruct storeNKlass(memory mem, rRegN src)
6021 %{
6022   match(Set mem (StoreNKlass mem src));
6023 
6024   ins_cost(125); // XXX
6025   format %{ "movl    $mem, $src\t# compressed klass ptr" %}
6026   ins_encode %{
6027     __ movl($mem$$Address, $src$$Register);
6028   %}
6029   ins_pipe(ialu_mem_reg);
6030 %}
6031 
6032 instruct storeImmN0(memory mem, immN0 zero)
6033 %{
6034   predicate(Universe::narrow_oop_base() == NULL && Universe::narrow_klass_base() == NULL);
6035   match(Set mem (StoreN mem zero));
6036 
6037   ins_cost(125); // XXX
6038   format %{ "movl    $mem, R12\t# compressed ptr (R12_heapbase==0)" %}
6039   ins_encode %{
6040     __ movl($mem$$Address, r12);
6041   %}
6042   ins_pipe(ialu_mem_reg);
6043 %}
6044 
6045 instruct storeImmN(memory mem, immN src)
6046 %{
6047   match(Set mem (StoreN mem src));
6048 
6049   ins_cost(150); // XXX
6050   format %{ "movl    $mem, $src\t# compressed ptr" %}
6051   ins_encode %{
6052     address con = (address)$src$$constant;
6053     if (con == NULL) {
6054       __ movl($mem$$Address, (int32_t)0);
6055     } else {
6056       __ set_narrow_oop($mem$$Address, (jobject)$src$$constant);
6057     }
6058   %}
6059   ins_pipe(ialu_mem_imm);
6060 %}
6061 
6062 instruct storeImmNKlass(memory mem, immNKlass src)
6063 %{
6064   match(Set mem (StoreNKlass mem src));
6065 
6066   ins_cost(150); // XXX
6067   format %{ "movl    $mem, $src\t# compressed klass ptr" %}
6068   ins_encode %{
6069     __ set_narrow_klass($mem$$Address, (Klass*)$src$$constant);
6070   %}
6071   ins_pipe(ialu_mem_imm);
6072 %}
6073 
6074 // Store Integer Immediate
6075 instruct storeImmI0(memory mem, immI0 zero)
6076 %{
6077   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
6078   match(Set mem (StoreI mem zero));
6079 
6080   ins_cost(125); // XXX
6081   format %{ "movl    $mem, R12\t# int (R12_heapbase==0)" %}
6082   ins_encode %{
6083     __ movl($mem$$Address, r12);
6084   %}
6085   ins_pipe(ialu_mem_reg);
6086 %}
6087 
6088 instruct storeImmI(memory mem, immI src)
6089 %{
6090   match(Set mem (StoreI mem src));
6091 
6092   ins_cost(150);
6093   format %{ "movl    $mem, $src\t# int" %}
6094   opcode(0xC7); /* C7 /0 */
6095   ins_encode(REX_mem(mem), OpcP, RM_opc_mem(0x00, mem), Con32(src));
6096   ins_pipe(ialu_mem_imm);
6097 %}
6098 
6099 // Store Long Immediate
6100 instruct storeImmL0(memory mem, immL0 zero)
6101 %{
6102   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
6103   match(Set mem (StoreL mem zero));
6104 
6105   ins_cost(125); // XXX
6106   format %{ "movq    $mem, R12\t# long (R12_heapbase==0)" %}
6107   ins_encode %{
6108     __ movq($mem$$Address, r12);
6109   %}
6110   ins_pipe(ialu_mem_reg);
6111 %}
6112 
6113 instruct storeImmL(memory mem, immL32 src)
6114 %{
6115   match(Set mem (StoreL mem src));
6116 
6117   ins_cost(150);
6118   format %{ "movq    $mem, $src\t# long" %}
6119   opcode(0xC7); /* C7 /0 */
6120   ins_encode(REX_mem_wide(mem), OpcP, RM_opc_mem(0x00, mem), Con32(src));
6121   ins_pipe(ialu_mem_imm);
6122 %}
6123 
6124 // Store Short/Char Immediate
6125 instruct storeImmC0(memory mem, immI0 zero)
6126 %{
6127   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
6128   match(Set mem (StoreC mem zero));
6129 
6130   ins_cost(125); // XXX
6131   format %{ "movw    $mem, R12\t# short/char (R12_heapbase==0)" %}
6132   ins_encode %{
6133     __ movw($mem$$Address, r12);
6134   %}
6135   ins_pipe(ialu_mem_reg);
6136 %}
6137 
6138 instruct storeImmI16(memory mem, immI16 src)
6139 %{
6140   predicate(UseStoreImmI16);
6141   match(Set mem (StoreC mem src));
6142 
6143   ins_cost(150);
6144   format %{ "movw    $mem, $src\t# short/char" %}
6145   opcode(0xC7); /* C7 /0 Same as 32 store immediate with prefix */
6146   ins_encode(SizePrefix, REX_mem(mem), OpcP, RM_opc_mem(0x00, mem),Con16(src));
6147   ins_pipe(ialu_mem_imm);
6148 %}
6149 
6150 // Store Byte Immediate
6151 instruct storeImmB0(memory mem, immI0 zero)
6152 %{
6153   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
6154   match(Set mem (StoreB mem zero));
6155 
6156   ins_cost(125); // XXX
6157   format %{ "movb    $mem, R12\t# short/char (R12_heapbase==0)" %}
6158   ins_encode %{
6159     __ movb($mem$$Address, r12);
6160   %}
6161   ins_pipe(ialu_mem_reg);
6162 %}
6163 
6164 instruct storeImmB(memory mem, immI8 src)
6165 %{
6166   match(Set mem (StoreB mem src));
6167 
6168   ins_cost(150); // XXX
6169   format %{ "movb    $mem, $src\t# byte" %}
6170   opcode(0xC6); /* C6 /0 */
6171   ins_encode(REX_mem(mem), OpcP, RM_opc_mem(0x00, mem), Con8or32(src));
6172   ins_pipe(ialu_mem_imm);
6173 %}
6174 
6175 // Store CMS card-mark Immediate
6176 instruct storeImmCM0_reg(memory mem, immI0 zero)
6177 %{
6178   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
6179   match(Set mem (StoreCM mem zero));
6180 
6181   ins_cost(125); // XXX
6182   format %{ "movb    $mem, R12\t# CMS card-mark byte 0 (R12_heapbase==0)" %}
6183   ins_encode %{
6184     __ movb($mem$$Address, r12);
6185   %}
6186   ins_pipe(ialu_mem_reg);
6187 %}
6188 
6189 instruct storeImmCM0(memory mem, immI0 src)
6190 %{
6191   match(Set mem (StoreCM mem src));
6192 
6193   ins_cost(150); // XXX
6194   format %{ "movb    $mem, $src\t# CMS card-mark byte 0" %}
6195   opcode(0xC6); /* C6 /0 */
6196   ins_encode(REX_mem(mem), OpcP, RM_opc_mem(0x00, mem), Con8or32(src));
6197   ins_pipe(ialu_mem_imm);
6198 %}
6199 
6200 // Store Float
6201 instruct storeF(memory mem, regF src)
6202 %{
6203   match(Set mem (StoreF mem src));
6204 
6205   ins_cost(95); // XXX
6206   format %{ "movss   $mem, $src\t# float" %}
6207   ins_encode %{
6208     __ movflt($mem$$Address, $src$$XMMRegister);
6209   %}
6210   ins_pipe(pipe_slow); // XXX
6211 %}
6212 
6213 // Store immediate Float value (it is faster than store from XMM register)
6214 instruct storeF0(memory mem, immF0 zero)
6215 %{
6216   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
6217   match(Set mem (StoreF mem zero));
6218 
6219   ins_cost(25); // XXX
6220   format %{ "movl    $mem, R12\t# float 0. (R12_heapbase==0)" %}
6221   ins_encode %{
6222     __ movl($mem$$Address, r12);
6223   %}
6224   ins_pipe(ialu_mem_reg);
6225 %}
6226 
6227 instruct storeF_imm(memory mem, immF src)
6228 %{
6229   match(Set mem (StoreF mem src));
6230 
6231   ins_cost(50);
6232   format %{ "movl    $mem, $src\t# float" %}
6233   opcode(0xC7); /* C7 /0 */
6234   ins_encode(REX_mem(mem), OpcP, RM_opc_mem(0x00, mem), Con32F_as_bits(src));
6235   ins_pipe(ialu_mem_imm);
6236 %}
6237 
6238 // Store Double
6239 instruct storeD(memory mem, regD src)
6240 %{
6241   match(Set mem (StoreD mem src));
6242 
6243   ins_cost(95); // XXX
6244   format %{ "movsd   $mem, $src\t# double" %}
6245   ins_encode %{
6246     __ movdbl($mem$$Address, $src$$XMMRegister);
6247   %}
6248   ins_pipe(pipe_slow); // XXX
6249 %}
6250 
6251 // Store immediate double 0.0 (it is faster than store from XMM register)
6252 instruct storeD0_imm(memory mem, immD0 src)
6253 %{
6254   predicate(!UseCompressedOops || (Universe::narrow_oop_base() != NULL));
6255   match(Set mem (StoreD mem src));
6256 
6257   ins_cost(50);
6258   format %{ "movq    $mem, $src\t# double 0." %}
6259   opcode(0xC7); /* C7 /0 */
6260   ins_encode(REX_mem_wide(mem), OpcP, RM_opc_mem(0x00, mem), Con32F_as_bits(src));
6261   ins_pipe(ialu_mem_imm);
6262 %}
6263 
6264 instruct storeD0(memory mem, immD0 zero)
6265 %{
6266   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
6267   match(Set mem (StoreD mem zero));
6268 
6269   ins_cost(25); // XXX
6270   format %{ "movq    $mem, R12\t# double 0. (R12_heapbase==0)" %}
6271   ins_encode %{
6272     __ movq($mem$$Address, r12);
6273   %}
6274   ins_pipe(ialu_mem_reg);
6275 %}
6276 
6277 instruct storeSSI(stackSlotI dst, rRegI src)
6278 %{
6279   match(Set dst src);
6280 
6281   ins_cost(100);
6282   format %{ "movl    $dst, $src\t# int stk" %}
6283   opcode(0x89);
6284   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
6285   ins_pipe( ialu_mem_reg );
6286 %}
6287 
6288 instruct storeSSL(stackSlotL dst, rRegL src)
6289 %{
6290   match(Set dst src);
6291 
6292   ins_cost(100);
6293   format %{ "movq    $dst, $src\t# long stk" %}
6294   opcode(0x89);
6295   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
6296   ins_pipe(ialu_mem_reg);
6297 %}
6298 
6299 instruct storeSSP(stackSlotP dst, rRegP src)
6300 %{
6301   match(Set dst src);
6302 
6303   ins_cost(100);
6304   format %{ "movq    $dst, $src\t# ptr stk" %}
6305   opcode(0x89);
6306   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
6307   ins_pipe(ialu_mem_reg);
6308 %}
6309 
6310 instruct storeSSF(stackSlotF dst, regF src)
6311 %{
6312   match(Set dst src);
6313 
6314   ins_cost(95); // XXX
6315   format %{ "movss   $dst, $src\t# float stk" %}
6316   ins_encode %{
6317     __ movflt(Address(rsp, $dst$$disp), $src$$XMMRegister);
6318   %}
6319   ins_pipe(pipe_slow); // XXX
6320 %}
6321 
6322 instruct storeSSD(stackSlotD dst, regD src)
6323 %{
6324   match(Set dst src);
6325 
6326   ins_cost(95); // XXX
6327   format %{ "movsd   $dst, $src\t# double stk" %}
6328   ins_encode %{
6329     __ movdbl(Address(rsp, $dst$$disp), $src$$XMMRegister);
6330   %}
6331   ins_pipe(pipe_slow); // XXX
6332 %}
6333 
6334 //----------BSWAP Instructions-------------------------------------------------
6335 instruct bytes_reverse_int(rRegI dst) %{
6336   match(Set dst (ReverseBytesI dst));
6337 
6338   format %{ "bswapl  $dst" %}
6339   opcode(0x0F, 0xC8);  /*Opcode 0F /C8 */
6340   ins_encode( REX_reg(dst), OpcP, opc2_reg(dst) );
6341   ins_pipe( ialu_reg );
6342 %}
6343 
6344 instruct bytes_reverse_long(rRegL dst) %{
6345   match(Set dst (ReverseBytesL dst));
6346 
6347   format %{ "bswapq  $dst" %}
6348   opcode(0x0F, 0xC8); /* Opcode 0F /C8 */
6349   ins_encode( REX_reg_wide(dst), OpcP, opc2_reg(dst) );
6350   ins_pipe( ialu_reg);
6351 %}
6352 
6353 instruct bytes_reverse_unsigned_short(rRegI dst, rFlagsReg cr) %{
6354   match(Set dst (ReverseBytesUS dst));
6355   effect(KILL cr);
6356 
6357   format %{ "bswapl  $dst\n\t"
6358             "shrl    $dst,16\n\t" %}
6359   ins_encode %{
6360     __ bswapl($dst$$Register);
6361     __ shrl($dst$$Register, 16);
6362   %}
6363   ins_pipe( ialu_reg );
6364 %}
6365 
6366 instruct bytes_reverse_short(rRegI dst, rFlagsReg cr) %{
6367   match(Set dst (ReverseBytesS dst));
6368   effect(KILL cr);
6369 
6370   format %{ "bswapl  $dst\n\t"
6371             "sar     $dst,16\n\t" %}
6372   ins_encode %{
6373     __ bswapl($dst$$Register);
6374     __ sarl($dst$$Register, 16);
6375   %}
6376   ins_pipe( ialu_reg );
6377 %}
6378 
6379 //---------- Zeros Count Instructions ------------------------------------------
6380 
6381 instruct countLeadingZerosI(rRegI dst, rRegI src, rFlagsReg cr) %{
6382   predicate(UseCountLeadingZerosInstruction);
6383   match(Set dst (CountLeadingZerosI src));
6384   effect(KILL cr);
6385 
6386   format %{ "lzcntl  $dst, $src\t# count leading zeros (int)" %}
6387   ins_encode %{
6388     __ lzcntl($dst$$Register, $src$$Register);
6389   %}
6390   ins_pipe(ialu_reg);
6391 %}
6392 
6393 instruct countLeadingZerosI_bsr(rRegI dst, rRegI src, rFlagsReg cr) %{
6394   predicate(!UseCountLeadingZerosInstruction);
6395   match(Set dst (CountLeadingZerosI src));
6396   effect(KILL cr);
6397 
6398   format %{ "bsrl    $dst, $src\t# count leading zeros (int)\n\t"
6399             "jnz     skip\n\t"
6400             "movl    $dst, -1\n"
6401       "skip:\n\t"
6402             "negl    $dst\n\t"
6403             "addl    $dst, 31" %}
6404   ins_encode %{
6405     Register Rdst = $dst$$Register;
6406     Register Rsrc = $src$$Register;
6407     Label skip;
6408     __ bsrl(Rdst, Rsrc);
6409     __ jccb(Assembler::notZero, skip);
6410     __ movl(Rdst, -1);
6411     __ bind(skip);
6412     __ negl(Rdst);
6413     __ addl(Rdst, BitsPerInt - 1);
6414   %}
6415   ins_pipe(ialu_reg);
6416 %}
6417 
6418 instruct countLeadingZerosL(rRegI dst, rRegL src, rFlagsReg cr) %{
6419   predicate(UseCountLeadingZerosInstruction);
6420   match(Set dst (CountLeadingZerosL src));
6421   effect(KILL cr);
6422 
6423   format %{ "lzcntq  $dst, $src\t# count leading zeros (long)" %}
6424   ins_encode %{
6425     __ lzcntq($dst$$Register, $src$$Register);
6426   %}
6427   ins_pipe(ialu_reg);
6428 %}
6429 
6430 instruct countLeadingZerosL_bsr(rRegI dst, rRegL src, rFlagsReg cr) %{
6431   predicate(!UseCountLeadingZerosInstruction);
6432   match(Set dst (CountLeadingZerosL src));
6433   effect(KILL cr);
6434 
6435   format %{ "bsrq    $dst, $src\t# count leading zeros (long)\n\t"
6436             "jnz     skip\n\t"
6437             "movl    $dst, -1\n"
6438       "skip:\n\t"
6439             "negl    $dst\n\t"
6440             "addl    $dst, 63" %}
6441   ins_encode %{
6442     Register Rdst = $dst$$Register;
6443     Register Rsrc = $src$$Register;
6444     Label skip;
6445     __ bsrq(Rdst, Rsrc);
6446     __ jccb(Assembler::notZero, skip);
6447     __ movl(Rdst, -1);
6448     __ bind(skip);
6449     __ negl(Rdst);
6450     __ addl(Rdst, BitsPerLong - 1);
6451   %}
6452   ins_pipe(ialu_reg);
6453 %}
6454 
6455 instruct countTrailingZerosI(rRegI dst, rRegI src, rFlagsReg cr) %{
6456   predicate(UseCountTrailingZerosInstruction);
6457   match(Set dst (CountTrailingZerosI src));
6458   effect(KILL cr);
6459 
6460   format %{ "tzcntl    $dst, $src\t# count trailing zeros (int)" %}
6461   ins_encode %{
6462     __ tzcntl($dst$$Register, $src$$Register);
6463   %}
6464   ins_pipe(ialu_reg);
6465 %}
6466 
6467 instruct countTrailingZerosI_bsf(rRegI dst, rRegI src, rFlagsReg cr) %{
6468   predicate(!UseCountTrailingZerosInstruction);
6469   match(Set dst (CountTrailingZerosI src));
6470   effect(KILL cr);
6471 
6472   format %{ "bsfl    $dst, $src\t# count trailing zeros (int)\n\t"
6473             "jnz     done\n\t"
6474             "movl    $dst, 32\n"
6475       "done:" %}
6476   ins_encode %{
6477     Register Rdst = $dst$$Register;
6478     Label done;
6479     __ bsfl(Rdst, $src$$Register);
6480     __ jccb(Assembler::notZero, done);
6481     __ movl(Rdst, BitsPerInt);
6482     __ bind(done);
6483   %}
6484   ins_pipe(ialu_reg);
6485 %}
6486 
6487 instruct countTrailingZerosL(rRegI dst, rRegL src, rFlagsReg cr) %{
6488   predicate(UseCountTrailingZerosInstruction);
6489   match(Set dst (CountTrailingZerosL src));
6490   effect(KILL cr);
6491 
6492   format %{ "tzcntq    $dst, $src\t# count trailing zeros (long)" %}
6493   ins_encode %{
6494     __ tzcntq($dst$$Register, $src$$Register);
6495   %}
6496   ins_pipe(ialu_reg);
6497 %}
6498 
6499 instruct countTrailingZerosL_bsf(rRegI dst, rRegL src, rFlagsReg cr) %{
6500   predicate(!UseCountTrailingZerosInstruction);
6501   match(Set dst (CountTrailingZerosL src));
6502   effect(KILL cr);
6503 
6504   format %{ "bsfq    $dst, $src\t# count trailing zeros (long)\n\t"
6505             "jnz     done\n\t"
6506             "movl    $dst, 64\n"
6507       "done:" %}
6508   ins_encode %{
6509     Register Rdst = $dst$$Register;
6510     Label done;
6511     __ bsfq(Rdst, $src$$Register);
6512     __ jccb(Assembler::notZero, done);
6513     __ movl(Rdst, BitsPerLong);
6514     __ bind(done);
6515   %}
6516   ins_pipe(ialu_reg);
6517 %}
6518 
6519 
6520 //---------- Population Count Instructions -------------------------------------
6521 
6522 instruct popCountI(rRegI dst, rRegI src, rFlagsReg cr) %{
6523   predicate(UsePopCountInstruction);
6524   match(Set dst (PopCountI src));
6525   effect(KILL cr);
6526 
6527   format %{ "popcnt  $dst, $src" %}
6528   ins_encode %{
6529     __ popcntl($dst$$Register, $src$$Register);
6530   %}
6531   ins_pipe(ialu_reg);
6532 %}
6533 
6534 instruct popCountI_mem(rRegI dst, memory mem, rFlagsReg cr) %{
6535   predicate(UsePopCountInstruction);
6536   match(Set dst (PopCountI (LoadI mem)));
6537   effect(KILL cr);
6538 
6539   format %{ "popcnt  $dst, $mem" %}
6540   ins_encode %{
6541     __ popcntl($dst$$Register, $mem$$Address);
6542   %}
6543   ins_pipe(ialu_reg);
6544 %}
6545 
6546 // Note: Long.bitCount(long) returns an int.
6547 instruct popCountL(rRegI dst, rRegL src, rFlagsReg cr) %{
6548   predicate(UsePopCountInstruction);
6549   match(Set dst (PopCountL src));
6550   effect(KILL cr);
6551 
6552   format %{ "popcnt  $dst, $src" %}
6553   ins_encode %{
6554     __ popcntq($dst$$Register, $src$$Register);
6555   %}
6556   ins_pipe(ialu_reg);
6557 %}
6558 
6559 // Note: Long.bitCount(long) returns an int.
6560 instruct popCountL_mem(rRegI dst, memory mem, rFlagsReg cr) %{
6561   predicate(UsePopCountInstruction);
6562   match(Set dst (PopCountL (LoadL mem)));
6563   effect(KILL cr);
6564 
6565   format %{ "popcnt  $dst, $mem" %}
6566   ins_encode %{
6567     __ popcntq($dst$$Register, $mem$$Address);
6568   %}
6569   ins_pipe(ialu_reg);
6570 %}
6571 
6572 
6573 //----------MemBar Instructions-----------------------------------------------
6574 // Memory barrier flavors
6575 
6576 instruct membar_acquire()
6577 %{
6578   match(MemBarAcquire);
6579   match(LoadFence);
6580   ins_cost(0);
6581 
6582   size(0);
6583   format %{ "MEMBAR-acquire ! (empty encoding)" %}
6584   ins_encode();
6585   ins_pipe(empty);
6586 %}
6587 
6588 instruct membar_acquire_lock()
6589 %{
6590   match(MemBarAcquireLock);
6591   ins_cost(0);
6592 
6593   size(0);
6594   format %{ "MEMBAR-acquire (prior CMPXCHG in FastLock so empty encoding)" %}
6595   ins_encode();
6596   ins_pipe(empty);
6597 %}
6598 
6599 instruct membar_release()
6600 %{
6601   match(MemBarRelease);
6602   match(StoreFence);
6603   ins_cost(0);
6604 
6605   size(0);
6606   format %{ "MEMBAR-release ! (empty encoding)" %}
6607   ins_encode();
6608   ins_pipe(empty);
6609 %}
6610 
6611 instruct membar_release_lock()
6612 %{
6613   match(MemBarReleaseLock);
6614   ins_cost(0);
6615 
6616   size(0);
6617   format %{ "MEMBAR-release (a FastUnlock follows so empty encoding)" %}
6618   ins_encode();
6619   ins_pipe(empty);
6620 %}
6621 
6622 instruct membar_volatile(rFlagsReg cr) %{
6623   match(MemBarVolatile);
6624   effect(KILL cr);
6625   ins_cost(400);
6626 
6627   format %{
6628     $$template
6629     if (os::is_MP()) {
6630       $$emit$$"lock addl [rsp + #0], 0\t! membar_volatile"
6631     } else {
6632       $$emit$$"MEMBAR-volatile ! (empty encoding)"
6633     }
6634   %}
6635   ins_encode %{
6636     __ membar(Assembler::StoreLoad);
6637   %}
6638   ins_pipe(pipe_slow);
6639 %}
6640 
6641 instruct unnecessary_membar_volatile()
6642 %{
6643   match(MemBarVolatile);
6644   predicate(Matcher::post_store_load_barrier(n));
6645   ins_cost(0);
6646 
6647   size(0);
6648   format %{ "MEMBAR-volatile (unnecessary so empty encoding)" %}
6649   ins_encode();
6650   ins_pipe(empty);
6651 %}
6652 
6653 instruct membar_storestore() %{
6654   match(MemBarStoreStore);
6655   ins_cost(0);
6656 
6657   size(0);
6658   format %{ "MEMBAR-storestore (empty encoding)" %}
6659   ins_encode( );
6660   ins_pipe(empty);
6661 %}
6662 
6663 //----------Move Instructions--------------------------------------------------
6664 
6665 instruct castX2P(rRegP dst, rRegL src)
6666 %{
6667   match(Set dst (CastX2P src));
6668 
6669   format %{ "movq    $dst, $src\t# long->ptr" %}
6670   ins_encode %{
6671     if ($dst$$reg != $src$$reg) {
6672       __ movptr($dst$$Register, $src$$Register);
6673     }
6674   %}
6675   ins_pipe(ialu_reg_reg); // XXX
6676 %}
6677 
6678 instruct castP2X(rRegL dst, rRegP src)
6679 %{
6680   match(Set dst (CastP2X src));
6681 
6682   format %{ "movq    $dst, $src\t# ptr -> long" %}
6683   ins_encode %{
6684     if ($dst$$reg != $src$$reg) {
6685       __ movptr($dst$$Register, $src$$Register);
6686     }
6687   %}
6688   ins_pipe(ialu_reg_reg); // XXX
6689 %}
6690 
6691 // Convert oop into int for vectors alignment masking
6692 instruct convP2I(rRegI dst, rRegP src)
6693 %{
6694   match(Set dst (ConvL2I (CastP2X src)));
6695 
6696   format %{ "movl    $dst, $src\t# ptr -> int" %}
6697   ins_encode %{
6698     __ movl($dst$$Register, $src$$Register);
6699   %}
6700   ins_pipe(ialu_reg_reg); // XXX
6701 %}
6702 
6703 // Convert compressed oop into int for vectors alignment masking
6704 // in case of 32bit oops (heap < 4Gb).
6705 instruct convN2I(rRegI dst, rRegN src)
6706 %{
6707   predicate(Universe::narrow_oop_shift() == 0);
6708   match(Set dst (ConvL2I (CastP2X (DecodeN src))));
6709 
6710   format %{ "movl    $dst, $src\t# compressed ptr -> int" %}
6711   ins_encode %{
6712     __ movl($dst$$Register, $src$$Register);
6713   %}
6714   ins_pipe(ialu_reg_reg); // XXX
6715 %}
6716 
6717 // Convert oop pointer into compressed form
6718 instruct encodeHeapOop(rRegN dst, rRegP src, rFlagsReg cr) %{
6719   predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
6720   match(Set dst (EncodeP src));
6721   effect(KILL cr);
6722   format %{ "encode_heap_oop $dst,$src" %}
6723   ins_encode %{
6724     Register s = $src$$Register;
6725     Register d = $dst$$Register;
6726     if (s != d) {
6727       __ movq(d, s);
6728     }
6729     __ encode_heap_oop(d);
6730   %}
6731   ins_pipe(ialu_reg_long);
6732 %}
6733 
6734 instruct encodeHeapOop_not_null(rRegN dst, rRegP src, rFlagsReg cr) %{
6735   predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
6736   match(Set dst (EncodeP src));
6737   effect(KILL cr);
6738   format %{ "encode_heap_oop_not_null $dst,$src" %}
6739   ins_encode %{
6740     __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
6741   %}
6742   ins_pipe(ialu_reg_long);
6743 %}
6744 
6745 instruct decodeHeapOop(rRegP dst, rRegN src, rFlagsReg cr) %{
6746   predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
6747             n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
6748   match(Set dst (DecodeN src));
6749   effect(KILL cr);
6750   format %{ "decode_heap_oop $dst,$src" %}
6751   ins_encode %{
6752     Register s = $src$$Register;
6753     Register d = $dst$$Register;
6754     if (s != d) {
6755       __ movq(d, s);
6756     }
6757     __ decode_heap_oop(d);
6758   %}
6759   ins_pipe(ialu_reg_long);
6760 %}
6761 
6762 instruct decodeHeapOop_not_null(rRegP dst, rRegN src, rFlagsReg cr) %{
6763   predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
6764             n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
6765   match(Set dst (DecodeN src));
6766   effect(KILL cr);
6767   format %{ "decode_heap_oop_not_null $dst,$src" %}
6768   ins_encode %{
6769     Register s = $src$$Register;
6770     Register d = $dst$$Register;
6771     if (s != d) {
6772       __ decode_heap_oop_not_null(d, s);
6773     } else {
6774       __ decode_heap_oop_not_null(d);
6775     }
6776   %}
6777   ins_pipe(ialu_reg_long);
6778 %}
6779 
6780 instruct encodeKlass_not_null(rRegN dst, rRegP src, rFlagsReg cr) %{
6781   match(Set dst (EncodePKlass src));
6782   effect(KILL cr);
6783   format %{ "encode_klass_not_null $dst,$src" %}
6784   ins_encode %{
6785     __ encode_klass_not_null($dst$$Register, $src$$Register);
6786   %}
6787   ins_pipe(ialu_reg_long);
6788 %}
6789 
6790 instruct decodeKlass_not_null(rRegP dst, rRegN src, rFlagsReg cr) %{
6791   match(Set dst (DecodeNKlass src));
6792   effect(KILL cr);
6793   format %{ "decode_klass_not_null $dst,$src" %}
6794   ins_encode %{
6795     Register s = $src$$Register;
6796     Register d = $dst$$Register;
6797     if (s != d) {
6798       __ decode_klass_not_null(d, s);
6799     } else {
6800       __ decode_klass_not_null(d);
6801     }
6802   %}
6803   ins_pipe(ialu_reg_long);
6804 %}
6805 
6806 
6807 //----------Conditional Move---------------------------------------------------
6808 // Jump
6809 // dummy instruction for generating temp registers
6810 instruct jumpXtnd_offset(rRegL switch_val, immI2 shift, rRegI dest) %{
6811   match(Jump (LShiftL switch_val shift));
6812   ins_cost(350);
6813   predicate(false);
6814   effect(TEMP dest);
6815 
6816   format %{ "leaq    $dest, [$constantaddress]\n\t"
6817             "jmp     [$dest + $switch_val << $shift]\n\t" %}
6818   ins_encode %{
6819     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
6820     // to do that and the compiler is using that register as one it can allocate.
6821     // So we build it all by hand.
6822     // Address index(noreg, switch_reg, (Address::ScaleFactor)$shift$$constant);
6823     // ArrayAddress dispatch(table, index);
6824     Address dispatch($dest$$Register, $switch_val$$Register, (Address::ScaleFactor) $shift$$constant);
6825     __ lea($dest$$Register, $constantaddress);
6826     __ jmp(dispatch);
6827   %}
6828   ins_pipe(pipe_jmp);
6829 %}
6830 
6831 instruct jumpXtnd_addr(rRegL switch_val, immI2 shift, immL32 offset, rRegI dest) %{
6832   match(Jump (AddL (LShiftL switch_val shift) offset));
6833   ins_cost(350);
6834   effect(TEMP dest);
6835 
6836   format %{ "leaq    $dest, [$constantaddress]\n\t"
6837             "jmp     [$dest + $switch_val << $shift + $offset]\n\t" %}
6838   ins_encode %{
6839     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
6840     // to do that and the compiler is using that register as one it can allocate.
6841     // So we build it all by hand.
6842     // Address index(noreg, switch_reg, (Address::ScaleFactor) $shift$$constant, (int) $offset$$constant);
6843     // ArrayAddress dispatch(table, index);
6844     Address dispatch($dest$$Register, $switch_val$$Register, (Address::ScaleFactor) $shift$$constant, (int) $offset$$constant);
6845     __ lea($dest$$Register, $constantaddress);
6846     __ jmp(dispatch);
6847   %}
6848   ins_pipe(pipe_jmp);
6849 %}
6850 
6851 instruct jumpXtnd(rRegL switch_val, rRegI dest) %{
6852   match(Jump switch_val);
6853   ins_cost(350);
6854   effect(TEMP dest);
6855 
6856   format %{ "leaq    $dest, [$constantaddress]\n\t"
6857             "jmp     [$dest + $switch_val]\n\t" %}
6858   ins_encode %{
6859     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
6860     // to do that and the compiler is using that register as one it can allocate.
6861     // So we build it all by hand.
6862     // Address index(noreg, switch_reg, Address::times_1);
6863     // ArrayAddress dispatch(table, index);
6864     Address dispatch($dest$$Register, $switch_val$$Register, Address::times_1);
6865     __ lea($dest$$Register, $constantaddress);
6866     __ jmp(dispatch);
6867   %}
6868   ins_pipe(pipe_jmp);
6869 %}
6870 
6871 // Conditional move
6872 instruct cmovI_reg(rRegI dst, rRegI src, rFlagsReg cr, cmpOp cop)
6873 %{
6874   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
6875 
6876   ins_cost(200); // XXX
6877   format %{ "cmovl$cop $dst, $src\t# signed, int" %}
6878   opcode(0x0F, 0x40);
6879   ins_encode(REX_reg_reg(dst, src), enc_cmov(cop), reg_reg(dst, src));
6880   ins_pipe(pipe_cmov_reg);
6881 %}
6882 
6883 instruct cmovI_regU(cmpOpU cop, rFlagsRegU cr, rRegI dst, rRegI src) %{
6884   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
6885 
6886   ins_cost(200); // XXX
6887   format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
6888   opcode(0x0F, 0x40);
6889   ins_encode(REX_reg_reg(dst, src), enc_cmov(cop), reg_reg(dst, src));
6890   ins_pipe(pipe_cmov_reg);
6891 %}
6892 
6893 instruct cmovI_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
6894   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
6895   ins_cost(200);
6896   expand %{
6897     cmovI_regU(cop, cr, dst, src);
6898   %}
6899 %}
6900 
6901 // Conditional move
6902 instruct cmovI_mem(cmpOp cop, rFlagsReg cr, rRegI dst, memory src) %{
6903   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
6904 
6905   ins_cost(250); // XXX
6906   format %{ "cmovl$cop $dst, $src\t# signed, int" %}
6907   opcode(0x0F, 0x40);
6908   ins_encode(REX_reg_mem(dst, src), enc_cmov(cop), reg_mem(dst, src));
6909   ins_pipe(pipe_cmov_mem);
6910 %}
6911 
6912 // Conditional move
6913 instruct cmovI_memU(cmpOpU cop, rFlagsRegU cr, rRegI dst, memory src)
6914 %{
6915   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
6916 
6917   ins_cost(250); // XXX
6918   format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
6919   opcode(0x0F, 0x40);
6920   ins_encode(REX_reg_mem(dst, src), enc_cmov(cop), reg_mem(dst, src));
6921   ins_pipe(pipe_cmov_mem);
6922 %}
6923 
6924 instruct cmovI_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, memory src) %{
6925   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
6926   ins_cost(250);
6927   expand %{
6928     cmovI_memU(cop, cr, dst, src);
6929   %}
6930 %}
6931 
6932 // Conditional move
6933 instruct cmovN_reg(rRegN dst, rRegN src, rFlagsReg cr, cmpOp cop)
6934 %{
6935   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
6936 
6937   ins_cost(200); // XXX
6938   format %{ "cmovl$cop $dst, $src\t# signed, compressed ptr" %}
6939   opcode(0x0F, 0x40);
6940   ins_encode(REX_reg_reg(dst, src), enc_cmov(cop), reg_reg(dst, src));
6941   ins_pipe(pipe_cmov_reg);
6942 %}
6943 
6944 // Conditional move
6945 instruct cmovN_regU(cmpOpU cop, rFlagsRegU cr, rRegN dst, rRegN src)
6946 %{
6947   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
6948 
6949   ins_cost(200); // XXX
6950   format %{ "cmovl$cop $dst, $src\t# unsigned, compressed ptr" %}
6951   opcode(0x0F, 0x40);
6952   ins_encode(REX_reg_reg(dst, src), enc_cmov(cop), reg_reg(dst, src));
6953   ins_pipe(pipe_cmov_reg);
6954 %}
6955 
6956 instruct cmovN_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegN dst, rRegN src) %{
6957   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
6958   ins_cost(200);
6959   expand %{
6960     cmovN_regU(cop, cr, dst, src);
6961   %}
6962 %}
6963 
6964 // Conditional move
6965 instruct cmovP_reg(rRegP dst, rRegP src, rFlagsReg cr, cmpOp cop)
6966 %{
6967   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
6968 
6969   ins_cost(200); // XXX
6970   format %{ "cmovq$cop $dst, $src\t# signed, ptr" %}
6971   opcode(0x0F, 0x40);
6972   ins_encode(REX_reg_reg_wide(dst, src), enc_cmov(cop), reg_reg(dst, src));
6973   ins_pipe(pipe_cmov_reg);  // XXX
6974 %}
6975 
6976 // Conditional move
6977 instruct cmovP_regU(cmpOpU cop, rFlagsRegU cr, rRegP dst, rRegP src)
6978 %{
6979   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
6980 
6981   ins_cost(200); // XXX
6982   format %{ "cmovq$cop $dst, $src\t# unsigned, ptr" %}
6983   opcode(0x0F, 0x40);
6984   ins_encode(REX_reg_reg_wide(dst, src), enc_cmov(cop), reg_reg(dst, src));
6985   ins_pipe(pipe_cmov_reg); // XXX
6986 %}
6987 
6988 instruct cmovP_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
6989   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
6990   ins_cost(200);
6991   expand %{
6992     cmovP_regU(cop, cr, dst, src);
6993   %}
6994 %}
6995 
6996 // DISABLED: Requires the ADLC to emit a bottom_type call that
6997 // correctly meets the two pointer arguments; one is an incoming
6998 // register but the other is a memory operand.  ALSO appears to
6999 // be buggy with implicit null checks.
7000 //
7001 //// Conditional move
7002 //instruct cmovP_mem(cmpOp cop, rFlagsReg cr, rRegP dst, memory src)
7003 //%{
7004 //  match(Set dst (CMoveP (Binary cop cr) (Binary dst (LoadP src))));
7005 //  ins_cost(250);
7006 //  format %{ "CMOV$cop $dst,$src\t# ptr" %}
7007 //  opcode(0x0F,0x40);
7008 //  ins_encode( enc_cmov(cop), reg_mem( dst, src ) );
7009 //  ins_pipe( pipe_cmov_mem );
7010 //%}
7011 //
7012 //// Conditional move
7013 //instruct cmovP_memU(cmpOpU cop, rFlagsRegU cr, rRegP dst, memory src)
7014 //%{
7015 //  match(Set dst (CMoveP (Binary cop cr) (Binary dst (LoadP src))));
7016 //  ins_cost(250);
7017 //  format %{ "CMOV$cop $dst,$src\t# ptr" %}
7018 //  opcode(0x0F,0x40);
7019 //  ins_encode( enc_cmov(cop), reg_mem( dst, src ) );
7020 //  ins_pipe( pipe_cmov_mem );
7021 //%}
7022 
7023 instruct cmovL_reg(cmpOp cop, rFlagsReg cr, rRegL dst, rRegL src)
7024 %{
7025   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
7026 
7027   ins_cost(200); // XXX
7028   format %{ "cmovq$cop $dst, $src\t# signed, long" %}
7029   opcode(0x0F, 0x40);
7030   ins_encode(REX_reg_reg_wide(dst, src), enc_cmov(cop), reg_reg(dst, src));
7031   ins_pipe(pipe_cmov_reg);  // XXX
7032 %}
7033 
7034 instruct cmovL_mem(cmpOp cop, rFlagsReg cr, rRegL dst, memory src)
7035 %{
7036   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
7037 
7038   ins_cost(200); // XXX
7039   format %{ "cmovq$cop $dst, $src\t# signed, long" %}
7040   opcode(0x0F, 0x40);
7041   ins_encode(REX_reg_mem_wide(dst, src), enc_cmov(cop), reg_mem(dst, src));
7042   ins_pipe(pipe_cmov_mem);  // XXX
7043 %}
7044 
7045 instruct cmovL_regU(cmpOpU cop, rFlagsRegU cr, rRegL dst, rRegL src)
7046 %{
7047   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
7048 
7049   ins_cost(200); // XXX
7050   format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
7051   opcode(0x0F, 0x40);
7052   ins_encode(REX_reg_reg_wide(dst, src), enc_cmov(cop), reg_reg(dst, src));
7053   ins_pipe(pipe_cmov_reg); // XXX
7054 %}
7055 
7056 instruct cmovL_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
7057   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
7058   ins_cost(200);
7059   expand %{
7060     cmovL_regU(cop, cr, dst, src);
7061   %}
7062 %}
7063 
7064 instruct cmovL_memU(cmpOpU cop, rFlagsRegU cr, rRegL dst, memory src)
7065 %{
7066   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
7067 
7068   ins_cost(200); // XXX
7069   format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
7070   opcode(0x0F, 0x40);
7071   ins_encode(REX_reg_mem_wide(dst, src), enc_cmov(cop), reg_mem(dst, src));
7072   ins_pipe(pipe_cmov_mem); // XXX
7073 %}
7074 
7075 instruct cmovL_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, memory src) %{
7076   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
7077   ins_cost(200);
7078   expand %{
7079     cmovL_memU(cop, cr, dst, src);
7080   %}
7081 %}
7082 
7083 instruct cmovF_reg(cmpOp cop, rFlagsReg cr, regF dst, regF src)
7084 %{
7085   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
7086 
7087   ins_cost(200); // XXX
7088   format %{ "jn$cop    skip\t# signed cmove float\n\t"
7089             "movss     $dst, $src\n"
7090     "skip:" %}
7091   ins_encode %{
7092     Label Lskip;
7093     // Invert sense of branch from sense of CMOV
7094     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
7095     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
7096     __ bind(Lskip);
7097   %}
7098   ins_pipe(pipe_slow);
7099 %}
7100 
7101 // instruct cmovF_mem(cmpOp cop, rFlagsReg cr, regF dst, memory src)
7102 // %{
7103 //   match(Set dst (CMoveF (Binary cop cr) (Binary dst (LoadL src))));
7104 
7105 //   ins_cost(200); // XXX
7106 //   format %{ "jn$cop    skip\t# signed cmove float\n\t"
7107 //             "movss     $dst, $src\n"
7108 //     "skip:" %}
7109 //   ins_encode(enc_cmovf_mem_branch(cop, dst, src));
7110 //   ins_pipe(pipe_slow);
7111 // %}
7112 
7113 instruct cmovF_regU(cmpOpU cop, rFlagsRegU cr, regF dst, regF src)
7114 %{
7115   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
7116 
7117   ins_cost(200); // XXX
7118   format %{ "jn$cop    skip\t# unsigned cmove float\n\t"
7119             "movss     $dst, $src\n"
7120     "skip:" %}
7121   ins_encode %{
7122     Label Lskip;
7123     // Invert sense of branch from sense of CMOV
7124     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
7125     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
7126     __ bind(Lskip);
7127   %}
7128   ins_pipe(pipe_slow);
7129 %}
7130 
7131 instruct cmovF_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regF dst, regF src) %{
7132   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
7133   ins_cost(200);
7134   expand %{
7135     cmovF_regU(cop, cr, dst, src);
7136   %}
7137 %}
7138 
7139 instruct cmovD_reg(cmpOp cop, rFlagsReg cr, regD dst, regD src)
7140 %{
7141   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
7142 
7143   ins_cost(200); // XXX
7144   format %{ "jn$cop    skip\t# signed cmove double\n\t"
7145             "movsd     $dst, $src\n"
7146     "skip:" %}
7147   ins_encode %{
7148     Label Lskip;
7149     // Invert sense of branch from sense of CMOV
7150     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
7151     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
7152     __ bind(Lskip);
7153   %}
7154   ins_pipe(pipe_slow);
7155 %}
7156 
7157 instruct cmovD_regU(cmpOpU cop, rFlagsRegU cr, regD dst, regD src)
7158 %{
7159   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
7160 
7161   ins_cost(200); // XXX
7162   format %{ "jn$cop    skip\t# unsigned cmove double\n\t"
7163             "movsd     $dst, $src\n"
7164     "skip:" %}
7165   ins_encode %{
7166     Label Lskip;
7167     // Invert sense of branch from sense of CMOV
7168     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
7169     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
7170     __ bind(Lskip);
7171   %}
7172   ins_pipe(pipe_slow);
7173 %}
7174 
7175 instruct cmovD_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regD dst, regD src) %{
7176   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
7177   ins_cost(200);
7178   expand %{
7179     cmovD_regU(cop, cr, dst, src);
7180   %}
7181 %}
7182 
7183 //----------Arithmetic Instructions--------------------------------------------
7184 //----------Addition Instructions----------------------------------------------
7185 
7186 instruct addI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
7187 %{
7188   match(Set dst (AddI dst src));
7189   effect(KILL cr);
7190 
7191   format %{ "addl    $dst, $src\t# int" %}
7192   opcode(0x03);
7193   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
7194   ins_pipe(ialu_reg_reg);
7195 %}
7196 
7197 instruct addI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
7198 %{
7199   match(Set dst (AddI dst src));
7200   effect(KILL cr);
7201 
7202   format %{ "addl    $dst, $src\t# int" %}
7203   opcode(0x81, 0x00); /* /0 id */
7204   ins_encode(OpcSErm(dst, src), Con8or32(src));
7205   ins_pipe( ialu_reg );
7206 %}
7207 
7208 instruct addI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
7209 %{
7210   match(Set dst (AddI dst (LoadI src)));
7211   effect(KILL cr);
7212 
7213   ins_cost(125); // XXX
7214   format %{ "addl    $dst, $src\t# int" %}
7215   opcode(0x03);
7216   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
7217   ins_pipe(ialu_reg_mem);
7218 %}
7219 
7220 instruct addI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
7221 %{
7222   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
7223   effect(KILL cr);
7224 
7225   ins_cost(150); // XXX
7226   format %{ "addl    $dst, $src\t# int" %}
7227   opcode(0x01); /* Opcode 01 /r */
7228   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
7229   ins_pipe(ialu_mem_reg);
7230 %}
7231 
7232 instruct addI_mem_imm(memory dst, immI src, rFlagsReg cr)
7233 %{
7234   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
7235   effect(KILL cr);
7236 
7237   ins_cost(125); // XXX
7238   format %{ "addl    $dst, $src\t# int" %}
7239   opcode(0x81); /* Opcode 81 /0 id */
7240   ins_encode(REX_mem(dst), OpcSE(src), RM_opc_mem(0x00, dst), Con8or32(src));
7241   ins_pipe(ialu_mem_imm);
7242 %}
7243 
7244 instruct incI_rReg(rRegI dst, immI1 src, rFlagsReg cr)
7245 %{
7246   predicate(UseIncDec);
7247   match(Set dst (AddI dst src));
7248   effect(KILL cr);
7249 
7250   format %{ "incl    $dst\t# int" %}
7251   opcode(0xFF, 0x00); // FF /0
7252   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
7253   ins_pipe(ialu_reg);
7254 %}
7255 
7256 instruct incI_mem(memory dst, immI1 src, rFlagsReg cr)
7257 %{
7258   predicate(UseIncDec);
7259   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
7260   effect(KILL cr);
7261 
7262   ins_cost(125); // XXX
7263   format %{ "incl    $dst\t# int" %}
7264   opcode(0xFF); /* Opcode FF /0 */
7265   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(0x00, dst));
7266   ins_pipe(ialu_mem_imm);
7267 %}
7268 
7269 // XXX why does that use AddI
7270 instruct decI_rReg(rRegI dst, immI_M1 src, rFlagsReg cr)
7271 %{
7272   predicate(UseIncDec);
7273   match(Set dst (AddI dst src));
7274   effect(KILL cr);
7275 
7276   format %{ "decl    $dst\t# int" %}
7277   opcode(0xFF, 0x01); // FF /1
7278   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
7279   ins_pipe(ialu_reg);
7280 %}
7281 
7282 // XXX why does that use AddI
7283 instruct decI_mem(memory dst, immI_M1 src, rFlagsReg cr)
7284 %{
7285   predicate(UseIncDec);
7286   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
7287   effect(KILL cr);
7288 
7289   ins_cost(125); // XXX
7290   format %{ "decl    $dst\t# int" %}
7291   opcode(0xFF); /* Opcode FF /1 */
7292   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(0x01, dst));
7293   ins_pipe(ialu_mem_imm);
7294 %}
7295 
7296 instruct leaI_rReg_immI(rRegI dst, rRegI src0, immI src1)
7297 %{
7298   match(Set dst (AddI src0 src1));
7299 
7300   ins_cost(110);
7301   format %{ "addr32 leal $dst, [$src0 + $src1]\t# int" %}
7302   opcode(0x8D); /* 0x8D /r */
7303   ins_encode(Opcode(0x67), REX_reg_reg(dst, src0), OpcP, reg_lea(dst, src0, src1)); // XXX
7304   ins_pipe(ialu_reg_reg);
7305 %}
7306 
7307 instruct addL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
7308 %{
7309   match(Set dst (AddL dst src));
7310   effect(KILL cr);
7311 
7312   format %{ "addq    $dst, $src\t# long" %}
7313   opcode(0x03);
7314   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
7315   ins_pipe(ialu_reg_reg);
7316 %}
7317 
7318 instruct addL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
7319 %{
7320   match(Set dst (AddL dst src));
7321   effect(KILL cr);
7322 
7323   format %{ "addq    $dst, $src\t# long" %}
7324   opcode(0x81, 0x00); /* /0 id */
7325   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
7326   ins_pipe( ialu_reg );
7327 %}
7328 
7329 instruct addL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
7330 %{
7331   match(Set dst (AddL dst (LoadL src)));
7332   effect(KILL cr);
7333 
7334   ins_cost(125); // XXX
7335   format %{ "addq    $dst, $src\t# long" %}
7336   opcode(0x03);
7337   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
7338   ins_pipe(ialu_reg_mem);
7339 %}
7340 
7341 instruct addL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
7342 %{
7343   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
7344   effect(KILL cr);
7345 
7346   ins_cost(150); // XXX
7347   format %{ "addq    $dst, $src\t# long" %}
7348   opcode(0x01); /* Opcode 01 /r */
7349   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
7350   ins_pipe(ialu_mem_reg);
7351 %}
7352 
7353 instruct addL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
7354 %{
7355   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
7356   effect(KILL cr);
7357 
7358   ins_cost(125); // XXX
7359   format %{ "addq    $dst, $src\t# long" %}
7360   opcode(0x81); /* Opcode 81 /0 id */
7361   ins_encode(REX_mem_wide(dst),
7362              OpcSE(src), RM_opc_mem(0x00, dst), Con8or32(src));
7363   ins_pipe(ialu_mem_imm);
7364 %}
7365 
7366 instruct incL_rReg(rRegI dst, immL1 src, rFlagsReg cr)
7367 %{
7368   predicate(UseIncDec);
7369   match(Set dst (AddL dst src));
7370   effect(KILL cr);
7371 
7372   format %{ "incq    $dst\t# long" %}
7373   opcode(0xFF, 0x00); // FF /0
7374   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
7375   ins_pipe(ialu_reg);
7376 %}
7377 
7378 instruct incL_mem(memory dst, immL1 src, rFlagsReg cr)
7379 %{
7380   predicate(UseIncDec);
7381   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
7382   effect(KILL cr);
7383 
7384   ins_cost(125); // XXX
7385   format %{ "incq    $dst\t# long" %}
7386   opcode(0xFF); /* Opcode FF /0 */
7387   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(0x00, dst));
7388   ins_pipe(ialu_mem_imm);
7389 %}
7390 
7391 // XXX why does that use AddL
7392 instruct decL_rReg(rRegL dst, immL_M1 src, rFlagsReg cr)
7393 %{
7394   predicate(UseIncDec);
7395   match(Set dst (AddL dst src));
7396   effect(KILL cr);
7397 
7398   format %{ "decq    $dst\t# long" %}
7399   opcode(0xFF, 0x01); // FF /1
7400   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
7401   ins_pipe(ialu_reg);
7402 %}
7403 
7404 // XXX why does that use AddL
7405 instruct decL_mem(memory dst, immL_M1 src, rFlagsReg cr)
7406 %{
7407   predicate(UseIncDec);
7408   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
7409   effect(KILL cr);
7410 
7411   ins_cost(125); // XXX
7412   format %{ "decq    $dst\t# long" %}
7413   opcode(0xFF); /* Opcode FF /1 */
7414   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(0x01, dst));
7415   ins_pipe(ialu_mem_imm);
7416 %}
7417 
7418 instruct leaL_rReg_immL(rRegL dst, rRegL src0, immL32 src1)
7419 %{
7420   match(Set dst (AddL src0 src1));
7421 
7422   ins_cost(110);
7423   format %{ "leaq    $dst, [$src0 + $src1]\t# long" %}
7424   opcode(0x8D); /* 0x8D /r */
7425   ins_encode(REX_reg_reg_wide(dst, src0), OpcP, reg_lea(dst, src0, src1)); // XXX
7426   ins_pipe(ialu_reg_reg);
7427 %}
7428 
7429 instruct addP_rReg(rRegP dst, rRegL src, rFlagsReg cr)
7430 %{
7431   match(Set dst (AddP dst src));
7432   effect(KILL cr);
7433 
7434   format %{ "addq    $dst, $src\t# ptr" %}
7435   opcode(0x03);
7436   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
7437   ins_pipe(ialu_reg_reg);
7438 %}
7439 
7440 instruct addP_rReg_imm(rRegP dst, immL32 src, rFlagsReg cr)
7441 %{
7442   match(Set dst (AddP dst src));
7443   effect(KILL cr);
7444 
7445   format %{ "addq    $dst, $src\t# ptr" %}
7446   opcode(0x81, 0x00); /* /0 id */
7447   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
7448   ins_pipe( ialu_reg );
7449 %}
7450 
7451 // XXX addP mem ops ????
7452 
7453 instruct leaP_rReg_imm(rRegP dst, rRegP src0, immL32 src1)
7454 %{
7455   match(Set dst (AddP src0 src1));
7456 
7457   ins_cost(110);
7458   format %{ "leaq    $dst, [$src0 + $src1]\t# ptr" %}
7459   opcode(0x8D); /* 0x8D /r */
7460   ins_encode(REX_reg_reg_wide(dst, src0), OpcP, reg_lea(dst, src0, src1));// XXX
7461   ins_pipe(ialu_reg_reg);
7462 %}
7463 
7464 instruct checkCastPP(rRegP dst)
7465 %{
7466   match(Set dst (CheckCastPP dst));
7467 
7468   size(0);
7469   format %{ "# checkcastPP of $dst" %}
7470   ins_encode(/* empty encoding */);
7471   ins_pipe(empty);
7472 %}
7473 
7474 instruct castPP(rRegP dst)
7475 %{
7476   match(Set dst (CastPP dst));
7477 
7478   size(0);
7479   format %{ "# castPP of $dst" %}
7480   ins_encode(/* empty encoding */);
7481   ins_pipe(empty);
7482 %}
7483 
7484 instruct castII(rRegI dst)
7485 %{
7486   match(Set dst (CastII dst));
7487 
7488   size(0);
7489   format %{ "# castII of $dst" %}
7490   ins_encode(/* empty encoding */);
7491   ins_cost(0);
7492   ins_pipe(empty);
7493 %}
7494 
7495 // LoadP-locked same as a regular LoadP when used with compare-swap
7496 instruct loadPLocked(rRegP dst, memory mem)
7497 %{
7498   match(Set dst (LoadPLocked mem));
7499 
7500   ins_cost(125); // XXX
7501   format %{ "movq    $dst, $mem\t# ptr locked" %}
7502   opcode(0x8B);
7503   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
7504   ins_pipe(ialu_reg_mem); // XXX
7505 %}
7506 
7507 // Conditional-store of the updated heap-top.
7508 // Used during allocation of the shared heap.
7509 // Sets flags (EQ) on success.  Implemented with a CMPXCHG on Intel.
7510 
7511 instruct storePConditional(memory heap_top_ptr,
7512                            rax_RegP oldval, rRegP newval,
7513                            rFlagsReg cr)
7514 %{
7515   match(Set cr (StorePConditional heap_top_ptr (Binary oldval newval)));
7516 
7517   format %{ "cmpxchgq $heap_top_ptr, $newval\t# (ptr) "
7518             "If rax == $heap_top_ptr then store $newval into $heap_top_ptr" %}
7519   opcode(0x0F, 0xB1);
7520   ins_encode(lock_prefix,
7521              REX_reg_mem_wide(newval, heap_top_ptr),
7522              OpcP, OpcS,
7523              reg_mem(newval, heap_top_ptr));
7524   ins_pipe(pipe_cmpxchg);
7525 %}
7526 
7527 // Conditional-store of an int value.
7528 // ZF flag is set on success, reset otherwise.  Implemented with a CMPXCHG.
7529 instruct storeIConditional(memory mem, rax_RegI oldval, rRegI newval, rFlagsReg cr)
7530 %{
7531   match(Set cr (StoreIConditional mem (Binary oldval newval)));
7532   effect(KILL oldval);
7533 
7534   format %{ "cmpxchgl $mem, $newval\t# If rax == $mem then store $newval into $mem" %}
7535   opcode(0x0F, 0xB1);
7536   ins_encode(lock_prefix,
7537              REX_reg_mem(newval, mem),
7538              OpcP, OpcS,
7539              reg_mem(newval, mem));
7540   ins_pipe(pipe_cmpxchg);
7541 %}
7542 
7543 // Conditional-store of a long value.
7544 // ZF flag is set on success, reset otherwise.  Implemented with a CMPXCHG.
7545 instruct storeLConditional(memory mem, rax_RegL oldval, rRegL newval, rFlagsReg cr)
7546 %{
7547   match(Set cr (StoreLConditional mem (Binary oldval newval)));
7548   effect(KILL oldval);
7549 
7550   format %{ "cmpxchgq $mem, $newval\t# If rax == $mem then store $newval into $mem" %}
7551   opcode(0x0F, 0xB1);
7552   ins_encode(lock_prefix,
7553              REX_reg_mem_wide(newval, mem),
7554              OpcP, OpcS,
7555              reg_mem(newval, mem));
7556   ins_pipe(pipe_cmpxchg);
7557 %}
7558 
7559 
7560 // XXX No flag versions for CompareAndSwap{P,I,L} because matcher can't match them
7561 instruct compareAndSwapP(rRegI res,
7562                          memory mem_ptr,
7563                          rax_RegP oldval, rRegP newval,
7564                          rFlagsReg cr)
7565 %{
7566   predicate(VM_Version::supports_cx8());
7567   match(Set res (CompareAndSwapP mem_ptr (Binary oldval newval)));
7568   match(Set res (WeakCompareAndSwapP mem_ptr (Binary oldval newval)));
7569   effect(KILL cr, KILL oldval);
7570 
7571   format %{ "cmpxchgq $mem_ptr,$newval\t# "
7572             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7573             "sete    $res\n\t"
7574             "movzbl  $res, $res" %}
7575   opcode(0x0F, 0xB1);
7576   ins_encode(lock_prefix,
7577              REX_reg_mem_wide(newval, mem_ptr),
7578              OpcP, OpcS,
7579              reg_mem(newval, mem_ptr),
7580              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7581              REX_reg_breg(res, res), // movzbl
7582              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7583   ins_pipe( pipe_cmpxchg );
7584 %}
7585 
7586 instruct compareAndSwapP_shenandoah(rRegI res,
7587                                     memory mem_ptr,
7588                                     rRegP tmp1, rRegP tmp2,
7589                                     rax_RegP oldval, rRegP newval,
7590                                     rFlagsReg cr)
7591 %{
7592   predicate(VM_Version::supports_cx8());
7593   match(Set res (ShenandoahCompareAndSwapP mem_ptr (Binary oldval newval)));
7594   match(Set res (ShenandoahWeakCompareAndSwapP mem_ptr (Binary oldval newval)));
7595   effect(TEMP tmp1, TEMP tmp2, KILL cr, KILL oldval);
7596 
7597   format %{ "shenandoah_cas_oop $mem_ptr,$newval" %}
7598 
7599   ins_encode %{
7600 #if INCLUDE_SHENANDOAHGC
7601     ShenandoahBarrierSet::assembler()->cmpxchg_oop(&_masm,
7602                                                    $res$$Register, $mem_ptr$$Address, $oldval$$Register, $newval$$Register,
7603                                                    false, // swap
7604                                                    $tmp1$$Register, $tmp2$$Register
7605                                                    );
7606 #else
7607     ShouldNotReachHere();
7608 #endif
7609   %}
7610   ins_pipe( pipe_cmpxchg );
7611 %}
7612 
7613 instruct compareAndSwapL(rRegI res,
7614                          memory mem_ptr,
7615                          rax_RegL oldval, rRegL newval,
7616                          rFlagsReg cr)
7617 %{
7618   predicate(VM_Version::supports_cx8());
7619   match(Set res (CompareAndSwapL mem_ptr (Binary oldval newval)));
7620   match(Set res (WeakCompareAndSwapL mem_ptr (Binary oldval newval)));
7621   effect(KILL cr, KILL oldval);
7622 
7623   format %{ "cmpxchgq $mem_ptr,$newval\t# "
7624             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7625             "sete    $res\n\t"
7626             "movzbl  $res, $res" %}
7627   opcode(0x0F, 0xB1);
7628   ins_encode(lock_prefix,
7629              REX_reg_mem_wide(newval, mem_ptr),
7630              OpcP, OpcS,
7631              reg_mem(newval, mem_ptr),
7632              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7633              REX_reg_breg(res, res), // movzbl
7634              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7635   ins_pipe( pipe_cmpxchg );
7636 %}
7637 
7638 instruct compareAndSwapI(rRegI res,
7639                          memory mem_ptr,
7640                          rax_RegI oldval, rRegI newval,
7641                          rFlagsReg cr)
7642 %{
7643   match(Set res (CompareAndSwapI mem_ptr (Binary oldval newval)));
7644   match(Set res (WeakCompareAndSwapI mem_ptr (Binary oldval newval)));
7645   effect(KILL cr, KILL oldval);
7646 
7647   format %{ "cmpxchgl $mem_ptr,$newval\t# "
7648             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7649             "sete    $res\n\t"
7650             "movzbl  $res, $res" %}
7651   opcode(0x0F, 0xB1);
7652   ins_encode(lock_prefix,
7653              REX_reg_mem(newval, mem_ptr),
7654              OpcP, OpcS,
7655              reg_mem(newval, mem_ptr),
7656              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7657              REX_reg_breg(res, res), // movzbl
7658              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7659   ins_pipe( pipe_cmpxchg );
7660 %}
7661 
7662 instruct compareAndSwapB(rRegI res,
7663                          memory mem_ptr,
7664                          rax_RegI oldval, rRegI newval,
7665                          rFlagsReg cr)
7666 %{
7667   match(Set res (CompareAndSwapB mem_ptr (Binary oldval newval)));
7668   match(Set res (WeakCompareAndSwapB mem_ptr (Binary oldval newval)));
7669   effect(KILL cr, KILL oldval);
7670 
7671   format %{ "cmpxchgb $mem_ptr,$newval\t# "
7672             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7673             "sete    $res\n\t"
7674             "movzbl  $res, $res" %}
7675   opcode(0x0F, 0xB0);
7676   ins_encode(lock_prefix,
7677              REX_breg_mem(newval, mem_ptr),
7678              OpcP, OpcS,
7679              reg_mem(newval, mem_ptr),
7680              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7681              REX_reg_breg(res, res), // movzbl
7682              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7683   ins_pipe( pipe_cmpxchg );
7684 %}
7685 
7686 instruct compareAndSwapS(rRegI res,
7687                          memory mem_ptr,
7688                          rax_RegI oldval, rRegI newval,
7689                          rFlagsReg cr)
7690 %{
7691   match(Set res (CompareAndSwapS mem_ptr (Binary oldval newval)));
7692   match(Set res (WeakCompareAndSwapS mem_ptr (Binary oldval newval)));
7693   effect(KILL cr, KILL oldval);
7694 
7695   format %{ "cmpxchgw $mem_ptr,$newval\t# "
7696             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7697             "sete    $res\n\t"
7698             "movzbl  $res, $res" %}
7699   opcode(0x0F, 0xB1);
7700   ins_encode(lock_prefix,
7701              SizePrefix,
7702              REX_reg_mem(newval, mem_ptr),
7703              OpcP, OpcS,
7704              reg_mem(newval, mem_ptr),
7705              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7706              REX_reg_breg(res, res), // movzbl
7707              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7708   ins_pipe( pipe_cmpxchg );
7709 %}
7710 
7711 instruct compareAndSwapN(rRegI res,
7712                           memory mem_ptr,
7713                           rax_RegN oldval, rRegN newval,
7714                           rFlagsReg cr) %{
7715   match(Set res (CompareAndSwapN mem_ptr (Binary oldval newval)));
7716   match(Set res (WeakCompareAndSwapN mem_ptr (Binary oldval newval)));
7717   effect(KILL cr, KILL oldval);
7718 
7719   format %{ "cmpxchgl $mem_ptr,$newval\t# "
7720             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7721             "sete    $res\n\t"
7722             "movzbl  $res, $res" %}
7723   opcode(0x0F, 0xB1);
7724   ins_encode(lock_prefix,
7725              REX_reg_mem(newval, mem_ptr),
7726              OpcP, OpcS,
7727              reg_mem(newval, mem_ptr),
7728              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7729              REX_reg_breg(res, res), // movzbl
7730              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7731   ins_pipe( pipe_cmpxchg );
7732 %}
7733 
7734 instruct compareAndSwapN_shenandoah(rRegI res,
7735                                     memory mem_ptr,
7736                                     rRegP tmp1, rRegP tmp2,
7737                                     rax_RegN oldval, rRegN newval,
7738                                     rFlagsReg cr) %{
7739   match(Set res (ShenandoahCompareAndSwapN mem_ptr (Binary oldval newval)));
7740   match(Set res (ShenandoahWeakCompareAndSwapN mem_ptr (Binary oldval newval)));
7741   effect(TEMP tmp1, TEMP tmp2, KILL cr, KILL oldval);
7742 
7743   format %{ "shenandoah_cas_oop $mem_ptr,$newval" %}
7744 
7745   ins_encode %{
7746 #if INCLUDE_SHENANDOAHGC
7747     ShenandoahBarrierSet::assembler()->cmpxchg_oop(&_masm,
7748                                                    $res$$Register, $mem_ptr$$Address, $oldval$$Register, $newval$$Register,
7749                                                    false, // swap
7750                                                    $tmp1$$Register, $tmp2$$Register
7751                                                    );
7752 #else
7753     ShouldNotReachHere();
7754 #endif
7755   %}
7756   ins_pipe( pipe_cmpxchg );
7757 %}
7758 
7759 instruct compareAndExchangeB(
7760                          memory mem_ptr,
7761                          rax_RegI oldval, rRegI newval,
7762                          rFlagsReg cr)
7763 %{
7764   match(Set oldval (CompareAndExchangeB mem_ptr (Binary oldval newval)));
7765   effect(KILL cr);
7766 
7767   format %{ "cmpxchgb $mem_ptr,$newval\t# "
7768             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
7769   opcode(0x0F, 0xB0);
7770   ins_encode(lock_prefix,
7771              REX_breg_mem(newval, mem_ptr),
7772              OpcP, OpcS,
7773              reg_mem(newval, mem_ptr) // lock cmpxchg
7774              );
7775   ins_pipe( pipe_cmpxchg );
7776 %}
7777 
7778 instruct compareAndExchangeS(
7779                          memory mem_ptr,
7780                          rax_RegI oldval, rRegI newval,
7781                          rFlagsReg cr)
7782 %{
7783   match(Set oldval (CompareAndExchangeS mem_ptr (Binary oldval newval)));
7784   effect(KILL cr);
7785 
7786   format %{ "cmpxchgw $mem_ptr,$newval\t# "
7787             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
7788   opcode(0x0F, 0xB1);
7789   ins_encode(lock_prefix,
7790              SizePrefix,
7791              REX_reg_mem(newval, mem_ptr),
7792              OpcP, OpcS,
7793              reg_mem(newval, mem_ptr) // lock cmpxchg
7794              );
7795   ins_pipe( pipe_cmpxchg );
7796 %}
7797 
7798 instruct compareAndExchangeI(
7799                          memory mem_ptr,
7800                          rax_RegI oldval, rRegI newval,
7801                          rFlagsReg cr)
7802 %{
7803   match(Set oldval (CompareAndExchangeI mem_ptr (Binary oldval newval)));
7804   effect(KILL cr);
7805 
7806   format %{ "cmpxchgl $mem_ptr,$newval\t# "
7807             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
7808   opcode(0x0F, 0xB1);
7809   ins_encode(lock_prefix,
7810              REX_reg_mem(newval, mem_ptr),
7811              OpcP, OpcS,
7812              reg_mem(newval, mem_ptr) // lock cmpxchg
7813              );
7814   ins_pipe( pipe_cmpxchg );
7815 %}
7816 
7817 instruct compareAndExchangeL(
7818                          memory mem_ptr,
7819                          rax_RegL oldval, rRegL newval,
7820                          rFlagsReg cr)
7821 %{
7822   predicate(VM_Version::supports_cx8());
7823   match(Set oldval (CompareAndExchangeL mem_ptr (Binary oldval newval)));
7824   effect(KILL cr);
7825 
7826   format %{ "cmpxchgq $mem_ptr,$newval\t# "
7827             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
7828   opcode(0x0F, 0xB1);
7829   ins_encode(lock_prefix,
7830              REX_reg_mem_wide(newval, mem_ptr),
7831              OpcP, OpcS,
7832              reg_mem(newval, mem_ptr)  // lock cmpxchg
7833             );
7834   ins_pipe( pipe_cmpxchg );
7835 %}
7836 
7837 instruct compareAndExchangeN(
7838                           memory mem_ptr,
7839                           rax_RegN oldval, rRegN newval,
7840                           rFlagsReg cr) %{
7841   match(Set oldval (CompareAndExchangeN mem_ptr (Binary oldval newval)));
7842   effect(KILL cr);
7843 
7844   format %{ "cmpxchgl $mem_ptr,$newval\t# "
7845             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %}
7846   opcode(0x0F, 0xB1);
7847   ins_encode(lock_prefix,
7848              REX_reg_mem(newval, mem_ptr),
7849              OpcP, OpcS,
7850              reg_mem(newval, mem_ptr)  // lock cmpxchg
7851           );
7852   ins_pipe( pipe_cmpxchg );
7853 %}
7854 
7855 instruct compareAndExchangeN_shenandoah(memory mem_ptr,
7856                                         rax_RegN oldval, rRegN newval,
7857                                         rRegP tmp1, rRegP tmp2,
7858                                         rFlagsReg cr) %{
7859   match(Set oldval (ShenandoahCompareAndExchangeN mem_ptr (Binary oldval newval)));
7860   effect(TEMP tmp1, TEMP tmp2, KILL cr);
7861 
7862   format %{ "shenandoah_cas_oop $mem_ptr,$newval" %}
7863 
7864   ins_encode %{
7865 #if INCLUDE_SHENANDOAHGC
7866     ShenandoahBarrierSet::assembler()->cmpxchg_oop(&_masm,
7867                                                    NULL, $mem_ptr$$Address, $oldval$$Register, $newval$$Register,
7868                                                    true, // exchange
7869                                                    $tmp1$$Register, $tmp2$$Register
7870                                                    );
7871 #else
7872     ShouldNotReachHere();
7873 #endif
7874   %}
7875   ins_pipe( pipe_cmpxchg );
7876 %}
7877 
7878 instruct compareAndExchangeP(
7879                          memory mem_ptr,
7880                          rax_RegP oldval, rRegP newval,
7881                          rFlagsReg cr)
7882 %{
7883   predicate(VM_Version::supports_cx8());
7884   match(Set oldval (CompareAndExchangeP mem_ptr (Binary oldval newval)));
7885   effect(KILL cr);
7886 
7887   format %{ "cmpxchgq $mem_ptr,$newval\t# "
7888             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %}
7889   opcode(0x0F, 0xB1);
7890   ins_encode(lock_prefix,
7891              REX_reg_mem_wide(newval, mem_ptr),
7892              OpcP, OpcS,
7893              reg_mem(newval, mem_ptr)  // lock cmpxchg
7894           );
7895   ins_pipe( pipe_cmpxchg );
7896 %}
7897 
7898 instruct compareAndExchangeP_shenandoah(memory mem_ptr,
7899                                         rax_RegP oldval, rRegP newval,
7900                                         rRegP tmp1, rRegP tmp2,
7901                                         rFlagsReg cr)
7902 %{
7903   predicate(VM_Version::supports_cx8());
7904   match(Set oldval (ShenandoahCompareAndExchangeP mem_ptr (Binary oldval newval)));
7905   effect(KILL cr, TEMP tmp1, TEMP tmp2);
7906   ins_cost(1000);
7907 
7908   format %{ "shenandoah_cas_oop $mem_ptr,$newval" %}
7909 
7910   ins_encode %{
7911 #if INCLUDE_SHENANDOAHGC
7912     ShenandoahBarrierSet::assembler()->cmpxchg_oop(&_masm,
7913                                                    NULL, $mem_ptr$$Address, $oldval$$Register, $newval$$Register,
7914                                                    true,  // exchange
7915                                                    $tmp1$$Register, $tmp2$$Register
7916                                                    );
7917 #else
7918     ShouldNotReachHere();
7919 #endif
7920   %}
7921   ins_pipe( pipe_cmpxchg );
7922 %}
7923 
7924 instruct xaddB_no_res( memory mem, Universe dummy, immI add, rFlagsReg cr) %{
7925   predicate(n->as_LoadStore()->result_not_used());
7926   match(Set dummy (GetAndAddB mem add));
7927   effect(KILL cr);
7928   format %{ "ADDB  [$mem],$add" %}
7929   ins_encode %{
7930     if (os::is_MP()) { __ lock(); }
7931     __ addb($mem$$Address, $add$$constant);
7932   %}
7933   ins_pipe( pipe_cmpxchg );
7934 %}
7935 
7936 instruct xaddB( memory mem, rRegI newval, rFlagsReg cr) %{
7937   match(Set newval (GetAndAddB mem newval));
7938   effect(KILL cr);
7939   format %{ "XADDB  [$mem],$newval" %}
7940   ins_encode %{
7941     if (os::is_MP()) { __ lock(); }
7942     __ xaddb($mem$$Address, $newval$$Register);
7943   %}
7944   ins_pipe( pipe_cmpxchg );
7945 %}
7946 
7947 instruct xaddS_no_res( memory mem, Universe dummy, immI add, rFlagsReg cr) %{
7948   predicate(n->as_LoadStore()->result_not_used());
7949   match(Set dummy (GetAndAddS mem add));
7950   effect(KILL cr);
7951   format %{ "ADDW  [$mem],$add" %}
7952   ins_encode %{
7953     if (os::is_MP()) { __ lock(); }
7954     __ addw($mem$$Address, $add$$constant);
7955   %}
7956   ins_pipe( pipe_cmpxchg );
7957 %}
7958 
7959 instruct xaddS( memory mem, rRegI newval, rFlagsReg cr) %{
7960   match(Set newval (GetAndAddS mem newval));
7961   effect(KILL cr);
7962   format %{ "XADDW  [$mem],$newval" %}
7963   ins_encode %{
7964     if (os::is_MP()) { __ lock(); }
7965     __ xaddw($mem$$Address, $newval$$Register);
7966   %}
7967   ins_pipe( pipe_cmpxchg );
7968 %}
7969 
7970 instruct xaddI_no_res( memory mem, Universe dummy, immI add, rFlagsReg cr) %{
7971   predicate(n->as_LoadStore()->result_not_used());
7972   match(Set dummy (GetAndAddI mem add));
7973   effect(KILL cr);
7974   format %{ "ADDL  [$mem],$add" %}
7975   ins_encode %{
7976     if (os::is_MP()) { __ lock(); }
7977     __ addl($mem$$Address, $add$$constant);
7978   %}
7979   ins_pipe( pipe_cmpxchg );
7980 %}
7981 
7982 instruct xaddI( memory mem, rRegI newval, rFlagsReg cr) %{
7983   match(Set newval (GetAndAddI mem newval));
7984   effect(KILL cr);
7985   format %{ "XADDL  [$mem],$newval" %}
7986   ins_encode %{
7987     if (os::is_MP()) { __ lock(); }
7988     __ xaddl($mem$$Address, $newval$$Register);
7989   %}
7990   ins_pipe( pipe_cmpxchg );
7991 %}
7992 
7993 instruct xaddL_no_res( memory mem, Universe dummy, immL32 add, rFlagsReg cr) %{
7994   predicate(n->as_LoadStore()->result_not_used());
7995   match(Set dummy (GetAndAddL mem add));
7996   effect(KILL cr);
7997   format %{ "ADDQ  [$mem],$add" %}
7998   ins_encode %{
7999     if (os::is_MP()) { __ lock(); }
8000     __ addq($mem$$Address, $add$$constant);
8001   %}
8002   ins_pipe( pipe_cmpxchg );
8003 %}
8004 
8005 instruct xaddL( memory mem, rRegL newval, rFlagsReg cr) %{
8006   match(Set newval (GetAndAddL mem newval));
8007   effect(KILL cr);
8008   format %{ "XADDQ  [$mem],$newval" %}
8009   ins_encode %{
8010     if (os::is_MP()) { __ lock(); }
8011     __ xaddq($mem$$Address, $newval$$Register);
8012   %}
8013   ins_pipe( pipe_cmpxchg );
8014 %}
8015 
8016 instruct xchgB( memory mem, rRegI newval) %{
8017   match(Set newval (GetAndSetB mem newval));
8018   format %{ "XCHGB  $newval,[$mem]" %}
8019   ins_encode %{
8020     __ xchgb($newval$$Register, $mem$$Address);
8021   %}
8022   ins_pipe( pipe_cmpxchg );
8023 %}
8024 
8025 instruct xchgS( memory mem, rRegI newval) %{
8026   match(Set newval (GetAndSetS mem newval));
8027   format %{ "XCHGW  $newval,[$mem]" %}
8028   ins_encode %{
8029     __ xchgw($newval$$Register, $mem$$Address);
8030   %}
8031   ins_pipe( pipe_cmpxchg );
8032 %}
8033 
8034 instruct xchgI( memory mem, rRegI newval) %{
8035   match(Set newval (GetAndSetI mem newval));
8036   format %{ "XCHGL  $newval,[$mem]" %}
8037   ins_encode %{
8038     __ xchgl($newval$$Register, $mem$$Address);
8039   %}
8040   ins_pipe( pipe_cmpxchg );
8041 %}
8042 
8043 instruct xchgL( memory mem, rRegL newval) %{
8044   match(Set newval (GetAndSetL mem newval));
8045   format %{ "XCHGL  $newval,[$mem]" %}
8046   ins_encode %{
8047     __ xchgq($newval$$Register, $mem$$Address);
8048   %}
8049   ins_pipe( pipe_cmpxchg );
8050 %}
8051 
8052 instruct xchgP( memory mem, rRegP newval) %{
8053   match(Set newval (GetAndSetP mem newval));
8054   format %{ "XCHGQ  $newval,[$mem]" %}
8055   ins_encode %{
8056     __ xchgq($newval$$Register, $mem$$Address);
8057   %}
8058   ins_pipe( pipe_cmpxchg );
8059 %}
8060 
8061 instruct xchgN( memory mem, rRegN newval) %{
8062   match(Set newval (GetAndSetN mem newval));
8063   format %{ "XCHGL  $newval,$mem]" %}
8064   ins_encode %{
8065     __ xchgl($newval$$Register, $mem$$Address);
8066   %}
8067   ins_pipe( pipe_cmpxchg );
8068 %}
8069 
8070 //----------Subtraction Instructions-------------------------------------------
8071 
8072 // Integer Subtraction Instructions
8073 instruct subI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
8074 %{
8075   match(Set dst (SubI dst src));
8076   effect(KILL cr);
8077 
8078   format %{ "subl    $dst, $src\t# int" %}
8079   opcode(0x2B);
8080   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
8081   ins_pipe(ialu_reg_reg);
8082 %}
8083 
8084 instruct subI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
8085 %{
8086   match(Set dst (SubI dst src));
8087   effect(KILL cr);
8088 
8089   format %{ "subl    $dst, $src\t# int" %}
8090   opcode(0x81, 0x05);  /* Opcode 81 /5 */
8091   ins_encode(OpcSErm(dst, src), Con8or32(src));
8092   ins_pipe(ialu_reg);
8093 %}
8094 
8095 instruct subI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
8096 %{
8097   match(Set dst (SubI dst (LoadI src)));
8098   effect(KILL cr);
8099 
8100   ins_cost(125);
8101   format %{ "subl    $dst, $src\t# int" %}
8102   opcode(0x2B);
8103   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
8104   ins_pipe(ialu_reg_mem);
8105 %}
8106 
8107 instruct subI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
8108 %{
8109   match(Set dst (StoreI dst (SubI (LoadI dst) src)));
8110   effect(KILL cr);
8111 
8112   ins_cost(150);
8113   format %{ "subl    $dst, $src\t# int" %}
8114   opcode(0x29); /* Opcode 29 /r */
8115   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
8116   ins_pipe(ialu_mem_reg);
8117 %}
8118 
8119 instruct subI_mem_imm(memory dst, immI src, rFlagsReg cr)
8120 %{
8121   match(Set dst (StoreI dst (SubI (LoadI dst) src)));
8122   effect(KILL cr);
8123 
8124   ins_cost(125); // XXX
8125   format %{ "subl    $dst, $src\t# int" %}
8126   opcode(0x81); /* Opcode 81 /5 id */
8127   ins_encode(REX_mem(dst), OpcSE(src), RM_opc_mem(0x05, dst), Con8or32(src));
8128   ins_pipe(ialu_mem_imm);
8129 %}
8130 
8131 instruct subL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
8132 %{
8133   match(Set dst (SubL dst src));
8134   effect(KILL cr);
8135 
8136   format %{ "subq    $dst, $src\t# long" %}
8137   opcode(0x2B);
8138   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
8139   ins_pipe(ialu_reg_reg);
8140 %}
8141 
8142 instruct subL_rReg_imm(rRegI dst, immL32 src, rFlagsReg cr)
8143 %{
8144   match(Set dst (SubL dst src));
8145   effect(KILL cr);
8146 
8147   format %{ "subq    $dst, $src\t# long" %}
8148   opcode(0x81, 0x05);  /* Opcode 81 /5 */
8149   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
8150   ins_pipe(ialu_reg);
8151 %}
8152 
8153 instruct subL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
8154 %{
8155   match(Set dst (SubL dst (LoadL src)));
8156   effect(KILL cr);
8157 
8158   ins_cost(125);
8159   format %{ "subq    $dst, $src\t# long" %}
8160   opcode(0x2B);
8161   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
8162   ins_pipe(ialu_reg_mem);
8163 %}
8164 
8165 instruct subL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
8166 %{
8167   match(Set dst (StoreL dst (SubL (LoadL dst) src)));
8168   effect(KILL cr);
8169 
8170   ins_cost(150);
8171   format %{ "subq    $dst, $src\t# long" %}
8172   opcode(0x29); /* Opcode 29 /r */
8173   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
8174   ins_pipe(ialu_mem_reg);
8175 %}
8176 
8177 instruct subL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
8178 %{
8179   match(Set dst (StoreL dst (SubL (LoadL dst) src)));
8180   effect(KILL cr);
8181 
8182   ins_cost(125); // XXX
8183   format %{ "subq    $dst, $src\t# long" %}
8184   opcode(0x81); /* Opcode 81 /5 id */
8185   ins_encode(REX_mem_wide(dst),
8186              OpcSE(src), RM_opc_mem(0x05, dst), Con8or32(src));
8187   ins_pipe(ialu_mem_imm);
8188 %}
8189 
8190 // Subtract from a pointer
8191 // XXX hmpf???
8192 instruct subP_rReg(rRegP dst, rRegI src, immI0 zero, rFlagsReg cr)
8193 %{
8194   match(Set dst (AddP dst (SubI zero src)));
8195   effect(KILL cr);
8196 
8197   format %{ "subq    $dst, $src\t# ptr - int" %}
8198   opcode(0x2B);
8199   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
8200   ins_pipe(ialu_reg_reg);
8201 %}
8202 
8203 instruct negI_rReg(rRegI dst, immI0 zero, rFlagsReg cr)
8204 %{
8205   match(Set dst (SubI zero dst));
8206   effect(KILL cr);
8207 
8208   format %{ "negl    $dst\t# int" %}
8209   opcode(0xF7, 0x03);  // Opcode F7 /3
8210   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8211   ins_pipe(ialu_reg);
8212 %}
8213 
8214 instruct negI_mem(memory dst, immI0 zero, rFlagsReg cr)
8215 %{
8216   match(Set dst (StoreI dst (SubI zero (LoadI dst))));
8217   effect(KILL cr);
8218 
8219   format %{ "negl    $dst\t# int" %}
8220   opcode(0xF7, 0x03);  // Opcode F7 /3
8221   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8222   ins_pipe(ialu_reg);
8223 %}
8224 
8225 instruct negL_rReg(rRegL dst, immL0 zero, rFlagsReg cr)
8226 %{
8227   match(Set dst (SubL zero dst));
8228   effect(KILL cr);
8229 
8230   format %{ "negq    $dst\t# long" %}
8231   opcode(0xF7, 0x03);  // Opcode F7 /3
8232   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8233   ins_pipe(ialu_reg);
8234 %}
8235 
8236 instruct negL_mem(memory dst, immL0 zero, rFlagsReg cr)
8237 %{
8238   match(Set dst (StoreL dst (SubL zero (LoadL dst))));
8239   effect(KILL cr);
8240 
8241   format %{ "negq    $dst\t# long" %}
8242   opcode(0xF7, 0x03);  // Opcode F7 /3
8243   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8244   ins_pipe(ialu_reg);
8245 %}
8246 
8247 //----------Multiplication/Division Instructions-------------------------------
8248 // Integer Multiplication Instructions
8249 // Multiply Register
8250 
8251 instruct mulI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
8252 %{
8253   match(Set dst (MulI dst src));
8254   effect(KILL cr);
8255 
8256   ins_cost(300);
8257   format %{ "imull   $dst, $src\t# int" %}
8258   opcode(0x0F, 0xAF);
8259   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
8260   ins_pipe(ialu_reg_reg_alu0);
8261 %}
8262 
8263 instruct mulI_rReg_imm(rRegI dst, rRegI src, immI imm, rFlagsReg cr)
8264 %{
8265   match(Set dst (MulI src imm));
8266   effect(KILL cr);
8267 
8268   ins_cost(300);
8269   format %{ "imull   $dst, $src, $imm\t# int" %}
8270   opcode(0x69); /* 69 /r id */
8271   ins_encode(REX_reg_reg(dst, src),
8272              OpcSE(imm), reg_reg(dst, src), Con8or32(imm));
8273   ins_pipe(ialu_reg_reg_alu0);
8274 %}
8275 
8276 instruct mulI_mem(rRegI dst, memory src, rFlagsReg cr)
8277 %{
8278   match(Set dst (MulI dst (LoadI src)));
8279   effect(KILL cr);
8280 
8281   ins_cost(350);
8282   format %{ "imull   $dst, $src\t# int" %}
8283   opcode(0x0F, 0xAF);
8284   ins_encode(REX_reg_mem(dst, src), OpcP, OpcS, reg_mem(dst, src));
8285   ins_pipe(ialu_reg_mem_alu0);
8286 %}
8287 
8288 instruct mulI_mem_imm(rRegI dst, memory src, immI imm, rFlagsReg cr)
8289 %{
8290   match(Set dst (MulI (LoadI src) imm));
8291   effect(KILL cr);
8292 
8293   ins_cost(300);
8294   format %{ "imull   $dst, $src, $imm\t# int" %}
8295   opcode(0x69); /* 69 /r id */
8296   ins_encode(REX_reg_mem(dst, src),
8297              OpcSE(imm), reg_mem(dst, src), Con8or32(imm));
8298   ins_pipe(ialu_reg_mem_alu0);
8299 %}
8300 
8301 instruct mulL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
8302 %{
8303   match(Set dst (MulL dst src));
8304   effect(KILL cr);
8305 
8306   ins_cost(300);
8307   format %{ "imulq   $dst, $src\t# long" %}
8308   opcode(0x0F, 0xAF);
8309   ins_encode(REX_reg_reg_wide(dst, src), OpcP, OpcS, reg_reg(dst, src));
8310   ins_pipe(ialu_reg_reg_alu0);
8311 %}
8312 
8313 instruct mulL_rReg_imm(rRegL dst, rRegL src, immL32 imm, rFlagsReg cr)
8314 %{
8315   match(Set dst (MulL src imm));
8316   effect(KILL cr);
8317 
8318   ins_cost(300);
8319   format %{ "imulq   $dst, $src, $imm\t# long" %}
8320   opcode(0x69); /* 69 /r id */
8321   ins_encode(REX_reg_reg_wide(dst, src),
8322              OpcSE(imm), reg_reg(dst, src), Con8or32(imm));
8323   ins_pipe(ialu_reg_reg_alu0);
8324 %}
8325 
8326 instruct mulL_mem(rRegL dst, memory src, rFlagsReg cr)
8327 %{
8328   match(Set dst (MulL dst (LoadL src)));
8329   effect(KILL cr);
8330 
8331   ins_cost(350);
8332   format %{ "imulq   $dst, $src\t# long" %}
8333   opcode(0x0F, 0xAF);
8334   ins_encode(REX_reg_mem_wide(dst, src), OpcP, OpcS, reg_mem(dst, src));
8335   ins_pipe(ialu_reg_mem_alu0);
8336 %}
8337 
8338 instruct mulL_mem_imm(rRegL dst, memory src, immL32 imm, rFlagsReg cr)
8339 %{
8340   match(Set dst (MulL (LoadL src) imm));
8341   effect(KILL cr);
8342 
8343   ins_cost(300);
8344   format %{ "imulq   $dst, $src, $imm\t# long" %}
8345   opcode(0x69); /* 69 /r id */
8346   ins_encode(REX_reg_mem_wide(dst, src),
8347              OpcSE(imm), reg_mem(dst, src), Con8or32(imm));
8348   ins_pipe(ialu_reg_mem_alu0);
8349 %}
8350 
8351 instruct mulHiL_rReg(rdx_RegL dst, no_rax_RegL src, rax_RegL rax, rFlagsReg cr)
8352 %{
8353   match(Set dst (MulHiL src rax));
8354   effect(USE_KILL rax, KILL cr);
8355 
8356   ins_cost(300);
8357   format %{ "imulq   RDX:RAX, RAX, $src\t# mulhi" %}
8358   opcode(0xF7, 0x5); /* Opcode F7 /5 */
8359   ins_encode(REX_reg_wide(src), OpcP, reg_opc(src));
8360   ins_pipe(ialu_reg_reg_alu0);
8361 %}
8362 
8363 instruct divI_rReg(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div,
8364                    rFlagsReg cr)
8365 %{
8366   match(Set rax (DivI rax div));
8367   effect(KILL rdx, KILL cr);
8368 
8369   ins_cost(30*100+10*100); // XXX
8370   format %{ "cmpl    rax, 0x80000000\t# idiv\n\t"
8371             "jne,s   normal\n\t"
8372             "xorl    rdx, rdx\n\t"
8373             "cmpl    $div, -1\n\t"
8374             "je,s    done\n"
8375     "normal: cdql\n\t"
8376             "idivl   $div\n"
8377     "done:"        %}
8378   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
8379   ins_encode(cdql_enc(div), REX_reg(div), OpcP, reg_opc(div));
8380   ins_pipe(ialu_reg_reg_alu0);
8381 %}
8382 
8383 instruct divL_rReg(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div,
8384                    rFlagsReg cr)
8385 %{
8386   match(Set rax (DivL rax div));
8387   effect(KILL rdx, KILL cr);
8388 
8389   ins_cost(30*100+10*100); // XXX
8390   format %{ "movq    rdx, 0x8000000000000000\t# ldiv\n\t"
8391             "cmpq    rax, rdx\n\t"
8392             "jne,s   normal\n\t"
8393             "xorl    rdx, rdx\n\t"
8394             "cmpq    $div, -1\n\t"
8395             "je,s    done\n"
8396     "normal: cdqq\n\t"
8397             "idivq   $div\n"
8398     "done:"        %}
8399   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
8400   ins_encode(cdqq_enc(div), REX_reg_wide(div), OpcP, reg_opc(div));
8401   ins_pipe(ialu_reg_reg_alu0);
8402 %}
8403 
8404 // Integer DIVMOD with Register, both quotient and mod results
8405 instruct divModI_rReg_divmod(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div,
8406                              rFlagsReg cr)
8407 %{
8408   match(DivModI rax div);
8409   effect(KILL cr);
8410 
8411   ins_cost(30*100+10*100); // XXX
8412   format %{ "cmpl    rax, 0x80000000\t# idiv\n\t"
8413             "jne,s   normal\n\t"
8414             "xorl    rdx, rdx\n\t"
8415             "cmpl    $div, -1\n\t"
8416             "je,s    done\n"
8417     "normal: cdql\n\t"
8418             "idivl   $div\n"
8419     "done:"        %}
8420   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
8421   ins_encode(cdql_enc(div), REX_reg(div), OpcP, reg_opc(div));
8422   ins_pipe(pipe_slow);
8423 %}
8424 
8425 // Long DIVMOD with Register, both quotient and mod results
8426 instruct divModL_rReg_divmod(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div,
8427                              rFlagsReg cr)
8428 %{
8429   match(DivModL rax div);
8430   effect(KILL cr);
8431 
8432   ins_cost(30*100+10*100); // XXX
8433   format %{ "movq    rdx, 0x8000000000000000\t# ldiv\n\t"
8434             "cmpq    rax, rdx\n\t"
8435             "jne,s   normal\n\t"
8436             "xorl    rdx, rdx\n\t"
8437             "cmpq    $div, -1\n\t"
8438             "je,s    done\n"
8439     "normal: cdqq\n\t"
8440             "idivq   $div\n"
8441     "done:"        %}
8442   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
8443   ins_encode(cdqq_enc(div), REX_reg_wide(div), OpcP, reg_opc(div));
8444   ins_pipe(pipe_slow);
8445 %}
8446 
8447 //----------- DivL-By-Constant-Expansions--------------------------------------
8448 // DivI cases are handled by the compiler
8449 
8450 // Magic constant, reciprocal of 10
8451 instruct loadConL_0x6666666666666667(rRegL dst)
8452 %{
8453   effect(DEF dst);
8454 
8455   format %{ "movq    $dst, #0x666666666666667\t# Used in div-by-10" %}
8456   ins_encode(load_immL(dst, 0x6666666666666667));
8457   ins_pipe(ialu_reg);
8458 %}
8459 
8460 instruct mul_hi(rdx_RegL dst, no_rax_RegL src, rax_RegL rax, rFlagsReg cr)
8461 %{
8462   effect(DEF dst, USE src, USE_KILL rax, KILL cr);
8463 
8464   format %{ "imulq   rdx:rax, rax, $src\t# Used in div-by-10" %}
8465   opcode(0xF7, 0x5); /* Opcode F7 /5 */
8466   ins_encode(REX_reg_wide(src), OpcP, reg_opc(src));
8467   ins_pipe(ialu_reg_reg_alu0);
8468 %}
8469 
8470 instruct sarL_rReg_63(rRegL dst, rFlagsReg cr)
8471 %{
8472   effect(USE_DEF dst, KILL cr);
8473 
8474   format %{ "sarq    $dst, #63\t# Used in div-by-10" %}
8475   opcode(0xC1, 0x7); /* C1 /7 ib */
8476   ins_encode(reg_opc_imm_wide(dst, 0x3F));
8477   ins_pipe(ialu_reg);
8478 %}
8479 
8480 instruct sarL_rReg_2(rRegL dst, rFlagsReg cr)
8481 %{
8482   effect(USE_DEF dst, KILL cr);
8483 
8484   format %{ "sarq    $dst, #2\t# Used in div-by-10" %}
8485   opcode(0xC1, 0x7); /* C1 /7 ib */
8486   ins_encode(reg_opc_imm_wide(dst, 0x2));
8487   ins_pipe(ialu_reg);
8488 %}
8489 
8490 instruct divL_10(rdx_RegL dst, no_rax_RegL src, immL10 div)
8491 %{
8492   match(Set dst (DivL src div));
8493 
8494   ins_cost((5+8)*100);
8495   expand %{
8496     rax_RegL rax;                     // Killed temp
8497     rFlagsReg cr;                     // Killed
8498     loadConL_0x6666666666666667(rax); // movq  rax, 0x6666666666666667
8499     mul_hi(dst, src, rax, cr);        // mulq  rdx:rax <= rax * $src
8500     sarL_rReg_63(src, cr);            // sarq  src, 63
8501     sarL_rReg_2(dst, cr);             // sarq  rdx, 2
8502     subL_rReg(dst, src, cr);          // subl  rdx, src
8503   %}
8504 %}
8505 
8506 //-----------------------------------------------------------------------------
8507 
8508 instruct modI_rReg(rdx_RegI rdx, rax_RegI rax, no_rax_rdx_RegI div,
8509                    rFlagsReg cr)
8510 %{
8511   match(Set rdx (ModI rax div));
8512   effect(KILL rax, KILL cr);
8513 
8514   ins_cost(300); // XXX
8515   format %{ "cmpl    rax, 0x80000000\t# irem\n\t"
8516             "jne,s   normal\n\t"
8517             "xorl    rdx, rdx\n\t"
8518             "cmpl    $div, -1\n\t"
8519             "je,s    done\n"
8520     "normal: cdql\n\t"
8521             "idivl   $div\n"
8522     "done:"        %}
8523   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
8524   ins_encode(cdql_enc(div), REX_reg(div), OpcP, reg_opc(div));
8525   ins_pipe(ialu_reg_reg_alu0);
8526 %}
8527 
8528 instruct modL_rReg(rdx_RegL rdx, rax_RegL rax, no_rax_rdx_RegL div,
8529                    rFlagsReg cr)
8530 %{
8531   match(Set rdx (ModL rax div));
8532   effect(KILL rax, KILL cr);
8533 
8534   ins_cost(300); // XXX
8535   format %{ "movq    rdx, 0x8000000000000000\t# lrem\n\t"
8536             "cmpq    rax, rdx\n\t"
8537             "jne,s   normal\n\t"
8538             "xorl    rdx, rdx\n\t"
8539             "cmpq    $div, -1\n\t"
8540             "je,s    done\n"
8541     "normal: cdqq\n\t"
8542             "idivq   $div\n"
8543     "done:"        %}
8544   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
8545   ins_encode(cdqq_enc(div), REX_reg_wide(div), OpcP, reg_opc(div));
8546   ins_pipe(ialu_reg_reg_alu0);
8547 %}
8548 
8549 // Integer Shift Instructions
8550 // Shift Left by one
8551 instruct salI_rReg_1(rRegI dst, immI1 shift, rFlagsReg cr)
8552 %{
8553   match(Set dst (LShiftI dst shift));
8554   effect(KILL cr);
8555 
8556   format %{ "sall    $dst, $shift" %}
8557   opcode(0xD1, 0x4); /* D1 /4 */
8558   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8559   ins_pipe(ialu_reg);
8560 %}
8561 
8562 // Shift Left by one
8563 instruct salI_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8564 %{
8565   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
8566   effect(KILL cr);
8567 
8568   format %{ "sall    $dst, $shift\t" %}
8569   opcode(0xD1, 0x4); /* D1 /4 */
8570   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8571   ins_pipe(ialu_mem_imm);
8572 %}
8573 
8574 // Shift Left by 8-bit immediate
8575 instruct salI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
8576 %{
8577   match(Set dst (LShiftI dst shift));
8578   effect(KILL cr);
8579 
8580   format %{ "sall    $dst, $shift" %}
8581   opcode(0xC1, 0x4); /* C1 /4 ib */
8582   ins_encode(reg_opc_imm(dst, shift));
8583   ins_pipe(ialu_reg);
8584 %}
8585 
8586 // Shift Left by 8-bit immediate
8587 instruct salI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8588 %{
8589   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
8590   effect(KILL cr);
8591 
8592   format %{ "sall    $dst, $shift" %}
8593   opcode(0xC1, 0x4); /* C1 /4 ib */
8594   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst), Con8or32(shift));
8595   ins_pipe(ialu_mem_imm);
8596 %}
8597 
8598 // Shift Left by variable
8599 instruct salI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
8600 %{
8601   match(Set dst (LShiftI dst shift));
8602   effect(KILL cr);
8603 
8604   format %{ "sall    $dst, $shift" %}
8605   opcode(0xD3, 0x4); /* D3 /4 */
8606   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8607   ins_pipe(ialu_reg_reg);
8608 %}
8609 
8610 // Shift Left by variable
8611 instruct salI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8612 %{
8613   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
8614   effect(KILL cr);
8615 
8616   format %{ "sall    $dst, $shift" %}
8617   opcode(0xD3, 0x4); /* D3 /4 */
8618   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8619   ins_pipe(ialu_mem_reg);
8620 %}
8621 
8622 // Arithmetic shift right by one
8623 instruct sarI_rReg_1(rRegI dst, immI1 shift, rFlagsReg cr)
8624 %{
8625   match(Set dst (RShiftI dst shift));
8626   effect(KILL cr);
8627 
8628   format %{ "sarl    $dst, $shift" %}
8629   opcode(0xD1, 0x7); /* D1 /7 */
8630   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8631   ins_pipe(ialu_reg);
8632 %}
8633 
8634 // Arithmetic shift right by one
8635 instruct sarI_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8636 %{
8637   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
8638   effect(KILL cr);
8639 
8640   format %{ "sarl    $dst, $shift" %}
8641   opcode(0xD1, 0x7); /* D1 /7 */
8642   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8643   ins_pipe(ialu_mem_imm);
8644 %}
8645 
8646 // Arithmetic Shift Right by 8-bit immediate
8647 instruct sarI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
8648 %{
8649   match(Set dst (RShiftI dst shift));
8650   effect(KILL cr);
8651 
8652   format %{ "sarl    $dst, $shift" %}
8653   opcode(0xC1, 0x7); /* C1 /7 ib */
8654   ins_encode(reg_opc_imm(dst, shift));
8655   ins_pipe(ialu_mem_imm);
8656 %}
8657 
8658 // Arithmetic Shift Right by 8-bit immediate
8659 instruct sarI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8660 %{
8661   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
8662   effect(KILL cr);
8663 
8664   format %{ "sarl    $dst, $shift" %}
8665   opcode(0xC1, 0x7); /* C1 /7 ib */
8666   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst), Con8or32(shift));
8667   ins_pipe(ialu_mem_imm);
8668 %}
8669 
8670 // Arithmetic Shift Right by variable
8671 instruct sarI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
8672 %{
8673   match(Set dst (RShiftI dst shift));
8674   effect(KILL cr);
8675 
8676   format %{ "sarl    $dst, $shift" %}
8677   opcode(0xD3, 0x7); /* D3 /7 */
8678   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8679   ins_pipe(ialu_reg_reg);
8680 %}
8681 
8682 // Arithmetic Shift Right by variable
8683 instruct sarI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8684 %{
8685   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
8686   effect(KILL cr);
8687 
8688   format %{ "sarl    $dst, $shift" %}
8689   opcode(0xD3, 0x7); /* D3 /7 */
8690   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8691   ins_pipe(ialu_mem_reg);
8692 %}
8693 
8694 // Logical shift right by one
8695 instruct shrI_rReg_1(rRegI dst, immI1 shift, rFlagsReg cr)
8696 %{
8697   match(Set dst (URShiftI dst shift));
8698   effect(KILL cr);
8699 
8700   format %{ "shrl    $dst, $shift" %}
8701   opcode(0xD1, 0x5); /* D1 /5 */
8702   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8703   ins_pipe(ialu_reg);
8704 %}
8705 
8706 // Logical shift right by one
8707 instruct shrI_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8708 %{
8709   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
8710   effect(KILL cr);
8711 
8712   format %{ "shrl    $dst, $shift" %}
8713   opcode(0xD1, 0x5); /* D1 /5 */
8714   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8715   ins_pipe(ialu_mem_imm);
8716 %}
8717 
8718 // Logical Shift Right by 8-bit immediate
8719 instruct shrI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
8720 %{
8721   match(Set dst (URShiftI dst shift));
8722   effect(KILL cr);
8723 
8724   format %{ "shrl    $dst, $shift" %}
8725   opcode(0xC1, 0x5); /* C1 /5 ib */
8726   ins_encode(reg_opc_imm(dst, shift));
8727   ins_pipe(ialu_reg);
8728 %}
8729 
8730 // Logical Shift Right by 8-bit immediate
8731 instruct shrI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8732 %{
8733   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
8734   effect(KILL cr);
8735 
8736   format %{ "shrl    $dst, $shift" %}
8737   opcode(0xC1, 0x5); /* C1 /5 ib */
8738   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst), Con8or32(shift));
8739   ins_pipe(ialu_mem_imm);
8740 %}
8741 
8742 // Logical Shift Right by variable
8743 instruct shrI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
8744 %{
8745   match(Set dst (URShiftI dst shift));
8746   effect(KILL cr);
8747 
8748   format %{ "shrl    $dst, $shift" %}
8749   opcode(0xD3, 0x5); /* D3 /5 */
8750   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8751   ins_pipe(ialu_reg_reg);
8752 %}
8753 
8754 // Logical Shift Right by variable
8755 instruct shrI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8756 %{
8757   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
8758   effect(KILL cr);
8759 
8760   format %{ "shrl    $dst, $shift" %}
8761   opcode(0xD3, 0x5); /* D3 /5 */
8762   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8763   ins_pipe(ialu_mem_reg);
8764 %}
8765 
8766 // Long Shift Instructions
8767 // Shift Left by one
8768 instruct salL_rReg_1(rRegL dst, immI1 shift, rFlagsReg cr)
8769 %{
8770   match(Set dst (LShiftL dst shift));
8771   effect(KILL cr);
8772 
8773   format %{ "salq    $dst, $shift" %}
8774   opcode(0xD1, 0x4); /* D1 /4 */
8775   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8776   ins_pipe(ialu_reg);
8777 %}
8778 
8779 // Shift Left by one
8780 instruct salL_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8781 %{
8782   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
8783   effect(KILL cr);
8784 
8785   format %{ "salq    $dst, $shift" %}
8786   opcode(0xD1, 0x4); /* D1 /4 */
8787   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8788   ins_pipe(ialu_mem_imm);
8789 %}
8790 
8791 // Shift Left by 8-bit immediate
8792 instruct salL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
8793 %{
8794   match(Set dst (LShiftL dst shift));
8795   effect(KILL cr);
8796 
8797   format %{ "salq    $dst, $shift" %}
8798   opcode(0xC1, 0x4); /* C1 /4 ib */
8799   ins_encode(reg_opc_imm_wide(dst, shift));
8800   ins_pipe(ialu_reg);
8801 %}
8802 
8803 // Shift Left by 8-bit immediate
8804 instruct salL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8805 %{
8806   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
8807   effect(KILL cr);
8808 
8809   format %{ "salq    $dst, $shift" %}
8810   opcode(0xC1, 0x4); /* C1 /4 ib */
8811   ins_encode(REX_mem_wide(dst), OpcP,
8812              RM_opc_mem(secondary, dst), Con8or32(shift));
8813   ins_pipe(ialu_mem_imm);
8814 %}
8815 
8816 // Shift Left by variable
8817 instruct salL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
8818 %{
8819   match(Set dst (LShiftL dst shift));
8820   effect(KILL cr);
8821 
8822   format %{ "salq    $dst, $shift" %}
8823   opcode(0xD3, 0x4); /* D3 /4 */
8824   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8825   ins_pipe(ialu_reg_reg);
8826 %}
8827 
8828 // Shift Left by variable
8829 instruct salL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8830 %{
8831   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
8832   effect(KILL cr);
8833 
8834   format %{ "salq    $dst, $shift" %}
8835   opcode(0xD3, 0x4); /* D3 /4 */
8836   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8837   ins_pipe(ialu_mem_reg);
8838 %}
8839 
8840 // Arithmetic shift right by one
8841 instruct sarL_rReg_1(rRegL dst, immI1 shift, rFlagsReg cr)
8842 %{
8843   match(Set dst (RShiftL dst shift));
8844   effect(KILL cr);
8845 
8846   format %{ "sarq    $dst, $shift" %}
8847   opcode(0xD1, 0x7); /* D1 /7 */
8848   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8849   ins_pipe(ialu_reg);
8850 %}
8851 
8852 // Arithmetic shift right by one
8853 instruct sarL_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8854 %{
8855   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
8856   effect(KILL cr);
8857 
8858   format %{ "sarq    $dst, $shift" %}
8859   opcode(0xD1, 0x7); /* D1 /7 */
8860   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8861   ins_pipe(ialu_mem_imm);
8862 %}
8863 
8864 // Arithmetic Shift Right by 8-bit immediate
8865 instruct sarL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
8866 %{
8867   match(Set dst (RShiftL dst shift));
8868   effect(KILL cr);
8869 
8870   format %{ "sarq    $dst, $shift" %}
8871   opcode(0xC1, 0x7); /* C1 /7 ib */
8872   ins_encode(reg_opc_imm_wide(dst, shift));
8873   ins_pipe(ialu_mem_imm);
8874 %}
8875 
8876 // Arithmetic Shift Right by 8-bit immediate
8877 instruct sarL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8878 %{
8879   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
8880   effect(KILL cr);
8881 
8882   format %{ "sarq    $dst, $shift" %}
8883   opcode(0xC1, 0x7); /* C1 /7 ib */
8884   ins_encode(REX_mem_wide(dst), OpcP,
8885              RM_opc_mem(secondary, dst), Con8or32(shift));
8886   ins_pipe(ialu_mem_imm);
8887 %}
8888 
8889 // Arithmetic Shift Right by variable
8890 instruct sarL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
8891 %{
8892   match(Set dst (RShiftL dst shift));
8893   effect(KILL cr);
8894 
8895   format %{ "sarq    $dst, $shift" %}
8896   opcode(0xD3, 0x7); /* D3 /7 */
8897   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8898   ins_pipe(ialu_reg_reg);
8899 %}
8900 
8901 // Arithmetic Shift Right by variable
8902 instruct sarL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8903 %{
8904   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
8905   effect(KILL cr);
8906 
8907   format %{ "sarq    $dst, $shift" %}
8908   opcode(0xD3, 0x7); /* D3 /7 */
8909   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8910   ins_pipe(ialu_mem_reg);
8911 %}
8912 
8913 // Logical shift right by one
8914 instruct shrL_rReg_1(rRegL dst, immI1 shift, rFlagsReg cr)
8915 %{
8916   match(Set dst (URShiftL dst shift));
8917   effect(KILL cr);
8918 
8919   format %{ "shrq    $dst, $shift" %}
8920   opcode(0xD1, 0x5); /* D1 /5 */
8921   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst ));
8922   ins_pipe(ialu_reg);
8923 %}
8924 
8925 // Logical shift right by one
8926 instruct shrL_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8927 %{
8928   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
8929   effect(KILL cr);
8930 
8931   format %{ "shrq    $dst, $shift" %}
8932   opcode(0xD1, 0x5); /* D1 /5 */
8933   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8934   ins_pipe(ialu_mem_imm);
8935 %}
8936 
8937 // Logical Shift Right by 8-bit immediate
8938 instruct shrL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
8939 %{
8940   match(Set dst (URShiftL dst shift));
8941   effect(KILL cr);
8942 
8943   format %{ "shrq    $dst, $shift" %}
8944   opcode(0xC1, 0x5); /* C1 /5 ib */
8945   ins_encode(reg_opc_imm_wide(dst, shift));
8946   ins_pipe(ialu_reg);
8947 %}
8948 
8949 
8950 // Logical Shift Right by 8-bit immediate
8951 instruct shrL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8952 %{
8953   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
8954   effect(KILL cr);
8955 
8956   format %{ "shrq    $dst, $shift" %}
8957   opcode(0xC1, 0x5); /* C1 /5 ib */
8958   ins_encode(REX_mem_wide(dst), OpcP,
8959              RM_opc_mem(secondary, dst), Con8or32(shift));
8960   ins_pipe(ialu_mem_imm);
8961 %}
8962 
8963 // Logical Shift Right by variable
8964 instruct shrL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
8965 %{
8966   match(Set dst (URShiftL dst shift));
8967   effect(KILL cr);
8968 
8969   format %{ "shrq    $dst, $shift" %}
8970   opcode(0xD3, 0x5); /* D3 /5 */
8971   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8972   ins_pipe(ialu_reg_reg);
8973 %}
8974 
8975 // Logical Shift Right by variable
8976 instruct shrL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8977 %{
8978   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
8979   effect(KILL cr);
8980 
8981   format %{ "shrq    $dst, $shift" %}
8982   opcode(0xD3, 0x5); /* D3 /5 */
8983   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8984   ins_pipe(ialu_mem_reg);
8985 %}
8986 
8987 // Logical Shift Right by 24, followed by Arithmetic Shift Left by 24.
8988 // This idiom is used by the compiler for the i2b bytecode.
8989 instruct i2b(rRegI dst, rRegI src, immI_24 twentyfour)
8990 %{
8991   match(Set dst (RShiftI (LShiftI src twentyfour) twentyfour));
8992 
8993   format %{ "movsbl  $dst, $src\t# i2b" %}
8994   opcode(0x0F, 0xBE);
8995   ins_encode(REX_reg_breg(dst, src), OpcP, OpcS, reg_reg(dst, src));
8996   ins_pipe(ialu_reg_reg);
8997 %}
8998 
8999 // Logical Shift Right by 16, followed by Arithmetic Shift Left by 16.
9000 // This idiom is used by the compiler the i2s bytecode.
9001 instruct i2s(rRegI dst, rRegI src, immI_16 sixteen)
9002 %{
9003   match(Set dst (RShiftI (LShiftI src sixteen) sixteen));
9004 
9005   format %{ "movswl  $dst, $src\t# i2s" %}
9006   opcode(0x0F, 0xBF);
9007   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
9008   ins_pipe(ialu_reg_reg);
9009 %}
9010 
9011 // ROL/ROR instructions
9012 
9013 // ROL expand
9014 instruct rolI_rReg_imm1(rRegI dst, rFlagsReg cr) %{
9015   effect(KILL cr, USE_DEF dst);
9016 
9017   format %{ "roll    $dst" %}
9018   opcode(0xD1, 0x0); /* Opcode  D1 /0 */
9019   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
9020   ins_pipe(ialu_reg);
9021 %}
9022 
9023 instruct rolI_rReg_imm8(rRegI dst, immI8 shift, rFlagsReg cr) %{
9024   effect(USE_DEF dst, USE shift, KILL cr);
9025 
9026   format %{ "roll    $dst, $shift" %}
9027   opcode(0xC1, 0x0); /* Opcode C1 /0 ib */
9028   ins_encode( reg_opc_imm(dst, shift) );
9029   ins_pipe(ialu_reg);
9030 %}
9031 
9032 instruct rolI_rReg_CL(no_rcx_RegI dst, rcx_RegI shift, rFlagsReg cr)
9033 %{
9034   effect(USE_DEF dst, USE shift, KILL cr);
9035 
9036   format %{ "roll    $dst, $shift" %}
9037   opcode(0xD3, 0x0); /* Opcode D3 /0 */
9038   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
9039   ins_pipe(ialu_reg_reg);
9040 %}
9041 // end of ROL expand
9042 
9043 // Rotate Left by one
9044 instruct rolI_rReg_i1(rRegI dst, immI1 lshift, immI_M1 rshift, rFlagsReg cr)
9045 %{
9046   match(Set dst (OrI (LShiftI dst lshift) (URShiftI dst rshift)));
9047 
9048   expand %{
9049     rolI_rReg_imm1(dst, cr);
9050   %}
9051 %}
9052 
9053 // Rotate Left by 8-bit immediate
9054 instruct rolI_rReg_i8(rRegI dst, immI8 lshift, immI8 rshift, rFlagsReg cr)
9055 %{
9056   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x1f));
9057   match(Set dst (OrI (LShiftI dst lshift) (URShiftI dst rshift)));
9058 
9059   expand %{
9060     rolI_rReg_imm8(dst, lshift, cr);
9061   %}
9062 %}
9063 
9064 // Rotate Left by variable
9065 instruct rolI_rReg_Var_C0(no_rcx_RegI dst, rcx_RegI shift, immI0 zero, rFlagsReg cr)
9066 %{
9067   match(Set dst (OrI (LShiftI dst shift) (URShiftI dst (SubI zero shift))));
9068 
9069   expand %{
9070     rolI_rReg_CL(dst, shift, cr);
9071   %}
9072 %}
9073 
9074 // Rotate Left by variable
9075 instruct rolI_rReg_Var_C32(no_rcx_RegI dst, rcx_RegI shift, immI_32 c32, rFlagsReg cr)
9076 %{
9077   match(Set dst (OrI (LShiftI dst shift) (URShiftI dst (SubI c32 shift))));
9078 
9079   expand %{
9080     rolI_rReg_CL(dst, shift, cr);
9081   %}
9082 %}
9083 
9084 // ROR expand
9085 instruct rorI_rReg_imm1(rRegI dst, rFlagsReg cr)
9086 %{
9087   effect(USE_DEF dst, KILL cr);
9088 
9089   format %{ "rorl    $dst" %}
9090   opcode(0xD1, 0x1); /* D1 /1 */
9091   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
9092   ins_pipe(ialu_reg);
9093 %}
9094 
9095 instruct rorI_rReg_imm8(rRegI dst, immI8 shift, rFlagsReg cr)
9096 %{
9097   effect(USE_DEF dst, USE shift, KILL cr);
9098 
9099   format %{ "rorl    $dst, $shift" %}
9100   opcode(0xC1, 0x1); /* C1 /1 ib */
9101   ins_encode(reg_opc_imm(dst, shift));
9102   ins_pipe(ialu_reg);
9103 %}
9104 
9105 instruct rorI_rReg_CL(no_rcx_RegI dst, rcx_RegI shift, rFlagsReg cr)
9106 %{
9107   effect(USE_DEF dst, USE shift, KILL cr);
9108 
9109   format %{ "rorl    $dst, $shift" %}
9110   opcode(0xD3, 0x1); /* D3 /1 */
9111   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
9112   ins_pipe(ialu_reg_reg);
9113 %}
9114 // end of ROR expand
9115 
9116 // Rotate Right by one
9117 instruct rorI_rReg_i1(rRegI dst, immI1 rshift, immI_M1 lshift, rFlagsReg cr)
9118 %{
9119   match(Set dst (OrI (URShiftI dst rshift) (LShiftI dst lshift)));
9120 
9121   expand %{
9122     rorI_rReg_imm1(dst, cr);
9123   %}
9124 %}
9125 
9126 // Rotate Right by 8-bit immediate
9127 instruct rorI_rReg_i8(rRegI dst, immI8 rshift, immI8 lshift, rFlagsReg cr)
9128 %{
9129   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x1f));
9130   match(Set dst (OrI (URShiftI dst rshift) (LShiftI dst lshift)));
9131 
9132   expand %{
9133     rorI_rReg_imm8(dst, rshift, cr);
9134   %}
9135 %}
9136 
9137 // Rotate Right by variable
9138 instruct rorI_rReg_Var_C0(no_rcx_RegI dst, rcx_RegI shift, immI0 zero, rFlagsReg cr)
9139 %{
9140   match(Set dst (OrI (URShiftI dst shift) (LShiftI dst (SubI zero shift))));
9141 
9142   expand %{
9143     rorI_rReg_CL(dst, shift, cr);
9144   %}
9145 %}
9146 
9147 // Rotate Right by variable
9148 instruct rorI_rReg_Var_C32(no_rcx_RegI dst, rcx_RegI shift, immI_32 c32, rFlagsReg cr)
9149 %{
9150   match(Set dst (OrI (URShiftI dst shift) (LShiftI dst (SubI c32 shift))));
9151 
9152   expand %{
9153     rorI_rReg_CL(dst, shift, cr);
9154   %}
9155 %}
9156 
9157 // for long rotate
9158 // ROL expand
9159 instruct rolL_rReg_imm1(rRegL dst, rFlagsReg cr) %{
9160   effect(USE_DEF dst, KILL cr);
9161 
9162   format %{ "rolq    $dst" %}
9163   opcode(0xD1, 0x0); /* Opcode  D1 /0 */
9164   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
9165   ins_pipe(ialu_reg);
9166 %}
9167 
9168 instruct rolL_rReg_imm8(rRegL dst, immI8 shift, rFlagsReg cr) %{
9169   effect(USE_DEF dst, USE shift, KILL cr);
9170 
9171   format %{ "rolq    $dst, $shift" %}
9172   opcode(0xC1, 0x0); /* Opcode C1 /0 ib */
9173   ins_encode( reg_opc_imm_wide(dst, shift) );
9174   ins_pipe(ialu_reg);
9175 %}
9176 
9177 instruct rolL_rReg_CL(no_rcx_RegL dst, rcx_RegI shift, rFlagsReg cr)
9178 %{
9179   effect(USE_DEF dst, USE shift, KILL cr);
9180 
9181   format %{ "rolq    $dst, $shift" %}
9182   opcode(0xD3, 0x0); /* Opcode D3 /0 */
9183   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
9184   ins_pipe(ialu_reg_reg);
9185 %}
9186 // end of ROL expand
9187 
9188 // Rotate Left by one
9189 instruct rolL_rReg_i1(rRegL dst, immI1 lshift, immI_M1 rshift, rFlagsReg cr)
9190 %{
9191   match(Set dst (OrL (LShiftL dst lshift) (URShiftL dst rshift)));
9192 
9193   expand %{
9194     rolL_rReg_imm1(dst, cr);
9195   %}
9196 %}
9197 
9198 // Rotate Left by 8-bit immediate
9199 instruct rolL_rReg_i8(rRegL dst, immI8 lshift, immI8 rshift, rFlagsReg cr)
9200 %{
9201   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x3f));
9202   match(Set dst (OrL (LShiftL dst lshift) (URShiftL dst rshift)));
9203 
9204   expand %{
9205     rolL_rReg_imm8(dst, lshift, cr);
9206   %}
9207 %}
9208 
9209 // Rotate Left by variable
9210 instruct rolL_rReg_Var_C0(no_rcx_RegL dst, rcx_RegI shift, immI0 zero, rFlagsReg cr)
9211 %{
9212   match(Set dst (OrL (LShiftL dst shift) (URShiftL dst (SubI zero shift))));
9213 
9214   expand %{
9215     rolL_rReg_CL(dst, shift, cr);
9216   %}
9217 %}
9218 
9219 // Rotate Left by variable
9220 instruct rolL_rReg_Var_C64(no_rcx_RegL dst, rcx_RegI shift, immI_64 c64, rFlagsReg cr)
9221 %{
9222   match(Set dst (OrL (LShiftL dst shift) (URShiftL dst (SubI c64 shift))));
9223 
9224   expand %{
9225     rolL_rReg_CL(dst, shift, cr);
9226   %}
9227 %}
9228 
9229 // ROR expand
9230 instruct rorL_rReg_imm1(rRegL dst, rFlagsReg cr)
9231 %{
9232   effect(USE_DEF dst, KILL cr);
9233 
9234   format %{ "rorq    $dst" %}
9235   opcode(0xD1, 0x1); /* D1 /1 */
9236   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
9237   ins_pipe(ialu_reg);
9238 %}
9239 
9240 instruct rorL_rReg_imm8(rRegL dst, immI8 shift, rFlagsReg cr)
9241 %{
9242   effect(USE_DEF dst, USE shift, KILL cr);
9243 
9244   format %{ "rorq    $dst, $shift" %}
9245   opcode(0xC1, 0x1); /* C1 /1 ib */
9246   ins_encode(reg_opc_imm_wide(dst, shift));
9247   ins_pipe(ialu_reg);
9248 %}
9249 
9250 instruct rorL_rReg_CL(no_rcx_RegL dst, rcx_RegI shift, rFlagsReg cr)
9251 %{
9252   effect(USE_DEF dst, USE shift, KILL cr);
9253 
9254   format %{ "rorq    $dst, $shift" %}
9255   opcode(0xD3, 0x1); /* D3 /1 */
9256   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
9257   ins_pipe(ialu_reg_reg);
9258 %}
9259 // end of ROR expand
9260 
9261 // Rotate Right by one
9262 instruct rorL_rReg_i1(rRegL dst, immI1 rshift, immI_M1 lshift, rFlagsReg cr)
9263 %{
9264   match(Set dst (OrL (URShiftL dst rshift) (LShiftL dst lshift)));
9265 
9266   expand %{
9267     rorL_rReg_imm1(dst, cr);
9268   %}
9269 %}
9270 
9271 // Rotate Right by 8-bit immediate
9272 instruct rorL_rReg_i8(rRegL dst, immI8 rshift, immI8 lshift, rFlagsReg cr)
9273 %{
9274   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x3f));
9275   match(Set dst (OrL (URShiftL dst rshift) (LShiftL dst lshift)));
9276 
9277   expand %{
9278     rorL_rReg_imm8(dst, rshift, cr);
9279   %}
9280 %}
9281 
9282 // Rotate Right by variable
9283 instruct rorL_rReg_Var_C0(no_rcx_RegL dst, rcx_RegI shift, immI0 zero, rFlagsReg cr)
9284 %{
9285   match(Set dst (OrL (URShiftL dst shift) (LShiftL dst (SubI zero shift))));
9286 
9287   expand %{
9288     rorL_rReg_CL(dst, shift, cr);
9289   %}
9290 %}
9291 
9292 // Rotate Right by variable
9293 instruct rorL_rReg_Var_C64(no_rcx_RegL dst, rcx_RegI shift, immI_64 c64, rFlagsReg cr)
9294 %{
9295   match(Set dst (OrL (URShiftL dst shift) (LShiftL dst (SubI c64 shift))));
9296 
9297   expand %{
9298     rorL_rReg_CL(dst, shift, cr);
9299   %}
9300 %}
9301 
9302 // Logical Instructions
9303 
9304 // Integer Logical Instructions
9305 
9306 // And Instructions
9307 // And Register with Register
9308 instruct andI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
9309 %{
9310   match(Set dst (AndI dst src));
9311   effect(KILL cr);
9312 
9313   format %{ "andl    $dst, $src\t# int" %}
9314   opcode(0x23);
9315   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
9316   ins_pipe(ialu_reg_reg);
9317 %}
9318 
9319 // And Register with Immediate 255
9320 instruct andI_rReg_imm255(rRegI dst, immI_255 src)
9321 %{
9322   match(Set dst (AndI dst src));
9323 
9324   format %{ "movzbl  $dst, $dst\t# int & 0xFF" %}
9325   opcode(0x0F, 0xB6);
9326   ins_encode(REX_reg_breg(dst, dst), OpcP, OpcS, reg_reg(dst, dst));
9327   ins_pipe(ialu_reg);
9328 %}
9329 
9330 // And Register with Immediate 255 and promote to long
9331 instruct andI2L_rReg_imm255(rRegL dst, rRegI src, immI_255 mask)
9332 %{
9333   match(Set dst (ConvI2L (AndI src mask)));
9334 
9335   format %{ "movzbl  $dst, $src\t# int & 0xFF -> long" %}
9336   opcode(0x0F, 0xB6);
9337   ins_encode(REX_reg_breg(dst, src), OpcP, OpcS, reg_reg(dst, src));
9338   ins_pipe(ialu_reg);
9339 %}
9340 
9341 // And Register with Immediate 65535
9342 instruct andI_rReg_imm65535(rRegI dst, immI_65535 src)
9343 %{
9344   match(Set dst (AndI dst src));
9345 
9346   format %{ "movzwl  $dst, $dst\t# int & 0xFFFF" %}
9347   opcode(0x0F, 0xB7);
9348   ins_encode(REX_reg_reg(dst, dst), OpcP, OpcS, reg_reg(dst, dst));
9349   ins_pipe(ialu_reg);
9350 %}
9351 
9352 // And Register with Immediate 65535 and promote to long
9353 instruct andI2L_rReg_imm65535(rRegL dst, rRegI src, immI_65535 mask)
9354 %{
9355   match(Set dst (ConvI2L (AndI src mask)));
9356 
9357   format %{ "movzwl  $dst, $src\t# int & 0xFFFF -> long" %}
9358   opcode(0x0F, 0xB7);
9359   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
9360   ins_pipe(ialu_reg);
9361 %}
9362 
9363 // And Register with Immediate
9364 instruct andI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
9365 %{
9366   match(Set dst (AndI dst src));
9367   effect(KILL cr);
9368 
9369   format %{ "andl    $dst, $src\t# int" %}
9370   opcode(0x81, 0x04); /* Opcode 81 /4 */
9371   ins_encode(OpcSErm(dst, src), Con8or32(src));
9372   ins_pipe(ialu_reg);
9373 %}
9374 
9375 // And Register with Memory
9376 instruct andI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
9377 %{
9378   match(Set dst (AndI dst (LoadI src)));
9379   effect(KILL cr);
9380 
9381   ins_cost(125);
9382   format %{ "andl    $dst, $src\t# int" %}
9383   opcode(0x23);
9384   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
9385   ins_pipe(ialu_reg_mem);
9386 %}
9387 
9388 // And Memory with Register
9389 instruct andI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
9390 %{
9391   match(Set dst (StoreI dst (AndI (LoadI dst) src)));
9392   effect(KILL cr);
9393 
9394   ins_cost(150);
9395   format %{ "andl    $dst, $src\t# int" %}
9396   opcode(0x21); /* Opcode 21 /r */
9397   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
9398   ins_pipe(ialu_mem_reg);
9399 %}
9400 
9401 // And Memory with Immediate
9402 instruct andI_mem_imm(memory dst, immI src, rFlagsReg cr)
9403 %{
9404   match(Set dst (StoreI dst (AndI (LoadI dst) src)));
9405   effect(KILL cr);
9406 
9407   ins_cost(125);
9408   format %{ "andl    $dst, $src\t# int" %}
9409   opcode(0x81, 0x4); /* Opcode 81 /4 id */
9410   ins_encode(REX_mem(dst), OpcSE(src),
9411              RM_opc_mem(secondary, dst), Con8or32(src));
9412   ins_pipe(ialu_mem_imm);
9413 %}
9414 
9415 // BMI1 instructions
9416 instruct andnI_rReg_rReg_mem(rRegI dst, rRegI src1, memory src2, immI_M1 minus_1, rFlagsReg cr) %{
9417   match(Set dst (AndI (XorI src1 minus_1) (LoadI src2)));
9418   predicate(UseBMI1Instructions);
9419   effect(KILL cr);
9420 
9421   ins_cost(125);
9422   format %{ "andnl  $dst, $src1, $src2" %}
9423 
9424   ins_encode %{
9425     __ andnl($dst$$Register, $src1$$Register, $src2$$Address);
9426   %}
9427   ins_pipe(ialu_reg_mem);
9428 %}
9429 
9430 instruct andnI_rReg_rReg_rReg(rRegI dst, rRegI src1, rRegI src2, immI_M1 minus_1, rFlagsReg cr) %{
9431   match(Set dst (AndI (XorI src1 minus_1) src2));
9432   predicate(UseBMI1Instructions);
9433   effect(KILL cr);
9434 
9435   format %{ "andnl  $dst, $src1, $src2" %}
9436 
9437   ins_encode %{
9438     __ andnl($dst$$Register, $src1$$Register, $src2$$Register);
9439   %}
9440   ins_pipe(ialu_reg);
9441 %}
9442 
9443 instruct blsiI_rReg_rReg(rRegI dst, rRegI src, immI0 imm_zero, rFlagsReg cr) %{
9444   match(Set dst (AndI (SubI imm_zero src) src));
9445   predicate(UseBMI1Instructions);
9446   effect(KILL cr);
9447 
9448   format %{ "blsil  $dst, $src" %}
9449 
9450   ins_encode %{
9451     __ blsil($dst$$Register, $src$$Register);
9452   %}
9453   ins_pipe(ialu_reg);
9454 %}
9455 
9456 instruct blsiI_rReg_mem(rRegI dst, memory src, immI0 imm_zero, rFlagsReg cr) %{
9457   match(Set dst (AndI (SubI imm_zero (LoadI src) ) (LoadI src) ));
9458   predicate(UseBMI1Instructions);
9459   effect(KILL cr);
9460 
9461   ins_cost(125);
9462   format %{ "blsil  $dst, $src" %}
9463 
9464   ins_encode %{
9465     __ blsil($dst$$Register, $src$$Address);
9466   %}
9467   ins_pipe(ialu_reg_mem);
9468 %}
9469 
9470 instruct blsmskI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, rFlagsReg cr)
9471 %{
9472   match(Set dst (XorI (AddI (LoadI src) minus_1) (LoadI src) ) );
9473   predicate(UseBMI1Instructions);
9474   effect(KILL cr);
9475 
9476   ins_cost(125);
9477   format %{ "blsmskl $dst, $src" %}
9478 
9479   ins_encode %{
9480     __ blsmskl($dst$$Register, $src$$Address);
9481   %}
9482   ins_pipe(ialu_reg_mem);
9483 %}
9484 
9485 instruct blsmskI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, rFlagsReg cr)
9486 %{
9487   match(Set dst (XorI (AddI src minus_1) src));
9488   predicate(UseBMI1Instructions);
9489   effect(KILL cr);
9490 
9491   format %{ "blsmskl $dst, $src" %}
9492 
9493   ins_encode %{
9494     __ blsmskl($dst$$Register, $src$$Register);
9495   %}
9496 
9497   ins_pipe(ialu_reg);
9498 %}
9499 
9500 instruct blsrI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, rFlagsReg cr)
9501 %{
9502   match(Set dst (AndI (AddI src minus_1) src) );
9503   predicate(UseBMI1Instructions);
9504   effect(KILL cr);
9505 
9506   format %{ "blsrl  $dst, $src" %}
9507 
9508   ins_encode %{
9509     __ blsrl($dst$$Register, $src$$Register);
9510   %}
9511 
9512   ins_pipe(ialu_reg_mem);
9513 %}
9514 
9515 instruct blsrI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, rFlagsReg cr)
9516 %{
9517   match(Set dst (AndI (AddI (LoadI src) minus_1) (LoadI src) ) );
9518   predicate(UseBMI1Instructions);
9519   effect(KILL cr);
9520 
9521   ins_cost(125);
9522   format %{ "blsrl  $dst, $src" %}
9523 
9524   ins_encode %{
9525     __ blsrl($dst$$Register, $src$$Address);
9526   %}
9527 
9528   ins_pipe(ialu_reg);
9529 %}
9530 
9531 // Or Instructions
9532 // Or Register with Register
9533 instruct orI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
9534 %{
9535   match(Set dst (OrI dst src));
9536   effect(KILL cr);
9537 
9538   format %{ "orl     $dst, $src\t# int" %}
9539   opcode(0x0B);
9540   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
9541   ins_pipe(ialu_reg_reg);
9542 %}
9543 
9544 // Or Register with Immediate
9545 instruct orI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
9546 %{
9547   match(Set dst (OrI dst src));
9548   effect(KILL cr);
9549 
9550   format %{ "orl     $dst, $src\t# int" %}
9551   opcode(0x81, 0x01); /* Opcode 81 /1 id */
9552   ins_encode(OpcSErm(dst, src), Con8or32(src));
9553   ins_pipe(ialu_reg);
9554 %}
9555 
9556 // Or Register with Memory
9557 instruct orI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
9558 %{
9559   match(Set dst (OrI dst (LoadI src)));
9560   effect(KILL cr);
9561 
9562   ins_cost(125);
9563   format %{ "orl     $dst, $src\t# int" %}
9564   opcode(0x0B);
9565   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
9566   ins_pipe(ialu_reg_mem);
9567 %}
9568 
9569 // Or Memory with Register
9570 instruct orI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
9571 %{
9572   match(Set dst (StoreI dst (OrI (LoadI dst) src)));
9573   effect(KILL cr);
9574 
9575   ins_cost(150);
9576   format %{ "orl     $dst, $src\t# int" %}
9577   opcode(0x09); /* Opcode 09 /r */
9578   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
9579   ins_pipe(ialu_mem_reg);
9580 %}
9581 
9582 // Or Memory with Immediate
9583 instruct orI_mem_imm(memory dst, immI src, rFlagsReg cr)
9584 %{
9585   match(Set dst (StoreI dst (OrI (LoadI dst) src)));
9586   effect(KILL cr);
9587 
9588   ins_cost(125);
9589   format %{ "orl     $dst, $src\t# int" %}
9590   opcode(0x81, 0x1); /* Opcode 81 /1 id */
9591   ins_encode(REX_mem(dst), OpcSE(src),
9592              RM_opc_mem(secondary, dst), Con8or32(src));
9593   ins_pipe(ialu_mem_imm);
9594 %}
9595 
9596 // Xor Instructions
9597 // Xor Register with Register
9598 instruct xorI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
9599 %{
9600   match(Set dst (XorI dst src));
9601   effect(KILL cr);
9602 
9603   format %{ "xorl    $dst, $src\t# int" %}
9604   opcode(0x33);
9605   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
9606   ins_pipe(ialu_reg_reg);
9607 %}
9608 
9609 // Xor Register with Immediate -1
9610 instruct xorI_rReg_im1(rRegI dst, immI_M1 imm) %{
9611   match(Set dst (XorI dst imm));
9612 
9613   format %{ "not    $dst" %}
9614   ins_encode %{
9615      __ notl($dst$$Register);
9616   %}
9617   ins_pipe(ialu_reg);
9618 %}
9619 
9620 // Xor Register with Immediate
9621 instruct xorI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
9622 %{
9623   match(Set dst (XorI dst src));
9624   effect(KILL cr);
9625 
9626   format %{ "xorl    $dst, $src\t# int" %}
9627   opcode(0x81, 0x06); /* Opcode 81 /6 id */
9628   ins_encode(OpcSErm(dst, src), Con8or32(src));
9629   ins_pipe(ialu_reg);
9630 %}
9631 
9632 // Xor Register with Memory
9633 instruct xorI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
9634 %{
9635   match(Set dst (XorI dst (LoadI src)));
9636   effect(KILL cr);
9637 
9638   ins_cost(125);
9639   format %{ "xorl    $dst, $src\t# int" %}
9640   opcode(0x33);
9641   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
9642   ins_pipe(ialu_reg_mem);
9643 %}
9644 
9645 // Xor Memory with Register
9646 instruct xorI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
9647 %{
9648   match(Set dst (StoreI dst (XorI (LoadI dst) src)));
9649   effect(KILL cr);
9650 
9651   ins_cost(150);
9652   format %{ "xorl    $dst, $src\t# int" %}
9653   opcode(0x31); /* Opcode 31 /r */
9654   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
9655   ins_pipe(ialu_mem_reg);
9656 %}
9657 
9658 // Xor Memory with Immediate
9659 instruct xorI_mem_imm(memory dst, immI src, rFlagsReg cr)
9660 %{
9661   match(Set dst (StoreI dst (XorI (LoadI dst) src)));
9662   effect(KILL cr);
9663 
9664   ins_cost(125);
9665   format %{ "xorl    $dst, $src\t# int" %}
9666   opcode(0x81, 0x6); /* Opcode 81 /6 id */
9667   ins_encode(REX_mem(dst), OpcSE(src),
9668              RM_opc_mem(secondary, dst), Con8or32(src));
9669   ins_pipe(ialu_mem_imm);
9670 %}
9671 
9672 
9673 // Long Logical Instructions
9674 
9675 // And Instructions
9676 // And Register with Register
9677 instruct andL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
9678 %{
9679   match(Set dst (AndL dst src));
9680   effect(KILL cr);
9681 
9682   format %{ "andq    $dst, $src\t# long" %}
9683   opcode(0x23);
9684   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
9685   ins_pipe(ialu_reg_reg);
9686 %}
9687 
9688 // And Register with Immediate 255
9689 instruct andL_rReg_imm255(rRegL dst, immL_255 src)
9690 %{
9691   match(Set dst (AndL dst src));
9692 
9693   format %{ "movzbq  $dst, $dst\t# long & 0xFF" %}
9694   opcode(0x0F, 0xB6);
9695   ins_encode(REX_reg_reg_wide(dst, dst), OpcP, OpcS, reg_reg(dst, dst));
9696   ins_pipe(ialu_reg);
9697 %}
9698 
9699 // And Register with Immediate 65535
9700 instruct andL_rReg_imm65535(rRegL dst, immL_65535 src)
9701 %{
9702   match(Set dst (AndL dst src));
9703 
9704   format %{ "movzwq  $dst, $dst\t# long & 0xFFFF" %}
9705   opcode(0x0F, 0xB7);
9706   ins_encode(REX_reg_reg_wide(dst, dst), OpcP, OpcS, reg_reg(dst, dst));
9707   ins_pipe(ialu_reg);
9708 %}
9709 
9710 // And Register with Immediate
9711 instruct andL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
9712 %{
9713   match(Set dst (AndL dst src));
9714   effect(KILL cr);
9715 
9716   format %{ "andq    $dst, $src\t# long" %}
9717   opcode(0x81, 0x04); /* Opcode 81 /4 */
9718   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
9719   ins_pipe(ialu_reg);
9720 %}
9721 
9722 // And Register with Memory
9723 instruct andL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
9724 %{
9725   match(Set dst (AndL dst (LoadL src)));
9726   effect(KILL cr);
9727 
9728   ins_cost(125);
9729   format %{ "andq    $dst, $src\t# long" %}
9730   opcode(0x23);
9731   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
9732   ins_pipe(ialu_reg_mem);
9733 %}
9734 
9735 // And Memory with Register
9736 instruct andL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
9737 %{
9738   match(Set dst (StoreL dst (AndL (LoadL dst) src)));
9739   effect(KILL cr);
9740 
9741   ins_cost(150);
9742   format %{ "andq    $dst, $src\t# long" %}
9743   opcode(0x21); /* Opcode 21 /r */
9744   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
9745   ins_pipe(ialu_mem_reg);
9746 %}
9747 
9748 // And Memory with Immediate
9749 instruct andL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
9750 %{
9751   match(Set dst (StoreL dst (AndL (LoadL dst) src)));
9752   effect(KILL cr);
9753 
9754   ins_cost(125);
9755   format %{ "andq    $dst, $src\t# long" %}
9756   opcode(0x81, 0x4); /* Opcode 81 /4 id */
9757   ins_encode(REX_mem_wide(dst), OpcSE(src),
9758              RM_opc_mem(secondary, dst), Con8or32(src));
9759   ins_pipe(ialu_mem_imm);
9760 %}
9761 
9762 // BMI1 instructions
9763 instruct andnL_rReg_rReg_mem(rRegL dst, rRegL src1, memory src2, immL_M1 minus_1, rFlagsReg cr) %{
9764   match(Set dst (AndL (XorL src1 minus_1) (LoadL src2)));
9765   predicate(UseBMI1Instructions);
9766   effect(KILL cr);
9767 
9768   ins_cost(125);
9769   format %{ "andnq  $dst, $src1, $src2" %}
9770 
9771   ins_encode %{
9772     __ andnq($dst$$Register, $src1$$Register, $src2$$Address);
9773   %}
9774   ins_pipe(ialu_reg_mem);
9775 %}
9776 
9777 instruct andnL_rReg_rReg_rReg(rRegL dst, rRegL src1, rRegL src2, immL_M1 minus_1, rFlagsReg cr) %{
9778   match(Set dst (AndL (XorL src1 minus_1) src2));
9779   predicate(UseBMI1Instructions);
9780   effect(KILL cr);
9781 
9782   format %{ "andnq  $dst, $src1, $src2" %}
9783 
9784   ins_encode %{
9785   __ andnq($dst$$Register, $src1$$Register, $src2$$Register);
9786   %}
9787   ins_pipe(ialu_reg_mem);
9788 %}
9789 
9790 instruct blsiL_rReg_rReg(rRegL dst, rRegL src, immL0 imm_zero, rFlagsReg cr) %{
9791   match(Set dst (AndL (SubL imm_zero src) src));
9792   predicate(UseBMI1Instructions);
9793   effect(KILL cr);
9794 
9795   format %{ "blsiq  $dst, $src" %}
9796 
9797   ins_encode %{
9798     __ blsiq($dst$$Register, $src$$Register);
9799   %}
9800   ins_pipe(ialu_reg);
9801 %}
9802 
9803 instruct blsiL_rReg_mem(rRegL dst, memory src, immL0 imm_zero, rFlagsReg cr) %{
9804   match(Set dst (AndL (SubL imm_zero (LoadL src) ) (LoadL src) ));
9805   predicate(UseBMI1Instructions);
9806   effect(KILL cr);
9807 
9808   ins_cost(125);
9809   format %{ "blsiq  $dst, $src" %}
9810 
9811   ins_encode %{
9812     __ blsiq($dst$$Register, $src$$Address);
9813   %}
9814   ins_pipe(ialu_reg_mem);
9815 %}
9816 
9817 instruct blsmskL_rReg_mem(rRegL dst, memory src, immL_M1 minus_1, rFlagsReg cr)
9818 %{
9819   match(Set dst (XorL (AddL (LoadL src) minus_1) (LoadL src) ) );
9820   predicate(UseBMI1Instructions);
9821   effect(KILL cr);
9822 
9823   ins_cost(125);
9824   format %{ "blsmskq $dst, $src" %}
9825 
9826   ins_encode %{
9827     __ blsmskq($dst$$Register, $src$$Address);
9828   %}
9829   ins_pipe(ialu_reg_mem);
9830 %}
9831 
9832 instruct blsmskL_rReg_rReg(rRegL dst, rRegL src, immL_M1 minus_1, rFlagsReg cr)
9833 %{
9834   match(Set dst (XorL (AddL src minus_1) src));
9835   predicate(UseBMI1Instructions);
9836   effect(KILL cr);
9837 
9838   format %{ "blsmskq $dst, $src" %}
9839 
9840   ins_encode %{
9841     __ blsmskq($dst$$Register, $src$$Register);
9842   %}
9843 
9844   ins_pipe(ialu_reg);
9845 %}
9846 
9847 instruct blsrL_rReg_rReg(rRegL dst, rRegL src, immL_M1 minus_1, rFlagsReg cr)
9848 %{
9849   match(Set dst (AndL (AddL src minus_1) src) );
9850   predicate(UseBMI1Instructions);
9851   effect(KILL cr);
9852 
9853   format %{ "blsrq  $dst, $src" %}
9854 
9855   ins_encode %{
9856     __ blsrq($dst$$Register, $src$$Register);
9857   %}
9858 
9859   ins_pipe(ialu_reg);
9860 %}
9861 
9862 instruct blsrL_rReg_mem(rRegL dst, memory src, immL_M1 minus_1, rFlagsReg cr)
9863 %{
9864   match(Set dst (AndL (AddL (LoadL src) minus_1) (LoadL src)) );
9865   predicate(UseBMI1Instructions);
9866   effect(KILL cr);
9867 
9868   ins_cost(125);
9869   format %{ "blsrq  $dst, $src" %}
9870 
9871   ins_encode %{
9872     __ blsrq($dst$$Register, $src$$Address);
9873   %}
9874 
9875   ins_pipe(ialu_reg);
9876 %}
9877 
9878 // Or Instructions
9879 // Or Register with Register
9880 instruct orL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
9881 %{
9882   match(Set dst (OrL dst src));
9883   effect(KILL cr);
9884 
9885   format %{ "orq     $dst, $src\t# long" %}
9886   opcode(0x0B);
9887   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
9888   ins_pipe(ialu_reg_reg);
9889 %}
9890 
9891 // Use any_RegP to match R15 (TLS register) without spilling.
9892 instruct orL_rReg_castP2X(rRegL dst, any_RegP src, rFlagsReg cr) %{
9893   match(Set dst (OrL dst (CastP2X src)));
9894   effect(KILL cr);
9895 
9896   format %{ "orq     $dst, $src\t# long" %}
9897   opcode(0x0B);
9898   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
9899   ins_pipe(ialu_reg_reg);
9900 %}
9901 
9902 
9903 // Or Register with Immediate
9904 instruct orL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
9905 %{
9906   match(Set dst (OrL dst src));
9907   effect(KILL cr);
9908 
9909   format %{ "orq     $dst, $src\t# long" %}
9910   opcode(0x81, 0x01); /* Opcode 81 /1 id */
9911   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
9912   ins_pipe(ialu_reg);
9913 %}
9914 
9915 // Or Register with Memory
9916 instruct orL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
9917 %{
9918   match(Set dst (OrL dst (LoadL src)));
9919   effect(KILL cr);
9920 
9921   ins_cost(125);
9922   format %{ "orq     $dst, $src\t# long" %}
9923   opcode(0x0B);
9924   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
9925   ins_pipe(ialu_reg_mem);
9926 %}
9927 
9928 // Or Memory with Register
9929 instruct orL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
9930 %{
9931   match(Set dst (StoreL dst (OrL (LoadL dst) src)));
9932   effect(KILL cr);
9933 
9934   ins_cost(150);
9935   format %{ "orq     $dst, $src\t# long" %}
9936   opcode(0x09); /* Opcode 09 /r */
9937   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
9938   ins_pipe(ialu_mem_reg);
9939 %}
9940 
9941 // Or Memory with Immediate
9942 instruct orL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
9943 %{
9944   match(Set dst (StoreL dst (OrL (LoadL dst) src)));
9945   effect(KILL cr);
9946 
9947   ins_cost(125);
9948   format %{ "orq     $dst, $src\t# long" %}
9949   opcode(0x81, 0x1); /* Opcode 81 /1 id */
9950   ins_encode(REX_mem_wide(dst), OpcSE(src),
9951              RM_opc_mem(secondary, dst), Con8or32(src));
9952   ins_pipe(ialu_mem_imm);
9953 %}
9954 
9955 // Xor Instructions
9956 // Xor Register with Register
9957 instruct xorL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
9958 %{
9959   match(Set dst (XorL dst src));
9960   effect(KILL cr);
9961 
9962   format %{ "xorq    $dst, $src\t# long" %}
9963   opcode(0x33);
9964   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
9965   ins_pipe(ialu_reg_reg);
9966 %}
9967 
9968 // Xor Register with Immediate -1
9969 instruct xorL_rReg_im1(rRegL dst, immL_M1 imm) %{
9970   match(Set dst (XorL dst imm));
9971 
9972   format %{ "notq   $dst" %}
9973   ins_encode %{
9974      __ notq($dst$$Register);
9975   %}
9976   ins_pipe(ialu_reg);
9977 %}
9978 
9979 // Xor Register with Immediate
9980 instruct xorL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
9981 %{
9982   match(Set dst (XorL dst src));
9983   effect(KILL cr);
9984 
9985   format %{ "xorq    $dst, $src\t# long" %}
9986   opcode(0x81, 0x06); /* Opcode 81 /6 id */
9987   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
9988   ins_pipe(ialu_reg);
9989 %}
9990 
9991 // Xor Register with Memory
9992 instruct xorL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
9993 %{
9994   match(Set dst (XorL dst (LoadL src)));
9995   effect(KILL cr);
9996 
9997   ins_cost(125);
9998   format %{ "xorq    $dst, $src\t# long" %}
9999   opcode(0x33);
10000   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
10001   ins_pipe(ialu_reg_mem);
10002 %}
10003 
10004 // Xor Memory with Register
10005 instruct xorL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
10006 %{
10007   match(Set dst (StoreL dst (XorL (LoadL dst) src)));
10008   effect(KILL cr);
10009 
10010   ins_cost(150);
10011   format %{ "xorq    $dst, $src\t# long" %}
10012   opcode(0x31); /* Opcode 31 /r */
10013   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
10014   ins_pipe(ialu_mem_reg);
10015 %}
10016 
10017 // Xor Memory with Immediate
10018 instruct xorL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
10019 %{
10020   match(Set dst (StoreL dst (XorL (LoadL dst) src)));
10021   effect(KILL cr);
10022 
10023   ins_cost(125);
10024   format %{ "xorq    $dst, $src\t# long" %}
10025   opcode(0x81, 0x6); /* Opcode 81 /6 id */
10026   ins_encode(REX_mem_wide(dst), OpcSE(src),
10027              RM_opc_mem(secondary, dst), Con8or32(src));
10028   ins_pipe(ialu_mem_imm);
10029 %}
10030 
10031 // Convert Int to Boolean
10032 instruct convI2B(rRegI dst, rRegI src, rFlagsReg cr)
10033 %{
10034   match(Set dst (Conv2B src));
10035   effect(KILL cr);
10036 
10037   format %{ "testl   $src, $src\t# ci2b\n\t"
10038             "setnz   $dst\n\t"
10039             "movzbl  $dst, $dst" %}
10040   ins_encode(REX_reg_reg(src, src), opc_reg_reg(0x85, src, src), // testl
10041              setNZ_reg(dst),
10042              REX_reg_breg(dst, dst), // movzbl
10043              Opcode(0x0F), Opcode(0xB6), reg_reg(dst, dst));
10044   ins_pipe(pipe_slow); // XXX
10045 %}
10046 
10047 // Convert Pointer to Boolean
10048 instruct convP2B(rRegI dst, rRegP src, rFlagsReg cr)
10049 %{
10050   match(Set dst (Conv2B src));
10051   effect(KILL cr);
10052 
10053   format %{ "testq   $src, $src\t# cp2b\n\t"
10054             "setnz   $dst\n\t"
10055             "movzbl  $dst, $dst" %}
10056   ins_encode(REX_reg_reg_wide(src, src), opc_reg_reg(0x85, src, src), // testq
10057              setNZ_reg(dst),
10058              REX_reg_breg(dst, dst), // movzbl
10059              Opcode(0x0F), Opcode(0xB6), reg_reg(dst, dst));
10060   ins_pipe(pipe_slow); // XXX
10061 %}
10062 
10063 instruct cmpLTMask(rRegI dst, rRegI p, rRegI q, rFlagsReg cr)
10064 %{
10065   match(Set dst (CmpLTMask p q));
10066   effect(KILL cr);
10067 
10068   ins_cost(400);
10069   format %{ "cmpl    $p, $q\t# cmpLTMask\n\t"
10070             "setlt   $dst\n\t"
10071             "movzbl  $dst, $dst\n\t"
10072             "negl    $dst" %}
10073   ins_encode(REX_reg_reg(p, q), opc_reg_reg(0x3B, p, q), // cmpl
10074              setLT_reg(dst),
10075              REX_reg_breg(dst, dst), // movzbl
10076              Opcode(0x0F), Opcode(0xB6), reg_reg(dst, dst),
10077              neg_reg(dst));
10078   ins_pipe(pipe_slow);
10079 %}
10080 
10081 instruct cmpLTMask0(rRegI dst, immI0 zero, rFlagsReg cr)
10082 %{
10083   match(Set dst (CmpLTMask dst zero));
10084   effect(KILL cr);
10085 
10086   ins_cost(100);
10087   format %{ "sarl    $dst, #31\t# cmpLTMask0" %}
10088   ins_encode %{
10089   __ sarl($dst$$Register, 31);
10090   %}
10091   ins_pipe(ialu_reg);
10092 %}
10093 
10094 /* Better to save a register than avoid a branch */
10095 instruct cadd_cmpLTMask(rRegI p, rRegI q, rRegI y, rFlagsReg cr)
10096 %{
10097   match(Set p (AddI (AndI (CmpLTMask p q) y) (SubI p q)));
10098   effect(KILL cr);
10099   ins_cost(300);
10100   format %{ "subl   $p,$q\t# cadd_cmpLTMask\n\t"
10101             "jge    done\n\t"
10102             "addl   $p,$y\n"
10103             "done:  " %}
10104   ins_encode %{
10105     Register Rp = $p$$Register;
10106     Register Rq = $q$$Register;
10107     Register Ry = $y$$Register;
10108     Label done;
10109     __ subl(Rp, Rq);
10110     __ jccb(Assembler::greaterEqual, done);
10111     __ addl(Rp, Ry);
10112     __ bind(done);
10113   %}
10114   ins_pipe(pipe_cmplt);
10115 %}
10116 
10117 /* Better to save a register than avoid a branch */
10118 instruct and_cmpLTMask(rRegI p, rRegI q, rRegI y, rFlagsReg cr)
10119 %{
10120   match(Set y (AndI (CmpLTMask p q) y));
10121   effect(KILL cr);
10122 
10123   ins_cost(300);
10124 
10125   format %{ "cmpl     $p, $q\t# and_cmpLTMask\n\t"
10126             "jlt      done\n\t"
10127             "xorl     $y, $y\n"
10128             "done:  " %}
10129   ins_encode %{
10130     Register Rp = $p$$Register;
10131     Register Rq = $q$$Register;
10132     Register Ry = $y$$Register;
10133     Label done;
10134     __ cmpl(Rp, Rq);
10135     __ jccb(Assembler::less, done);
10136     __ xorl(Ry, Ry);
10137     __ bind(done);
10138   %}
10139   ins_pipe(pipe_cmplt);
10140 %}
10141 
10142 
10143 //---------- FP Instructions------------------------------------------------
10144 
10145 instruct cmpF_cc_reg(rFlagsRegU cr, regF src1, regF src2)
10146 %{
10147   match(Set cr (CmpF src1 src2));
10148 
10149   ins_cost(145);
10150   format %{ "ucomiss $src1, $src2\n\t"
10151             "jnp,s   exit\n\t"
10152             "pushfq\t# saw NaN, set CF\n\t"
10153             "andq    [rsp], #0xffffff2b\n\t"
10154             "popfq\n"
10155     "exit:" %}
10156   ins_encode %{
10157     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
10158     emit_cmpfp_fixup(_masm);
10159   %}
10160   ins_pipe(pipe_slow);
10161 %}
10162 
10163 instruct cmpF_cc_reg_CF(rFlagsRegUCF cr, regF src1, regF src2) %{
10164   match(Set cr (CmpF src1 src2));
10165 
10166   ins_cost(100);
10167   format %{ "ucomiss $src1, $src2" %}
10168   ins_encode %{
10169     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
10170   %}
10171   ins_pipe(pipe_slow);
10172 %}
10173 
10174 instruct cmpF_cc_mem(rFlagsRegU cr, regF src1, memory src2)
10175 %{
10176   match(Set cr (CmpF src1 (LoadF src2)));
10177 
10178   ins_cost(145);
10179   format %{ "ucomiss $src1, $src2\n\t"
10180             "jnp,s   exit\n\t"
10181             "pushfq\t# saw NaN, set CF\n\t"
10182             "andq    [rsp], #0xffffff2b\n\t"
10183             "popfq\n"
10184     "exit:" %}
10185   ins_encode %{
10186     __ ucomiss($src1$$XMMRegister, $src2$$Address);
10187     emit_cmpfp_fixup(_masm);
10188   %}
10189   ins_pipe(pipe_slow);
10190 %}
10191 
10192 instruct cmpF_cc_memCF(rFlagsRegUCF cr, regF src1, memory src2) %{
10193   match(Set cr (CmpF src1 (LoadF src2)));
10194 
10195   ins_cost(100);
10196   format %{ "ucomiss $src1, $src2" %}
10197   ins_encode %{
10198     __ ucomiss($src1$$XMMRegister, $src2$$Address);
10199   %}
10200   ins_pipe(pipe_slow);
10201 %}
10202 
10203 instruct cmpF_cc_imm(rFlagsRegU cr, regF src, immF con) %{
10204   match(Set cr (CmpF src con));
10205 
10206   ins_cost(145);
10207   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con\n\t"
10208             "jnp,s   exit\n\t"
10209             "pushfq\t# saw NaN, set CF\n\t"
10210             "andq    [rsp], #0xffffff2b\n\t"
10211             "popfq\n"
10212     "exit:" %}
10213   ins_encode %{
10214     __ ucomiss($src$$XMMRegister, $constantaddress($con));
10215     emit_cmpfp_fixup(_masm);
10216   %}
10217   ins_pipe(pipe_slow);
10218 %}
10219 
10220 instruct cmpF_cc_immCF(rFlagsRegUCF cr, regF src, immF con) %{
10221   match(Set cr (CmpF src con));
10222   ins_cost(100);
10223   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con" %}
10224   ins_encode %{
10225     __ ucomiss($src$$XMMRegister, $constantaddress($con));
10226   %}
10227   ins_pipe(pipe_slow);
10228 %}
10229 
10230 instruct cmpD_cc_reg(rFlagsRegU cr, regD src1, regD src2)
10231 %{
10232   match(Set cr (CmpD src1 src2));
10233 
10234   ins_cost(145);
10235   format %{ "ucomisd $src1, $src2\n\t"
10236             "jnp,s   exit\n\t"
10237             "pushfq\t# saw NaN, set CF\n\t"
10238             "andq    [rsp], #0xffffff2b\n\t"
10239             "popfq\n"
10240     "exit:" %}
10241   ins_encode %{
10242     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
10243     emit_cmpfp_fixup(_masm);
10244   %}
10245   ins_pipe(pipe_slow);
10246 %}
10247 
10248 instruct cmpD_cc_reg_CF(rFlagsRegUCF cr, regD src1, regD src2) %{
10249   match(Set cr (CmpD src1 src2));
10250 
10251   ins_cost(100);
10252   format %{ "ucomisd $src1, $src2 test" %}
10253   ins_encode %{
10254     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
10255   %}
10256   ins_pipe(pipe_slow);
10257 %}
10258 
10259 instruct cmpD_cc_mem(rFlagsRegU cr, regD src1, memory src2)
10260 %{
10261   match(Set cr (CmpD src1 (LoadD src2)));
10262 
10263   ins_cost(145);
10264   format %{ "ucomisd $src1, $src2\n\t"
10265             "jnp,s   exit\n\t"
10266             "pushfq\t# saw NaN, set CF\n\t"
10267             "andq    [rsp], #0xffffff2b\n\t"
10268             "popfq\n"
10269     "exit:" %}
10270   ins_encode %{
10271     __ ucomisd($src1$$XMMRegister, $src2$$Address);
10272     emit_cmpfp_fixup(_masm);
10273   %}
10274   ins_pipe(pipe_slow);
10275 %}
10276 
10277 instruct cmpD_cc_memCF(rFlagsRegUCF cr, regD src1, memory src2) %{
10278   match(Set cr (CmpD src1 (LoadD src2)));
10279 
10280   ins_cost(100);
10281   format %{ "ucomisd $src1, $src2" %}
10282   ins_encode %{
10283     __ ucomisd($src1$$XMMRegister, $src2$$Address);
10284   %}
10285   ins_pipe(pipe_slow);
10286 %}
10287 
10288 instruct cmpD_cc_imm(rFlagsRegU cr, regD src, immD con) %{
10289   match(Set cr (CmpD src con));
10290 
10291   ins_cost(145);
10292   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con\n\t"
10293             "jnp,s   exit\n\t"
10294             "pushfq\t# saw NaN, set CF\n\t"
10295             "andq    [rsp], #0xffffff2b\n\t"
10296             "popfq\n"
10297     "exit:" %}
10298   ins_encode %{
10299     __ ucomisd($src$$XMMRegister, $constantaddress($con));
10300     emit_cmpfp_fixup(_masm);
10301   %}
10302   ins_pipe(pipe_slow);
10303 %}
10304 
10305 instruct cmpD_cc_immCF(rFlagsRegUCF cr, regD src, immD con) %{
10306   match(Set cr (CmpD src con));
10307   ins_cost(100);
10308   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con" %}
10309   ins_encode %{
10310     __ ucomisd($src$$XMMRegister, $constantaddress($con));
10311   %}
10312   ins_pipe(pipe_slow);
10313 %}
10314 
10315 // Compare into -1,0,1
10316 instruct cmpF_reg(rRegI dst, regF src1, regF src2, rFlagsReg cr)
10317 %{
10318   match(Set dst (CmpF3 src1 src2));
10319   effect(KILL cr);
10320 
10321   ins_cost(275);
10322   format %{ "ucomiss $src1, $src2\n\t"
10323             "movl    $dst, #-1\n\t"
10324             "jp,s    done\n\t"
10325             "jb,s    done\n\t"
10326             "setne   $dst\n\t"
10327             "movzbl  $dst, $dst\n"
10328     "done:" %}
10329   ins_encode %{
10330     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
10331     emit_cmpfp3(_masm, $dst$$Register);
10332   %}
10333   ins_pipe(pipe_slow);
10334 %}
10335 
10336 // Compare into -1,0,1
10337 instruct cmpF_mem(rRegI dst, regF src1, memory src2, rFlagsReg cr)
10338 %{
10339   match(Set dst (CmpF3 src1 (LoadF src2)));
10340   effect(KILL cr);
10341 
10342   ins_cost(275);
10343   format %{ "ucomiss $src1, $src2\n\t"
10344             "movl    $dst, #-1\n\t"
10345             "jp,s    done\n\t"
10346             "jb,s    done\n\t"
10347             "setne   $dst\n\t"
10348             "movzbl  $dst, $dst\n"
10349     "done:" %}
10350   ins_encode %{
10351     __ ucomiss($src1$$XMMRegister, $src2$$Address);
10352     emit_cmpfp3(_masm, $dst$$Register);
10353   %}
10354   ins_pipe(pipe_slow);
10355 %}
10356 
10357 // Compare into -1,0,1
10358 instruct cmpF_imm(rRegI dst, regF src, immF con, rFlagsReg cr) %{
10359   match(Set dst (CmpF3 src con));
10360   effect(KILL cr);
10361 
10362   ins_cost(275);
10363   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con\n\t"
10364             "movl    $dst, #-1\n\t"
10365             "jp,s    done\n\t"
10366             "jb,s    done\n\t"
10367             "setne   $dst\n\t"
10368             "movzbl  $dst, $dst\n"
10369     "done:" %}
10370   ins_encode %{
10371     __ ucomiss($src$$XMMRegister, $constantaddress($con));
10372     emit_cmpfp3(_masm, $dst$$Register);
10373   %}
10374   ins_pipe(pipe_slow);
10375 %}
10376 
10377 // Compare into -1,0,1
10378 instruct cmpD_reg(rRegI dst, regD src1, regD src2, rFlagsReg cr)
10379 %{
10380   match(Set dst (CmpD3 src1 src2));
10381   effect(KILL cr);
10382 
10383   ins_cost(275);
10384   format %{ "ucomisd $src1, $src2\n\t"
10385             "movl    $dst, #-1\n\t"
10386             "jp,s    done\n\t"
10387             "jb,s    done\n\t"
10388             "setne   $dst\n\t"
10389             "movzbl  $dst, $dst\n"
10390     "done:" %}
10391   ins_encode %{
10392     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
10393     emit_cmpfp3(_masm, $dst$$Register);
10394   %}
10395   ins_pipe(pipe_slow);
10396 %}
10397 
10398 // Compare into -1,0,1
10399 instruct cmpD_mem(rRegI dst, regD src1, memory src2, rFlagsReg cr)
10400 %{
10401   match(Set dst (CmpD3 src1 (LoadD src2)));
10402   effect(KILL cr);
10403 
10404   ins_cost(275);
10405   format %{ "ucomisd $src1, $src2\n\t"
10406             "movl    $dst, #-1\n\t"
10407             "jp,s    done\n\t"
10408             "jb,s    done\n\t"
10409             "setne   $dst\n\t"
10410             "movzbl  $dst, $dst\n"
10411     "done:" %}
10412   ins_encode %{
10413     __ ucomisd($src1$$XMMRegister, $src2$$Address);
10414     emit_cmpfp3(_masm, $dst$$Register);
10415   %}
10416   ins_pipe(pipe_slow);
10417 %}
10418 
10419 // Compare into -1,0,1
10420 instruct cmpD_imm(rRegI dst, regD src, immD con, rFlagsReg cr) %{
10421   match(Set dst (CmpD3 src con));
10422   effect(KILL cr);
10423 
10424   ins_cost(275);
10425   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con\n\t"
10426             "movl    $dst, #-1\n\t"
10427             "jp,s    done\n\t"
10428             "jb,s    done\n\t"
10429             "setne   $dst\n\t"
10430             "movzbl  $dst, $dst\n"
10431     "done:" %}
10432   ins_encode %{
10433     __ ucomisd($src$$XMMRegister, $constantaddress($con));
10434     emit_cmpfp3(_masm, $dst$$Register);
10435   %}
10436   ins_pipe(pipe_slow);
10437 %}
10438 
10439 //----------Arithmetic Conversion Instructions---------------------------------
10440 
10441 instruct roundFloat_nop(regF dst)
10442 %{
10443   match(Set dst (RoundFloat dst));
10444 
10445   ins_cost(0);
10446   ins_encode();
10447   ins_pipe(empty);
10448 %}
10449 
10450 instruct roundDouble_nop(regD dst)
10451 %{
10452   match(Set dst (RoundDouble dst));
10453 
10454   ins_cost(0);
10455   ins_encode();
10456   ins_pipe(empty);
10457 %}
10458 
10459 instruct convF2D_reg_reg(regD dst, regF src)
10460 %{
10461   match(Set dst (ConvF2D src));
10462 
10463   format %{ "cvtss2sd $dst, $src" %}
10464   ins_encode %{
10465     __ cvtss2sd ($dst$$XMMRegister, $src$$XMMRegister);
10466   %}
10467   ins_pipe(pipe_slow); // XXX
10468 %}
10469 
10470 instruct convF2D_reg_mem(regD dst, memory src)
10471 %{
10472   match(Set dst (ConvF2D (LoadF src)));
10473 
10474   format %{ "cvtss2sd $dst, $src" %}
10475   ins_encode %{
10476     __ cvtss2sd ($dst$$XMMRegister, $src$$Address);
10477   %}
10478   ins_pipe(pipe_slow); // XXX
10479 %}
10480 
10481 instruct convD2F_reg_reg(regF dst, regD src)
10482 %{
10483   match(Set dst (ConvD2F src));
10484 
10485   format %{ "cvtsd2ss $dst, $src" %}
10486   ins_encode %{
10487     __ cvtsd2ss ($dst$$XMMRegister, $src$$XMMRegister);
10488   %}
10489   ins_pipe(pipe_slow); // XXX
10490 %}
10491 
10492 instruct convD2F_reg_mem(regF dst, memory src)
10493 %{
10494   match(Set dst (ConvD2F (LoadD src)));
10495 
10496   format %{ "cvtsd2ss $dst, $src" %}
10497   ins_encode %{
10498     __ cvtsd2ss ($dst$$XMMRegister, $src$$Address);
10499   %}
10500   ins_pipe(pipe_slow); // XXX
10501 %}
10502 
10503 // XXX do mem variants
10504 instruct convF2I_reg_reg(rRegI dst, regF src, rFlagsReg cr)
10505 %{
10506   match(Set dst (ConvF2I src));
10507   effect(KILL cr);
10508 
10509   format %{ "cvttss2sil $dst, $src\t# f2i\n\t"
10510             "cmpl    $dst, #0x80000000\n\t"
10511             "jne,s   done\n\t"
10512             "subq    rsp, #8\n\t"
10513             "movss   [rsp], $src\n\t"
10514             "call    f2i_fixup\n\t"
10515             "popq    $dst\n"
10516     "done:   "%}
10517   ins_encode %{
10518     Label done;
10519     __ cvttss2sil($dst$$Register, $src$$XMMRegister);
10520     __ cmpl($dst$$Register, 0x80000000);
10521     __ jccb(Assembler::notEqual, done);
10522     __ subptr(rsp, 8);
10523     __ movflt(Address(rsp, 0), $src$$XMMRegister);
10524     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::f2i_fixup())));
10525     __ pop($dst$$Register);
10526     __ bind(done);
10527   %}
10528   ins_pipe(pipe_slow);
10529 %}
10530 
10531 instruct convF2L_reg_reg(rRegL dst, regF src, rFlagsReg cr)
10532 %{
10533   match(Set dst (ConvF2L src));
10534   effect(KILL cr);
10535 
10536   format %{ "cvttss2siq $dst, $src\t# f2l\n\t"
10537             "cmpq    $dst, [0x8000000000000000]\n\t"
10538             "jne,s   done\n\t"
10539             "subq    rsp, #8\n\t"
10540             "movss   [rsp], $src\n\t"
10541             "call    f2l_fixup\n\t"
10542             "popq    $dst\n"
10543     "done:   "%}
10544   ins_encode %{
10545     Label done;
10546     __ cvttss2siq($dst$$Register, $src$$XMMRegister);
10547     __ cmp64($dst$$Register,
10548              ExternalAddress((address) StubRoutines::x86::double_sign_flip()));
10549     __ jccb(Assembler::notEqual, done);
10550     __ subptr(rsp, 8);
10551     __ movflt(Address(rsp, 0), $src$$XMMRegister);
10552     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::f2l_fixup())));
10553     __ pop($dst$$Register);
10554     __ bind(done);
10555   %}
10556   ins_pipe(pipe_slow);
10557 %}
10558 
10559 instruct convD2I_reg_reg(rRegI dst, regD src, rFlagsReg cr)
10560 %{
10561   match(Set dst (ConvD2I src));
10562   effect(KILL cr);
10563 
10564   format %{ "cvttsd2sil $dst, $src\t# d2i\n\t"
10565             "cmpl    $dst, #0x80000000\n\t"
10566             "jne,s   done\n\t"
10567             "subq    rsp, #8\n\t"
10568             "movsd   [rsp], $src\n\t"
10569             "call    d2i_fixup\n\t"
10570             "popq    $dst\n"
10571     "done:   "%}
10572   ins_encode %{
10573     Label done;
10574     __ cvttsd2sil($dst$$Register, $src$$XMMRegister);
10575     __ cmpl($dst$$Register, 0x80000000);
10576     __ jccb(Assembler::notEqual, done);
10577     __ subptr(rsp, 8);
10578     __ movdbl(Address(rsp, 0), $src$$XMMRegister);
10579     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::d2i_fixup())));
10580     __ pop($dst$$Register);
10581     __ bind(done);
10582   %}
10583   ins_pipe(pipe_slow);
10584 %}
10585 
10586 instruct convD2L_reg_reg(rRegL dst, regD src, rFlagsReg cr)
10587 %{
10588   match(Set dst (ConvD2L src));
10589   effect(KILL cr);
10590 
10591   format %{ "cvttsd2siq $dst, $src\t# d2l\n\t"
10592             "cmpq    $dst, [0x8000000000000000]\n\t"
10593             "jne,s   done\n\t"
10594             "subq    rsp, #8\n\t"
10595             "movsd   [rsp], $src\n\t"
10596             "call    d2l_fixup\n\t"
10597             "popq    $dst\n"
10598     "done:   "%}
10599   ins_encode %{
10600     Label done;
10601     __ cvttsd2siq($dst$$Register, $src$$XMMRegister);
10602     __ cmp64($dst$$Register,
10603              ExternalAddress((address) StubRoutines::x86::double_sign_flip()));
10604     __ jccb(Assembler::notEqual, done);
10605     __ subptr(rsp, 8);
10606     __ movdbl(Address(rsp, 0), $src$$XMMRegister);
10607     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::d2l_fixup())));
10608     __ pop($dst$$Register);
10609     __ bind(done);
10610   %}
10611   ins_pipe(pipe_slow);
10612 %}
10613 
10614 instruct convI2F_reg_reg(regF dst, rRegI src)
10615 %{
10616   predicate(!UseXmmI2F);
10617   match(Set dst (ConvI2F src));
10618 
10619   format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
10620   ins_encode %{
10621     __ cvtsi2ssl ($dst$$XMMRegister, $src$$Register);
10622   %}
10623   ins_pipe(pipe_slow); // XXX
10624 %}
10625 
10626 instruct convI2F_reg_mem(regF dst, memory src)
10627 %{
10628   match(Set dst (ConvI2F (LoadI src)));
10629 
10630   format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
10631   ins_encode %{
10632     __ cvtsi2ssl ($dst$$XMMRegister, $src$$Address);
10633   %}
10634   ins_pipe(pipe_slow); // XXX
10635 %}
10636 
10637 instruct convI2D_reg_reg(regD dst, rRegI src)
10638 %{
10639   predicate(!UseXmmI2D);
10640   match(Set dst (ConvI2D src));
10641 
10642   format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
10643   ins_encode %{
10644     __ cvtsi2sdl ($dst$$XMMRegister, $src$$Register);
10645   %}
10646   ins_pipe(pipe_slow); // XXX
10647 %}
10648 
10649 instruct convI2D_reg_mem(regD dst, memory src)
10650 %{
10651   match(Set dst (ConvI2D (LoadI src)));
10652 
10653   format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
10654   ins_encode %{
10655     __ cvtsi2sdl ($dst$$XMMRegister, $src$$Address);
10656   %}
10657   ins_pipe(pipe_slow); // XXX
10658 %}
10659 
10660 instruct convXI2F_reg(regF dst, rRegI src)
10661 %{
10662   predicate(UseXmmI2F);
10663   match(Set dst (ConvI2F src));
10664 
10665   format %{ "movdl $dst, $src\n\t"
10666             "cvtdq2psl $dst, $dst\t# i2f" %}
10667   ins_encode %{
10668     __ movdl($dst$$XMMRegister, $src$$Register);
10669     __ cvtdq2ps($dst$$XMMRegister, $dst$$XMMRegister);
10670   %}
10671   ins_pipe(pipe_slow); // XXX
10672 %}
10673 
10674 instruct convXI2D_reg(regD dst, rRegI src)
10675 %{
10676   predicate(UseXmmI2D);
10677   match(Set dst (ConvI2D src));
10678 
10679   format %{ "movdl $dst, $src\n\t"
10680             "cvtdq2pdl $dst, $dst\t# i2d" %}
10681   ins_encode %{
10682     __ movdl($dst$$XMMRegister, $src$$Register);
10683     __ cvtdq2pd($dst$$XMMRegister, $dst$$XMMRegister);
10684   %}
10685   ins_pipe(pipe_slow); // XXX
10686 %}
10687 
10688 instruct convL2F_reg_reg(regF dst, rRegL src)
10689 %{
10690   match(Set dst (ConvL2F src));
10691 
10692   format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
10693   ins_encode %{
10694     __ cvtsi2ssq ($dst$$XMMRegister, $src$$Register);
10695   %}
10696   ins_pipe(pipe_slow); // XXX
10697 %}
10698 
10699 instruct convL2F_reg_mem(regF dst, memory src)
10700 %{
10701   match(Set dst (ConvL2F (LoadL src)));
10702 
10703   format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
10704   ins_encode %{
10705     __ cvtsi2ssq ($dst$$XMMRegister, $src$$Address);
10706   %}
10707   ins_pipe(pipe_slow); // XXX
10708 %}
10709 
10710 instruct convL2D_reg_reg(regD dst, rRegL src)
10711 %{
10712   match(Set dst (ConvL2D src));
10713 
10714   format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
10715   ins_encode %{
10716     __ cvtsi2sdq ($dst$$XMMRegister, $src$$Register);
10717   %}
10718   ins_pipe(pipe_slow); // XXX
10719 %}
10720 
10721 instruct convL2D_reg_mem(regD dst, memory src)
10722 %{
10723   match(Set dst (ConvL2D (LoadL src)));
10724 
10725   format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
10726   ins_encode %{
10727     __ cvtsi2sdq ($dst$$XMMRegister, $src$$Address);
10728   %}
10729   ins_pipe(pipe_slow); // XXX
10730 %}
10731 
10732 instruct convI2L_reg_reg(rRegL dst, rRegI src)
10733 %{
10734   match(Set dst (ConvI2L src));
10735 
10736   ins_cost(125);
10737   format %{ "movslq  $dst, $src\t# i2l" %}
10738   ins_encode %{
10739     __ movslq($dst$$Register, $src$$Register);
10740   %}
10741   ins_pipe(ialu_reg_reg);
10742 %}
10743 
10744 // instruct convI2L_reg_reg_foo(rRegL dst, rRegI src)
10745 // %{
10746 //   match(Set dst (ConvI2L src));
10747 // //   predicate(_kids[0]->_leaf->as_Type()->type()->is_int()->_lo >= 0 &&
10748 // //             _kids[0]->_leaf->as_Type()->type()->is_int()->_hi >= 0);
10749 //   predicate(((const TypeNode*) n)->type()->is_long()->_hi ==
10750 //             (unsigned int) ((const TypeNode*) n)->type()->is_long()->_hi &&
10751 //             ((const TypeNode*) n)->type()->is_long()->_lo ==
10752 //             (unsigned int) ((const TypeNode*) n)->type()->is_long()->_lo);
10753 
10754 //   format %{ "movl    $dst, $src\t# unsigned i2l" %}
10755 //   ins_encode(enc_copy(dst, src));
10756 // //   opcode(0x63); // needs REX.W
10757 // //   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst,src));
10758 //   ins_pipe(ialu_reg_reg);
10759 // %}
10760 
10761 // Zero-extend convert int to long
10762 instruct convI2L_reg_reg_zex(rRegL dst, rRegI src, immL_32bits mask)
10763 %{
10764   match(Set dst (AndL (ConvI2L src) mask));
10765 
10766   format %{ "movl    $dst, $src\t# i2l zero-extend\n\t" %}
10767   ins_encode %{
10768     if ($dst$$reg != $src$$reg) {
10769       __ movl($dst$$Register, $src$$Register);
10770     }
10771   %}
10772   ins_pipe(ialu_reg_reg);
10773 %}
10774 
10775 // Zero-extend convert int to long
10776 instruct convI2L_reg_mem_zex(rRegL dst, memory src, immL_32bits mask)
10777 %{
10778   match(Set dst (AndL (ConvI2L (LoadI src)) mask));
10779 
10780   format %{ "movl    $dst, $src\t# i2l zero-extend\n\t" %}
10781   ins_encode %{
10782     __ movl($dst$$Register, $src$$Address);
10783   %}
10784   ins_pipe(ialu_reg_mem);
10785 %}
10786 
10787 instruct zerox_long_reg_reg(rRegL dst, rRegL src, immL_32bits mask)
10788 %{
10789   match(Set dst (AndL src mask));
10790 
10791   format %{ "movl    $dst, $src\t# zero-extend long" %}
10792   ins_encode %{
10793     __ movl($dst$$Register, $src$$Register);
10794   %}
10795   ins_pipe(ialu_reg_reg);
10796 %}
10797 
10798 instruct convL2I_reg_reg(rRegI dst, rRegL src)
10799 %{
10800   match(Set dst (ConvL2I src));
10801 
10802   format %{ "movl    $dst, $src\t# l2i" %}
10803   ins_encode %{
10804     __ movl($dst$$Register, $src$$Register);
10805   %}
10806   ins_pipe(ialu_reg_reg);
10807 %}
10808 
10809 
10810 instruct MoveF2I_stack_reg(rRegI dst, stackSlotF src) %{
10811   match(Set dst (MoveF2I src));
10812   effect(DEF dst, USE src);
10813 
10814   ins_cost(125);
10815   format %{ "movl    $dst, $src\t# MoveF2I_stack_reg" %}
10816   ins_encode %{
10817     __ movl($dst$$Register, Address(rsp, $src$$disp));
10818   %}
10819   ins_pipe(ialu_reg_mem);
10820 %}
10821 
10822 instruct MoveI2F_stack_reg(regF dst, stackSlotI src) %{
10823   match(Set dst (MoveI2F src));
10824   effect(DEF dst, USE src);
10825 
10826   ins_cost(125);
10827   format %{ "movss   $dst, $src\t# MoveI2F_stack_reg" %}
10828   ins_encode %{
10829     __ movflt($dst$$XMMRegister, Address(rsp, $src$$disp));
10830   %}
10831   ins_pipe(pipe_slow);
10832 %}
10833 
10834 instruct MoveD2L_stack_reg(rRegL dst, stackSlotD src) %{
10835   match(Set dst (MoveD2L src));
10836   effect(DEF dst, USE src);
10837 
10838   ins_cost(125);
10839   format %{ "movq    $dst, $src\t# MoveD2L_stack_reg" %}
10840   ins_encode %{
10841     __ movq($dst$$Register, Address(rsp, $src$$disp));
10842   %}
10843   ins_pipe(ialu_reg_mem);
10844 %}
10845 
10846 instruct MoveL2D_stack_reg_partial(regD dst, stackSlotL src) %{
10847   predicate(!UseXmmLoadAndClearUpper);
10848   match(Set dst (MoveL2D src));
10849   effect(DEF dst, USE src);
10850 
10851   ins_cost(125);
10852   format %{ "movlpd  $dst, $src\t# MoveL2D_stack_reg" %}
10853   ins_encode %{
10854     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
10855   %}
10856   ins_pipe(pipe_slow);
10857 %}
10858 
10859 instruct MoveL2D_stack_reg(regD dst, stackSlotL src) %{
10860   predicate(UseXmmLoadAndClearUpper);
10861   match(Set dst (MoveL2D src));
10862   effect(DEF dst, USE src);
10863 
10864   ins_cost(125);
10865   format %{ "movsd   $dst, $src\t# MoveL2D_stack_reg" %}
10866   ins_encode %{
10867     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
10868   %}
10869   ins_pipe(pipe_slow);
10870 %}
10871 
10872 
10873 instruct MoveF2I_reg_stack(stackSlotI dst, regF src) %{
10874   match(Set dst (MoveF2I src));
10875   effect(DEF dst, USE src);
10876 
10877   ins_cost(95); // XXX
10878   format %{ "movss   $dst, $src\t# MoveF2I_reg_stack" %}
10879   ins_encode %{
10880     __ movflt(Address(rsp, $dst$$disp), $src$$XMMRegister);
10881   %}
10882   ins_pipe(pipe_slow);
10883 %}
10884 
10885 instruct MoveI2F_reg_stack(stackSlotF dst, rRegI src) %{
10886   match(Set dst (MoveI2F src));
10887   effect(DEF dst, USE src);
10888 
10889   ins_cost(100);
10890   format %{ "movl    $dst, $src\t# MoveI2F_reg_stack" %}
10891   ins_encode %{
10892     __ movl(Address(rsp, $dst$$disp), $src$$Register);
10893   %}
10894   ins_pipe( ialu_mem_reg );
10895 %}
10896 
10897 instruct MoveD2L_reg_stack(stackSlotL dst, regD src) %{
10898   match(Set dst (MoveD2L src));
10899   effect(DEF dst, USE src);
10900 
10901   ins_cost(95); // XXX
10902   format %{ "movsd   $dst, $src\t# MoveL2D_reg_stack" %}
10903   ins_encode %{
10904     __ movdbl(Address(rsp, $dst$$disp), $src$$XMMRegister);
10905   %}
10906   ins_pipe(pipe_slow);
10907 %}
10908 
10909 instruct MoveL2D_reg_stack(stackSlotD dst, rRegL src) %{
10910   match(Set dst (MoveL2D src));
10911   effect(DEF dst, USE src);
10912 
10913   ins_cost(100);
10914   format %{ "movq    $dst, $src\t# MoveL2D_reg_stack" %}
10915   ins_encode %{
10916     __ movq(Address(rsp, $dst$$disp), $src$$Register);
10917   %}
10918   ins_pipe(ialu_mem_reg);
10919 %}
10920 
10921 instruct MoveF2I_reg_reg(rRegI dst, regF src) %{
10922   match(Set dst (MoveF2I src));
10923   effect(DEF dst, USE src);
10924   ins_cost(85);
10925   format %{ "movd    $dst,$src\t# MoveF2I" %}
10926   ins_encode %{
10927     __ movdl($dst$$Register, $src$$XMMRegister);
10928   %}
10929   ins_pipe( pipe_slow );
10930 %}
10931 
10932 instruct MoveD2L_reg_reg(rRegL dst, regD src) %{
10933   match(Set dst (MoveD2L src));
10934   effect(DEF dst, USE src);
10935   ins_cost(85);
10936   format %{ "movd    $dst,$src\t# MoveD2L" %}
10937   ins_encode %{
10938     __ movdq($dst$$Register, $src$$XMMRegister);
10939   %}
10940   ins_pipe( pipe_slow );
10941 %}
10942 
10943 instruct MoveI2F_reg_reg(regF dst, rRegI src) %{
10944   match(Set dst (MoveI2F src));
10945   effect(DEF dst, USE src);
10946   ins_cost(100);
10947   format %{ "movd    $dst,$src\t# MoveI2F" %}
10948   ins_encode %{
10949     __ movdl($dst$$XMMRegister, $src$$Register);
10950   %}
10951   ins_pipe( pipe_slow );
10952 %}
10953 
10954 instruct MoveL2D_reg_reg(regD dst, rRegL src) %{
10955   match(Set dst (MoveL2D src));
10956   effect(DEF dst, USE src);
10957   ins_cost(100);
10958   format %{ "movd    $dst,$src\t# MoveL2D" %}
10959   ins_encode %{
10960      __ movdq($dst$$XMMRegister, $src$$Register);
10961   %}
10962   ins_pipe( pipe_slow );
10963 %}
10964 
10965 
10966 // =======================================================================
10967 // fast clearing of an array
10968 instruct rep_stos(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegI zero,
10969                   Universe dummy, rFlagsReg cr)
10970 %{
10971   predicate(!((ClearArrayNode*)n)->is_large());
10972   match(Set dummy (ClearArray cnt base));
10973   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, KILL zero, KILL cr);
10974 
10975   format %{ $$template
10976     $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
10977     $$emit$$"cmp     InitArrayShortSize,rcx\n\t"
10978     $$emit$$"jg      LARGE\n\t"
10979     $$emit$$"dec     rcx\n\t"
10980     $$emit$$"js      DONE\t# Zero length\n\t"
10981     $$emit$$"mov     rax,(rdi,rcx,8)\t# LOOP\n\t"
10982     $$emit$$"dec     rcx\n\t"
10983     $$emit$$"jge     LOOP\n\t"
10984     $$emit$$"jmp     DONE\n\t"
10985     $$emit$$"# LARGE:\n\t"
10986     if (UseFastStosb) {
10987        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
10988        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--\n\t"
10989     } else if (UseXMMForObjInit) {
10990        $$emit$$"mov     rdi,rax\n\t"
10991        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
10992        $$emit$$"jmpq    L_zero_64_bytes\n\t"
10993        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
10994        $$emit$$"vmovdqu ymm0,(rax)\n\t"
10995        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
10996        $$emit$$"add     0x40,rax\n\t"
10997        $$emit$$"# L_zero_64_bytes:\n\t"
10998        $$emit$$"sub     0x8,rcx\n\t"
10999        $$emit$$"jge     L_loop\n\t"
11000        $$emit$$"add     0x4,rcx\n\t"
11001        $$emit$$"jl      L_tail\n\t"
11002        $$emit$$"vmovdqu ymm0,(rax)\n\t"
11003        $$emit$$"add     0x20,rax\n\t"
11004        $$emit$$"sub     0x4,rcx\n\t"
11005        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
11006        $$emit$$"add     0x4,rcx\n\t"
11007        $$emit$$"jle     L_end\n\t"
11008        $$emit$$"dec     rcx\n\t"
11009        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
11010        $$emit$$"vmovq   xmm0,(rax)\n\t"
11011        $$emit$$"add     0x8,rax\n\t"
11012        $$emit$$"dec     rcx\n\t"
11013        $$emit$$"jge     L_sloop\n\t"
11014        $$emit$$"# L_end:\n\t"
11015     } else {
11016        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--\n\t"
11017     }
11018     $$emit$$"# DONE"
11019   %}
11020   ins_encode %{
11021     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
11022                  $tmp$$XMMRegister, false);
11023   %}
11024   ins_pipe(pipe_slow);
11025 %}
11026 
11027 instruct rep_stos_large(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegI zero, 
11028                         Universe dummy, rFlagsReg cr)
11029 %{
11030   predicate(((ClearArrayNode*)n)->is_large());
11031   match(Set dummy (ClearArray cnt base));
11032   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, KILL zero, KILL cr);
11033 
11034   format %{ $$template
11035     if (UseFastStosb) {
11036        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
11037        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
11038        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--"
11039     } else if (UseXMMForObjInit) {
11040        $$emit$$"mov     rdi,rax\t# ClearArray:\n\t"
11041        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
11042        $$emit$$"jmpq    L_zero_64_bytes\n\t"
11043        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
11044        $$emit$$"vmovdqu ymm0,(rax)\n\t"
11045        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
11046        $$emit$$"add     0x40,rax\n\t"
11047        $$emit$$"# L_zero_64_bytes:\n\t"
11048        $$emit$$"sub     0x8,rcx\n\t"
11049        $$emit$$"jge     L_loop\n\t"
11050        $$emit$$"add     0x4,rcx\n\t"
11051        $$emit$$"jl      L_tail\n\t"
11052        $$emit$$"vmovdqu ymm0,(rax)\n\t"
11053        $$emit$$"add     0x20,rax\n\t"
11054        $$emit$$"sub     0x4,rcx\n\t"
11055        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
11056        $$emit$$"add     0x4,rcx\n\t"
11057        $$emit$$"jle     L_end\n\t"
11058        $$emit$$"dec     rcx\n\t"
11059        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
11060        $$emit$$"vmovq   xmm0,(rax)\n\t"
11061        $$emit$$"add     0x8,rax\n\t"
11062        $$emit$$"dec     rcx\n\t"
11063        $$emit$$"jge     L_sloop\n\t"
11064        $$emit$$"# L_end:\n\t"
11065     } else {
11066        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
11067        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--"
11068     }
11069   %}
11070   ins_encode %{
11071     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register, 
11072                  $tmp$$XMMRegister, true);
11073   %}
11074   ins_pipe(pipe_slow);
11075 %}
11076 
11077 instruct string_compareL(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
11078                          rax_RegI result, legVecS tmp1, rFlagsReg cr)
11079 %{
11080   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL);
11081   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11082   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11083 
11084   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11085   ins_encode %{
11086     __ string_compare($str1$$Register, $str2$$Register,
11087                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
11088                       $tmp1$$XMMRegister, StrIntrinsicNode::LL);
11089   %}
11090   ins_pipe( pipe_slow );
11091 %}
11092 
11093 instruct string_compareU(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
11094                          rax_RegI result, legVecS tmp1, rFlagsReg cr)
11095 %{
11096   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU);
11097   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11098   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11099 
11100   format %{ "String Compare char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11101   ins_encode %{
11102     __ string_compare($str1$$Register, $str2$$Register,
11103                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
11104                       $tmp1$$XMMRegister, StrIntrinsicNode::UU);
11105   %}
11106   ins_pipe( pipe_slow );
11107 %}
11108 
11109 instruct string_compareLU(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
11110                           rax_RegI result, legVecS tmp1, rFlagsReg cr)
11111 %{
11112   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU);
11113   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11114   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11115 
11116   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11117   ins_encode %{
11118     __ string_compare($str1$$Register, $str2$$Register,
11119                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
11120                       $tmp1$$XMMRegister, StrIntrinsicNode::LU);
11121   %}
11122   ins_pipe( pipe_slow );
11123 %}
11124 
11125 instruct string_compareUL(rsi_RegP str1, rdx_RegI cnt1, rdi_RegP str2, rcx_RegI cnt2,
11126                           rax_RegI result, legVecS tmp1, rFlagsReg cr)
11127 %{
11128   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL);
11129   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11130   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11131 
11132   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11133   ins_encode %{
11134     __ string_compare($str2$$Register, $str1$$Register,
11135                       $cnt2$$Register, $cnt1$$Register, $result$$Register,
11136                       $tmp1$$XMMRegister, StrIntrinsicNode::UL);
11137   %}
11138   ins_pipe( pipe_slow );
11139 %}
11140 
11141 // fast search of substring with known size.
11142 instruct string_indexof_conL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
11143                              rbx_RegI result, legVecS vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
11144 %{
11145   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL));
11146   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
11147   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
11148 
11149   format %{ "String IndexOf byte[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $vec, $cnt1, $cnt2, $tmp" %}
11150   ins_encode %{
11151     int icnt2 = (int)$int_cnt2$$constant;
11152     if (icnt2 >= 16) {
11153       // IndexOf for constant substrings with size >= 16 elements
11154       // which don't need to be loaded through stack.
11155       __ string_indexofC8($str1$$Register, $str2$$Register,
11156                           $cnt1$$Register, $cnt2$$Register,
11157                           icnt2, $result$$Register,
11158                           $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
11159     } else {
11160       // Small strings are loaded through stack if they cross page boundary.
11161       __ string_indexof($str1$$Register, $str2$$Register,
11162                         $cnt1$$Register, $cnt2$$Register,
11163                         icnt2, $result$$Register,
11164                         $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
11165     }
11166   %}
11167   ins_pipe( pipe_slow );
11168 %}
11169 
11170 // fast search of substring with known size.
11171 instruct string_indexof_conU(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
11172                              rbx_RegI result, legVecS vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
11173 %{
11174   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU));
11175   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
11176   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
11177 
11178   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $vec, $cnt1, $cnt2, $tmp" %}
11179   ins_encode %{
11180     int icnt2 = (int)$int_cnt2$$constant;
11181     if (icnt2 >= 8) {
11182       // IndexOf for constant substrings with size >= 8 elements
11183       // which don't need to be loaded through stack.
11184       __ string_indexofC8($str1$$Register, $str2$$Register,
11185                           $cnt1$$Register, $cnt2$$Register,
11186                           icnt2, $result$$Register,
11187                           $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
11188     } else {
11189       // Small strings are loaded through stack if they cross page boundary.
11190       __ string_indexof($str1$$Register, $str2$$Register,
11191                         $cnt1$$Register, $cnt2$$Register,
11192                         icnt2, $result$$Register,
11193                         $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
11194     }
11195   %}
11196   ins_pipe( pipe_slow );
11197 %}
11198 
11199 // fast search of substring with known size.
11200 instruct string_indexof_conUL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
11201                              rbx_RegI result, legVecS vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
11202 %{
11203   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL));
11204   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
11205   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
11206 
11207   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $vec, $cnt1, $cnt2, $tmp" %}
11208   ins_encode %{
11209     int icnt2 = (int)$int_cnt2$$constant;
11210     if (icnt2 >= 8) {
11211       // IndexOf for constant substrings with size >= 8 elements
11212       // which don't need to be loaded through stack.
11213       __ string_indexofC8($str1$$Register, $str2$$Register,
11214                           $cnt1$$Register, $cnt2$$Register,
11215                           icnt2, $result$$Register,
11216                           $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
11217     } else {
11218       // Small strings are loaded through stack if they cross page boundary.
11219       __ string_indexof($str1$$Register, $str2$$Register,
11220                         $cnt1$$Register, $cnt2$$Register,
11221                         icnt2, $result$$Register,
11222                         $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
11223     }
11224   %}
11225   ins_pipe( pipe_slow );
11226 %}
11227 
11228 instruct string_indexofL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
11229                          rbx_RegI result, legVecS vec, rcx_RegI tmp, rFlagsReg cr)
11230 %{
11231   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL));
11232   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
11233   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
11234 
11235   format %{ "String IndexOf byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
11236   ins_encode %{
11237     __ string_indexof($str1$$Register, $str2$$Register,
11238                       $cnt1$$Register, $cnt2$$Register,
11239                       (-1), $result$$Register,
11240                       $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
11241   %}
11242   ins_pipe( pipe_slow );
11243 %}
11244 
11245 instruct string_indexofU(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
11246                          rbx_RegI result, legVecS vec, rcx_RegI tmp, rFlagsReg cr)
11247 %{
11248   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU));
11249   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
11250   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
11251 
11252   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
11253   ins_encode %{
11254     __ string_indexof($str1$$Register, $str2$$Register,
11255                       $cnt1$$Register, $cnt2$$Register,
11256                       (-1), $result$$Register,
11257                       $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
11258   %}
11259   ins_pipe( pipe_slow );
11260 %}
11261 
11262 instruct string_indexofUL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
11263                          rbx_RegI result, legVecS vec, rcx_RegI tmp, rFlagsReg cr)
11264 %{
11265   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL));
11266   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
11267   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
11268 
11269   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
11270   ins_encode %{
11271     __ string_indexof($str1$$Register, $str2$$Register,
11272                       $cnt1$$Register, $cnt2$$Register,
11273                       (-1), $result$$Register,
11274                       $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
11275   %}
11276   ins_pipe( pipe_slow );
11277 %}
11278 
11279 instruct string_indexofU_char(rdi_RegP str1, rdx_RegI cnt1, rax_RegI ch,
11280                               rbx_RegI result, legVecS vec1, legVecS vec2, legVecS vec3, rcx_RegI tmp, rFlagsReg cr)
11281 %{
11282   predicate(UseSSE42Intrinsics);
11283   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
11284   effect(TEMP vec1, TEMP vec2, TEMP vec3, USE_KILL str1, USE_KILL cnt1, USE_KILL ch, TEMP tmp, KILL cr);
11285   format %{ "String IndexOf char[] $str1,$cnt1,$ch -> $result   // KILL all" %}
11286   ins_encode %{
11287     __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register, $result$$Register,
11288                            $vec1$$XMMRegister, $vec2$$XMMRegister, $vec3$$XMMRegister, $tmp$$Register);
11289   %}
11290   ins_pipe( pipe_slow );
11291 %}
11292 
11293 // fast string equals
11294 instruct string_equals(rdi_RegP str1, rsi_RegP str2, rcx_RegI cnt, rax_RegI result,
11295                        legVecS tmp1, legVecS tmp2, rbx_RegI tmp3, rFlagsReg cr)
11296 %{
11297   match(Set result (StrEquals (Binary str1 str2) cnt));
11298   effect(TEMP tmp1, TEMP tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL tmp3, KILL cr);
11299 
11300   format %{ "String Equals $str1,$str2,$cnt -> $result    // KILL $tmp1, $tmp2, $tmp3" %}
11301   ins_encode %{
11302     __ arrays_equals(false, $str1$$Register, $str2$$Register,
11303                      $cnt$$Register, $result$$Register, $tmp3$$Register,
11304                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */);
11305   %}
11306   ins_pipe( pipe_slow );
11307 %}
11308 
11309 // fast array equals
11310 instruct array_equalsB(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
11311                        legVecS tmp1, legVecS tmp2, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
11312 %{
11313   predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
11314   match(Set result (AryEq ary1 ary2));
11315   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
11316 
11317   format %{ "Array Equals byte[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
11318   ins_encode %{
11319     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
11320                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
11321                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */);
11322   %}
11323   ins_pipe( pipe_slow );
11324 %}
11325 
11326 instruct array_equalsC(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
11327                       legVecS tmp1, legVecS tmp2, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
11328 %{
11329   predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
11330   match(Set result (AryEq ary1 ary2));
11331   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
11332 
11333   format %{ "Array Equals char[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
11334   ins_encode %{
11335     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
11336                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
11337                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, true /* char */);
11338   %}
11339   ins_pipe( pipe_slow );
11340 %}
11341 
11342 instruct has_negatives(rsi_RegP ary1, rcx_RegI len, rax_RegI result,
11343                       legVecS tmp1, legVecS tmp2, rbx_RegI tmp3, rFlagsReg cr)
11344 %{
11345   match(Set result (HasNegatives ary1 len));
11346   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL len, KILL tmp3, KILL cr);
11347 
11348   format %{ "has negatives byte[] $ary1,$len -> $result   // KILL $tmp1, $tmp2, $tmp3" %}
11349   ins_encode %{
11350     __ has_negatives($ary1$$Register, $len$$Register,
11351                      $result$$Register, $tmp3$$Register,
11352                      $tmp1$$XMMRegister, $tmp2$$XMMRegister);
11353   %}
11354   ins_pipe( pipe_slow );
11355 %}
11356 
11357 // fast char[] to byte[] compression
11358 instruct string_compress(rsi_RegP src, rdi_RegP dst, rdx_RegI len, legVecS tmp1, legVecS tmp2, legVecS tmp3, legVecS tmp4,
11359                          rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
11360   match(Set result (StrCompressedCopy src (Binary dst len)));
11361   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);
11362 
11363   format %{ "String Compress $src,$dst -> $result    // KILL RAX, RCX, RDX" %}
11364   ins_encode %{
11365     __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
11366                            $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
11367                            $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register);
11368   %}
11369   ins_pipe( pipe_slow );
11370 %}
11371 
11372 // fast byte[] to char[] inflation
11373 instruct string_inflate(Universe dummy, rsi_RegP src, rdi_RegP dst, rdx_RegI len,
11374                         legVecS tmp1, rcx_RegI tmp2, rFlagsReg cr) %{
11375   match(Set dummy (StrInflatedCopy src (Binary dst len)));
11376   effect(TEMP tmp1, TEMP tmp2, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
11377 
11378   format %{ "String Inflate $src,$dst    // KILL $tmp1, $tmp2" %}
11379   ins_encode %{
11380     __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
11381                           $tmp1$$XMMRegister, $tmp2$$Register);
11382   %}
11383   ins_pipe( pipe_slow );
11384 %}
11385 
11386 // encode char[] to byte[] in ISO_8859_1
11387 instruct encode_iso_array(rsi_RegP src, rdi_RegP dst, rdx_RegI len,
11388                           legVecS tmp1, legVecS tmp2, legVecS tmp3, legVecS tmp4,
11389                           rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
11390   match(Set result (EncodeISOArray src (Binary dst len)));
11391   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);
11392 
11393   format %{ "Encode array $src,$dst,$len -> $result    // KILL RCX, RDX, $tmp1, $tmp2, $tmp3, $tmp4, RSI, RDI " %}
11394   ins_encode %{
11395     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
11396                         $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
11397                         $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register);
11398   %}
11399   ins_pipe( pipe_slow );
11400 %}
11401 
11402 //----------Overflow Math Instructions-----------------------------------------
11403 
11404 instruct overflowAddI_rReg(rFlagsReg cr, rax_RegI op1, rRegI op2)
11405 %{
11406   match(Set cr (OverflowAddI op1 op2));
11407   effect(DEF cr, USE_KILL op1, USE op2);
11408 
11409   format %{ "addl    $op1, $op2\t# overflow check int" %}
11410 
11411   ins_encode %{
11412     __ addl($op1$$Register, $op2$$Register);
11413   %}
11414   ins_pipe(ialu_reg_reg);
11415 %}
11416 
11417 instruct overflowAddI_rReg_imm(rFlagsReg cr, rax_RegI op1, immI op2)
11418 %{
11419   match(Set cr (OverflowAddI op1 op2));
11420   effect(DEF cr, USE_KILL op1, USE op2);
11421 
11422   format %{ "addl    $op1, $op2\t# overflow check int" %}
11423 
11424   ins_encode %{
11425     __ addl($op1$$Register, $op2$$constant);
11426   %}
11427   ins_pipe(ialu_reg_reg);
11428 %}
11429 
11430 instruct overflowAddL_rReg(rFlagsReg cr, rax_RegL op1, rRegL op2)
11431 %{
11432   match(Set cr (OverflowAddL op1 op2));
11433   effect(DEF cr, USE_KILL op1, USE op2);
11434 
11435   format %{ "addq    $op1, $op2\t# overflow check long" %}
11436   ins_encode %{
11437     __ addq($op1$$Register, $op2$$Register);
11438   %}
11439   ins_pipe(ialu_reg_reg);
11440 %}
11441 
11442 instruct overflowAddL_rReg_imm(rFlagsReg cr, rax_RegL op1, immL32 op2)
11443 %{
11444   match(Set cr (OverflowAddL op1 op2));
11445   effect(DEF cr, USE_KILL op1, USE op2);
11446 
11447   format %{ "addq    $op1, $op2\t# overflow check long" %}
11448   ins_encode %{
11449     __ addq($op1$$Register, $op2$$constant);
11450   %}
11451   ins_pipe(ialu_reg_reg);
11452 %}
11453 
11454 instruct overflowSubI_rReg(rFlagsReg cr, rRegI op1, rRegI op2)
11455 %{
11456   match(Set cr (OverflowSubI op1 op2));
11457 
11458   format %{ "cmpl    $op1, $op2\t# overflow check int" %}
11459   ins_encode %{
11460     __ cmpl($op1$$Register, $op2$$Register);
11461   %}
11462   ins_pipe(ialu_reg_reg);
11463 %}
11464 
11465 instruct overflowSubI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2)
11466 %{
11467   match(Set cr (OverflowSubI op1 op2));
11468 
11469   format %{ "cmpl    $op1, $op2\t# overflow check int" %}
11470   ins_encode %{
11471     __ cmpl($op1$$Register, $op2$$constant);
11472   %}
11473   ins_pipe(ialu_reg_reg);
11474 %}
11475 
11476 instruct overflowSubL_rReg(rFlagsReg cr, rRegL op1, rRegL op2)
11477 %{
11478   match(Set cr (OverflowSubL op1 op2));
11479 
11480   format %{ "cmpq    $op1, $op2\t# overflow check long" %}
11481   ins_encode %{
11482     __ cmpq($op1$$Register, $op2$$Register);
11483   %}
11484   ins_pipe(ialu_reg_reg);
11485 %}
11486 
11487 instruct overflowSubL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2)
11488 %{
11489   match(Set cr (OverflowSubL op1 op2));
11490 
11491   format %{ "cmpq    $op1, $op2\t# overflow check long" %}
11492   ins_encode %{
11493     __ cmpq($op1$$Register, $op2$$constant);
11494   %}
11495   ins_pipe(ialu_reg_reg);
11496 %}
11497 
11498 instruct overflowNegI_rReg(rFlagsReg cr, immI0 zero, rax_RegI op2)
11499 %{
11500   match(Set cr (OverflowSubI zero op2));
11501   effect(DEF cr, USE_KILL op2);
11502 
11503   format %{ "negl    $op2\t# overflow check int" %}
11504   ins_encode %{
11505     __ negl($op2$$Register);
11506   %}
11507   ins_pipe(ialu_reg_reg);
11508 %}
11509 
11510 instruct overflowNegL_rReg(rFlagsReg cr, immL0 zero, rax_RegL op2)
11511 %{
11512   match(Set cr (OverflowSubL zero op2));
11513   effect(DEF cr, USE_KILL op2);
11514 
11515   format %{ "negq    $op2\t# overflow check long" %}
11516   ins_encode %{
11517     __ negq($op2$$Register);
11518   %}
11519   ins_pipe(ialu_reg_reg);
11520 %}
11521 
11522 instruct overflowMulI_rReg(rFlagsReg cr, rax_RegI op1, rRegI op2)
11523 %{
11524   match(Set cr (OverflowMulI op1 op2));
11525   effect(DEF cr, USE_KILL op1, USE op2);
11526 
11527   format %{ "imull    $op1, $op2\t# overflow check int" %}
11528   ins_encode %{
11529     __ imull($op1$$Register, $op2$$Register);
11530   %}
11531   ins_pipe(ialu_reg_reg_alu0);
11532 %}
11533 
11534 instruct overflowMulI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2, rRegI tmp)
11535 %{
11536   match(Set cr (OverflowMulI op1 op2));
11537   effect(DEF cr, TEMP tmp, USE op1, USE op2);
11538 
11539   format %{ "imull    $tmp, $op1, $op2\t# overflow check int" %}
11540   ins_encode %{
11541     __ imull($tmp$$Register, $op1$$Register, $op2$$constant);
11542   %}
11543   ins_pipe(ialu_reg_reg_alu0);
11544 %}
11545 
11546 instruct overflowMulL_rReg(rFlagsReg cr, rax_RegL op1, rRegL op2)
11547 %{
11548   match(Set cr (OverflowMulL op1 op2));
11549   effect(DEF cr, USE_KILL op1, USE op2);
11550 
11551   format %{ "imulq    $op1, $op2\t# overflow check long" %}
11552   ins_encode %{
11553     __ imulq($op1$$Register, $op2$$Register);
11554   %}
11555   ins_pipe(ialu_reg_reg_alu0);
11556 %}
11557 
11558 instruct overflowMulL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2, rRegL tmp)
11559 %{
11560   match(Set cr (OverflowMulL op1 op2));
11561   effect(DEF cr, TEMP tmp, USE op1, USE op2);
11562 
11563   format %{ "imulq    $tmp, $op1, $op2\t# overflow check long" %}
11564   ins_encode %{
11565     __ imulq($tmp$$Register, $op1$$Register, $op2$$constant);
11566   %}
11567   ins_pipe(ialu_reg_reg_alu0);
11568 %}
11569 
11570 
11571 //----------Control Flow Instructions------------------------------------------
11572 // Signed compare Instructions
11573 
11574 // XXX more variants!!
11575 instruct compI_rReg(rFlagsReg cr, rRegI op1, rRegI op2)
11576 %{
11577   match(Set cr (CmpI op1 op2));
11578   effect(DEF cr, USE op1, USE op2);
11579 
11580   format %{ "cmpl    $op1, $op2" %}
11581   opcode(0x3B);  /* Opcode 3B /r */
11582   ins_encode(REX_reg_reg(op1, op2), OpcP, reg_reg(op1, op2));
11583   ins_pipe(ialu_cr_reg_reg);
11584 %}
11585 
11586 instruct compI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2)
11587 %{
11588   match(Set cr (CmpI op1 op2));
11589 
11590   format %{ "cmpl    $op1, $op2" %}
11591   opcode(0x81, 0x07); /* Opcode 81 /7 */
11592   ins_encode(OpcSErm(op1, op2), Con8or32(op2));
11593   ins_pipe(ialu_cr_reg_imm);
11594 %}
11595 
11596 instruct compI_rReg_mem(rFlagsReg cr, rRegI op1, memory op2)
11597 %{
11598   match(Set cr (CmpI op1 (LoadI op2)));
11599 
11600   ins_cost(500); // XXX
11601   format %{ "cmpl    $op1, $op2" %}
11602   opcode(0x3B); /* Opcode 3B /r */
11603   ins_encode(REX_reg_mem(op1, op2), OpcP, reg_mem(op1, op2));
11604   ins_pipe(ialu_cr_reg_mem);
11605 %}
11606 
11607 instruct testI_reg(rFlagsReg cr, rRegI src, immI0 zero)
11608 %{
11609   match(Set cr (CmpI src zero));
11610 
11611   format %{ "testl   $src, $src" %}
11612   opcode(0x85);
11613   ins_encode(REX_reg_reg(src, src), OpcP, reg_reg(src, src));
11614   ins_pipe(ialu_cr_reg_imm);
11615 %}
11616 
11617 instruct testI_reg_imm(rFlagsReg cr, rRegI src, immI con, immI0 zero)
11618 %{
11619   match(Set cr (CmpI (AndI src con) zero));
11620 
11621   format %{ "testl   $src, $con" %}
11622   opcode(0xF7, 0x00);
11623   ins_encode(REX_reg(src), OpcP, reg_opc(src), Con32(con));
11624   ins_pipe(ialu_cr_reg_imm);
11625 %}
11626 
11627 instruct testI_reg_mem(rFlagsReg cr, rRegI src, memory mem, immI0 zero)
11628 %{
11629   match(Set cr (CmpI (AndI src (LoadI mem)) zero));
11630 
11631   format %{ "testl   $src, $mem" %}
11632   opcode(0x85);
11633   ins_encode(REX_reg_mem(src, mem), OpcP, reg_mem(src, mem));
11634   ins_pipe(ialu_cr_reg_mem);
11635 %}
11636 
11637 // Unsigned compare Instructions; really, same as signed except they
11638 // produce an rFlagsRegU instead of rFlagsReg.
11639 instruct compU_rReg(rFlagsRegU cr, rRegI op1, rRegI op2)
11640 %{
11641   match(Set cr (CmpU op1 op2));
11642 
11643   format %{ "cmpl    $op1, $op2\t# unsigned" %}
11644   opcode(0x3B); /* Opcode 3B /r */
11645   ins_encode(REX_reg_reg(op1, op2), OpcP, reg_reg(op1, op2));
11646   ins_pipe(ialu_cr_reg_reg);
11647 %}
11648 
11649 instruct compU_rReg_imm(rFlagsRegU cr, rRegI op1, immI op2)
11650 %{
11651   match(Set cr (CmpU op1 op2));
11652 
11653   format %{ "cmpl    $op1, $op2\t# unsigned" %}
11654   opcode(0x81,0x07); /* Opcode 81 /7 */
11655   ins_encode(OpcSErm(op1, op2), Con8or32(op2));
11656   ins_pipe(ialu_cr_reg_imm);
11657 %}
11658 
11659 instruct compU_rReg_mem(rFlagsRegU cr, rRegI op1, memory op2)
11660 %{
11661   match(Set cr (CmpU op1 (LoadI op2)));
11662 
11663   ins_cost(500); // XXX
11664   format %{ "cmpl    $op1, $op2\t# unsigned" %}
11665   opcode(0x3B); /* Opcode 3B /r */
11666   ins_encode(REX_reg_mem(op1, op2), OpcP, reg_mem(op1, op2));
11667   ins_pipe(ialu_cr_reg_mem);
11668 %}
11669 
11670 // // // Cisc-spilled version of cmpU_rReg
11671 // //instruct compU_mem_rReg(rFlagsRegU cr, memory op1, rRegI op2)
11672 // //%{
11673 // //  match(Set cr (CmpU (LoadI op1) op2));
11674 // //
11675 // //  format %{ "CMPu   $op1,$op2" %}
11676 // //  ins_cost(500);
11677 // //  opcode(0x39);  /* Opcode 39 /r */
11678 // //  ins_encode( OpcP, reg_mem( op1, op2) );
11679 // //%}
11680 
11681 instruct testU_reg(rFlagsRegU cr, rRegI src, immI0 zero)
11682 %{
11683   match(Set cr (CmpU src zero));
11684 
11685   format %{ "testl  $src, $src\t# unsigned" %}
11686   opcode(0x85);
11687   ins_encode(REX_reg_reg(src, src), OpcP, reg_reg(src, src));
11688   ins_pipe(ialu_cr_reg_imm);
11689 %}
11690 
11691 instruct compP_rReg(rFlagsRegU cr, rRegP op1, rRegP op2)
11692 %{
11693   match(Set cr (CmpP op1 op2));
11694 
11695   format %{ "cmpq    $op1, $op2\t# ptr" %}
11696   opcode(0x3B); /* Opcode 3B /r */
11697   ins_encode(REX_reg_reg_wide(op1, op2), OpcP, reg_reg(op1, op2));
11698   ins_pipe(ialu_cr_reg_reg);
11699 %}
11700 
11701 instruct compP_rReg_mem(rFlagsRegU cr, rRegP op1, memory op2)
11702 %{
11703   match(Set cr (CmpP op1 (LoadP op2)));
11704 
11705   ins_cost(500); // XXX
11706   format %{ "cmpq    $op1, $op2\t# ptr" %}
11707   opcode(0x3B); /* Opcode 3B /r */
11708   ins_encode(REX_reg_mem_wide(op1, op2), OpcP, reg_mem(op1, op2));
11709   ins_pipe(ialu_cr_reg_mem);
11710 %}
11711 
11712 // // // Cisc-spilled version of cmpP_rReg
11713 // //instruct compP_mem_rReg(rFlagsRegU cr, memory op1, rRegP op2)
11714 // //%{
11715 // //  match(Set cr (CmpP (LoadP op1) op2));
11716 // //
11717 // //  format %{ "CMPu   $op1,$op2" %}
11718 // //  ins_cost(500);
11719 // //  opcode(0x39);  /* Opcode 39 /r */
11720 // //  ins_encode( OpcP, reg_mem( op1, op2) );
11721 // //%}
11722 
11723 // XXX this is generalized by compP_rReg_mem???
11724 // Compare raw pointer (used in out-of-heap check).
11725 // Only works because non-oop pointers must be raw pointers
11726 // and raw pointers have no anti-dependencies.
11727 instruct compP_mem_rReg(rFlagsRegU cr, rRegP op1, memory op2)
11728 %{
11729   predicate(n->in(2)->in(2)->bottom_type()->reloc() == relocInfo::none);
11730   match(Set cr (CmpP op1 (LoadP op2)));
11731 
11732   format %{ "cmpq    $op1, $op2\t# raw ptr" %}
11733   opcode(0x3B); /* Opcode 3B /r */
11734   ins_encode(REX_reg_mem_wide(op1, op2), OpcP, reg_mem(op1, op2));
11735   ins_pipe(ialu_cr_reg_mem);
11736 %}
11737 
11738 // This will generate a signed flags result. This should be OK since
11739 // any compare to a zero should be eq/neq.
11740 instruct testP_reg(rFlagsReg cr, rRegP src, immP0 zero)
11741 %{
11742   match(Set cr (CmpP src zero));
11743 
11744   format %{ "testq   $src, $src\t# ptr" %}
11745   opcode(0x85);
11746   ins_encode(REX_reg_reg_wide(src, src), OpcP, reg_reg(src, src));
11747   ins_pipe(ialu_cr_reg_imm);
11748 %}
11749 
11750 // This will generate a signed flags result. This should be OK since
11751 // any compare to a zero should be eq/neq.
11752 instruct testP_mem(rFlagsReg cr, memory op, immP0 zero)
11753 %{
11754   predicate(!UseCompressedOops || (Universe::narrow_oop_base() != NULL));
11755   match(Set cr (CmpP (LoadP op) zero));
11756 
11757   ins_cost(500); // XXX
11758   format %{ "testq   $op, 0xffffffffffffffff\t# ptr" %}
11759   opcode(0xF7); /* Opcode F7 /0 */
11760   ins_encode(REX_mem_wide(op),
11761              OpcP, RM_opc_mem(0x00, op), Con_d32(0xFFFFFFFF));
11762   ins_pipe(ialu_cr_reg_imm);
11763 %}
11764 
11765 instruct testP_mem_reg0(rFlagsReg cr, memory mem, immP0 zero)
11766 %{
11767   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
11768   match(Set cr (CmpP (LoadP mem) zero));
11769 
11770   format %{ "cmpq    R12, $mem\t# ptr (R12_heapbase==0)" %}
11771   ins_encode %{
11772     __ cmpq(r12, $mem$$Address);
11773   %}
11774   ins_pipe(ialu_cr_reg_mem);
11775 %}
11776 
11777 instruct compN_rReg(rFlagsRegU cr, rRegN op1, rRegN op2)
11778 %{
11779   match(Set cr (CmpN op1 op2));
11780 
11781   format %{ "cmpl    $op1, $op2\t# compressed ptr" %}
11782   ins_encode %{ __ cmpl($op1$$Register, $op2$$Register); %}
11783   ins_pipe(ialu_cr_reg_reg);
11784 %}
11785 
11786 instruct compN_rReg_mem(rFlagsRegU cr, rRegN src, memory mem)
11787 %{
11788   match(Set cr (CmpN src (LoadN mem)));
11789 
11790   format %{ "cmpl    $src, $mem\t# compressed ptr" %}
11791   ins_encode %{
11792     __ cmpl($src$$Register, $mem$$Address);
11793   %}
11794   ins_pipe(ialu_cr_reg_mem);
11795 %}
11796 
11797 instruct compN_rReg_imm(rFlagsRegU cr, rRegN op1, immN op2) %{
11798   match(Set cr (CmpN op1 op2));
11799 
11800   format %{ "cmpl    $op1, $op2\t# compressed ptr" %}
11801   ins_encode %{
11802     __ cmp_narrow_oop($op1$$Register, (jobject)$op2$$constant);
11803   %}
11804   ins_pipe(ialu_cr_reg_imm);
11805 %}
11806 
11807 instruct compN_mem_imm(rFlagsRegU cr, memory mem, immN src)
11808 %{
11809   match(Set cr (CmpN src (LoadN mem)));
11810 
11811   format %{ "cmpl    $mem, $src\t# compressed ptr" %}
11812   ins_encode %{
11813     __ cmp_narrow_oop($mem$$Address, (jobject)$src$$constant);
11814   %}
11815   ins_pipe(ialu_cr_reg_mem);
11816 %}
11817 
11818 instruct compN_rReg_imm_klass(rFlagsRegU cr, rRegN op1, immNKlass op2) %{
11819   match(Set cr (CmpN op1 op2));
11820 
11821   format %{ "cmpl    $op1, $op2\t# compressed klass ptr" %}
11822   ins_encode %{
11823     __ cmp_narrow_klass($op1$$Register, (Klass*)$op2$$constant);
11824   %}
11825   ins_pipe(ialu_cr_reg_imm);
11826 %}
11827 
11828 instruct compN_mem_imm_klass(rFlagsRegU cr, memory mem, immNKlass src)
11829 %{
11830   match(Set cr (CmpN src (LoadNKlass mem)));
11831 
11832   format %{ "cmpl    $mem, $src\t# compressed klass ptr" %}
11833   ins_encode %{
11834     __ cmp_narrow_klass($mem$$Address, (Klass*)$src$$constant);
11835   %}
11836   ins_pipe(ialu_cr_reg_mem);
11837 %}
11838 
11839 instruct testN_reg(rFlagsReg cr, rRegN src, immN0 zero) %{
11840   match(Set cr (CmpN src zero));
11841 
11842   format %{ "testl   $src, $src\t# compressed ptr" %}
11843   ins_encode %{ __ testl($src$$Register, $src$$Register); %}
11844   ins_pipe(ialu_cr_reg_imm);
11845 %}
11846 
11847 instruct testN_mem(rFlagsReg cr, memory mem, immN0 zero)
11848 %{
11849   predicate(Universe::narrow_oop_base() != NULL);
11850   match(Set cr (CmpN (LoadN mem) zero));
11851 
11852   ins_cost(500); // XXX
11853   format %{ "testl   $mem, 0xffffffff\t# compressed ptr" %}
11854   ins_encode %{
11855     __ cmpl($mem$$Address, (int)0xFFFFFFFF);
11856   %}
11857   ins_pipe(ialu_cr_reg_mem);
11858 %}
11859 
11860 instruct testN_mem_reg0(rFlagsReg cr, memory mem, immN0 zero)
11861 %{
11862   predicate(Universe::narrow_oop_base() == NULL && (Universe::narrow_klass_base() == NULL));
11863   match(Set cr (CmpN (LoadN mem) zero));
11864 
11865   format %{ "cmpl    R12, $mem\t# compressed ptr (R12_heapbase==0)" %}
11866   ins_encode %{
11867     __ cmpl(r12, $mem$$Address);
11868   %}
11869   ins_pipe(ialu_cr_reg_mem);
11870 %}
11871 
11872 // Yanked all unsigned pointer compare operations.
11873 // Pointer compares are done with CmpP which is already unsigned.
11874 
11875 instruct compL_rReg(rFlagsReg cr, rRegL op1, rRegL op2)
11876 %{
11877   match(Set cr (CmpL op1 op2));
11878 
11879   format %{ "cmpq    $op1, $op2" %}
11880   opcode(0x3B);  /* Opcode 3B /r */
11881   ins_encode(REX_reg_reg_wide(op1, op2), OpcP, reg_reg(op1, op2));
11882   ins_pipe(ialu_cr_reg_reg);
11883 %}
11884 
11885 instruct compL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2)
11886 %{
11887   match(Set cr (CmpL op1 op2));
11888 
11889   format %{ "cmpq    $op1, $op2" %}
11890   opcode(0x81, 0x07); /* Opcode 81 /7 */
11891   ins_encode(OpcSErm_wide(op1, op2), Con8or32(op2));
11892   ins_pipe(ialu_cr_reg_imm);
11893 %}
11894 
11895 instruct compL_rReg_mem(rFlagsReg cr, rRegL op1, memory op2)
11896 %{
11897   match(Set cr (CmpL op1 (LoadL op2)));
11898 
11899   format %{ "cmpq    $op1, $op2" %}
11900   opcode(0x3B); /* Opcode 3B /r */
11901   ins_encode(REX_reg_mem_wide(op1, op2), OpcP, reg_mem(op1, op2));
11902   ins_pipe(ialu_cr_reg_mem);
11903 %}
11904 
11905 instruct testL_reg(rFlagsReg cr, rRegL src, immL0 zero)
11906 %{
11907   match(Set cr (CmpL src zero));
11908 
11909   format %{ "testq   $src, $src" %}
11910   opcode(0x85);
11911   ins_encode(REX_reg_reg_wide(src, src), OpcP, reg_reg(src, src));
11912   ins_pipe(ialu_cr_reg_imm);
11913 %}
11914 
11915 instruct testL_reg_imm(rFlagsReg cr, rRegL src, immL32 con, immL0 zero)
11916 %{
11917   match(Set cr (CmpL (AndL src con) zero));
11918 
11919   format %{ "testq   $src, $con\t# long" %}
11920   opcode(0xF7, 0x00);
11921   ins_encode(REX_reg_wide(src), OpcP, reg_opc(src), Con32(con));
11922   ins_pipe(ialu_cr_reg_imm);
11923 %}
11924 
11925 instruct testL_reg_mem(rFlagsReg cr, rRegL src, memory mem, immL0 zero)
11926 %{
11927   match(Set cr (CmpL (AndL src (LoadL mem)) zero));
11928 
11929   format %{ "testq   $src, $mem" %}
11930   opcode(0x85);
11931   ins_encode(REX_reg_mem_wide(src, mem), OpcP, reg_mem(src, mem));
11932   ins_pipe(ialu_cr_reg_mem);
11933 %}
11934 
11935 instruct testL_reg_mem2(rFlagsReg cr, rRegP src, memory mem, immL0 zero)
11936 %{
11937   match(Set cr (CmpL (AndL (CastP2X src) (LoadL mem)) zero));
11938 
11939   format %{ "testq   $src, $mem" %}
11940   opcode(0x85);
11941   ins_encode(REX_reg_mem_wide(src, mem), OpcP, reg_mem(src, mem));
11942   ins_pipe(ialu_cr_reg_mem);
11943 %}
11944 
11945 // Manifest a CmpL result in an integer register.  Very painful.
11946 // This is the test to avoid.
11947 instruct cmpL3_reg_reg(rRegI dst, rRegL src1, rRegL src2, rFlagsReg flags)
11948 %{
11949   match(Set dst (CmpL3 src1 src2));
11950   effect(KILL flags);
11951 
11952   ins_cost(275); // XXX
11953   format %{ "cmpq    $src1, $src2\t# CmpL3\n\t"
11954             "movl    $dst, -1\n\t"
11955             "jl,s    done\n\t"
11956             "setne   $dst\n\t"
11957             "movzbl  $dst, $dst\n\t"
11958     "done:" %}
11959   ins_encode(cmpl3_flag(src1, src2, dst));
11960   ins_pipe(pipe_slow);
11961 %}
11962 
11963 // Unsigned long compare Instructions; really, same as signed long except they
11964 // produce an rFlagsRegU instead of rFlagsReg.
11965 instruct compUL_rReg(rFlagsRegU cr, rRegL op1, rRegL op2)
11966 %{
11967   match(Set cr (CmpUL op1 op2));
11968 
11969   format %{ "cmpq    $op1, $op2\t# unsigned" %}
11970   opcode(0x3B);  /* Opcode 3B /r */
11971   ins_encode(REX_reg_reg_wide(op1, op2), OpcP, reg_reg(op1, op2));
11972   ins_pipe(ialu_cr_reg_reg);
11973 %}
11974 
11975 instruct compUL_rReg_imm(rFlagsRegU cr, rRegL op1, immL32 op2)
11976 %{
11977   match(Set cr (CmpUL op1 op2));
11978 
11979   format %{ "cmpq    $op1, $op2\t# unsigned" %}
11980   opcode(0x81, 0x07); /* Opcode 81 /7 */
11981   ins_encode(OpcSErm_wide(op1, op2), Con8or32(op2));
11982   ins_pipe(ialu_cr_reg_imm);
11983 %}
11984 
11985 instruct compUL_rReg_mem(rFlagsRegU cr, rRegL op1, memory op2)
11986 %{
11987   match(Set cr (CmpUL op1 (LoadL op2)));
11988 
11989   format %{ "cmpq    $op1, $op2\t# unsigned" %}
11990   opcode(0x3B); /* Opcode 3B /r */
11991   ins_encode(REX_reg_mem_wide(op1, op2), OpcP, reg_mem(op1, op2));
11992   ins_pipe(ialu_cr_reg_mem);
11993 %}
11994 
11995 instruct testUL_reg(rFlagsRegU cr, rRegL src, immL0 zero)
11996 %{
11997   match(Set cr (CmpUL src zero));
11998 
11999   format %{ "testq   $src, $src\t# unsigned" %}
12000   opcode(0x85);
12001   ins_encode(REX_reg_reg_wide(src, src), OpcP, reg_reg(src, src));
12002   ins_pipe(ialu_cr_reg_imm);
12003 %}
12004 
12005 instruct compB_mem_imm(rFlagsReg cr, memory mem, immI8 imm)
12006 %{
12007   match(Set cr (CmpI (LoadB mem) imm));
12008 
12009   ins_cost(125);
12010   format %{ "cmpb    $mem, $imm" %}
12011   ins_encode %{ __ cmpb($mem$$Address, $imm$$constant); %}
12012   ins_pipe(ialu_cr_reg_mem);
12013 %}
12014 
12015 instruct testB_mem_imm(rFlagsReg cr, memory mem, immI8 imm, immI0 zero)
12016 %{
12017   match(Set cr (CmpI (AndI (LoadB mem) imm) zero));
12018 
12019   ins_cost(125);
12020   format %{ "testb   $mem, $imm" %}
12021   ins_encode %{ __ testb($mem$$Address, $imm$$constant); %}
12022   ins_pipe(ialu_cr_reg_mem);
12023 %}
12024 
12025 //----------Max and Min--------------------------------------------------------
12026 // Min Instructions
12027 
12028 instruct cmovI_reg_g(rRegI dst, rRegI src, rFlagsReg cr)
12029 %{
12030   effect(USE_DEF dst, USE src, USE cr);
12031 
12032   format %{ "cmovlgt $dst, $src\t# min" %}
12033   opcode(0x0F, 0x4F);
12034   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
12035   ins_pipe(pipe_cmov_reg);
12036 %}
12037 
12038 
12039 instruct minI_rReg(rRegI dst, rRegI src)
12040 %{
12041   match(Set dst (MinI dst src));
12042 
12043   ins_cost(200);
12044   expand %{
12045     rFlagsReg cr;
12046     compI_rReg(cr, dst, src);
12047     cmovI_reg_g(dst, src, cr);
12048   %}
12049 %}
12050 
12051 instruct cmovI_reg_l(rRegI dst, rRegI src, rFlagsReg cr)
12052 %{
12053   effect(USE_DEF dst, USE src, USE cr);
12054 
12055   format %{ "cmovllt $dst, $src\t# max" %}
12056   opcode(0x0F, 0x4C);
12057   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
12058   ins_pipe(pipe_cmov_reg);
12059 %}
12060 
12061 
12062 instruct maxI_rReg(rRegI dst, rRegI src)
12063 %{
12064   match(Set dst (MaxI dst src));
12065 
12066   ins_cost(200);
12067   expand %{
12068     rFlagsReg cr;
12069     compI_rReg(cr, dst, src);
12070     cmovI_reg_l(dst, src, cr);
12071   %}
12072 %}
12073 
12074 // ============================================================================
12075 // Branch Instructions
12076 
12077 // Jump Direct - Label defines a relative address from JMP+1
12078 instruct jmpDir(label labl)
12079 %{
12080   match(Goto);
12081   effect(USE labl);
12082 
12083   ins_cost(300);
12084   format %{ "jmp     $labl" %}
12085   size(5);
12086   ins_encode %{
12087     Label* L = $labl$$label;
12088     __ jmp(*L, false); // Always long jump
12089   %}
12090   ins_pipe(pipe_jmp);
12091 %}
12092 
12093 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12094 instruct jmpCon(cmpOp cop, rFlagsReg cr, label labl)
12095 %{
12096   match(If cop cr);
12097   effect(USE labl);
12098 
12099   ins_cost(300);
12100   format %{ "j$cop     $labl" %}
12101   size(6);
12102   ins_encode %{
12103     Label* L = $labl$$label;
12104     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12105   %}
12106   ins_pipe(pipe_jcc);
12107 %}
12108 
12109 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12110 instruct jmpLoopEnd(cmpOp cop, rFlagsReg cr, label labl)
12111 %{
12112   predicate(!n->has_vector_mask_set());
12113   match(CountedLoopEnd cop cr);
12114   effect(USE labl);
12115 
12116   ins_cost(300);
12117   format %{ "j$cop     $labl\t# loop end" %}
12118   size(6);
12119   ins_encode %{
12120     Label* L = $labl$$label;
12121     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12122   %}
12123   ins_pipe(pipe_jcc);
12124 %}
12125 
12126 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12127 instruct jmpLoopEndU(cmpOpU cop, rFlagsRegU cmp, label labl) %{
12128   predicate(!n->has_vector_mask_set());
12129   match(CountedLoopEnd cop cmp);
12130   effect(USE labl);
12131 
12132   ins_cost(300);
12133   format %{ "j$cop,u   $labl\t# loop end" %}
12134   size(6);
12135   ins_encode %{
12136     Label* L = $labl$$label;
12137     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12138   %}
12139   ins_pipe(pipe_jcc);
12140 %}
12141 
12142 instruct jmpLoopEndUCF(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
12143   predicate(!n->has_vector_mask_set());
12144   match(CountedLoopEnd cop cmp);
12145   effect(USE labl);
12146 
12147   ins_cost(200);
12148   format %{ "j$cop,u   $labl\t# loop end" %}
12149   size(6);
12150   ins_encode %{
12151     Label* L = $labl$$label;
12152     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12153   %}
12154   ins_pipe(pipe_jcc);
12155 %}
12156 
12157 // mask version
12158 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12159 instruct jmpLoopEnd_and_restoreMask(cmpOp cop, rFlagsReg cr, label labl)
12160 %{
12161   predicate(n->has_vector_mask_set());
12162   match(CountedLoopEnd cop cr);
12163   effect(USE labl);
12164 
12165   ins_cost(400);
12166   format %{ "j$cop     $labl\t# loop end\n\t"
12167             "restorevectmask \t# vector mask restore for loops" %}
12168   size(10);
12169   ins_encode %{
12170     Label* L = $labl$$label;
12171     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12172     __ restorevectmask();
12173   %}
12174   ins_pipe(pipe_jcc);
12175 %}
12176 
12177 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12178 instruct jmpLoopEndU_and_restoreMask(cmpOpU cop, rFlagsRegU cmp, label labl) %{
12179   predicate(n->has_vector_mask_set());
12180   match(CountedLoopEnd cop cmp);
12181   effect(USE labl);
12182 
12183   ins_cost(400);
12184   format %{ "j$cop,u   $labl\t# loop end\n\t"
12185             "restorevectmask \t# vector mask restore for loops" %}
12186   size(10);
12187   ins_encode %{
12188     Label* L = $labl$$label;
12189     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12190     __ restorevectmask();
12191   %}
12192   ins_pipe(pipe_jcc);
12193 %}
12194 
12195 instruct jmpLoopEndUCF_and_restoreMask(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
12196   predicate(n->has_vector_mask_set());
12197   match(CountedLoopEnd cop cmp);
12198   effect(USE labl);
12199 
12200   ins_cost(300);
12201   format %{ "j$cop,u   $labl\t# loop end\n\t"
12202             "restorevectmask \t# vector mask restore for loops" %}
12203   size(10);
12204   ins_encode %{
12205     Label* L = $labl$$label;
12206     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12207     __ restorevectmask();
12208   %}
12209   ins_pipe(pipe_jcc);
12210 %}
12211 
12212 // Jump Direct Conditional - using unsigned comparison
12213 instruct jmpConU(cmpOpU cop, rFlagsRegU cmp, label labl) %{
12214   match(If cop cmp);
12215   effect(USE labl);
12216 
12217   ins_cost(300);
12218   format %{ "j$cop,u  $labl" %}
12219   size(6);
12220   ins_encode %{
12221     Label* L = $labl$$label;
12222     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12223   %}
12224   ins_pipe(pipe_jcc);
12225 %}
12226 
12227 instruct jmpConUCF(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
12228   match(If cop cmp);
12229   effect(USE labl);
12230 
12231   ins_cost(200);
12232   format %{ "j$cop,u  $labl" %}
12233   size(6);
12234   ins_encode %{
12235     Label* L = $labl$$label;
12236     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12237   %}
12238   ins_pipe(pipe_jcc);
12239 %}
12240 
12241 instruct jmpConUCF2(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
12242   match(If cop cmp);
12243   effect(USE labl);
12244 
12245   ins_cost(200);
12246   format %{ $$template
12247     if ($cop$$cmpcode == Assembler::notEqual) {
12248       $$emit$$"jp,u   $labl\n\t"
12249       $$emit$$"j$cop,u   $labl"
12250     } else {
12251       $$emit$$"jp,u   done\n\t"
12252       $$emit$$"j$cop,u   $labl\n\t"
12253       $$emit$$"done:"
12254     }
12255   %}
12256   ins_encode %{
12257     Label* l = $labl$$label;
12258     if ($cop$$cmpcode == Assembler::notEqual) {
12259       __ jcc(Assembler::parity, *l, false);
12260       __ jcc(Assembler::notEqual, *l, false);
12261     } else if ($cop$$cmpcode == Assembler::equal) {
12262       Label done;
12263       __ jccb(Assembler::parity, done);
12264       __ jcc(Assembler::equal, *l, false);
12265       __ bind(done);
12266     } else {
12267        ShouldNotReachHere();
12268     }
12269   %}
12270   ins_pipe(pipe_jcc);
12271 %}
12272 
12273 // ============================================================================
12274 // The 2nd slow-half of a subtype check.  Scan the subklass's 2ndary
12275 // superklass array for an instance of the superklass.  Set a hidden
12276 // internal cache on a hit (cache is checked with exposed code in
12277 // gen_subtype_check()).  Return NZ for a miss or zero for a hit.  The
12278 // encoding ALSO sets flags.
12279 
12280 instruct partialSubtypeCheck(rdi_RegP result,
12281                              rsi_RegP sub, rax_RegP super, rcx_RegI rcx,
12282                              rFlagsReg cr)
12283 %{
12284   match(Set result (PartialSubtypeCheck sub super));
12285   effect(KILL rcx, KILL cr);
12286 
12287   ins_cost(1100);  // slightly larger than the next version
12288   format %{ "movq    rdi, [$sub + in_bytes(Klass::secondary_supers_offset())]\n\t"
12289             "movl    rcx, [rdi + Array<Klass*>::length_offset_in_bytes()]\t# length to scan\n\t"
12290             "addq    rdi, Array<Klass*>::base_offset_in_bytes()\t# Skip to start of data; set NZ in case count is zero\n\t"
12291             "repne   scasq\t# Scan *rdi++ for a match with rax while rcx--\n\t"
12292             "jne,s   miss\t\t# Missed: rdi not-zero\n\t"
12293             "movq    [$sub + in_bytes(Klass::secondary_super_cache_offset())], $super\t# Hit: update cache\n\t"
12294             "xorq    $result, $result\t\t Hit: rdi zero\n\t"
12295     "miss:\t" %}
12296 
12297   opcode(0x1); // Force a XOR of RDI
12298   ins_encode(enc_PartialSubtypeCheck());
12299   ins_pipe(pipe_slow);
12300 %}
12301 
12302 instruct partialSubtypeCheck_vs_Zero(rFlagsReg cr,
12303                                      rsi_RegP sub, rax_RegP super, rcx_RegI rcx,
12304                                      immP0 zero,
12305                                      rdi_RegP result)
12306 %{
12307   match(Set cr (CmpP (PartialSubtypeCheck sub super) zero));
12308   effect(KILL rcx, KILL result);
12309 
12310   ins_cost(1000);
12311   format %{ "movq    rdi, [$sub + in_bytes(Klass::secondary_supers_offset())]\n\t"
12312             "movl    rcx, [rdi + Array<Klass*>::length_offset_in_bytes()]\t# length to scan\n\t"
12313             "addq    rdi, Array<Klass*>::base_offset_in_bytes()\t# Skip to start of data; set NZ in case count is zero\n\t"
12314             "repne   scasq\t# Scan *rdi++ for a match with rax while cx-- != 0\n\t"
12315             "jne,s   miss\t\t# Missed: flags nz\n\t"
12316             "movq    [$sub + in_bytes(Klass::secondary_super_cache_offset())], $super\t# Hit: update cache\n\t"
12317     "miss:\t" %}
12318 
12319   opcode(0x0); // No need to XOR RDI
12320   ins_encode(enc_PartialSubtypeCheck());
12321   ins_pipe(pipe_slow);
12322 %}
12323 
12324 // ============================================================================
12325 // Branch Instructions -- short offset versions
12326 //
12327 // These instructions are used to replace jumps of a long offset (the default
12328 // match) with jumps of a shorter offset.  These instructions are all tagged
12329 // with the ins_short_branch attribute, which causes the ADLC to suppress the
12330 // match rules in general matching.  Instead, the ADLC generates a conversion
12331 // method in the MachNode which can be used to do in-place replacement of the
12332 // long variant with the shorter variant.  The compiler will determine if a
12333 // branch can be taken by the is_short_branch_offset() predicate in the machine
12334 // specific code section of the file.
12335 
12336 // Jump Direct - Label defines a relative address from JMP+1
12337 instruct jmpDir_short(label labl) %{
12338   match(Goto);
12339   effect(USE labl);
12340 
12341   ins_cost(300);
12342   format %{ "jmp,s   $labl" %}
12343   size(2);
12344   ins_encode %{
12345     Label* L = $labl$$label;
12346     __ jmpb(*L);
12347   %}
12348   ins_pipe(pipe_jmp);
12349   ins_short_branch(1);
12350 %}
12351 
12352 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12353 instruct jmpCon_short(cmpOp cop, rFlagsReg cr, label labl) %{
12354   match(If cop cr);
12355   effect(USE labl);
12356 
12357   ins_cost(300);
12358   format %{ "j$cop,s   $labl" %}
12359   size(2);
12360   ins_encode %{
12361     Label* L = $labl$$label;
12362     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12363   %}
12364   ins_pipe(pipe_jcc);
12365   ins_short_branch(1);
12366 %}
12367 
12368 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12369 instruct jmpLoopEnd_short(cmpOp cop, rFlagsReg cr, label labl) %{
12370   match(CountedLoopEnd cop cr);
12371   effect(USE labl);
12372 
12373   ins_cost(300);
12374   format %{ "j$cop,s   $labl\t# loop end" %}
12375   size(2);
12376   ins_encode %{
12377     Label* L = $labl$$label;
12378     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12379   %}
12380   ins_pipe(pipe_jcc);
12381   ins_short_branch(1);
12382 %}
12383 
12384 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12385 instruct jmpLoopEndU_short(cmpOpU cop, rFlagsRegU cmp, label labl) %{
12386   match(CountedLoopEnd cop cmp);
12387   effect(USE labl);
12388 
12389   ins_cost(300);
12390   format %{ "j$cop,us  $labl\t# loop end" %}
12391   size(2);
12392   ins_encode %{
12393     Label* L = $labl$$label;
12394     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12395   %}
12396   ins_pipe(pipe_jcc);
12397   ins_short_branch(1);
12398 %}
12399 
12400 instruct jmpLoopEndUCF_short(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
12401   match(CountedLoopEnd cop cmp);
12402   effect(USE labl);
12403 
12404   ins_cost(300);
12405   format %{ "j$cop,us  $labl\t# loop end" %}
12406   size(2);
12407   ins_encode %{
12408     Label* L = $labl$$label;
12409     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12410   %}
12411   ins_pipe(pipe_jcc);
12412   ins_short_branch(1);
12413 %}
12414 
12415 // Jump Direct Conditional - using unsigned comparison
12416 instruct jmpConU_short(cmpOpU cop, rFlagsRegU cmp, label labl) %{
12417   match(If cop cmp);
12418   effect(USE labl);
12419 
12420   ins_cost(300);
12421   format %{ "j$cop,us  $labl" %}
12422   size(2);
12423   ins_encode %{
12424     Label* L = $labl$$label;
12425     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12426   %}
12427   ins_pipe(pipe_jcc);
12428   ins_short_branch(1);
12429 %}
12430 
12431 instruct jmpConUCF_short(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
12432   match(If cop cmp);
12433   effect(USE labl);
12434 
12435   ins_cost(300);
12436   format %{ "j$cop,us  $labl" %}
12437   size(2);
12438   ins_encode %{
12439     Label* L = $labl$$label;
12440     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12441   %}
12442   ins_pipe(pipe_jcc);
12443   ins_short_branch(1);
12444 %}
12445 
12446 instruct jmpConUCF2_short(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
12447   match(If cop cmp);
12448   effect(USE labl);
12449 
12450   ins_cost(300);
12451   format %{ $$template
12452     if ($cop$$cmpcode == Assembler::notEqual) {
12453       $$emit$$"jp,u,s   $labl\n\t"
12454       $$emit$$"j$cop,u,s   $labl"
12455     } else {
12456       $$emit$$"jp,u,s   done\n\t"
12457       $$emit$$"j$cop,u,s  $labl\n\t"
12458       $$emit$$"done:"
12459     }
12460   %}
12461   size(4);
12462   ins_encode %{
12463     Label* l = $labl$$label;
12464     if ($cop$$cmpcode == Assembler::notEqual) {
12465       __ jccb(Assembler::parity, *l);
12466       __ jccb(Assembler::notEqual, *l);
12467     } else if ($cop$$cmpcode == Assembler::equal) {
12468       Label done;
12469       __ jccb(Assembler::parity, done);
12470       __ jccb(Assembler::equal, *l);
12471       __ bind(done);
12472     } else {
12473        ShouldNotReachHere();
12474     }
12475   %}
12476   ins_pipe(pipe_jcc);
12477   ins_short_branch(1);
12478 %}
12479 
12480 // ============================================================================
12481 // inlined locking and unlocking
12482 
12483 instruct cmpFastLockRTM(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI tmp, rdx_RegI scr, rRegI cx1, rRegI cx2) %{
12484   predicate(Compile::current()->use_rtm());
12485   match(Set cr (FastLock object box));
12486   effect(TEMP tmp, TEMP scr, TEMP cx1, TEMP cx2, USE_KILL box);
12487   ins_cost(300);
12488   format %{ "fastlock $object,$box\t! kills $box,$tmp,$scr,$cx1,$cx2" %}
12489   ins_encode %{
12490     __ fast_lock($object$$Register, $box$$Register, $tmp$$Register,
12491                  $scr$$Register, $cx1$$Register, $cx2$$Register,
12492                  _counters, _rtm_counters, _stack_rtm_counters,
12493                  ((Method*)(ra_->C->method()->constant_encoding()))->method_data(),
12494                  true, ra_->C->profile_rtm());
12495   %}
12496   ins_pipe(pipe_slow);
12497 %}
12498 
12499 instruct cmpFastLock(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI tmp, rRegP scr) %{
12500   predicate(!Compile::current()->use_rtm());
12501   match(Set cr (FastLock object box));
12502   effect(TEMP tmp, TEMP scr, USE_KILL box);
12503   ins_cost(300);
12504   format %{ "fastlock $object,$box\t! kills $box,$tmp,$scr" %}
12505   ins_encode %{
12506     __ fast_lock($object$$Register, $box$$Register, $tmp$$Register,
12507                  $scr$$Register, noreg, noreg, _counters, NULL, NULL, NULL, false, false);
12508   %}
12509   ins_pipe(pipe_slow);
12510 %}
12511 
12512 instruct cmpFastUnlock(rFlagsReg cr, rRegP object, rax_RegP box, rRegP tmp) %{
12513   match(Set cr (FastUnlock object box));
12514   effect(TEMP tmp, USE_KILL box);
12515   ins_cost(300);
12516   format %{ "fastunlock $object,$box\t! kills $box,$tmp" %}
12517   ins_encode %{
12518     __ fast_unlock($object$$Register, $box$$Register, $tmp$$Register, ra_->C->use_rtm());
12519   %}
12520   ins_pipe(pipe_slow);
12521 %}
12522 
12523 
12524 // ============================================================================
12525 // Safepoint Instructions
12526 instruct safePoint_poll(rFlagsReg cr)
12527 %{
12528   predicate(!Assembler::is_polling_page_far() && SafepointMechanism::uses_global_page_poll());
12529   match(SafePoint);
12530   effect(KILL cr);
12531 
12532   format %{ "testl  rax, [rip + #offset_to_poll_page]\t"
12533             "# Safepoint: poll for GC" %}
12534   ins_cost(125);
12535   ins_encode %{
12536     AddressLiteral addr(os::get_polling_page(), relocInfo::poll_type);
12537     __ testl(rax, addr);
12538   %}
12539   ins_pipe(ialu_reg_mem);
12540 %}
12541 
12542 instruct safePoint_poll_far(rFlagsReg cr, rRegP poll)
12543 %{
12544   predicate(Assembler::is_polling_page_far() && SafepointMechanism::uses_global_page_poll());
12545   match(SafePoint poll);
12546   effect(KILL cr, USE poll);
12547 
12548   format %{ "testl  rax, [$poll]\t"
12549             "# Safepoint: poll for GC" %}
12550   ins_cost(125);
12551   ins_encode %{
12552     __ relocate(relocInfo::poll_type);
12553     __ testl(rax, Address($poll$$Register, 0));
12554   %}
12555   ins_pipe(ialu_reg_mem);
12556 %}
12557 
12558 instruct safePoint_poll_tls(rFlagsReg cr, rex_RegP poll)
12559 %{
12560   predicate(SafepointMechanism::uses_thread_local_poll());
12561   match(SafePoint poll);
12562   effect(KILL cr, USE poll);
12563 
12564   format %{ "testl  rax, [$poll]\t"
12565             "# Safepoint: poll for GC" %}
12566   ins_cost(125);
12567   size(3); /* setting an explicit size will cause debug builds to assert if size is incorrect */
12568   ins_encode %{
12569     __ relocate(relocInfo::poll_type);
12570     address pre_pc = __ pc();
12571     __ testl(rax, Address($poll$$Register, 0));
12572     address post_pc = __ pc();
12573     guarantee(pre_pc[0] == 0x41 && pre_pc[1] == 0x85, "must emit #rex test-ax [reg]");
12574   %}
12575   ins_pipe(ialu_reg_mem);
12576 %}
12577 
12578 // ============================================================================
12579 // Procedure Call/Return Instructions
12580 // Call Java Static Instruction
12581 // Note: If this code changes, the corresponding ret_addr_offset() and
12582 //       compute_padding() functions will have to be adjusted.
12583 instruct CallStaticJavaDirect(method meth) %{
12584   match(CallStaticJava);
12585   effect(USE meth);
12586 
12587   ins_cost(300);
12588   format %{ "call,static " %}
12589   opcode(0xE8); /* E8 cd */
12590   ins_encode(clear_avx, Java_Static_Call(meth), call_epilog);
12591   ins_pipe(pipe_slow);
12592   ins_alignment(4);
12593 %}
12594 
12595 // Call Java Dynamic Instruction
12596 // Note: If this code changes, the corresponding ret_addr_offset() and
12597 //       compute_padding() functions will have to be adjusted.
12598 instruct CallDynamicJavaDirect(method meth)
12599 %{
12600   match(CallDynamicJava);
12601   effect(USE meth);
12602 
12603   ins_cost(300);
12604   format %{ "movq    rax, #Universe::non_oop_word()\n\t"
12605             "call,dynamic " %}
12606   ins_encode(clear_avx, Java_Dynamic_Call(meth), call_epilog);
12607   ins_pipe(pipe_slow);
12608   ins_alignment(4);
12609 %}
12610 
12611 // Call Runtime Instruction
12612 instruct CallRuntimeDirect(method meth)
12613 %{
12614   match(CallRuntime);
12615   effect(USE meth);
12616 
12617   ins_cost(300);
12618   format %{ "call,runtime " %}
12619   ins_encode(clear_avx, Java_To_Runtime(meth));
12620   ins_pipe(pipe_slow);
12621 %}
12622 
12623 // Call runtime without safepoint
12624 instruct CallLeafDirect(method meth)
12625 %{
12626   match(CallLeaf);
12627   effect(USE meth);
12628 
12629   ins_cost(300);
12630   format %{ "call_leaf,runtime " %}
12631   ins_encode(clear_avx, Java_To_Runtime(meth));
12632   ins_pipe(pipe_slow);
12633 %}
12634 
12635 // Call runtime without safepoint
12636 instruct CallLeafNoFPDirect(method meth)
12637 %{
12638   match(CallLeafNoFP);
12639   effect(USE meth);
12640 
12641   ins_cost(300);
12642   format %{ "call_leaf_nofp,runtime " %}
12643   ins_encode(clear_avx, Java_To_Runtime(meth));
12644   ins_pipe(pipe_slow);
12645 %}
12646 
12647 // Return Instruction
12648 // Remove the return address & jump to it.
12649 // Notice: We always emit a nop after a ret to make sure there is room
12650 // for safepoint patching
12651 instruct Ret()
12652 %{
12653   match(Return);
12654 
12655   format %{ "ret" %}
12656   opcode(0xC3);
12657   ins_encode(OpcP);
12658   ins_pipe(pipe_jmp);
12659 %}
12660 
12661 // Tail Call; Jump from runtime stub to Java code.
12662 // Also known as an 'interprocedural jump'.
12663 // Target of jump will eventually return to caller.
12664 // TailJump below removes the return address.
12665 instruct TailCalljmpInd(no_rbp_RegP jump_target, rbx_RegP method_oop)
12666 %{
12667   match(TailCall jump_target method_oop);
12668 
12669   ins_cost(300);
12670   format %{ "jmp     $jump_target\t# rbx holds method oop" %}
12671   opcode(0xFF, 0x4); /* Opcode FF /4 */
12672   ins_encode(REX_reg(jump_target), OpcP, reg_opc(jump_target));
12673   ins_pipe(pipe_jmp);
12674 %}
12675 
12676 // Tail Jump; remove the return address; jump to target.
12677 // TailCall above leaves the return address around.
12678 instruct tailjmpInd(no_rbp_RegP jump_target, rax_RegP ex_oop)
12679 %{
12680   match(TailJump jump_target ex_oop);
12681 
12682   ins_cost(300);
12683   format %{ "popq    rdx\t# pop return address\n\t"
12684             "jmp     $jump_target" %}
12685   opcode(0xFF, 0x4); /* Opcode FF /4 */
12686   ins_encode(Opcode(0x5a), // popq rdx
12687              REX_reg(jump_target), OpcP, reg_opc(jump_target));
12688   ins_pipe(pipe_jmp);
12689 %}
12690 
12691 // Create exception oop: created by stack-crawling runtime code.
12692 // Created exception is now available to this handler, and is setup
12693 // just prior to jumping to this handler.  No code emitted.
12694 instruct CreateException(rax_RegP ex_oop)
12695 %{
12696   match(Set ex_oop (CreateEx));
12697 
12698   size(0);
12699   // use the following format syntax
12700   format %{ "# exception oop is in rax; no code emitted" %}
12701   ins_encode();
12702   ins_pipe(empty);
12703 %}
12704 
12705 // Rethrow exception:
12706 // The exception oop will come in the first argument position.
12707 // Then JUMP (not call) to the rethrow stub code.
12708 instruct RethrowException()
12709 %{
12710   match(Rethrow);
12711 
12712   // use the following format syntax
12713   format %{ "jmp     rethrow_stub" %}
12714   ins_encode(enc_rethrow);
12715   ins_pipe(pipe_jmp);
12716 %}
12717 
12718 //
12719 // Execute ZGC load barrier (strong) slow path
12720 //
12721 
12722 // When running without XMM regs
12723 instruct loadBarrierSlowRegNoVec(rRegP dst, memory mem, rFlagsReg cr) %{
12724 
12725   match(Set dst (LoadBarrierSlowReg mem));
12726   predicate(MaxVectorSize < 16);
12727 
12728   effect(DEF dst, KILL cr);
12729 
12730   format %{"LoadBarrierSlowRegNoVec $dst, $mem" %}
12731   ins_encode %{
12732 #if INCLUDE_ZGC
12733     Register d = $dst$$Register;
12734     ZBarrierSetAssembler* bs = (ZBarrierSetAssembler*)BarrierSet::barrier_set()->barrier_set_assembler();
12735 
12736     assert(d != r12, "Can't be R12!");
12737     assert(d != r15, "Can't be R15!");
12738     assert(d != rsp, "Can't be RSP!");
12739 
12740     __ lea(d, $mem$$Address);
12741     __ call(RuntimeAddress(bs->load_barrier_slow_stub(d)));
12742 #else
12743     ShouldNotReachHere();
12744 #endif
12745   %}
12746   ins_pipe(pipe_slow);
12747 %}
12748 
12749 // For XMM and YMM enabled processors
12750 instruct loadBarrierSlowRegXmmAndYmm(rRegP dst, memory mem, rFlagsReg cr,
12751                                      rxmm0 x0, rxmm1 x1, rxmm2 x2,rxmm3 x3,
12752                                      rxmm4 x4, rxmm5 x5, rxmm6 x6, rxmm7 x7,
12753                                      rxmm8 x8, rxmm9 x9, rxmm10 x10, rxmm11 x11,
12754                                      rxmm12 x12, rxmm13 x13, rxmm14 x14, rxmm15 x15) %{
12755 
12756   match(Set dst (LoadBarrierSlowReg mem));
12757   predicate((UseSSE > 0) && (UseAVX <= 2) && (MaxVectorSize >= 16));
12758 
12759   effect(DEF dst, KILL cr,
12760          KILL x0, KILL x1, KILL x2, KILL x3,
12761          KILL x4, KILL x5, KILL x6, KILL x7,
12762          KILL x8, KILL x9, KILL x10, KILL x11,
12763          KILL x12, KILL x13, KILL x14, KILL x15);
12764 
12765   format %{"LoadBarrierSlowRegXmm $dst, $mem" %}
12766   ins_encode %{
12767 #if INCLUDE_ZGC
12768     Register d = $dst$$Register;
12769     ZBarrierSetAssembler* bs = (ZBarrierSetAssembler*)BarrierSet::barrier_set()->barrier_set_assembler();
12770 
12771     assert(d != r12, "Can't be R12!");
12772     assert(d != r15, "Can't be R15!");
12773     assert(d != rsp, "Can't be RSP!");
12774 
12775     __ lea(d, $mem$$Address);
12776     __ call(RuntimeAddress(bs->load_barrier_slow_stub(d)));
12777 #else
12778     ShouldNotReachHere();
12779 #endif
12780   %}
12781   ins_pipe(pipe_slow);
12782 %}
12783 
12784 // For ZMM enabled processors
12785 instruct loadBarrierSlowRegZmm(rRegP dst, memory mem, rFlagsReg cr,
12786                                rxmm0 x0, rxmm1 x1, rxmm2 x2,rxmm3 x3,
12787                                rxmm4 x4, rxmm5 x5, rxmm6 x6, rxmm7 x7,
12788                                rxmm8 x8, rxmm9 x9, rxmm10 x10, rxmm11 x11,
12789                                rxmm12 x12, rxmm13 x13, rxmm14 x14, rxmm15 x15,
12790                                rxmm16 x16, rxmm17 x17, rxmm18 x18, rxmm19 x19,
12791                                rxmm20 x20, rxmm21 x21, rxmm22 x22, rxmm23 x23,
12792                                rxmm24 x24, rxmm25 x25, rxmm26 x26, rxmm27 x27,
12793                                rxmm28 x28, rxmm29 x29, rxmm30 x30, rxmm31 x31) %{
12794 
12795   match(Set dst (LoadBarrierSlowReg mem));
12796   predicate((UseAVX == 3) && (MaxVectorSize >= 16));
12797 
12798   effect(DEF dst, KILL cr,
12799          KILL x0, KILL x1, KILL x2, KILL x3,
12800          KILL x4, KILL x5, KILL x6, KILL x7,
12801          KILL x8, KILL x9, KILL x10, KILL x11,
12802          KILL x12, KILL x13, KILL x14, KILL x15,
12803          KILL x16, KILL x17, KILL x18, KILL x19,
12804          KILL x20, KILL x21, KILL x22, KILL x23,
12805          KILL x24, KILL x25, KILL x26, KILL x27,
12806          KILL x28, KILL x29, KILL x30, KILL x31);
12807 
12808   format %{"LoadBarrierSlowRegZmm $dst, $mem" %}
12809   ins_encode %{
12810 #if INCLUDE_ZGC
12811     Register d = $dst$$Register;
12812     ZBarrierSetAssembler* bs = (ZBarrierSetAssembler*)BarrierSet::barrier_set()->barrier_set_assembler();
12813 
12814     assert(d != r12, "Can't be R12!");
12815     assert(d != r15, "Can't be R15!");
12816     assert(d != rsp, "Can't be RSP!");
12817 
12818     __ lea(d, $mem$$Address);
12819     __ call(RuntimeAddress(bs->load_barrier_slow_stub(d)));
12820 #else
12821     ShouldNotReachHere();
12822 #endif
12823   %}
12824   ins_pipe(pipe_slow);
12825 %}
12826 
12827 //
12828 // Execute ZGC load barrier (weak) slow path
12829 //
12830 
12831 // When running without XMM regs
12832 instruct loadBarrierWeakSlowRegNoVec(rRegP dst, memory mem, rFlagsReg cr) %{
12833 
12834   match(Set dst (LoadBarrierSlowReg mem));
12835   predicate(MaxVectorSize < 16);
12836 
12837   effect(DEF dst, KILL cr);
12838 
12839   format %{"LoadBarrierSlowRegNoVec $dst, $mem" %}
12840   ins_encode %{
12841 #if INCLUDE_ZGC
12842     Register d = $dst$$Register;
12843     ZBarrierSetAssembler* bs = (ZBarrierSetAssembler*)BarrierSet::barrier_set()->barrier_set_assembler();
12844 
12845     assert(d != r12, "Can't be R12!");
12846     assert(d != r15, "Can't be R15!");
12847     assert(d != rsp, "Can't be RSP!");
12848 
12849     __ lea(d, $mem$$Address);
12850     __ call(RuntimeAddress(bs->load_barrier_weak_slow_stub(d)));
12851 #else
12852     ShouldNotReachHere();
12853 #endif
12854   %}
12855   ins_pipe(pipe_slow);
12856 %}
12857 
12858 // For XMM and YMM enabled processors
12859 instruct loadBarrierWeakSlowRegXmmAndYmm(rRegP dst, memory mem, rFlagsReg cr,
12860                                          rxmm0 x0, rxmm1 x1, rxmm2 x2,rxmm3 x3,
12861                                          rxmm4 x4, rxmm5 x5, rxmm6 x6, rxmm7 x7,
12862                                          rxmm8 x8, rxmm9 x9, rxmm10 x10, rxmm11 x11,
12863                                          rxmm12 x12, rxmm13 x13, rxmm14 x14, rxmm15 x15) %{
12864 
12865   match(Set dst (LoadBarrierWeakSlowReg mem));
12866   predicate((UseSSE > 0) && (UseAVX <= 2) && (MaxVectorSize >= 16));
12867 
12868   effect(DEF dst, KILL cr,
12869          KILL x0, KILL x1, KILL x2, KILL x3,
12870          KILL x4, KILL x5, KILL x6, KILL x7,
12871          KILL x8, KILL x9, KILL x10, KILL x11,
12872          KILL x12, KILL x13, KILL x14, KILL x15);
12873 
12874   format %{"LoadBarrierWeakSlowRegXmm $dst, $mem" %}
12875   ins_encode %{
12876 #if INCLUDE_ZGC
12877     Register d = $dst$$Register;
12878     ZBarrierSetAssembler* bs = (ZBarrierSetAssembler*)BarrierSet::barrier_set()->barrier_set_assembler();
12879 
12880     assert(d != r12, "Can't be R12!");
12881     assert(d != r15, "Can't be R15!");
12882     assert(d != rsp, "Can't be RSP!");
12883 
12884     __ lea(d,$mem$$Address);
12885     __ call(RuntimeAddress(bs->load_barrier_weak_slow_stub(d)));
12886 #else
12887     ShouldNotReachHere();
12888 #endif
12889   %}
12890   ins_pipe(pipe_slow);
12891 %}
12892 
12893 // For ZMM enabled processors
12894 instruct loadBarrierWeakSlowRegZmm(rRegP dst, memory mem, rFlagsReg cr,
12895                                    rxmm0 x0, rxmm1 x1, rxmm2 x2,rxmm3 x3,
12896                                    rxmm4 x4, rxmm5 x5, rxmm6 x6, rxmm7 x7,
12897                                    rxmm8 x8, rxmm9 x9, rxmm10 x10, rxmm11 x11,
12898                                    rxmm12 x12, rxmm13 x13, rxmm14 x14, rxmm15 x15,
12899                                    rxmm16 x16, rxmm17 x17, rxmm18 x18, rxmm19 x19,
12900                                    rxmm20 x20, rxmm21 x21, rxmm22 x22, rxmm23 x23,
12901                                    rxmm24 x24, rxmm25 x25, rxmm26 x26, rxmm27 x27,
12902                                    rxmm28 x28, rxmm29 x29, rxmm30 x30, rxmm31 x31) %{
12903 
12904   match(Set dst (LoadBarrierWeakSlowReg mem));
12905   predicate((UseAVX == 3) && (MaxVectorSize >= 16));
12906 
12907   effect(DEF dst, KILL cr,
12908          KILL x0, KILL x1, KILL x2, KILL x3,
12909          KILL x4, KILL x5, KILL x6, KILL x7,
12910          KILL x8, KILL x9, KILL x10, KILL x11,
12911          KILL x12, KILL x13, KILL x14, KILL x15,
12912          KILL x16, KILL x17, KILL x18, KILL x19,
12913          KILL x20, KILL x21, KILL x22, KILL x23,
12914          KILL x24, KILL x25, KILL x26, KILL x27,
12915          KILL x28, KILL x29, KILL x30, KILL x31);
12916 
12917   format %{"LoadBarrierWeakSlowRegZmm $dst, $mem" %}
12918   ins_encode %{
12919 #if INCLUDE_ZGC
12920     Register d = $dst$$Register;
12921     ZBarrierSetAssembler* bs = (ZBarrierSetAssembler*)BarrierSet::barrier_set()->barrier_set_assembler();
12922 
12923     assert(d != r12, "Can't be R12!");
12924     assert(d != r15, "Can't be R15!");
12925     assert(d != rsp, "Can't be RSP!");
12926 
12927     __ lea(d,$mem$$Address);
12928     __ call(RuntimeAddress(bs->load_barrier_weak_slow_stub(d)));
12929 #else
12930     ShouldNotReachHere();
12931 #endif
12932   %}
12933   ins_pipe(pipe_slow);
12934 %}
12935 
12936 // ============================================================================
12937 // This name is KNOWN by the ADLC and cannot be changed.
12938 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
12939 // for this guy.
12940 instruct tlsLoadP(r15_RegP dst) %{
12941   match(Set dst (ThreadLocal));
12942   effect(DEF dst);
12943 
12944   size(0);
12945   format %{ "# TLS is in R15" %}
12946   ins_encode( /*empty encoding*/ );
12947   ins_pipe(ialu_reg_reg);
12948 %}
12949 
12950 
12951 //----------PEEPHOLE RULES-----------------------------------------------------
12952 // These must follow all instruction definitions as they use the names
12953 // defined in the instructions definitions.
12954 //
12955 // peepmatch ( root_instr_name [preceding_instruction]* );
12956 //
12957 // peepconstraint %{
12958 // (instruction_number.operand_name relational_op instruction_number.operand_name
12959 //  [, ...] );
12960 // // instruction numbers are zero-based using left to right order in peepmatch
12961 //
12962 // peepreplace ( instr_name  ( [instruction_number.operand_name]* ) );
12963 // // provide an instruction_number.operand_name for each operand that appears
12964 // // in the replacement instruction's match rule
12965 //
12966 // ---------VM FLAGS---------------------------------------------------------
12967 //
12968 // All peephole optimizations can be turned off using -XX:-OptoPeephole
12969 //
12970 // Each peephole rule is given an identifying number starting with zero and
12971 // increasing by one in the order seen by the parser.  An individual peephole
12972 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
12973 // on the command-line.
12974 //
12975 // ---------CURRENT LIMITATIONS----------------------------------------------
12976 //
12977 // Only match adjacent instructions in same basic block
12978 // Only equality constraints
12979 // Only constraints between operands, not (0.dest_reg == RAX_enc)
12980 // Only one replacement instruction
12981 //
12982 // ---------EXAMPLE----------------------------------------------------------
12983 //
12984 // // pertinent parts of existing instructions in architecture description
12985 // instruct movI(rRegI dst, rRegI src)
12986 // %{
12987 //   match(Set dst (CopyI src));
12988 // %}
12989 //
12990 // instruct incI_rReg(rRegI dst, immI1 src, rFlagsReg cr)
12991 // %{
12992 //   match(Set dst (AddI dst src));
12993 //   effect(KILL cr);
12994 // %}
12995 //
12996 // // Change (inc mov) to lea
12997 // peephole %{
12998 //   // increment preceeded by register-register move
12999 //   peepmatch ( incI_rReg movI );
13000 //   // require that the destination register of the increment
13001 //   // match the destination register of the move
13002 //   peepconstraint ( 0.dst == 1.dst );
13003 //   // construct a replacement instruction that sets
13004 //   // the destination to ( move's source register + one )
13005 //   peepreplace ( leaI_rReg_immI( 0.dst 1.src 0.src ) );
13006 // %}
13007 //
13008 
13009 // Implementation no longer uses movX instructions since
13010 // machine-independent system no longer uses CopyX nodes.
13011 //
13012 // peephole
13013 // %{
13014 //   peepmatch (incI_rReg movI);
13015 //   peepconstraint (0.dst == 1.dst);
13016 //   peepreplace (leaI_rReg_immI(0.dst 1.src 0.src));
13017 // %}
13018 
13019 // peephole
13020 // %{
13021 //   peepmatch (decI_rReg movI);
13022 //   peepconstraint (0.dst == 1.dst);
13023 //   peepreplace (leaI_rReg_immI(0.dst 1.src 0.src));
13024 // %}
13025 
13026 // peephole
13027 // %{
13028 //   peepmatch (addI_rReg_imm movI);
13029 //   peepconstraint (0.dst == 1.dst);
13030 //   peepreplace (leaI_rReg_immI(0.dst 1.src 0.src));
13031 // %}
13032 
13033 // peephole
13034 // %{
13035 //   peepmatch (incL_rReg movL);
13036 //   peepconstraint (0.dst == 1.dst);
13037 //   peepreplace (leaL_rReg_immL(0.dst 1.src 0.src));
13038 // %}
13039 
13040 // peephole
13041 // %{
13042 //   peepmatch (decL_rReg movL);
13043 //   peepconstraint (0.dst == 1.dst);
13044 //   peepreplace (leaL_rReg_immL(0.dst 1.src 0.src));
13045 // %}
13046 
13047 // peephole
13048 // %{
13049 //   peepmatch (addL_rReg_imm movL);
13050 //   peepconstraint (0.dst == 1.dst);
13051 //   peepreplace (leaL_rReg_immL(0.dst 1.src 0.src));
13052 // %}
13053 
13054 // peephole
13055 // %{
13056 //   peepmatch (addP_rReg_imm movP);
13057 //   peepconstraint (0.dst == 1.dst);
13058 //   peepreplace (leaP_rReg_imm(0.dst 1.src 0.src));
13059 // %}
13060 
13061 // // Change load of spilled value to only a spill
13062 // instruct storeI(memory mem, rRegI src)
13063 // %{
13064 //   match(Set mem (StoreI mem src));
13065 // %}
13066 //
13067 // instruct loadI(rRegI dst, memory mem)
13068 // %{
13069 //   match(Set dst (LoadI mem));
13070 // %}
13071 //
13072 
13073 peephole
13074 %{
13075   peepmatch (loadI storeI);
13076   peepconstraint (1.src == 0.dst, 1.mem == 0.mem);
13077   peepreplace (storeI(1.mem 1.mem 1.src));
13078 %}
13079 
13080 peephole
13081 %{
13082   peepmatch (loadL storeL);
13083   peepconstraint (1.src == 0.dst, 1.mem == 0.mem);
13084   peepreplace (storeL(1.mem 1.mem 1.src));
13085 %}
13086 
13087 //----------SMARTSPILL RULES---------------------------------------------------
13088 // These must follow all instruction definitions as they use the names
13089 // defined in the instructions definitions.
--- EOF ---