1 //
   2 // Copyright (c) 2003, 2013, Oracle and/or its affiliates. All rights reserved.
   3 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4 //
   5 // This code is free software; you can redistribute it and/or modify it
   6 // under the terms of the GNU General Public License version 2 only, as
   7 // published by the Free Software Foundation.
   8 //
   9 // This code is distributed in the hope that it will be useful, but WITHOUT
  10 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11 // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12 // version 2 for more details (a copy is included in the LICENSE file that
  13 // accompanied this code).
  14 //
  15 // You should have received a copy of the GNU General Public License version
  16 // 2 along with this work; if not, write to the Free Software Foundation,
  17 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18 //
  19 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20 // or visit www.oracle.com if you need additional information or have any
  21 // questions.
  22 //
  23 //
  24 
  25 // AMD64 Architecture Description File
  26 
  27 //----------REGISTER DEFINITION BLOCK------------------------------------------
  28 // This information is used by the matcher and the register allocator to
  29 // describe individual registers and classes of registers within the target
  30 // archtecture.
  31 
  32 register %{
  33 //----------Architecture Description Register Definitions----------------------
  34 // General Registers
  35 // "reg_def"  name ( register save type, C convention save type,
  36 //                   ideal register type, encoding );
  37 // Register Save Types:
  38 //
  39 // NS  = No-Save:       The register allocator assumes that these registers
  40 //                      can be used without saving upon entry to the method, &
  41 //                      that they do not need to be saved at call sites.
  42 //
  43 // SOC = Save-On-Call:  The register allocator assumes that these registers
  44 //                      can be used without saving upon entry to the method,
  45 //                      but that they must be saved at call sites.
  46 //
  47 // SOE = Save-On-Entry: The register allocator assumes that these registers
  48 //                      must be saved before using them upon entry to the
  49 //                      method, but they do not need to be saved at call
  50 //                      sites.
  51 //
  52 // AS  = Always-Save:   The register allocator assumes that these registers
  53 //                      must be saved before using them upon entry to the
  54 //                      method, & that they must be saved at call sites.
  55 //
  56 // Ideal Register Type is used to determine how to save & restore a
  57 // register.  Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
  58 // spilled with LoadP/StoreP.  If the register supports both, use Op_RegI.
  59 //
  60 // The encoding number is the actual bit-pattern placed into the opcodes.
  61 
  62 // General Registers
  63 // R8-R15 must be encoded with REX.  (RSP, RBP, RSI, RDI need REX when
  64 // used as byte registers)
  65 
  66 // Previously set RBX, RSI, and RDI as save-on-entry for java code
  67 // Turn off SOE in java-code due to frequent use of uncommon-traps.
  68 // Now that allocator is better, turn on RSI and RDI as SOE registers.
  69 
  70 reg_def RAX  (SOC, SOC, Op_RegI,  0, rax->as_VMReg());
  71 reg_def RAX_H(SOC, SOC, Op_RegI,  0, rax->as_VMReg()->next());
  72 
  73 reg_def RCX  (SOC, SOC, Op_RegI,  1, rcx->as_VMReg());
  74 reg_def RCX_H(SOC, SOC, Op_RegI,  1, rcx->as_VMReg()->next());
  75 
  76 reg_def RDX  (SOC, SOC, Op_RegI,  2, rdx->as_VMReg());
  77 reg_def RDX_H(SOC, SOC, Op_RegI,  2, rdx->as_VMReg()->next());
  78 
  79 reg_def RBX  (SOC, SOE, Op_RegI,  3, rbx->as_VMReg());
  80 reg_def RBX_H(SOC, SOE, Op_RegI,  3, rbx->as_VMReg()->next());
  81 
  82 reg_def RSP  (NS,  NS,  Op_RegI,  4, rsp->as_VMReg());
  83 reg_def RSP_H(NS,  NS,  Op_RegI,  4, rsp->as_VMReg()->next());
  84 
  85 // now that adapter frames are gone RBP is always saved and restored by the prolog/epilog code
  86 reg_def RBP  (NS, SOE, Op_RegI,  5, rbp->as_VMReg());
  87 reg_def RBP_H(NS, SOE, Op_RegI,  5, rbp->as_VMReg()->next());
  88 
  89 #ifdef _WIN64
  90 
  91 reg_def RSI  (SOC, SOE, Op_RegI,  6, rsi->as_VMReg());
  92 reg_def RSI_H(SOC, SOE, Op_RegI,  6, rsi->as_VMReg()->next());
  93 
  94 reg_def RDI  (SOC, SOE, Op_RegI,  7, rdi->as_VMReg());
  95 reg_def RDI_H(SOC, SOE, Op_RegI,  7, rdi->as_VMReg()->next());
  96 
  97 #else
  98 
  99 reg_def RSI  (SOC, SOC, Op_RegI,  6, rsi->as_VMReg());
 100 reg_def RSI_H(SOC, SOC, Op_RegI,  6, rsi->as_VMReg()->next());
 101 
 102 reg_def RDI  (SOC, SOC, Op_RegI,  7, rdi->as_VMReg());
 103 reg_def RDI_H(SOC, SOC, Op_RegI,  7, rdi->as_VMReg()->next());
 104 
 105 #endif
 106 
 107 reg_def R8   (SOC, SOC, Op_RegI,  8, r8->as_VMReg());
 108 reg_def R8_H (SOC, SOC, Op_RegI,  8, r8->as_VMReg()->next());
 109 
 110 reg_def R9   (SOC, SOC, Op_RegI,  9, r9->as_VMReg());
 111 reg_def R9_H (SOC, SOC, Op_RegI,  9, r9->as_VMReg()->next());
 112 
 113 reg_def R10  (SOC, SOC, Op_RegI, 10, r10->as_VMReg());
 114 reg_def R10_H(SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
 115 
 116 reg_def R11  (SOC, SOC, Op_RegI, 11, r11->as_VMReg());
 117 reg_def R11_H(SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
 118 
 119 reg_def R12  (SOC, SOE, Op_RegI, 12, r12->as_VMReg());
 120 reg_def R12_H(SOC, SOE, Op_RegI, 12, r12->as_VMReg()->next());
 121 
 122 reg_def R13  (SOC, SOE, Op_RegI, 13, r13->as_VMReg());
 123 reg_def R13_H(SOC, SOE, Op_RegI, 13, r13->as_VMReg()->next());
 124 
 125 reg_def R14  (SOC, SOE, Op_RegI, 14, r14->as_VMReg());
 126 reg_def R14_H(SOC, SOE, Op_RegI, 14, r14->as_VMReg()->next());
 127 
 128 reg_def R15  (SOC, SOE, Op_RegI, 15, r15->as_VMReg());
 129 reg_def R15_H(SOC, SOE, Op_RegI, 15, r15->as_VMReg()->next());
 130 
 131 
 132 // Floating Point Registers
 133 
 134 // Specify priority of register selection within phases of register
 135 // allocation.  Highest priority is first.  A useful heuristic is to
 136 // give registers a low priority when they are required by machine
 137 // instructions, like EAX and EDX on I486, and choose no-save registers
 138 // before save-on-call, & save-on-call before save-on-entry.  Registers
 139 // which participate in fixed calling sequences should come last.
 140 // Registers which are used as pairs must fall on an even boundary.
 141 
 142 alloc_class chunk0(R10,         R10_H,
 143                    R11,         R11_H,
 144                    R8,          R8_H,
 145                    R9,          R9_H,
 146                    R12,         R12_H,
 147                    RCX,         RCX_H,
 148                    RBX,         RBX_H,
 149                    RDI,         RDI_H,
 150                    RDX,         RDX_H,
 151                    RSI,         RSI_H,
 152                    RAX,         RAX_H,
 153                    RBP,         RBP_H,
 154                    R13,         R13_H,
 155                    R14,         R14_H,
 156                    R15,         R15_H,
 157                    RSP,         RSP_H);
 158 
 159 
 160 //----------Architecture Description Register Classes--------------------------
 161 // Several register classes are automatically defined based upon information in
 162 // this architecture description.
 163 // 1) reg_class inline_cache_reg           ( /* as def'd in frame section */ )
 164 // 2) reg_class compiler_method_oop_reg    ( /* as def'd in frame section */ )
 165 // 2) reg_class interpreter_method_oop_reg ( /* as def'd in frame section */ )
 166 // 3) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
 167 //
 168 
 169 // Empty register class.
 170 reg_class no_reg();
 171 
 172 // Class for all pointer registers (including RSP and RBP)
 173 reg_class any_reg_with_rbp(RAX, RAX_H,
 174                            RDX, RDX_H,
 175                            RBP, RBP_H,               
 176                            RDI, RDI_H,
 177                            RSI, RSI_H,
 178                            RCX, RCX_H,
 179                            RBX, RBX_H,
 180                            RSP, RSP_H,
 181                            R8,  R8_H,
 182                            R9,  R9_H,
 183                            R10, R10_H,
 184                            R11, R11_H,
 185                            R12, R12_H,
 186                            R13, R13_H,
 187                            R14, R14_H,
 188                            R15, R15_H);
 189 
 190 // Class for all pointer registers (including RSP, but excluding RBP)
 191 reg_class any_reg_no_rbp(RAX, RAX_H,
 192                          RDX, RDX_H,                
 193                          RDI, RDI_H,
 194                          RSI, RSI_H,
 195                          RCX, RCX_H,
 196                          RBX, RBX_H,
 197                          RSP, RSP_H,
 198                          R8,  R8_H,
 199                          R9,  R9_H,
 200                          R10, R10_H,
 201                          R11, R11_H,
 202                          R12, R12_H,
 203                          R13, R13_H,
 204                          R14, R14_H,
 205                          R15, R15_H);
 206 
 207 // Dynamic register class that selects at runtime between register classes
 208 // any_reg_no_rbp and any_reg_with_rbp (depending on the value of the flag PreserveFramePointer). 
 209 // Equivalent to: return PreserveFramePointer ? any_reg_no_rbp : any_reg_with_rbp;
 210 reg_class_dynamic any_reg(any_reg_no_rbp, any_reg_with_rbp, %{ PreserveFramePointer %});
 211                   
 212 // Class for all pointer registers (excluding RSP)
 213 reg_class ptr_reg_with_rbp(RAX, RAX_H,
 214                            RDX, RDX_H,
 215                            RBP, RBP_H,
 216                            RDI, RDI_H,
 217                            RSI, RSI_H,
 218                            RCX, RCX_H,
 219                            RBX, RBX_H,
 220                            R8,  R8_H,
 221                            R9,  R9_H,
 222                            R10, R10_H,
 223                            R11, R11_H,
 224                            R13, R13_H,
 225                            R14, R14_H);
 226 
 227 // Class for all pointer registers (excluding RSP and RBP)
 228 reg_class ptr_reg_no_rbp(RAX, RAX_H,
 229                          RDX, RDX_H,                         
 230                          RDI, RDI_H,
 231                          RSI, RSI_H,
 232                          RCX, RCX_H,
 233                          RBX, RBX_H,
 234                          R8,  R8_H,
 235                          R9,  R9_H,
 236                          R10, R10_H,
 237                          R11, R11_H,
 238                          R13, R13_H,
 239                          R14, R14_H);
 240 
 241 // Dynamic register class that selects between ptr_reg_no_rbp and ptr_reg_with_rbp.
 242 reg_class_dynamic ptr_reg(ptr_reg_no_rbp, ptr_reg_with_rbp, %{ PreserveFramePointer %});
 243 
 244 // Class for all pointer registers (excluding RAX and RSP)
 245 reg_class ptr_no_rax_reg_with_rbp(RDX, RDX_H,
 246                                   RBP, RBP_H,
 247                                   RDI, RDI_H,
 248                                   RSI, RSI_H,
 249                                   RCX, RCX_H,
 250                                   RBX, RBX_H,
 251                                   R8,  R8_H,
 252                                   R9,  R9_H,
 253                                   R10, R10_H,
 254                                   R11, R11_H,
 255                                   R13, R13_H,
 256                                   R14, R14_H);
 257 
 258 // Class for all pointer registers (excluding RAX, RSP, and RBP)
 259 reg_class ptr_no_rax_reg_no_rbp(RDX, RDX_H,
 260                                 RDI, RDI_H,
 261                                 RSI, RSI_H,
 262                                 RCX, RCX_H,
 263                                 RBX, RBX_H,
 264                                 R8,  R8_H,
 265                                 R9,  R9_H,
 266                                 R10, R10_H,
 267                                 R11, R11_H,
 268                                 R13, R13_H,
 269                                 R14, R14_H);
 270 
 271 // Dynamic register class that selects between ptr_no_rax_reg_no_rbp and ptr_no_rax_reg_with_rbp.
 272 reg_class_dynamic ptr_no_rax_reg(ptr_no_rax_reg_no_rbp, ptr_no_rax_reg_with_rbp, %{ PreserveFramePointer %});
 273 
 274 // Class for all pointer registers (excluding RAX, RBX, and RSP)
 275 reg_class ptr_no_rax_rbx_reg_with_rbp(RDX, RDX_H,
 276                                       RBP, RBP_H,
 277                                       RDI, RDI_H,
 278                                       RSI, RSI_H,
 279                                       RCX, RCX_H,
 280                                       R8,  R8_H,
 281                                       R9,  R9_H,
 282                                       R10, R10_H,
 283                                       R11, R11_H,
 284                                       R13, R13_H,
 285                                       R14, R14_H);
 286 
 287 // Class for all pointer registers (excluding RAX, RBX, RSP, and RBP)
 288 reg_class ptr_no_rax_rbx_reg_no_rbp(RDX, RDX_H,
 289                                     RDI, RDI_H,
 290                                     RSI, RSI_H,
 291                                     RCX, RCX_H,
 292                                     R8,  R8_H,
 293                                     R9,  R9_H,
 294                                     R10, R10_H,
 295                                     R11, R11_H,
 296                                     R13, R13_H,
 297                                     R14, R14_H);
 298 
 299 // Dynamic register class that selects between ptr_no_rax_rbx_reg_no_rbp and ptr_no_rax_rbx_reg_with_rbp.
 300 reg_class_dynamic ptr_no_rax_rbx_reg(ptr_no_rax_rbx_reg_no_rbp, ptr_no_rax_rbx_reg_with_rbp, %{ PreserveFramePointer %});
 301 
 302 // Singleton class for RAX pointer register
 303 reg_class ptr_rax_reg(RAX, RAX_H);
 304 
 305 // Singleton class for RBX pointer register
 306 reg_class ptr_rbx_reg(RBX, RBX_H);
 307 
 308 // Singleton class for RSI pointer register
 309 reg_class ptr_rsi_reg(RSI, RSI_H);
 310 
 311 // Singleton class for RDI pointer register
 312 reg_class ptr_rdi_reg(RDI, RDI_H);
 313 
 314 // Singleton class for stack pointer
 315 reg_class ptr_rsp_reg(RSP, RSP_H);
 316 
 317 // Singleton class for TLS pointer
 318 reg_class ptr_r15_reg(R15, R15_H);
 319 
 320 // Class for all long registers (excluding RSP)
 321 reg_class long_reg_with_rbp(RAX, RAX_H,
 322                             RDX, RDX_H,
 323                             RBP, RBP_H,
 324                             RDI, RDI_H,
 325                             RSI, RSI_H,
 326                             RCX, RCX_H,
 327                             RBX, RBX_H,
 328                             R8,  R8_H,
 329                             R9,  R9_H,
 330                             R10, R10_H,
 331                             R11, R11_H,
 332                             R13, R13_H,
 333                             R14, R14_H);
 334 
 335 // Class for all long registers (excluding RSP and RBP)
 336 reg_class long_reg_no_rbp(RAX, RAX_H,
 337                           RDX, RDX_H,
 338                           RDI, RDI_H,
 339                           RSI, RSI_H,
 340                           RCX, RCX_H,
 341                           RBX, RBX_H,
 342                           R8,  R8_H,
 343                           R9,  R9_H,
 344                           R10, R10_H,
 345                           R11, R11_H,
 346                           R13, R13_H,
 347                           R14, R14_H);
 348 
 349 // Dynamic register class that selects between long_reg_no_rbp and long_reg_with_rbp.
 350 reg_class_dynamic long_reg(long_reg_no_rbp, long_reg_with_rbp, %{ PreserveFramePointer %});
 351 
 352 // Class for all long registers (excluding RAX, RDX and RSP)
 353 reg_class long_no_rax_rdx_reg_with_rbp(RBP, RBP_H,
 354                                        RDI, RDI_H,
 355                                        RSI, RSI_H,
 356                                        RCX, RCX_H,
 357                                        RBX, RBX_H,
 358                                        R8,  R8_H,
 359                                        R9,  R9_H,
 360                                        R10, R10_H,
 361                                        R11, R11_H,
 362                                        R13, R13_H,
 363                                        R14, R14_H);
 364 
 365 // Class for all long registers (excluding RAX, RDX, RSP, and RBP)
 366 reg_class long_no_rax_rdx_reg_no_rbp(RDI, RDI_H,
 367                                      RSI, RSI_H,
 368                                      RCX, RCX_H,
 369                                      RBX, RBX_H,
 370                                      R8,  R8_H,
 371                                      R9,  R9_H,
 372                                      R10, R10_H,
 373                                      R11, R11_H,
 374                                      R13, R13_H,
 375                                      R14, R14_H);
 376 
 377 // Dynamic register class that selects between long_no_rax_rdx_reg_no_rbp and long_no_rax_rdx_reg_with_rbp.
 378 reg_class_dynamic long_no_rax_rdx_reg(long_no_rax_rdx_reg_no_rbp, long_no_rax_rdx_reg_with_rbp, %{ PreserveFramePointer %});
 379 
 380 // Class for all long registers (excluding RCX and RSP)
 381 reg_class long_no_rcx_reg_with_rbp(RBP, RBP_H,
 382                                    RDI, RDI_H,
 383                                    RSI, RSI_H,
 384                                    RAX, RAX_H,
 385                                    RDX, RDX_H,
 386                                    RBX, RBX_H,
 387                                    R8,  R8_H,
 388                                    R9,  R9_H,
 389                                    R10, R10_H,
 390                                    R11, R11_H,
 391                                    R13, R13_H,
 392                                    R14, R14_H);
 393 
 394 // Class for all long registers (excluding RCX, RSP, and RBP)
 395 reg_class long_no_rcx_reg_no_rbp(RDI, RDI_H,
 396                                  RSI, RSI_H,
 397                                  RAX, RAX_H,
 398                                  RDX, RDX_H,
 399                                  RBX, RBX_H,
 400                                  R8,  R8_H,
 401                                  R9,  R9_H,
 402                                  R10, R10_H,
 403                                  R11, R11_H,
 404                                  R13, R13_H,
 405                                  R14, R14_H);
 406 
 407 // Dynamic register class that selects between long_no_rcx_reg_no_rbp and long_no_rcx_reg_with_rbp.
 408 reg_class_dynamic long_no_rcx_reg(long_no_rcx_reg_no_rbp, long_no_rcx_reg_with_rbp, %{ PreserveFramePointer %});
 409 
 410 // Singleton class for RAX long register
 411 reg_class long_rax_reg(RAX, RAX_H);
 412 
 413 // Singleton class for RCX long register
 414 reg_class long_rcx_reg(RCX, RCX_H);
 415 
 416 // Singleton class for RDX long register
 417 reg_class long_rdx_reg(RDX, RDX_H);
 418 
 419 // Class for all int registers (excluding RSP)
 420 reg_class int_reg_with_rbp(RAX,
 421                            RDX,
 422                            RBP,
 423                            RDI,
 424                            RSI,
 425                            RCX,
 426                            RBX,
 427                            R8,
 428                            R9,
 429                            R10,
 430                            R11,
 431                            R13,
 432                            R14);
 433 
 434 // Class for all int registers (excluding RSP and RBP)
 435 reg_class int_reg_no_rbp(RAX,
 436                          RDX,
 437                          RDI,
 438                          RSI,
 439                          RCX,
 440                          RBX,
 441                          R8,
 442                          R9,
 443                          R10,
 444                          R11,
 445                          R13,
 446                          R14);
 447 
 448 // Dynamic register class that selects between int_reg_no_rbp and int_reg_with_rbp.
 449 reg_class_dynamic int_reg(int_reg_no_rbp, int_reg_with_rbp, %{ PreserveFramePointer %});
 450 
 451 // Class for all int registers (excluding RCX and RSP)
 452 reg_class int_no_rcx_reg_with_rbp(RAX,
 453                                   RDX,
 454                                   RBP,
 455                                   RDI,
 456                                   RSI,
 457                                   RBX,
 458                                   R8,
 459                                   R9,
 460                                   R10,
 461                                   R11,
 462                                   R13,
 463                                   R14);
 464 
 465 // Class for all int registers (excluding RCX, RSP, and RBP)
 466 reg_class int_no_rcx_reg_no_rbp(RAX,
 467                                 RDX,
 468                                 RDI,
 469                                 RSI,
 470                                 RBX,
 471                                 R8,
 472                                 R9,
 473                                 R10,
 474                                 R11,
 475                                 R13,
 476                                 R14);
 477 
 478 // Dynamic register class that selects between int_no_rcx_reg_no_rbp and int_no_rcx_reg_with_rbp.
 479 reg_class_dynamic int_no_rcx_reg(int_no_rcx_reg_no_rbp, int_no_rcx_reg_with_rbp, %{ PreserveFramePointer %});
 480 
 481 // Class for all int registers (excluding RAX, RDX, and RSP)
 482 reg_class int_no_rax_rdx_reg_with_rbp(RBP,
 483                                       RDI,
 484                                       RSI,
 485                                       RCX,
 486                                       RBX,
 487                                       R8,
 488                                       R9,
 489                                       R10,
 490                                       R11,
 491                                       R13,
 492                                       R14);
 493 
 494 // Class for all int registers (excluding RAX, RDX, RSP, and RBP)
 495 reg_class int_no_rax_rdx_reg_no_rbp(RDI,
 496                                     RSI,
 497                                     RCX,
 498                                     RBX,
 499                                     R8,
 500                                     R9,
 501                                     R10,
 502                                     R11,
 503                                     R13,
 504                                     R14);
 505 
 506 // Dynamic register class that selects between int_no_rax_rdx_reg_no_rbp and int_no_rax_rdx_reg_with_rbp.
 507 reg_class_dynamic int_no_rax_rdx_reg(int_no_rax_rdx_reg_no_rbp, int_no_rax_rdx_reg_with_rbp, %{ PreserveFramePointer %});
 508 
 509 // Singleton class for RAX int register
 510 reg_class int_rax_reg(RAX);
 511 
 512 // Singleton class for RBX int register
 513 reg_class int_rbx_reg(RBX);
 514 
 515 // Singleton class for RCX int register
 516 reg_class int_rcx_reg(RCX);
 517 
 518 // Singleton class for RCX int register
 519 reg_class int_rdx_reg(RDX);
 520 
 521 // Singleton class for RCX int register
 522 reg_class int_rdi_reg(RDI);
 523 
 524 // Singleton class for instruction pointer
 525 // reg_class ip_reg(RIP);
 526 
 527 %}
 528 
 529 //----------SOURCE BLOCK-------------------------------------------------------
 530 // This is a block of C++ code which provides values, functions, and
 531 // definitions necessary in the rest of the architecture description
 532 source %{
 533 #define   RELOC_IMM64    Assembler::imm_operand
 534 #define   RELOC_DISP32   Assembler::disp32_operand
 535 
 536 #define __ _masm.
 537 
 538 static int clear_avx_size() {
 539   return (Compile::current()->max_vector_size() > 16) ? 3 : 0;  // vzeroupper
 540 }
 541 
 542 // !!!!! Special hack to get all types of calls to specify the byte offset
 543 //       from the start of the call to the point where the return address
 544 //       will point.
 545 int MachCallStaticJavaNode::ret_addr_offset()
 546 {
 547   int offset = 5; // 5 bytes from start of call to where return address points
 548   offset += clear_avx_size();  
 549   return offset;
 550 }
 551 
 552 int MachCallDynamicJavaNode::ret_addr_offset()
 553 {
 554   int offset = 15; // 15 bytes from start of call to where return address points
 555   offset += clear_avx_size();
 556   return offset;
 557 }
 558 
 559 int MachCallRuntimeNode::ret_addr_offset() {
 560   int offset = 13; // movq r10,#addr; callq (r10)
 561   offset += clear_avx_size();
 562   return offset;
 563 }
 564 
 565 // Indicate if the safepoint node needs the polling page as an input,
 566 // it does if the polling page is more than disp32 away.
 567 bool SafePointNode::needs_polling_address_input()
 568 {
 569   return Assembler::is_polling_page_far();
 570 }
 571 
 572 //
 573 // Compute padding required for nodes which need alignment
 574 //
 575 
 576 // The address of the call instruction needs to be 4-byte aligned to
 577 // ensure that it does not span a cache line so that it can be patched.
 578 int CallStaticJavaDirectNode::compute_padding(int current_offset) const
 579 {
 580   current_offset += clear_avx_size(); // skip vzeroupper
 581   current_offset += 1; // skip call opcode byte
 582   return round_to(current_offset, alignment_required()) - current_offset;
 583 }
 584 
 585 // The address of the call instruction needs to be 4-byte aligned to
 586 // ensure that it does not span a cache line so that it can be patched.
 587 int CallDynamicJavaDirectNode::compute_padding(int current_offset) const
 588 {
 589   current_offset += clear_avx_size(); // skip vzeroupper
 590   current_offset += 11; // skip movq instruction + call opcode byte
 591   return round_to(current_offset, alignment_required()) - current_offset;
 592 }
 593 
 594 // EMIT_RM()
 595 void emit_rm(CodeBuffer &cbuf, int f1, int f2, int f3) {
 596   unsigned char c = (unsigned char) ((f1 << 6) | (f2 << 3) | f3);
 597   cbuf.insts()->emit_int8(c);
 598 }
 599 
 600 // EMIT_CC()
 601 void emit_cc(CodeBuffer &cbuf, int f1, int f2) {
 602   unsigned char c = (unsigned char) (f1 | f2);
 603   cbuf.insts()->emit_int8(c);
 604 }
 605 
 606 // EMIT_OPCODE()
 607 void emit_opcode(CodeBuffer &cbuf, int code) {
 608   cbuf.insts()->emit_int8((unsigned char) code);
 609 }
 610 
 611 // EMIT_OPCODE() w/ relocation information
 612 void emit_opcode(CodeBuffer &cbuf,
 613                  int code, relocInfo::relocType reloc, int offset, int format)
 614 {
 615   cbuf.relocate(cbuf.insts_mark() + offset, reloc, format);
 616   emit_opcode(cbuf, code);
 617 }
 618 
 619 // EMIT_D8()
 620 void emit_d8(CodeBuffer &cbuf, int d8) {
 621   cbuf.insts()->emit_int8((unsigned char) d8);
 622 }
 623 
 624 // EMIT_D16()
 625 void emit_d16(CodeBuffer &cbuf, int d16) {
 626   cbuf.insts()->emit_int16(d16);
 627 }
 628 
 629 // EMIT_D32()
 630 void emit_d32(CodeBuffer &cbuf, int d32) {
 631   cbuf.insts()->emit_int32(d32);
 632 }
 633 
 634 // EMIT_D64()
 635 void emit_d64(CodeBuffer &cbuf, int64_t d64) {
 636   cbuf.insts()->emit_int64(d64);
 637 }
 638 
 639 // emit 32 bit value and construct relocation entry from relocInfo::relocType
 640 void emit_d32_reloc(CodeBuffer& cbuf,
 641                     int d32,
 642                     relocInfo::relocType reloc,
 643                     int format)
 644 {
 645   assert(reloc != relocInfo::external_word_type, "use 2-arg emit_d32_reloc");
 646   cbuf.relocate(cbuf.insts_mark(), reloc, format);
 647   cbuf.insts()->emit_int32(d32);
 648 }
 649 
 650 // emit 32 bit value and construct relocation entry from RelocationHolder
 651 void emit_d32_reloc(CodeBuffer& cbuf, int d32, RelocationHolder const& rspec, int format) {
 652 #ifdef ASSERT
 653   if (rspec.reloc()->type() == relocInfo::oop_type &&
 654       d32 != 0 && d32 != (intptr_t) Universe::non_oop_word()) {
 655     assert(Universe::heap()->is_in_reserved((address)(intptr_t)d32), "should be real oop");
 656     assert(cast_to_oop((intptr_t)d32)->is_oop() && (ScavengeRootsInCode || !cast_to_oop((intptr_t)d32)->is_scavengable()), "cannot embed scavengable oops in code");
 657   }
 658 #endif
 659   cbuf.relocate(cbuf.insts_mark(), rspec, format);
 660   cbuf.insts()->emit_int32(d32);
 661 }
 662 
 663 void emit_d32_reloc(CodeBuffer& cbuf, address addr) {
 664   address next_ip = cbuf.insts_end() + 4;
 665   emit_d32_reloc(cbuf, (int) (addr - next_ip),
 666                  external_word_Relocation::spec(addr),
 667                  RELOC_DISP32);
 668 }
 669 
 670 
 671 // emit 64 bit value and construct relocation entry from relocInfo::relocType
 672 void emit_d64_reloc(CodeBuffer& cbuf, int64_t d64, relocInfo::relocType reloc, int format) {
 673   cbuf.relocate(cbuf.insts_mark(), reloc, format);
 674   cbuf.insts()->emit_int64(d64);
 675 }
 676 
 677 // emit 64 bit value and construct relocation entry from RelocationHolder
 678 void emit_d64_reloc(CodeBuffer& cbuf, int64_t d64, RelocationHolder const& rspec, int format) {
 679 #ifdef ASSERT
 680   if (rspec.reloc()->type() == relocInfo::oop_type &&
 681       d64 != 0 && d64 != (int64_t) Universe::non_oop_word()) {
 682     assert(Universe::heap()->is_in_reserved((address)d64), "should be real oop");
 683     assert(cast_to_oop(d64)->is_oop() && (ScavengeRootsInCode || !cast_to_oop(d64)->is_scavengable()),
 684            "cannot embed scavengable oops in code");
 685   }
 686 #endif
 687   cbuf.relocate(cbuf.insts_mark(), rspec, format);
 688   cbuf.insts()->emit_int64(d64);
 689 }
 690 
 691 // Access stack slot for load or store
 692 void store_to_stackslot(CodeBuffer &cbuf, int opcode, int rm_field, int disp)
 693 {
 694   emit_opcode(cbuf, opcode);                  // (e.g., FILD   [RSP+src])
 695   if (-0x80 <= disp && disp < 0x80) {
 696     emit_rm(cbuf, 0x01, rm_field, RSP_enc);   // R/M byte
 697     emit_rm(cbuf, 0x00, RSP_enc, RSP_enc);    // SIB byte
 698     emit_d8(cbuf, disp);     // Displacement  // R/M byte
 699   } else {
 700     emit_rm(cbuf, 0x02, rm_field, RSP_enc);   // R/M byte
 701     emit_rm(cbuf, 0x00, RSP_enc, RSP_enc);    // SIB byte
 702     emit_d32(cbuf, disp);     // Displacement // R/M byte
 703   }
 704 }
 705 
 706    // rRegI ereg, memory mem) %{    // emit_reg_mem
 707 void encode_RegMem(CodeBuffer &cbuf,
 708                    int reg,
 709                    int base, int index, int scale, int disp, relocInfo::relocType disp_reloc)
 710 {
 711   assert(disp_reloc == relocInfo::none, "cannot have disp");
 712   int regenc = reg & 7;
 713   int baseenc = base & 7;
 714   int indexenc = index & 7;
 715 
 716   // There is no index & no scale, use form without SIB byte
 717   if (index == 0x4 && scale == 0 && base != RSP_enc && base != R12_enc) {
 718     // If no displacement, mode is 0x0; unless base is [RBP] or [R13]
 719     if (disp == 0 && base != RBP_enc && base != R13_enc) {
 720       emit_rm(cbuf, 0x0, regenc, baseenc); // *
 721     } else if (-0x80 <= disp && disp < 0x80 && disp_reloc == relocInfo::none) {
 722       // If 8-bit displacement, mode 0x1
 723       emit_rm(cbuf, 0x1, regenc, baseenc); // *
 724       emit_d8(cbuf, disp);
 725     } else {
 726       // If 32-bit displacement
 727       if (base == -1) { // Special flag for absolute address
 728         emit_rm(cbuf, 0x0, regenc, 0x5); // *
 729         if (disp_reloc != relocInfo::none) {
 730           emit_d32_reloc(cbuf, disp, relocInfo::oop_type, RELOC_DISP32);
 731         } else {
 732           emit_d32(cbuf, disp);
 733         }
 734       } else {
 735         // Normal base + offset
 736         emit_rm(cbuf, 0x2, regenc, baseenc); // *
 737         if (disp_reloc != relocInfo::none) {
 738           emit_d32_reloc(cbuf, disp, relocInfo::oop_type, RELOC_DISP32);
 739         } else {
 740           emit_d32(cbuf, disp);
 741         }
 742       }
 743     }
 744   } else {
 745     // Else, encode with the SIB byte
 746     // If no displacement, mode is 0x0; unless base is [RBP] or [R13]
 747     if (disp == 0 && base != RBP_enc && base != R13_enc) {
 748       // If no displacement
 749       emit_rm(cbuf, 0x0, regenc, 0x4); // *
 750       emit_rm(cbuf, scale, indexenc, baseenc);
 751     } else {
 752       if (-0x80 <= disp && disp < 0x80 && disp_reloc == relocInfo::none) {
 753         // If 8-bit displacement, mode 0x1
 754         emit_rm(cbuf, 0x1, regenc, 0x4); // *
 755         emit_rm(cbuf, scale, indexenc, baseenc);
 756         emit_d8(cbuf, disp);
 757       } else {
 758         // If 32-bit displacement
 759         if (base == 0x04 ) {
 760           emit_rm(cbuf, 0x2, regenc, 0x4);
 761           emit_rm(cbuf, scale, indexenc, 0x04); // XXX is this valid???
 762         } else {
 763           emit_rm(cbuf, 0x2, regenc, 0x4);
 764           emit_rm(cbuf, scale, indexenc, baseenc); // *
 765         }
 766         if (disp_reloc != relocInfo::none) {
 767           emit_d32_reloc(cbuf, disp, relocInfo::oop_type, RELOC_DISP32);
 768         } else {
 769           emit_d32(cbuf, disp);
 770         }
 771       }
 772     }
 773   }
 774 }
 775 
 776 // This could be in MacroAssembler but it's fairly C2 specific
 777 void emit_cmpfp_fixup(MacroAssembler& _masm) {
 778   Label exit;
 779   __ jccb(Assembler::noParity, exit);
 780   __ pushf();
 781   //
 782   // comiss/ucomiss instructions set ZF,PF,CF flags and
 783   // zero OF,AF,SF for NaN values.
 784   // Fixup flags by zeroing ZF,PF so that compare of NaN
 785   // values returns 'less than' result (CF is set).
 786   // Leave the rest of flags unchanged.
 787   //
 788   //    7 6 5 4 3 2 1 0
 789   //   |S|Z|r|A|r|P|r|C|  (r - reserved bit)
 790   //    0 0 1 0 1 0 1 1   (0x2B)
 791   //
 792   __ andq(Address(rsp, 0), 0xffffff2b);
 793   __ popf();
 794   __ bind(exit);
 795 }
 796 
 797 void emit_cmpfp3(MacroAssembler& _masm, Register dst) {
 798   Label done;
 799   __ movl(dst, -1);
 800   __ jcc(Assembler::parity, done);
 801   __ jcc(Assembler::below, done);
 802   __ setb(Assembler::notEqual, dst);
 803   __ movzbl(dst, dst);
 804   __ bind(done);
 805 }
 806 
 807 
 808 //=============================================================================
 809 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
 810 
 811 int Compile::ConstantTable::calculate_table_base_offset() const {
 812   return 0;  // absolute addressing, no offset
 813 }
 814 
 815 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
 816 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
 817   ShouldNotReachHere();
 818 }
 819 
 820 void MachConstantBaseNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const {
 821   // Empty encoding
 822 }
 823 
 824 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
 825   return 0;
 826 }
 827 
 828 #ifndef PRODUCT
 829 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
 830   st->print("# MachConstantBaseNode (empty encoding)");
 831 }
 832 #endif
 833 
 834 
 835 //=============================================================================
 836 #ifndef PRODUCT
 837 void MachPrologNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
 838   Compile* C = ra_->C;
 839 
 840   int framesize = C->frame_size_in_bytes();
 841   int bangsize = C->bang_size_in_bytes();
 842   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
 843   // Remove wordSize for return addr which is already pushed.
 844   framesize -= wordSize;
 845 
 846   if (C->need_stack_bang(bangsize)) {
 847     framesize -= wordSize;
 848     st->print("# stack bang (%d bytes)", bangsize);
 849     st->print("\n\t");
 850     st->print("pushq   rbp\t# Save rbp");
 851     if (PreserveFramePointer) {
 852         st->print("\n\t");
 853         st->print("movq    rbp, rsp\t# Save the caller's SP into rbp");
 854     }
 855     if (framesize) {
 856       st->print("\n\t");
 857       st->print("subq    rsp, #%d\t# Create frame",framesize);
 858     }
 859   } else {
 860     st->print("subq    rsp, #%d\t# Create frame",framesize);
 861     st->print("\n\t");
 862     framesize -= wordSize;
 863     st->print("movq    [rsp + #%d], rbp\t# Save rbp",framesize);    
 864     if (PreserveFramePointer) {
 865       st->print("\n\t");
 866       st->print("movq    rbp, rsp\t# Save the caller's SP into rbp");
 867       if (framesize > 0) {
 868         st->print("\n\t");
 869         st->print("addq    rbp, #%d", framesize);
 870       }      
 871     }
 872   }
 873 
 874   if (VerifyStackAtCalls) {
 875     st->print("\n\t");
 876     framesize -= wordSize;
 877     st->print("movq    [rsp + #%d], 0xbadb100d\t# Majik cookie for stack depth check",framesize);
 878 #ifdef ASSERT
 879     st->print("\n\t");
 880     st->print("# stack alignment check");
 881 #endif
 882   }
 883   st->cr();
 884 }
 885 #endif
 886 
 887 void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 888   Compile* C = ra_->C;
 889   MacroAssembler _masm(&cbuf);
 890 
 891   int framesize = C->frame_size_in_bytes();
 892   int bangsize = C->bang_size_in_bytes();
 893 
 894   __ verified_entry(framesize, C->need_stack_bang(bangsize)?bangsize:0, false);
 895 
 896   C->set_frame_complete(cbuf.insts_size());
 897 
 898   if (C->has_mach_constant_base_node()) {
 899     // NOTE: We set the table base offset here because users might be
 900     // emitted before MachConstantBaseNode.
 901     Compile::ConstantTable& constant_table = C->constant_table();
 902     constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
 903   }
 904 }
 905 
 906 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
 907 {
 908   return MachNode::size(ra_); // too many variables; just compute it
 909                               // the hard way
 910 }
 911 
 912 int MachPrologNode::reloc() const
 913 {
 914   return 0; // a large enough number
 915 }
 916 
 917 //=============================================================================
 918 #ifndef PRODUCT
 919 void MachEpilogNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 920 {
 921   Compile* C = ra_->C;
 922   if (C->max_vector_size() > 16) {
 923     st->print("vzeroupper");
 924     st->cr(); st->print("\t");
 925   }
 926 
 927   int framesize = C->frame_size_in_bytes();
 928   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
 929   // Remove word for return adr already pushed
 930   // and RBP
 931   framesize -= 2*wordSize;
 932 
 933   if (framesize) {
 934     st->print_cr("addq    rsp, %d\t# Destroy frame", framesize);
 935     st->print("\t");
 936   }
 937 
 938   st->print_cr("popq   rbp");
 939   if (do_polling() && C->is_method_compilation()) {
 940     st->print("\t");
 941     if (Assembler::is_polling_page_far()) {
 942       st->print_cr("movq   rscratch1, #polling_page_address\n\t"
 943                    "testl  rax, [rscratch1]\t"
 944                    "# Safepoint: poll for GC");
 945     } else {
 946       st->print_cr("testl  rax, [rip + #offset_to_poll_page]\t"
 947                    "# Safepoint: poll for GC");
 948     }
 949   }
 950 }
 951 #endif
 952 
 953 void MachEpilogNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
 954 {
 955   Compile* C = ra_->C;
 956   if (C->max_vector_size() > 16) {
 957     // Clear upper bits of YMM registers when current compiled code uses
 958     // wide vectors to avoid AVX <-> SSE transition penalty during call.
 959     MacroAssembler _masm(&cbuf);
 960     __ vzeroupper();
 961   }
 962 
 963   int framesize = C->frame_size_in_bytes();
 964   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
 965   // Remove word for return adr already pushed
 966   // and RBP
 967   framesize -= 2*wordSize;
 968 
 969   // Note that VerifyStackAtCalls' Majik cookie does not change the frame size popped here
 970 
 971   if (framesize) {
 972     emit_opcode(cbuf, Assembler::REX_W);
 973     if (framesize < 0x80) {
 974       emit_opcode(cbuf, 0x83); // addq rsp, #framesize
 975       emit_rm(cbuf, 0x3, 0x00, RSP_enc);
 976       emit_d8(cbuf, framesize);
 977     } else {
 978       emit_opcode(cbuf, 0x81); // addq rsp, #framesize
 979       emit_rm(cbuf, 0x3, 0x00, RSP_enc);
 980       emit_d32(cbuf, framesize);
 981     }
 982   }
 983 
 984   // popq rbp
 985   emit_opcode(cbuf, 0x58 | RBP_enc);
 986 
 987   if (do_polling() && C->is_method_compilation()) {
 988     MacroAssembler _masm(&cbuf);
 989     AddressLiteral polling_page(os::get_polling_page(), relocInfo::poll_return_type);
 990     if (Assembler::is_polling_page_far()) {
 991       __ lea(rscratch1, polling_page);
 992       __ relocate(relocInfo::poll_return_type);
 993       __ testl(rax, Address(rscratch1, 0));
 994     } else {
 995       __ testl(rax, polling_page);
 996     }
 997   }
 998 }
 999 
1000 uint MachEpilogNode::size(PhaseRegAlloc* ra_) const
1001 {
1002   return MachNode::size(ra_); // too many variables; just compute it
1003                               // the hard way
1004 }
1005 
1006 int MachEpilogNode::reloc() const
1007 {
1008   return 2; // a large enough number
1009 }
1010 
1011 const Pipeline* MachEpilogNode::pipeline() const
1012 {
1013   return MachNode::pipeline_class();
1014 }
1015 
1016 int MachEpilogNode::safepoint_offset() const
1017 {
1018   return 0;
1019 }
1020 
1021 //=============================================================================
1022 
1023 enum RC {
1024   rc_bad,
1025   rc_int,
1026   rc_float,
1027   rc_stack
1028 };
1029 
1030 static enum RC rc_class(OptoReg::Name reg)
1031 {
1032   if( !OptoReg::is_valid(reg)  ) return rc_bad;
1033 
1034   if (OptoReg::is_stack(reg)) return rc_stack;
1035 
1036   VMReg r = OptoReg::as_VMReg(reg);
1037 
1038   if (r->is_Register()) return rc_int;
1039 
1040   assert(r->is_XMMRegister(), "must be");
1041   return rc_float;
1042 }
1043 
1044 // Next two methods are shared by 32- and 64-bit VM. They are defined in x86.ad.
1045 static int vec_mov_helper(CodeBuffer *cbuf, bool do_size, int src_lo, int dst_lo,
1046                           int src_hi, int dst_hi, uint ireg, outputStream* st);
1047 
1048 static int vec_spill_helper(CodeBuffer *cbuf, bool do_size, bool is_load,
1049                             int stack_offset, int reg, uint ireg, outputStream* st);
1050 
1051 static void vec_stack_to_stack_helper(CodeBuffer *cbuf, int src_offset,
1052                                       int dst_offset, uint ireg, outputStream* st) {
1053   if (cbuf) {
1054     MacroAssembler _masm(cbuf);
1055     switch (ireg) {
1056     case Op_VecS:
1057       __ movq(Address(rsp, -8), rax);
1058       __ movl(rax, Address(rsp, src_offset));
1059       __ movl(Address(rsp, dst_offset), rax);
1060       __ movq(rax, Address(rsp, -8));
1061       break;
1062     case Op_VecD:
1063       __ pushq(Address(rsp, src_offset));
1064       __ popq (Address(rsp, dst_offset));
1065       break;
1066     case Op_VecX:
1067       __ pushq(Address(rsp, src_offset));
1068       __ popq (Address(rsp, dst_offset));
1069       __ pushq(Address(rsp, src_offset+8));
1070       __ popq (Address(rsp, dst_offset+8));
1071       break;
1072     case Op_VecY:
1073       __ vmovdqu(Address(rsp, -32), xmm0);
1074       __ vmovdqu(xmm0, Address(rsp, src_offset));
1075       __ vmovdqu(Address(rsp, dst_offset), xmm0);
1076       __ vmovdqu(xmm0, Address(rsp, -32));
1077       break;
1078     default:
1079       ShouldNotReachHere();
1080     }
1081 #ifndef PRODUCT
1082   } else {
1083     switch (ireg) {
1084     case Op_VecS:
1085       st->print("movq    [rsp - #8], rax\t# 32-bit mem-mem spill\n\t"
1086                 "movl    rax, [rsp + #%d]\n\t"
1087                 "movl    [rsp + #%d], rax\n\t"
1088                 "movq    rax, [rsp - #8]",
1089                 src_offset, dst_offset);
1090       break;
1091     case Op_VecD:
1092       st->print("pushq   [rsp + #%d]\t# 64-bit mem-mem spill\n\t"
1093                 "popq    [rsp + #%d]",
1094                 src_offset, dst_offset);
1095       break;
1096      case Op_VecX:
1097       st->print("pushq   [rsp + #%d]\t# 128-bit mem-mem spill\n\t"
1098                 "popq    [rsp + #%d]\n\t"
1099                 "pushq   [rsp + #%d]\n\t"
1100                 "popq    [rsp + #%d]",
1101                 src_offset, dst_offset, src_offset+8, dst_offset+8);
1102       break;
1103     case Op_VecY:
1104       st->print("vmovdqu [rsp - #32], xmm0\t# 256-bit mem-mem spill\n\t"
1105                 "vmovdqu xmm0, [rsp + #%d]\n\t"
1106                 "vmovdqu [rsp + #%d], xmm0\n\t"
1107                 "vmovdqu xmm0, [rsp - #32]",
1108                 src_offset, dst_offset);
1109       break;
1110     default:
1111       ShouldNotReachHere();
1112     }
1113 #endif
1114   }
1115 }
1116 
1117 uint MachSpillCopyNode::implementation(CodeBuffer* cbuf,
1118                                        PhaseRegAlloc* ra_,
1119                                        bool do_size,
1120                                        outputStream* st) const {
1121   assert(cbuf != NULL || st  != NULL, "sanity");
1122   // Get registers to move
1123   OptoReg::Name src_second = ra_->get_reg_second(in(1));
1124   OptoReg::Name src_first = ra_->get_reg_first(in(1));
1125   OptoReg::Name dst_second = ra_->get_reg_second(this);
1126   OptoReg::Name dst_first = ra_->get_reg_first(this);
1127 
1128   enum RC src_second_rc = rc_class(src_second);
1129   enum RC src_first_rc = rc_class(src_first);
1130   enum RC dst_second_rc = rc_class(dst_second);
1131   enum RC dst_first_rc = rc_class(dst_first);
1132 
1133   assert(OptoReg::is_valid(src_first) && OptoReg::is_valid(dst_first),
1134          "must move at least 1 register" );
1135 
1136   if (src_first == dst_first && src_second == dst_second) {
1137     // Self copy, no move
1138     return 0;
1139   }
1140   if (bottom_type()->isa_vect() != NULL) {
1141     uint ireg = ideal_reg();
1142     assert((src_first_rc != rc_int && dst_first_rc != rc_int), "sanity");
1143     assert((ireg == Op_VecS || ireg == Op_VecD || ireg == Op_VecX || ireg == Op_VecY), "sanity");
1144     if( src_first_rc == rc_stack && dst_first_rc == rc_stack ) {
1145       // mem -> mem
1146       int src_offset = ra_->reg2offset(src_first);
1147       int dst_offset = ra_->reg2offset(dst_first);
1148       vec_stack_to_stack_helper(cbuf, src_offset, dst_offset, ireg, st);
1149     } else if (src_first_rc == rc_float && dst_first_rc == rc_float ) {
1150       vec_mov_helper(cbuf, false, src_first, dst_first, src_second, dst_second, ireg, st);
1151     } else if (src_first_rc == rc_float && dst_first_rc == rc_stack ) {
1152       int stack_offset = ra_->reg2offset(dst_first);
1153       vec_spill_helper(cbuf, false, false, stack_offset, src_first, ireg, st);
1154     } else if (src_first_rc == rc_stack && dst_first_rc == rc_float ) {
1155       int stack_offset = ra_->reg2offset(src_first);
1156       vec_spill_helper(cbuf, false, true,  stack_offset, dst_first, ireg, st);
1157     } else {
1158       ShouldNotReachHere();
1159     }
1160     return 0;
1161   }
1162   if (src_first_rc == rc_stack) {
1163     // mem ->
1164     if (dst_first_rc == rc_stack) {
1165       // mem -> mem
1166       assert(src_second != dst_first, "overlap");
1167       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1168           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1169         // 64-bit
1170         int src_offset = ra_->reg2offset(src_first);
1171         int dst_offset = ra_->reg2offset(dst_first);
1172         if (cbuf) {
1173           MacroAssembler _masm(cbuf);
1174           __ pushq(Address(rsp, src_offset));
1175           __ popq (Address(rsp, dst_offset));
1176 #ifndef PRODUCT
1177         } else {
1178           st->print("pushq   [rsp + #%d]\t# 64-bit mem-mem spill\n\t"
1179                     "popq    [rsp + #%d]",
1180                      src_offset, dst_offset);
1181 #endif
1182         }
1183       } else {
1184         // 32-bit
1185         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1186         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1187         // No pushl/popl, so:
1188         int src_offset = ra_->reg2offset(src_first);
1189         int dst_offset = ra_->reg2offset(dst_first);
1190         if (cbuf) {
1191           MacroAssembler _masm(cbuf);
1192           __ movq(Address(rsp, -8), rax);
1193           __ movl(rax, Address(rsp, src_offset));
1194           __ movl(Address(rsp, dst_offset), rax);
1195           __ movq(rax, Address(rsp, -8));
1196 #ifndef PRODUCT
1197         } else {
1198           st->print("movq    [rsp - #8], rax\t# 32-bit mem-mem spill\n\t"
1199                     "movl    rax, [rsp + #%d]\n\t"
1200                     "movl    [rsp + #%d], rax\n\t"
1201                     "movq    rax, [rsp - #8]",
1202                      src_offset, dst_offset);
1203 #endif
1204         }
1205       }
1206       return 0;
1207     } else if (dst_first_rc == rc_int) {
1208       // mem -> gpr
1209       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1210           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1211         // 64-bit
1212         int offset = ra_->reg2offset(src_first);
1213         if (cbuf) {
1214           MacroAssembler _masm(cbuf);
1215           __ movq(as_Register(Matcher::_regEncode[dst_first]), Address(rsp, offset));
1216 #ifndef PRODUCT
1217         } else {
1218           st->print("movq    %s, [rsp + #%d]\t# spill",
1219                      Matcher::regName[dst_first],
1220                      offset);
1221 #endif
1222         }
1223       } else {
1224         // 32-bit
1225         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1226         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1227         int offset = ra_->reg2offset(src_first);
1228         if (cbuf) {
1229           MacroAssembler _masm(cbuf);
1230           __ movl(as_Register(Matcher::_regEncode[dst_first]), Address(rsp, offset));
1231 #ifndef PRODUCT
1232         } else {
1233           st->print("movl    %s, [rsp + #%d]\t# spill",
1234                      Matcher::regName[dst_first],
1235                      offset);
1236 #endif
1237         }
1238       }
1239       return 0;
1240     } else if (dst_first_rc == rc_float) {
1241       // mem-> xmm
1242       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1243           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1244         // 64-bit
1245         int offset = ra_->reg2offset(src_first);
1246         if (cbuf) {
1247           MacroAssembler _masm(cbuf);
1248           __ movdbl( as_XMMRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
1249 #ifndef PRODUCT
1250         } else {
1251           st->print("%s  %s, [rsp + #%d]\t# spill",
1252                      UseXmmLoadAndClearUpper ? "movsd " : "movlpd",
1253                      Matcher::regName[dst_first],
1254                      offset);
1255 #endif
1256         }
1257       } else {
1258         // 32-bit
1259         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1260         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1261         int offset = ra_->reg2offset(src_first);
1262         if (cbuf) {
1263           MacroAssembler _masm(cbuf);
1264           __ movflt( as_XMMRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
1265 #ifndef PRODUCT
1266         } else {
1267           st->print("movss   %s, [rsp + #%d]\t# spill",
1268                      Matcher::regName[dst_first],
1269                      offset);
1270 #endif
1271         }
1272       }
1273       return 0;
1274     }
1275   } else if (src_first_rc == rc_int) {
1276     // gpr ->
1277     if (dst_first_rc == rc_stack) {
1278       // gpr -> mem
1279       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1280           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1281         // 64-bit
1282         int offset = ra_->reg2offset(dst_first);
1283         if (cbuf) {
1284           MacroAssembler _masm(cbuf);
1285           __ movq(Address(rsp, offset), as_Register(Matcher::_regEncode[src_first]));
1286 #ifndef PRODUCT
1287         } else {
1288           st->print("movq    [rsp + #%d], %s\t# spill",
1289                      offset,
1290                      Matcher::regName[src_first]);
1291 #endif
1292         }
1293       } else {
1294         // 32-bit
1295         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1296         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1297         int offset = ra_->reg2offset(dst_first);
1298         if (cbuf) {
1299           MacroAssembler _masm(cbuf);
1300           __ movl(Address(rsp, offset), as_Register(Matcher::_regEncode[src_first]));
1301 #ifndef PRODUCT
1302         } else {
1303           st->print("movl    [rsp + #%d], %s\t# spill",
1304                      offset,
1305                      Matcher::regName[src_first]);
1306 #endif
1307         }
1308       }
1309       return 0;
1310     } else if (dst_first_rc == rc_int) {
1311       // gpr -> gpr
1312       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1313           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1314         // 64-bit
1315         if (cbuf) {
1316           MacroAssembler _masm(cbuf);
1317           __ movq(as_Register(Matcher::_regEncode[dst_first]),
1318                   as_Register(Matcher::_regEncode[src_first]));
1319 #ifndef PRODUCT
1320         } else {
1321           st->print("movq    %s, %s\t# spill",
1322                      Matcher::regName[dst_first],
1323                      Matcher::regName[src_first]);
1324 #endif
1325         }
1326         return 0;
1327       } else {
1328         // 32-bit
1329         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1330         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1331         if (cbuf) {
1332           MacroAssembler _masm(cbuf);
1333           __ movl(as_Register(Matcher::_regEncode[dst_first]),
1334                   as_Register(Matcher::_regEncode[src_first]));
1335 #ifndef PRODUCT
1336         } else {
1337           st->print("movl    %s, %s\t# spill",
1338                      Matcher::regName[dst_first],
1339                      Matcher::regName[src_first]);
1340 #endif
1341         }
1342         return 0;
1343       }
1344     } else if (dst_first_rc == rc_float) {
1345       // gpr -> xmm
1346       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1347           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1348         // 64-bit
1349         if (cbuf) {
1350           MacroAssembler _masm(cbuf);
1351           __ movdq( as_XMMRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
1352 #ifndef PRODUCT
1353         } else {
1354           st->print("movdq   %s, %s\t# spill",
1355                      Matcher::regName[dst_first],
1356                      Matcher::regName[src_first]);
1357 #endif
1358         }
1359       } else {
1360         // 32-bit
1361         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1362         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1363         if (cbuf) {
1364           MacroAssembler _masm(cbuf);
1365           __ movdl( as_XMMRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
1366 #ifndef PRODUCT
1367         } else {
1368           st->print("movdl   %s, %s\t# spill",
1369                      Matcher::regName[dst_first],
1370                      Matcher::regName[src_first]);
1371 #endif
1372         }
1373       }
1374       return 0;
1375     }
1376   } else if (src_first_rc == rc_float) {
1377     // xmm ->
1378     if (dst_first_rc == rc_stack) {
1379       // xmm -> mem
1380       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1381           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1382         // 64-bit
1383         int offset = ra_->reg2offset(dst_first);
1384         if (cbuf) {
1385           MacroAssembler _masm(cbuf);
1386           __ movdbl( Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[src_first]));
1387 #ifndef PRODUCT
1388         } else {
1389           st->print("movsd   [rsp + #%d], %s\t# spill",
1390                      offset,
1391                      Matcher::regName[src_first]);
1392 #endif
1393         }
1394       } else {
1395         // 32-bit
1396         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1397         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1398         int offset = ra_->reg2offset(dst_first);
1399         if (cbuf) {
1400           MacroAssembler _masm(cbuf);
1401           __ movflt(Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[src_first]));
1402 #ifndef PRODUCT
1403         } else {
1404           st->print("movss   [rsp + #%d], %s\t# spill",
1405                      offset,
1406                      Matcher::regName[src_first]);
1407 #endif
1408         }
1409       }
1410       return 0;
1411     } else if (dst_first_rc == rc_int) {
1412       // xmm -> gpr
1413       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1414           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1415         // 64-bit
1416         if (cbuf) {
1417           MacroAssembler _masm(cbuf);
1418           __ movdq( as_Register(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
1419 #ifndef PRODUCT
1420         } else {
1421           st->print("movdq   %s, %s\t# spill",
1422                      Matcher::regName[dst_first],
1423                      Matcher::regName[src_first]);
1424 #endif
1425         }
1426       } else {
1427         // 32-bit
1428         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1429         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1430         if (cbuf) {
1431           MacroAssembler _masm(cbuf);
1432           __ movdl( as_Register(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
1433 #ifndef PRODUCT
1434         } else {
1435           st->print("movdl   %s, %s\t# spill",
1436                      Matcher::regName[dst_first],
1437                      Matcher::regName[src_first]);
1438 #endif
1439         }
1440       }
1441       return 0;
1442     } else if (dst_first_rc == rc_float) {
1443       // xmm -> xmm
1444       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1445           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1446         // 64-bit
1447         if (cbuf) {
1448           MacroAssembler _masm(cbuf);
1449           __ movdbl( as_XMMRegister(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
1450 #ifndef PRODUCT
1451         } else {
1452           st->print("%s  %s, %s\t# spill",
1453                      UseXmmRegToRegMoveAll ? "movapd" : "movsd ",
1454                      Matcher::regName[dst_first],
1455                      Matcher::regName[src_first]);
1456 #endif
1457         }
1458       } else {
1459         // 32-bit
1460         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1461         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1462         if (cbuf) {
1463           MacroAssembler _masm(cbuf);
1464           __ movflt( as_XMMRegister(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
1465 #ifndef PRODUCT
1466         } else {
1467           st->print("%s  %s, %s\t# spill",
1468                      UseXmmRegToRegMoveAll ? "movaps" : "movss ",
1469                      Matcher::regName[dst_first],
1470                      Matcher::regName[src_first]);
1471 #endif
1472         }
1473       }
1474       return 0;
1475     }
1476   }
1477 
1478   assert(0," foo ");
1479   Unimplemented();
1480   return 0;
1481 }
1482 
1483 #ifndef PRODUCT
1484 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream* st) const {
1485   implementation(NULL, ra_, false, st);
1486 }
1487 #endif
1488 
1489 void MachSpillCopyNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1490   implementation(&cbuf, ra_, false, NULL);
1491 }
1492 
1493 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
1494   return MachNode::size(ra_);
1495 }
1496 
1497 //=============================================================================
1498 #ifndef PRODUCT
1499 void BoxLockNode::format(PhaseRegAlloc* ra_, outputStream* st) const
1500 {
1501   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
1502   int reg = ra_->get_reg_first(this);
1503   st->print("leaq    %s, [rsp + #%d]\t# box lock",
1504             Matcher::regName[reg], offset);
1505 }
1506 #endif
1507 
1508 void BoxLockNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
1509 {
1510   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
1511   int reg = ra_->get_encode(this);
1512   if (offset >= 0x80) {
1513     emit_opcode(cbuf, reg < 8 ? Assembler::REX_W : Assembler::REX_WR);
1514     emit_opcode(cbuf, 0x8D); // LEA  reg,[SP+offset]
1515     emit_rm(cbuf, 0x2, reg & 7, 0x04);
1516     emit_rm(cbuf, 0x0, 0x04, RSP_enc);
1517     emit_d32(cbuf, offset);
1518   } else {
1519     emit_opcode(cbuf, reg < 8 ? Assembler::REX_W : Assembler::REX_WR);
1520     emit_opcode(cbuf, 0x8D); // LEA  reg,[SP+offset]
1521     emit_rm(cbuf, 0x1, reg & 7, 0x04);
1522     emit_rm(cbuf, 0x0, 0x04, RSP_enc);
1523     emit_d8(cbuf, offset);
1524   }
1525 }
1526 
1527 uint BoxLockNode::size(PhaseRegAlloc *ra_) const
1528 {
1529   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
1530   return (offset < 0x80) ? 5 : 8; // REX
1531 }
1532 
1533 //=============================================================================
1534 #ifndef PRODUCT
1535 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
1536 {
1537   if (UseCompressedClassPointers) {
1538     st->print_cr("movl    rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
1539     st->print_cr("\tdecode_klass_not_null rscratch1, rscratch1");
1540     st->print_cr("\tcmpq    rax, rscratch1\t # Inline cache check");
1541   } else {
1542     st->print_cr("\tcmpq    rax, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t"
1543                  "# Inline cache check");
1544   }
1545   st->print_cr("\tjne     SharedRuntime::_ic_miss_stub");
1546   st->print_cr("\tnop\t# nops to align entry point");
1547 }
1548 #endif
1549 
1550 void MachUEPNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
1551 {
1552   MacroAssembler masm(&cbuf);
1553   uint insts_size = cbuf.insts_size();
1554   if (UseCompressedClassPointers) {
1555     masm.load_klass(rscratch1, j_rarg0);
1556     masm.cmpptr(rax, rscratch1);
1557   } else {
1558     masm.cmpptr(rax, Address(j_rarg0, oopDesc::klass_offset_in_bytes()));
1559   }
1560 
1561   masm.jump_cc(Assembler::notEqual, RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
1562 
1563   /* WARNING these NOPs are critical so that verified entry point is properly
1564      4 bytes aligned for patching by NativeJump::patch_verified_entry() */
1565   int nops_cnt = 4 - ((cbuf.insts_size() - insts_size) & 0x3);
1566   if (OptoBreakpoint) {
1567     // Leave space for int3
1568     nops_cnt -= 1;
1569   }
1570   nops_cnt &= 0x3; // Do not add nops if code is aligned.
1571   if (nops_cnt > 0)
1572     masm.nop(nops_cnt);
1573 }
1574 
1575 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
1576 {
1577   return MachNode::size(ra_); // too many variables; just compute it
1578                               // the hard way
1579 }
1580  
1581 
1582 //=============================================================================
1583 
1584 int Matcher::regnum_to_fpu_offset(int regnum)
1585 {
1586   return regnum - 32; // The FP registers are in the second chunk
1587 }
1588 
1589 // This is UltraSparc specific, true just means we have fast l2f conversion
1590 const bool Matcher::convL2FSupported(void) {
1591   return true;
1592 }
1593 
1594 // Is this branch offset short enough that a short branch can be used?
1595 //
1596 // NOTE: If the platform does not provide any short branch variants, then
1597 //       this method should return false for offset 0.
1598 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
1599   // The passed offset is relative to address of the branch.
1600   // On 86 a branch displacement is calculated relative to address
1601   // of a next instruction.
1602   offset -= br_size;
1603 
1604   // the short version of jmpConUCF2 contains multiple branches,
1605   // making the reach slightly less
1606   if (rule == jmpConUCF2_rule)
1607     return (-126 <= offset && offset <= 125);
1608   return (-128 <= offset && offset <= 127);
1609 }
1610 
1611 const bool Matcher::isSimpleConstant64(jlong value) {
1612   // Will one (StoreL ConL) be cheaper than two (StoreI ConI)?.
1613   //return value == (int) value;  // Cf. storeImmL and immL32.
1614 
1615   // Probably always true, even if a temp register is required.
1616   return true;
1617 }
1618 
1619 // The ecx parameter to rep stosq for the ClearArray node is in words.
1620 const bool Matcher::init_array_count_is_in_bytes = false;
1621 
1622 // Threshold size for cleararray.
1623 const int Matcher::init_array_short_size = 8 * BytesPerLong;
1624 
1625 // No additional cost for CMOVL.
1626 const int Matcher::long_cmove_cost() { return 0; }
1627 
1628 // No CMOVF/CMOVD with SSE2
1629 const int Matcher::float_cmove_cost() { return ConditionalMoveLimit; }
1630 
1631 // Does the CPU require late expand (see block.cpp for description of late expand)?
1632 const bool Matcher::require_postalloc_expand = false;
1633 
1634 // Should the Matcher clone shifts on addressing modes, expecting them
1635 // to be subsumed into complex addressing expressions or compute them
1636 // into registers?  True for Intel but false for most RISCs
1637 const bool Matcher::clone_shift_expressions = true;
1638 
1639 // Do we need to mask the count passed to shift instructions or does
1640 // the cpu only look at the lower 5/6 bits anyway?
1641 const bool Matcher::need_masked_shift_count = false;
1642 
1643 bool Matcher::narrow_oop_use_complex_address() {
1644   assert(UseCompressedOops, "only for compressed oops code");
1645   return (LogMinObjAlignmentInBytes <= 3);
1646 }
1647 
1648 bool Matcher::narrow_klass_use_complex_address() {
1649   assert(UseCompressedClassPointers, "only for compressed klass code");
1650   return (LogKlassAlignmentInBytes <= 3);
1651 }
1652 
1653 // Is it better to copy float constants, or load them directly from
1654 // memory?  Intel can load a float constant from a direct address,
1655 // requiring no extra registers.  Most RISCs will have to materialize
1656 // an address into a register first, so they would do better to copy
1657 // the constant from stack.
1658 const bool Matcher::rematerialize_float_constants = true; // XXX
1659 
1660 // If CPU can load and store mis-aligned doubles directly then no
1661 // fixup is needed.  Else we split the double into 2 integer pieces
1662 // and move it piece-by-piece.  Only happens when passing doubles into
1663 // C code as the Java calling convention forces doubles to be aligned.
1664 const bool Matcher::misaligned_doubles_ok = true;
1665 
1666 // No-op on amd64
1667 void Matcher::pd_implicit_null_fixup(MachNode *node, uint idx) {}
1668 
1669 // Advertise here if the CPU requires explicit rounding operations to
1670 // implement the UseStrictFP mode.
1671 const bool Matcher::strict_fp_requires_explicit_rounding = true;
1672 
1673 // Are floats conerted to double when stored to stack during deoptimization?
1674 // On x64 it is stored without convertion so we can use normal access.
1675 bool Matcher::float_in_double() { return false; }
1676 
1677 // Do ints take an entire long register or just half?
1678 const bool Matcher::int_in_long = true;
1679 
1680 // Return whether or not this register is ever used as an argument.
1681 // This function is used on startup to build the trampoline stubs in
1682 // generateOptoStub.  Registers not mentioned will be killed by the VM
1683 // call in the trampoline, and arguments in those registers not be
1684 // available to the callee.
1685 bool Matcher::can_be_java_arg(int reg)
1686 {
1687   return
1688     reg ==  RDI_num || reg == RDI_H_num ||
1689     reg ==  RSI_num || reg == RSI_H_num ||
1690     reg ==  RDX_num || reg == RDX_H_num ||
1691     reg ==  RCX_num || reg == RCX_H_num ||
1692     reg ==   R8_num || reg ==  R8_H_num ||
1693     reg ==   R9_num || reg ==  R9_H_num ||
1694     reg ==  R12_num || reg == R12_H_num ||
1695     reg == XMM0_num || reg == XMM0b_num ||
1696     reg == XMM1_num || reg == XMM1b_num ||
1697     reg == XMM2_num || reg == XMM2b_num ||
1698     reg == XMM3_num || reg == XMM3b_num ||
1699     reg == XMM4_num || reg == XMM4b_num ||
1700     reg == XMM5_num || reg == XMM5b_num ||
1701     reg == XMM6_num || reg == XMM6b_num ||
1702     reg == XMM7_num || reg == XMM7b_num;
1703 }
1704 
1705 bool Matcher::is_spillable_arg(int reg)
1706 {
1707   return can_be_java_arg(reg);
1708 }
1709 
1710 bool Matcher::use_asm_for_ldiv_by_con( jlong divisor ) {
1711   // In 64 bit mode a code which use multiply when
1712   // devisor is constant is faster than hardware
1713   // DIV instruction (it uses MulHiL).
1714   return false;
1715 }
1716 
1717 // Register for DIVI projection of divmodI
1718 RegMask Matcher::divI_proj_mask() {
1719   return INT_RAX_REG_mask();
1720 }
1721 
1722 // Register for MODI projection of divmodI
1723 RegMask Matcher::modI_proj_mask() {
1724   return INT_RDX_REG_mask();
1725 }
1726 
1727 // Register for DIVL projection of divmodL
1728 RegMask Matcher::divL_proj_mask() {
1729   return LONG_RAX_REG_mask();
1730 }
1731 
1732 // Register for MODL projection of divmodL
1733 RegMask Matcher::modL_proj_mask() {
1734   return LONG_RDX_REG_mask();
1735 }
1736 
1737 // Register for saving SP into on method handle invokes. Not used on x86_64.
1738 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
1739     return NO_REG_mask();
1740 }
1741 
1742 %}
1743 
1744 //----------ENCODING BLOCK-----------------------------------------------------
1745 // This block specifies the encoding classes used by the compiler to
1746 // output byte streams.  Encoding classes are parameterized macros
1747 // used by Machine Instruction Nodes in order to generate the bit
1748 // encoding of the instruction.  Operands specify their base encoding
1749 // interface with the interface keyword.  There are currently
1750 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
1751 // COND_INTER.  REG_INTER causes an operand to generate a function
1752 // which returns its register number when queried.  CONST_INTER causes
1753 // an operand to generate a function which returns the value of the
1754 // constant when queried.  MEMORY_INTER causes an operand to generate
1755 // four functions which return the Base Register, the Index Register,
1756 // the Scale Value, and the Offset Value of the operand when queried.
1757 // COND_INTER causes an operand to generate six functions which return
1758 // the encoding code (ie - encoding bits for the instruction)
1759 // associated with each basic boolean condition for a conditional
1760 // instruction.
1761 //
1762 // Instructions specify two basic values for encoding.  Again, a
1763 // function is available to check if the constant displacement is an
1764 // oop. They use the ins_encode keyword to specify their encoding
1765 // classes (which must be a sequence of enc_class names, and their
1766 // parameters, specified in the encoding block), and they use the
1767 // opcode keyword to specify, in order, their primary, secondary, and
1768 // tertiary opcode.  Only the opcode sections which a particular
1769 // instruction needs for encoding need to be specified.
1770 encode %{
1771   // Build emit functions for each basic byte or larger field in the
1772   // intel encoding scheme (opcode, rm, sib, immediate), and call them
1773   // from C++ code in the enc_class source block.  Emit functions will
1774   // live in the main source block for now.  In future, we can
1775   // generalize this by adding a syntax that specifies the sizes of
1776   // fields in an order, so that the adlc can build the emit functions
1777   // automagically
1778 
1779   // Emit primary opcode
1780   enc_class OpcP
1781   %{
1782     emit_opcode(cbuf, $primary);
1783   %}
1784 
1785   // Emit secondary opcode
1786   enc_class OpcS
1787   %{
1788     emit_opcode(cbuf, $secondary);
1789   %}
1790 
1791   // Emit tertiary opcode
1792   enc_class OpcT
1793   %{
1794     emit_opcode(cbuf, $tertiary);
1795   %}
1796 
1797   // Emit opcode directly
1798   enc_class Opcode(immI d8)
1799   %{
1800     emit_opcode(cbuf, $d8$$constant);
1801   %}
1802 
1803   // Emit size prefix
1804   enc_class SizePrefix
1805   %{
1806     emit_opcode(cbuf, 0x66);
1807   %}
1808 
1809   enc_class reg(rRegI reg)
1810   %{
1811     emit_rm(cbuf, 0x3, 0, $reg$$reg & 7);
1812   %}
1813 
1814   enc_class reg_reg(rRegI dst, rRegI src)
1815   %{
1816     emit_rm(cbuf, 0x3, $dst$$reg & 7, $src$$reg & 7);
1817   %}
1818 
1819   enc_class opc_reg_reg(immI opcode, rRegI dst, rRegI src)
1820   %{
1821     emit_opcode(cbuf, $opcode$$constant);
1822     emit_rm(cbuf, 0x3, $dst$$reg & 7, $src$$reg & 7);
1823   %}
1824 
1825   enc_class cdql_enc(no_rax_rdx_RegI div)
1826   %{
1827     // Full implementation of Java idiv and irem; checks for
1828     // special case as described in JVM spec., p.243 & p.271.
1829     //
1830     //         normal case                           special case
1831     //
1832     // input : rax: dividend                         min_int
1833     //         reg: divisor                          -1
1834     //
1835     // output: rax: quotient  (= rax idiv reg)       min_int
1836     //         rdx: remainder (= rax irem reg)       0
1837     //
1838     //  Code sequnce:
1839     //
1840     //    0:   3d 00 00 00 80          cmp    $0x80000000,%eax
1841     //    5:   75 07/08                jne    e <normal>
1842     //    7:   33 d2                   xor    %edx,%edx
1843     //  [div >= 8 -> offset + 1]
1844     //  [REX_B]
1845     //    9:   83 f9 ff                cmp    $0xffffffffffffffff,$div
1846     //    c:   74 03/04                je     11 <done>
1847     // 000000000000000e <normal>:
1848     //    e:   99                      cltd
1849     //  [div >= 8 -> offset + 1]
1850     //  [REX_B]
1851     //    f:   f7 f9                   idiv   $div
1852     // 0000000000000011 <done>:
1853 
1854     // cmp    $0x80000000,%eax
1855     emit_opcode(cbuf, 0x3d);
1856     emit_d8(cbuf, 0x00);
1857     emit_d8(cbuf, 0x00);
1858     emit_d8(cbuf, 0x00);
1859     emit_d8(cbuf, 0x80);
1860 
1861     // jne    e <normal>
1862     emit_opcode(cbuf, 0x75);
1863     emit_d8(cbuf, $div$$reg < 8 ? 0x07 : 0x08);
1864 
1865     // xor    %edx,%edx
1866     emit_opcode(cbuf, 0x33);
1867     emit_d8(cbuf, 0xD2);
1868 
1869     // cmp    $0xffffffffffffffff,%ecx
1870     if ($div$$reg >= 8) {
1871       emit_opcode(cbuf, Assembler::REX_B);
1872     }
1873     emit_opcode(cbuf, 0x83);
1874     emit_rm(cbuf, 0x3, 0x7, $div$$reg & 7);
1875     emit_d8(cbuf, 0xFF);
1876 
1877     // je     11 <done>
1878     emit_opcode(cbuf, 0x74);
1879     emit_d8(cbuf, $div$$reg < 8 ? 0x03 : 0x04);
1880 
1881     // <normal>
1882     // cltd
1883     emit_opcode(cbuf, 0x99);
1884 
1885     // idivl (note: must be emitted by the user of this rule)
1886     // <done>
1887   %}
1888 
1889   enc_class cdqq_enc(no_rax_rdx_RegL div)
1890   %{
1891     // Full implementation of Java ldiv and lrem; checks for
1892     // special case as described in JVM spec., p.243 & p.271.
1893     //
1894     //         normal case                           special case
1895     //
1896     // input : rax: dividend                         min_long
1897     //         reg: divisor                          -1
1898     //
1899     // output: rax: quotient  (= rax idiv reg)       min_long
1900     //         rdx: remainder (= rax irem reg)       0
1901     //
1902     //  Code sequnce:
1903     //
1904     //    0:   48 ba 00 00 00 00 00    mov    $0x8000000000000000,%rdx
1905     //    7:   00 00 80
1906     //    a:   48 39 d0                cmp    %rdx,%rax
1907     //    d:   75 08                   jne    17 <normal>
1908     //    f:   33 d2                   xor    %edx,%edx
1909     //   11:   48 83 f9 ff             cmp    $0xffffffffffffffff,$div
1910     //   15:   74 05                   je     1c <done>
1911     // 0000000000000017 <normal>:
1912     //   17:   48 99                   cqto
1913     //   19:   48 f7 f9                idiv   $div
1914     // 000000000000001c <done>:
1915 
1916     // mov    $0x8000000000000000,%rdx
1917     emit_opcode(cbuf, Assembler::REX_W);
1918     emit_opcode(cbuf, 0xBA);
1919     emit_d8(cbuf, 0x00);
1920     emit_d8(cbuf, 0x00);
1921     emit_d8(cbuf, 0x00);
1922     emit_d8(cbuf, 0x00);
1923     emit_d8(cbuf, 0x00);
1924     emit_d8(cbuf, 0x00);
1925     emit_d8(cbuf, 0x00);
1926     emit_d8(cbuf, 0x80);
1927 
1928     // cmp    %rdx,%rax
1929     emit_opcode(cbuf, Assembler::REX_W);
1930     emit_opcode(cbuf, 0x39);
1931     emit_d8(cbuf, 0xD0);
1932 
1933     // jne    17 <normal>
1934     emit_opcode(cbuf, 0x75);
1935     emit_d8(cbuf, 0x08);
1936 
1937     // xor    %edx,%edx
1938     emit_opcode(cbuf, 0x33);
1939     emit_d8(cbuf, 0xD2);
1940 
1941     // cmp    $0xffffffffffffffff,$div
1942     emit_opcode(cbuf, $div$$reg < 8 ? Assembler::REX_W : Assembler::REX_WB);
1943     emit_opcode(cbuf, 0x83);
1944     emit_rm(cbuf, 0x3, 0x7, $div$$reg & 7);
1945     emit_d8(cbuf, 0xFF);
1946 
1947     // je     1e <done>
1948     emit_opcode(cbuf, 0x74);
1949     emit_d8(cbuf, 0x05);
1950 
1951     // <normal>
1952     // cqto
1953     emit_opcode(cbuf, Assembler::REX_W);
1954     emit_opcode(cbuf, 0x99);
1955 
1956     // idivq (note: must be emitted by the user of this rule)
1957     // <done>
1958   %}
1959 
1960   // Opcde enc_class for 8/32 bit immediate instructions with sign-extension
1961   enc_class OpcSE(immI imm)
1962   %{
1963     // Emit primary opcode and set sign-extend bit
1964     // Check for 8-bit immediate, and set sign extend bit in opcode
1965     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
1966       emit_opcode(cbuf, $primary | 0x02);
1967     } else {
1968       // 32-bit immediate
1969       emit_opcode(cbuf, $primary);
1970     }
1971   %}
1972 
1973   enc_class OpcSErm(rRegI dst, immI imm)
1974   %{
1975     // OpcSEr/m
1976     int dstenc = $dst$$reg;
1977     if (dstenc >= 8) {
1978       emit_opcode(cbuf, Assembler::REX_B);
1979       dstenc -= 8;
1980     }
1981     // Emit primary opcode and set sign-extend bit
1982     // Check for 8-bit immediate, and set sign extend bit in opcode
1983     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
1984       emit_opcode(cbuf, $primary | 0x02);
1985     } else {
1986       // 32-bit immediate
1987       emit_opcode(cbuf, $primary);
1988     }
1989     // Emit r/m byte with secondary opcode, after primary opcode.
1990     emit_rm(cbuf, 0x3, $secondary, dstenc);
1991   %}
1992 
1993   enc_class OpcSErm_wide(rRegL dst, immI imm)
1994   %{
1995     // OpcSEr/m
1996     int dstenc = $dst$$reg;
1997     if (dstenc < 8) {
1998       emit_opcode(cbuf, Assembler::REX_W);
1999     } else {
2000       emit_opcode(cbuf, Assembler::REX_WB);
2001       dstenc -= 8;
2002     }
2003     // Emit primary opcode and set sign-extend bit
2004     // Check for 8-bit immediate, and set sign extend bit in opcode
2005     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
2006       emit_opcode(cbuf, $primary | 0x02);
2007     } else {
2008       // 32-bit immediate
2009       emit_opcode(cbuf, $primary);
2010     }
2011     // Emit r/m byte with secondary opcode, after primary opcode.
2012     emit_rm(cbuf, 0x3, $secondary, dstenc);
2013   %}
2014 
2015   enc_class Con8or32(immI imm)
2016   %{
2017     // Check for 8-bit immediate, and set sign extend bit in opcode
2018     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
2019       $$$emit8$imm$$constant;
2020     } else {
2021       // 32-bit immediate
2022       $$$emit32$imm$$constant;
2023     }
2024   %}
2025 
2026   enc_class opc2_reg(rRegI dst)
2027   %{
2028     // BSWAP
2029     emit_cc(cbuf, $secondary, $dst$$reg);
2030   %}
2031 
2032   enc_class opc3_reg(rRegI dst)
2033   %{
2034     // BSWAP
2035     emit_cc(cbuf, $tertiary, $dst$$reg);
2036   %}
2037 
2038   enc_class reg_opc(rRegI div)
2039   %{
2040     // INC, DEC, IDIV, IMOD, JMP indirect, ...
2041     emit_rm(cbuf, 0x3, $secondary, $div$$reg & 7);
2042   %}
2043 
2044   enc_class enc_cmov(cmpOp cop)
2045   %{
2046     // CMOV
2047     $$$emit8$primary;
2048     emit_cc(cbuf, $secondary, $cop$$cmpcode);
2049   %}
2050 
2051   enc_class enc_PartialSubtypeCheck()
2052   %{
2053     Register Rrdi = as_Register(RDI_enc); // result register
2054     Register Rrax = as_Register(RAX_enc); // super class
2055     Register Rrcx = as_Register(RCX_enc); // killed
2056     Register Rrsi = as_Register(RSI_enc); // sub class
2057     Label miss;
2058     const bool set_cond_codes = true;
2059 
2060     MacroAssembler _masm(&cbuf);
2061     __ check_klass_subtype_slow_path(Rrsi, Rrax, Rrcx, Rrdi,
2062                                      NULL, &miss,
2063                                      /*set_cond_codes:*/ true);
2064     if ($primary) {
2065       __ xorptr(Rrdi, Rrdi);
2066     }
2067     __ bind(miss);
2068   %}
2069 
2070   enc_class clear_avx %{
2071     debug_only(int off0 = cbuf.insts_size());
2072     if (ra_->C->max_vector_size() > 16) {
2073       // Clear upper bits of YMM registers when current compiled code uses
2074       // wide vectors to avoid AVX <-> SSE transition penalty during call.
2075       MacroAssembler _masm(&cbuf);
2076       __ vzeroupper();
2077     }
2078     debug_only(int off1 = cbuf.insts_size());
2079     assert(off1 - off0 == clear_avx_size(), "correct size prediction");
2080   %}
2081 
2082   enc_class Java_To_Runtime(method meth) %{
2083     // No relocation needed
2084     MacroAssembler _masm(&cbuf);
2085     __ mov64(r10, (int64_t) $meth$$method);
2086     __ call(r10);
2087   %}
2088 
2089   enc_class Java_To_Interpreter(method meth)
2090   %{
2091     // CALL Java_To_Interpreter
2092     // This is the instruction starting address for relocation info.
2093     cbuf.set_insts_mark();
2094     $$$emit8$primary;
2095     // CALL directly to the runtime
2096     emit_d32_reloc(cbuf,
2097                    (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
2098                    runtime_call_Relocation::spec(),
2099                    RELOC_DISP32);
2100   %}
2101 
2102   enc_class Java_Static_Call(method meth)
2103   %{
2104     // JAVA STATIC CALL
2105     // CALL to fixup routine.  Fixup routine uses ScopeDesc info to
2106     // determine who we intended to call.
2107     cbuf.set_insts_mark();
2108     $$$emit8$primary;
2109 
2110     if (!_method) {
2111       emit_d32_reloc(cbuf,
2112                      (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
2113                      runtime_call_Relocation::spec(),
2114                      RELOC_DISP32);
2115     } else if (_optimized_virtual) {
2116       emit_d32_reloc(cbuf,
2117                      (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
2118                      opt_virtual_call_Relocation::spec(),
2119                      RELOC_DISP32);
2120     } else {
2121       emit_d32_reloc(cbuf,
2122                      (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
2123                      static_call_Relocation::spec(),
2124                      RELOC_DISP32);
2125     }
2126     if (_method) {
2127       // Emit stub for static call.
2128       address stub = CompiledStaticCall::emit_to_interp_stub(cbuf);
2129       if (stub == NULL) {
2130         ciEnv::current()->record_failure("CodeCache is full");
2131         return;
2132       } 
2133     }
2134   %}
2135 
2136   enc_class Java_Dynamic_Call(method meth) %{
2137     MacroAssembler _masm(&cbuf);
2138     __ ic_call((address)$meth$$method);
2139   %}
2140 
2141   enc_class Java_Compiled_Call(method meth)
2142   %{
2143     // JAVA COMPILED CALL
2144     int disp = in_bytes(Method:: from_compiled_offset());
2145 
2146     // XXX XXX offset is 128 is 1.5 NON-PRODUCT !!!
2147     // assert(-0x80 <= disp && disp < 0x80, "compiled_code_offset isn't small");
2148 
2149     // callq *disp(%rax)
2150     cbuf.set_insts_mark();
2151     $$$emit8$primary;
2152     if (disp < 0x80) {
2153       emit_rm(cbuf, 0x01, $secondary, RAX_enc); // R/M byte
2154       emit_d8(cbuf, disp); // Displacement
2155     } else {
2156       emit_rm(cbuf, 0x02, $secondary, RAX_enc); // R/M byte
2157       emit_d32(cbuf, disp); // Displacement
2158     }
2159   %}
2160 
2161   enc_class reg_opc_imm(rRegI dst, immI8 shift)
2162   %{
2163     // SAL, SAR, SHR
2164     int dstenc = $dst$$reg;
2165     if (dstenc >= 8) {
2166       emit_opcode(cbuf, Assembler::REX_B);
2167       dstenc -= 8;
2168     }
2169     $$$emit8$primary;
2170     emit_rm(cbuf, 0x3, $secondary, dstenc);
2171     $$$emit8$shift$$constant;
2172   %}
2173 
2174   enc_class reg_opc_imm_wide(rRegL dst, immI8 shift)
2175   %{
2176     // SAL, SAR, SHR
2177     int dstenc = $dst$$reg;
2178     if (dstenc < 8) {
2179       emit_opcode(cbuf, Assembler::REX_W);
2180     } else {
2181       emit_opcode(cbuf, Assembler::REX_WB);
2182       dstenc -= 8;
2183     }
2184     $$$emit8$primary;
2185     emit_rm(cbuf, 0x3, $secondary, dstenc);
2186     $$$emit8$shift$$constant;
2187   %}
2188 
2189   enc_class load_immI(rRegI dst, immI src)
2190   %{
2191     int dstenc = $dst$$reg;
2192     if (dstenc >= 8) {
2193       emit_opcode(cbuf, Assembler::REX_B);
2194       dstenc -= 8;
2195     }
2196     emit_opcode(cbuf, 0xB8 | dstenc);
2197     $$$emit32$src$$constant;
2198   %}
2199 
2200   enc_class load_immL(rRegL dst, immL src)
2201   %{
2202     int dstenc = $dst$$reg;
2203     if (dstenc < 8) {
2204       emit_opcode(cbuf, Assembler::REX_W);
2205     } else {
2206       emit_opcode(cbuf, Assembler::REX_WB);
2207       dstenc -= 8;
2208     }
2209     emit_opcode(cbuf, 0xB8 | dstenc);
2210     emit_d64(cbuf, $src$$constant);
2211   %}
2212 
2213   enc_class load_immUL32(rRegL dst, immUL32 src)
2214   %{
2215     // same as load_immI, but this time we care about zeroes in the high word
2216     int dstenc = $dst$$reg;
2217     if (dstenc >= 8) {
2218       emit_opcode(cbuf, Assembler::REX_B);
2219       dstenc -= 8;
2220     }
2221     emit_opcode(cbuf, 0xB8 | dstenc);
2222     $$$emit32$src$$constant;
2223   %}
2224 
2225   enc_class load_immL32(rRegL dst, immL32 src)
2226   %{
2227     int dstenc = $dst$$reg;
2228     if (dstenc < 8) {
2229       emit_opcode(cbuf, Assembler::REX_W);
2230     } else {
2231       emit_opcode(cbuf, Assembler::REX_WB);
2232       dstenc -= 8;
2233     }
2234     emit_opcode(cbuf, 0xC7);
2235     emit_rm(cbuf, 0x03, 0x00, dstenc);
2236     $$$emit32$src$$constant;
2237   %}
2238 
2239   enc_class load_immP31(rRegP dst, immP32 src)
2240   %{
2241     // same as load_immI, but this time we care about zeroes in the high word
2242     int dstenc = $dst$$reg;
2243     if (dstenc >= 8) {
2244       emit_opcode(cbuf, Assembler::REX_B);
2245       dstenc -= 8;
2246     }
2247     emit_opcode(cbuf, 0xB8 | dstenc);
2248     $$$emit32$src$$constant;
2249   %}
2250 
2251   enc_class load_immP(rRegP dst, immP src)
2252   %{
2253     int dstenc = $dst$$reg;
2254     if (dstenc < 8) {
2255       emit_opcode(cbuf, Assembler::REX_W);
2256     } else {
2257       emit_opcode(cbuf, Assembler::REX_WB);
2258       dstenc -= 8;
2259     }
2260     emit_opcode(cbuf, 0xB8 | dstenc);
2261     // This next line should be generated from ADLC
2262     if ($src->constant_reloc() != relocInfo::none) {
2263       emit_d64_reloc(cbuf, $src$$constant, $src->constant_reloc(), RELOC_IMM64);
2264     } else {
2265       emit_d64(cbuf, $src$$constant);
2266     }
2267   %}
2268 
2269   enc_class Con32(immI src)
2270   %{
2271     // Output immediate
2272     $$$emit32$src$$constant;
2273   %}
2274 
2275   enc_class Con32F_as_bits(immF src)
2276   %{
2277     // Output Float immediate bits
2278     jfloat jf = $src$$constant;
2279     jint jf_as_bits = jint_cast(jf);
2280     emit_d32(cbuf, jf_as_bits);
2281   %}
2282 
2283   enc_class Con16(immI src)
2284   %{
2285     // Output immediate
2286     $$$emit16$src$$constant;
2287   %}
2288 
2289   // How is this different from Con32??? XXX
2290   enc_class Con_d32(immI src)
2291   %{
2292     emit_d32(cbuf,$src$$constant);
2293   %}
2294 
2295   enc_class conmemref (rRegP t1) %{    // Con32(storeImmI)
2296     // Output immediate memory reference
2297     emit_rm(cbuf, 0x00, $t1$$reg, 0x05 );
2298     emit_d32(cbuf, 0x00);
2299   %}
2300 
2301   enc_class lock_prefix()
2302   %{
2303     if (os::is_MP()) {
2304       emit_opcode(cbuf, 0xF0); // lock
2305     }
2306   %}
2307 
2308   enc_class REX_mem(memory mem)
2309   %{
2310     if ($mem$$base >= 8) {
2311       if ($mem$$index < 8) {
2312         emit_opcode(cbuf, Assembler::REX_B);
2313       } else {
2314         emit_opcode(cbuf, Assembler::REX_XB);
2315       }
2316     } else {
2317       if ($mem$$index >= 8) {
2318         emit_opcode(cbuf, Assembler::REX_X);
2319       }
2320     }
2321   %}
2322 
2323   enc_class REX_mem_wide(memory mem)
2324   %{
2325     if ($mem$$base >= 8) {
2326       if ($mem$$index < 8) {
2327         emit_opcode(cbuf, Assembler::REX_WB);
2328       } else {
2329         emit_opcode(cbuf, Assembler::REX_WXB);
2330       }
2331     } else {
2332       if ($mem$$index < 8) {
2333         emit_opcode(cbuf, Assembler::REX_W);
2334       } else {
2335         emit_opcode(cbuf, Assembler::REX_WX);
2336       }
2337     }
2338   %}
2339 
2340   // for byte regs
2341   enc_class REX_breg(rRegI reg)
2342   %{
2343     if ($reg$$reg >= 4) {
2344       emit_opcode(cbuf, $reg$$reg < 8 ? Assembler::REX : Assembler::REX_B);
2345     }
2346   %}
2347 
2348   // for byte regs
2349   enc_class REX_reg_breg(rRegI dst, rRegI src)
2350   %{
2351     if ($dst$$reg < 8) {
2352       if ($src$$reg >= 4) {
2353         emit_opcode(cbuf, $src$$reg < 8 ? Assembler::REX : Assembler::REX_B);
2354       }
2355     } else {
2356       if ($src$$reg < 8) {
2357         emit_opcode(cbuf, Assembler::REX_R);
2358       } else {
2359         emit_opcode(cbuf, Assembler::REX_RB);
2360       }
2361     }
2362   %}
2363 
2364   // for byte regs
2365   enc_class REX_breg_mem(rRegI reg, memory mem)
2366   %{
2367     if ($reg$$reg < 8) {
2368       if ($mem$$base < 8) {
2369         if ($mem$$index >= 8) {
2370           emit_opcode(cbuf, Assembler::REX_X);
2371         } else if ($reg$$reg >= 4) {
2372           emit_opcode(cbuf, Assembler::REX);
2373         }
2374       } else {
2375         if ($mem$$index < 8) {
2376           emit_opcode(cbuf, Assembler::REX_B);
2377         } else {
2378           emit_opcode(cbuf, Assembler::REX_XB);
2379         }
2380       }
2381     } else {
2382       if ($mem$$base < 8) {
2383         if ($mem$$index < 8) {
2384           emit_opcode(cbuf, Assembler::REX_R);
2385         } else {
2386           emit_opcode(cbuf, Assembler::REX_RX);
2387         }
2388       } else {
2389         if ($mem$$index < 8) {
2390           emit_opcode(cbuf, Assembler::REX_RB);
2391         } else {
2392           emit_opcode(cbuf, Assembler::REX_RXB);
2393         }
2394       }
2395     }
2396   %}
2397 
2398   enc_class REX_reg(rRegI reg)
2399   %{
2400     if ($reg$$reg >= 8) {
2401       emit_opcode(cbuf, Assembler::REX_B);
2402     }
2403   %}
2404 
2405   enc_class REX_reg_wide(rRegI reg)
2406   %{
2407     if ($reg$$reg < 8) {
2408       emit_opcode(cbuf, Assembler::REX_W);
2409     } else {
2410       emit_opcode(cbuf, Assembler::REX_WB);
2411     }
2412   %}
2413 
2414   enc_class REX_reg_reg(rRegI dst, rRegI src)
2415   %{
2416     if ($dst$$reg < 8) {
2417       if ($src$$reg >= 8) {
2418         emit_opcode(cbuf, Assembler::REX_B);
2419       }
2420     } else {
2421       if ($src$$reg < 8) {
2422         emit_opcode(cbuf, Assembler::REX_R);
2423       } else {
2424         emit_opcode(cbuf, Assembler::REX_RB);
2425       }
2426     }
2427   %}
2428 
2429   enc_class REX_reg_reg_wide(rRegI dst, rRegI src)
2430   %{
2431     if ($dst$$reg < 8) {
2432       if ($src$$reg < 8) {
2433         emit_opcode(cbuf, Assembler::REX_W);
2434       } else {
2435         emit_opcode(cbuf, Assembler::REX_WB);
2436       }
2437     } else {
2438       if ($src$$reg < 8) {
2439         emit_opcode(cbuf, Assembler::REX_WR);
2440       } else {
2441         emit_opcode(cbuf, Assembler::REX_WRB);
2442       }
2443     }
2444   %}
2445 
2446   enc_class REX_reg_mem(rRegI reg, memory mem)
2447   %{
2448     if ($reg$$reg < 8) {
2449       if ($mem$$base < 8) {
2450         if ($mem$$index >= 8) {
2451           emit_opcode(cbuf, Assembler::REX_X);
2452         }
2453       } else {
2454         if ($mem$$index < 8) {
2455           emit_opcode(cbuf, Assembler::REX_B);
2456         } else {
2457           emit_opcode(cbuf, Assembler::REX_XB);
2458         }
2459       }
2460     } else {
2461       if ($mem$$base < 8) {
2462         if ($mem$$index < 8) {
2463           emit_opcode(cbuf, Assembler::REX_R);
2464         } else {
2465           emit_opcode(cbuf, Assembler::REX_RX);
2466         }
2467       } else {
2468         if ($mem$$index < 8) {
2469           emit_opcode(cbuf, Assembler::REX_RB);
2470         } else {
2471           emit_opcode(cbuf, Assembler::REX_RXB);
2472         }
2473       }
2474     }
2475   %}
2476 
2477   enc_class REX_reg_mem_wide(rRegL reg, memory mem)
2478   %{
2479     if ($reg$$reg < 8) {
2480       if ($mem$$base < 8) {
2481         if ($mem$$index < 8) {
2482           emit_opcode(cbuf, Assembler::REX_W);
2483         } else {
2484           emit_opcode(cbuf, Assembler::REX_WX);
2485         }
2486       } else {
2487         if ($mem$$index < 8) {
2488           emit_opcode(cbuf, Assembler::REX_WB);
2489         } else {
2490           emit_opcode(cbuf, Assembler::REX_WXB);
2491         }
2492       }
2493     } else {
2494       if ($mem$$base < 8) {
2495         if ($mem$$index < 8) {
2496           emit_opcode(cbuf, Assembler::REX_WR);
2497         } else {
2498           emit_opcode(cbuf, Assembler::REX_WRX);
2499         }
2500       } else {
2501         if ($mem$$index < 8) {
2502           emit_opcode(cbuf, Assembler::REX_WRB);
2503         } else {
2504           emit_opcode(cbuf, Assembler::REX_WRXB);
2505         }
2506       }
2507     }
2508   %}
2509 
2510   enc_class reg_mem(rRegI ereg, memory mem)
2511   %{
2512     // High registers handle in encode_RegMem
2513     int reg = $ereg$$reg;
2514     int base = $mem$$base;
2515     int index = $mem$$index;
2516     int scale = $mem$$scale;
2517     int disp = $mem$$disp;
2518     relocInfo::relocType disp_reloc = $mem->disp_reloc();
2519 
2520     encode_RegMem(cbuf, reg, base, index, scale, disp, disp_reloc);
2521   %}
2522 
2523   enc_class RM_opc_mem(immI rm_opcode, memory mem)
2524   %{
2525     int rm_byte_opcode = $rm_opcode$$constant;
2526 
2527     // High registers handle in encode_RegMem
2528     int base = $mem$$base;
2529     int index = $mem$$index;
2530     int scale = $mem$$scale;
2531     int displace = $mem$$disp;
2532 
2533     relocInfo::relocType disp_reloc = $mem->disp_reloc();       // disp-as-oop when
2534                                             // working with static
2535                                             // globals
2536     encode_RegMem(cbuf, rm_byte_opcode, base, index, scale, displace,
2537                   disp_reloc);
2538   %}
2539 
2540   enc_class reg_lea(rRegI dst, rRegI src0, immI src1)
2541   %{
2542     int reg_encoding = $dst$$reg;
2543     int base         = $src0$$reg;      // 0xFFFFFFFF indicates no base
2544     int index        = 0x04;            // 0x04 indicates no index
2545     int scale        = 0x00;            // 0x00 indicates no scale
2546     int displace     = $src1$$constant; // 0x00 indicates no displacement
2547     relocInfo::relocType disp_reloc = relocInfo::none;
2548     encode_RegMem(cbuf, reg_encoding, base, index, scale, displace,
2549                   disp_reloc);
2550   %}
2551 
2552   enc_class neg_reg(rRegI dst)
2553   %{
2554     int dstenc = $dst$$reg;
2555     if (dstenc >= 8) {
2556       emit_opcode(cbuf, Assembler::REX_B);
2557       dstenc -= 8;
2558     }
2559     // NEG $dst
2560     emit_opcode(cbuf, 0xF7);
2561     emit_rm(cbuf, 0x3, 0x03, dstenc);
2562   %}
2563 
2564   enc_class neg_reg_wide(rRegI dst)
2565   %{
2566     int dstenc = $dst$$reg;
2567     if (dstenc < 8) {
2568       emit_opcode(cbuf, Assembler::REX_W);
2569     } else {
2570       emit_opcode(cbuf, Assembler::REX_WB);
2571       dstenc -= 8;
2572     }
2573     // NEG $dst
2574     emit_opcode(cbuf, 0xF7);
2575     emit_rm(cbuf, 0x3, 0x03, dstenc);
2576   %}
2577 
2578   enc_class setLT_reg(rRegI dst)
2579   %{
2580     int dstenc = $dst$$reg;
2581     if (dstenc >= 8) {
2582       emit_opcode(cbuf, Assembler::REX_B);
2583       dstenc -= 8;
2584     } else if (dstenc >= 4) {
2585       emit_opcode(cbuf, Assembler::REX);
2586     }
2587     // SETLT $dst
2588     emit_opcode(cbuf, 0x0F);
2589     emit_opcode(cbuf, 0x9C);
2590     emit_rm(cbuf, 0x3, 0x0, dstenc);
2591   %}
2592 
2593   enc_class setNZ_reg(rRegI dst)
2594   %{
2595     int dstenc = $dst$$reg;
2596     if (dstenc >= 8) {
2597       emit_opcode(cbuf, Assembler::REX_B);
2598       dstenc -= 8;
2599     } else if (dstenc >= 4) {
2600       emit_opcode(cbuf, Assembler::REX);
2601     }
2602     // SETNZ $dst
2603     emit_opcode(cbuf, 0x0F);
2604     emit_opcode(cbuf, 0x95);
2605     emit_rm(cbuf, 0x3, 0x0, dstenc);
2606   %}
2607 
2608 
2609   // Compare the lonogs and set -1, 0, or 1 into dst
2610   enc_class cmpl3_flag(rRegL src1, rRegL src2, rRegI dst)
2611   %{
2612     int src1enc = $src1$$reg;
2613     int src2enc = $src2$$reg;
2614     int dstenc = $dst$$reg;
2615 
2616     // cmpq $src1, $src2
2617     if (src1enc < 8) {
2618       if (src2enc < 8) {
2619         emit_opcode(cbuf, Assembler::REX_W);
2620       } else {
2621         emit_opcode(cbuf, Assembler::REX_WB);
2622       }
2623     } else {
2624       if (src2enc < 8) {
2625         emit_opcode(cbuf, Assembler::REX_WR);
2626       } else {
2627         emit_opcode(cbuf, Assembler::REX_WRB);
2628       }
2629     }
2630     emit_opcode(cbuf, 0x3B);
2631     emit_rm(cbuf, 0x3, src1enc & 7, src2enc & 7);
2632 
2633     // movl $dst, -1
2634     if (dstenc >= 8) {
2635       emit_opcode(cbuf, Assembler::REX_B);
2636     }
2637     emit_opcode(cbuf, 0xB8 | (dstenc & 7));
2638     emit_d32(cbuf, -1);
2639 
2640     // jl,s done
2641     emit_opcode(cbuf, 0x7C);
2642     emit_d8(cbuf, dstenc < 4 ? 0x06 : 0x08);
2643 
2644     // setne $dst
2645     if (dstenc >= 4) {
2646       emit_opcode(cbuf, dstenc < 8 ? Assembler::REX : Assembler::REX_B);
2647     }
2648     emit_opcode(cbuf, 0x0F);
2649     emit_opcode(cbuf, 0x95);
2650     emit_opcode(cbuf, 0xC0 | (dstenc & 7));
2651 
2652     // movzbl $dst, $dst
2653     if (dstenc >= 4) {
2654       emit_opcode(cbuf, dstenc < 8 ? Assembler::REX : Assembler::REX_RB);
2655     }
2656     emit_opcode(cbuf, 0x0F);
2657     emit_opcode(cbuf, 0xB6);
2658     emit_rm(cbuf, 0x3, dstenc & 7, dstenc & 7);
2659   %}
2660 
2661   enc_class Push_ResultXD(regD dst) %{
2662     MacroAssembler _masm(&cbuf);
2663     __ fstp_d(Address(rsp, 0));
2664     __ movdbl($dst$$XMMRegister, Address(rsp, 0));
2665     __ addptr(rsp, 8);
2666   %}
2667 
2668   enc_class Push_SrcXD(regD src) %{
2669     MacroAssembler _masm(&cbuf);
2670     __ subptr(rsp, 8);
2671     __ movdbl(Address(rsp, 0), $src$$XMMRegister);
2672     __ fld_d(Address(rsp, 0));
2673   %}
2674 
2675 
2676   enc_class enc_rethrow()
2677   %{
2678     cbuf.set_insts_mark();
2679     emit_opcode(cbuf, 0xE9); // jmp entry
2680     emit_d32_reloc(cbuf,
2681                    (int) (OptoRuntime::rethrow_stub() - cbuf.insts_end() - 4),
2682                    runtime_call_Relocation::spec(),
2683                    RELOC_DISP32);
2684   %}
2685 
2686 %}
2687 
2688 
2689 
2690 //----------FRAME--------------------------------------------------------------
2691 // Definition of frame structure and management information.
2692 //
2693 //  S T A C K   L A Y O U T    Allocators stack-slot number
2694 //                             |   (to get allocators register number
2695 //  G  Owned by    |        |  v    add OptoReg::stack0())
2696 //  r   CALLER     |        |
2697 //  o     |        +--------+      pad to even-align allocators stack-slot
2698 //  w     V        |  pad0  |        numbers; owned by CALLER
2699 //  t   -----------+--------+----> Matcher::_in_arg_limit, unaligned
2700 //  h     ^        |   in   |  5
2701 //        |        |  args  |  4   Holes in incoming args owned by SELF
2702 //  |     |        |        |  3
2703 //  |     |        +--------+
2704 //  V     |        | old out|      Empty on Intel, window on Sparc
2705 //        |    old |preserve|      Must be even aligned.
2706 //        |     SP-+--------+----> Matcher::_old_SP, even aligned
2707 //        |        |   in   |  3   area for Intel ret address
2708 //     Owned by    |preserve|      Empty on Sparc.
2709 //       SELF      +--------+
2710 //        |        |  pad2  |  2   pad to align old SP
2711 //        |        +--------+  1
2712 //        |        | locks  |  0
2713 //        |        +--------+----> OptoReg::stack0(), even aligned
2714 //        |        |  pad1  | 11   pad to align new SP
2715 //        |        +--------+
2716 //        |        |        | 10
2717 //        |        | spills |  9   spills
2718 //        V        |        |  8   (pad0 slot for callee)
2719 //      -----------+--------+----> Matcher::_out_arg_limit, unaligned
2720 //        ^        |  out   |  7
2721 //        |        |  args  |  6   Holes in outgoing args owned by CALLEE
2722 //     Owned by    +--------+
2723 //      CALLEE     | new out|  6   Empty on Intel, window on Sparc
2724 //        |    new |preserve|      Must be even-aligned.
2725 //        |     SP-+--------+----> Matcher::_new_SP, even aligned
2726 //        |        |        |
2727 //
2728 // Note 1: Only region 8-11 is determined by the allocator.  Region 0-5 is
2729 //         known from SELF's arguments and the Java calling convention.
2730 //         Region 6-7 is determined per call site.
2731 // Note 2: If the calling convention leaves holes in the incoming argument
2732 //         area, those holes are owned by SELF.  Holes in the outgoing area
2733 //         are owned by the CALLEE.  Holes should not be nessecary in the
2734 //         incoming area, as the Java calling convention is completely under
2735 //         the control of the AD file.  Doubles can be sorted and packed to
2736 //         avoid holes.  Holes in the outgoing arguments may be nessecary for
2737 //         varargs C calling conventions.
2738 // Note 3: Region 0-3 is even aligned, with pad2 as needed.  Region 3-5 is
2739 //         even aligned with pad0 as needed.
2740 //         Region 6 is even aligned.  Region 6-7 is NOT even aligned;
2741 //         region 6-11 is even aligned; it may be padded out more so that
2742 //         the region from SP to FP meets the minimum stack alignment.
2743 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
2744 //         alignment.  Region 11, pad1, may be dynamically extended so that
2745 //         SP meets the minimum alignment.
2746 
2747 frame
2748 %{
2749   // What direction does stack grow in (assumed to be same for C & Java)
2750   stack_direction(TOWARDS_LOW);
2751 
2752   // These three registers define part of the calling convention
2753   // between compiled code and the interpreter.
2754   inline_cache_reg(RAX);                // Inline Cache Register
2755   interpreter_method_oop_reg(RBX);      // Method Oop Register when
2756                                         // calling interpreter
2757 
2758   // Optional: name the operand used by cisc-spilling to access
2759   // [stack_pointer + offset]
2760   cisc_spilling_operand_name(indOffset32);
2761 
2762   // Number of stack slots consumed by locking an object
2763   sync_stack_slots(2);
2764 
2765   // Compiled code's Frame Pointer
2766   frame_pointer(RSP);
2767 
2768   // Interpreter stores its frame pointer in a register which is
2769   // stored to the stack by I2CAdaptors.
2770   // I2CAdaptors convert from interpreted java to compiled java.
2771   interpreter_frame_pointer(RBP);
2772 
2773   // Stack alignment requirement
2774   stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
2775 
2776   // Number of stack slots between incoming argument block and the start of
2777   // a new frame.  The PROLOG must add this many slots to the stack.  The
2778   // EPILOG must remove this many slots.  amd64 needs two slots for
2779   // return address.
2780   in_preserve_stack_slots(4 + 2 * VerifyStackAtCalls);
2781 
2782   // Number of outgoing stack slots killed above the out_preserve_stack_slots
2783   // for calls to C.  Supports the var-args backing area for register parms.
2784   varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
2785 
2786   // The after-PROLOG location of the return address.  Location of
2787   // return address specifies a type (REG or STACK) and a number
2788   // representing the register number (i.e. - use a register name) or
2789   // stack slot.
2790   // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
2791   // Otherwise, it is above the locks and verification slot and alignment word
2792   return_addr(STACK - 2 +
2793               round_to((Compile::current()->in_preserve_stack_slots() +
2794                         Compile::current()->fixed_slots()),
2795                        stack_alignment_in_slots()));
2796 
2797   // Body of function which returns an integer array locating
2798   // arguments either in registers or in stack slots.  Passed an array
2799   // of ideal registers called "sig" and a "length" count.  Stack-slot
2800   // offsets are based on outgoing arguments, i.e. a CALLER setting up
2801   // arguments for a CALLEE.  Incoming stack arguments are
2802   // automatically biased by the preserve_stack_slots field above.
2803 
2804   calling_convention
2805   %{
2806     // No difference between ingoing/outgoing just pass false
2807     SharedRuntime::java_calling_convention(sig_bt, regs, length, false);
2808   %}
2809 
2810   c_calling_convention
2811   %{
2812     // This is obviously always outgoing
2813     (void) SharedRuntime::c_calling_convention(sig_bt, regs, /*regs2=*/NULL, length);
2814   %}
2815 
2816   // Location of compiled Java return values.  Same as C for now.
2817   return_value
2818   %{
2819     assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
2820            "only return normal values");
2821 
2822     static const int lo[Op_RegL + 1] = {
2823       0,
2824       0,
2825       RAX_num,  // Op_RegN
2826       RAX_num,  // Op_RegI
2827       RAX_num,  // Op_RegP
2828       XMM0_num, // Op_RegF
2829       XMM0_num, // Op_RegD
2830       RAX_num   // Op_RegL
2831     };
2832     static const int hi[Op_RegL + 1] = {
2833       0,
2834       0,
2835       OptoReg::Bad, // Op_RegN
2836       OptoReg::Bad, // Op_RegI
2837       RAX_H_num,    // Op_RegP
2838       OptoReg::Bad, // Op_RegF
2839       XMM0b_num,    // Op_RegD
2840       RAX_H_num     // Op_RegL
2841     };
2842     // Excluded flags and vector registers.
2843     assert(ARRAY_SIZE(hi) == _last_machine_leaf - 5, "missing type");
2844     return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
2845   %}
2846 %}
2847 
2848 //----------ATTRIBUTES---------------------------------------------------------
2849 //----------Operand Attributes-------------------------------------------------
2850 op_attrib op_cost(0);        // Required cost attribute
2851 
2852 //----------Instruction Attributes---------------------------------------------
2853 ins_attrib ins_cost(100);       // Required cost attribute
2854 ins_attrib ins_size(8);         // Required size attribute (in bits)
2855 ins_attrib ins_short_branch(0); // Required flag: is this instruction
2856                                 // a non-matching short branch variant
2857                                 // of some long branch?
2858 ins_attrib ins_alignment(1);    // Required alignment attribute (must
2859                                 // be a power of 2) specifies the
2860                                 // alignment that some part of the
2861                                 // instruction (not necessarily the
2862                                 // start) requires.  If > 1, a
2863                                 // compute_padding() function must be
2864                                 // provided for the instruction
2865 
2866 //----------OPERANDS-----------------------------------------------------------
2867 // Operand definitions must precede instruction definitions for correct parsing
2868 // in the ADLC because operands constitute user defined types which are used in
2869 // instruction definitions.
2870 
2871 //----------Simple Operands----------------------------------------------------
2872 // Immediate Operands
2873 // Integer Immediate
2874 operand immI()
2875 %{
2876   match(ConI);
2877 
2878   op_cost(10);
2879   format %{ %}
2880   interface(CONST_INTER);
2881 %}
2882 
2883 // Constant for test vs zero
2884 operand immI0()
2885 %{
2886   predicate(n->get_int() == 0);
2887   match(ConI);
2888 
2889   op_cost(0);
2890   format %{ %}
2891   interface(CONST_INTER);
2892 %}
2893 
2894 // Constant for increment
2895 operand immI1()
2896 %{
2897   predicate(n->get_int() == 1);
2898   match(ConI);
2899 
2900   op_cost(0);
2901   format %{ %}
2902   interface(CONST_INTER);
2903 %}
2904 
2905 // Constant for decrement
2906 operand immI_M1()
2907 %{
2908   predicate(n->get_int() == -1);
2909   match(ConI);
2910 
2911   op_cost(0);
2912   format %{ %}
2913   interface(CONST_INTER);
2914 %}
2915 
2916 // Valid scale values for addressing modes
2917 operand immI2()
2918 %{
2919   predicate(0 <= n->get_int() && (n->get_int() <= 3));
2920   match(ConI);
2921 
2922   format %{ %}
2923   interface(CONST_INTER);
2924 %}
2925 
2926 operand immI8()
2927 %{
2928   predicate((-0x80 <= n->get_int()) && (n->get_int() < 0x80));
2929   match(ConI);
2930 
2931   op_cost(5);
2932   format %{ %}
2933   interface(CONST_INTER);
2934 %}
2935 
2936 operand immI16()
2937 %{
2938   predicate((-32768 <= n->get_int()) && (n->get_int() <= 32767));
2939   match(ConI);
2940 
2941   op_cost(10);
2942   format %{ %}
2943   interface(CONST_INTER);
2944 %}
2945 
2946 // Int Immediate non-negative
2947 operand immU31()
2948 %{
2949   predicate(n->get_int() >= 0);
2950   match(ConI);
2951 
2952   op_cost(0);
2953   format %{ %}
2954   interface(CONST_INTER);
2955 %}
2956 
2957 // Constant for long shifts
2958 operand immI_32()
2959 %{
2960   predicate( n->get_int() == 32 );
2961   match(ConI);
2962 
2963   op_cost(0);
2964   format %{ %}
2965   interface(CONST_INTER);
2966 %}
2967 
2968 // Constant for long shifts
2969 operand immI_64()
2970 %{
2971   predicate( n->get_int() == 64 );
2972   match(ConI);
2973 
2974   op_cost(0);
2975   format %{ %}
2976   interface(CONST_INTER);
2977 %}
2978 
2979 // Pointer Immediate
2980 operand immP()
2981 %{
2982   match(ConP);
2983 
2984   op_cost(10);
2985   format %{ %}
2986   interface(CONST_INTER);
2987 %}
2988 
2989 // NULL Pointer Immediate
2990 operand immP0()
2991 %{
2992   predicate(n->get_ptr() == 0);
2993   match(ConP);
2994 
2995   op_cost(5);
2996   format %{ %}
2997   interface(CONST_INTER);
2998 %}
2999 
3000 // Pointer Immediate
3001 operand immN() %{
3002   match(ConN);
3003 
3004   op_cost(10);
3005   format %{ %}
3006   interface(CONST_INTER);
3007 %}
3008 
3009 operand immNKlass() %{
3010   match(ConNKlass);
3011 
3012   op_cost(10);
3013   format %{ %}
3014   interface(CONST_INTER);
3015 %}
3016 
3017 // NULL Pointer Immediate
3018 operand immN0() %{
3019   predicate(n->get_narrowcon() == 0);
3020   match(ConN);
3021 
3022   op_cost(5);
3023   format %{ %}
3024   interface(CONST_INTER);
3025 %}
3026 
3027 operand immP31()
3028 %{
3029   predicate(n->as_Type()->type()->reloc() == relocInfo::none
3030             && (n->get_ptr() >> 31) == 0);
3031   match(ConP);
3032 
3033   op_cost(5);
3034   format %{ %}
3035   interface(CONST_INTER);
3036 %}
3037 
3038 
3039 // Long Immediate
3040 operand immL()
3041 %{
3042   match(ConL);
3043 
3044   op_cost(20);
3045   format %{ %}
3046   interface(CONST_INTER);
3047 %}
3048 
3049 // Long Immediate 8-bit
3050 operand immL8()
3051 %{
3052   predicate(-0x80L <= n->get_long() && n->get_long() < 0x80L);
3053   match(ConL);
3054 
3055   op_cost(5);
3056   format %{ %}
3057   interface(CONST_INTER);
3058 %}
3059 
3060 // Long Immediate 32-bit unsigned
3061 operand immUL32()
3062 %{
3063   predicate(n->get_long() == (unsigned int) (n->get_long()));
3064   match(ConL);
3065 
3066   op_cost(10);
3067   format %{ %}
3068   interface(CONST_INTER);
3069 %}
3070 
3071 // Long Immediate 32-bit signed
3072 operand immL32()
3073 %{
3074   predicate(n->get_long() == (int) (n->get_long()));
3075   match(ConL);
3076 
3077   op_cost(15);
3078   format %{ %}
3079   interface(CONST_INTER);
3080 %}
3081 
3082 // Long Immediate zero
3083 operand immL0()
3084 %{
3085   predicate(n->get_long() == 0L);
3086   match(ConL);
3087 
3088   op_cost(10);
3089   format %{ %}
3090   interface(CONST_INTER);
3091 %}
3092 
3093 // Constant for increment
3094 operand immL1()
3095 %{
3096   predicate(n->get_long() == 1);
3097   match(ConL);
3098 
3099   format %{ %}
3100   interface(CONST_INTER);
3101 %}
3102 
3103 // Constant for decrement
3104 operand immL_M1()
3105 %{
3106   predicate(n->get_long() == -1);
3107   match(ConL);
3108 
3109   format %{ %}
3110   interface(CONST_INTER);
3111 %}
3112 
3113 // Long Immediate: the value 10
3114 operand immL10()
3115 %{
3116   predicate(n->get_long() == 10);
3117   match(ConL);
3118 
3119   format %{ %}
3120   interface(CONST_INTER);
3121 %}
3122 
3123 // Long immediate from 0 to 127.
3124 // Used for a shorter form of long mul by 10.
3125 operand immL_127()
3126 %{
3127   predicate(0 <= n->get_long() && n->get_long() < 0x80);
3128   match(ConL);
3129 
3130   op_cost(10);
3131   format %{ %}
3132   interface(CONST_INTER);
3133 %}
3134 
3135 // Long Immediate: low 32-bit mask
3136 operand immL_32bits()
3137 %{
3138   predicate(n->get_long() == 0xFFFFFFFFL);
3139   match(ConL);
3140   op_cost(20);
3141 
3142   format %{ %}
3143   interface(CONST_INTER);
3144 %}
3145 
3146 // Float Immediate zero
3147 operand immF0()
3148 %{
3149   predicate(jint_cast(n->getf()) == 0);
3150   match(ConF);
3151 
3152   op_cost(5);
3153   format %{ %}
3154   interface(CONST_INTER);
3155 %}
3156 
3157 // Float Immediate
3158 operand immF()
3159 %{
3160   match(ConF);
3161 
3162   op_cost(15);
3163   format %{ %}
3164   interface(CONST_INTER);
3165 %}
3166 
3167 // Double Immediate zero
3168 operand immD0()
3169 %{
3170   predicate(jlong_cast(n->getd()) == 0);
3171   match(ConD);
3172 
3173   op_cost(5);
3174   format %{ %}
3175   interface(CONST_INTER);
3176 %}
3177 
3178 // Double Immediate
3179 operand immD()
3180 %{
3181   match(ConD);
3182 
3183   op_cost(15);
3184   format %{ %}
3185   interface(CONST_INTER);
3186 %}
3187 
3188 // Immediates for special shifts (sign extend)
3189 
3190 // Constants for increment
3191 operand immI_16()
3192 %{
3193   predicate(n->get_int() == 16);
3194   match(ConI);
3195 
3196   format %{ %}
3197   interface(CONST_INTER);
3198 %}
3199 
3200 operand immI_24()
3201 %{
3202   predicate(n->get_int() == 24);
3203   match(ConI);
3204 
3205   format %{ %}
3206   interface(CONST_INTER);
3207 %}
3208 
3209 // Constant for byte-wide masking
3210 operand immI_255()
3211 %{
3212   predicate(n->get_int() == 255);
3213   match(ConI);
3214 
3215   format %{ %}
3216   interface(CONST_INTER);
3217 %}
3218 
3219 // Constant for short-wide masking
3220 operand immI_65535()
3221 %{
3222   predicate(n->get_int() == 65535);
3223   match(ConI);
3224 
3225   format %{ %}
3226   interface(CONST_INTER);
3227 %}
3228 
3229 // Constant for byte-wide masking
3230 operand immL_255()
3231 %{
3232   predicate(n->get_long() == 255);
3233   match(ConL);
3234 
3235   format %{ %}
3236   interface(CONST_INTER);
3237 %}
3238 
3239 // Constant for short-wide masking
3240 operand immL_65535()
3241 %{
3242   predicate(n->get_long() == 65535);
3243   match(ConL);
3244 
3245   format %{ %}
3246   interface(CONST_INTER);
3247 %}
3248 
3249 // Register Operands
3250 // Integer Register
3251 operand rRegI()
3252 %{
3253   constraint(ALLOC_IN_RC(int_reg));
3254   match(RegI);
3255 
3256   match(rax_RegI);
3257   match(rbx_RegI);
3258   match(rcx_RegI);
3259   match(rdx_RegI);
3260   match(rdi_RegI);
3261 
3262   format %{ %}
3263   interface(REG_INTER);
3264 %}
3265 
3266 // Special Registers
3267 operand rax_RegI()
3268 %{
3269   constraint(ALLOC_IN_RC(int_rax_reg));
3270   match(RegI);
3271   match(rRegI);
3272 
3273   format %{ "RAX" %}
3274   interface(REG_INTER);
3275 %}
3276 
3277 // Special Registers
3278 operand rbx_RegI()
3279 %{
3280   constraint(ALLOC_IN_RC(int_rbx_reg));
3281   match(RegI);
3282   match(rRegI);
3283 
3284   format %{ "RBX" %}
3285   interface(REG_INTER);
3286 %}
3287 
3288 operand rcx_RegI()
3289 %{
3290   constraint(ALLOC_IN_RC(int_rcx_reg));
3291   match(RegI);
3292   match(rRegI);
3293 
3294   format %{ "RCX" %}
3295   interface(REG_INTER);
3296 %}
3297 
3298 operand rdx_RegI()
3299 %{
3300   constraint(ALLOC_IN_RC(int_rdx_reg));
3301   match(RegI);
3302   match(rRegI);
3303 
3304   format %{ "RDX" %}
3305   interface(REG_INTER);
3306 %}
3307 
3308 operand rdi_RegI()
3309 %{
3310   constraint(ALLOC_IN_RC(int_rdi_reg));
3311   match(RegI);
3312   match(rRegI);
3313 
3314   format %{ "RDI" %}
3315   interface(REG_INTER);
3316 %}
3317 
3318 operand no_rcx_RegI()
3319 %{
3320   constraint(ALLOC_IN_RC(int_no_rcx_reg));
3321   match(RegI);
3322   match(rax_RegI);
3323   match(rbx_RegI);
3324   match(rdx_RegI);
3325   match(rdi_RegI);
3326 
3327   format %{ %}
3328   interface(REG_INTER);
3329 %}
3330 
3331 operand no_rax_rdx_RegI()
3332 %{
3333   constraint(ALLOC_IN_RC(int_no_rax_rdx_reg));
3334   match(RegI);
3335   match(rbx_RegI);
3336   match(rcx_RegI);
3337   match(rdi_RegI);
3338 
3339   format %{ %}
3340   interface(REG_INTER);
3341 %}
3342 
3343 // Pointer Register
3344 operand any_RegP()
3345 %{
3346   constraint(ALLOC_IN_RC(any_reg));  
3347   match(RegP);
3348   match(rax_RegP);
3349   match(rbx_RegP);
3350   match(rdi_RegP);
3351   match(rsi_RegP);
3352   match(rbp_RegP);
3353   match(r15_RegP);
3354   match(rRegP);
3355 
3356   format %{ %}
3357   interface(REG_INTER);
3358 %}
3359 
3360 operand rRegP()
3361 %{
3362   constraint(ALLOC_IN_RC(ptr_reg));
3363   match(RegP);
3364   match(rax_RegP);
3365   match(rbx_RegP);
3366   match(rdi_RegP);
3367   match(rsi_RegP);
3368   match(rbp_RegP);  // See Q&A below about
3369   match(r15_RegP);  // r15_RegP and rbp_RegP.
3370 
3371   format %{ %}
3372   interface(REG_INTER);
3373 %}
3374 
3375 operand rRegN() %{
3376   constraint(ALLOC_IN_RC(int_reg));
3377   match(RegN);
3378 
3379   format %{ %}
3380   interface(REG_INTER);
3381 %}
3382 
3383 // Question: Why is r15_RegP (the read-only TLS register) a match for rRegP?
3384 // Answer: Operand match rules govern the DFA as it processes instruction inputs.
3385 // It's fine for an instruction input that expects rRegP to match a r15_RegP.
3386 // The output of an instruction is controlled by the allocator, which respects
3387 // register class masks, not match rules.  Unless an instruction mentions
3388 // r15_RegP or any_RegP explicitly as its output, r15 will not be considered
3389 // by the allocator as an input.
3390 // The same logic applies to rbp_RegP being a match for rRegP: If PreserveFramePointer==true,
3391 // the RBP is used as a proper frame pointer and is not included in ptr_reg. As a
3392 // result, RBP is not included in the output of the instruction either.
3393 
3394 operand no_rax_RegP()
3395 %{
3396   constraint(ALLOC_IN_RC(ptr_no_rax_reg));
3397   match(RegP);
3398   match(rbx_RegP);
3399   match(rsi_RegP);
3400   match(rdi_RegP);
3401 
3402   format %{ %}
3403   interface(REG_INTER);
3404 %}
3405 
3406 // This operand is not allowed to use RBP even if
3407 // RBP is not used to hold the frame pointer.
3408 operand no_rbp_RegP()
3409 %{
3410   constraint(ALLOC_IN_RC(ptr_reg_no_rbp));
3411   match(RegP);
3412   match(rbx_RegP);
3413   match(rsi_RegP);
3414   match(rdi_RegP);
3415 
3416   format %{ %}
3417   interface(REG_INTER);
3418 %}
3419 
3420 operand no_rax_rbx_RegP()
3421 %{
3422   constraint(ALLOC_IN_RC(ptr_no_rax_rbx_reg));
3423   match(RegP);
3424   match(rsi_RegP);
3425   match(rdi_RegP);
3426 
3427   format %{ %}
3428   interface(REG_INTER);
3429 %}
3430 
3431 // Special Registers
3432 // Return a pointer value
3433 operand rax_RegP()
3434 %{
3435   constraint(ALLOC_IN_RC(ptr_rax_reg));
3436   match(RegP);
3437   match(rRegP);
3438 
3439   format %{ %}
3440   interface(REG_INTER);
3441 %}
3442 
3443 // Special Registers
3444 // Return a compressed pointer value
3445 operand rax_RegN()
3446 %{
3447   constraint(ALLOC_IN_RC(int_rax_reg));
3448   match(RegN);
3449   match(rRegN);
3450 
3451   format %{ %}
3452   interface(REG_INTER);
3453 %}
3454 
3455 // Used in AtomicAdd
3456 operand rbx_RegP()
3457 %{
3458   constraint(ALLOC_IN_RC(ptr_rbx_reg));
3459   match(RegP);
3460   match(rRegP);
3461 
3462   format %{ %}
3463   interface(REG_INTER);
3464 %}
3465 
3466 operand rsi_RegP()
3467 %{
3468   constraint(ALLOC_IN_RC(ptr_rsi_reg));
3469   match(RegP);
3470   match(rRegP);
3471 
3472   format %{ %}
3473   interface(REG_INTER);
3474 %}
3475 
3476 // Used in rep stosq
3477 operand rdi_RegP()
3478 %{
3479   constraint(ALLOC_IN_RC(ptr_rdi_reg));
3480   match(RegP);
3481   match(rRegP);
3482 
3483   format %{ %}
3484   interface(REG_INTER);
3485 %}
3486 
3487 operand r15_RegP()
3488 %{
3489   constraint(ALLOC_IN_RC(ptr_r15_reg));
3490   match(RegP);
3491   match(rRegP);
3492 
3493   format %{ %}
3494   interface(REG_INTER);
3495 %}
3496 
3497 operand rRegL()
3498 %{
3499   constraint(ALLOC_IN_RC(long_reg));
3500   match(RegL);
3501   match(rax_RegL);
3502   match(rdx_RegL);
3503 
3504   format %{ %}
3505   interface(REG_INTER);
3506 %}
3507 
3508 // Special Registers
3509 operand no_rax_rdx_RegL()
3510 %{
3511   constraint(ALLOC_IN_RC(long_no_rax_rdx_reg));
3512   match(RegL);
3513   match(rRegL);
3514 
3515   format %{ %}
3516   interface(REG_INTER);
3517 %}
3518 
3519 operand no_rax_RegL()
3520 %{
3521   constraint(ALLOC_IN_RC(long_no_rax_rdx_reg));
3522   match(RegL);
3523   match(rRegL);
3524   match(rdx_RegL);
3525 
3526   format %{ %}
3527   interface(REG_INTER);
3528 %}
3529 
3530 operand no_rcx_RegL()
3531 %{
3532   constraint(ALLOC_IN_RC(long_no_rcx_reg));
3533   match(RegL);
3534   match(rRegL);
3535 
3536   format %{ %}
3537   interface(REG_INTER);
3538 %}
3539 
3540 operand rax_RegL()
3541 %{
3542   constraint(ALLOC_IN_RC(long_rax_reg));
3543   match(RegL);
3544   match(rRegL);
3545 
3546   format %{ "RAX" %}
3547   interface(REG_INTER);
3548 %}
3549 
3550 operand rcx_RegL()
3551 %{
3552   constraint(ALLOC_IN_RC(long_rcx_reg));
3553   match(RegL);
3554   match(rRegL);
3555 
3556   format %{ %}
3557   interface(REG_INTER);
3558 %}
3559 
3560 operand rdx_RegL()
3561 %{
3562   constraint(ALLOC_IN_RC(long_rdx_reg));
3563   match(RegL);
3564   match(rRegL);
3565 
3566   format %{ %}
3567   interface(REG_INTER);
3568 %}
3569 
3570 // Flags register, used as output of compare instructions
3571 operand rFlagsReg()
3572 %{
3573   constraint(ALLOC_IN_RC(int_flags));
3574   match(RegFlags);
3575 
3576   format %{ "RFLAGS" %}
3577   interface(REG_INTER);
3578 %}
3579 
3580 // Flags register, used as output of FLOATING POINT compare instructions
3581 operand rFlagsRegU()
3582 %{
3583   constraint(ALLOC_IN_RC(int_flags));
3584   match(RegFlags);
3585 
3586   format %{ "RFLAGS_U" %}
3587   interface(REG_INTER);
3588 %}
3589 
3590 operand rFlagsRegUCF() %{
3591   constraint(ALLOC_IN_RC(int_flags));
3592   match(RegFlags);
3593   predicate(false);
3594 
3595   format %{ "RFLAGS_U_CF" %}
3596   interface(REG_INTER);
3597 %}
3598 
3599 // Float register operands
3600 operand regF()
3601 %{
3602   constraint(ALLOC_IN_RC(float_reg));
3603   match(RegF);
3604 
3605   format %{ %}
3606   interface(REG_INTER);
3607 %}
3608 
3609 // Double register operands
3610 operand regD()
3611 %{
3612   constraint(ALLOC_IN_RC(double_reg));
3613   match(RegD);
3614 
3615   format %{ %}
3616   interface(REG_INTER);
3617 %}
3618 
3619 //----------Memory Operands----------------------------------------------------
3620 // Direct Memory Operand
3621 // operand direct(immP addr)
3622 // %{
3623 //   match(addr);
3624 
3625 //   format %{ "[$addr]" %}
3626 //   interface(MEMORY_INTER) %{
3627 //     base(0xFFFFFFFF);
3628 //     index(0x4);
3629 //     scale(0x0);
3630 //     disp($addr);
3631 //   %}
3632 // %}
3633 
3634 // Indirect Memory Operand
3635 operand indirect(any_RegP reg)
3636 %{
3637   constraint(ALLOC_IN_RC(ptr_reg));
3638   match(reg);
3639 
3640   format %{ "[$reg]" %}
3641   interface(MEMORY_INTER) %{
3642     base($reg);
3643     index(0x4);
3644     scale(0x0);
3645     disp(0x0);
3646   %}
3647 %}
3648 
3649 // Indirect Memory Plus Short Offset Operand
3650 operand indOffset8(any_RegP reg, immL8 off)
3651 %{
3652   constraint(ALLOC_IN_RC(ptr_reg));
3653   match(AddP reg off);
3654 
3655   format %{ "[$reg + $off (8-bit)]" %}
3656   interface(MEMORY_INTER) %{
3657     base($reg);
3658     index(0x4);
3659     scale(0x0);
3660     disp($off);
3661   %}
3662 %}
3663 
3664 // Indirect Memory Plus Long Offset Operand
3665 operand indOffset32(any_RegP reg, immL32 off)
3666 %{
3667   constraint(ALLOC_IN_RC(ptr_reg));
3668   match(AddP reg off);
3669 
3670   format %{ "[$reg + $off (32-bit)]" %}
3671   interface(MEMORY_INTER) %{
3672     base($reg);
3673     index(0x4);
3674     scale(0x0);
3675     disp($off);
3676   %}
3677 %}
3678 
3679 // Indirect Memory Plus Index Register Plus Offset Operand
3680 operand indIndexOffset(any_RegP reg, rRegL lreg, immL32 off)
3681 %{
3682   constraint(ALLOC_IN_RC(ptr_reg));
3683   match(AddP (AddP reg lreg) off);
3684 
3685   op_cost(10);
3686   format %{"[$reg + $off + $lreg]" %}
3687   interface(MEMORY_INTER) %{
3688     base($reg);
3689     index($lreg);
3690     scale(0x0);
3691     disp($off);
3692   %}
3693 %}
3694 
3695 // Indirect Memory Plus Index Register Plus Offset Operand
3696 operand indIndex(any_RegP reg, rRegL lreg)
3697 %{
3698   constraint(ALLOC_IN_RC(ptr_reg));
3699   match(AddP reg lreg);
3700 
3701   op_cost(10);
3702   format %{"[$reg + $lreg]" %}
3703   interface(MEMORY_INTER) %{
3704     base($reg);
3705     index($lreg);
3706     scale(0x0);
3707     disp(0x0);
3708   %}
3709 %}
3710 
3711 // Indirect Memory Times Scale Plus Index Register
3712 operand indIndexScale(any_RegP reg, rRegL lreg, immI2 scale)
3713 %{
3714   constraint(ALLOC_IN_RC(ptr_reg));
3715   match(AddP reg (LShiftL lreg scale));
3716 
3717   op_cost(10);
3718   format %{"[$reg + $lreg << $scale]" %}
3719   interface(MEMORY_INTER) %{
3720     base($reg);
3721     index($lreg);
3722     scale($scale);
3723     disp(0x0);
3724   %}
3725 %}
3726 
3727 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
3728 operand indIndexScaleOffset(any_RegP reg, immL32 off, rRegL lreg, immI2 scale)
3729 %{
3730   constraint(ALLOC_IN_RC(ptr_reg));
3731   match(AddP (AddP reg (LShiftL lreg scale)) off);
3732 
3733   op_cost(10);
3734   format %{"[$reg + $off + $lreg << $scale]" %}
3735   interface(MEMORY_INTER) %{
3736     base($reg);
3737     index($lreg);
3738     scale($scale);
3739     disp($off);
3740   %}
3741 %}
3742 
3743 // Indirect Memory Plus Positive Index Register Plus Offset Operand
3744 operand indPosIndexOffset(any_RegP reg, immL32 off, rRegI idx)
3745 %{
3746   constraint(ALLOC_IN_RC(ptr_reg));
3747   predicate(n->in(2)->in(3)->as_Type()->type()->is_long()->_lo >= 0);
3748   match(AddP (AddP reg (ConvI2L idx)) off);
3749 
3750   op_cost(10);
3751   format %{"[$reg + $off + $idx]" %}
3752   interface(MEMORY_INTER) %{
3753     base($reg);
3754     index($idx);
3755     scale(0x0);
3756     disp($off);
3757   %}
3758 %}
3759 
3760 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
3761 operand indPosIndexScaleOffset(any_RegP reg, immL32 off, rRegI idx, immI2 scale)
3762 %{
3763   constraint(ALLOC_IN_RC(ptr_reg));
3764   predicate(n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
3765   match(AddP (AddP reg (LShiftL (ConvI2L idx) scale)) off);
3766 
3767   op_cost(10);
3768   format %{"[$reg + $off + $idx << $scale]" %}
3769   interface(MEMORY_INTER) %{
3770     base($reg);
3771     index($idx);
3772     scale($scale);
3773     disp($off);
3774   %}
3775 %}
3776 
3777 // Indirect Narrow Oop Plus Offset Operand
3778 // Note: x86 architecture doesn't support "scale * index + offset" without a base
3779 // we can't free r12 even with Universe::narrow_oop_base() == NULL.
3780 operand indCompressedOopOffset(rRegN reg, immL32 off) %{
3781   predicate(UseCompressedOops && (Universe::narrow_oop_shift() == Address::times_8));
3782   constraint(ALLOC_IN_RC(ptr_reg));
3783   match(AddP (DecodeN reg) off);
3784 
3785   op_cost(10);
3786   format %{"[R12 + $reg << 3 + $off] (compressed oop addressing)" %}
3787   interface(MEMORY_INTER) %{
3788     base(0xc); // R12
3789     index($reg);
3790     scale(0x3);
3791     disp($off);
3792   %}
3793 %}
3794 
3795 // Indirect Memory Operand
3796 operand indirectNarrow(rRegN reg)
3797 %{
3798   predicate(Universe::narrow_oop_shift() == 0);
3799   constraint(ALLOC_IN_RC(ptr_reg));
3800   match(DecodeN reg);
3801 
3802   format %{ "[$reg]" %}
3803   interface(MEMORY_INTER) %{
3804     base($reg);
3805     index(0x4);
3806     scale(0x0);
3807     disp(0x0);
3808   %}
3809 %}
3810 
3811 // Indirect Memory Plus Short Offset Operand
3812 operand indOffset8Narrow(rRegN reg, immL8 off)
3813 %{
3814   predicate(Universe::narrow_oop_shift() == 0);
3815   constraint(ALLOC_IN_RC(ptr_reg));
3816   match(AddP (DecodeN reg) off);
3817 
3818   format %{ "[$reg + $off (8-bit)]" %}
3819   interface(MEMORY_INTER) %{
3820     base($reg);
3821     index(0x4);
3822     scale(0x0);
3823     disp($off);
3824   %}
3825 %}
3826 
3827 // Indirect Memory Plus Long Offset Operand
3828 operand indOffset32Narrow(rRegN reg, immL32 off)
3829 %{
3830   predicate(Universe::narrow_oop_shift() == 0);
3831   constraint(ALLOC_IN_RC(ptr_reg));
3832   match(AddP (DecodeN reg) off);
3833 
3834   format %{ "[$reg + $off (32-bit)]" %}
3835   interface(MEMORY_INTER) %{
3836     base($reg);
3837     index(0x4);
3838     scale(0x0);
3839     disp($off);
3840   %}
3841 %}
3842 
3843 // Indirect Memory Plus Index Register Plus Offset Operand
3844 operand indIndexOffsetNarrow(rRegN reg, rRegL lreg, immL32 off)
3845 %{
3846   predicate(Universe::narrow_oop_shift() == 0);
3847   constraint(ALLOC_IN_RC(ptr_reg));
3848   match(AddP (AddP (DecodeN reg) lreg) off);
3849 
3850   op_cost(10);
3851   format %{"[$reg + $off + $lreg]" %}
3852   interface(MEMORY_INTER) %{
3853     base($reg);
3854     index($lreg);
3855     scale(0x0);
3856     disp($off);
3857   %}
3858 %}
3859 
3860 // Indirect Memory Plus Index Register Plus Offset Operand
3861 operand indIndexNarrow(rRegN reg, rRegL lreg)
3862 %{
3863   predicate(Universe::narrow_oop_shift() == 0);
3864   constraint(ALLOC_IN_RC(ptr_reg));
3865   match(AddP (DecodeN reg) lreg);
3866 
3867   op_cost(10);
3868   format %{"[$reg + $lreg]" %}
3869   interface(MEMORY_INTER) %{
3870     base($reg);
3871     index($lreg);
3872     scale(0x0);
3873     disp(0x0);
3874   %}
3875 %}
3876 
3877 // Indirect Memory Times Scale Plus Index Register
3878 operand indIndexScaleNarrow(rRegN reg, rRegL lreg, immI2 scale)
3879 %{
3880   predicate(Universe::narrow_oop_shift() == 0);
3881   constraint(ALLOC_IN_RC(ptr_reg));
3882   match(AddP (DecodeN reg) (LShiftL lreg scale));
3883 
3884   op_cost(10);
3885   format %{"[$reg + $lreg << $scale]" %}
3886   interface(MEMORY_INTER) %{
3887     base($reg);
3888     index($lreg);
3889     scale($scale);
3890     disp(0x0);
3891   %}
3892 %}
3893 
3894 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
3895 operand indIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegL lreg, immI2 scale)
3896 %{
3897   predicate(Universe::narrow_oop_shift() == 0);
3898   constraint(ALLOC_IN_RC(ptr_reg));
3899   match(AddP (AddP (DecodeN reg) (LShiftL lreg scale)) off);
3900 
3901   op_cost(10);
3902   format %{"[$reg + $off + $lreg << $scale]" %}
3903   interface(MEMORY_INTER) %{
3904     base($reg);
3905     index($lreg);
3906     scale($scale);
3907     disp($off);
3908   %}
3909 %}
3910 
3911 // Indirect Memory Times Plus Positive Index Register Plus Offset Operand
3912 operand indPosIndexOffsetNarrow(rRegN reg, immL32 off, rRegI idx)
3913 %{
3914   constraint(ALLOC_IN_RC(ptr_reg));
3915   predicate(Universe::narrow_oop_shift() == 0 && n->in(2)->in(3)->as_Type()->type()->is_long()->_lo >= 0);
3916   match(AddP (AddP (DecodeN reg) (ConvI2L idx)) off);
3917 
3918   op_cost(10);
3919   format %{"[$reg + $off + $idx]" %}
3920   interface(MEMORY_INTER) %{
3921     base($reg);
3922     index($idx);
3923     scale(0x0);
3924     disp($off);
3925   %}
3926 %}
3927 
3928 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
3929 operand indPosIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegI idx, immI2 scale)
3930 %{
3931   constraint(ALLOC_IN_RC(ptr_reg));
3932   predicate(Universe::narrow_oop_shift() == 0 && n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
3933   match(AddP (AddP (DecodeN reg) (LShiftL (ConvI2L idx) scale)) off);
3934 
3935   op_cost(10);
3936   format %{"[$reg + $off + $idx << $scale]" %}
3937   interface(MEMORY_INTER) %{
3938     base($reg);
3939     index($idx);
3940     scale($scale);
3941     disp($off);
3942   %}
3943 %}
3944 
3945 //----------Special Memory Operands--------------------------------------------
3946 // Stack Slot Operand - This operand is used for loading and storing temporary
3947 //                      values on the stack where a match requires a value to
3948 //                      flow through memory.
3949 operand stackSlotP(sRegP reg)
3950 %{
3951   constraint(ALLOC_IN_RC(stack_slots));
3952   // No match rule because this operand is only generated in matching
3953 
3954   format %{ "[$reg]" %}
3955   interface(MEMORY_INTER) %{
3956     base(0x4);   // RSP
3957     index(0x4);  // No Index
3958     scale(0x0);  // No Scale
3959     disp($reg);  // Stack Offset
3960   %}
3961 %}
3962 
3963 operand stackSlotI(sRegI reg)
3964 %{
3965   constraint(ALLOC_IN_RC(stack_slots));
3966   // No match rule because this operand is only generated in matching
3967 
3968   format %{ "[$reg]" %}
3969   interface(MEMORY_INTER) %{
3970     base(0x4);   // RSP
3971     index(0x4);  // No Index
3972     scale(0x0);  // No Scale
3973     disp($reg);  // Stack Offset
3974   %}
3975 %}
3976 
3977 operand stackSlotF(sRegF reg)
3978 %{
3979   constraint(ALLOC_IN_RC(stack_slots));
3980   // No match rule because this operand is only generated in matching
3981 
3982   format %{ "[$reg]" %}
3983   interface(MEMORY_INTER) %{
3984     base(0x4);   // RSP
3985     index(0x4);  // No Index
3986     scale(0x0);  // No Scale
3987     disp($reg);  // Stack Offset
3988   %}
3989 %}
3990 
3991 operand stackSlotD(sRegD reg)
3992 %{
3993   constraint(ALLOC_IN_RC(stack_slots));
3994   // No match rule because this operand is only generated in matching
3995 
3996   format %{ "[$reg]" %}
3997   interface(MEMORY_INTER) %{
3998     base(0x4);   // RSP
3999     index(0x4);  // No Index
4000     scale(0x0);  // No Scale
4001     disp($reg);  // Stack Offset
4002   %}
4003 %}
4004 operand stackSlotL(sRegL reg)
4005 %{
4006   constraint(ALLOC_IN_RC(stack_slots));
4007   // No match rule because this operand is only generated in matching
4008 
4009   format %{ "[$reg]" %}
4010   interface(MEMORY_INTER) %{
4011     base(0x4);   // RSP
4012     index(0x4);  // No Index
4013     scale(0x0);  // No Scale
4014     disp($reg);  // Stack Offset
4015   %}
4016 %}
4017 
4018 //----------Conditional Branch Operands----------------------------------------
4019 // Comparison Op  - This is the operation of the comparison, and is limited to
4020 //                  the following set of codes:
4021 //                  L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
4022 //
4023 // Other attributes of the comparison, such as unsignedness, are specified
4024 // by the comparison instruction that sets a condition code flags register.
4025 // That result is represented by a flags operand whose subtype is appropriate
4026 // to the unsignedness (etc.) of the comparison.
4027 //
4028 // Later, the instruction which matches both the Comparison Op (a Bool) and
4029 // the flags (produced by the Cmp) specifies the coding of the comparison op
4030 // by matching a specific subtype of Bool operand below, such as cmpOpU.
4031 
4032 // Comparision Code
4033 operand cmpOp()
4034 %{
4035   match(Bool);
4036 
4037   format %{ "" %}
4038   interface(COND_INTER) %{
4039     equal(0x4, "e");
4040     not_equal(0x5, "ne");
4041     less(0xC, "l");
4042     greater_equal(0xD, "ge");
4043     less_equal(0xE, "le");
4044     greater(0xF, "g");
4045     overflow(0x0, "o");
4046     no_overflow(0x1, "no");
4047   %}
4048 %}
4049 
4050 // Comparison Code, unsigned compare.  Used by FP also, with
4051 // C2 (unordered) turned into GT or LT already.  The other bits
4052 // C0 and C3 are turned into Carry & Zero flags.
4053 operand cmpOpU()
4054 %{
4055   match(Bool);
4056 
4057   format %{ "" %}
4058   interface(COND_INTER) %{
4059     equal(0x4, "e");
4060     not_equal(0x5, "ne");
4061     less(0x2, "b");
4062     greater_equal(0x3, "nb");
4063     less_equal(0x6, "be");
4064     greater(0x7, "nbe");
4065     overflow(0x0, "o");
4066     no_overflow(0x1, "no");
4067   %}
4068 %}
4069 
4070 
4071 // Floating comparisons that don't require any fixup for the unordered case
4072 operand cmpOpUCF() %{
4073   match(Bool);
4074   predicate(n->as_Bool()->_test._test == BoolTest::lt ||
4075             n->as_Bool()->_test._test == BoolTest::ge ||
4076             n->as_Bool()->_test._test == BoolTest::le ||
4077             n->as_Bool()->_test._test == BoolTest::gt);
4078   format %{ "" %}
4079   interface(COND_INTER) %{
4080     equal(0x4, "e");
4081     not_equal(0x5, "ne");
4082     less(0x2, "b");
4083     greater_equal(0x3, "nb");
4084     less_equal(0x6, "be");
4085     greater(0x7, "nbe");
4086     overflow(0x0, "o");
4087     no_overflow(0x1, "no");
4088   %}
4089 %}
4090 
4091 
4092 // Floating comparisons that can be fixed up with extra conditional jumps
4093 operand cmpOpUCF2() %{
4094   match(Bool);
4095   predicate(n->as_Bool()->_test._test == BoolTest::ne ||
4096             n->as_Bool()->_test._test == BoolTest::eq);
4097   format %{ "" %}
4098   interface(COND_INTER) %{
4099     equal(0x4, "e");
4100     not_equal(0x5, "ne");
4101     less(0x2, "b");
4102     greater_equal(0x3, "nb");
4103     less_equal(0x6, "be");
4104     greater(0x7, "nbe");
4105     overflow(0x0, "o");
4106     no_overflow(0x1, "no");
4107   %}
4108 %}
4109 
4110 
4111 //----------OPERAND CLASSES----------------------------------------------------
4112 // Operand Classes are groups of operands that are used as to simplify
4113 // instruction definitions by not requiring the AD writer to specify separate
4114 // instructions for every form of operand when the instruction accepts
4115 // multiple operand types with the same basic encoding and format.  The classic
4116 // case of this is memory operands.
4117 
4118 opclass memory(indirect, indOffset8, indOffset32, indIndexOffset, indIndex,
4119                indIndexScale, indIndexScaleOffset, indPosIndexOffset, indPosIndexScaleOffset,
4120                indCompressedOopOffset,
4121                indirectNarrow, indOffset8Narrow, indOffset32Narrow,
4122                indIndexOffsetNarrow, indIndexNarrow, indIndexScaleNarrow,
4123                indIndexScaleOffsetNarrow, indPosIndexOffsetNarrow, indPosIndexScaleOffsetNarrow);
4124 
4125 //----------PIPELINE-----------------------------------------------------------
4126 // Rules which define the behavior of the target architectures pipeline.
4127 pipeline %{
4128 
4129 //----------ATTRIBUTES---------------------------------------------------------
4130 attributes %{
4131   variable_size_instructions;        // Fixed size instructions
4132   max_instructions_per_bundle = 3;   // Up to 3 instructions per bundle
4133   instruction_unit_size = 1;         // An instruction is 1 bytes long
4134   instruction_fetch_unit_size = 16;  // The processor fetches one line
4135   instruction_fetch_units = 1;       // of 16 bytes
4136 
4137   // List of nop instructions
4138   nops( MachNop );
4139 %}
4140 
4141 //----------RESOURCES----------------------------------------------------------
4142 // Resources are the functional units available to the machine
4143 
4144 // Generic P2/P3 pipeline
4145 // 3 decoders, only D0 handles big operands; a "bundle" is the limit of
4146 // 3 instructions decoded per cycle.
4147 // 2 load/store ops per cycle, 1 branch, 1 FPU,
4148 // 3 ALU op, only ALU0 handles mul instructions.
4149 resources( D0, D1, D2, DECODE = D0 | D1 | D2,
4150            MS0, MS1, MS2, MEM = MS0 | MS1 | MS2,
4151            BR, FPU,
4152            ALU0, ALU1, ALU2, ALU = ALU0 | ALU1 | ALU2);
4153 
4154 //----------PIPELINE DESCRIPTION-----------------------------------------------
4155 // Pipeline Description specifies the stages in the machine's pipeline
4156 
4157 // Generic P2/P3 pipeline
4158 pipe_desc(S0, S1, S2, S3, S4, S5);
4159 
4160 //----------PIPELINE CLASSES---------------------------------------------------
4161 // Pipeline Classes describe the stages in which input and output are
4162 // referenced by the hardware pipeline.
4163 
4164 // Naming convention: ialu or fpu
4165 // Then: _reg
4166 // Then: _reg if there is a 2nd register
4167 // Then: _long if it's a pair of instructions implementing a long
4168 // Then: _fat if it requires the big decoder
4169 //   Or: _mem if it requires the big decoder and a memory unit.
4170 
4171 // Integer ALU reg operation
4172 pipe_class ialu_reg(rRegI dst)
4173 %{
4174     single_instruction;
4175     dst    : S4(write);
4176     dst    : S3(read);
4177     DECODE : S0;        // any decoder
4178     ALU    : S3;        // any alu
4179 %}
4180 
4181 // Long ALU reg operation
4182 pipe_class ialu_reg_long(rRegL dst)
4183 %{
4184     instruction_count(2);
4185     dst    : S4(write);
4186     dst    : S3(read);
4187     DECODE : S0(2);     // any 2 decoders
4188     ALU    : S3(2);     // both alus
4189 %}
4190 
4191 // Integer ALU reg operation using big decoder
4192 pipe_class ialu_reg_fat(rRegI dst)
4193 %{
4194     single_instruction;
4195     dst    : S4(write);
4196     dst    : S3(read);
4197     D0     : S0;        // big decoder only
4198     ALU    : S3;        // any alu
4199 %}
4200 
4201 // Long ALU reg operation using big decoder
4202 pipe_class ialu_reg_long_fat(rRegL dst)
4203 %{
4204     instruction_count(2);
4205     dst    : S4(write);
4206     dst    : S3(read);
4207     D0     : S0(2);     // big decoder only; twice
4208     ALU    : S3(2);     // any 2 alus
4209 %}
4210 
4211 // Integer ALU reg-reg operation
4212 pipe_class ialu_reg_reg(rRegI dst, rRegI src)
4213 %{
4214     single_instruction;
4215     dst    : S4(write);
4216     src    : S3(read);
4217     DECODE : S0;        // any decoder
4218     ALU    : S3;        // any alu
4219 %}
4220 
4221 // Long ALU reg-reg operation
4222 pipe_class ialu_reg_reg_long(rRegL dst, rRegL src)
4223 %{
4224     instruction_count(2);
4225     dst    : S4(write);
4226     src    : S3(read);
4227     DECODE : S0(2);     // any 2 decoders
4228     ALU    : S3(2);     // both alus
4229 %}
4230 
4231 // Integer ALU reg-reg operation
4232 pipe_class ialu_reg_reg_fat(rRegI dst, memory src)
4233 %{
4234     single_instruction;
4235     dst    : S4(write);
4236     src    : S3(read);
4237     D0     : S0;        // big decoder only
4238     ALU    : S3;        // any alu
4239 %}
4240 
4241 // Long ALU reg-reg operation
4242 pipe_class ialu_reg_reg_long_fat(rRegL dst, rRegL src)
4243 %{
4244     instruction_count(2);
4245     dst    : S4(write);
4246     src    : S3(read);
4247     D0     : S0(2);     // big decoder only; twice
4248     ALU    : S3(2);     // both alus
4249 %}
4250 
4251 // Integer ALU reg-mem operation
4252 pipe_class ialu_reg_mem(rRegI dst, memory mem)
4253 %{
4254     single_instruction;
4255     dst    : S5(write);
4256     mem    : S3(read);
4257     D0     : S0;        // big decoder only
4258     ALU    : S4;        // any alu
4259     MEM    : S3;        // any mem
4260 %}
4261 
4262 // Integer mem operation (prefetch)
4263 pipe_class ialu_mem(memory mem)
4264 %{
4265     single_instruction;
4266     mem    : S3(read);
4267     D0     : S0;        // big decoder only
4268     MEM    : S3;        // any mem
4269 %}
4270 
4271 // Integer Store to Memory
4272 pipe_class ialu_mem_reg(memory mem, rRegI src)
4273 %{
4274     single_instruction;
4275     mem    : S3(read);
4276     src    : S5(read);
4277     D0     : S0;        // big decoder only
4278     ALU    : S4;        // any alu
4279     MEM    : S3;
4280 %}
4281 
4282 // // Long Store to Memory
4283 // pipe_class ialu_mem_long_reg(memory mem, rRegL src)
4284 // %{
4285 //     instruction_count(2);
4286 //     mem    : S3(read);
4287 //     src    : S5(read);
4288 //     D0     : S0(2);          // big decoder only; twice
4289 //     ALU    : S4(2);     // any 2 alus
4290 //     MEM    : S3(2);  // Both mems
4291 // %}
4292 
4293 // Integer Store to Memory
4294 pipe_class ialu_mem_imm(memory mem)
4295 %{
4296     single_instruction;
4297     mem    : S3(read);
4298     D0     : S0;        // big decoder only
4299     ALU    : S4;        // any alu
4300     MEM    : S3;
4301 %}
4302 
4303 // Integer ALU0 reg-reg operation
4304 pipe_class ialu_reg_reg_alu0(rRegI dst, rRegI src)
4305 %{
4306     single_instruction;
4307     dst    : S4(write);
4308     src    : S3(read);
4309     D0     : S0;        // Big decoder only
4310     ALU0   : S3;        // only alu0
4311 %}
4312 
4313 // Integer ALU0 reg-mem operation
4314 pipe_class ialu_reg_mem_alu0(rRegI dst, memory mem)
4315 %{
4316     single_instruction;
4317     dst    : S5(write);
4318     mem    : S3(read);
4319     D0     : S0;        // big decoder only
4320     ALU0   : S4;        // ALU0 only
4321     MEM    : S3;        // any mem
4322 %}
4323 
4324 // Integer ALU reg-reg operation
4325 pipe_class ialu_cr_reg_reg(rFlagsReg cr, rRegI src1, rRegI src2)
4326 %{
4327     single_instruction;
4328     cr     : S4(write);
4329     src1   : S3(read);
4330     src2   : S3(read);
4331     DECODE : S0;        // any decoder
4332     ALU    : S3;        // any alu
4333 %}
4334 
4335 // Integer ALU reg-imm operation
4336 pipe_class ialu_cr_reg_imm(rFlagsReg cr, rRegI src1)
4337 %{
4338     single_instruction;
4339     cr     : S4(write);
4340     src1   : S3(read);
4341     DECODE : S0;        // any decoder
4342     ALU    : S3;        // any alu
4343 %}
4344 
4345 // Integer ALU reg-mem operation
4346 pipe_class ialu_cr_reg_mem(rFlagsReg cr, rRegI src1, memory src2)
4347 %{
4348     single_instruction;
4349     cr     : S4(write);
4350     src1   : S3(read);
4351     src2   : S3(read);
4352     D0     : S0;        // big decoder only
4353     ALU    : S4;        // any alu
4354     MEM    : S3;
4355 %}
4356 
4357 // Conditional move reg-reg
4358 pipe_class pipe_cmplt( rRegI p, rRegI q, rRegI y)
4359 %{
4360     instruction_count(4);
4361     y      : S4(read);
4362     q      : S3(read);
4363     p      : S3(read);
4364     DECODE : S0(4);     // any decoder
4365 %}
4366 
4367 // Conditional move reg-reg
4368 pipe_class pipe_cmov_reg( rRegI dst, rRegI src, rFlagsReg cr)
4369 %{
4370     single_instruction;
4371     dst    : S4(write);
4372     src    : S3(read);
4373     cr     : S3(read);
4374     DECODE : S0;        // any decoder
4375 %}
4376 
4377 // Conditional move reg-mem
4378 pipe_class pipe_cmov_mem( rFlagsReg cr, rRegI dst, memory src)
4379 %{
4380     single_instruction;
4381     dst    : S4(write);
4382     src    : S3(read);
4383     cr     : S3(read);
4384     DECODE : S0;        // any decoder
4385     MEM    : S3;
4386 %}
4387 
4388 // Conditional move reg-reg long
4389 pipe_class pipe_cmov_reg_long( rFlagsReg cr, rRegL dst, rRegL src)
4390 %{
4391     single_instruction;
4392     dst    : S4(write);
4393     src    : S3(read);
4394     cr     : S3(read);
4395     DECODE : S0(2);     // any 2 decoders
4396 %}
4397 
4398 // XXX
4399 // // Conditional move double reg-reg
4400 // pipe_class pipe_cmovD_reg( rFlagsReg cr, regDPR1 dst, regD src)
4401 // %{
4402 //     single_instruction;
4403 //     dst    : S4(write);
4404 //     src    : S3(read);
4405 //     cr     : S3(read);
4406 //     DECODE : S0;     // any decoder
4407 // %}
4408 
4409 // Float reg-reg operation
4410 pipe_class fpu_reg(regD dst)
4411 %{
4412     instruction_count(2);
4413     dst    : S3(read);
4414     DECODE : S0(2);     // any 2 decoders
4415     FPU    : S3;
4416 %}
4417 
4418 // Float reg-reg operation
4419 pipe_class fpu_reg_reg(regD dst, regD src)
4420 %{
4421     instruction_count(2);
4422     dst    : S4(write);
4423     src    : S3(read);
4424     DECODE : S0(2);     // any 2 decoders
4425     FPU    : S3;
4426 %}
4427 
4428 // Float reg-reg operation
4429 pipe_class fpu_reg_reg_reg(regD dst, regD src1, regD src2)
4430 %{
4431     instruction_count(3);
4432     dst    : S4(write);
4433     src1   : S3(read);
4434     src2   : S3(read);
4435     DECODE : S0(3);     // any 3 decoders
4436     FPU    : S3(2);
4437 %}
4438 
4439 // Float reg-reg operation
4440 pipe_class fpu_reg_reg_reg_reg(regD dst, regD src1, regD src2, regD src3)
4441 %{
4442     instruction_count(4);
4443     dst    : S4(write);
4444     src1   : S3(read);
4445     src2   : S3(read);
4446     src3   : S3(read);
4447     DECODE : S0(4);     // any 3 decoders
4448     FPU    : S3(2);
4449 %}
4450 
4451 // Float reg-reg operation
4452 pipe_class fpu_reg_mem_reg_reg(regD dst, memory src1, regD src2, regD src3)
4453 %{
4454     instruction_count(4);
4455     dst    : S4(write);
4456     src1   : S3(read);
4457     src2   : S3(read);
4458     src3   : S3(read);
4459     DECODE : S1(3);     // any 3 decoders
4460     D0     : S0;        // Big decoder only
4461     FPU    : S3(2);
4462     MEM    : S3;
4463 %}
4464 
4465 // Float reg-mem operation
4466 pipe_class fpu_reg_mem(regD dst, memory mem)
4467 %{
4468     instruction_count(2);
4469     dst    : S5(write);
4470     mem    : S3(read);
4471     D0     : S0;        // big decoder only
4472     DECODE : S1;        // any decoder for FPU POP
4473     FPU    : S4;
4474     MEM    : S3;        // any mem
4475 %}
4476 
4477 // Float reg-mem operation
4478 pipe_class fpu_reg_reg_mem(regD dst, regD src1, memory mem)
4479 %{
4480     instruction_count(3);
4481     dst    : S5(write);
4482     src1   : S3(read);
4483     mem    : S3(read);
4484     D0     : S0;        // big decoder only
4485     DECODE : S1(2);     // any decoder for FPU POP
4486     FPU    : S4;
4487     MEM    : S3;        // any mem
4488 %}
4489 
4490 // Float mem-reg operation
4491 pipe_class fpu_mem_reg(memory mem, regD src)
4492 %{
4493     instruction_count(2);
4494     src    : S5(read);
4495     mem    : S3(read);
4496     DECODE : S0;        // any decoder for FPU PUSH
4497     D0     : S1;        // big decoder only
4498     FPU    : S4;
4499     MEM    : S3;        // any mem
4500 %}
4501 
4502 pipe_class fpu_mem_reg_reg(memory mem, regD src1, regD src2)
4503 %{
4504     instruction_count(3);
4505     src1   : S3(read);
4506     src2   : S3(read);
4507     mem    : S3(read);
4508     DECODE : S0(2);     // any decoder for FPU PUSH
4509     D0     : S1;        // big decoder only
4510     FPU    : S4;
4511     MEM    : S3;        // any mem
4512 %}
4513 
4514 pipe_class fpu_mem_reg_mem(memory mem, regD src1, memory src2)
4515 %{
4516     instruction_count(3);
4517     src1   : S3(read);
4518     src2   : S3(read);
4519     mem    : S4(read);
4520     DECODE : S0;        // any decoder for FPU PUSH
4521     D0     : S0(2);     // big decoder only
4522     FPU    : S4;
4523     MEM    : S3(2);     // any mem
4524 %}
4525 
4526 pipe_class fpu_mem_mem(memory dst, memory src1)
4527 %{
4528     instruction_count(2);
4529     src1   : S3(read);
4530     dst    : S4(read);
4531     D0     : S0(2);     // big decoder only
4532     MEM    : S3(2);     // any mem
4533 %}
4534 
4535 pipe_class fpu_mem_mem_mem(memory dst, memory src1, memory src2)
4536 %{
4537     instruction_count(3);
4538     src1   : S3(read);
4539     src2   : S3(read);
4540     dst    : S4(read);
4541     D0     : S0(3);     // big decoder only
4542     FPU    : S4;
4543     MEM    : S3(3);     // any mem
4544 %}
4545 
4546 pipe_class fpu_mem_reg_con(memory mem, regD src1)
4547 %{
4548     instruction_count(3);
4549     src1   : S4(read);
4550     mem    : S4(read);
4551     DECODE : S0;        // any decoder for FPU PUSH
4552     D0     : S0(2);     // big decoder only
4553     FPU    : S4;
4554     MEM    : S3(2);     // any mem
4555 %}
4556 
4557 // Float load constant
4558 pipe_class fpu_reg_con(regD dst)
4559 %{
4560     instruction_count(2);
4561     dst    : S5(write);
4562     D0     : S0;        // big decoder only for the load
4563     DECODE : S1;        // any decoder for FPU POP
4564     FPU    : S4;
4565     MEM    : S3;        // any mem
4566 %}
4567 
4568 // Float load constant
4569 pipe_class fpu_reg_reg_con(regD dst, regD src)
4570 %{
4571     instruction_count(3);
4572     dst    : S5(write);
4573     src    : S3(read);
4574     D0     : S0;        // big decoder only for the load
4575     DECODE : S1(2);     // any decoder for FPU POP
4576     FPU    : S4;
4577     MEM    : S3;        // any mem
4578 %}
4579 
4580 // UnConditional branch
4581 pipe_class pipe_jmp(label labl)
4582 %{
4583     single_instruction;
4584     BR   : S3;
4585 %}
4586 
4587 // Conditional branch
4588 pipe_class pipe_jcc(cmpOp cmp, rFlagsReg cr, label labl)
4589 %{
4590     single_instruction;
4591     cr    : S1(read);
4592     BR    : S3;
4593 %}
4594 
4595 // Allocation idiom
4596 pipe_class pipe_cmpxchg(rRegP dst, rRegP heap_ptr)
4597 %{
4598     instruction_count(1); force_serialization;
4599     fixed_latency(6);
4600     heap_ptr : S3(read);
4601     DECODE   : S0(3);
4602     D0       : S2;
4603     MEM      : S3;
4604     ALU      : S3(2);
4605     dst      : S5(write);
4606     BR       : S5;
4607 %}
4608 
4609 // Generic big/slow expanded idiom
4610 pipe_class pipe_slow()
4611 %{
4612     instruction_count(10); multiple_bundles; force_serialization;
4613     fixed_latency(100);
4614     D0  : S0(2);
4615     MEM : S3(2);
4616 %}
4617 
4618 // The real do-nothing guy
4619 pipe_class empty()
4620 %{
4621     instruction_count(0);
4622 %}
4623 
4624 // Define the class for the Nop node
4625 define
4626 %{
4627    MachNop = empty;
4628 %}
4629 
4630 %}
4631 
4632 //----------INSTRUCTIONS-------------------------------------------------------
4633 //
4634 // match      -- States which machine-independent subtree may be replaced
4635 //               by this instruction.
4636 // ins_cost   -- The estimated cost of this instruction is used by instruction
4637 //               selection to identify a minimum cost tree of machine
4638 //               instructions that matches a tree of machine-independent
4639 //               instructions.
4640 // format     -- A string providing the disassembly for this instruction.
4641 //               The value of an instruction's operand may be inserted
4642 //               by referring to it with a '$' prefix.
4643 // opcode     -- Three instruction opcodes may be provided.  These are referred
4644 //               to within an encode class as $primary, $secondary, and $tertiary
4645 //               rrspectively.  The primary opcode is commonly used to
4646 //               indicate the type of machine instruction, while secondary
4647 //               and tertiary are often used for prefix options or addressing
4648 //               modes.
4649 // ins_encode -- A list of encode classes with parameters. The encode class
4650 //               name must have been defined in an 'enc_class' specification
4651 //               in the encode section of the architecture description.
4652 
4653 
4654 //----------Load/Store/Move Instructions---------------------------------------
4655 //----------Load Instructions--------------------------------------------------
4656 
4657 // Load Byte (8 bit signed)
4658 instruct loadB(rRegI dst, memory mem)
4659 %{
4660   match(Set dst (LoadB mem));
4661 
4662   ins_cost(125);
4663   format %{ "movsbl  $dst, $mem\t# byte" %}
4664 
4665   ins_encode %{
4666     __ movsbl($dst$$Register, $mem$$Address);
4667   %}
4668 
4669   ins_pipe(ialu_reg_mem);
4670 %}
4671 
4672 // Load Byte (8 bit signed) into Long Register
4673 instruct loadB2L(rRegL dst, memory mem)
4674 %{
4675   match(Set dst (ConvI2L (LoadB mem)));
4676 
4677   ins_cost(125);
4678   format %{ "movsbq  $dst, $mem\t# byte -> long" %}
4679 
4680   ins_encode %{
4681     __ movsbq($dst$$Register, $mem$$Address);
4682   %}
4683 
4684   ins_pipe(ialu_reg_mem);
4685 %}
4686 
4687 // Load Unsigned Byte (8 bit UNsigned)
4688 instruct loadUB(rRegI dst, memory mem)
4689 %{
4690   match(Set dst (LoadUB mem));
4691 
4692   ins_cost(125);
4693   format %{ "movzbl  $dst, $mem\t# ubyte" %}
4694 
4695   ins_encode %{
4696     __ movzbl($dst$$Register, $mem$$Address);
4697   %}
4698 
4699   ins_pipe(ialu_reg_mem);
4700 %}
4701 
4702 // Load Unsigned Byte (8 bit UNsigned) into Long Register
4703 instruct loadUB2L(rRegL dst, memory mem)
4704 %{
4705   match(Set dst (ConvI2L (LoadUB mem)));
4706 
4707   ins_cost(125);
4708   format %{ "movzbq  $dst, $mem\t# ubyte -> long" %}
4709 
4710   ins_encode %{
4711     __ movzbq($dst$$Register, $mem$$Address);
4712   %}
4713 
4714   ins_pipe(ialu_reg_mem);
4715 %}
4716 
4717 // Load Unsigned Byte (8 bit UNsigned) with a 8-bit mask into Long Register
4718 instruct loadUB2L_immI8(rRegL dst, memory mem, immI8 mask, rFlagsReg cr) %{
4719   match(Set dst (ConvI2L (AndI (LoadUB mem) mask)));
4720   effect(KILL cr);
4721 
4722   format %{ "movzbq  $dst, $mem\t# ubyte & 8-bit mask -> long\n\t"
4723             "andl    $dst, $mask" %}
4724   ins_encode %{
4725     Register Rdst = $dst$$Register;
4726     __ movzbq(Rdst, $mem$$Address);
4727     __ andl(Rdst, $mask$$constant);
4728   %}
4729   ins_pipe(ialu_reg_mem);
4730 %}
4731 
4732 // Load Short (16 bit signed)
4733 instruct loadS(rRegI dst, memory mem)
4734 %{
4735   match(Set dst (LoadS mem));
4736 
4737   ins_cost(125);
4738   format %{ "movswl $dst, $mem\t# short" %}
4739 
4740   ins_encode %{
4741     __ movswl($dst$$Register, $mem$$Address);
4742   %}
4743 
4744   ins_pipe(ialu_reg_mem);
4745 %}
4746 
4747 // Load Short (16 bit signed) to Byte (8 bit signed)
4748 instruct loadS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
4749   match(Set dst (RShiftI (LShiftI (LoadS mem) twentyfour) twentyfour));
4750 
4751   ins_cost(125);
4752   format %{ "movsbl $dst, $mem\t# short -> byte" %}
4753   ins_encode %{
4754     __ movsbl($dst$$Register, $mem$$Address);
4755   %}
4756   ins_pipe(ialu_reg_mem);
4757 %}
4758 
4759 // Load Short (16 bit signed) into Long Register
4760 instruct loadS2L(rRegL dst, memory mem)
4761 %{
4762   match(Set dst (ConvI2L (LoadS mem)));
4763 
4764   ins_cost(125);
4765   format %{ "movswq $dst, $mem\t# short -> long" %}
4766 
4767   ins_encode %{
4768     __ movswq($dst$$Register, $mem$$Address);
4769   %}
4770 
4771   ins_pipe(ialu_reg_mem);
4772 %}
4773 
4774 // Load Unsigned Short/Char (16 bit UNsigned)
4775 instruct loadUS(rRegI dst, memory mem)
4776 %{
4777   match(Set dst (LoadUS mem));
4778 
4779   ins_cost(125);
4780   format %{ "movzwl  $dst, $mem\t# ushort/char" %}
4781 
4782   ins_encode %{
4783     __ movzwl($dst$$Register, $mem$$Address);
4784   %}
4785 
4786   ins_pipe(ialu_reg_mem);
4787 %}
4788 
4789 // Load Unsigned Short/Char (16 bit UNsigned) to Byte (8 bit signed)
4790 instruct loadUS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
4791   match(Set dst (RShiftI (LShiftI (LoadUS mem) twentyfour) twentyfour));
4792 
4793   ins_cost(125);
4794   format %{ "movsbl $dst, $mem\t# ushort -> byte" %}
4795   ins_encode %{
4796     __ movsbl($dst$$Register, $mem$$Address);
4797   %}
4798   ins_pipe(ialu_reg_mem);
4799 %}
4800 
4801 // Load Unsigned Short/Char (16 bit UNsigned) into Long Register
4802 instruct loadUS2L(rRegL dst, memory mem)
4803 %{
4804   match(Set dst (ConvI2L (LoadUS mem)));
4805 
4806   ins_cost(125);
4807   format %{ "movzwq  $dst, $mem\t# ushort/char -> long" %}
4808 
4809   ins_encode %{
4810     __ movzwq($dst$$Register, $mem$$Address);
4811   %}
4812 
4813   ins_pipe(ialu_reg_mem);
4814 %}
4815 
4816 // Load Unsigned Short/Char (16 bit UNsigned) with mask 0xFF into Long Register
4817 instruct loadUS2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
4818   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
4819 
4820   format %{ "movzbq  $dst, $mem\t# ushort/char & 0xFF -> long" %}
4821   ins_encode %{
4822     __ movzbq($dst$$Register, $mem$$Address);
4823   %}
4824   ins_pipe(ialu_reg_mem);
4825 %}
4826 
4827 // Load Unsigned Short/Char (16 bit UNsigned) with mask into Long Register
4828 instruct loadUS2L_immI16(rRegL dst, memory mem, immI16 mask, rFlagsReg cr) %{
4829   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
4830   effect(KILL cr);
4831 
4832   format %{ "movzwq  $dst, $mem\t# ushort/char & 16-bit mask -> long\n\t"
4833             "andl    $dst, $mask" %}
4834   ins_encode %{
4835     Register Rdst = $dst$$Register;
4836     __ movzwq(Rdst, $mem$$Address);
4837     __ andl(Rdst, $mask$$constant);
4838   %}
4839   ins_pipe(ialu_reg_mem);
4840 %}
4841 
4842 // Load Integer
4843 instruct loadI(rRegI dst, memory mem)
4844 %{
4845   match(Set dst (LoadI mem));
4846 
4847   ins_cost(125);
4848   format %{ "movl    $dst, $mem\t# int" %}
4849 
4850   ins_encode %{
4851     __ movl($dst$$Register, $mem$$Address);
4852   %}
4853 
4854   ins_pipe(ialu_reg_mem);
4855 %}
4856 
4857 // Load Integer (32 bit signed) to Byte (8 bit signed)
4858 instruct loadI2B(rRegI dst, memory mem, immI_24 twentyfour) %{
4859   match(Set dst (RShiftI (LShiftI (LoadI mem) twentyfour) twentyfour));
4860 
4861   ins_cost(125);
4862   format %{ "movsbl  $dst, $mem\t# int -> byte" %}
4863   ins_encode %{
4864     __ movsbl($dst$$Register, $mem$$Address);
4865   %}
4866   ins_pipe(ialu_reg_mem);
4867 %}
4868 
4869 // Load Integer (32 bit signed) to Unsigned Byte (8 bit UNsigned)
4870 instruct loadI2UB(rRegI dst, memory mem, immI_255 mask) %{
4871   match(Set dst (AndI (LoadI mem) mask));
4872 
4873   ins_cost(125);
4874   format %{ "movzbl  $dst, $mem\t# int -> ubyte" %}
4875   ins_encode %{
4876     __ movzbl($dst$$Register, $mem$$Address);
4877   %}
4878   ins_pipe(ialu_reg_mem);
4879 %}
4880 
4881 // Load Integer (32 bit signed) to Short (16 bit signed)
4882 instruct loadI2S(rRegI dst, memory mem, immI_16 sixteen) %{
4883   match(Set dst (RShiftI (LShiftI (LoadI mem) sixteen) sixteen));
4884 
4885   ins_cost(125);
4886   format %{ "movswl  $dst, $mem\t# int -> short" %}
4887   ins_encode %{
4888     __ movswl($dst$$Register, $mem$$Address);
4889   %}
4890   ins_pipe(ialu_reg_mem);
4891 %}
4892 
4893 // Load Integer (32 bit signed) to Unsigned Short/Char (16 bit UNsigned)
4894 instruct loadI2US(rRegI dst, memory mem, immI_65535 mask) %{
4895   match(Set dst (AndI (LoadI mem) mask));
4896 
4897   ins_cost(125);
4898   format %{ "movzwl  $dst, $mem\t# int -> ushort/char" %}
4899   ins_encode %{
4900     __ movzwl($dst$$Register, $mem$$Address);
4901   %}
4902   ins_pipe(ialu_reg_mem);
4903 %}
4904 
4905 // Load Integer into Long Register
4906 instruct loadI2L(rRegL dst, memory mem)
4907 %{
4908   match(Set dst (ConvI2L (LoadI mem)));
4909 
4910   ins_cost(125);
4911   format %{ "movslq  $dst, $mem\t# int -> long" %}
4912 
4913   ins_encode %{
4914     __ movslq($dst$$Register, $mem$$Address);
4915   %}
4916 
4917   ins_pipe(ialu_reg_mem);
4918 %}
4919 
4920 // Load Integer with mask 0xFF into Long Register
4921 instruct loadI2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
4922   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
4923 
4924   format %{ "movzbq  $dst, $mem\t# int & 0xFF -> long" %}
4925   ins_encode %{
4926     __ movzbq($dst$$Register, $mem$$Address);
4927   %}
4928   ins_pipe(ialu_reg_mem);
4929 %}
4930 
4931 // Load Integer with mask 0xFFFF into Long Register
4932 instruct loadI2L_immI_65535(rRegL dst, memory mem, immI_65535 mask) %{
4933   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
4934 
4935   format %{ "movzwq  $dst, $mem\t# int & 0xFFFF -> long" %}
4936   ins_encode %{
4937     __ movzwq($dst$$Register, $mem$$Address);
4938   %}
4939   ins_pipe(ialu_reg_mem);
4940 %}
4941 
4942 // Load Integer with a 31-bit mask into Long Register
4943 instruct loadI2L_immU31(rRegL dst, memory mem, immU31 mask, rFlagsReg cr) %{
4944   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
4945   effect(KILL cr);
4946 
4947   format %{ "movl    $dst, $mem\t# int & 31-bit mask -> long\n\t"
4948             "andl    $dst, $mask" %}
4949   ins_encode %{
4950     Register Rdst = $dst$$Register;
4951     __ movl(Rdst, $mem$$Address);
4952     __ andl(Rdst, $mask$$constant);
4953   %}
4954   ins_pipe(ialu_reg_mem);
4955 %}
4956 
4957 // Load Unsigned Integer into Long Register
4958 instruct loadUI2L(rRegL dst, memory mem, immL_32bits mask) 
4959 %{
4960   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
4961 
4962   ins_cost(125);
4963   format %{ "movl    $dst, $mem\t# uint -> long" %}
4964 
4965   ins_encode %{
4966     __ movl($dst$$Register, $mem$$Address);
4967   %}
4968 
4969   ins_pipe(ialu_reg_mem);
4970 %}
4971 
4972 // Load Long
4973 instruct loadL(rRegL dst, memory mem)
4974 %{
4975   match(Set dst (LoadL mem));
4976 
4977   ins_cost(125);
4978   format %{ "movq    $dst, $mem\t# long" %}
4979 
4980   ins_encode %{
4981     __ movq($dst$$Register, $mem$$Address);
4982   %}
4983 
4984   ins_pipe(ialu_reg_mem); // XXX
4985 %}
4986 
4987 // Load Range
4988 instruct loadRange(rRegI dst, memory mem)
4989 %{
4990   match(Set dst (LoadRange mem));
4991 
4992   ins_cost(125); // XXX
4993   format %{ "movl    $dst, $mem\t# range" %}
4994   opcode(0x8B);
4995   ins_encode(REX_reg_mem(dst, mem), OpcP, reg_mem(dst, mem));
4996   ins_pipe(ialu_reg_mem);
4997 %}
4998 
4999 // Load Pointer
5000 instruct loadP(rRegP dst, memory mem)
5001 %{
5002   match(Set dst (LoadP mem));
5003 
5004   ins_cost(125); // XXX
5005   format %{ "movq    $dst, $mem\t# ptr" %}
5006   opcode(0x8B);
5007   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5008   ins_pipe(ialu_reg_mem); // XXX
5009 %}
5010 
5011 // Load Compressed Pointer
5012 instruct loadN(rRegN dst, memory mem)
5013 %{
5014    match(Set dst (LoadN mem));
5015 
5016    ins_cost(125); // XXX
5017    format %{ "movl    $dst, $mem\t# compressed ptr" %}
5018    ins_encode %{
5019      __ movl($dst$$Register, $mem$$Address);
5020    %}
5021    ins_pipe(ialu_reg_mem); // XXX
5022 %}
5023 
5024 
5025 // Load Klass Pointer
5026 instruct loadKlass(rRegP dst, memory mem)
5027 %{
5028   match(Set dst (LoadKlass mem));
5029 
5030   ins_cost(125); // XXX
5031   format %{ "movq    $dst, $mem\t# class" %}
5032   opcode(0x8B);
5033   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5034   ins_pipe(ialu_reg_mem); // XXX
5035 %}
5036 
5037 // Load narrow Klass Pointer
5038 instruct loadNKlass(rRegN dst, memory mem)
5039 %{
5040   match(Set dst (LoadNKlass mem));
5041 
5042   ins_cost(125); // XXX
5043   format %{ "movl    $dst, $mem\t# compressed klass ptr" %}
5044   ins_encode %{
5045     __ movl($dst$$Register, $mem$$Address);
5046   %}
5047   ins_pipe(ialu_reg_mem); // XXX
5048 %}
5049 
5050 // Load Float
5051 instruct loadF(regF dst, memory mem)
5052 %{
5053   match(Set dst (LoadF mem));
5054 
5055   ins_cost(145); // XXX
5056   format %{ "movss   $dst, $mem\t# float" %}
5057   ins_encode %{
5058     __ movflt($dst$$XMMRegister, $mem$$Address);
5059   %}
5060   ins_pipe(pipe_slow); // XXX
5061 %}
5062 
5063 // Load Double
5064 instruct loadD_partial(regD dst, memory mem)
5065 %{
5066   predicate(!UseXmmLoadAndClearUpper);
5067   match(Set dst (LoadD mem));
5068 
5069   ins_cost(145); // XXX
5070   format %{ "movlpd  $dst, $mem\t# double" %}
5071   ins_encode %{
5072     __ movdbl($dst$$XMMRegister, $mem$$Address);
5073   %}
5074   ins_pipe(pipe_slow); // XXX
5075 %}
5076 
5077 instruct loadD(regD dst, memory mem)
5078 %{
5079   predicate(UseXmmLoadAndClearUpper);
5080   match(Set dst (LoadD mem));
5081 
5082   ins_cost(145); // XXX
5083   format %{ "movsd   $dst, $mem\t# double" %}
5084   ins_encode %{
5085     __ movdbl($dst$$XMMRegister, $mem$$Address);
5086   %}
5087   ins_pipe(pipe_slow); // XXX
5088 %}
5089 
5090 // Load Effective Address
5091 instruct leaP8(rRegP dst, indOffset8 mem)
5092 %{
5093   match(Set dst mem);
5094 
5095   ins_cost(110); // XXX
5096   format %{ "leaq    $dst, $mem\t# ptr 8" %}
5097   opcode(0x8D);
5098   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5099   ins_pipe(ialu_reg_reg_fat);
5100 %}
5101 
5102 instruct leaP32(rRegP dst, indOffset32 mem)
5103 %{
5104   match(Set dst mem);
5105 
5106   ins_cost(110);
5107   format %{ "leaq    $dst, $mem\t# ptr 32" %}
5108   opcode(0x8D);
5109   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5110   ins_pipe(ialu_reg_reg_fat);
5111 %}
5112 
5113 // instruct leaPIdx(rRegP dst, indIndex mem)
5114 // %{
5115 //   match(Set dst mem);
5116 
5117 //   ins_cost(110);
5118 //   format %{ "leaq    $dst, $mem\t# ptr idx" %}
5119 //   opcode(0x8D);
5120 //   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5121 //   ins_pipe(ialu_reg_reg_fat);
5122 // %}
5123 
5124 instruct leaPIdxOff(rRegP dst, indIndexOffset mem)
5125 %{
5126   match(Set dst mem);
5127 
5128   ins_cost(110);
5129   format %{ "leaq    $dst, $mem\t# ptr idxoff" %}
5130   opcode(0x8D);
5131   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5132   ins_pipe(ialu_reg_reg_fat);
5133 %}
5134 
5135 instruct leaPIdxScale(rRegP dst, indIndexScale mem)
5136 %{
5137   match(Set dst mem);
5138 
5139   ins_cost(110);
5140   format %{ "leaq    $dst, $mem\t# ptr idxscale" %}
5141   opcode(0x8D);
5142   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5143   ins_pipe(ialu_reg_reg_fat);
5144 %}
5145 
5146 instruct leaPIdxScaleOff(rRegP dst, indIndexScaleOffset mem)
5147 %{
5148   match(Set dst mem);
5149 
5150   ins_cost(110);
5151   format %{ "leaq    $dst, $mem\t# ptr idxscaleoff" %}
5152   opcode(0x8D);
5153   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5154   ins_pipe(ialu_reg_reg_fat);
5155 %}
5156 
5157 instruct leaPPosIdxOff(rRegP dst, indPosIndexOffset mem)
5158 %{
5159   match(Set dst mem);
5160 
5161   ins_cost(110);
5162   format %{ "leaq    $dst, $mem\t# ptr posidxoff" %}
5163   opcode(0x8D);
5164   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5165   ins_pipe(ialu_reg_reg_fat);
5166 %}
5167 
5168 instruct leaPPosIdxScaleOff(rRegP dst, indPosIndexScaleOffset mem)
5169 %{
5170   match(Set dst mem);
5171 
5172   ins_cost(110);
5173   format %{ "leaq    $dst, $mem\t# ptr posidxscaleoff" %}
5174   opcode(0x8D);
5175   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5176   ins_pipe(ialu_reg_reg_fat);
5177 %}
5178 
5179 // Load Effective Address which uses Narrow (32-bits) oop
5180 instruct leaPCompressedOopOffset(rRegP dst, indCompressedOopOffset mem)
5181 %{
5182   predicate(UseCompressedOops && (Universe::narrow_oop_shift() != 0));
5183   match(Set dst mem);
5184 
5185   ins_cost(110);
5186   format %{ "leaq    $dst, $mem\t# ptr compressedoopoff32" %}
5187   opcode(0x8D);
5188   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5189   ins_pipe(ialu_reg_reg_fat);
5190 %}
5191 
5192 instruct leaP8Narrow(rRegP dst, indOffset8Narrow mem)
5193 %{
5194   predicate(Universe::narrow_oop_shift() == 0);
5195   match(Set dst mem);
5196 
5197   ins_cost(110); // XXX
5198   format %{ "leaq    $dst, $mem\t# ptr off8narrow" %}
5199   opcode(0x8D);
5200   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5201   ins_pipe(ialu_reg_reg_fat);
5202 %}
5203 
5204 instruct leaP32Narrow(rRegP dst, indOffset32Narrow mem)
5205 %{
5206   predicate(Universe::narrow_oop_shift() == 0);
5207   match(Set dst mem);
5208 
5209   ins_cost(110);
5210   format %{ "leaq    $dst, $mem\t# ptr off32narrow" %}
5211   opcode(0x8D);
5212   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5213   ins_pipe(ialu_reg_reg_fat);
5214 %}
5215 
5216 instruct leaPIdxOffNarrow(rRegP dst, indIndexOffsetNarrow mem)
5217 %{
5218   predicate(Universe::narrow_oop_shift() == 0);
5219   match(Set dst mem);
5220 
5221   ins_cost(110);
5222   format %{ "leaq    $dst, $mem\t# ptr idxoffnarrow" %}
5223   opcode(0x8D);
5224   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5225   ins_pipe(ialu_reg_reg_fat);
5226 %}
5227 
5228 instruct leaPIdxScaleNarrow(rRegP dst, indIndexScaleNarrow mem)
5229 %{
5230   predicate(Universe::narrow_oop_shift() == 0);
5231   match(Set dst mem);
5232 
5233   ins_cost(110);
5234   format %{ "leaq    $dst, $mem\t# ptr idxscalenarrow" %}
5235   opcode(0x8D);
5236   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5237   ins_pipe(ialu_reg_reg_fat);
5238 %}
5239 
5240 instruct leaPIdxScaleOffNarrow(rRegP dst, indIndexScaleOffsetNarrow mem)
5241 %{
5242   predicate(Universe::narrow_oop_shift() == 0);
5243   match(Set dst mem);
5244 
5245   ins_cost(110);
5246   format %{ "leaq    $dst, $mem\t# ptr idxscaleoffnarrow" %}
5247   opcode(0x8D);
5248   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5249   ins_pipe(ialu_reg_reg_fat);
5250 %}
5251 
5252 instruct leaPPosIdxOffNarrow(rRegP dst, indPosIndexOffsetNarrow mem)
5253 %{
5254   predicate(Universe::narrow_oop_shift() == 0);
5255   match(Set dst mem);
5256 
5257   ins_cost(110);
5258   format %{ "leaq    $dst, $mem\t# ptr posidxoffnarrow" %}
5259   opcode(0x8D);
5260   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5261   ins_pipe(ialu_reg_reg_fat);
5262 %}
5263 
5264 instruct leaPPosIdxScaleOffNarrow(rRegP dst, indPosIndexScaleOffsetNarrow mem)
5265 %{
5266   predicate(Universe::narrow_oop_shift() == 0);
5267   match(Set dst mem);
5268 
5269   ins_cost(110);
5270   format %{ "leaq    $dst, $mem\t# ptr posidxscaleoffnarrow" %}
5271   opcode(0x8D);
5272   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5273   ins_pipe(ialu_reg_reg_fat);
5274 %}
5275 
5276 instruct loadConI(rRegI dst, immI src)
5277 %{
5278   match(Set dst src);
5279 
5280   format %{ "movl    $dst, $src\t# int" %}
5281   ins_encode(load_immI(dst, src));
5282   ins_pipe(ialu_reg_fat); // XXX
5283 %}
5284 
5285 instruct loadConI0(rRegI dst, immI0 src, rFlagsReg cr)
5286 %{
5287   match(Set dst src);
5288   effect(KILL cr);
5289 
5290   ins_cost(50);
5291   format %{ "xorl    $dst, $dst\t# int" %}
5292   opcode(0x33); /* + rd */
5293   ins_encode(REX_reg_reg(dst, dst), OpcP, reg_reg(dst, dst));
5294   ins_pipe(ialu_reg);
5295 %}
5296 
5297 instruct loadConL(rRegL dst, immL src)
5298 %{
5299   match(Set dst src);
5300 
5301   ins_cost(150);
5302   format %{ "movq    $dst, $src\t# long" %}
5303   ins_encode(load_immL(dst, src));
5304   ins_pipe(ialu_reg);
5305 %}
5306 
5307 instruct loadConL0(rRegL dst, immL0 src, rFlagsReg cr)
5308 %{
5309   match(Set dst src);
5310   effect(KILL cr);
5311 
5312   ins_cost(50);
5313   format %{ "xorl    $dst, $dst\t# long" %}
5314   opcode(0x33); /* + rd */
5315   ins_encode(REX_reg_reg(dst, dst), OpcP, reg_reg(dst, dst));
5316   ins_pipe(ialu_reg); // XXX
5317 %}
5318 
5319 instruct loadConUL32(rRegL dst, immUL32 src)
5320 %{
5321   match(Set dst src);
5322 
5323   ins_cost(60);
5324   format %{ "movl    $dst, $src\t# long (unsigned 32-bit)" %}
5325   ins_encode(load_immUL32(dst, src));
5326   ins_pipe(ialu_reg);
5327 %}
5328 
5329 instruct loadConL32(rRegL dst, immL32 src)
5330 %{
5331   match(Set dst src);
5332 
5333   ins_cost(70);
5334   format %{ "movq    $dst, $src\t# long (32-bit)" %}
5335   ins_encode(load_immL32(dst, src));
5336   ins_pipe(ialu_reg);
5337 %}
5338 
5339 instruct loadConP(rRegP dst, immP con) %{
5340   match(Set dst con);
5341 
5342   format %{ "movq    $dst, $con\t# ptr" %}
5343   ins_encode(load_immP(dst, con));
5344   ins_pipe(ialu_reg_fat); // XXX
5345 %}
5346 
5347 instruct loadConP0(rRegP dst, immP0 src, rFlagsReg cr)
5348 %{
5349   match(Set dst src);
5350   effect(KILL cr);
5351 
5352   ins_cost(50);
5353   format %{ "xorl    $dst, $dst\t# ptr" %}
5354   opcode(0x33); /* + rd */
5355   ins_encode(REX_reg_reg(dst, dst), OpcP, reg_reg(dst, dst));
5356   ins_pipe(ialu_reg);
5357 %}
5358 
5359 instruct loadConP31(rRegP dst, immP31 src, rFlagsReg cr)
5360 %{
5361   match(Set dst src);
5362   effect(KILL cr);
5363 
5364   ins_cost(60);
5365   format %{ "movl    $dst, $src\t# ptr (positive 32-bit)" %}
5366   ins_encode(load_immP31(dst, src));
5367   ins_pipe(ialu_reg);
5368 %}
5369 
5370 instruct loadConF(regF dst, immF con) %{
5371   match(Set dst con);
5372   ins_cost(125);
5373   format %{ "movss   $dst, [$constantaddress]\t# load from constant table: float=$con" %}
5374   ins_encode %{
5375     __ movflt($dst$$XMMRegister, $constantaddress($con));
5376   %}
5377   ins_pipe(pipe_slow);
5378 %}
5379 
5380 instruct loadConN0(rRegN dst, immN0 src, rFlagsReg cr) %{
5381   match(Set dst src);
5382   effect(KILL cr);
5383   format %{ "xorq    $dst, $src\t# compressed NULL ptr" %}
5384   ins_encode %{
5385     __ xorq($dst$$Register, $dst$$Register);
5386   %}
5387   ins_pipe(ialu_reg);
5388 %}
5389 
5390 instruct loadConN(rRegN dst, immN src) %{
5391   match(Set dst src);
5392 
5393   ins_cost(125);
5394   format %{ "movl    $dst, $src\t# compressed ptr" %}
5395   ins_encode %{
5396     address con = (address)$src$$constant;
5397     if (con == NULL) {
5398       ShouldNotReachHere();
5399     } else {
5400       __ set_narrow_oop($dst$$Register, (jobject)$src$$constant);
5401     }
5402   %}
5403   ins_pipe(ialu_reg_fat); // XXX
5404 %}
5405 
5406 instruct loadConNKlass(rRegN dst, immNKlass src) %{
5407   match(Set dst src);
5408 
5409   ins_cost(125);
5410   format %{ "movl    $dst, $src\t# compressed klass ptr" %}
5411   ins_encode %{
5412     address con = (address)$src$$constant;
5413     if (con == NULL) {
5414       ShouldNotReachHere();
5415     } else {
5416       __ set_narrow_klass($dst$$Register, (Klass*)$src$$constant);
5417     }
5418   %}
5419   ins_pipe(ialu_reg_fat); // XXX
5420 %}
5421 
5422 instruct loadConF0(regF dst, immF0 src)
5423 %{
5424   match(Set dst src);
5425   ins_cost(100);
5426 
5427   format %{ "xorps   $dst, $dst\t# float 0.0" %}
5428   ins_encode %{
5429     __ xorps($dst$$XMMRegister, $dst$$XMMRegister);
5430   %}
5431   ins_pipe(pipe_slow);
5432 %}
5433 
5434 // Use the same format since predicate() can not be used here.
5435 instruct loadConD(regD dst, immD con) %{
5436   match(Set dst con);
5437   ins_cost(125);
5438   format %{ "movsd   $dst, [$constantaddress]\t# load from constant table: double=$con" %}
5439   ins_encode %{
5440     __ movdbl($dst$$XMMRegister, $constantaddress($con));
5441   %}
5442   ins_pipe(pipe_slow);
5443 %}
5444 
5445 instruct loadConD0(regD dst, immD0 src)
5446 %{
5447   match(Set dst src);
5448   ins_cost(100);
5449 
5450   format %{ "xorpd   $dst, $dst\t# double 0.0" %}
5451   ins_encode %{
5452     __ xorpd ($dst$$XMMRegister, $dst$$XMMRegister);
5453   %}
5454   ins_pipe(pipe_slow);
5455 %}
5456 
5457 instruct loadSSI(rRegI dst, stackSlotI src)
5458 %{
5459   match(Set dst src);
5460 
5461   ins_cost(125);
5462   format %{ "movl    $dst, $src\t# int stk" %}
5463   opcode(0x8B);
5464   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
5465   ins_pipe(ialu_reg_mem);
5466 %}
5467 
5468 instruct loadSSL(rRegL dst, stackSlotL src)
5469 %{
5470   match(Set dst src);
5471 
5472   ins_cost(125);
5473   format %{ "movq    $dst, $src\t# long stk" %}
5474   opcode(0x8B);
5475   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
5476   ins_pipe(ialu_reg_mem);
5477 %}
5478 
5479 instruct loadSSP(rRegP dst, stackSlotP src)
5480 %{
5481   match(Set dst src);
5482 
5483   ins_cost(125);
5484   format %{ "movq    $dst, $src\t# ptr stk" %}
5485   opcode(0x8B);
5486   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
5487   ins_pipe(ialu_reg_mem);
5488 %}
5489 
5490 instruct loadSSF(regF dst, stackSlotF src)
5491 %{
5492   match(Set dst src);
5493 
5494   ins_cost(125);
5495   format %{ "movss   $dst, $src\t# float stk" %}
5496   ins_encode %{
5497     __ movflt($dst$$XMMRegister, Address(rsp, $src$$disp));
5498   %}
5499   ins_pipe(pipe_slow); // XXX
5500 %}
5501 
5502 // Use the same format since predicate() can not be used here.
5503 instruct loadSSD(regD dst, stackSlotD src)
5504 %{
5505   match(Set dst src);
5506 
5507   ins_cost(125);
5508   format %{ "movsd   $dst, $src\t# double stk" %}
5509   ins_encode  %{
5510     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
5511   %}
5512   ins_pipe(pipe_slow); // XXX
5513 %}
5514 
5515 // Prefetch instructions.
5516 // Must be safe to execute with invalid address (cannot fault).
5517 
5518 instruct prefetchr( memory mem ) %{
5519   predicate(ReadPrefetchInstr==3);
5520   match(PrefetchRead mem);
5521   ins_cost(125);
5522 
5523   format %{ "PREFETCHR $mem\t# Prefetch into level 1 cache" %}
5524   ins_encode %{
5525     __ prefetchr($mem$$Address);
5526   %}
5527   ins_pipe(ialu_mem);
5528 %}
5529 
5530 instruct prefetchrNTA( memory mem ) %{
5531   predicate(ReadPrefetchInstr==0);
5532   match(PrefetchRead mem);
5533   ins_cost(125);
5534 
5535   format %{ "PREFETCHNTA $mem\t# Prefetch into non-temporal cache for read" %}
5536   ins_encode %{
5537     __ prefetchnta($mem$$Address);
5538   %}
5539   ins_pipe(ialu_mem);
5540 %}
5541 
5542 instruct prefetchrT0( memory mem ) %{
5543   predicate(ReadPrefetchInstr==1);
5544   match(PrefetchRead mem);
5545   ins_cost(125);
5546 
5547   format %{ "PREFETCHT0 $mem\t# prefetch into L1 and L2 caches for read" %}
5548   ins_encode %{
5549     __ prefetcht0($mem$$Address);
5550   %}
5551   ins_pipe(ialu_mem);
5552 %}
5553 
5554 instruct prefetchrT2( memory mem ) %{
5555   predicate(ReadPrefetchInstr==2);
5556   match(PrefetchRead mem);
5557   ins_cost(125);
5558 
5559   format %{ "PREFETCHT2 $mem\t# prefetch into L2 caches for read" %}
5560   ins_encode %{
5561     __ prefetcht2($mem$$Address);
5562   %}
5563   ins_pipe(ialu_mem);
5564 %}
5565 
5566 instruct prefetchwNTA( memory mem ) %{
5567   match(PrefetchWrite mem);
5568   ins_cost(125);
5569 
5570   format %{ "PREFETCHNTA $mem\t# Prefetch to non-temporal cache for write" %}
5571   ins_encode %{
5572     __ prefetchnta($mem$$Address);
5573   %}
5574   ins_pipe(ialu_mem);
5575 %}
5576 
5577 // Prefetch instructions for allocation.
5578 
5579 instruct prefetchAlloc( memory mem ) %{
5580   predicate(AllocatePrefetchInstr==3);
5581   match(PrefetchAllocation mem);
5582   ins_cost(125);
5583 
5584   format %{ "PREFETCHW $mem\t# Prefetch allocation into level 1 cache and mark modified" %}
5585   ins_encode %{
5586     __ prefetchw($mem$$Address);
5587   %}
5588   ins_pipe(ialu_mem);
5589 %}
5590 
5591 instruct prefetchAllocNTA( memory mem ) %{
5592   predicate(AllocatePrefetchInstr==0);
5593   match(PrefetchAllocation mem);
5594   ins_cost(125);
5595 
5596   format %{ "PREFETCHNTA $mem\t# Prefetch allocation to non-temporal cache for write" %}
5597   ins_encode %{
5598     __ prefetchnta($mem$$Address);
5599   %}
5600   ins_pipe(ialu_mem);
5601 %}
5602 
5603 instruct prefetchAllocT0( memory mem ) %{
5604   predicate(AllocatePrefetchInstr==1);
5605   match(PrefetchAllocation mem);
5606   ins_cost(125);
5607 
5608   format %{ "PREFETCHT0 $mem\t# Prefetch allocation to level 1 and 2 caches for write" %}
5609   ins_encode %{
5610     __ prefetcht0($mem$$Address);
5611   %}
5612   ins_pipe(ialu_mem);
5613 %}
5614 
5615 instruct prefetchAllocT2( memory mem ) %{
5616   predicate(AllocatePrefetchInstr==2);
5617   match(PrefetchAllocation mem);
5618   ins_cost(125);
5619 
5620   format %{ "PREFETCHT2 $mem\t# Prefetch allocation to level 2 cache for write" %}
5621   ins_encode %{
5622     __ prefetcht2($mem$$Address);
5623   %}
5624   ins_pipe(ialu_mem);
5625 %}
5626 
5627 //----------Store Instructions-------------------------------------------------
5628 
5629 // Store Byte
5630 instruct storeB(memory mem, rRegI src)
5631 %{
5632   match(Set mem (StoreB mem src));
5633 
5634   ins_cost(125); // XXX
5635   format %{ "movb    $mem, $src\t# byte" %}
5636   opcode(0x88);
5637   ins_encode(REX_breg_mem(src, mem), OpcP, reg_mem(src, mem));
5638   ins_pipe(ialu_mem_reg);
5639 %}
5640 
5641 // Store Char/Short
5642 instruct storeC(memory mem, rRegI src)
5643 %{
5644   match(Set mem (StoreC mem src));
5645 
5646   ins_cost(125); // XXX
5647   format %{ "movw    $mem, $src\t# char/short" %}
5648   opcode(0x89);
5649   ins_encode(SizePrefix, REX_reg_mem(src, mem), OpcP, reg_mem(src, mem));
5650   ins_pipe(ialu_mem_reg);
5651 %}
5652 
5653 // Store Integer
5654 instruct storeI(memory mem, rRegI src)
5655 %{
5656   match(Set mem (StoreI mem src));
5657 
5658   ins_cost(125); // XXX
5659   format %{ "movl    $mem, $src\t# int" %}
5660   opcode(0x89);
5661   ins_encode(REX_reg_mem(src, mem), OpcP, reg_mem(src, mem));
5662   ins_pipe(ialu_mem_reg);
5663 %}
5664 
5665 // Store Long
5666 instruct storeL(memory mem, rRegL src)
5667 %{
5668   match(Set mem (StoreL mem src));
5669 
5670   ins_cost(125); // XXX
5671   format %{ "movq    $mem, $src\t# long" %}
5672   opcode(0x89);
5673   ins_encode(REX_reg_mem_wide(src, mem), OpcP, reg_mem(src, mem));
5674   ins_pipe(ialu_mem_reg); // XXX
5675 %}
5676 
5677 // Store Pointer
5678 instruct storeP(memory mem, any_RegP src)
5679 %{
5680   match(Set mem (StoreP mem src));
5681 
5682   ins_cost(125); // XXX
5683   format %{ "movq    $mem, $src\t# ptr" %}
5684   opcode(0x89);
5685   ins_encode(REX_reg_mem_wide(src, mem), OpcP, reg_mem(src, mem));
5686   ins_pipe(ialu_mem_reg);
5687 %}
5688 
5689 instruct storeImmP0(memory mem, immP0 zero)
5690 %{
5691   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5692   match(Set mem (StoreP mem zero));
5693 
5694   ins_cost(125); // XXX
5695   format %{ "movq    $mem, R12\t# ptr (R12_heapbase==0)" %}
5696   ins_encode %{
5697     __ movq($mem$$Address, r12);
5698   %}
5699   ins_pipe(ialu_mem_reg);
5700 %}
5701 
5702 // Store NULL Pointer, mark word, or other simple pointer constant.
5703 instruct storeImmP(memory mem, immP31 src)
5704 %{
5705   match(Set mem (StoreP mem src));
5706 
5707   ins_cost(150); // XXX
5708   format %{ "movq    $mem, $src\t# ptr" %}
5709   opcode(0xC7); /* C7 /0 */
5710   ins_encode(REX_mem_wide(mem), OpcP, RM_opc_mem(0x00, mem), Con32(src));
5711   ins_pipe(ialu_mem_imm);
5712 %}
5713 
5714 // Store Compressed Pointer
5715 instruct storeN(memory mem, rRegN src)
5716 %{
5717   match(Set mem (StoreN mem src));
5718 
5719   ins_cost(125); // XXX
5720   format %{ "movl    $mem, $src\t# compressed ptr" %}
5721   ins_encode %{
5722     __ movl($mem$$Address, $src$$Register);
5723   %}
5724   ins_pipe(ialu_mem_reg);
5725 %}
5726 
5727 instruct storeNKlass(memory mem, rRegN src)
5728 %{
5729   match(Set mem (StoreNKlass mem src));
5730 
5731   ins_cost(125); // XXX
5732   format %{ "movl    $mem, $src\t# compressed klass ptr" %}
5733   ins_encode %{
5734     __ movl($mem$$Address, $src$$Register);
5735   %}
5736   ins_pipe(ialu_mem_reg);
5737 %}
5738 
5739 instruct storeImmN0(memory mem, immN0 zero)
5740 %{
5741   predicate(Universe::narrow_oop_base() == NULL && Universe::narrow_klass_base() == NULL);
5742   match(Set mem (StoreN mem zero));
5743 
5744   ins_cost(125); // XXX
5745   format %{ "movl    $mem, R12\t# compressed ptr (R12_heapbase==0)" %}
5746   ins_encode %{
5747     __ movl($mem$$Address, r12);
5748   %}
5749   ins_pipe(ialu_mem_reg);
5750 %}
5751 
5752 instruct storeImmN(memory mem, immN src)
5753 %{
5754   match(Set mem (StoreN mem src));
5755 
5756   ins_cost(150); // XXX
5757   format %{ "movl    $mem, $src\t# compressed ptr" %}
5758   ins_encode %{
5759     address con = (address)$src$$constant;
5760     if (con == NULL) {
5761       __ movl($mem$$Address, (int32_t)0);
5762     } else {
5763       __ set_narrow_oop($mem$$Address, (jobject)$src$$constant);
5764     }
5765   %}
5766   ins_pipe(ialu_mem_imm);
5767 %}
5768 
5769 instruct storeImmNKlass(memory mem, immNKlass src)
5770 %{
5771   match(Set mem (StoreNKlass mem src));
5772 
5773   ins_cost(150); // XXX
5774   format %{ "movl    $mem, $src\t# compressed klass ptr" %}
5775   ins_encode %{
5776     __ set_narrow_klass($mem$$Address, (Klass*)$src$$constant);
5777   %}
5778   ins_pipe(ialu_mem_imm);
5779 %}
5780 
5781 // Store Integer Immediate
5782 instruct storeImmI0(memory mem, immI0 zero)
5783 %{
5784   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5785   match(Set mem (StoreI mem zero));
5786 
5787   ins_cost(125); // XXX
5788   format %{ "movl    $mem, R12\t# int (R12_heapbase==0)" %}
5789   ins_encode %{
5790     __ movl($mem$$Address, r12);
5791   %}
5792   ins_pipe(ialu_mem_reg);
5793 %}
5794 
5795 instruct storeImmI(memory mem, immI src)
5796 %{
5797   match(Set mem (StoreI mem src));
5798 
5799   ins_cost(150);
5800   format %{ "movl    $mem, $src\t# int" %}
5801   opcode(0xC7); /* C7 /0 */
5802   ins_encode(REX_mem(mem), OpcP, RM_opc_mem(0x00, mem), Con32(src));
5803   ins_pipe(ialu_mem_imm);
5804 %}
5805 
5806 // Store Long Immediate
5807 instruct storeImmL0(memory mem, immL0 zero)
5808 %{
5809   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5810   match(Set mem (StoreL mem zero));
5811 
5812   ins_cost(125); // XXX
5813   format %{ "movq    $mem, R12\t# long (R12_heapbase==0)" %}
5814   ins_encode %{
5815     __ movq($mem$$Address, r12);
5816   %}
5817   ins_pipe(ialu_mem_reg);
5818 %}
5819 
5820 instruct storeImmL(memory mem, immL32 src)
5821 %{
5822   match(Set mem (StoreL mem src));
5823 
5824   ins_cost(150);
5825   format %{ "movq    $mem, $src\t# long" %}
5826   opcode(0xC7); /* C7 /0 */
5827   ins_encode(REX_mem_wide(mem), OpcP, RM_opc_mem(0x00, mem), Con32(src));
5828   ins_pipe(ialu_mem_imm);
5829 %}
5830 
5831 // Store Short/Char Immediate
5832 instruct storeImmC0(memory mem, immI0 zero)
5833 %{
5834   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5835   match(Set mem (StoreC mem zero));
5836 
5837   ins_cost(125); // XXX
5838   format %{ "movw    $mem, R12\t# short/char (R12_heapbase==0)" %}
5839   ins_encode %{
5840     __ movw($mem$$Address, r12);
5841   %}
5842   ins_pipe(ialu_mem_reg);
5843 %}
5844 
5845 instruct storeImmI16(memory mem, immI16 src)
5846 %{
5847   predicate(UseStoreImmI16);
5848   match(Set mem (StoreC mem src));
5849 
5850   ins_cost(150);
5851   format %{ "movw    $mem, $src\t# short/char" %}
5852   opcode(0xC7); /* C7 /0 Same as 32 store immediate with prefix */
5853   ins_encode(SizePrefix, REX_mem(mem), OpcP, RM_opc_mem(0x00, mem),Con16(src));
5854   ins_pipe(ialu_mem_imm);
5855 %}
5856 
5857 // Store Byte Immediate
5858 instruct storeImmB0(memory mem, immI0 zero)
5859 %{
5860   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5861   match(Set mem (StoreB mem zero));
5862 
5863   ins_cost(125); // XXX
5864   format %{ "movb    $mem, R12\t# short/char (R12_heapbase==0)" %}
5865   ins_encode %{
5866     __ movb($mem$$Address, r12);
5867   %}
5868   ins_pipe(ialu_mem_reg);
5869 %}
5870 
5871 instruct storeImmB(memory mem, immI8 src)
5872 %{
5873   match(Set mem (StoreB mem src));
5874 
5875   ins_cost(150); // XXX
5876   format %{ "movb    $mem, $src\t# byte" %}
5877   opcode(0xC6); /* C6 /0 */
5878   ins_encode(REX_mem(mem), OpcP, RM_opc_mem(0x00, mem), Con8or32(src));
5879   ins_pipe(ialu_mem_imm);
5880 %}
5881 
5882 // Store CMS card-mark Immediate
5883 instruct storeImmCM0_reg(memory mem, immI0 zero)
5884 %{
5885   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5886   match(Set mem (StoreCM mem zero));
5887 
5888   ins_cost(125); // XXX
5889   format %{ "movb    $mem, R12\t# CMS card-mark byte 0 (R12_heapbase==0)" %}
5890   ins_encode %{
5891     __ movb($mem$$Address, r12);
5892   %}
5893   ins_pipe(ialu_mem_reg);
5894 %}
5895 
5896 instruct storeImmCM0(memory mem, immI0 src)
5897 %{
5898   match(Set mem (StoreCM mem src));
5899 
5900   ins_cost(150); // XXX
5901   format %{ "movb    $mem, $src\t# CMS card-mark byte 0" %}
5902   opcode(0xC6); /* C6 /0 */
5903   ins_encode(REX_mem(mem), OpcP, RM_opc_mem(0x00, mem), Con8or32(src));
5904   ins_pipe(ialu_mem_imm);
5905 %}
5906 
5907 // Store Float
5908 instruct storeF(memory mem, regF src)
5909 %{
5910   match(Set mem (StoreF mem src));
5911 
5912   ins_cost(95); // XXX
5913   format %{ "movss   $mem, $src\t# float" %}
5914   ins_encode %{
5915     __ movflt($mem$$Address, $src$$XMMRegister);
5916   %}
5917   ins_pipe(pipe_slow); // XXX
5918 %}
5919 
5920 // Store immediate Float value (it is faster than store from XMM register)
5921 instruct storeF0(memory mem, immF0 zero)
5922 %{
5923   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5924   match(Set mem (StoreF mem zero));
5925 
5926   ins_cost(25); // XXX
5927   format %{ "movl    $mem, R12\t# float 0. (R12_heapbase==0)" %}
5928   ins_encode %{
5929     __ movl($mem$$Address, r12);
5930   %}
5931   ins_pipe(ialu_mem_reg);
5932 %}
5933 
5934 instruct storeF_imm(memory mem, immF src)
5935 %{
5936   match(Set mem (StoreF mem src));
5937 
5938   ins_cost(50);
5939   format %{ "movl    $mem, $src\t# float" %}
5940   opcode(0xC7); /* C7 /0 */
5941   ins_encode(REX_mem(mem), OpcP, RM_opc_mem(0x00, mem), Con32F_as_bits(src));
5942   ins_pipe(ialu_mem_imm);
5943 %}
5944 
5945 // Store Double
5946 instruct storeD(memory mem, regD src)
5947 %{
5948   match(Set mem (StoreD mem src));
5949 
5950   ins_cost(95); // XXX
5951   format %{ "movsd   $mem, $src\t# double" %}
5952   ins_encode %{
5953     __ movdbl($mem$$Address, $src$$XMMRegister);
5954   %}
5955   ins_pipe(pipe_slow); // XXX
5956 %}
5957 
5958 // Store immediate double 0.0 (it is faster than store from XMM register)
5959 instruct storeD0_imm(memory mem, immD0 src)
5960 %{
5961   predicate(!UseCompressedOops || (Universe::narrow_oop_base() != NULL));
5962   match(Set mem (StoreD mem src));
5963 
5964   ins_cost(50);
5965   format %{ "movq    $mem, $src\t# double 0." %}
5966   opcode(0xC7); /* C7 /0 */
5967   ins_encode(REX_mem_wide(mem), OpcP, RM_opc_mem(0x00, mem), Con32F_as_bits(src));
5968   ins_pipe(ialu_mem_imm);
5969 %}
5970 
5971 instruct storeD0(memory mem, immD0 zero)
5972 %{
5973   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5974   match(Set mem (StoreD mem zero));
5975 
5976   ins_cost(25); // XXX
5977   format %{ "movq    $mem, R12\t# double 0. (R12_heapbase==0)" %}
5978   ins_encode %{
5979     __ movq($mem$$Address, r12);
5980   %}
5981   ins_pipe(ialu_mem_reg);
5982 %}
5983 
5984 instruct storeSSI(stackSlotI dst, rRegI src)
5985 %{
5986   match(Set dst src);
5987 
5988   ins_cost(100);
5989   format %{ "movl    $dst, $src\t# int stk" %}
5990   opcode(0x89);
5991   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
5992   ins_pipe( ialu_mem_reg );
5993 %}
5994 
5995 instruct storeSSL(stackSlotL dst, rRegL src)
5996 %{
5997   match(Set dst src);
5998 
5999   ins_cost(100);
6000   format %{ "movq    $dst, $src\t# long stk" %}
6001   opcode(0x89);
6002   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
6003   ins_pipe(ialu_mem_reg);
6004 %}
6005 
6006 instruct storeSSP(stackSlotP dst, rRegP src)
6007 %{
6008   match(Set dst src);
6009 
6010   ins_cost(100);
6011   format %{ "movq    $dst, $src\t# ptr stk" %}
6012   opcode(0x89);
6013   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
6014   ins_pipe(ialu_mem_reg);
6015 %}
6016 
6017 instruct storeSSF(stackSlotF dst, regF src)
6018 %{
6019   match(Set dst src);
6020 
6021   ins_cost(95); // XXX
6022   format %{ "movss   $dst, $src\t# float stk" %}
6023   ins_encode %{
6024     __ movflt(Address(rsp, $dst$$disp), $src$$XMMRegister);
6025   %}
6026   ins_pipe(pipe_slow); // XXX
6027 %}
6028 
6029 instruct storeSSD(stackSlotD dst, regD src)
6030 %{
6031   match(Set dst src);
6032 
6033   ins_cost(95); // XXX
6034   format %{ "movsd   $dst, $src\t# double stk" %}
6035   ins_encode %{
6036     __ movdbl(Address(rsp, $dst$$disp), $src$$XMMRegister);
6037   %}
6038   ins_pipe(pipe_slow); // XXX
6039 %}
6040 
6041 //----------BSWAP Instructions-------------------------------------------------
6042 instruct bytes_reverse_int(rRegI dst) %{
6043   match(Set dst (ReverseBytesI dst));
6044 
6045   format %{ "bswapl  $dst" %}
6046   opcode(0x0F, 0xC8);  /*Opcode 0F /C8 */
6047   ins_encode( REX_reg(dst), OpcP, opc2_reg(dst) );
6048   ins_pipe( ialu_reg );
6049 %}
6050 
6051 instruct bytes_reverse_long(rRegL dst) %{
6052   match(Set dst (ReverseBytesL dst));
6053 
6054   format %{ "bswapq  $dst" %}
6055   opcode(0x0F, 0xC8); /* Opcode 0F /C8 */
6056   ins_encode( REX_reg_wide(dst), OpcP, opc2_reg(dst) );
6057   ins_pipe( ialu_reg);
6058 %}
6059 
6060 instruct bytes_reverse_unsigned_short(rRegI dst, rFlagsReg cr) %{
6061   match(Set dst (ReverseBytesUS dst));
6062   effect(KILL cr);
6063 
6064   format %{ "bswapl  $dst\n\t"
6065             "shrl    $dst,16\n\t" %}
6066   ins_encode %{
6067     __ bswapl($dst$$Register);
6068     __ shrl($dst$$Register, 16);
6069   %}
6070   ins_pipe( ialu_reg );
6071 %}
6072 
6073 instruct bytes_reverse_short(rRegI dst, rFlagsReg cr) %{
6074   match(Set dst (ReverseBytesS dst));
6075   effect(KILL cr);
6076 
6077   format %{ "bswapl  $dst\n\t"
6078             "sar     $dst,16\n\t" %}
6079   ins_encode %{
6080     __ bswapl($dst$$Register);
6081     __ sarl($dst$$Register, 16);
6082   %}
6083   ins_pipe( ialu_reg );
6084 %}
6085 
6086 //---------- Zeros Count Instructions ------------------------------------------
6087 
6088 instruct countLeadingZerosI(rRegI dst, rRegI src, rFlagsReg cr) %{
6089   predicate(UseCountLeadingZerosInstruction);
6090   match(Set dst (CountLeadingZerosI src));
6091   effect(KILL cr);
6092 
6093   format %{ "lzcntl  $dst, $src\t# count leading zeros (int)" %}
6094   ins_encode %{
6095     __ lzcntl($dst$$Register, $src$$Register);
6096   %}
6097   ins_pipe(ialu_reg);
6098 %}
6099 
6100 instruct countLeadingZerosI_bsr(rRegI dst, rRegI src, rFlagsReg cr) %{
6101   predicate(!UseCountLeadingZerosInstruction);
6102   match(Set dst (CountLeadingZerosI src));
6103   effect(KILL cr);
6104 
6105   format %{ "bsrl    $dst, $src\t# count leading zeros (int)\n\t"
6106             "jnz     skip\n\t"
6107             "movl    $dst, -1\n"
6108       "skip:\n\t"
6109             "negl    $dst\n\t"
6110             "addl    $dst, 31" %}
6111   ins_encode %{
6112     Register Rdst = $dst$$Register;
6113     Register Rsrc = $src$$Register;
6114     Label skip;
6115     __ bsrl(Rdst, Rsrc);
6116     __ jccb(Assembler::notZero, skip);
6117     __ movl(Rdst, -1);
6118     __ bind(skip);
6119     __ negl(Rdst);
6120     __ addl(Rdst, BitsPerInt - 1);
6121   %}
6122   ins_pipe(ialu_reg);
6123 %}
6124 
6125 instruct countLeadingZerosL(rRegI dst, rRegL src, rFlagsReg cr) %{
6126   predicate(UseCountLeadingZerosInstruction);
6127   match(Set dst (CountLeadingZerosL src));
6128   effect(KILL cr);
6129 
6130   format %{ "lzcntq  $dst, $src\t# count leading zeros (long)" %}
6131   ins_encode %{
6132     __ lzcntq($dst$$Register, $src$$Register);
6133   %}
6134   ins_pipe(ialu_reg);
6135 %}
6136 
6137 instruct countLeadingZerosL_bsr(rRegI dst, rRegL src, rFlagsReg cr) %{
6138   predicate(!UseCountLeadingZerosInstruction);
6139   match(Set dst (CountLeadingZerosL src));
6140   effect(KILL cr);
6141 
6142   format %{ "bsrq    $dst, $src\t# count leading zeros (long)\n\t"
6143             "jnz     skip\n\t"
6144             "movl    $dst, -1\n"
6145       "skip:\n\t"
6146             "negl    $dst\n\t"
6147             "addl    $dst, 63" %}
6148   ins_encode %{
6149     Register Rdst = $dst$$Register;
6150     Register Rsrc = $src$$Register;
6151     Label skip;
6152     __ bsrq(Rdst, Rsrc);
6153     __ jccb(Assembler::notZero, skip);
6154     __ movl(Rdst, -1);
6155     __ bind(skip);
6156     __ negl(Rdst);
6157     __ addl(Rdst, BitsPerLong - 1);
6158   %}
6159   ins_pipe(ialu_reg);
6160 %}
6161 
6162 instruct countTrailingZerosI(rRegI dst, rRegI src, rFlagsReg cr) %{
6163   predicate(UseCountTrailingZerosInstruction);
6164   match(Set dst (CountTrailingZerosI src));
6165   effect(KILL cr);
6166 
6167   format %{ "tzcntl    $dst, $src\t# count trailing zeros (int)" %}
6168   ins_encode %{
6169     __ tzcntl($dst$$Register, $src$$Register);
6170   %}
6171   ins_pipe(ialu_reg);
6172 %}
6173 
6174 instruct countTrailingZerosI_bsf(rRegI dst, rRegI src, rFlagsReg cr) %{
6175   predicate(!UseCountTrailingZerosInstruction);
6176   match(Set dst (CountTrailingZerosI src));
6177   effect(KILL cr);
6178 
6179   format %{ "bsfl    $dst, $src\t# count trailing zeros (int)\n\t"
6180             "jnz     done\n\t"
6181             "movl    $dst, 32\n"
6182       "done:" %}
6183   ins_encode %{
6184     Register Rdst = $dst$$Register;
6185     Label done;
6186     __ bsfl(Rdst, $src$$Register);
6187     __ jccb(Assembler::notZero, done);
6188     __ movl(Rdst, BitsPerInt);
6189     __ bind(done);
6190   %}
6191   ins_pipe(ialu_reg);
6192 %}
6193 
6194 instruct countTrailingZerosL(rRegI dst, rRegL src, rFlagsReg cr) %{
6195   predicate(UseCountTrailingZerosInstruction);
6196   match(Set dst (CountTrailingZerosL src));
6197   effect(KILL cr);
6198 
6199   format %{ "tzcntq    $dst, $src\t# count trailing zeros (long)" %}
6200   ins_encode %{
6201     __ tzcntq($dst$$Register, $src$$Register);
6202   %}
6203   ins_pipe(ialu_reg);
6204 %}
6205 
6206 instruct countTrailingZerosL_bsf(rRegI dst, rRegL src, rFlagsReg cr) %{
6207   predicate(!UseCountTrailingZerosInstruction);
6208   match(Set dst (CountTrailingZerosL src));
6209   effect(KILL cr);
6210 
6211   format %{ "bsfq    $dst, $src\t# count trailing zeros (long)\n\t"
6212             "jnz     done\n\t"
6213             "movl    $dst, 64\n"
6214       "done:" %}
6215   ins_encode %{
6216     Register Rdst = $dst$$Register;
6217     Label done;
6218     __ bsfq(Rdst, $src$$Register);
6219     __ jccb(Assembler::notZero, done);
6220     __ movl(Rdst, BitsPerLong);
6221     __ bind(done);
6222   %}
6223   ins_pipe(ialu_reg);
6224 %}
6225 
6226 
6227 //---------- Population Count Instructions -------------------------------------
6228 
6229 instruct popCountI(rRegI dst, rRegI src, rFlagsReg cr) %{
6230   predicate(UsePopCountInstruction);
6231   match(Set dst (PopCountI src));
6232   effect(KILL cr);
6233 
6234   format %{ "popcnt  $dst, $src" %}
6235   ins_encode %{
6236     __ popcntl($dst$$Register, $src$$Register);
6237   %}
6238   ins_pipe(ialu_reg);
6239 %}
6240 
6241 instruct popCountI_mem(rRegI dst, memory mem, rFlagsReg cr) %{
6242   predicate(UsePopCountInstruction);
6243   match(Set dst (PopCountI (LoadI mem)));
6244   effect(KILL cr);
6245 
6246   format %{ "popcnt  $dst, $mem" %}
6247   ins_encode %{
6248     __ popcntl($dst$$Register, $mem$$Address);
6249   %}
6250   ins_pipe(ialu_reg);
6251 %}
6252 
6253 // Note: Long.bitCount(long) returns an int.
6254 instruct popCountL(rRegI dst, rRegL src, rFlagsReg cr) %{
6255   predicate(UsePopCountInstruction);
6256   match(Set dst (PopCountL src));
6257   effect(KILL cr);
6258 
6259   format %{ "popcnt  $dst, $src" %}
6260   ins_encode %{
6261     __ popcntq($dst$$Register, $src$$Register);
6262   %}
6263   ins_pipe(ialu_reg);
6264 %}
6265 
6266 // Note: Long.bitCount(long) returns an int.
6267 instruct popCountL_mem(rRegI dst, memory mem, rFlagsReg cr) %{
6268   predicate(UsePopCountInstruction);
6269   match(Set dst (PopCountL (LoadL mem)));
6270   effect(KILL cr);
6271 
6272   format %{ "popcnt  $dst, $mem" %}
6273   ins_encode %{
6274     __ popcntq($dst$$Register, $mem$$Address);
6275   %}
6276   ins_pipe(ialu_reg);
6277 %}
6278 
6279 
6280 //----------MemBar Instructions-----------------------------------------------
6281 // Memory barrier flavors
6282 
6283 instruct membar_acquire()
6284 %{
6285   match(MemBarAcquire);
6286   match(LoadFence);
6287   ins_cost(0);
6288 
6289   size(0);
6290   format %{ "MEMBAR-acquire ! (empty encoding)" %}
6291   ins_encode();
6292   ins_pipe(empty);
6293 %}
6294 
6295 instruct membar_acquire_lock()
6296 %{
6297   match(MemBarAcquireLock);
6298   ins_cost(0);
6299 
6300   size(0);
6301   format %{ "MEMBAR-acquire (prior CMPXCHG in FastLock so empty encoding)" %}
6302   ins_encode();
6303   ins_pipe(empty);
6304 %}
6305 
6306 instruct membar_release()
6307 %{
6308   match(MemBarRelease);
6309   match(StoreFence);
6310   ins_cost(0);
6311 
6312   size(0);
6313   format %{ "MEMBAR-release ! (empty encoding)" %}
6314   ins_encode();
6315   ins_pipe(empty);
6316 %}
6317 
6318 instruct membar_release_lock()
6319 %{
6320   match(MemBarReleaseLock);
6321   ins_cost(0);
6322 
6323   size(0);
6324   format %{ "MEMBAR-release (a FastUnlock follows so empty encoding)" %}
6325   ins_encode();
6326   ins_pipe(empty);
6327 %}
6328 
6329 instruct membar_volatile(rFlagsReg cr) %{
6330   match(MemBarVolatile);
6331   effect(KILL cr);
6332   ins_cost(400);
6333 
6334   format %{
6335     $$template
6336     if (os::is_MP()) {
6337       $$emit$$"lock addl [rsp + #0], 0\t! membar_volatile"
6338     } else {
6339       $$emit$$"MEMBAR-volatile ! (empty encoding)"
6340     }
6341   %}
6342   ins_encode %{
6343     __ membar(Assembler::StoreLoad);
6344   %}
6345   ins_pipe(pipe_slow);
6346 %}
6347 
6348 instruct unnecessary_membar_volatile()
6349 %{
6350   match(MemBarVolatile);
6351   predicate(Matcher::post_store_load_barrier(n));
6352   ins_cost(0);
6353 
6354   size(0);
6355   format %{ "MEMBAR-volatile (unnecessary so empty encoding)" %}
6356   ins_encode();
6357   ins_pipe(empty);
6358 %}
6359 
6360 instruct membar_storestore() %{
6361   match(MemBarStoreStore);
6362   ins_cost(0);
6363 
6364   size(0);
6365   format %{ "MEMBAR-storestore (empty encoding)" %}
6366   ins_encode( );
6367   ins_pipe(empty);
6368 %}
6369 
6370 //----------Move Instructions--------------------------------------------------
6371 
6372 instruct castX2P(rRegP dst, rRegL src)
6373 %{
6374   match(Set dst (CastX2P src));
6375 
6376   format %{ "movq    $dst, $src\t# long->ptr" %}
6377   ins_encode %{
6378     if ($dst$$reg != $src$$reg) {
6379       __ movptr($dst$$Register, $src$$Register);
6380     }
6381   %}
6382   ins_pipe(ialu_reg_reg); // XXX
6383 %}
6384 
6385 instruct castP2X(rRegL dst, rRegP src)
6386 %{
6387   match(Set dst (CastP2X src));
6388 
6389   format %{ "movq    $dst, $src\t# ptr -> long" %}
6390   ins_encode %{
6391     if ($dst$$reg != $src$$reg) {
6392       __ movptr($dst$$Register, $src$$Register);
6393     }
6394   %}
6395   ins_pipe(ialu_reg_reg); // XXX
6396 %}
6397 
6398 // Convert oop into int for vectors alignment masking
6399 instruct convP2I(rRegI dst, rRegP src)
6400 %{
6401   match(Set dst (ConvL2I (CastP2X src)));
6402 
6403   format %{ "movl    $dst, $src\t# ptr -> int" %}
6404   ins_encode %{
6405     __ movl($dst$$Register, $src$$Register);
6406   %}
6407   ins_pipe(ialu_reg_reg); // XXX
6408 %}
6409 
6410 // Convert compressed oop into int for vectors alignment masking
6411 // in case of 32bit oops (heap < 4Gb).
6412 instruct convN2I(rRegI dst, rRegN src)
6413 %{
6414   predicate(Universe::narrow_oop_shift() == 0);
6415   match(Set dst (ConvL2I (CastP2X (DecodeN src))));
6416 
6417   format %{ "movl    $dst, $src\t# compressed ptr -> int" %}
6418   ins_encode %{
6419     __ movl($dst$$Register, $src$$Register);
6420   %}
6421   ins_pipe(ialu_reg_reg); // XXX
6422 %}
6423 
6424 // Convert oop pointer into compressed form
6425 instruct encodeHeapOop(rRegN dst, rRegP src, rFlagsReg cr) %{
6426   predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
6427   match(Set dst (EncodeP src));
6428   effect(KILL cr);
6429   format %{ "encode_heap_oop $dst,$src" %}
6430   ins_encode %{
6431     Register s = $src$$Register;
6432     Register d = $dst$$Register;
6433     if (s != d) {
6434       __ movq(d, s);
6435     }
6436     __ encode_heap_oop(d);
6437   %}
6438   ins_pipe(ialu_reg_long);
6439 %}
6440 
6441 instruct encodeHeapOop_not_null(rRegN dst, rRegP src, rFlagsReg cr) %{
6442   predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
6443   match(Set dst (EncodeP src));
6444   effect(KILL cr);
6445   format %{ "encode_heap_oop_not_null $dst,$src" %}
6446   ins_encode %{
6447     __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
6448   %}
6449   ins_pipe(ialu_reg_long);
6450 %}
6451 
6452 instruct decodeHeapOop(rRegP dst, rRegN src, rFlagsReg cr) %{
6453   predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
6454             n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
6455   match(Set dst (DecodeN src));
6456   effect(KILL cr);
6457   format %{ "decode_heap_oop $dst,$src" %}
6458   ins_encode %{
6459     Register s = $src$$Register;
6460     Register d = $dst$$Register;
6461     if (s != d) {
6462       __ movq(d, s);
6463     }
6464     __ decode_heap_oop(d);
6465   %}
6466   ins_pipe(ialu_reg_long);
6467 %}
6468 
6469 instruct decodeHeapOop_not_null(rRegP dst, rRegN src, rFlagsReg cr) %{
6470   predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
6471             n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
6472   match(Set dst (DecodeN src));
6473   effect(KILL cr);
6474   format %{ "decode_heap_oop_not_null $dst,$src" %}
6475   ins_encode %{
6476     Register s = $src$$Register;
6477     Register d = $dst$$Register;
6478     if (s != d) {
6479       __ decode_heap_oop_not_null(d, s);
6480     } else {
6481       __ decode_heap_oop_not_null(d);
6482     }
6483   %}
6484   ins_pipe(ialu_reg_long);
6485 %}
6486 
6487 instruct encodeKlass_not_null(rRegN dst, rRegP src, rFlagsReg cr) %{
6488   match(Set dst (EncodePKlass src));
6489   effect(KILL cr);
6490   format %{ "encode_klass_not_null $dst,$src" %}
6491   ins_encode %{
6492     __ encode_klass_not_null($dst$$Register, $src$$Register);
6493   %}
6494   ins_pipe(ialu_reg_long);
6495 %}
6496 
6497 instruct decodeKlass_not_null(rRegP dst, rRegN src, rFlagsReg cr) %{
6498   match(Set dst (DecodeNKlass src));
6499   effect(KILL cr);
6500   format %{ "decode_klass_not_null $dst,$src" %}
6501   ins_encode %{
6502     Register s = $src$$Register;
6503     Register d = $dst$$Register;
6504     if (s != d) {
6505       __ decode_klass_not_null(d, s);
6506     } else {
6507       __ decode_klass_not_null(d);
6508     }
6509   %}
6510   ins_pipe(ialu_reg_long);
6511 %}
6512 
6513 
6514 //----------Conditional Move---------------------------------------------------
6515 // Jump
6516 // dummy instruction for generating temp registers
6517 instruct jumpXtnd_offset(rRegL switch_val, immI2 shift, rRegI dest) %{
6518   match(Jump (LShiftL switch_val shift));
6519   ins_cost(350);
6520   predicate(false);
6521   effect(TEMP dest);
6522 
6523   format %{ "leaq    $dest, [$constantaddress]\n\t"
6524             "jmp     [$dest + $switch_val << $shift]\n\t" %}
6525   ins_encode %{
6526     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
6527     // to do that and the compiler is using that register as one it can allocate.
6528     // So we build it all by hand.
6529     // Address index(noreg, switch_reg, (Address::ScaleFactor)$shift$$constant);
6530     // ArrayAddress dispatch(table, index);
6531     Address dispatch($dest$$Register, $switch_val$$Register, (Address::ScaleFactor) $shift$$constant);
6532     __ lea($dest$$Register, $constantaddress);
6533     __ jmp(dispatch);
6534   %}
6535   ins_pipe(pipe_jmp);
6536 %}
6537 
6538 instruct jumpXtnd_addr(rRegL switch_val, immI2 shift, immL32 offset, rRegI dest) %{
6539   match(Jump (AddL (LShiftL switch_val shift) offset));
6540   ins_cost(350);
6541   effect(TEMP dest);
6542 
6543   format %{ "leaq    $dest, [$constantaddress]\n\t"
6544             "jmp     [$dest + $switch_val << $shift + $offset]\n\t" %}
6545   ins_encode %{
6546     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
6547     // to do that and the compiler is using that register as one it can allocate.
6548     // So we build it all by hand.
6549     // Address index(noreg, switch_reg, (Address::ScaleFactor) $shift$$constant, (int) $offset$$constant);
6550     // ArrayAddress dispatch(table, index);
6551     Address dispatch($dest$$Register, $switch_val$$Register, (Address::ScaleFactor) $shift$$constant, (int) $offset$$constant);
6552     __ lea($dest$$Register, $constantaddress);
6553     __ jmp(dispatch);
6554   %}
6555   ins_pipe(pipe_jmp);
6556 %}
6557 
6558 instruct jumpXtnd(rRegL switch_val, rRegI dest) %{
6559   match(Jump switch_val);
6560   ins_cost(350);
6561   effect(TEMP dest);
6562 
6563   format %{ "leaq    $dest, [$constantaddress]\n\t"
6564             "jmp     [$dest + $switch_val]\n\t" %}
6565   ins_encode %{
6566     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
6567     // to do that and the compiler is using that register as one it can allocate.
6568     // So we build it all by hand.
6569     // Address index(noreg, switch_reg, Address::times_1);
6570     // ArrayAddress dispatch(table, index);
6571     Address dispatch($dest$$Register, $switch_val$$Register, Address::times_1);
6572     __ lea($dest$$Register, $constantaddress);
6573     __ jmp(dispatch);
6574   %}
6575   ins_pipe(pipe_jmp);
6576 %}
6577 
6578 // Conditional move
6579 instruct cmovI_reg(rRegI dst, rRegI src, rFlagsReg cr, cmpOp cop)
6580 %{
6581   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
6582 
6583   ins_cost(200); // XXX
6584   format %{ "cmovl$cop $dst, $src\t# signed, int" %}
6585   opcode(0x0F, 0x40);
6586   ins_encode(REX_reg_reg(dst, src), enc_cmov(cop), reg_reg(dst, src));
6587   ins_pipe(pipe_cmov_reg);
6588 %}
6589 
6590 instruct cmovI_regU(cmpOpU cop, rFlagsRegU cr, rRegI dst, rRegI src) %{
6591   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
6592 
6593   ins_cost(200); // XXX
6594   format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
6595   opcode(0x0F, 0x40);
6596   ins_encode(REX_reg_reg(dst, src), enc_cmov(cop), reg_reg(dst, src));
6597   ins_pipe(pipe_cmov_reg);
6598 %}
6599 
6600 instruct cmovI_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
6601   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
6602   ins_cost(200);
6603   expand %{
6604     cmovI_regU(cop, cr, dst, src);
6605   %}
6606 %}
6607 
6608 // Conditional move
6609 instruct cmovI_mem(cmpOp cop, rFlagsReg cr, rRegI dst, memory src) %{
6610   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
6611 
6612   ins_cost(250); // XXX
6613   format %{ "cmovl$cop $dst, $src\t# signed, int" %}
6614   opcode(0x0F, 0x40);
6615   ins_encode(REX_reg_mem(dst, src), enc_cmov(cop), reg_mem(dst, src));
6616   ins_pipe(pipe_cmov_mem);
6617 %}
6618 
6619 // Conditional move
6620 instruct cmovI_memU(cmpOpU cop, rFlagsRegU cr, rRegI dst, memory src)
6621 %{
6622   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
6623 
6624   ins_cost(250); // XXX
6625   format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
6626   opcode(0x0F, 0x40);
6627   ins_encode(REX_reg_mem(dst, src), enc_cmov(cop), reg_mem(dst, src));
6628   ins_pipe(pipe_cmov_mem);
6629 %}
6630 
6631 instruct cmovI_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, memory src) %{
6632   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
6633   ins_cost(250);
6634   expand %{
6635     cmovI_memU(cop, cr, dst, src);
6636   %}
6637 %}
6638 
6639 // Conditional move
6640 instruct cmovN_reg(rRegN dst, rRegN src, rFlagsReg cr, cmpOp cop)
6641 %{
6642   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
6643 
6644   ins_cost(200); // XXX
6645   format %{ "cmovl$cop $dst, $src\t# signed, compressed ptr" %}
6646   opcode(0x0F, 0x40);
6647   ins_encode(REX_reg_reg(dst, src), enc_cmov(cop), reg_reg(dst, src));
6648   ins_pipe(pipe_cmov_reg);
6649 %}
6650 
6651 // Conditional move
6652 instruct cmovN_regU(cmpOpU cop, rFlagsRegU cr, rRegN dst, rRegN src)
6653 %{
6654   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
6655 
6656   ins_cost(200); // XXX
6657   format %{ "cmovl$cop $dst, $src\t# unsigned, compressed ptr" %}
6658   opcode(0x0F, 0x40);
6659   ins_encode(REX_reg_reg(dst, src), enc_cmov(cop), reg_reg(dst, src));
6660   ins_pipe(pipe_cmov_reg);
6661 %}
6662 
6663 instruct cmovN_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegN dst, rRegN src) %{
6664   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
6665   ins_cost(200);
6666   expand %{
6667     cmovN_regU(cop, cr, dst, src);
6668   %}
6669 %}
6670 
6671 // Conditional move
6672 instruct cmovP_reg(rRegP dst, rRegP src, rFlagsReg cr, cmpOp cop)
6673 %{
6674   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
6675 
6676   ins_cost(200); // XXX
6677   format %{ "cmovq$cop $dst, $src\t# signed, ptr" %}
6678   opcode(0x0F, 0x40);
6679   ins_encode(REX_reg_reg_wide(dst, src), enc_cmov(cop), reg_reg(dst, src));
6680   ins_pipe(pipe_cmov_reg);  // XXX
6681 %}
6682 
6683 // Conditional move
6684 instruct cmovP_regU(cmpOpU cop, rFlagsRegU cr, rRegP dst, rRegP src)
6685 %{
6686   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
6687 
6688   ins_cost(200); // XXX
6689   format %{ "cmovq$cop $dst, $src\t# unsigned, ptr" %}
6690   opcode(0x0F, 0x40);
6691   ins_encode(REX_reg_reg_wide(dst, src), enc_cmov(cop), reg_reg(dst, src));
6692   ins_pipe(pipe_cmov_reg); // XXX
6693 %}
6694 
6695 instruct cmovP_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
6696   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
6697   ins_cost(200);
6698   expand %{
6699     cmovP_regU(cop, cr, dst, src);
6700   %}
6701 %}
6702 
6703 // DISABLED: Requires the ADLC to emit a bottom_type call that
6704 // correctly meets the two pointer arguments; one is an incoming
6705 // register but the other is a memory operand.  ALSO appears to
6706 // be buggy with implicit null checks.
6707 //
6708 //// Conditional move
6709 //instruct cmovP_mem(cmpOp cop, rFlagsReg cr, rRegP dst, memory src)
6710 //%{
6711 //  match(Set dst (CMoveP (Binary cop cr) (Binary dst (LoadP src))));
6712 //  ins_cost(250);
6713 //  format %{ "CMOV$cop $dst,$src\t# ptr" %}
6714 //  opcode(0x0F,0x40);
6715 //  ins_encode( enc_cmov(cop), reg_mem( dst, src ) );
6716 //  ins_pipe( pipe_cmov_mem );
6717 //%}
6718 //
6719 //// Conditional move
6720 //instruct cmovP_memU(cmpOpU cop, rFlagsRegU cr, rRegP dst, memory src)
6721 //%{
6722 //  match(Set dst (CMoveP (Binary cop cr) (Binary dst (LoadP src))));
6723 //  ins_cost(250);
6724 //  format %{ "CMOV$cop $dst,$src\t# ptr" %}
6725 //  opcode(0x0F,0x40);
6726 //  ins_encode( enc_cmov(cop), reg_mem( dst, src ) );
6727 //  ins_pipe( pipe_cmov_mem );
6728 //%}
6729 
6730 instruct cmovL_reg(cmpOp cop, rFlagsReg cr, rRegL dst, rRegL src)
6731 %{
6732   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
6733 
6734   ins_cost(200); // XXX
6735   format %{ "cmovq$cop $dst, $src\t# signed, long" %}
6736   opcode(0x0F, 0x40);
6737   ins_encode(REX_reg_reg_wide(dst, src), enc_cmov(cop), reg_reg(dst, src));
6738   ins_pipe(pipe_cmov_reg);  // XXX
6739 %}
6740 
6741 instruct cmovL_mem(cmpOp cop, rFlagsReg cr, rRegL dst, memory src)
6742 %{
6743   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
6744 
6745   ins_cost(200); // XXX
6746   format %{ "cmovq$cop $dst, $src\t# signed, long" %}
6747   opcode(0x0F, 0x40);
6748   ins_encode(REX_reg_mem_wide(dst, src), enc_cmov(cop), reg_mem(dst, src));
6749   ins_pipe(pipe_cmov_mem);  // XXX
6750 %}
6751 
6752 instruct cmovL_regU(cmpOpU cop, rFlagsRegU cr, rRegL dst, rRegL src)
6753 %{
6754   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
6755 
6756   ins_cost(200); // XXX
6757   format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
6758   opcode(0x0F, 0x40);
6759   ins_encode(REX_reg_reg_wide(dst, src), enc_cmov(cop), reg_reg(dst, src));
6760   ins_pipe(pipe_cmov_reg); // XXX
6761 %}
6762 
6763 instruct cmovL_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
6764   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
6765   ins_cost(200);
6766   expand %{
6767     cmovL_regU(cop, cr, dst, src);
6768   %}
6769 %}
6770 
6771 instruct cmovL_memU(cmpOpU cop, rFlagsRegU cr, rRegL dst, memory src)
6772 %{
6773   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
6774 
6775   ins_cost(200); // XXX
6776   format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
6777   opcode(0x0F, 0x40);
6778   ins_encode(REX_reg_mem_wide(dst, src), enc_cmov(cop), reg_mem(dst, src));
6779   ins_pipe(pipe_cmov_mem); // XXX
6780 %}
6781 
6782 instruct cmovL_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, memory src) %{
6783   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
6784   ins_cost(200);
6785   expand %{
6786     cmovL_memU(cop, cr, dst, src);
6787   %}
6788 %}
6789 
6790 instruct cmovF_reg(cmpOp cop, rFlagsReg cr, regF dst, regF src)
6791 %{
6792   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
6793 
6794   ins_cost(200); // XXX
6795   format %{ "jn$cop    skip\t# signed cmove float\n\t"
6796             "movss     $dst, $src\n"
6797     "skip:" %}
6798   ins_encode %{
6799     Label Lskip;
6800     // Invert sense of branch from sense of CMOV
6801     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
6802     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
6803     __ bind(Lskip);
6804   %}
6805   ins_pipe(pipe_slow);
6806 %}
6807 
6808 // instruct cmovF_mem(cmpOp cop, rFlagsReg cr, regF dst, memory src)
6809 // %{
6810 //   match(Set dst (CMoveF (Binary cop cr) (Binary dst (LoadL src))));
6811 
6812 //   ins_cost(200); // XXX
6813 //   format %{ "jn$cop    skip\t# signed cmove float\n\t"
6814 //             "movss     $dst, $src\n"
6815 //     "skip:" %}
6816 //   ins_encode(enc_cmovf_mem_branch(cop, dst, src));
6817 //   ins_pipe(pipe_slow);
6818 // %}
6819 
6820 instruct cmovF_regU(cmpOpU cop, rFlagsRegU cr, regF dst, regF src)
6821 %{
6822   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
6823 
6824   ins_cost(200); // XXX
6825   format %{ "jn$cop    skip\t# unsigned cmove float\n\t"
6826             "movss     $dst, $src\n"
6827     "skip:" %}
6828   ins_encode %{
6829     Label Lskip;
6830     // Invert sense of branch from sense of CMOV
6831     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
6832     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
6833     __ bind(Lskip);
6834   %}
6835   ins_pipe(pipe_slow);
6836 %}
6837 
6838 instruct cmovF_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regF dst, regF src) %{
6839   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
6840   ins_cost(200);
6841   expand %{
6842     cmovF_regU(cop, cr, dst, src);
6843   %}
6844 %}
6845 
6846 instruct cmovD_reg(cmpOp cop, rFlagsReg cr, regD dst, regD src)
6847 %{
6848   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
6849 
6850   ins_cost(200); // XXX
6851   format %{ "jn$cop    skip\t# signed cmove double\n\t"
6852             "movsd     $dst, $src\n"
6853     "skip:" %}
6854   ins_encode %{
6855     Label Lskip;
6856     // Invert sense of branch from sense of CMOV
6857     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
6858     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
6859     __ bind(Lskip);
6860   %}
6861   ins_pipe(pipe_slow);
6862 %}
6863 
6864 instruct cmovD_regU(cmpOpU cop, rFlagsRegU cr, regD dst, regD src)
6865 %{
6866   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
6867 
6868   ins_cost(200); // XXX
6869   format %{ "jn$cop    skip\t# unsigned cmove double\n\t"
6870             "movsd     $dst, $src\n"
6871     "skip:" %}
6872   ins_encode %{
6873     Label Lskip;
6874     // Invert sense of branch from sense of CMOV
6875     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
6876     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
6877     __ bind(Lskip);
6878   %}
6879   ins_pipe(pipe_slow);
6880 %}
6881 
6882 instruct cmovD_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regD dst, regD src) %{
6883   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
6884   ins_cost(200);
6885   expand %{
6886     cmovD_regU(cop, cr, dst, src);
6887   %}
6888 %}
6889 
6890 //----------Arithmetic Instructions--------------------------------------------
6891 //----------Addition Instructions----------------------------------------------
6892 
6893 instruct addI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
6894 %{
6895   match(Set dst (AddI dst src));
6896   effect(KILL cr);
6897 
6898   format %{ "addl    $dst, $src\t# int" %}
6899   opcode(0x03);
6900   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
6901   ins_pipe(ialu_reg_reg);
6902 %}
6903 
6904 instruct addI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
6905 %{
6906   match(Set dst (AddI dst src));
6907   effect(KILL cr);
6908 
6909   format %{ "addl    $dst, $src\t# int" %}
6910   opcode(0x81, 0x00); /* /0 id */
6911   ins_encode(OpcSErm(dst, src), Con8or32(src));
6912   ins_pipe( ialu_reg );
6913 %}
6914 
6915 instruct addI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
6916 %{
6917   match(Set dst (AddI dst (LoadI src)));
6918   effect(KILL cr);
6919 
6920   ins_cost(125); // XXX
6921   format %{ "addl    $dst, $src\t# int" %}
6922   opcode(0x03);
6923   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
6924   ins_pipe(ialu_reg_mem);
6925 %}
6926 
6927 instruct addI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
6928 %{
6929   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
6930   effect(KILL cr);
6931 
6932   ins_cost(150); // XXX
6933   format %{ "addl    $dst, $src\t# int" %}
6934   opcode(0x01); /* Opcode 01 /r */
6935   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
6936   ins_pipe(ialu_mem_reg);
6937 %}
6938 
6939 instruct addI_mem_imm(memory dst, immI src, rFlagsReg cr)
6940 %{
6941   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
6942   effect(KILL cr);
6943 
6944   ins_cost(125); // XXX
6945   format %{ "addl    $dst, $src\t# int" %}
6946   opcode(0x81); /* Opcode 81 /0 id */
6947   ins_encode(REX_mem(dst), OpcSE(src), RM_opc_mem(0x00, dst), Con8or32(src));
6948   ins_pipe(ialu_mem_imm);
6949 %}
6950 
6951 instruct incI_rReg(rRegI dst, immI1 src, rFlagsReg cr)
6952 %{
6953   predicate(UseIncDec);
6954   match(Set dst (AddI dst src));
6955   effect(KILL cr);
6956 
6957   format %{ "incl    $dst\t# int" %}
6958   opcode(0xFF, 0x00); // FF /0
6959   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
6960   ins_pipe(ialu_reg);
6961 %}
6962 
6963 instruct incI_mem(memory dst, immI1 src, rFlagsReg cr)
6964 %{
6965   predicate(UseIncDec);
6966   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
6967   effect(KILL cr);
6968 
6969   ins_cost(125); // XXX
6970   format %{ "incl    $dst\t# int" %}
6971   opcode(0xFF); /* Opcode FF /0 */
6972   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(0x00, dst));
6973   ins_pipe(ialu_mem_imm);
6974 %}
6975 
6976 // XXX why does that use AddI
6977 instruct decI_rReg(rRegI dst, immI_M1 src, rFlagsReg cr)
6978 %{
6979   predicate(UseIncDec);
6980   match(Set dst (AddI dst src));
6981   effect(KILL cr);
6982 
6983   format %{ "decl    $dst\t# int" %}
6984   opcode(0xFF, 0x01); // FF /1
6985   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
6986   ins_pipe(ialu_reg);
6987 %}
6988 
6989 // XXX why does that use AddI
6990 instruct decI_mem(memory dst, immI_M1 src, rFlagsReg cr)
6991 %{
6992   predicate(UseIncDec);
6993   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
6994   effect(KILL cr);
6995 
6996   ins_cost(125); // XXX
6997   format %{ "decl    $dst\t# int" %}
6998   opcode(0xFF); /* Opcode FF /1 */
6999   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(0x01, dst));
7000   ins_pipe(ialu_mem_imm);
7001 %}
7002 
7003 instruct leaI_rReg_immI(rRegI dst, rRegI src0, immI src1)
7004 %{
7005   match(Set dst (AddI src0 src1));
7006 
7007   ins_cost(110);
7008   format %{ "addr32 leal $dst, [$src0 + $src1]\t# int" %}
7009   opcode(0x8D); /* 0x8D /r */
7010   ins_encode(Opcode(0x67), REX_reg_reg(dst, src0), OpcP, reg_lea(dst, src0, src1)); // XXX
7011   ins_pipe(ialu_reg_reg);
7012 %}
7013 
7014 instruct addL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
7015 %{
7016   match(Set dst (AddL dst src));
7017   effect(KILL cr);
7018 
7019   format %{ "addq    $dst, $src\t# long" %}
7020   opcode(0x03);
7021   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
7022   ins_pipe(ialu_reg_reg);
7023 %}
7024 
7025 instruct addL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
7026 %{
7027   match(Set dst (AddL dst src));
7028   effect(KILL cr);
7029 
7030   format %{ "addq    $dst, $src\t# long" %}
7031   opcode(0x81, 0x00); /* /0 id */
7032   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
7033   ins_pipe( ialu_reg );
7034 %}
7035 
7036 instruct addL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
7037 %{
7038   match(Set dst (AddL dst (LoadL src)));
7039   effect(KILL cr);
7040 
7041   ins_cost(125); // XXX
7042   format %{ "addq    $dst, $src\t# long" %}
7043   opcode(0x03);
7044   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
7045   ins_pipe(ialu_reg_mem);
7046 %}
7047 
7048 instruct addL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
7049 %{
7050   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
7051   effect(KILL cr);
7052 
7053   ins_cost(150); // XXX
7054   format %{ "addq    $dst, $src\t# long" %}
7055   opcode(0x01); /* Opcode 01 /r */
7056   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
7057   ins_pipe(ialu_mem_reg);
7058 %}
7059 
7060 instruct addL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
7061 %{
7062   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
7063   effect(KILL cr);
7064 
7065   ins_cost(125); // XXX
7066   format %{ "addq    $dst, $src\t# long" %}
7067   opcode(0x81); /* Opcode 81 /0 id */
7068   ins_encode(REX_mem_wide(dst),
7069              OpcSE(src), RM_opc_mem(0x00, dst), Con8or32(src));
7070   ins_pipe(ialu_mem_imm);
7071 %}
7072 
7073 instruct incL_rReg(rRegI dst, immL1 src, rFlagsReg cr)
7074 %{
7075   predicate(UseIncDec);
7076   match(Set dst (AddL dst src));
7077   effect(KILL cr);
7078 
7079   format %{ "incq    $dst\t# long" %}
7080   opcode(0xFF, 0x00); // FF /0
7081   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
7082   ins_pipe(ialu_reg);
7083 %}
7084 
7085 instruct incL_mem(memory dst, immL1 src, rFlagsReg cr)
7086 %{
7087   predicate(UseIncDec);
7088   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
7089   effect(KILL cr);
7090 
7091   ins_cost(125); // XXX
7092   format %{ "incq    $dst\t# long" %}
7093   opcode(0xFF); /* Opcode FF /0 */
7094   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(0x00, dst));
7095   ins_pipe(ialu_mem_imm);
7096 %}
7097 
7098 // XXX why does that use AddL
7099 instruct decL_rReg(rRegL dst, immL_M1 src, rFlagsReg cr)
7100 %{
7101   predicate(UseIncDec);
7102   match(Set dst (AddL dst src));
7103   effect(KILL cr);
7104 
7105   format %{ "decq    $dst\t# long" %}
7106   opcode(0xFF, 0x01); // FF /1
7107   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
7108   ins_pipe(ialu_reg);
7109 %}
7110 
7111 // XXX why does that use AddL
7112 instruct decL_mem(memory dst, immL_M1 src, rFlagsReg cr)
7113 %{
7114   predicate(UseIncDec);
7115   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
7116   effect(KILL cr);
7117 
7118   ins_cost(125); // XXX
7119   format %{ "decq    $dst\t# long" %}
7120   opcode(0xFF); /* Opcode FF /1 */
7121   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(0x01, dst));
7122   ins_pipe(ialu_mem_imm);
7123 %}
7124 
7125 instruct leaL_rReg_immL(rRegL dst, rRegL src0, immL32 src1)
7126 %{
7127   match(Set dst (AddL src0 src1));
7128 
7129   ins_cost(110);
7130   format %{ "leaq    $dst, [$src0 + $src1]\t# long" %}
7131   opcode(0x8D); /* 0x8D /r */
7132   ins_encode(REX_reg_reg_wide(dst, src0), OpcP, reg_lea(dst, src0, src1)); // XXX
7133   ins_pipe(ialu_reg_reg);
7134 %}
7135 
7136 instruct addP_rReg(rRegP dst, rRegL src, rFlagsReg cr)
7137 %{
7138   match(Set dst (AddP dst src));
7139   effect(KILL cr);
7140 
7141   format %{ "addq    $dst, $src\t# ptr" %}
7142   opcode(0x03);
7143   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
7144   ins_pipe(ialu_reg_reg);
7145 %}
7146 
7147 instruct addP_rReg_imm(rRegP dst, immL32 src, rFlagsReg cr)
7148 %{
7149   match(Set dst (AddP dst src));
7150   effect(KILL cr);
7151 
7152   format %{ "addq    $dst, $src\t# ptr" %}
7153   opcode(0x81, 0x00); /* /0 id */
7154   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
7155   ins_pipe( ialu_reg );
7156 %}
7157 
7158 // XXX addP mem ops ????
7159 
7160 instruct leaP_rReg_imm(rRegP dst, rRegP src0, immL32 src1)
7161 %{
7162   match(Set dst (AddP src0 src1));
7163 
7164   ins_cost(110);
7165   format %{ "leaq    $dst, [$src0 + $src1]\t# ptr" %}
7166   opcode(0x8D); /* 0x8D /r */
7167   ins_encode(REX_reg_reg_wide(dst, src0), OpcP, reg_lea(dst, src0, src1));// XXX
7168   ins_pipe(ialu_reg_reg);
7169 %}
7170 
7171 instruct checkCastPP(rRegP dst)
7172 %{
7173   match(Set dst (CheckCastPP dst));
7174 
7175   size(0);
7176   format %{ "# checkcastPP of $dst" %}
7177   ins_encode(/* empty encoding */);
7178   ins_pipe(empty);
7179 %}
7180 
7181 instruct castPP(rRegP dst)
7182 %{
7183   match(Set dst (CastPP dst));
7184 
7185   size(0);
7186   format %{ "# castPP of $dst" %}
7187   ins_encode(/* empty encoding */);
7188   ins_pipe(empty);
7189 %}
7190 
7191 instruct castII(rRegI dst)
7192 %{
7193   match(Set dst (CastII dst));
7194 
7195   size(0);
7196   format %{ "# castII of $dst" %}
7197   ins_encode(/* empty encoding */);
7198   ins_cost(0);
7199   ins_pipe(empty);
7200 %}
7201 
7202 // LoadP-locked same as a regular LoadP when used with compare-swap
7203 instruct loadPLocked(rRegP dst, memory mem)
7204 %{
7205   match(Set dst (LoadPLocked mem));
7206 
7207   ins_cost(125); // XXX
7208   format %{ "movq    $dst, $mem\t# ptr locked" %}
7209   opcode(0x8B);
7210   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
7211   ins_pipe(ialu_reg_mem); // XXX
7212 %}
7213 
7214 // Conditional-store of the updated heap-top.
7215 // Used during allocation of the shared heap.
7216 // Sets flags (EQ) on success.  Implemented with a CMPXCHG on Intel.
7217 
7218 instruct storePConditional(memory heap_top_ptr,
7219                            rax_RegP oldval, rRegP newval,
7220                            rFlagsReg cr)
7221 %{
7222   match(Set cr (StorePConditional heap_top_ptr (Binary oldval newval)));
7223 
7224   format %{ "cmpxchgq $heap_top_ptr, $newval\t# (ptr) "
7225             "If rax == $heap_top_ptr then store $newval into $heap_top_ptr" %}
7226   opcode(0x0F, 0xB1);
7227   ins_encode(lock_prefix,
7228              REX_reg_mem_wide(newval, heap_top_ptr),
7229              OpcP, OpcS,
7230              reg_mem(newval, heap_top_ptr));
7231   ins_pipe(pipe_cmpxchg);
7232 %}
7233 
7234 // Conditional-store of an int value.
7235 // ZF flag is set on success, reset otherwise.  Implemented with a CMPXCHG.
7236 instruct storeIConditional(memory mem, rax_RegI oldval, rRegI newval, rFlagsReg cr)
7237 %{
7238   match(Set cr (StoreIConditional mem (Binary oldval newval)));
7239   effect(KILL oldval);
7240 
7241   format %{ "cmpxchgl $mem, $newval\t# If rax == $mem then store $newval into $mem" %}
7242   opcode(0x0F, 0xB1);
7243   ins_encode(lock_prefix,
7244              REX_reg_mem(newval, mem),
7245              OpcP, OpcS,
7246              reg_mem(newval, mem));
7247   ins_pipe(pipe_cmpxchg);
7248 %}
7249 
7250 // Conditional-store of a long value.
7251 // ZF flag is set on success, reset otherwise.  Implemented with a CMPXCHG.
7252 instruct storeLConditional(memory mem, rax_RegL oldval, rRegL newval, rFlagsReg cr)
7253 %{
7254   match(Set cr (StoreLConditional mem (Binary oldval newval)));
7255   effect(KILL oldval);
7256 
7257   format %{ "cmpxchgq $mem, $newval\t# If rax == $mem then store $newval into $mem" %}
7258   opcode(0x0F, 0xB1);
7259   ins_encode(lock_prefix,
7260              REX_reg_mem_wide(newval, mem),
7261              OpcP, OpcS,
7262              reg_mem(newval, mem));
7263   ins_pipe(pipe_cmpxchg);
7264 %}
7265 
7266 
7267 // XXX No flag versions for CompareAndSwap{P,I,L} because matcher can't match them
7268 instruct compareAndSwapP(rRegI res,
7269                          memory mem_ptr,
7270                          rax_RegP oldval, rRegP newval,
7271                          rFlagsReg cr)
7272 %{
7273   predicate(VM_Version::supports_cx8());
7274   match(Set res (CompareAndSwapP mem_ptr (Binary oldval newval)));
7275   effect(KILL cr, KILL oldval);
7276 
7277   format %{ "cmpxchgq $mem_ptr,$newval\t# "
7278             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7279             "sete    $res\n\t"
7280             "movzbl  $res, $res" %}
7281   opcode(0x0F, 0xB1);
7282   ins_encode(lock_prefix,
7283              REX_reg_mem_wide(newval, mem_ptr),
7284              OpcP, OpcS,
7285              reg_mem(newval, mem_ptr),
7286              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7287              REX_reg_breg(res, res), // movzbl
7288              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7289   ins_pipe( pipe_cmpxchg );
7290 %}
7291 
7292 instruct compareAndSwapL(rRegI res,
7293                          memory mem_ptr,
7294                          rax_RegL oldval, rRegL newval,
7295                          rFlagsReg cr)
7296 %{
7297   predicate(VM_Version::supports_cx8());
7298   match(Set res (CompareAndSwapL mem_ptr (Binary oldval newval)));
7299   effect(KILL cr, KILL oldval);
7300 
7301   format %{ "cmpxchgq $mem_ptr,$newval\t# "
7302             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7303             "sete    $res\n\t"
7304             "movzbl  $res, $res" %}
7305   opcode(0x0F, 0xB1);
7306   ins_encode(lock_prefix,
7307              REX_reg_mem_wide(newval, mem_ptr),
7308              OpcP, OpcS,
7309              reg_mem(newval, mem_ptr),
7310              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7311              REX_reg_breg(res, res), // movzbl
7312              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7313   ins_pipe( pipe_cmpxchg );
7314 %}
7315 
7316 instruct compareAndSwapI(rRegI res,
7317                          memory mem_ptr,
7318                          rax_RegI oldval, rRegI newval,
7319                          rFlagsReg cr)
7320 %{
7321   match(Set res (CompareAndSwapI mem_ptr (Binary oldval newval)));
7322   effect(KILL cr, KILL oldval);
7323 
7324   format %{ "cmpxchgl $mem_ptr,$newval\t# "
7325             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7326             "sete    $res\n\t"
7327             "movzbl  $res, $res" %}
7328   opcode(0x0F, 0xB1);
7329   ins_encode(lock_prefix,
7330              REX_reg_mem(newval, mem_ptr),
7331              OpcP, OpcS,
7332              reg_mem(newval, mem_ptr),
7333              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7334              REX_reg_breg(res, res), // movzbl
7335              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7336   ins_pipe( pipe_cmpxchg );
7337 %}
7338 
7339 
7340 instruct compareAndSwapN(rRegI res,
7341                           memory mem_ptr,
7342                           rax_RegN oldval, rRegN newval,
7343                           rFlagsReg cr) %{
7344   match(Set res (CompareAndSwapN mem_ptr (Binary oldval newval)));
7345   effect(KILL cr, KILL oldval);
7346 
7347   format %{ "cmpxchgl $mem_ptr,$newval\t# "
7348             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7349             "sete    $res\n\t"
7350             "movzbl  $res, $res" %}
7351   opcode(0x0F, 0xB1);
7352   ins_encode(lock_prefix,
7353              REX_reg_mem(newval, mem_ptr),
7354              OpcP, OpcS,
7355              reg_mem(newval, mem_ptr),
7356              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7357              REX_reg_breg(res, res), // movzbl
7358              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7359   ins_pipe( pipe_cmpxchg );
7360 %}
7361 
7362 instruct xaddI_no_res( memory mem, Universe dummy, immI add, rFlagsReg cr) %{
7363   predicate(n->as_LoadStore()->result_not_used());
7364   match(Set dummy (GetAndAddI mem add));
7365   effect(KILL cr);
7366   format %{ "ADDL  [$mem],$add" %}
7367   ins_encode %{
7368     if (os::is_MP()) { __ lock(); }
7369     __ addl($mem$$Address, $add$$constant);
7370   %}
7371   ins_pipe( pipe_cmpxchg );
7372 %}
7373 
7374 instruct xaddI( memory mem, rRegI newval, rFlagsReg cr) %{
7375   match(Set newval (GetAndAddI mem newval));
7376   effect(KILL cr);
7377   format %{ "XADDL  [$mem],$newval" %}
7378   ins_encode %{
7379     if (os::is_MP()) { __ lock(); }
7380     __ xaddl($mem$$Address, $newval$$Register);
7381   %}
7382   ins_pipe( pipe_cmpxchg );
7383 %}
7384 
7385 instruct xaddL_no_res( memory mem, Universe dummy, immL32 add, rFlagsReg cr) %{
7386   predicate(n->as_LoadStore()->result_not_used());
7387   match(Set dummy (GetAndAddL mem add));
7388   effect(KILL cr);
7389   format %{ "ADDQ  [$mem],$add" %}
7390   ins_encode %{
7391     if (os::is_MP()) { __ lock(); }
7392     __ addq($mem$$Address, $add$$constant);
7393   %}
7394   ins_pipe( pipe_cmpxchg );
7395 %}
7396 
7397 instruct xaddL( memory mem, rRegL newval, rFlagsReg cr) %{
7398   match(Set newval (GetAndAddL mem newval));
7399   effect(KILL cr);
7400   format %{ "XADDQ  [$mem],$newval" %}
7401   ins_encode %{
7402     if (os::is_MP()) { __ lock(); }
7403     __ xaddq($mem$$Address, $newval$$Register);
7404   %}
7405   ins_pipe( pipe_cmpxchg );
7406 %}
7407 
7408 instruct xchgI( memory mem, rRegI newval) %{
7409   match(Set newval (GetAndSetI mem newval));
7410   format %{ "XCHGL  $newval,[$mem]" %}
7411   ins_encode %{
7412     __ xchgl($newval$$Register, $mem$$Address);
7413   %}
7414   ins_pipe( pipe_cmpxchg );
7415 %}
7416 
7417 instruct xchgL( memory mem, rRegL newval) %{
7418   match(Set newval (GetAndSetL mem newval));
7419   format %{ "XCHGL  $newval,[$mem]" %}
7420   ins_encode %{
7421     __ xchgq($newval$$Register, $mem$$Address);
7422   %}
7423   ins_pipe( pipe_cmpxchg );
7424 %}
7425 
7426 instruct xchgP( memory mem, rRegP newval) %{
7427   match(Set newval (GetAndSetP mem newval));
7428   format %{ "XCHGQ  $newval,[$mem]" %}
7429   ins_encode %{
7430     __ xchgq($newval$$Register, $mem$$Address);
7431   %}
7432   ins_pipe( pipe_cmpxchg );
7433 %}
7434 
7435 instruct xchgN( memory mem, rRegN newval) %{
7436   match(Set newval (GetAndSetN mem newval));
7437   format %{ "XCHGL  $newval,$mem]" %}
7438   ins_encode %{
7439     __ xchgl($newval$$Register, $mem$$Address);
7440   %}
7441   ins_pipe( pipe_cmpxchg );
7442 %}
7443 
7444 //----------Subtraction Instructions-------------------------------------------
7445 
7446 // Integer Subtraction Instructions
7447 instruct subI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
7448 %{
7449   match(Set dst (SubI dst src));
7450   effect(KILL cr);
7451 
7452   format %{ "subl    $dst, $src\t# int" %}
7453   opcode(0x2B);
7454   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
7455   ins_pipe(ialu_reg_reg);
7456 %}
7457 
7458 instruct subI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
7459 %{
7460   match(Set dst (SubI dst src));
7461   effect(KILL cr);
7462 
7463   format %{ "subl    $dst, $src\t# int" %}
7464   opcode(0x81, 0x05);  /* Opcode 81 /5 */
7465   ins_encode(OpcSErm(dst, src), Con8or32(src));
7466   ins_pipe(ialu_reg);
7467 %}
7468 
7469 instruct subI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
7470 %{
7471   match(Set dst (SubI dst (LoadI src)));
7472   effect(KILL cr);
7473 
7474   ins_cost(125);
7475   format %{ "subl    $dst, $src\t# int" %}
7476   opcode(0x2B);
7477   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
7478   ins_pipe(ialu_reg_mem);
7479 %}
7480 
7481 instruct subI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
7482 %{
7483   match(Set dst (StoreI dst (SubI (LoadI dst) src)));
7484   effect(KILL cr);
7485 
7486   ins_cost(150);
7487   format %{ "subl    $dst, $src\t# int" %}
7488   opcode(0x29); /* Opcode 29 /r */
7489   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
7490   ins_pipe(ialu_mem_reg);
7491 %}
7492 
7493 instruct subI_mem_imm(memory dst, immI src, rFlagsReg cr)
7494 %{
7495   match(Set dst (StoreI dst (SubI (LoadI dst) src)));
7496   effect(KILL cr);
7497 
7498   ins_cost(125); // XXX
7499   format %{ "subl    $dst, $src\t# int" %}
7500   opcode(0x81); /* Opcode 81 /5 id */
7501   ins_encode(REX_mem(dst), OpcSE(src), RM_opc_mem(0x05, dst), Con8or32(src));
7502   ins_pipe(ialu_mem_imm);
7503 %}
7504 
7505 instruct subL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
7506 %{
7507   match(Set dst (SubL dst src));
7508   effect(KILL cr);
7509 
7510   format %{ "subq    $dst, $src\t# long" %}
7511   opcode(0x2B);
7512   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
7513   ins_pipe(ialu_reg_reg);
7514 %}
7515 
7516 instruct subL_rReg_imm(rRegI dst, immL32 src, rFlagsReg cr)
7517 %{
7518   match(Set dst (SubL dst src));
7519   effect(KILL cr);
7520 
7521   format %{ "subq    $dst, $src\t# long" %}
7522   opcode(0x81, 0x05);  /* Opcode 81 /5 */
7523   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
7524   ins_pipe(ialu_reg);
7525 %}
7526 
7527 instruct subL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
7528 %{
7529   match(Set dst (SubL dst (LoadL src)));
7530   effect(KILL cr);
7531 
7532   ins_cost(125);
7533   format %{ "subq    $dst, $src\t# long" %}
7534   opcode(0x2B);
7535   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
7536   ins_pipe(ialu_reg_mem);
7537 %}
7538 
7539 instruct subL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
7540 %{
7541   match(Set dst (StoreL dst (SubL (LoadL dst) src)));
7542   effect(KILL cr);
7543 
7544   ins_cost(150);
7545   format %{ "subq    $dst, $src\t# long" %}
7546   opcode(0x29); /* Opcode 29 /r */
7547   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
7548   ins_pipe(ialu_mem_reg);
7549 %}
7550 
7551 instruct subL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
7552 %{
7553   match(Set dst (StoreL dst (SubL (LoadL dst) src)));
7554   effect(KILL cr);
7555 
7556   ins_cost(125); // XXX
7557   format %{ "subq    $dst, $src\t# long" %}
7558   opcode(0x81); /* Opcode 81 /5 id */
7559   ins_encode(REX_mem_wide(dst),
7560              OpcSE(src), RM_opc_mem(0x05, dst), Con8or32(src));
7561   ins_pipe(ialu_mem_imm);
7562 %}
7563 
7564 // Subtract from a pointer
7565 // XXX hmpf???
7566 instruct subP_rReg(rRegP dst, rRegI src, immI0 zero, rFlagsReg cr)
7567 %{
7568   match(Set dst (AddP dst (SubI zero src)));
7569   effect(KILL cr);
7570 
7571   format %{ "subq    $dst, $src\t# ptr - int" %}
7572   opcode(0x2B);
7573   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
7574   ins_pipe(ialu_reg_reg);
7575 %}
7576 
7577 instruct negI_rReg(rRegI dst, immI0 zero, rFlagsReg cr)
7578 %{
7579   match(Set dst (SubI zero dst));
7580   effect(KILL cr);
7581 
7582   format %{ "negl    $dst\t# int" %}
7583   opcode(0xF7, 0x03);  // Opcode F7 /3
7584   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
7585   ins_pipe(ialu_reg);
7586 %}
7587 
7588 instruct negI_mem(memory dst, immI0 zero, rFlagsReg cr)
7589 %{
7590   match(Set dst (StoreI dst (SubI zero (LoadI dst))));
7591   effect(KILL cr);
7592 
7593   format %{ "negl    $dst\t# int" %}
7594   opcode(0xF7, 0x03);  // Opcode F7 /3
7595   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
7596   ins_pipe(ialu_reg);
7597 %}
7598 
7599 instruct negL_rReg(rRegL dst, immL0 zero, rFlagsReg cr)
7600 %{
7601   match(Set dst (SubL zero dst));
7602   effect(KILL cr);
7603 
7604   format %{ "negq    $dst\t# long" %}
7605   opcode(0xF7, 0x03);  // Opcode F7 /3
7606   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
7607   ins_pipe(ialu_reg);
7608 %}
7609 
7610 instruct negL_mem(memory dst, immL0 zero, rFlagsReg cr)
7611 %{
7612   match(Set dst (StoreL dst (SubL zero (LoadL dst))));
7613   effect(KILL cr);
7614 
7615   format %{ "negq    $dst\t# long" %}
7616   opcode(0xF7, 0x03);  // Opcode F7 /3
7617   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
7618   ins_pipe(ialu_reg);
7619 %}
7620 
7621 //----------Multiplication/Division Instructions-------------------------------
7622 // Integer Multiplication Instructions
7623 // Multiply Register
7624 
7625 instruct mulI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
7626 %{
7627   match(Set dst (MulI dst src));
7628   effect(KILL cr);
7629 
7630   ins_cost(300);
7631   format %{ "imull   $dst, $src\t# int" %}
7632   opcode(0x0F, 0xAF);
7633   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
7634   ins_pipe(ialu_reg_reg_alu0);
7635 %}
7636 
7637 instruct mulI_rReg_imm(rRegI dst, rRegI src, immI imm, rFlagsReg cr)
7638 %{
7639   match(Set dst (MulI src imm));
7640   effect(KILL cr);
7641 
7642   ins_cost(300);
7643   format %{ "imull   $dst, $src, $imm\t# int" %}
7644   opcode(0x69); /* 69 /r id */
7645   ins_encode(REX_reg_reg(dst, src),
7646              OpcSE(imm), reg_reg(dst, src), Con8or32(imm));
7647   ins_pipe(ialu_reg_reg_alu0);
7648 %}
7649 
7650 instruct mulI_mem(rRegI dst, memory src, rFlagsReg cr)
7651 %{
7652   match(Set dst (MulI dst (LoadI src)));
7653   effect(KILL cr);
7654 
7655   ins_cost(350);
7656   format %{ "imull   $dst, $src\t# int" %}
7657   opcode(0x0F, 0xAF);
7658   ins_encode(REX_reg_mem(dst, src), OpcP, OpcS, reg_mem(dst, src));
7659   ins_pipe(ialu_reg_mem_alu0);
7660 %}
7661 
7662 instruct mulI_mem_imm(rRegI dst, memory src, immI imm, rFlagsReg cr)
7663 %{
7664   match(Set dst (MulI (LoadI src) imm));
7665   effect(KILL cr);
7666 
7667   ins_cost(300);
7668   format %{ "imull   $dst, $src, $imm\t# int" %}
7669   opcode(0x69); /* 69 /r id */
7670   ins_encode(REX_reg_mem(dst, src),
7671              OpcSE(imm), reg_mem(dst, src), Con8or32(imm));
7672   ins_pipe(ialu_reg_mem_alu0);
7673 %}
7674 
7675 instruct mulL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
7676 %{
7677   match(Set dst (MulL dst src));
7678   effect(KILL cr);
7679 
7680   ins_cost(300);
7681   format %{ "imulq   $dst, $src\t# long" %}
7682   opcode(0x0F, 0xAF);
7683   ins_encode(REX_reg_reg_wide(dst, src), OpcP, OpcS, reg_reg(dst, src));
7684   ins_pipe(ialu_reg_reg_alu0);
7685 %}
7686 
7687 instruct mulL_rReg_imm(rRegL dst, rRegL src, immL32 imm, rFlagsReg cr)
7688 %{
7689   match(Set dst (MulL src imm));
7690   effect(KILL cr);
7691 
7692   ins_cost(300);
7693   format %{ "imulq   $dst, $src, $imm\t# long" %}
7694   opcode(0x69); /* 69 /r id */
7695   ins_encode(REX_reg_reg_wide(dst, src),
7696              OpcSE(imm), reg_reg(dst, src), Con8or32(imm));
7697   ins_pipe(ialu_reg_reg_alu0);
7698 %}
7699 
7700 instruct mulL_mem(rRegL dst, memory src, rFlagsReg cr)
7701 %{
7702   match(Set dst (MulL dst (LoadL src)));
7703   effect(KILL cr);
7704 
7705   ins_cost(350);
7706   format %{ "imulq   $dst, $src\t# long" %}
7707   opcode(0x0F, 0xAF);
7708   ins_encode(REX_reg_mem_wide(dst, src), OpcP, OpcS, reg_mem(dst, src));
7709   ins_pipe(ialu_reg_mem_alu0);
7710 %}
7711 
7712 instruct mulL_mem_imm(rRegL dst, memory src, immL32 imm, rFlagsReg cr)
7713 %{
7714   match(Set dst (MulL (LoadL src) imm));
7715   effect(KILL cr);
7716 
7717   ins_cost(300);
7718   format %{ "imulq   $dst, $src, $imm\t# long" %}
7719   opcode(0x69); /* 69 /r id */
7720   ins_encode(REX_reg_mem_wide(dst, src),
7721              OpcSE(imm), reg_mem(dst, src), Con8or32(imm));
7722   ins_pipe(ialu_reg_mem_alu0);
7723 %}
7724 
7725 instruct mulHiL_rReg(rdx_RegL dst, no_rax_RegL src, rax_RegL rax, rFlagsReg cr)
7726 %{
7727   match(Set dst (MulHiL src rax));
7728   effect(USE_KILL rax, KILL cr);
7729 
7730   ins_cost(300);
7731   format %{ "imulq   RDX:RAX, RAX, $src\t# mulhi" %}
7732   opcode(0xF7, 0x5); /* Opcode F7 /5 */
7733   ins_encode(REX_reg_wide(src), OpcP, reg_opc(src));
7734   ins_pipe(ialu_reg_reg_alu0);
7735 %}
7736 
7737 instruct divI_rReg(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div,
7738                    rFlagsReg cr)
7739 %{
7740   match(Set rax (DivI rax div));
7741   effect(KILL rdx, KILL cr);
7742 
7743   ins_cost(30*100+10*100); // XXX
7744   format %{ "cmpl    rax, 0x80000000\t# idiv\n\t"
7745             "jne,s   normal\n\t"
7746             "xorl    rdx, rdx\n\t"
7747             "cmpl    $div, -1\n\t"
7748             "je,s    done\n"
7749     "normal: cdql\n\t"
7750             "idivl   $div\n"
7751     "done:"        %}
7752   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
7753   ins_encode(cdql_enc(div), REX_reg(div), OpcP, reg_opc(div));
7754   ins_pipe(ialu_reg_reg_alu0);
7755 %}
7756 
7757 instruct divL_rReg(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div,
7758                    rFlagsReg cr)
7759 %{
7760   match(Set rax (DivL rax div));
7761   effect(KILL rdx, KILL cr);
7762 
7763   ins_cost(30*100+10*100); // XXX
7764   format %{ "movq    rdx, 0x8000000000000000\t# ldiv\n\t"
7765             "cmpq    rax, rdx\n\t"
7766             "jne,s   normal\n\t"
7767             "xorl    rdx, rdx\n\t"
7768             "cmpq    $div, -1\n\t"
7769             "je,s    done\n"
7770     "normal: cdqq\n\t"
7771             "idivq   $div\n"
7772     "done:"        %}
7773   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
7774   ins_encode(cdqq_enc(div), REX_reg_wide(div), OpcP, reg_opc(div));
7775   ins_pipe(ialu_reg_reg_alu0);
7776 %}
7777 
7778 // Integer DIVMOD with Register, both quotient and mod results
7779 instruct divModI_rReg_divmod(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div,
7780                              rFlagsReg cr)
7781 %{
7782   match(DivModI rax div);
7783   effect(KILL cr);
7784 
7785   ins_cost(30*100+10*100); // XXX
7786   format %{ "cmpl    rax, 0x80000000\t# idiv\n\t"
7787             "jne,s   normal\n\t"
7788             "xorl    rdx, rdx\n\t"
7789             "cmpl    $div, -1\n\t"
7790             "je,s    done\n"
7791     "normal: cdql\n\t"
7792             "idivl   $div\n"
7793     "done:"        %}
7794   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
7795   ins_encode(cdql_enc(div), REX_reg(div), OpcP, reg_opc(div));
7796   ins_pipe(pipe_slow);
7797 %}
7798 
7799 // Long DIVMOD with Register, both quotient and mod results
7800 instruct divModL_rReg_divmod(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div,
7801                              rFlagsReg cr)
7802 %{
7803   match(DivModL rax div);
7804   effect(KILL cr);
7805 
7806   ins_cost(30*100+10*100); // XXX
7807   format %{ "movq    rdx, 0x8000000000000000\t# ldiv\n\t"
7808             "cmpq    rax, rdx\n\t"
7809             "jne,s   normal\n\t"
7810             "xorl    rdx, rdx\n\t"
7811             "cmpq    $div, -1\n\t"
7812             "je,s    done\n"
7813     "normal: cdqq\n\t"
7814             "idivq   $div\n"
7815     "done:"        %}
7816   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
7817   ins_encode(cdqq_enc(div), REX_reg_wide(div), OpcP, reg_opc(div));
7818   ins_pipe(pipe_slow);
7819 %}
7820 
7821 //----------- DivL-By-Constant-Expansions--------------------------------------
7822 // DivI cases are handled by the compiler
7823 
7824 // Magic constant, reciprocal of 10
7825 instruct loadConL_0x6666666666666667(rRegL dst)
7826 %{
7827   effect(DEF dst);
7828 
7829   format %{ "movq    $dst, #0x666666666666667\t# Used in div-by-10" %}
7830   ins_encode(load_immL(dst, 0x6666666666666667));
7831   ins_pipe(ialu_reg);
7832 %}
7833 
7834 instruct mul_hi(rdx_RegL dst, no_rax_RegL src, rax_RegL rax, rFlagsReg cr)
7835 %{
7836   effect(DEF dst, USE src, USE_KILL rax, KILL cr);
7837 
7838   format %{ "imulq   rdx:rax, rax, $src\t# Used in div-by-10" %}
7839   opcode(0xF7, 0x5); /* Opcode F7 /5 */
7840   ins_encode(REX_reg_wide(src), OpcP, reg_opc(src));
7841   ins_pipe(ialu_reg_reg_alu0);
7842 %}
7843 
7844 instruct sarL_rReg_63(rRegL dst, rFlagsReg cr)
7845 %{
7846   effect(USE_DEF dst, KILL cr);
7847 
7848   format %{ "sarq    $dst, #63\t# Used in div-by-10" %}
7849   opcode(0xC1, 0x7); /* C1 /7 ib */
7850   ins_encode(reg_opc_imm_wide(dst, 0x3F));
7851   ins_pipe(ialu_reg);
7852 %}
7853 
7854 instruct sarL_rReg_2(rRegL dst, rFlagsReg cr)
7855 %{
7856   effect(USE_DEF dst, KILL cr);
7857 
7858   format %{ "sarq    $dst, #2\t# Used in div-by-10" %}
7859   opcode(0xC1, 0x7); /* C1 /7 ib */
7860   ins_encode(reg_opc_imm_wide(dst, 0x2));
7861   ins_pipe(ialu_reg);
7862 %}
7863 
7864 instruct divL_10(rdx_RegL dst, no_rax_RegL src, immL10 div)
7865 %{
7866   match(Set dst (DivL src div));
7867 
7868   ins_cost((5+8)*100);
7869   expand %{
7870     rax_RegL rax;                     // Killed temp
7871     rFlagsReg cr;                     // Killed
7872     loadConL_0x6666666666666667(rax); // movq  rax, 0x6666666666666667
7873     mul_hi(dst, src, rax, cr);        // mulq  rdx:rax <= rax * $src
7874     sarL_rReg_63(src, cr);            // sarq  src, 63
7875     sarL_rReg_2(dst, cr);             // sarq  rdx, 2
7876     subL_rReg(dst, src, cr);          // subl  rdx, src
7877   %}
7878 %}
7879 
7880 //-----------------------------------------------------------------------------
7881 
7882 instruct modI_rReg(rdx_RegI rdx, rax_RegI rax, no_rax_rdx_RegI div,
7883                    rFlagsReg cr)
7884 %{
7885   match(Set rdx (ModI rax div));
7886   effect(KILL rax, KILL cr);
7887 
7888   ins_cost(300); // XXX
7889   format %{ "cmpl    rax, 0x80000000\t# irem\n\t"
7890             "jne,s   normal\n\t"
7891             "xorl    rdx, rdx\n\t"
7892             "cmpl    $div, -1\n\t"
7893             "je,s    done\n"
7894     "normal: cdql\n\t"
7895             "idivl   $div\n"
7896     "done:"        %}
7897   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
7898   ins_encode(cdql_enc(div), REX_reg(div), OpcP, reg_opc(div));
7899   ins_pipe(ialu_reg_reg_alu0);
7900 %}
7901 
7902 instruct modL_rReg(rdx_RegL rdx, rax_RegL rax, no_rax_rdx_RegL div,
7903                    rFlagsReg cr)
7904 %{
7905   match(Set rdx (ModL rax div));
7906   effect(KILL rax, KILL cr);
7907 
7908   ins_cost(300); // XXX
7909   format %{ "movq    rdx, 0x8000000000000000\t# lrem\n\t"
7910             "cmpq    rax, rdx\n\t"
7911             "jne,s   normal\n\t"
7912             "xorl    rdx, rdx\n\t"
7913             "cmpq    $div, -1\n\t"
7914             "je,s    done\n"
7915     "normal: cdqq\n\t"
7916             "idivq   $div\n"
7917     "done:"        %}
7918   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
7919   ins_encode(cdqq_enc(div), REX_reg_wide(div), OpcP, reg_opc(div));
7920   ins_pipe(ialu_reg_reg_alu0);
7921 %}
7922 
7923 // Integer Shift Instructions
7924 // Shift Left by one
7925 instruct salI_rReg_1(rRegI dst, immI1 shift, rFlagsReg cr)
7926 %{
7927   match(Set dst (LShiftI dst shift));
7928   effect(KILL cr);
7929 
7930   format %{ "sall    $dst, $shift" %}
7931   opcode(0xD1, 0x4); /* D1 /4 */
7932   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
7933   ins_pipe(ialu_reg);
7934 %}
7935 
7936 // Shift Left by one
7937 instruct salI_mem_1(memory dst, immI1 shift, rFlagsReg cr)
7938 %{
7939   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
7940   effect(KILL cr);
7941 
7942   format %{ "sall    $dst, $shift\t" %}
7943   opcode(0xD1, 0x4); /* D1 /4 */
7944   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
7945   ins_pipe(ialu_mem_imm);
7946 %}
7947 
7948 // Shift Left by 8-bit immediate
7949 instruct salI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
7950 %{
7951   match(Set dst (LShiftI dst shift));
7952   effect(KILL cr);
7953 
7954   format %{ "sall    $dst, $shift" %}
7955   opcode(0xC1, 0x4); /* C1 /4 ib */
7956   ins_encode(reg_opc_imm(dst, shift));
7957   ins_pipe(ialu_reg);
7958 %}
7959 
7960 // Shift Left by 8-bit immediate
7961 instruct salI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
7962 %{
7963   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
7964   effect(KILL cr);
7965 
7966   format %{ "sall    $dst, $shift" %}
7967   opcode(0xC1, 0x4); /* C1 /4 ib */
7968   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst), Con8or32(shift));
7969   ins_pipe(ialu_mem_imm);
7970 %}
7971 
7972 // Shift Left by variable
7973 instruct salI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
7974 %{
7975   match(Set dst (LShiftI dst shift));
7976   effect(KILL cr);
7977 
7978   format %{ "sall    $dst, $shift" %}
7979   opcode(0xD3, 0x4); /* D3 /4 */
7980   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
7981   ins_pipe(ialu_reg_reg);
7982 %}
7983 
7984 // Shift Left by variable
7985 instruct salI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
7986 %{
7987   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
7988   effect(KILL cr);
7989 
7990   format %{ "sall    $dst, $shift" %}
7991   opcode(0xD3, 0x4); /* D3 /4 */
7992   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
7993   ins_pipe(ialu_mem_reg);
7994 %}
7995 
7996 // Arithmetic shift right by one
7997 instruct sarI_rReg_1(rRegI dst, immI1 shift, rFlagsReg cr)
7998 %{
7999   match(Set dst (RShiftI dst shift));
8000   effect(KILL cr);
8001 
8002   format %{ "sarl    $dst, $shift" %}
8003   opcode(0xD1, 0x7); /* D1 /7 */
8004   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8005   ins_pipe(ialu_reg);
8006 %}
8007 
8008 // Arithmetic shift right by one
8009 instruct sarI_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8010 %{
8011   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
8012   effect(KILL cr);
8013 
8014   format %{ "sarl    $dst, $shift" %}
8015   opcode(0xD1, 0x7); /* D1 /7 */
8016   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8017   ins_pipe(ialu_mem_imm);
8018 %}
8019 
8020 // Arithmetic Shift Right by 8-bit immediate
8021 instruct sarI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
8022 %{
8023   match(Set dst (RShiftI dst shift));
8024   effect(KILL cr);
8025 
8026   format %{ "sarl    $dst, $shift" %}
8027   opcode(0xC1, 0x7); /* C1 /7 ib */
8028   ins_encode(reg_opc_imm(dst, shift));
8029   ins_pipe(ialu_mem_imm);
8030 %}
8031 
8032 // Arithmetic Shift Right by 8-bit immediate
8033 instruct sarI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8034 %{
8035   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
8036   effect(KILL cr);
8037 
8038   format %{ "sarl    $dst, $shift" %}
8039   opcode(0xC1, 0x7); /* C1 /7 ib */
8040   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst), Con8or32(shift));
8041   ins_pipe(ialu_mem_imm);
8042 %}
8043 
8044 // Arithmetic Shift Right by variable
8045 instruct sarI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
8046 %{
8047   match(Set dst (RShiftI dst shift));
8048   effect(KILL cr);
8049 
8050   format %{ "sarl    $dst, $shift" %}
8051   opcode(0xD3, 0x7); /* D3 /7 */
8052   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8053   ins_pipe(ialu_reg_reg);
8054 %}
8055 
8056 // Arithmetic Shift Right by variable
8057 instruct sarI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8058 %{
8059   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
8060   effect(KILL cr);
8061 
8062   format %{ "sarl    $dst, $shift" %}
8063   opcode(0xD3, 0x7); /* D3 /7 */
8064   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8065   ins_pipe(ialu_mem_reg);
8066 %}
8067 
8068 // Logical shift right by one
8069 instruct shrI_rReg_1(rRegI dst, immI1 shift, rFlagsReg cr)
8070 %{
8071   match(Set dst (URShiftI dst shift));
8072   effect(KILL cr);
8073 
8074   format %{ "shrl    $dst, $shift" %}
8075   opcode(0xD1, 0x5); /* D1 /5 */
8076   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8077   ins_pipe(ialu_reg);
8078 %}
8079 
8080 // Logical shift right by one
8081 instruct shrI_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8082 %{
8083   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
8084   effect(KILL cr);
8085 
8086   format %{ "shrl    $dst, $shift" %}
8087   opcode(0xD1, 0x5); /* D1 /5 */
8088   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8089   ins_pipe(ialu_mem_imm);
8090 %}
8091 
8092 // Logical Shift Right by 8-bit immediate
8093 instruct shrI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
8094 %{
8095   match(Set dst (URShiftI dst shift));
8096   effect(KILL cr);
8097 
8098   format %{ "shrl    $dst, $shift" %}
8099   opcode(0xC1, 0x5); /* C1 /5 ib */
8100   ins_encode(reg_opc_imm(dst, shift));
8101   ins_pipe(ialu_reg);
8102 %}
8103 
8104 // Logical Shift Right by 8-bit immediate
8105 instruct shrI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8106 %{
8107   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
8108   effect(KILL cr);
8109 
8110   format %{ "shrl    $dst, $shift" %}
8111   opcode(0xC1, 0x5); /* C1 /5 ib */
8112   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst), Con8or32(shift));
8113   ins_pipe(ialu_mem_imm);
8114 %}
8115 
8116 // Logical Shift Right by variable
8117 instruct shrI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
8118 %{
8119   match(Set dst (URShiftI dst shift));
8120   effect(KILL cr);
8121 
8122   format %{ "shrl    $dst, $shift" %}
8123   opcode(0xD3, 0x5); /* D3 /5 */
8124   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8125   ins_pipe(ialu_reg_reg);
8126 %}
8127 
8128 // Logical Shift Right by variable
8129 instruct shrI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8130 %{
8131   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
8132   effect(KILL cr);
8133 
8134   format %{ "shrl    $dst, $shift" %}
8135   opcode(0xD3, 0x5); /* D3 /5 */
8136   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8137   ins_pipe(ialu_mem_reg);
8138 %}
8139 
8140 // Long Shift Instructions
8141 // Shift Left by one
8142 instruct salL_rReg_1(rRegL dst, immI1 shift, rFlagsReg cr)
8143 %{
8144   match(Set dst (LShiftL dst shift));
8145   effect(KILL cr);
8146 
8147   format %{ "salq    $dst, $shift" %}
8148   opcode(0xD1, 0x4); /* D1 /4 */
8149   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8150   ins_pipe(ialu_reg);
8151 %}
8152 
8153 // Shift Left by one
8154 instruct salL_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8155 %{
8156   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
8157   effect(KILL cr);
8158 
8159   format %{ "salq    $dst, $shift" %}
8160   opcode(0xD1, 0x4); /* D1 /4 */
8161   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8162   ins_pipe(ialu_mem_imm);
8163 %}
8164 
8165 // Shift Left by 8-bit immediate
8166 instruct salL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
8167 %{
8168   match(Set dst (LShiftL dst shift));
8169   effect(KILL cr);
8170 
8171   format %{ "salq    $dst, $shift" %}
8172   opcode(0xC1, 0x4); /* C1 /4 ib */
8173   ins_encode(reg_opc_imm_wide(dst, shift));
8174   ins_pipe(ialu_reg);
8175 %}
8176 
8177 // Shift Left by 8-bit immediate
8178 instruct salL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8179 %{
8180   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
8181   effect(KILL cr);
8182 
8183   format %{ "salq    $dst, $shift" %}
8184   opcode(0xC1, 0x4); /* C1 /4 ib */
8185   ins_encode(REX_mem_wide(dst), OpcP,
8186              RM_opc_mem(secondary, dst), Con8or32(shift));
8187   ins_pipe(ialu_mem_imm);
8188 %}
8189 
8190 // Shift Left by variable
8191 instruct salL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
8192 %{
8193   match(Set dst (LShiftL dst shift));
8194   effect(KILL cr);
8195 
8196   format %{ "salq    $dst, $shift" %}
8197   opcode(0xD3, 0x4); /* D3 /4 */
8198   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8199   ins_pipe(ialu_reg_reg);
8200 %}
8201 
8202 // Shift Left by variable
8203 instruct salL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8204 %{
8205   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
8206   effect(KILL cr);
8207 
8208   format %{ "salq    $dst, $shift" %}
8209   opcode(0xD3, 0x4); /* D3 /4 */
8210   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8211   ins_pipe(ialu_mem_reg);
8212 %}
8213 
8214 // Arithmetic shift right by one
8215 instruct sarL_rReg_1(rRegL dst, immI1 shift, rFlagsReg cr)
8216 %{
8217   match(Set dst (RShiftL dst shift));
8218   effect(KILL cr);
8219 
8220   format %{ "sarq    $dst, $shift" %}
8221   opcode(0xD1, 0x7); /* D1 /7 */
8222   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8223   ins_pipe(ialu_reg);
8224 %}
8225 
8226 // Arithmetic shift right by one
8227 instruct sarL_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8228 %{
8229   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
8230   effect(KILL cr);
8231 
8232   format %{ "sarq    $dst, $shift" %}
8233   opcode(0xD1, 0x7); /* D1 /7 */
8234   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8235   ins_pipe(ialu_mem_imm);
8236 %}
8237 
8238 // Arithmetic Shift Right by 8-bit immediate
8239 instruct sarL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
8240 %{
8241   match(Set dst (RShiftL dst shift));
8242   effect(KILL cr);
8243 
8244   format %{ "sarq    $dst, $shift" %}
8245   opcode(0xC1, 0x7); /* C1 /7 ib */
8246   ins_encode(reg_opc_imm_wide(dst, shift));
8247   ins_pipe(ialu_mem_imm);
8248 %}
8249 
8250 // Arithmetic Shift Right by 8-bit immediate
8251 instruct sarL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8252 %{
8253   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
8254   effect(KILL cr);
8255 
8256   format %{ "sarq    $dst, $shift" %}
8257   opcode(0xC1, 0x7); /* C1 /7 ib */
8258   ins_encode(REX_mem_wide(dst), OpcP,
8259              RM_opc_mem(secondary, dst), Con8or32(shift));
8260   ins_pipe(ialu_mem_imm);
8261 %}
8262 
8263 // Arithmetic Shift Right by variable
8264 instruct sarL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
8265 %{
8266   match(Set dst (RShiftL dst shift));
8267   effect(KILL cr);
8268 
8269   format %{ "sarq    $dst, $shift" %}
8270   opcode(0xD3, 0x7); /* D3 /7 */
8271   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8272   ins_pipe(ialu_reg_reg);
8273 %}
8274 
8275 // Arithmetic Shift Right by variable
8276 instruct sarL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8277 %{
8278   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
8279   effect(KILL cr);
8280 
8281   format %{ "sarq    $dst, $shift" %}
8282   opcode(0xD3, 0x7); /* D3 /7 */
8283   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8284   ins_pipe(ialu_mem_reg);
8285 %}
8286 
8287 // Logical shift right by one
8288 instruct shrL_rReg_1(rRegL dst, immI1 shift, rFlagsReg cr)
8289 %{
8290   match(Set dst (URShiftL dst shift));
8291   effect(KILL cr);
8292 
8293   format %{ "shrq    $dst, $shift" %}
8294   opcode(0xD1, 0x5); /* D1 /5 */
8295   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst ));
8296   ins_pipe(ialu_reg);
8297 %}
8298 
8299 // Logical shift right by one
8300 instruct shrL_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8301 %{
8302   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
8303   effect(KILL cr);
8304 
8305   format %{ "shrq    $dst, $shift" %}
8306   opcode(0xD1, 0x5); /* D1 /5 */
8307   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8308   ins_pipe(ialu_mem_imm);
8309 %}
8310 
8311 // Logical Shift Right by 8-bit immediate
8312 instruct shrL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
8313 %{
8314   match(Set dst (URShiftL dst shift));
8315   effect(KILL cr);
8316 
8317   format %{ "shrq    $dst, $shift" %}
8318   opcode(0xC1, 0x5); /* C1 /5 ib */
8319   ins_encode(reg_opc_imm_wide(dst, shift));
8320   ins_pipe(ialu_reg);
8321 %}
8322 
8323 
8324 // Logical Shift Right by 8-bit immediate
8325 instruct shrL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8326 %{
8327   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
8328   effect(KILL cr);
8329 
8330   format %{ "shrq    $dst, $shift" %}
8331   opcode(0xC1, 0x5); /* C1 /5 ib */
8332   ins_encode(REX_mem_wide(dst), OpcP,
8333              RM_opc_mem(secondary, dst), Con8or32(shift));
8334   ins_pipe(ialu_mem_imm);
8335 %}
8336 
8337 // Logical Shift Right by variable
8338 instruct shrL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
8339 %{
8340   match(Set dst (URShiftL dst shift));
8341   effect(KILL cr);
8342 
8343   format %{ "shrq    $dst, $shift" %}
8344   opcode(0xD3, 0x5); /* D3 /5 */
8345   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8346   ins_pipe(ialu_reg_reg);
8347 %}
8348 
8349 // Logical Shift Right by variable
8350 instruct shrL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8351 %{
8352   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
8353   effect(KILL cr);
8354 
8355   format %{ "shrq    $dst, $shift" %}
8356   opcode(0xD3, 0x5); /* D3 /5 */
8357   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8358   ins_pipe(ialu_mem_reg);
8359 %}
8360 
8361 // Logical Shift Right by 24, followed by Arithmetic Shift Left by 24.
8362 // This idiom is used by the compiler for the i2b bytecode.
8363 instruct i2b(rRegI dst, rRegI src, immI_24 twentyfour)
8364 %{
8365   match(Set dst (RShiftI (LShiftI src twentyfour) twentyfour));
8366 
8367   format %{ "movsbl  $dst, $src\t# i2b" %}
8368   opcode(0x0F, 0xBE);
8369   ins_encode(REX_reg_breg(dst, src), OpcP, OpcS, reg_reg(dst, src));
8370   ins_pipe(ialu_reg_reg);
8371 %}
8372 
8373 // Logical Shift Right by 16, followed by Arithmetic Shift Left by 16.
8374 // This idiom is used by the compiler the i2s bytecode.
8375 instruct i2s(rRegI dst, rRegI src, immI_16 sixteen)
8376 %{
8377   match(Set dst (RShiftI (LShiftI src sixteen) sixteen));
8378 
8379   format %{ "movswl  $dst, $src\t# i2s" %}
8380   opcode(0x0F, 0xBF);
8381   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
8382   ins_pipe(ialu_reg_reg);
8383 %}
8384 
8385 // ROL/ROR instructions
8386 
8387 // ROL expand
8388 instruct rolI_rReg_imm1(rRegI dst, rFlagsReg cr) %{
8389   effect(KILL cr, USE_DEF dst);
8390 
8391   format %{ "roll    $dst" %}
8392   opcode(0xD1, 0x0); /* Opcode  D1 /0 */
8393   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8394   ins_pipe(ialu_reg);
8395 %}
8396 
8397 instruct rolI_rReg_imm8(rRegI dst, immI8 shift, rFlagsReg cr) %{
8398   effect(USE_DEF dst, USE shift, KILL cr);
8399 
8400   format %{ "roll    $dst, $shift" %}
8401   opcode(0xC1, 0x0); /* Opcode C1 /0 ib */
8402   ins_encode( reg_opc_imm(dst, shift) );
8403   ins_pipe(ialu_reg);
8404 %}
8405 
8406 instruct rolI_rReg_CL(no_rcx_RegI dst, rcx_RegI shift, rFlagsReg cr)
8407 %{
8408   effect(USE_DEF dst, USE shift, KILL cr);
8409 
8410   format %{ "roll    $dst, $shift" %}
8411   opcode(0xD3, 0x0); /* Opcode D3 /0 */
8412   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8413   ins_pipe(ialu_reg_reg);
8414 %}
8415 // end of ROL expand
8416 
8417 // Rotate Left by one
8418 instruct rolI_rReg_i1(rRegI dst, immI1 lshift, immI_M1 rshift, rFlagsReg cr)
8419 %{
8420   match(Set dst (OrI (LShiftI dst lshift) (URShiftI dst rshift)));
8421 
8422   expand %{
8423     rolI_rReg_imm1(dst, cr);
8424   %}
8425 %}
8426 
8427 // Rotate Left by 8-bit immediate
8428 instruct rolI_rReg_i8(rRegI dst, immI8 lshift, immI8 rshift, rFlagsReg cr)
8429 %{
8430   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x1f));
8431   match(Set dst (OrI (LShiftI dst lshift) (URShiftI dst rshift)));
8432 
8433   expand %{
8434     rolI_rReg_imm8(dst, lshift, cr);
8435   %}
8436 %}
8437 
8438 // Rotate Left by variable
8439 instruct rolI_rReg_Var_C0(no_rcx_RegI dst, rcx_RegI shift, immI0 zero, rFlagsReg cr)
8440 %{
8441   match(Set dst (OrI (LShiftI dst shift) (URShiftI dst (SubI zero shift))));
8442 
8443   expand %{
8444     rolI_rReg_CL(dst, shift, cr);
8445   %}
8446 %}
8447 
8448 // Rotate Left by variable
8449 instruct rolI_rReg_Var_C32(no_rcx_RegI dst, rcx_RegI shift, immI_32 c32, rFlagsReg cr)
8450 %{
8451   match(Set dst (OrI (LShiftI dst shift) (URShiftI dst (SubI c32 shift))));
8452 
8453   expand %{
8454     rolI_rReg_CL(dst, shift, cr);
8455   %}
8456 %}
8457 
8458 // ROR expand
8459 instruct rorI_rReg_imm1(rRegI dst, rFlagsReg cr)
8460 %{
8461   effect(USE_DEF dst, KILL cr);
8462 
8463   format %{ "rorl    $dst" %}
8464   opcode(0xD1, 0x1); /* D1 /1 */
8465   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8466   ins_pipe(ialu_reg);
8467 %}
8468 
8469 instruct rorI_rReg_imm8(rRegI dst, immI8 shift, rFlagsReg cr)
8470 %{
8471   effect(USE_DEF dst, USE shift, KILL cr);
8472 
8473   format %{ "rorl    $dst, $shift" %}
8474   opcode(0xC1, 0x1); /* C1 /1 ib */
8475   ins_encode(reg_opc_imm(dst, shift));
8476   ins_pipe(ialu_reg);
8477 %}
8478 
8479 instruct rorI_rReg_CL(no_rcx_RegI dst, rcx_RegI shift, rFlagsReg cr)
8480 %{
8481   effect(USE_DEF dst, USE shift, KILL cr);
8482 
8483   format %{ "rorl    $dst, $shift" %}
8484   opcode(0xD3, 0x1); /* D3 /1 */
8485   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8486   ins_pipe(ialu_reg_reg);
8487 %}
8488 // end of ROR expand
8489 
8490 // Rotate Right by one
8491 instruct rorI_rReg_i1(rRegI dst, immI1 rshift, immI_M1 lshift, rFlagsReg cr)
8492 %{
8493   match(Set dst (OrI (URShiftI dst rshift) (LShiftI dst lshift)));
8494 
8495   expand %{
8496     rorI_rReg_imm1(dst, cr);
8497   %}
8498 %}
8499 
8500 // Rotate Right by 8-bit immediate
8501 instruct rorI_rReg_i8(rRegI dst, immI8 rshift, immI8 lshift, rFlagsReg cr)
8502 %{
8503   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x1f));
8504   match(Set dst (OrI (URShiftI dst rshift) (LShiftI dst lshift)));
8505 
8506   expand %{
8507     rorI_rReg_imm8(dst, rshift, cr);
8508   %}
8509 %}
8510 
8511 // Rotate Right by variable
8512 instruct rorI_rReg_Var_C0(no_rcx_RegI dst, rcx_RegI shift, immI0 zero, rFlagsReg cr)
8513 %{
8514   match(Set dst (OrI (URShiftI dst shift) (LShiftI dst (SubI zero shift))));
8515 
8516   expand %{
8517     rorI_rReg_CL(dst, shift, cr);
8518   %}
8519 %}
8520 
8521 // Rotate Right by variable
8522 instruct rorI_rReg_Var_C32(no_rcx_RegI dst, rcx_RegI shift, immI_32 c32, rFlagsReg cr)
8523 %{
8524   match(Set dst (OrI (URShiftI dst shift) (LShiftI dst (SubI c32 shift))));
8525 
8526   expand %{
8527     rorI_rReg_CL(dst, shift, cr);
8528   %}
8529 %}
8530 
8531 // for long rotate
8532 // ROL expand
8533 instruct rolL_rReg_imm1(rRegL dst, rFlagsReg cr) %{
8534   effect(USE_DEF dst, KILL cr);
8535 
8536   format %{ "rolq    $dst" %}
8537   opcode(0xD1, 0x0); /* Opcode  D1 /0 */
8538   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8539   ins_pipe(ialu_reg);
8540 %}
8541 
8542 instruct rolL_rReg_imm8(rRegL dst, immI8 shift, rFlagsReg cr) %{
8543   effect(USE_DEF dst, USE shift, KILL cr);
8544 
8545   format %{ "rolq    $dst, $shift" %}
8546   opcode(0xC1, 0x0); /* Opcode C1 /0 ib */
8547   ins_encode( reg_opc_imm_wide(dst, shift) );
8548   ins_pipe(ialu_reg);
8549 %}
8550 
8551 instruct rolL_rReg_CL(no_rcx_RegL dst, rcx_RegI shift, rFlagsReg cr)
8552 %{
8553   effect(USE_DEF dst, USE shift, KILL cr);
8554 
8555   format %{ "rolq    $dst, $shift" %}
8556   opcode(0xD3, 0x0); /* Opcode D3 /0 */
8557   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8558   ins_pipe(ialu_reg_reg);
8559 %}
8560 // end of ROL expand
8561 
8562 // Rotate Left by one
8563 instruct rolL_rReg_i1(rRegL dst, immI1 lshift, immI_M1 rshift, rFlagsReg cr)
8564 %{
8565   match(Set dst (OrL (LShiftL dst lshift) (URShiftL dst rshift)));
8566 
8567   expand %{
8568     rolL_rReg_imm1(dst, cr);
8569   %}
8570 %}
8571 
8572 // Rotate Left by 8-bit immediate
8573 instruct rolL_rReg_i8(rRegL dst, immI8 lshift, immI8 rshift, rFlagsReg cr)
8574 %{
8575   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x3f));
8576   match(Set dst (OrL (LShiftL dst lshift) (URShiftL dst rshift)));
8577 
8578   expand %{
8579     rolL_rReg_imm8(dst, lshift, cr);
8580   %}
8581 %}
8582 
8583 // Rotate Left by variable
8584 instruct rolL_rReg_Var_C0(no_rcx_RegL dst, rcx_RegI shift, immI0 zero, rFlagsReg cr)
8585 %{
8586   match(Set dst (OrL (LShiftL dst shift) (URShiftL dst (SubI zero shift))));
8587 
8588   expand %{
8589     rolL_rReg_CL(dst, shift, cr);
8590   %}
8591 %}
8592 
8593 // Rotate Left by variable
8594 instruct rolL_rReg_Var_C64(no_rcx_RegL dst, rcx_RegI shift, immI_64 c64, rFlagsReg cr)
8595 %{
8596   match(Set dst (OrL (LShiftL dst shift) (URShiftL dst (SubI c64 shift))));
8597 
8598   expand %{
8599     rolL_rReg_CL(dst, shift, cr);
8600   %}
8601 %}
8602 
8603 // ROR expand
8604 instruct rorL_rReg_imm1(rRegL dst, rFlagsReg cr)
8605 %{
8606   effect(USE_DEF dst, KILL cr);
8607 
8608   format %{ "rorq    $dst" %}
8609   opcode(0xD1, 0x1); /* D1 /1 */
8610   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8611   ins_pipe(ialu_reg);
8612 %}
8613 
8614 instruct rorL_rReg_imm8(rRegL dst, immI8 shift, rFlagsReg cr)
8615 %{
8616   effect(USE_DEF dst, USE shift, KILL cr);
8617 
8618   format %{ "rorq    $dst, $shift" %}
8619   opcode(0xC1, 0x1); /* C1 /1 ib */
8620   ins_encode(reg_opc_imm_wide(dst, shift));
8621   ins_pipe(ialu_reg);
8622 %}
8623 
8624 instruct rorL_rReg_CL(no_rcx_RegL dst, rcx_RegI shift, rFlagsReg cr)
8625 %{
8626   effect(USE_DEF dst, USE shift, KILL cr);
8627 
8628   format %{ "rorq    $dst, $shift" %}
8629   opcode(0xD3, 0x1); /* D3 /1 */
8630   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8631   ins_pipe(ialu_reg_reg);
8632 %}
8633 // end of ROR expand
8634 
8635 // Rotate Right by one
8636 instruct rorL_rReg_i1(rRegL dst, immI1 rshift, immI_M1 lshift, rFlagsReg cr)
8637 %{
8638   match(Set dst (OrL (URShiftL dst rshift) (LShiftL dst lshift)));
8639 
8640   expand %{
8641     rorL_rReg_imm1(dst, cr);
8642   %}
8643 %}
8644 
8645 // Rotate Right by 8-bit immediate
8646 instruct rorL_rReg_i8(rRegL dst, immI8 rshift, immI8 lshift, rFlagsReg cr)
8647 %{
8648   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x3f));
8649   match(Set dst (OrL (URShiftL dst rshift) (LShiftL dst lshift)));
8650 
8651   expand %{
8652     rorL_rReg_imm8(dst, rshift, cr);
8653   %}
8654 %}
8655 
8656 // Rotate Right by variable
8657 instruct rorL_rReg_Var_C0(no_rcx_RegL dst, rcx_RegI shift, immI0 zero, rFlagsReg cr)
8658 %{
8659   match(Set dst (OrL (URShiftL dst shift) (LShiftL dst (SubI zero shift))));
8660 
8661   expand %{
8662     rorL_rReg_CL(dst, shift, cr);
8663   %}
8664 %}
8665 
8666 // Rotate Right by variable
8667 instruct rorL_rReg_Var_C64(no_rcx_RegL dst, rcx_RegI shift, immI_64 c64, rFlagsReg cr)
8668 %{
8669   match(Set dst (OrL (URShiftL dst shift) (LShiftL dst (SubI c64 shift))));
8670 
8671   expand %{
8672     rorL_rReg_CL(dst, shift, cr);
8673   %}
8674 %}
8675 
8676 // Logical Instructions
8677 
8678 // Integer Logical Instructions
8679 
8680 // And Instructions
8681 // And Register with Register
8682 instruct andI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
8683 %{
8684   match(Set dst (AndI dst src));
8685   effect(KILL cr);
8686 
8687   format %{ "andl    $dst, $src\t# int" %}
8688   opcode(0x23);
8689   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
8690   ins_pipe(ialu_reg_reg);
8691 %}
8692 
8693 // And Register with Immediate 255
8694 instruct andI_rReg_imm255(rRegI dst, immI_255 src)
8695 %{
8696   match(Set dst (AndI dst src));
8697 
8698   format %{ "movzbl  $dst, $dst\t# int & 0xFF" %}
8699   opcode(0x0F, 0xB6);
8700   ins_encode(REX_reg_breg(dst, dst), OpcP, OpcS, reg_reg(dst, dst));
8701   ins_pipe(ialu_reg);
8702 %}
8703 
8704 // And Register with Immediate 255 and promote to long
8705 instruct andI2L_rReg_imm255(rRegL dst, rRegI src, immI_255 mask)
8706 %{
8707   match(Set dst (ConvI2L (AndI src mask)));
8708 
8709   format %{ "movzbl  $dst, $src\t# int & 0xFF -> long" %}
8710   opcode(0x0F, 0xB6);
8711   ins_encode(REX_reg_breg(dst, src), OpcP, OpcS, reg_reg(dst, src));
8712   ins_pipe(ialu_reg);
8713 %}
8714 
8715 // And Register with Immediate 65535
8716 instruct andI_rReg_imm65535(rRegI dst, immI_65535 src)
8717 %{
8718   match(Set dst (AndI dst src));
8719 
8720   format %{ "movzwl  $dst, $dst\t# int & 0xFFFF" %}
8721   opcode(0x0F, 0xB7);
8722   ins_encode(REX_reg_reg(dst, dst), OpcP, OpcS, reg_reg(dst, dst));
8723   ins_pipe(ialu_reg);
8724 %}
8725 
8726 // And Register with Immediate 65535 and promote to long
8727 instruct andI2L_rReg_imm65535(rRegL dst, rRegI src, immI_65535 mask)
8728 %{
8729   match(Set dst (ConvI2L (AndI src mask)));
8730 
8731   format %{ "movzwl  $dst, $src\t# int & 0xFFFF -> long" %}
8732   opcode(0x0F, 0xB7);
8733   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
8734   ins_pipe(ialu_reg);
8735 %}
8736 
8737 // And Register with Immediate
8738 instruct andI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
8739 %{
8740   match(Set dst (AndI dst src));
8741   effect(KILL cr);
8742 
8743   format %{ "andl    $dst, $src\t# int" %}
8744   opcode(0x81, 0x04); /* Opcode 81 /4 */
8745   ins_encode(OpcSErm(dst, src), Con8or32(src));
8746   ins_pipe(ialu_reg);
8747 %}
8748 
8749 // And Register with Memory
8750 instruct andI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
8751 %{
8752   match(Set dst (AndI dst (LoadI src)));
8753   effect(KILL cr);
8754 
8755   ins_cost(125);
8756   format %{ "andl    $dst, $src\t# int" %}
8757   opcode(0x23);
8758   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
8759   ins_pipe(ialu_reg_mem);
8760 %}
8761 
8762 // And Memory with Register
8763 instruct andI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
8764 %{
8765   match(Set dst (StoreI dst (AndI (LoadI dst) src)));
8766   effect(KILL cr);
8767 
8768   ins_cost(150);
8769   format %{ "andl    $dst, $src\t# int" %}
8770   opcode(0x21); /* Opcode 21 /r */
8771   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
8772   ins_pipe(ialu_mem_reg);
8773 %}
8774 
8775 // And Memory with Immediate
8776 instruct andI_mem_imm(memory dst, immI src, rFlagsReg cr)
8777 %{
8778   match(Set dst (StoreI dst (AndI (LoadI dst) src)));
8779   effect(KILL cr);
8780 
8781   ins_cost(125);
8782   format %{ "andl    $dst, $src\t# int" %}
8783   opcode(0x81, 0x4); /* Opcode 81 /4 id */
8784   ins_encode(REX_mem(dst), OpcSE(src),
8785              RM_opc_mem(secondary, dst), Con8or32(src));
8786   ins_pipe(ialu_mem_imm);
8787 %}
8788 
8789 // BMI1 instructions
8790 instruct andnI_rReg_rReg_mem(rRegI dst, rRegI src1, memory src2, immI_M1 minus_1, rFlagsReg cr) %{
8791   match(Set dst (AndI (XorI src1 minus_1) (LoadI src2)));
8792   predicate(UseBMI1Instructions);
8793   effect(KILL cr);
8794 
8795   ins_cost(125);
8796   format %{ "andnl  $dst, $src1, $src2" %}
8797 
8798   ins_encode %{
8799     __ andnl($dst$$Register, $src1$$Register, $src2$$Address);
8800   %}
8801   ins_pipe(ialu_reg_mem);
8802 %}
8803 
8804 instruct andnI_rReg_rReg_rReg(rRegI dst, rRegI src1, rRegI src2, immI_M1 minus_1, rFlagsReg cr) %{
8805   match(Set dst (AndI (XorI src1 minus_1) src2));
8806   predicate(UseBMI1Instructions);
8807   effect(KILL cr);
8808 
8809   format %{ "andnl  $dst, $src1, $src2" %}
8810 
8811   ins_encode %{
8812     __ andnl($dst$$Register, $src1$$Register, $src2$$Register);
8813   %}
8814   ins_pipe(ialu_reg);
8815 %}
8816 
8817 instruct blsiI_rReg_rReg(rRegI dst, rRegI src, immI0 imm_zero, rFlagsReg cr) %{
8818   match(Set dst (AndI (SubI imm_zero src) src));
8819   predicate(UseBMI1Instructions);
8820   effect(KILL cr);
8821 
8822   format %{ "blsil  $dst, $src" %}
8823 
8824   ins_encode %{
8825     __ blsil($dst$$Register, $src$$Register);
8826   %}
8827   ins_pipe(ialu_reg);
8828 %}
8829 
8830 instruct blsiI_rReg_mem(rRegI dst, memory src, immI0 imm_zero, rFlagsReg cr) %{
8831   match(Set dst (AndI (SubI imm_zero (LoadI src) ) (LoadI src) ));
8832   predicate(UseBMI1Instructions);
8833   effect(KILL cr);
8834 
8835   ins_cost(125);
8836   format %{ "blsil  $dst, $src" %}
8837 
8838   ins_encode %{
8839     __ blsil($dst$$Register, $src$$Address);
8840   %}
8841   ins_pipe(ialu_reg_mem);
8842 %}
8843 
8844 instruct blsmskI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, rFlagsReg cr)
8845 %{
8846   match(Set dst (XorI (AddI (LoadI src) minus_1) (LoadI src) ) );
8847   predicate(UseBMI1Instructions);
8848   effect(KILL cr);
8849 
8850   ins_cost(125);
8851   format %{ "blsmskl $dst, $src" %}
8852 
8853   ins_encode %{
8854     __ blsmskl($dst$$Register, $src$$Address);
8855   %}
8856   ins_pipe(ialu_reg_mem);
8857 %}
8858 
8859 instruct blsmskI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, rFlagsReg cr)
8860 %{
8861   match(Set dst (XorI (AddI src minus_1) src));
8862   predicate(UseBMI1Instructions);
8863   effect(KILL cr);
8864 
8865   format %{ "blsmskl $dst, $src" %}
8866 
8867   ins_encode %{
8868     __ blsmskl($dst$$Register, $src$$Register);
8869   %}
8870 
8871   ins_pipe(ialu_reg);
8872 %}
8873 
8874 instruct blsrI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, rFlagsReg cr)
8875 %{
8876   match(Set dst (AndI (AddI src minus_1) src) );
8877   predicate(UseBMI1Instructions);
8878   effect(KILL cr);
8879 
8880   format %{ "blsrl  $dst, $src" %}
8881 
8882   ins_encode %{
8883     __ blsrl($dst$$Register, $src$$Register);
8884   %}
8885 
8886   ins_pipe(ialu_reg_mem);
8887 %}
8888 
8889 instruct blsrI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, rFlagsReg cr)
8890 %{
8891   match(Set dst (AndI (AddI (LoadI src) minus_1) (LoadI src) ) );
8892   predicate(UseBMI1Instructions);
8893   effect(KILL cr);
8894 
8895   ins_cost(125);
8896   format %{ "blsrl  $dst, $src" %}
8897 
8898   ins_encode %{
8899     __ blsrl($dst$$Register, $src$$Address);
8900   %}
8901 
8902   ins_pipe(ialu_reg);
8903 %}
8904 
8905 // Or Instructions
8906 // Or Register with Register
8907 instruct orI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
8908 %{
8909   match(Set dst (OrI dst src));
8910   effect(KILL cr);
8911 
8912   format %{ "orl     $dst, $src\t# int" %}
8913   opcode(0x0B);
8914   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
8915   ins_pipe(ialu_reg_reg);
8916 %}
8917 
8918 // Or Register with Immediate
8919 instruct orI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
8920 %{
8921   match(Set dst (OrI dst src));
8922   effect(KILL cr);
8923 
8924   format %{ "orl     $dst, $src\t# int" %}
8925   opcode(0x81, 0x01); /* Opcode 81 /1 id */
8926   ins_encode(OpcSErm(dst, src), Con8or32(src));
8927   ins_pipe(ialu_reg);
8928 %}
8929 
8930 // Or Register with Memory
8931 instruct orI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
8932 %{
8933   match(Set dst (OrI dst (LoadI src)));
8934   effect(KILL cr);
8935 
8936   ins_cost(125);
8937   format %{ "orl     $dst, $src\t# int" %}
8938   opcode(0x0B);
8939   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
8940   ins_pipe(ialu_reg_mem);
8941 %}
8942 
8943 // Or Memory with Register
8944 instruct orI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
8945 %{
8946   match(Set dst (StoreI dst (OrI (LoadI dst) src)));
8947   effect(KILL cr);
8948 
8949   ins_cost(150);
8950   format %{ "orl     $dst, $src\t# int" %}
8951   opcode(0x09); /* Opcode 09 /r */
8952   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
8953   ins_pipe(ialu_mem_reg);
8954 %}
8955 
8956 // Or Memory with Immediate
8957 instruct orI_mem_imm(memory dst, immI src, rFlagsReg cr)
8958 %{
8959   match(Set dst (StoreI dst (OrI (LoadI dst) src)));
8960   effect(KILL cr);
8961 
8962   ins_cost(125);
8963   format %{ "orl     $dst, $src\t# int" %}
8964   opcode(0x81, 0x1); /* Opcode 81 /1 id */
8965   ins_encode(REX_mem(dst), OpcSE(src),
8966              RM_opc_mem(secondary, dst), Con8or32(src));
8967   ins_pipe(ialu_mem_imm);
8968 %}
8969 
8970 // Xor Instructions
8971 // Xor Register with Register
8972 instruct xorI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
8973 %{
8974   match(Set dst (XorI dst src));
8975   effect(KILL cr);
8976 
8977   format %{ "xorl    $dst, $src\t# int" %}
8978   opcode(0x33);
8979   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
8980   ins_pipe(ialu_reg_reg);
8981 %}
8982 
8983 // Xor Register with Immediate -1
8984 instruct xorI_rReg_im1(rRegI dst, immI_M1 imm) %{
8985   match(Set dst (XorI dst imm));
8986 
8987   format %{ "not    $dst" %}
8988   ins_encode %{
8989      __ notl($dst$$Register);
8990   %}
8991   ins_pipe(ialu_reg);
8992 %}
8993 
8994 // Xor Register with Immediate
8995 instruct xorI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
8996 %{
8997   match(Set dst (XorI dst src));
8998   effect(KILL cr);
8999 
9000   format %{ "xorl    $dst, $src\t# int" %}
9001   opcode(0x81, 0x06); /* Opcode 81 /6 id */
9002   ins_encode(OpcSErm(dst, src), Con8or32(src));
9003   ins_pipe(ialu_reg);
9004 %}
9005 
9006 // Xor Register with Memory
9007 instruct xorI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
9008 %{
9009   match(Set dst (XorI dst (LoadI src)));
9010   effect(KILL cr);
9011 
9012   ins_cost(125);
9013   format %{ "xorl    $dst, $src\t# int" %}
9014   opcode(0x33);
9015   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
9016   ins_pipe(ialu_reg_mem);
9017 %}
9018 
9019 // Xor Memory with Register
9020 instruct xorI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
9021 %{
9022   match(Set dst (StoreI dst (XorI (LoadI dst) src)));
9023   effect(KILL cr);
9024 
9025   ins_cost(150);
9026   format %{ "xorl    $dst, $src\t# int" %}
9027   opcode(0x31); /* Opcode 31 /r */
9028   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
9029   ins_pipe(ialu_mem_reg);
9030 %}
9031 
9032 // Xor Memory with Immediate
9033 instruct xorI_mem_imm(memory dst, immI src, rFlagsReg cr)
9034 %{
9035   match(Set dst (StoreI dst (XorI (LoadI dst) src)));
9036   effect(KILL cr);
9037 
9038   ins_cost(125);
9039   format %{ "xorl    $dst, $src\t# int" %}
9040   opcode(0x81, 0x6); /* Opcode 81 /6 id */
9041   ins_encode(REX_mem(dst), OpcSE(src),
9042              RM_opc_mem(secondary, dst), Con8or32(src));
9043   ins_pipe(ialu_mem_imm);
9044 %}
9045 
9046 
9047 // Long Logical Instructions
9048 
9049 // And Instructions
9050 // And Register with Register
9051 instruct andL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
9052 %{
9053   match(Set dst (AndL dst src));
9054   effect(KILL cr);
9055 
9056   format %{ "andq    $dst, $src\t# long" %}
9057   opcode(0x23);
9058   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
9059   ins_pipe(ialu_reg_reg);
9060 %}
9061 
9062 // And Register with Immediate 255
9063 instruct andL_rReg_imm255(rRegL dst, immL_255 src)
9064 %{
9065   match(Set dst (AndL dst src));
9066 
9067   format %{ "movzbq  $dst, $dst\t# long & 0xFF" %}
9068   opcode(0x0F, 0xB6);
9069   ins_encode(REX_reg_reg_wide(dst, dst), OpcP, OpcS, reg_reg(dst, dst));
9070   ins_pipe(ialu_reg);
9071 %}
9072 
9073 // And Register with Immediate 65535
9074 instruct andL_rReg_imm65535(rRegL dst, immL_65535 src)
9075 %{
9076   match(Set dst (AndL dst src));
9077 
9078   format %{ "movzwq  $dst, $dst\t# long & 0xFFFF" %}
9079   opcode(0x0F, 0xB7);
9080   ins_encode(REX_reg_reg_wide(dst, dst), OpcP, OpcS, reg_reg(dst, dst));
9081   ins_pipe(ialu_reg);
9082 %}
9083 
9084 // And Register with Immediate
9085 instruct andL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
9086 %{
9087   match(Set dst (AndL dst src));
9088   effect(KILL cr);
9089 
9090   format %{ "andq    $dst, $src\t# long" %}
9091   opcode(0x81, 0x04); /* Opcode 81 /4 */
9092   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
9093   ins_pipe(ialu_reg);
9094 %}
9095 
9096 // And Register with Memory
9097 instruct andL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
9098 %{
9099   match(Set dst (AndL dst (LoadL src)));
9100   effect(KILL cr);
9101 
9102   ins_cost(125);
9103   format %{ "andq    $dst, $src\t# long" %}
9104   opcode(0x23);
9105   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
9106   ins_pipe(ialu_reg_mem);
9107 %}
9108 
9109 // And Memory with Register
9110 instruct andL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
9111 %{
9112   match(Set dst (StoreL dst (AndL (LoadL dst) src)));
9113   effect(KILL cr);
9114 
9115   ins_cost(150);
9116   format %{ "andq    $dst, $src\t# long" %}
9117   opcode(0x21); /* Opcode 21 /r */
9118   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
9119   ins_pipe(ialu_mem_reg);
9120 %}
9121 
9122 // And Memory with Immediate
9123 instruct andL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
9124 %{
9125   match(Set dst (StoreL dst (AndL (LoadL dst) src)));
9126   effect(KILL cr);
9127 
9128   ins_cost(125);
9129   format %{ "andq    $dst, $src\t# long" %}
9130   opcode(0x81, 0x4); /* Opcode 81 /4 id */
9131   ins_encode(REX_mem_wide(dst), OpcSE(src),
9132              RM_opc_mem(secondary, dst), Con8or32(src));
9133   ins_pipe(ialu_mem_imm);
9134 %}
9135 
9136 // BMI1 instructions
9137 instruct andnL_rReg_rReg_mem(rRegL dst, rRegL src1, memory src2, immL_M1 minus_1, rFlagsReg cr) %{
9138   match(Set dst (AndL (XorL src1 minus_1) (LoadL src2)));
9139   predicate(UseBMI1Instructions);
9140   effect(KILL cr);
9141 
9142   ins_cost(125);
9143   format %{ "andnq  $dst, $src1, $src2" %}
9144 
9145   ins_encode %{
9146     __ andnq($dst$$Register, $src1$$Register, $src2$$Address);
9147   %}
9148   ins_pipe(ialu_reg_mem);
9149 %}
9150 
9151 instruct andnL_rReg_rReg_rReg(rRegL dst, rRegL src1, rRegL src2, immL_M1 minus_1, rFlagsReg cr) %{
9152   match(Set dst (AndL (XorL src1 minus_1) src2));
9153   predicate(UseBMI1Instructions);
9154   effect(KILL cr);
9155 
9156   format %{ "andnq  $dst, $src1, $src2" %}
9157 
9158   ins_encode %{
9159   __ andnq($dst$$Register, $src1$$Register, $src2$$Register);
9160   %}
9161   ins_pipe(ialu_reg_mem);
9162 %}
9163 
9164 instruct blsiL_rReg_rReg(rRegL dst, rRegL src, immL0 imm_zero, rFlagsReg cr) %{
9165   match(Set dst (AndL (SubL imm_zero src) src));
9166   predicate(UseBMI1Instructions);
9167   effect(KILL cr);
9168 
9169   format %{ "blsiq  $dst, $src" %}
9170 
9171   ins_encode %{
9172     __ blsiq($dst$$Register, $src$$Register);
9173   %}
9174   ins_pipe(ialu_reg);
9175 %}
9176 
9177 instruct blsiL_rReg_mem(rRegL dst, memory src, immL0 imm_zero, rFlagsReg cr) %{
9178   match(Set dst (AndL (SubL imm_zero (LoadL src) ) (LoadL src) ));
9179   predicate(UseBMI1Instructions);
9180   effect(KILL cr);
9181 
9182   ins_cost(125);
9183   format %{ "blsiq  $dst, $src" %}
9184 
9185   ins_encode %{
9186     __ blsiq($dst$$Register, $src$$Address);
9187   %}
9188   ins_pipe(ialu_reg_mem);
9189 %}
9190 
9191 instruct blsmskL_rReg_mem(rRegL dst, memory src, immL_M1 minus_1, rFlagsReg cr)
9192 %{
9193   match(Set dst (XorL (AddL (LoadL src) minus_1) (LoadL src) ) );
9194   predicate(UseBMI1Instructions);
9195   effect(KILL cr);
9196 
9197   ins_cost(125);
9198   format %{ "blsmskq $dst, $src" %}
9199 
9200   ins_encode %{
9201     __ blsmskq($dst$$Register, $src$$Address);
9202   %}
9203   ins_pipe(ialu_reg_mem);
9204 %}
9205 
9206 instruct blsmskL_rReg_rReg(rRegL dst, rRegL src, immL_M1 minus_1, rFlagsReg cr)
9207 %{
9208   match(Set dst (XorL (AddL src minus_1) src));
9209   predicate(UseBMI1Instructions);
9210   effect(KILL cr);
9211 
9212   format %{ "blsmskq $dst, $src" %}
9213 
9214   ins_encode %{
9215     __ blsmskq($dst$$Register, $src$$Register);
9216   %}
9217 
9218   ins_pipe(ialu_reg);
9219 %}
9220 
9221 instruct blsrL_rReg_rReg(rRegL dst, rRegL src, immL_M1 minus_1, rFlagsReg cr)
9222 %{
9223   match(Set dst (AndL (AddL src minus_1) src) );
9224   predicate(UseBMI1Instructions);
9225   effect(KILL cr);
9226 
9227   format %{ "blsrq  $dst, $src" %}
9228 
9229   ins_encode %{
9230     __ blsrq($dst$$Register, $src$$Register);
9231   %}
9232 
9233   ins_pipe(ialu_reg);
9234 %}
9235 
9236 instruct blsrL_rReg_mem(rRegL dst, memory src, immL_M1 minus_1, rFlagsReg cr)
9237 %{
9238   match(Set dst (AndL (AddL (LoadL src) minus_1) (LoadL src)) );
9239   predicate(UseBMI1Instructions);
9240   effect(KILL cr);
9241 
9242   ins_cost(125);
9243   format %{ "blsrq  $dst, $src" %}
9244 
9245   ins_encode %{
9246     __ blsrq($dst$$Register, $src$$Address);
9247   %}
9248 
9249   ins_pipe(ialu_reg);
9250 %}
9251 
9252 // Or Instructions
9253 // Or Register with Register
9254 instruct orL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
9255 %{
9256   match(Set dst (OrL dst src));
9257   effect(KILL cr);
9258 
9259   format %{ "orq     $dst, $src\t# long" %}
9260   opcode(0x0B);
9261   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
9262   ins_pipe(ialu_reg_reg);
9263 %}
9264 
9265 // Use any_RegP to match R15 (TLS register) without spilling.
9266 instruct orL_rReg_castP2X(rRegL dst, any_RegP src, rFlagsReg cr) %{
9267   match(Set dst (OrL dst (CastP2X src)));
9268   effect(KILL cr);
9269 
9270   format %{ "orq     $dst, $src\t# long" %}
9271   opcode(0x0B);
9272   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
9273   ins_pipe(ialu_reg_reg);
9274 %}
9275 
9276 
9277 // Or Register with Immediate
9278 instruct orL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
9279 %{
9280   match(Set dst (OrL dst src));
9281   effect(KILL cr);
9282 
9283   format %{ "orq     $dst, $src\t# long" %}
9284   opcode(0x81, 0x01); /* Opcode 81 /1 id */
9285   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
9286   ins_pipe(ialu_reg);
9287 %}
9288 
9289 // Or Register with Memory
9290 instruct orL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
9291 %{
9292   match(Set dst (OrL dst (LoadL src)));
9293   effect(KILL cr);
9294 
9295   ins_cost(125);
9296   format %{ "orq     $dst, $src\t# long" %}
9297   opcode(0x0B);
9298   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
9299   ins_pipe(ialu_reg_mem);
9300 %}
9301 
9302 // Or Memory with Register
9303 instruct orL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
9304 %{
9305   match(Set dst (StoreL dst (OrL (LoadL dst) src)));
9306   effect(KILL cr);
9307 
9308   ins_cost(150);
9309   format %{ "orq     $dst, $src\t# long" %}
9310   opcode(0x09); /* Opcode 09 /r */
9311   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
9312   ins_pipe(ialu_mem_reg);
9313 %}
9314 
9315 // Or Memory with Immediate
9316 instruct orL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
9317 %{
9318   match(Set dst (StoreL dst (OrL (LoadL dst) src)));
9319   effect(KILL cr);
9320 
9321   ins_cost(125);
9322   format %{ "orq     $dst, $src\t# long" %}
9323   opcode(0x81, 0x1); /* Opcode 81 /1 id */
9324   ins_encode(REX_mem_wide(dst), OpcSE(src),
9325              RM_opc_mem(secondary, dst), Con8or32(src));
9326   ins_pipe(ialu_mem_imm);
9327 %}
9328 
9329 // Xor Instructions
9330 // Xor Register with Register
9331 instruct xorL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
9332 %{
9333   match(Set dst (XorL dst src));
9334   effect(KILL cr);
9335 
9336   format %{ "xorq    $dst, $src\t# long" %}
9337   opcode(0x33);
9338   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
9339   ins_pipe(ialu_reg_reg);
9340 %}
9341 
9342 // Xor Register with Immediate -1
9343 instruct xorL_rReg_im1(rRegL dst, immL_M1 imm) %{
9344   match(Set dst (XorL dst imm));
9345 
9346   format %{ "notq   $dst" %}
9347   ins_encode %{
9348      __ notq($dst$$Register);
9349   %}
9350   ins_pipe(ialu_reg);
9351 %}
9352 
9353 // Xor Register with Immediate
9354 instruct xorL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
9355 %{
9356   match(Set dst (XorL dst src));
9357   effect(KILL cr);
9358 
9359   format %{ "xorq    $dst, $src\t# long" %}
9360   opcode(0x81, 0x06); /* Opcode 81 /6 id */
9361   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
9362   ins_pipe(ialu_reg);
9363 %}
9364 
9365 // Xor Register with Memory
9366 instruct xorL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
9367 %{
9368   match(Set dst (XorL dst (LoadL src)));
9369   effect(KILL cr);
9370 
9371   ins_cost(125);
9372   format %{ "xorq    $dst, $src\t# long" %}
9373   opcode(0x33);
9374   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
9375   ins_pipe(ialu_reg_mem);
9376 %}
9377 
9378 // Xor Memory with Register
9379 instruct xorL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
9380 %{
9381   match(Set dst (StoreL dst (XorL (LoadL dst) src)));
9382   effect(KILL cr);
9383 
9384   ins_cost(150);
9385   format %{ "xorq    $dst, $src\t# long" %}
9386   opcode(0x31); /* Opcode 31 /r */
9387   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
9388   ins_pipe(ialu_mem_reg);
9389 %}
9390 
9391 // Xor Memory with Immediate
9392 instruct xorL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
9393 %{
9394   match(Set dst (StoreL dst (XorL (LoadL dst) src)));
9395   effect(KILL cr);
9396 
9397   ins_cost(125);
9398   format %{ "xorq    $dst, $src\t# long" %}
9399   opcode(0x81, 0x6); /* Opcode 81 /6 id */
9400   ins_encode(REX_mem_wide(dst), OpcSE(src),
9401              RM_opc_mem(secondary, dst), Con8or32(src));
9402   ins_pipe(ialu_mem_imm);
9403 %}
9404 
9405 // Convert Int to Boolean
9406 instruct convI2B(rRegI dst, rRegI src, rFlagsReg cr)
9407 %{
9408   match(Set dst (Conv2B src));
9409   effect(KILL cr);
9410 
9411   format %{ "testl   $src, $src\t# ci2b\n\t"
9412             "setnz   $dst\n\t"
9413             "movzbl  $dst, $dst" %}
9414   ins_encode(REX_reg_reg(src, src), opc_reg_reg(0x85, src, src), // testl
9415              setNZ_reg(dst),
9416              REX_reg_breg(dst, dst), // movzbl
9417              Opcode(0x0F), Opcode(0xB6), reg_reg(dst, dst));
9418   ins_pipe(pipe_slow); // XXX
9419 %}
9420 
9421 // Convert Pointer to Boolean
9422 instruct convP2B(rRegI dst, rRegP src, rFlagsReg cr)
9423 %{
9424   match(Set dst (Conv2B src));
9425   effect(KILL cr);
9426 
9427   format %{ "testq   $src, $src\t# cp2b\n\t"
9428             "setnz   $dst\n\t"
9429             "movzbl  $dst, $dst" %}
9430   ins_encode(REX_reg_reg_wide(src, src), opc_reg_reg(0x85, src, src), // testq
9431              setNZ_reg(dst),
9432              REX_reg_breg(dst, dst), // movzbl
9433              Opcode(0x0F), Opcode(0xB6), reg_reg(dst, dst));
9434   ins_pipe(pipe_slow); // XXX
9435 %}
9436 
9437 instruct cmpLTMask(rRegI dst, rRegI p, rRegI q, rFlagsReg cr)
9438 %{
9439   match(Set dst (CmpLTMask p q));
9440   effect(KILL cr);
9441 
9442   ins_cost(400);
9443   format %{ "cmpl    $p, $q\t# cmpLTMask\n\t"
9444             "setlt   $dst\n\t"
9445             "movzbl  $dst, $dst\n\t"
9446             "negl    $dst" %}
9447   ins_encode(REX_reg_reg(p, q), opc_reg_reg(0x3B, p, q), // cmpl
9448              setLT_reg(dst),
9449              REX_reg_breg(dst, dst), // movzbl
9450              Opcode(0x0F), Opcode(0xB6), reg_reg(dst, dst),
9451              neg_reg(dst));
9452   ins_pipe(pipe_slow);
9453 %}
9454 
9455 instruct cmpLTMask0(rRegI dst, immI0 zero, rFlagsReg cr)
9456 %{
9457   match(Set dst (CmpLTMask dst zero));
9458   effect(KILL cr);
9459 
9460   ins_cost(100);
9461   format %{ "sarl    $dst, #31\t# cmpLTMask0" %}
9462   ins_encode %{
9463   __ sarl($dst$$Register, 31);
9464   %}
9465   ins_pipe(ialu_reg);
9466 %}
9467 
9468 /* Better to save a register than avoid a branch */
9469 instruct cadd_cmpLTMask(rRegI p, rRegI q, rRegI y, rFlagsReg cr)
9470 %{
9471   match(Set p (AddI (AndI (CmpLTMask p q) y) (SubI p q)));
9472   effect(KILL cr);
9473   ins_cost(300);
9474   format %{ "subl   $p,$q\t# cadd_cmpLTMask\n\t"
9475             "jge    done\n\t"
9476             "addl   $p,$y\n"
9477             "done:  " %}
9478   ins_encode %{
9479     Register Rp = $p$$Register;
9480     Register Rq = $q$$Register;
9481     Register Ry = $y$$Register;
9482     Label done;
9483     __ subl(Rp, Rq);
9484     __ jccb(Assembler::greaterEqual, done);
9485     __ addl(Rp, Ry);
9486     __ bind(done);
9487   %}
9488   ins_pipe(pipe_cmplt);
9489 %}
9490 
9491 /* Better to save a register than avoid a branch */
9492 instruct and_cmpLTMask(rRegI p, rRegI q, rRegI y, rFlagsReg cr)
9493 %{
9494   match(Set y (AndI (CmpLTMask p q) y));
9495   effect(KILL cr);
9496 
9497   ins_cost(300);
9498 
9499   format %{ "cmpl     $p, $q\t# and_cmpLTMask\n\t"
9500             "jlt      done\n\t"
9501             "xorl     $y, $y\n"
9502             "done:  " %}
9503   ins_encode %{
9504     Register Rp = $p$$Register;
9505     Register Rq = $q$$Register;
9506     Register Ry = $y$$Register;
9507     Label done;
9508     __ cmpl(Rp, Rq);
9509     __ jccb(Assembler::less, done);
9510     __ xorl(Ry, Ry);
9511     __ bind(done);
9512   %}
9513   ins_pipe(pipe_cmplt);
9514 %}
9515 
9516 
9517 //---------- FP Instructions------------------------------------------------
9518 
9519 instruct cmpF_cc_reg(rFlagsRegU cr, regF src1, regF src2)
9520 %{
9521   match(Set cr (CmpF src1 src2));
9522 
9523   ins_cost(145);
9524   format %{ "ucomiss $src1, $src2\n\t"
9525             "jnp,s   exit\n\t"
9526             "pushfq\t# saw NaN, set CF\n\t"
9527             "andq    [rsp], #0xffffff2b\n\t"
9528             "popfq\n"
9529     "exit:" %}
9530   ins_encode %{
9531     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
9532     emit_cmpfp_fixup(_masm);
9533   %}
9534   ins_pipe(pipe_slow);
9535 %}
9536 
9537 instruct cmpF_cc_reg_CF(rFlagsRegUCF cr, regF src1, regF src2) %{
9538   match(Set cr (CmpF src1 src2));
9539 
9540   ins_cost(100);
9541   format %{ "ucomiss $src1, $src2" %}
9542   ins_encode %{
9543     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
9544   %}
9545   ins_pipe(pipe_slow);
9546 %}
9547 
9548 instruct cmpF_cc_mem(rFlagsRegU cr, regF src1, memory src2)
9549 %{
9550   match(Set cr (CmpF src1 (LoadF src2)));
9551 
9552   ins_cost(145);
9553   format %{ "ucomiss $src1, $src2\n\t"
9554             "jnp,s   exit\n\t"
9555             "pushfq\t# saw NaN, set CF\n\t"
9556             "andq    [rsp], #0xffffff2b\n\t"
9557             "popfq\n"
9558     "exit:" %}
9559   ins_encode %{
9560     __ ucomiss($src1$$XMMRegister, $src2$$Address);
9561     emit_cmpfp_fixup(_masm);
9562   %}
9563   ins_pipe(pipe_slow);
9564 %}
9565 
9566 instruct cmpF_cc_memCF(rFlagsRegUCF cr, regF src1, memory src2) %{
9567   match(Set cr (CmpF src1 (LoadF src2)));
9568 
9569   ins_cost(100);
9570   format %{ "ucomiss $src1, $src2" %}
9571   ins_encode %{
9572     __ ucomiss($src1$$XMMRegister, $src2$$Address);
9573   %}
9574   ins_pipe(pipe_slow);
9575 %}
9576 
9577 instruct cmpF_cc_imm(rFlagsRegU cr, regF src, immF con) %{
9578   match(Set cr (CmpF src con));
9579 
9580   ins_cost(145);
9581   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con\n\t"
9582             "jnp,s   exit\n\t"
9583             "pushfq\t# saw NaN, set CF\n\t"
9584             "andq    [rsp], #0xffffff2b\n\t"
9585             "popfq\n"
9586     "exit:" %}
9587   ins_encode %{
9588     __ ucomiss($src$$XMMRegister, $constantaddress($con));
9589     emit_cmpfp_fixup(_masm);
9590   %}
9591   ins_pipe(pipe_slow);
9592 %}
9593 
9594 instruct cmpF_cc_immCF(rFlagsRegUCF cr, regF src, immF con) %{
9595   match(Set cr (CmpF src con));
9596   ins_cost(100);
9597   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con" %}
9598   ins_encode %{
9599     __ ucomiss($src$$XMMRegister, $constantaddress($con));
9600   %}
9601   ins_pipe(pipe_slow);
9602 %}
9603 
9604 instruct cmpD_cc_reg(rFlagsRegU cr, regD src1, regD src2)
9605 %{
9606   match(Set cr (CmpD src1 src2));
9607 
9608   ins_cost(145);
9609   format %{ "ucomisd $src1, $src2\n\t"
9610             "jnp,s   exit\n\t"
9611             "pushfq\t# saw NaN, set CF\n\t"
9612             "andq    [rsp], #0xffffff2b\n\t"
9613             "popfq\n"
9614     "exit:" %}
9615   ins_encode %{
9616     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
9617     emit_cmpfp_fixup(_masm);
9618   %}
9619   ins_pipe(pipe_slow);
9620 %}
9621 
9622 instruct cmpD_cc_reg_CF(rFlagsRegUCF cr, regD src1, regD src2) %{
9623   match(Set cr (CmpD src1 src2));
9624 
9625   ins_cost(100);
9626   format %{ "ucomisd $src1, $src2 test" %}
9627   ins_encode %{
9628     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
9629   %}
9630   ins_pipe(pipe_slow);
9631 %}
9632 
9633 instruct cmpD_cc_mem(rFlagsRegU cr, regD src1, memory src2)
9634 %{
9635   match(Set cr (CmpD src1 (LoadD src2)));
9636 
9637   ins_cost(145);
9638   format %{ "ucomisd $src1, $src2\n\t"
9639             "jnp,s   exit\n\t"
9640             "pushfq\t# saw NaN, set CF\n\t"
9641             "andq    [rsp], #0xffffff2b\n\t"
9642             "popfq\n"
9643     "exit:" %}
9644   ins_encode %{
9645     __ ucomisd($src1$$XMMRegister, $src2$$Address);
9646     emit_cmpfp_fixup(_masm);
9647   %}
9648   ins_pipe(pipe_slow);
9649 %}
9650 
9651 instruct cmpD_cc_memCF(rFlagsRegUCF cr, regD src1, memory src2) %{
9652   match(Set cr (CmpD src1 (LoadD src2)));
9653 
9654   ins_cost(100);
9655   format %{ "ucomisd $src1, $src2" %}
9656   ins_encode %{
9657     __ ucomisd($src1$$XMMRegister, $src2$$Address);
9658   %}
9659   ins_pipe(pipe_slow);
9660 %}
9661 
9662 instruct cmpD_cc_imm(rFlagsRegU cr, regD src, immD con) %{
9663   match(Set cr (CmpD src con));
9664 
9665   ins_cost(145);
9666   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con\n\t"
9667             "jnp,s   exit\n\t"
9668             "pushfq\t# saw NaN, set CF\n\t"
9669             "andq    [rsp], #0xffffff2b\n\t"
9670             "popfq\n"
9671     "exit:" %}
9672   ins_encode %{
9673     __ ucomisd($src$$XMMRegister, $constantaddress($con));
9674     emit_cmpfp_fixup(_masm);
9675   %}
9676   ins_pipe(pipe_slow);
9677 %}
9678 
9679 instruct cmpD_cc_immCF(rFlagsRegUCF cr, regD src, immD con) %{
9680   match(Set cr (CmpD src con));
9681   ins_cost(100);
9682   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con" %}
9683   ins_encode %{
9684     __ ucomisd($src$$XMMRegister, $constantaddress($con));
9685   %}
9686   ins_pipe(pipe_slow);
9687 %}
9688 
9689 // Compare into -1,0,1
9690 instruct cmpF_reg(rRegI dst, regF src1, regF src2, rFlagsReg cr)
9691 %{
9692   match(Set dst (CmpF3 src1 src2));
9693   effect(KILL cr);
9694 
9695   ins_cost(275);
9696   format %{ "ucomiss $src1, $src2\n\t"
9697             "movl    $dst, #-1\n\t"
9698             "jp,s    done\n\t"
9699             "jb,s    done\n\t"
9700             "setne   $dst\n\t"
9701             "movzbl  $dst, $dst\n"
9702     "done:" %}
9703   ins_encode %{
9704     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
9705     emit_cmpfp3(_masm, $dst$$Register);
9706   %}
9707   ins_pipe(pipe_slow);
9708 %}
9709 
9710 // Compare into -1,0,1
9711 instruct cmpF_mem(rRegI dst, regF src1, memory src2, rFlagsReg cr)
9712 %{
9713   match(Set dst (CmpF3 src1 (LoadF src2)));
9714   effect(KILL cr);
9715 
9716   ins_cost(275);
9717   format %{ "ucomiss $src1, $src2\n\t"
9718             "movl    $dst, #-1\n\t"
9719             "jp,s    done\n\t"
9720             "jb,s    done\n\t"
9721             "setne   $dst\n\t"
9722             "movzbl  $dst, $dst\n"
9723     "done:" %}
9724   ins_encode %{
9725     __ ucomiss($src1$$XMMRegister, $src2$$Address);
9726     emit_cmpfp3(_masm, $dst$$Register);
9727   %}
9728   ins_pipe(pipe_slow);
9729 %}
9730 
9731 // Compare into -1,0,1
9732 instruct cmpF_imm(rRegI dst, regF src, immF con, rFlagsReg cr) %{
9733   match(Set dst (CmpF3 src con));
9734   effect(KILL cr);
9735 
9736   ins_cost(275);
9737   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con\n\t"
9738             "movl    $dst, #-1\n\t"
9739             "jp,s    done\n\t"
9740             "jb,s    done\n\t"
9741             "setne   $dst\n\t"
9742             "movzbl  $dst, $dst\n"
9743     "done:" %}
9744   ins_encode %{
9745     __ ucomiss($src$$XMMRegister, $constantaddress($con));
9746     emit_cmpfp3(_masm, $dst$$Register);
9747   %}
9748   ins_pipe(pipe_slow);
9749 %}
9750 
9751 // Compare into -1,0,1
9752 instruct cmpD_reg(rRegI dst, regD src1, regD src2, rFlagsReg cr)
9753 %{
9754   match(Set dst (CmpD3 src1 src2));
9755   effect(KILL cr);
9756 
9757   ins_cost(275);
9758   format %{ "ucomisd $src1, $src2\n\t"
9759             "movl    $dst, #-1\n\t"
9760             "jp,s    done\n\t"
9761             "jb,s    done\n\t"
9762             "setne   $dst\n\t"
9763             "movzbl  $dst, $dst\n"
9764     "done:" %}
9765   ins_encode %{
9766     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
9767     emit_cmpfp3(_masm, $dst$$Register);
9768   %}
9769   ins_pipe(pipe_slow);
9770 %}
9771 
9772 // Compare into -1,0,1
9773 instruct cmpD_mem(rRegI dst, regD src1, memory src2, rFlagsReg cr)
9774 %{
9775   match(Set dst (CmpD3 src1 (LoadD src2)));
9776   effect(KILL cr);
9777 
9778   ins_cost(275);
9779   format %{ "ucomisd $src1, $src2\n\t"
9780             "movl    $dst, #-1\n\t"
9781             "jp,s    done\n\t"
9782             "jb,s    done\n\t"
9783             "setne   $dst\n\t"
9784             "movzbl  $dst, $dst\n"
9785     "done:" %}
9786   ins_encode %{
9787     __ ucomisd($src1$$XMMRegister, $src2$$Address);
9788     emit_cmpfp3(_masm, $dst$$Register);
9789   %}
9790   ins_pipe(pipe_slow);
9791 %}
9792 
9793 // Compare into -1,0,1
9794 instruct cmpD_imm(rRegI dst, regD src, immD con, rFlagsReg cr) %{
9795   match(Set dst (CmpD3 src con));
9796   effect(KILL cr);
9797 
9798   ins_cost(275);
9799   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con\n\t"
9800             "movl    $dst, #-1\n\t"
9801             "jp,s    done\n\t"
9802             "jb,s    done\n\t"
9803             "setne   $dst\n\t"
9804             "movzbl  $dst, $dst\n"
9805     "done:" %}
9806   ins_encode %{
9807     __ ucomisd($src$$XMMRegister, $constantaddress($con));
9808     emit_cmpfp3(_masm, $dst$$Register);
9809   %}
9810   ins_pipe(pipe_slow);
9811 %}
9812 
9813 // -----------Trig and Trancendental Instructions------------------------------
9814 instruct cosD_reg(regD dst) %{
9815   match(Set dst (CosD dst));
9816 
9817   format %{ "dcos   $dst\n\t" %}
9818   opcode(0xD9, 0xFF);
9819   ins_encode( Push_SrcXD(dst), OpcP, OpcS, Push_ResultXD(dst) );
9820   ins_pipe( pipe_slow );
9821 %}
9822 
9823 instruct sinD_reg(regD dst) %{
9824   match(Set dst (SinD dst));
9825 
9826   format %{ "dsin   $dst\n\t" %}
9827   opcode(0xD9, 0xFE);
9828   ins_encode( Push_SrcXD(dst), OpcP, OpcS, Push_ResultXD(dst) );
9829   ins_pipe( pipe_slow );
9830 %}
9831 
9832 instruct tanD_reg(regD dst) %{
9833   match(Set dst (TanD dst));
9834 
9835   format %{ "dtan   $dst\n\t" %}
9836   ins_encode( Push_SrcXD(dst),
9837               Opcode(0xD9), Opcode(0xF2),   //fptan
9838               Opcode(0xDD), Opcode(0xD8),   //fstp st
9839               Push_ResultXD(dst) );
9840   ins_pipe( pipe_slow );
9841 %}
9842 
9843 instruct log10D_reg(regD dst) %{
9844   // The source and result Double operands in XMM registers
9845   match(Set dst (Log10D dst));
9846   // fldlg2       ; push log_10(2) on the FPU stack; full 80-bit number
9847   // fyl2x        ; compute log_10(2) * log_2(x)
9848   format %{ "fldlg2\t\t\t#Log10\n\t"
9849             "fyl2x\t\t\t# Q=Log10*Log_2(x)\n\t"
9850          %}
9851    ins_encode(Opcode(0xD9), Opcode(0xEC),   // fldlg2
9852               Push_SrcXD(dst),
9853               Opcode(0xD9), Opcode(0xF1),   // fyl2x
9854               Push_ResultXD(dst));
9855 
9856   ins_pipe( pipe_slow );
9857 %}
9858 
9859 instruct logD_reg(regD dst) %{
9860   // The source and result Double operands in XMM registers
9861   match(Set dst (LogD dst));
9862   // fldln2       ; push log_e(2) on the FPU stack; full 80-bit number
9863   // fyl2x        ; compute log_e(2) * log_2(x)
9864   format %{ "fldln2\t\t\t#Log_e\n\t"
9865             "fyl2x\t\t\t# Q=Log_e*Log_2(x)\n\t"
9866          %}
9867   ins_encode( Opcode(0xD9), Opcode(0xED),   // fldln2
9868               Push_SrcXD(dst),
9869               Opcode(0xD9), Opcode(0xF1),   // fyl2x
9870               Push_ResultXD(dst));
9871   ins_pipe( pipe_slow );
9872 %}
9873 
9874 instruct powD_reg(regD dst, regD src0, regD src1, rax_RegI rax, rdx_RegI rdx, rcx_RegI rcx, rFlagsReg cr) %{
9875   match(Set dst (PowD src0 src1));  // Raise src0 to the src1'th power
9876   effect(KILL rax, KILL rdx, KILL rcx, KILL cr);
9877   format %{ "fast_pow $src0 $src1 -> $dst  // KILL $rax, $rcx, $rdx" %}
9878   ins_encode %{
9879     __ subptr(rsp, 8);
9880     __ movdbl(Address(rsp, 0), $src1$$XMMRegister);
9881     __ fld_d(Address(rsp, 0));
9882     __ movdbl(Address(rsp, 0), $src0$$XMMRegister);
9883     __ fld_d(Address(rsp, 0));
9884     __ fast_pow();
9885     __ fstp_d(Address(rsp, 0));
9886     __ movdbl($dst$$XMMRegister, Address(rsp, 0));
9887     __ addptr(rsp, 8);
9888   %}
9889   ins_pipe( pipe_slow );
9890 %}
9891 
9892 instruct expD_reg(regD dst, regD src, rax_RegI rax, rdx_RegI rdx, rcx_RegI rcx, rFlagsReg cr) %{
9893   match(Set dst (ExpD src));
9894   effect(KILL rax, KILL rcx, KILL rdx, KILL cr);
9895   format %{ "fast_exp $dst -> $src  // KILL $rax, $rcx, $rdx" %}
9896   ins_encode %{
9897     __ subptr(rsp, 8);
9898     __ movdbl(Address(rsp, 0), $src$$XMMRegister);
9899     __ fld_d(Address(rsp, 0));
9900     __ fast_exp();
9901     __ fstp_d(Address(rsp, 0));
9902     __ movdbl($dst$$XMMRegister, Address(rsp, 0));
9903     __ addptr(rsp, 8);
9904   %}
9905   ins_pipe( pipe_slow );
9906 %}
9907 
9908 //----------Arithmetic Conversion Instructions---------------------------------
9909 
9910 instruct roundFloat_nop(regF dst)
9911 %{
9912   match(Set dst (RoundFloat dst));
9913 
9914   ins_cost(0);
9915   ins_encode();
9916   ins_pipe(empty);
9917 %}
9918 
9919 instruct roundDouble_nop(regD dst)
9920 %{
9921   match(Set dst (RoundDouble dst));
9922 
9923   ins_cost(0);
9924   ins_encode();
9925   ins_pipe(empty);
9926 %}
9927 
9928 instruct convF2D_reg_reg(regD dst, regF src)
9929 %{
9930   match(Set dst (ConvF2D src));
9931 
9932   format %{ "cvtss2sd $dst, $src" %}
9933   ins_encode %{
9934     __ cvtss2sd ($dst$$XMMRegister, $src$$XMMRegister);
9935   %}
9936   ins_pipe(pipe_slow); // XXX
9937 %}
9938 
9939 instruct convF2D_reg_mem(regD dst, memory src)
9940 %{
9941   match(Set dst (ConvF2D (LoadF src)));
9942 
9943   format %{ "cvtss2sd $dst, $src" %}
9944   ins_encode %{
9945     __ cvtss2sd ($dst$$XMMRegister, $src$$Address);
9946   %}
9947   ins_pipe(pipe_slow); // XXX
9948 %}
9949 
9950 instruct convD2F_reg_reg(regF dst, regD src)
9951 %{
9952   match(Set dst (ConvD2F src));
9953 
9954   format %{ "cvtsd2ss $dst, $src" %}
9955   ins_encode %{
9956     __ cvtsd2ss ($dst$$XMMRegister, $src$$XMMRegister);
9957   %}
9958   ins_pipe(pipe_slow); // XXX
9959 %}
9960 
9961 instruct convD2F_reg_mem(regF dst, memory src)
9962 %{
9963   match(Set dst (ConvD2F (LoadD src)));
9964 
9965   format %{ "cvtsd2ss $dst, $src" %}
9966   ins_encode %{
9967     __ cvtsd2ss ($dst$$XMMRegister, $src$$Address);
9968   %}
9969   ins_pipe(pipe_slow); // XXX
9970 %}
9971 
9972 // XXX do mem variants
9973 instruct convF2I_reg_reg(rRegI dst, regF src, rFlagsReg cr)
9974 %{
9975   match(Set dst (ConvF2I src));
9976   effect(KILL cr);
9977 
9978   format %{ "cvttss2sil $dst, $src\t# f2i\n\t"
9979             "cmpl    $dst, #0x80000000\n\t"
9980             "jne,s   done\n\t"
9981             "subq    rsp, #8\n\t"
9982             "movss   [rsp], $src\n\t"
9983             "call    f2i_fixup\n\t"
9984             "popq    $dst\n"
9985     "done:   "%}
9986   ins_encode %{
9987     Label done;
9988     __ cvttss2sil($dst$$Register, $src$$XMMRegister);
9989     __ cmpl($dst$$Register, 0x80000000);
9990     __ jccb(Assembler::notEqual, done);
9991     __ subptr(rsp, 8);
9992     __ movflt(Address(rsp, 0), $src$$XMMRegister);
9993     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::f2i_fixup())));
9994     __ pop($dst$$Register);
9995     __ bind(done);
9996   %}
9997   ins_pipe(pipe_slow);
9998 %}
9999 
10000 instruct convF2L_reg_reg(rRegL dst, regF src, rFlagsReg cr)
10001 %{
10002   match(Set dst (ConvF2L src));
10003   effect(KILL cr);
10004 
10005   format %{ "cvttss2siq $dst, $src\t# f2l\n\t"
10006             "cmpq    $dst, [0x8000000000000000]\n\t"
10007             "jne,s   done\n\t"
10008             "subq    rsp, #8\n\t"
10009             "movss   [rsp], $src\n\t"
10010             "call    f2l_fixup\n\t"
10011             "popq    $dst\n"
10012     "done:   "%}
10013   ins_encode %{
10014     Label done;
10015     __ cvttss2siq($dst$$Register, $src$$XMMRegister);
10016     __ cmp64($dst$$Register,
10017              ExternalAddress((address) StubRoutines::x86::double_sign_flip()));
10018     __ jccb(Assembler::notEqual, done);
10019     __ subptr(rsp, 8);
10020     __ movflt(Address(rsp, 0), $src$$XMMRegister);
10021     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::f2l_fixup())));
10022     __ pop($dst$$Register);
10023     __ bind(done);
10024   %}
10025   ins_pipe(pipe_slow);
10026 %}
10027 
10028 instruct convD2I_reg_reg(rRegI dst, regD src, rFlagsReg cr)
10029 %{
10030   match(Set dst (ConvD2I src));
10031   effect(KILL cr);
10032 
10033   format %{ "cvttsd2sil $dst, $src\t# d2i\n\t"
10034             "cmpl    $dst, #0x80000000\n\t"
10035             "jne,s   done\n\t"
10036             "subq    rsp, #8\n\t"
10037             "movsd   [rsp], $src\n\t"
10038             "call    d2i_fixup\n\t"
10039             "popq    $dst\n"
10040     "done:   "%}
10041   ins_encode %{
10042     Label done;
10043     __ cvttsd2sil($dst$$Register, $src$$XMMRegister);
10044     __ cmpl($dst$$Register, 0x80000000);
10045     __ jccb(Assembler::notEqual, done);
10046     __ subptr(rsp, 8);
10047     __ movdbl(Address(rsp, 0), $src$$XMMRegister);
10048     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::d2i_fixup())));
10049     __ pop($dst$$Register);
10050     __ bind(done);
10051   %}
10052   ins_pipe(pipe_slow);
10053 %}
10054 
10055 instruct convD2L_reg_reg(rRegL dst, regD src, rFlagsReg cr)
10056 %{
10057   match(Set dst (ConvD2L src));
10058   effect(KILL cr);
10059 
10060   format %{ "cvttsd2siq $dst, $src\t# d2l\n\t"
10061             "cmpq    $dst, [0x8000000000000000]\n\t"
10062             "jne,s   done\n\t"
10063             "subq    rsp, #8\n\t"
10064             "movsd   [rsp], $src\n\t"
10065             "call    d2l_fixup\n\t"
10066             "popq    $dst\n"
10067     "done:   "%}
10068   ins_encode %{
10069     Label done;
10070     __ cvttsd2siq($dst$$Register, $src$$XMMRegister);
10071     __ cmp64($dst$$Register,
10072              ExternalAddress((address) StubRoutines::x86::double_sign_flip()));
10073     __ jccb(Assembler::notEqual, done);
10074     __ subptr(rsp, 8);
10075     __ movdbl(Address(rsp, 0), $src$$XMMRegister);
10076     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::d2l_fixup())));
10077     __ pop($dst$$Register);
10078     __ bind(done);
10079   %}
10080   ins_pipe(pipe_slow);
10081 %}
10082 
10083 instruct convI2F_reg_reg(regF dst, rRegI src)
10084 %{
10085   predicate(!UseXmmI2F);
10086   match(Set dst (ConvI2F src));
10087 
10088   format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
10089   ins_encode %{
10090     __ cvtsi2ssl ($dst$$XMMRegister, $src$$Register);
10091   %}
10092   ins_pipe(pipe_slow); // XXX
10093 %}
10094 
10095 instruct convI2F_reg_mem(regF dst, memory src)
10096 %{
10097   match(Set dst (ConvI2F (LoadI src)));
10098 
10099   format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
10100   ins_encode %{
10101     __ cvtsi2ssl ($dst$$XMMRegister, $src$$Address);
10102   %}
10103   ins_pipe(pipe_slow); // XXX
10104 %}
10105 
10106 instruct convI2D_reg_reg(regD dst, rRegI src)
10107 %{
10108   predicate(!UseXmmI2D);
10109   match(Set dst (ConvI2D src));
10110 
10111   format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
10112   ins_encode %{
10113     __ cvtsi2sdl ($dst$$XMMRegister, $src$$Register);
10114   %}
10115   ins_pipe(pipe_slow); // XXX
10116 %}
10117 
10118 instruct convI2D_reg_mem(regD dst, memory src)
10119 %{
10120   match(Set dst (ConvI2D (LoadI src)));
10121 
10122   format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
10123   ins_encode %{
10124     __ cvtsi2sdl ($dst$$XMMRegister, $src$$Address);
10125   %}
10126   ins_pipe(pipe_slow); // XXX
10127 %}
10128 
10129 instruct convXI2F_reg(regF dst, rRegI src)
10130 %{
10131   predicate(UseXmmI2F);
10132   match(Set dst (ConvI2F src));
10133 
10134   format %{ "movdl $dst, $src\n\t"
10135             "cvtdq2psl $dst, $dst\t# i2f" %}
10136   ins_encode %{
10137     __ movdl($dst$$XMMRegister, $src$$Register);
10138     __ cvtdq2ps($dst$$XMMRegister, $dst$$XMMRegister);
10139   %}
10140   ins_pipe(pipe_slow); // XXX
10141 %}
10142 
10143 instruct convXI2D_reg(regD dst, rRegI src)
10144 %{
10145   predicate(UseXmmI2D);
10146   match(Set dst (ConvI2D src));
10147 
10148   format %{ "movdl $dst, $src\n\t"
10149             "cvtdq2pdl $dst, $dst\t# i2d" %}
10150   ins_encode %{
10151     __ movdl($dst$$XMMRegister, $src$$Register);
10152     __ cvtdq2pd($dst$$XMMRegister, $dst$$XMMRegister);
10153   %}
10154   ins_pipe(pipe_slow); // XXX
10155 %}
10156 
10157 instruct convL2F_reg_reg(regF dst, rRegL src)
10158 %{
10159   match(Set dst (ConvL2F src));
10160 
10161   format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
10162   ins_encode %{
10163     __ cvtsi2ssq ($dst$$XMMRegister, $src$$Register);
10164   %}
10165   ins_pipe(pipe_slow); // XXX
10166 %}
10167 
10168 instruct convL2F_reg_mem(regF dst, memory src)
10169 %{
10170   match(Set dst (ConvL2F (LoadL src)));
10171 
10172   format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
10173   ins_encode %{
10174     __ cvtsi2ssq ($dst$$XMMRegister, $src$$Address);
10175   %}
10176   ins_pipe(pipe_slow); // XXX
10177 %}
10178 
10179 instruct convL2D_reg_reg(regD dst, rRegL src)
10180 %{
10181   match(Set dst (ConvL2D src));
10182 
10183   format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
10184   ins_encode %{
10185     __ cvtsi2sdq ($dst$$XMMRegister, $src$$Register);
10186   %}
10187   ins_pipe(pipe_slow); // XXX
10188 %}
10189 
10190 instruct convL2D_reg_mem(regD dst, memory src)
10191 %{
10192   match(Set dst (ConvL2D (LoadL src)));
10193 
10194   format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
10195   ins_encode %{
10196     __ cvtsi2sdq ($dst$$XMMRegister, $src$$Address);
10197   %}
10198   ins_pipe(pipe_slow); // XXX
10199 %}
10200 
10201 instruct convI2L_reg_reg(rRegL dst, rRegI src)
10202 %{
10203   match(Set dst (ConvI2L src));
10204 
10205   ins_cost(125);
10206   format %{ "movslq  $dst, $src\t# i2l" %}
10207   ins_encode %{
10208     __ movslq($dst$$Register, $src$$Register);
10209   %}
10210   ins_pipe(ialu_reg_reg);
10211 %}
10212 
10213 // instruct convI2L_reg_reg_foo(rRegL dst, rRegI src)
10214 // %{
10215 //   match(Set dst (ConvI2L src));
10216 // //   predicate(_kids[0]->_leaf->as_Type()->type()->is_int()->_lo >= 0 &&
10217 // //             _kids[0]->_leaf->as_Type()->type()->is_int()->_hi >= 0);
10218 //   predicate(((const TypeNode*) n)->type()->is_long()->_hi ==
10219 //             (unsigned int) ((const TypeNode*) n)->type()->is_long()->_hi &&
10220 //             ((const TypeNode*) n)->type()->is_long()->_lo ==
10221 //             (unsigned int) ((const TypeNode*) n)->type()->is_long()->_lo);
10222 
10223 //   format %{ "movl    $dst, $src\t# unsigned i2l" %}
10224 //   ins_encode(enc_copy(dst, src));
10225 // //   opcode(0x63); // needs REX.W
10226 // //   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst,src));
10227 //   ins_pipe(ialu_reg_reg);
10228 // %}
10229 
10230 // Zero-extend convert int to long
10231 instruct convI2L_reg_reg_zex(rRegL dst, rRegI src, immL_32bits mask)
10232 %{
10233   match(Set dst (AndL (ConvI2L src) mask));
10234 
10235   format %{ "movl    $dst, $src\t# i2l zero-extend\n\t" %}
10236   ins_encode %{
10237     if ($dst$$reg != $src$$reg) {
10238       __ movl($dst$$Register, $src$$Register);
10239     }
10240   %}
10241   ins_pipe(ialu_reg_reg);
10242 %}
10243 
10244 // Zero-extend convert int to long
10245 instruct convI2L_reg_mem_zex(rRegL dst, memory src, immL_32bits mask)
10246 %{
10247   match(Set dst (AndL (ConvI2L (LoadI src)) mask));
10248 
10249   format %{ "movl    $dst, $src\t# i2l zero-extend\n\t" %}
10250   ins_encode %{
10251     __ movl($dst$$Register, $src$$Address);
10252   %}
10253   ins_pipe(ialu_reg_mem);
10254 %}
10255 
10256 instruct zerox_long_reg_reg(rRegL dst, rRegL src, immL_32bits mask)
10257 %{
10258   match(Set dst (AndL src mask));
10259 
10260   format %{ "movl    $dst, $src\t# zero-extend long" %}
10261   ins_encode %{
10262     __ movl($dst$$Register, $src$$Register);
10263   %}
10264   ins_pipe(ialu_reg_reg);
10265 %}
10266 
10267 instruct convL2I_reg_reg(rRegI dst, rRegL src)
10268 %{
10269   match(Set dst (ConvL2I src));
10270 
10271   format %{ "movl    $dst, $src\t# l2i" %}
10272   ins_encode %{
10273     __ movl($dst$$Register, $src$$Register);
10274   %}
10275   ins_pipe(ialu_reg_reg);
10276 %}
10277 
10278 
10279 instruct MoveF2I_stack_reg(rRegI dst, stackSlotF src) %{
10280   match(Set dst (MoveF2I src));
10281   effect(DEF dst, USE src);
10282 
10283   ins_cost(125);
10284   format %{ "movl    $dst, $src\t# MoveF2I_stack_reg" %}
10285   ins_encode %{
10286     __ movl($dst$$Register, Address(rsp, $src$$disp));
10287   %}
10288   ins_pipe(ialu_reg_mem);
10289 %}
10290 
10291 instruct MoveI2F_stack_reg(regF dst, stackSlotI src) %{
10292   match(Set dst (MoveI2F src));
10293   effect(DEF dst, USE src);
10294 
10295   ins_cost(125);
10296   format %{ "movss   $dst, $src\t# MoveI2F_stack_reg" %}
10297   ins_encode %{
10298     __ movflt($dst$$XMMRegister, Address(rsp, $src$$disp));
10299   %}
10300   ins_pipe(pipe_slow);
10301 %}
10302 
10303 instruct MoveD2L_stack_reg(rRegL dst, stackSlotD src) %{
10304   match(Set dst (MoveD2L src));
10305   effect(DEF dst, USE src);
10306 
10307   ins_cost(125);
10308   format %{ "movq    $dst, $src\t# MoveD2L_stack_reg" %}
10309   ins_encode %{
10310     __ movq($dst$$Register, Address(rsp, $src$$disp));
10311   %}
10312   ins_pipe(ialu_reg_mem);
10313 %}
10314 
10315 instruct MoveL2D_stack_reg_partial(regD dst, stackSlotL src) %{
10316   predicate(!UseXmmLoadAndClearUpper);
10317   match(Set dst (MoveL2D src));
10318   effect(DEF dst, USE src);
10319 
10320   ins_cost(125);
10321   format %{ "movlpd  $dst, $src\t# MoveL2D_stack_reg" %}
10322   ins_encode %{
10323     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
10324   %}
10325   ins_pipe(pipe_slow);
10326 %}
10327 
10328 instruct MoveL2D_stack_reg(regD dst, stackSlotL src) %{
10329   predicate(UseXmmLoadAndClearUpper);
10330   match(Set dst (MoveL2D src));
10331   effect(DEF dst, USE src);
10332 
10333   ins_cost(125);
10334   format %{ "movsd   $dst, $src\t# MoveL2D_stack_reg" %}
10335   ins_encode %{
10336     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
10337   %}
10338   ins_pipe(pipe_slow);
10339 %}
10340 
10341 
10342 instruct MoveF2I_reg_stack(stackSlotI dst, regF src) %{
10343   match(Set dst (MoveF2I src));
10344   effect(DEF dst, USE src);
10345 
10346   ins_cost(95); // XXX
10347   format %{ "movss   $dst, $src\t# MoveF2I_reg_stack" %}
10348   ins_encode %{
10349     __ movflt(Address(rsp, $dst$$disp), $src$$XMMRegister);
10350   %}
10351   ins_pipe(pipe_slow);
10352 %}
10353 
10354 instruct MoveI2F_reg_stack(stackSlotF dst, rRegI src) %{
10355   match(Set dst (MoveI2F src));
10356   effect(DEF dst, USE src);
10357 
10358   ins_cost(100);
10359   format %{ "movl    $dst, $src\t# MoveI2F_reg_stack" %}
10360   ins_encode %{
10361     __ movl(Address(rsp, $dst$$disp), $src$$Register);
10362   %}
10363   ins_pipe( ialu_mem_reg );
10364 %}
10365 
10366 instruct MoveD2L_reg_stack(stackSlotL dst, regD src) %{
10367   match(Set dst (MoveD2L src));
10368   effect(DEF dst, USE src);
10369 
10370   ins_cost(95); // XXX
10371   format %{ "movsd   $dst, $src\t# MoveL2D_reg_stack" %}
10372   ins_encode %{
10373     __ movdbl(Address(rsp, $dst$$disp), $src$$XMMRegister);
10374   %}
10375   ins_pipe(pipe_slow);
10376 %}
10377 
10378 instruct MoveL2D_reg_stack(stackSlotD dst, rRegL src) %{
10379   match(Set dst (MoveL2D src));
10380   effect(DEF dst, USE src);
10381 
10382   ins_cost(100);
10383   format %{ "movq    $dst, $src\t# MoveL2D_reg_stack" %}
10384   ins_encode %{
10385     __ movq(Address(rsp, $dst$$disp), $src$$Register);
10386   %}
10387   ins_pipe(ialu_mem_reg);
10388 %}
10389 
10390 instruct MoveF2I_reg_reg(rRegI dst, regF src) %{
10391   match(Set dst (MoveF2I src));
10392   effect(DEF dst, USE src);
10393   ins_cost(85);
10394   format %{ "movd    $dst,$src\t# MoveF2I" %}
10395   ins_encode %{
10396     __ movdl($dst$$Register, $src$$XMMRegister);
10397   %}
10398   ins_pipe( pipe_slow );
10399 %}
10400 
10401 instruct MoveD2L_reg_reg(rRegL dst, regD src) %{
10402   match(Set dst (MoveD2L src));
10403   effect(DEF dst, USE src);
10404   ins_cost(85);
10405   format %{ "movd    $dst,$src\t# MoveD2L" %}
10406   ins_encode %{
10407     __ movdq($dst$$Register, $src$$XMMRegister);
10408   %}
10409   ins_pipe( pipe_slow );
10410 %}
10411 
10412 instruct MoveI2F_reg_reg(regF dst, rRegI src) %{
10413   match(Set dst (MoveI2F src));
10414   effect(DEF dst, USE src);
10415   ins_cost(100);
10416   format %{ "movd    $dst,$src\t# MoveI2F" %}
10417   ins_encode %{
10418     __ movdl($dst$$XMMRegister, $src$$Register);
10419   %}
10420   ins_pipe( pipe_slow );
10421 %}
10422 
10423 instruct MoveL2D_reg_reg(regD dst, rRegL src) %{
10424   match(Set dst (MoveL2D src));
10425   effect(DEF dst, USE src);
10426   ins_cost(100);
10427   format %{ "movd    $dst,$src\t# MoveL2D" %}
10428   ins_encode %{
10429      __ movdq($dst$$XMMRegister, $src$$Register);
10430   %}
10431   ins_pipe( pipe_slow );
10432 %}
10433 
10434 
10435 // =======================================================================
10436 // fast clearing of an array
10437 instruct rep_stos(rcx_RegL cnt, rdi_RegP base, rax_RegI zero, Universe dummy,
10438                   rFlagsReg cr)
10439 %{
10440   predicate(!UseFastStosb);
10441   match(Set dummy (ClearArray cnt base));
10442   effect(USE_KILL cnt, USE_KILL base, KILL zero, KILL cr);
10443 
10444   format %{ "xorq    rax, rax\t# ClearArray:\n\t"
10445             "rep     stosq\t# Store rax to *rdi++ while rcx--" %}
10446   ins_encode %{ 
10447     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register);
10448   %}
10449   ins_pipe(pipe_slow);
10450 %}
10451 
10452 instruct rep_fast_stosb(rcx_RegL cnt, rdi_RegP base, rax_RegI zero, Universe dummy,
10453                         rFlagsReg cr)
10454 %{
10455   predicate(UseFastStosb);
10456   match(Set dummy (ClearArray cnt base));
10457   effect(USE_KILL cnt, USE_KILL base, KILL zero, KILL cr);
10458   format %{ "xorq    rax, rax\t# ClearArray:\n\t"
10459             "shlq    rcx,3\t# Convert doublewords to bytes\n\t"
10460             "rep     stosb\t# Store rax to *rdi++ while rcx--" %}
10461   ins_encode %{ 
10462     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register);
10463   %}
10464   ins_pipe( pipe_slow );
10465 %}
10466 
10467 instruct string_compare(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
10468                         rax_RegI result, regD tmp1, rFlagsReg cr)
10469 %{
10470   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
10471   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
10472 
10473   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
10474   ins_encode %{
10475     __ string_compare($str1$$Register, $str2$$Register,
10476                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
10477                       $tmp1$$XMMRegister);
10478   %}
10479   ins_pipe( pipe_slow );
10480 %}
10481 
10482 // fast search of substring with known size.
10483 instruct string_indexof_con(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
10484                             rbx_RegI result, regD vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
10485 %{
10486   predicate(UseSSE42Intrinsics);
10487   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
10488   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
10489 
10490   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $vec, $cnt1, $cnt2, $tmp" %}
10491   ins_encode %{
10492     int icnt2 = (int)$int_cnt2$$constant;
10493     if (icnt2 >= 8) {
10494       // IndexOf for constant substrings with size >= 8 elements
10495       // which don't need to be loaded through stack.
10496       __ string_indexofC8($str1$$Register, $str2$$Register,
10497                           $cnt1$$Register, $cnt2$$Register,
10498                           icnt2, $result$$Register,
10499                           $vec$$XMMRegister, $tmp$$Register);
10500     } else {
10501       // Small strings are loaded through stack if they cross page boundary.
10502       __ string_indexof($str1$$Register, $str2$$Register,
10503                         $cnt1$$Register, $cnt2$$Register,
10504                         icnt2, $result$$Register,
10505                         $vec$$XMMRegister, $tmp$$Register);
10506     }
10507   %}
10508   ins_pipe( pipe_slow );
10509 %}
10510 
10511 instruct string_indexof(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
10512                         rbx_RegI result, regD vec, rcx_RegI tmp, rFlagsReg cr)
10513 %{
10514   predicate(UseSSE42Intrinsics);
10515   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
10516   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
10517 
10518   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
10519   ins_encode %{
10520     __ string_indexof($str1$$Register, $str2$$Register,
10521                       $cnt1$$Register, $cnt2$$Register,
10522                       (-1), $result$$Register,
10523                       $vec$$XMMRegister, $tmp$$Register);
10524   %}
10525   ins_pipe( pipe_slow );
10526 %}
10527 
10528 // fast string equals
10529 instruct string_equals(rdi_RegP str1, rsi_RegP str2, rcx_RegI cnt, rax_RegI result,
10530                        regD tmp1, regD tmp2, rbx_RegI tmp3, rFlagsReg cr)
10531 %{
10532   match(Set result (StrEquals (Binary str1 str2) cnt));
10533   effect(TEMP tmp1, TEMP tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL tmp3, KILL cr);
10534 
10535   format %{ "String Equals $str1,$str2,$cnt -> $result    // KILL $tmp1, $tmp2, $tmp3" %}
10536   ins_encode %{
10537     __ char_arrays_equals(false, $str1$$Register, $str2$$Register,
10538                           $cnt$$Register, $result$$Register, $tmp3$$Register,
10539                           $tmp1$$XMMRegister, $tmp2$$XMMRegister);
10540   %}
10541   ins_pipe( pipe_slow );
10542 %}
10543 
10544 // fast array equals
10545 instruct array_equals(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
10546                       regD tmp1, regD tmp2, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
10547 %{
10548   match(Set result (AryEq ary1 ary2));
10549   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
10550   //ins_cost(300);
10551 
10552   format %{ "Array Equals $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
10553   ins_encode %{
10554     __ char_arrays_equals(true, $ary1$$Register, $ary2$$Register,
10555                           $tmp3$$Register, $result$$Register, $tmp4$$Register,
10556                           $tmp1$$XMMRegister, $tmp2$$XMMRegister);
10557   %}
10558   ins_pipe( pipe_slow );
10559 %}
10560 
10561 // encode char[] to byte[] in ISO_8859_1
10562 instruct encode_iso_array(rsi_RegP src, rdi_RegP dst, rdx_RegI len,
10563                           regD tmp1, regD tmp2, regD tmp3, regD tmp4,
10564                           rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
10565   match(Set result (EncodeISOArray src (Binary dst len)));
10566   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);
10567 
10568   format %{ "Encode array $src,$dst,$len -> $result    // KILL RCX, RDX, $tmp1, $tmp2, $tmp3, $tmp4, RSI, RDI " %}
10569   ins_encode %{
10570     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
10571                         $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
10572                         $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register);
10573   %}
10574   ins_pipe( pipe_slow );
10575 %}
10576 
10577 //----------Overflow Math Instructions-----------------------------------------
10578 
10579 instruct overflowAddI_rReg(rFlagsReg cr, rax_RegI op1, rRegI op2)
10580 %{
10581   match(Set cr (OverflowAddI op1 op2));
10582   effect(DEF cr, USE_KILL op1, USE op2);
10583 
10584   format %{ "addl    $op1, $op2\t# overflow check int" %}
10585 
10586   ins_encode %{
10587     __ addl($op1$$Register, $op2$$Register);
10588   %}
10589   ins_pipe(ialu_reg_reg);
10590 %}
10591 
10592 instruct overflowAddI_rReg_imm(rFlagsReg cr, rax_RegI op1, immI op2)
10593 %{
10594   match(Set cr (OverflowAddI op1 op2));
10595   effect(DEF cr, USE_KILL op1, USE op2);
10596 
10597   format %{ "addl    $op1, $op2\t# overflow check int" %}
10598 
10599   ins_encode %{
10600     __ addl($op1$$Register, $op2$$constant);
10601   %}
10602   ins_pipe(ialu_reg_reg);
10603 %}
10604 
10605 instruct overflowAddL_rReg(rFlagsReg cr, rax_RegL op1, rRegL op2)
10606 %{
10607   match(Set cr (OverflowAddL op1 op2));
10608   effect(DEF cr, USE_KILL op1, USE op2);
10609 
10610   format %{ "addq    $op1, $op2\t# overflow check long" %}
10611   ins_encode %{
10612     __ addq($op1$$Register, $op2$$Register);
10613   %}
10614   ins_pipe(ialu_reg_reg);
10615 %}
10616 
10617 instruct overflowAddL_rReg_imm(rFlagsReg cr, rax_RegL op1, immL32 op2)
10618 %{
10619   match(Set cr (OverflowAddL op1 op2));
10620   effect(DEF cr, USE_KILL op1, USE op2);
10621 
10622   format %{ "addq    $op1, $op2\t# overflow check long" %}
10623   ins_encode %{
10624     __ addq($op1$$Register, $op2$$constant);
10625   %}
10626   ins_pipe(ialu_reg_reg);
10627 %}
10628 
10629 instruct overflowSubI_rReg(rFlagsReg cr, rRegI op1, rRegI op2)
10630 %{
10631   match(Set cr (OverflowSubI op1 op2));
10632 
10633   format %{ "cmpl    $op1, $op2\t# overflow check int" %}
10634   ins_encode %{
10635     __ cmpl($op1$$Register, $op2$$Register);
10636   %}
10637   ins_pipe(ialu_reg_reg);
10638 %}
10639 
10640 instruct overflowSubI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2)
10641 %{
10642   match(Set cr (OverflowSubI op1 op2));
10643 
10644   format %{ "cmpl    $op1, $op2\t# overflow check int" %}
10645   ins_encode %{
10646     __ cmpl($op1$$Register, $op2$$constant);
10647   %}
10648   ins_pipe(ialu_reg_reg);
10649 %}
10650 
10651 instruct overflowSubL_rReg(rFlagsReg cr, rRegL op1, rRegL op2)
10652 %{
10653   match(Set cr (OverflowSubL op1 op2));
10654 
10655   format %{ "cmpq    $op1, $op2\t# overflow check long" %}
10656   ins_encode %{
10657     __ cmpq($op1$$Register, $op2$$Register);
10658   %}
10659   ins_pipe(ialu_reg_reg);
10660 %}
10661 
10662 instruct overflowSubL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2)
10663 %{
10664   match(Set cr (OverflowSubL op1 op2));
10665 
10666   format %{ "cmpq    $op1, $op2\t# overflow check long" %}
10667   ins_encode %{
10668     __ cmpq($op1$$Register, $op2$$constant);
10669   %}
10670   ins_pipe(ialu_reg_reg);
10671 %}
10672 
10673 instruct overflowNegI_rReg(rFlagsReg cr, immI0 zero, rax_RegI op2)
10674 %{
10675   match(Set cr (OverflowSubI zero op2));
10676   effect(DEF cr, USE_KILL op2);
10677 
10678   format %{ "negl    $op2\t# overflow check int" %}
10679   ins_encode %{
10680     __ negl($op2$$Register);
10681   %}
10682   ins_pipe(ialu_reg_reg);
10683 %}
10684 
10685 instruct overflowNegL_rReg(rFlagsReg cr, immL0 zero, rax_RegL op2)
10686 %{
10687   match(Set cr (OverflowSubL zero op2));
10688   effect(DEF cr, USE_KILL op2);
10689 
10690   format %{ "negq    $op2\t# overflow check long" %}
10691   ins_encode %{
10692     __ negq($op2$$Register);
10693   %}
10694   ins_pipe(ialu_reg_reg);
10695 %}
10696 
10697 instruct overflowMulI_rReg(rFlagsReg cr, rax_RegI op1, rRegI op2)
10698 %{
10699   match(Set cr (OverflowMulI op1 op2));
10700   effect(DEF cr, USE_KILL op1, USE op2);
10701 
10702   format %{ "imull    $op1, $op2\t# overflow check int" %}
10703   ins_encode %{
10704     __ imull($op1$$Register, $op2$$Register);
10705   %}
10706   ins_pipe(ialu_reg_reg_alu0);
10707 %}
10708 
10709 instruct overflowMulI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2, rRegI tmp)
10710 %{
10711   match(Set cr (OverflowMulI op1 op2));
10712   effect(DEF cr, TEMP tmp, USE op1, USE op2);
10713 
10714   format %{ "imull    $tmp, $op1, $op2\t# overflow check int" %}
10715   ins_encode %{
10716     __ imull($tmp$$Register, $op1$$Register, $op2$$constant);
10717   %}
10718   ins_pipe(ialu_reg_reg_alu0);
10719 %}
10720 
10721 instruct overflowMulL_rReg(rFlagsReg cr, rax_RegL op1, rRegL op2)
10722 %{
10723   match(Set cr (OverflowMulL op1 op2));
10724   effect(DEF cr, USE_KILL op1, USE op2);
10725 
10726   format %{ "imulq    $op1, $op2\t# overflow check long" %}
10727   ins_encode %{
10728     __ imulq($op1$$Register, $op2$$Register);
10729   %}
10730   ins_pipe(ialu_reg_reg_alu0);
10731 %}
10732 
10733 instruct overflowMulL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2, rRegL tmp)
10734 %{
10735   match(Set cr (OverflowMulL op1 op2));
10736   effect(DEF cr, TEMP tmp, USE op1, USE op2);
10737 
10738   format %{ "imulq    $tmp, $op1, $op2\t# overflow check long" %}
10739   ins_encode %{
10740     __ imulq($tmp$$Register, $op1$$Register, $op2$$constant);
10741   %}
10742   ins_pipe(ialu_reg_reg_alu0);
10743 %}
10744 
10745 
10746 //----------Control Flow Instructions------------------------------------------
10747 // Signed compare Instructions
10748 
10749 // XXX more variants!!
10750 instruct compI_rReg(rFlagsReg cr, rRegI op1, rRegI op2)
10751 %{
10752   match(Set cr (CmpI op1 op2));
10753   effect(DEF cr, USE op1, USE op2);
10754 
10755   format %{ "cmpl    $op1, $op2" %}
10756   opcode(0x3B);  /* Opcode 3B /r */
10757   ins_encode(REX_reg_reg(op1, op2), OpcP, reg_reg(op1, op2));
10758   ins_pipe(ialu_cr_reg_reg);
10759 %}
10760 
10761 instruct compI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2)
10762 %{
10763   match(Set cr (CmpI op1 op2));
10764 
10765   format %{ "cmpl    $op1, $op2" %}
10766   opcode(0x81, 0x07); /* Opcode 81 /7 */
10767   ins_encode(OpcSErm(op1, op2), Con8or32(op2));
10768   ins_pipe(ialu_cr_reg_imm);
10769 %}
10770 
10771 instruct compI_rReg_mem(rFlagsReg cr, rRegI op1, memory op2)
10772 %{
10773   match(Set cr (CmpI op1 (LoadI op2)));
10774 
10775   ins_cost(500); // XXX
10776   format %{ "cmpl    $op1, $op2" %}
10777   opcode(0x3B); /* Opcode 3B /r */
10778   ins_encode(REX_reg_mem(op1, op2), OpcP, reg_mem(op1, op2));
10779   ins_pipe(ialu_cr_reg_mem);
10780 %}
10781 
10782 instruct testI_reg(rFlagsReg cr, rRegI src, immI0 zero)
10783 %{
10784   match(Set cr (CmpI src zero));
10785 
10786   format %{ "testl   $src, $src" %}
10787   opcode(0x85);
10788   ins_encode(REX_reg_reg(src, src), OpcP, reg_reg(src, src));
10789   ins_pipe(ialu_cr_reg_imm);
10790 %}
10791 
10792 instruct testI_reg_imm(rFlagsReg cr, rRegI src, immI con, immI0 zero)
10793 %{
10794   match(Set cr (CmpI (AndI src con) zero));
10795 
10796   format %{ "testl   $src, $con" %}
10797   opcode(0xF7, 0x00);
10798   ins_encode(REX_reg(src), OpcP, reg_opc(src), Con32(con));
10799   ins_pipe(ialu_cr_reg_imm);
10800 %}
10801 
10802 instruct testI_reg_mem(rFlagsReg cr, rRegI src, memory mem, immI0 zero)
10803 %{
10804   match(Set cr (CmpI (AndI src (LoadI mem)) zero));
10805 
10806   format %{ "testl   $src, $mem" %}
10807   opcode(0x85);
10808   ins_encode(REX_reg_mem(src, mem), OpcP, reg_mem(src, mem));
10809   ins_pipe(ialu_cr_reg_mem);
10810 %}
10811 
10812 // Unsigned compare Instructions; really, same as signed except they
10813 // produce an rFlagsRegU instead of rFlagsReg.
10814 instruct compU_rReg(rFlagsRegU cr, rRegI op1, rRegI op2)
10815 %{
10816   match(Set cr (CmpU op1 op2));
10817 
10818   format %{ "cmpl    $op1, $op2\t# unsigned" %}
10819   opcode(0x3B); /* Opcode 3B /r */
10820   ins_encode(REX_reg_reg(op1, op2), OpcP, reg_reg(op1, op2));
10821   ins_pipe(ialu_cr_reg_reg);
10822 %}
10823 
10824 instruct compU_rReg_imm(rFlagsRegU cr, rRegI op1, immI op2)
10825 %{
10826   match(Set cr (CmpU op1 op2));
10827 
10828   format %{ "cmpl    $op1, $op2\t# unsigned" %}
10829   opcode(0x81,0x07); /* Opcode 81 /7 */
10830   ins_encode(OpcSErm(op1, op2), Con8or32(op2));
10831   ins_pipe(ialu_cr_reg_imm);
10832 %}
10833 
10834 instruct compU_rReg_mem(rFlagsRegU cr, rRegI op1, memory op2)
10835 %{
10836   match(Set cr (CmpU op1 (LoadI op2)));
10837 
10838   ins_cost(500); // XXX
10839   format %{ "cmpl    $op1, $op2\t# unsigned" %}
10840   opcode(0x3B); /* Opcode 3B /r */
10841   ins_encode(REX_reg_mem(op1, op2), OpcP, reg_mem(op1, op2));
10842   ins_pipe(ialu_cr_reg_mem);
10843 %}
10844 
10845 // // // Cisc-spilled version of cmpU_rReg
10846 // //instruct compU_mem_rReg(rFlagsRegU cr, memory op1, rRegI op2)
10847 // //%{
10848 // //  match(Set cr (CmpU (LoadI op1) op2));
10849 // //
10850 // //  format %{ "CMPu   $op1,$op2" %}
10851 // //  ins_cost(500);
10852 // //  opcode(0x39);  /* Opcode 39 /r */
10853 // //  ins_encode( OpcP, reg_mem( op1, op2) );
10854 // //%}
10855 
10856 instruct testU_reg(rFlagsRegU cr, rRegI src, immI0 zero)
10857 %{
10858   match(Set cr (CmpU src zero));
10859 
10860   format %{ "testl  $src, $src\t# unsigned" %}
10861   opcode(0x85);
10862   ins_encode(REX_reg_reg(src, src), OpcP, reg_reg(src, src));
10863   ins_pipe(ialu_cr_reg_imm);
10864 %}
10865 
10866 instruct compP_rReg(rFlagsRegU cr, rRegP op1, rRegP op2)
10867 %{
10868   match(Set cr (CmpP op1 op2));
10869 
10870   format %{ "cmpq    $op1, $op2\t# ptr" %}
10871   opcode(0x3B); /* Opcode 3B /r */
10872   ins_encode(REX_reg_reg_wide(op1, op2), OpcP, reg_reg(op1, op2));
10873   ins_pipe(ialu_cr_reg_reg);
10874 %}
10875 
10876 instruct compP_rReg_mem(rFlagsRegU cr, rRegP op1, memory op2)
10877 %{
10878   match(Set cr (CmpP op1 (LoadP op2)));
10879 
10880   ins_cost(500); // XXX
10881   format %{ "cmpq    $op1, $op2\t# ptr" %}
10882   opcode(0x3B); /* Opcode 3B /r */
10883   ins_encode(REX_reg_mem_wide(op1, op2), OpcP, reg_mem(op1, op2));
10884   ins_pipe(ialu_cr_reg_mem);
10885 %}
10886 
10887 // // // Cisc-spilled version of cmpP_rReg
10888 // //instruct compP_mem_rReg(rFlagsRegU cr, memory op1, rRegP op2)
10889 // //%{
10890 // //  match(Set cr (CmpP (LoadP op1) op2));
10891 // //
10892 // //  format %{ "CMPu   $op1,$op2" %}
10893 // //  ins_cost(500);
10894 // //  opcode(0x39);  /* Opcode 39 /r */
10895 // //  ins_encode( OpcP, reg_mem( op1, op2) );
10896 // //%}
10897 
10898 // XXX this is generalized by compP_rReg_mem???
10899 // Compare raw pointer (used in out-of-heap check).
10900 // Only works because non-oop pointers must be raw pointers
10901 // and raw pointers have no anti-dependencies.
10902 instruct compP_mem_rReg(rFlagsRegU cr, rRegP op1, memory op2)
10903 %{
10904   predicate(n->in(2)->in(2)->bottom_type()->reloc() == relocInfo::none);
10905   match(Set cr (CmpP op1 (LoadP op2)));
10906 
10907   format %{ "cmpq    $op1, $op2\t# raw ptr" %}
10908   opcode(0x3B); /* Opcode 3B /r */
10909   ins_encode(REX_reg_mem_wide(op1, op2), OpcP, reg_mem(op1, op2));
10910   ins_pipe(ialu_cr_reg_mem);
10911 %}
10912 
10913 // This will generate a signed flags result. This should be OK since
10914 // any compare to a zero should be eq/neq.
10915 instruct testP_reg(rFlagsReg cr, rRegP src, immP0 zero)
10916 %{
10917   match(Set cr (CmpP src zero));
10918 
10919   format %{ "testq   $src, $src\t# ptr" %}
10920   opcode(0x85);
10921   ins_encode(REX_reg_reg_wide(src, src), OpcP, reg_reg(src, src));
10922   ins_pipe(ialu_cr_reg_imm);
10923 %}
10924 
10925 // This will generate a signed flags result. This should be OK since
10926 // any compare to a zero should be eq/neq.
10927 instruct testP_mem(rFlagsReg cr, memory op, immP0 zero)
10928 %{
10929   predicate(!UseCompressedOops || (Universe::narrow_oop_base() != NULL));
10930   match(Set cr (CmpP (LoadP op) zero));
10931 
10932   ins_cost(500); // XXX
10933   format %{ "testq   $op, 0xffffffffffffffff\t# ptr" %}
10934   opcode(0xF7); /* Opcode F7 /0 */
10935   ins_encode(REX_mem_wide(op),
10936              OpcP, RM_opc_mem(0x00, op), Con_d32(0xFFFFFFFF));
10937   ins_pipe(ialu_cr_reg_imm);
10938 %}
10939 
10940 instruct testP_mem_reg0(rFlagsReg cr, memory mem, immP0 zero)
10941 %{
10942   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
10943   match(Set cr (CmpP (LoadP mem) zero));
10944 
10945   format %{ "cmpq    R12, $mem\t# ptr (R12_heapbase==0)" %}
10946   ins_encode %{
10947     __ cmpq(r12, $mem$$Address);
10948   %}
10949   ins_pipe(ialu_cr_reg_mem);
10950 %}
10951 
10952 instruct compN_rReg(rFlagsRegU cr, rRegN op1, rRegN op2)
10953 %{
10954   match(Set cr (CmpN op1 op2));
10955 
10956   format %{ "cmpl    $op1, $op2\t# compressed ptr" %}
10957   ins_encode %{ __ cmpl($op1$$Register, $op2$$Register); %}
10958   ins_pipe(ialu_cr_reg_reg);
10959 %}
10960 
10961 instruct compN_rReg_mem(rFlagsRegU cr, rRegN src, memory mem)
10962 %{
10963   match(Set cr (CmpN src (LoadN mem)));
10964 
10965   format %{ "cmpl    $src, $mem\t# compressed ptr" %}
10966   ins_encode %{
10967     __ cmpl($src$$Register, $mem$$Address);
10968   %}
10969   ins_pipe(ialu_cr_reg_mem);
10970 %}
10971 
10972 instruct compN_rReg_imm(rFlagsRegU cr, rRegN op1, immN op2) %{
10973   match(Set cr (CmpN op1 op2));
10974 
10975   format %{ "cmpl    $op1, $op2\t# compressed ptr" %}
10976   ins_encode %{
10977     __ cmp_narrow_oop($op1$$Register, (jobject)$op2$$constant);
10978   %}
10979   ins_pipe(ialu_cr_reg_imm);
10980 %}
10981 
10982 instruct compN_mem_imm(rFlagsRegU cr, memory mem, immN src)
10983 %{
10984   match(Set cr (CmpN src (LoadN mem)));
10985 
10986   format %{ "cmpl    $mem, $src\t# compressed ptr" %}
10987   ins_encode %{
10988     __ cmp_narrow_oop($mem$$Address, (jobject)$src$$constant);
10989   %}
10990   ins_pipe(ialu_cr_reg_mem);
10991 %}
10992 
10993 instruct compN_rReg_imm_klass(rFlagsRegU cr, rRegN op1, immNKlass op2) %{
10994   match(Set cr (CmpN op1 op2));
10995 
10996   format %{ "cmpl    $op1, $op2\t# compressed klass ptr" %}
10997   ins_encode %{
10998     __ cmp_narrow_klass($op1$$Register, (Klass*)$op2$$constant);
10999   %}
11000   ins_pipe(ialu_cr_reg_imm);
11001 %}
11002 
11003 instruct compN_mem_imm_klass(rFlagsRegU cr, memory mem, immNKlass src)
11004 %{
11005   match(Set cr (CmpN src (LoadNKlass mem)));
11006 
11007   format %{ "cmpl    $mem, $src\t# compressed klass ptr" %}
11008   ins_encode %{
11009     __ cmp_narrow_klass($mem$$Address, (Klass*)$src$$constant);
11010   %}
11011   ins_pipe(ialu_cr_reg_mem);
11012 %}
11013 
11014 instruct testN_reg(rFlagsReg cr, rRegN src, immN0 zero) %{
11015   match(Set cr (CmpN src zero));
11016 
11017   format %{ "testl   $src, $src\t# compressed ptr" %}
11018   ins_encode %{ __ testl($src$$Register, $src$$Register); %}
11019   ins_pipe(ialu_cr_reg_imm);
11020 %}
11021 
11022 instruct testN_mem(rFlagsReg cr, memory mem, immN0 zero)
11023 %{
11024   predicate(Universe::narrow_oop_base() != NULL);
11025   match(Set cr (CmpN (LoadN mem) zero));
11026 
11027   ins_cost(500); // XXX
11028   format %{ "testl   $mem, 0xffffffff\t# compressed ptr" %}
11029   ins_encode %{
11030     __ cmpl($mem$$Address, (int)0xFFFFFFFF);
11031   %}
11032   ins_pipe(ialu_cr_reg_mem);
11033 %}
11034 
11035 instruct testN_mem_reg0(rFlagsReg cr, memory mem, immN0 zero)
11036 %{
11037   predicate(Universe::narrow_oop_base() == NULL && (Universe::narrow_klass_base() == NULL));
11038   match(Set cr (CmpN (LoadN mem) zero));
11039 
11040   format %{ "cmpl    R12, $mem\t# compressed ptr (R12_heapbase==0)" %}
11041   ins_encode %{
11042     __ cmpl(r12, $mem$$Address);
11043   %}
11044   ins_pipe(ialu_cr_reg_mem);
11045 %}
11046 
11047 // Yanked all unsigned pointer compare operations.
11048 // Pointer compares are done with CmpP which is already unsigned.
11049 
11050 instruct compL_rReg(rFlagsReg cr, rRegL op1, rRegL op2)
11051 %{
11052   match(Set cr (CmpL op1 op2));
11053 
11054   format %{ "cmpq    $op1, $op2" %}
11055   opcode(0x3B);  /* Opcode 3B /r */
11056   ins_encode(REX_reg_reg_wide(op1, op2), OpcP, reg_reg(op1, op2));
11057   ins_pipe(ialu_cr_reg_reg);
11058 %}
11059 
11060 instruct compL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2)
11061 %{
11062   match(Set cr (CmpL op1 op2));
11063 
11064   format %{ "cmpq    $op1, $op2" %}
11065   opcode(0x81, 0x07); /* Opcode 81 /7 */
11066   ins_encode(OpcSErm_wide(op1, op2), Con8or32(op2));
11067   ins_pipe(ialu_cr_reg_imm);
11068 %}
11069 
11070 instruct compL_rReg_mem(rFlagsReg cr, rRegL op1, memory op2)
11071 %{
11072   match(Set cr (CmpL op1 (LoadL op2)));
11073 
11074   format %{ "cmpq    $op1, $op2" %}
11075   opcode(0x3B); /* Opcode 3B /r */
11076   ins_encode(REX_reg_mem_wide(op1, op2), OpcP, reg_mem(op1, op2));
11077   ins_pipe(ialu_cr_reg_mem);
11078 %}
11079 
11080 instruct testL_reg(rFlagsReg cr, rRegL src, immL0 zero)
11081 %{
11082   match(Set cr (CmpL src zero));
11083 
11084   format %{ "testq   $src, $src" %}
11085   opcode(0x85);
11086   ins_encode(REX_reg_reg_wide(src, src), OpcP, reg_reg(src, src));
11087   ins_pipe(ialu_cr_reg_imm);
11088 %}
11089 
11090 instruct testL_reg_imm(rFlagsReg cr, rRegL src, immL32 con, immL0 zero)
11091 %{
11092   match(Set cr (CmpL (AndL src con) zero));
11093 
11094   format %{ "testq   $src, $con\t# long" %}
11095   opcode(0xF7, 0x00);
11096   ins_encode(REX_reg_wide(src), OpcP, reg_opc(src), Con32(con));
11097   ins_pipe(ialu_cr_reg_imm);
11098 %}
11099 
11100 instruct testL_reg_mem(rFlagsReg cr, rRegL src, memory mem, immL0 zero)
11101 %{
11102   match(Set cr (CmpL (AndL src (LoadL mem)) zero));
11103 
11104   format %{ "testq   $src, $mem" %}
11105   opcode(0x85);
11106   ins_encode(REX_reg_mem_wide(src, mem), OpcP, reg_mem(src, mem));
11107   ins_pipe(ialu_cr_reg_mem);
11108 %}
11109 
11110 // Manifest a CmpL result in an integer register.  Very painful.
11111 // This is the test to avoid.
11112 instruct cmpL3_reg_reg(rRegI dst, rRegL src1, rRegL src2, rFlagsReg flags)
11113 %{
11114   match(Set dst (CmpL3 src1 src2));
11115   effect(KILL flags);
11116 
11117   ins_cost(275); // XXX
11118   format %{ "cmpq    $src1, $src2\t# CmpL3\n\t"
11119             "movl    $dst, -1\n\t"
11120             "jl,s    done\n\t"
11121             "setne   $dst\n\t"
11122             "movzbl  $dst, $dst\n\t"
11123     "done:" %}
11124   ins_encode(cmpl3_flag(src1, src2, dst));
11125   ins_pipe(pipe_slow);
11126 %}
11127 
11128 // Unsigned long compare Instructions; really, same as signed long except they
11129 // produce an rFlagsRegU instead of rFlagsReg.
11130 instruct compUL_rReg(rFlagsRegU cr, rRegL op1, rRegL op2)
11131 %{
11132   match(Set cr (CmpUL op1 op2));
11133 
11134   format %{ "cmpq    $op1, $op2\t# unsigned" %}
11135   opcode(0x3B);  /* Opcode 3B /r */
11136   ins_encode(REX_reg_reg_wide(op1, op2), OpcP, reg_reg(op1, op2));
11137   ins_pipe(ialu_cr_reg_reg);
11138 %}
11139 
11140 instruct compUL_rReg_imm(rFlagsRegU cr, rRegL op1, immL32 op2)
11141 %{
11142   match(Set cr (CmpUL op1 op2));
11143 
11144   format %{ "cmpq    $op1, $op2\t# unsigned" %}
11145   opcode(0x81, 0x07); /* Opcode 81 /7 */
11146   ins_encode(OpcSErm_wide(op1, op2), Con8or32(op2));
11147   ins_pipe(ialu_cr_reg_imm);
11148 %}
11149 
11150 instruct compUL_rReg_mem(rFlagsRegU cr, rRegL op1, memory op2)
11151 %{
11152   match(Set cr (CmpUL op1 (LoadL op2)));
11153 
11154   format %{ "cmpq    $op1, $op2\t# unsigned" %}
11155   opcode(0x3B); /* Opcode 3B /r */
11156   ins_encode(REX_reg_mem_wide(op1, op2), OpcP, reg_mem(op1, op2));
11157   ins_pipe(ialu_cr_reg_mem);
11158 %}
11159 
11160 instruct testUL_reg(rFlagsRegU cr, rRegL src, immL0 zero)
11161 %{
11162   match(Set cr (CmpUL src zero));
11163 
11164   format %{ "testq   $src, $src\t# unsigned" %}
11165   opcode(0x85);
11166   ins_encode(REX_reg_reg_wide(src, src), OpcP, reg_reg(src, src));
11167   ins_pipe(ialu_cr_reg_imm);
11168 %}
11169 
11170 //----------Max and Min--------------------------------------------------------
11171 // Min Instructions
11172 
11173 instruct cmovI_reg_g(rRegI dst, rRegI src, rFlagsReg cr)
11174 %{
11175   effect(USE_DEF dst, USE src, USE cr);
11176 
11177   format %{ "cmovlgt $dst, $src\t# min" %}
11178   opcode(0x0F, 0x4F);
11179   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
11180   ins_pipe(pipe_cmov_reg);
11181 %}
11182 
11183 
11184 instruct minI_rReg(rRegI dst, rRegI src)
11185 %{
11186   match(Set dst (MinI dst src));
11187 
11188   ins_cost(200);
11189   expand %{
11190     rFlagsReg cr;
11191     compI_rReg(cr, dst, src);
11192     cmovI_reg_g(dst, src, cr);
11193   %}
11194 %}
11195 
11196 instruct cmovI_reg_l(rRegI dst, rRegI src, rFlagsReg cr)
11197 %{
11198   effect(USE_DEF dst, USE src, USE cr);
11199 
11200   format %{ "cmovllt $dst, $src\t# max" %}
11201   opcode(0x0F, 0x4C);
11202   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
11203   ins_pipe(pipe_cmov_reg);
11204 %}
11205 
11206 
11207 instruct maxI_rReg(rRegI dst, rRegI src)
11208 %{
11209   match(Set dst (MaxI dst src));
11210 
11211   ins_cost(200);
11212   expand %{
11213     rFlagsReg cr;
11214     compI_rReg(cr, dst, src);
11215     cmovI_reg_l(dst, src, cr);
11216   %}
11217 %}
11218 
11219 // ============================================================================
11220 // Branch Instructions
11221 
11222 // Jump Direct - Label defines a relative address from JMP+1
11223 instruct jmpDir(label labl)
11224 %{
11225   match(Goto);
11226   effect(USE labl);
11227 
11228   ins_cost(300);
11229   format %{ "jmp     $labl" %}
11230   size(5);
11231   ins_encode %{
11232     Label* L = $labl$$label;
11233     __ jmp(*L, false); // Always long jump
11234   %}
11235   ins_pipe(pipe_jmp);
11236 %}
11237 
11238 // Jump Direct Conditional - Label defines a relative address from Jcc+1
11239 instruct jmpCon(cmpOp cop, rFlagsReg cr, label labl)
11240 %{
11241   match(If cop cr);
11242   effect(USE labl);
11243 
11244   ins_cost(300);
11245   format %{ "j$cop     $labl" %}
11246   size(6);
11247   ins_encode %{
11248     Label* L = $labl$$label;
11249     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
11250   %}
11251   ins_pipe(pipe_jcc);
11252 %}
11253 
11254 // Jump Direct Conditional - Label defines a relative address from Jcc+1
11255 instruct jmpLoopEnd(cmpOp cop, rFlagsReg cr, label labl)
11256 %{
11257   match(CountedLoopEnd cop cr);
11258   effect(USE labl);
11259 
11260   ins_cost(300);
11261   format %{ "j$cop     $labl\t# loop end" %}
11262   size(6);
11263   ins_encode %{
11264     Label* L = $labl$$label;
11265     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
11266   %}
11267   ins_pipe(pipe_jcc);
11268 %}
11269 
11270 // Jump Direct Conditional - Label defines a relative address from Jcc+1
11271 instruct jmpLoopEndU(cmpOpU cop, rFlagsRegU cmp, label labl) %{
11272   match(CountedLoopEnd cop cmp);
11273   effect(USE labl);
11274 
11275   ins_cost(300);
11276   format %{ "j$cop,u   $labl\t# loop end" %}
11277   size(6);
11278   ins_encode %{
11279     Label* L = $labl$$label;
11280     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
11281   %}
11282   ins_pipe(pipe_jcc);
11283 %}
11284 
11285 instruct jmpLoopEndUCF(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
11286   match(CountedLoopEnd cop cmp);
11287   effect(USE labl);
11288 
11289   ins_cost(200);
11290   format %{ "j$cop,u   $labl\t# loop end" %}
11291   size(6);
11292   ins_encode %{
11293     Label* L = $labl$$label;
11294     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
11295   %}
11296   ins_pipe(pipe_jcc);
11297 %}
11298 
11299 // Jump Direct Conditional - using unsigned comparison
11300 instruct jmpConU(cmpOpU cop, rFlagsRegU cmp, label labl) %{
11301   match(If cop cmp);
11302   effect(USE labl);
11303 
11304   ins_cost(300);
11305   format %{ "j$cop,u  $labl" %}
11306   size(6);
11307   ins_encode %{
11308     Label* L = $labl$$label;
11309     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
11310   %}
11311   ins_pipe(pipe_jcc);
11312 %}
11313 
11314 instruct jmpConUCF(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
11315   match(If cop cmp);
11316   effect(USE labl);
11317 
11318   ins_cost(200);
11319   format %{ "j$cop,u  $labl" %}
11320   size(6);
11321   ins_encode %{
11322     Label* L = $labl$$label;
11323     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
11324   %}
11325   ins_pipe(pipe_jcc);
11326 %}
11327 
11328 instruct jmpConUCF2(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
11329   match(If cop cmp);
11330   effect(USE labl);
11331 
11332   ins_cost(200);
11333   format %{ $$template
11334     if ($cop$$cmpcode == Assembler::notEqual) {
11335       $$emit$$"jp,u   $labl\n\t"
11336       $$emit$$"j$cop,u   $labl"
11337     } else {
11338       $$emit$$"jp,u   done\n\t"
11339       $$emit$$"j$cop,u   $labl\n\t"
11340       $$emit$$"done:"
11341     }
11342   %}
11343   ins_encode %{
11344     Label* l = $labl$$label;
11345     if ($cop$$cmpcode == Assembler::notEqual) {
11346       __ jcc(Assembler::parity, *l, false);
11347       __ jcc(Assembler::notEqual, *l, false);
11348     } else if ($cop$$cmpcode == Assembler::equal) {
11349       Label done;
11350       __ jccb(Assembler::parity, done);
11351       __ jcc(Assembler::equal, *l, false);
11352       __ bind(done);
11353     } else {
11354        ShouldNotReachHere();
11355     }
11356   %}
11357   ins_pipe(pipe_jcc);
11358 %}
11359 
11360 // ============================================================================
11361 // The 2nd slow-half of a subtype check.  Scan the subklass's 2ndary
11362 // superklass array for an instance of the superklass.  Set a hidden
11363 // internal cache on a hit (cache is checked with exposed code in
11364 // gen_subtype_check()).  Return NZ for a miss or zero for a hit.  The
11365 // encoding ALSO sets flags.
11366 
11367 instruct partialSubtypeCheck(rdi_RegP result,
11368                              rsi_RegP sub, rax_RegP super, rcx_RegI rcx,
11369                              rFlagsReg cr)
11370 %{
11371   match(Set result (PartialSubtypeCheck sub super));
11372   effect(KILL rcx, KILL cr);
11373 
11374   ins_cost(1100);  // slightly larger than the next version
11375   format %{ "movq    rdi, [$sub + in_bytes(Klass::secondary_supers_offset())]\n\t"
11376             "movl    rcx, [rdi + Array<Klass*>::length_offset_in_bytes()]\t# length to scan\n\t"
11377             "addq    rdi, Array<Klass*>::base_offset_in_bytes()\t# Skip to start of data; set NZ in case count is zero\n\t"
11378             "repne   scasq\t# Scan *rdi++ for a match with rax while rcx--\n\t"
11379             "jne,s   miss\t\t# Missed: rdi not-zero\n\t"
11380             "movq    [$sub + in_bytes(Klass::secondary_super_cache_offset())], $super\t# Hit: update cache\n\t"
11381             "xorq    $result, $result\t\t Hit: rdi zero\n\t"
11382     "miss:\t" %}
11383 
11384   opcode(0x1); // Force a XOR of RDI
11385   ins_encode(enc_PartialSubtypeCheck());
11386   ins_pipe(pipe_slow);
11387 %}
11388 
11389 instruct partialSubtypeCheck_vs_Zero(rFlagsReg cr,
11390                                      rsi_RegP sub, rax_RegP super, rcx_RegI rcx,
11391                                      immP0 zero,
11392                                      rdi_RegP result)
11393 %{
11394   match(Set cr (CmpP (PartialSubtypeCheck sub super) zero));
11395   effect(KILL rcx, KILL result);
11396 
11397   ins_cost(1000);
11398   format %{ "movq    rdi, [$sub + in_bytes(Klass::secondary_supers_offset())]\n\t"
11399             "movl    rcx, [rdi + Array<Klass*>::length_offset_in_bytes()]\t# length to scan\n\t"
11400             "addq    rdi, Array<Klass*>::base_offset_in_bytes()\t# Skip to start of data; set NZ in case count is zero\n\t"
11401             "repne   scasq\t# Scan *rdi++ for a match with rax while cx-- != 0\n\t"
11402             "jne,s   miss\t\t# Missed: flags nz\n\t"
11403             "movq    [$sub + in_bytes(Klass::secondary_super_cache_offset())], $super\t# Hit: update cache\n\t"
11404     "miss:\t" %}
11405 
11406   opcode(0x0); // No need to XOR RDI
11407   ins_encode(enc_PartialSubtypeCheck());
11408   ins_pipe(pipe_slow);
11409 %}
11410 
11411 // ============================================================================
11412 // Branch Instructions -- short offset versions
11413 //
11414 // These instructions are used to replace jumps of a long offset (the default
11415 // match) with jumps of a shorter offset.  These instructions are all tagged
11416 // with the ins_short_branch attribute, which causes the ADLC to suppress the
11417 // match rules in general matching.  Instead, the ADLC generates a conversion
11418 // method in the MachNode which can be used to do in-place replacement of the
11419 // long variant with the shorter variant.  The compiler will determine if a
11420 // branch can be taken by the is_short_branch_offset() predicate in the machine
11421 // specific code section of the file.
11422 
11423 // Jump Direct - Label defines a relative address from JMP+1
11424 instruct jmpDir_short(label labl) %{
11425   match(Goto);
11426   effect(USE labl);
11427 
11428   ins_cost(300);
11429   format %{ "jmp,s   $labl" %}
11430   size(2);
11431   ins_encode %{
11432     Label* L = $labl$$label;
11433     __ jmpb(*L);
11434   %}
11435   ins_pipe(pipe_jmp);
11436   ins_short_branch(1);
11437 %}
11438 
11439 // Jump Direct Conditional - Label defines a relative address from Jcc+1
11440 instruct jmpCon_short(cmpOp cop, rFlagsReg cr, label labl) %{
11441   match(If cop cr);
11442   effect(USE labl);
11443 
11444   ins_cost(300);
11445   format %{ "j$cop,s   $labl" %}
11446   size(2);
11447   ins_encode %{
11448     Label* L = $labl$$label;
11449     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
11450   %}
11451   ins_pipe(pipe_jcc);
11452   ins_short_branch(1);
11453 %}
11454 
11455 // Jump Direct Conditional - Label defines a relative address from Jcc+1
11456 instruct jmpLoopEnd_short(cmpOp cop, rFlagsReg cr, label labl) %{
11457   match(CountedLoopEnd cop cr);
11458   effect(USE labl);
11459 
11460   ins_cost(300);
11461   format %{ "j$cop,s   $labl\t# loop end" %}
11462   size(2);
11463   ins_encode %{
11464     Label* L = $labl$$label;
11465     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
11466   %}
11467   ins_pipe(pipe_jcc);
11468   ins_short_branch(1);
11469 %}
11470 
11471 // Jump Direct Conditional - Label defines a relative address from Jcc+1
11472 instruct jmpLoopEndU_short(cmpOpU cop, rFlagsRegU cmp, label labl) %{
11473   match(CountedLoopEnd cop cmp);
11474   effect(USE labl);
11475 
11476   ins_cost(300);
11477   format %{ "j$cop,us  $labl\t# loop end" %}
11478   size(2);
11479   ins_encode %{
11480     Label* L = $labl$$label;
11481     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
11482   %}
11483   ins_pipe(pipe_jcc);
11484   ins_short_branch(1);
11485 %}
11486 
11487 instruct jmpLoopEndUCF_short(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
11488   match(CountedLoopEnd cop cmp);
11489   effect(USE labl);
11490 
11491   ins_cost(300);
11492   format %{ "j$cop,us  $labl\t# loop end" %}
11493   size(2);
11494   ins_encode %{
11495     Label* L = $labl$$label;
11496     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
11497   %}
11498   ins_pipe(pipe_jcc);
11499   ins_short_branch(1);
11500 %}
11501 
11502 // Jump Direct Conditional - using unsigned comparison
11503 instruct jmpConU_short(cmpOpU cop, rFlagsRegU cmp, label labl) %{
11504   match(If cop cmp);
11505   effect(USE labl);
11506 
11507   ins_cost(300);
11508   format %{ "j$cop,us  $labl" %}
11509   size(2);
11510   ins_encode %{
11511     Label* L = $labl$$label;
11512     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
11513   %}
11514   ins_pipe(pipe_jcc);
11515   ins_short_branch(1);
11516 %}
11517 
11518 instruct jmpConUCF_short(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
11519   match(If cop cmp);
11520   effect(USE labl);
11521 
11522   ins_cost(300);
11523   format %{ "j$cop,us  $labl" %}
11524   size(2);
11525   ins_encode %{
11526     Label* L = $labl$$label;
11527     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
11528   %}
11529   ins_pipe(pipe_jcc);
11530   ins_short_branch(1);
11531 %}
11532 
11533 instruct jmpConUCF2_short(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
11534   match(If cop cmp);
11535   effect(USE labl);
11536 
11537   ins_cost(300);
11538   format %{ $$template
11539     if ($cop$$cmpcode == Assembler::notEqual) {
11540       $$emit$$"jp,u,s   $labl\n\t"
11541       $$emit$$"j$cop,u,s   $labl"
11542     } else {
11543       $$emit$$"jp,u,s   done\n\t"
11544       $$emit$$"j$cop,u,s  $labl\n\t"
11545       $$emit$$"done:"
11546     }
11547   %}
11548   size(4);
11549   ins_encode %{
11550     Label* l = $labl$$label;
11551     if ($cop$$cmpcode == Assembler::notEqual) {
11552       __ jccb(Assembler::parity, *l);
11553       __ jccb(Assembler::notEqual, *l);
11554     } else if ($cop$$cmpcode == Assembler::equal) {
11555       Label done;
11556       __ jccb(Assembler::parity, done);
11557       __ jccb(Assembler::equal, *l);
11558       __ bind(done);
11559     } else {
11560        ShouldNotReachHere();
11561     }
11562   %}
11563   ins_pipe(pipe_jcc);
11564   ins_short_branch(1);
11565 %}
11566 
11567 // ============================================================================
11568 // inlined locking and unlocking
11569 
11570 instruct cmpFastLockRTM(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI tmp, rdx_RegI scr, rRegI cx1, rRegI cx2) %{
11571   predicate(Compile::current()->use_rtm());
11572   match(Set cr (FastLock object box));
11573   effect(TEMP tmp, TEMP scr, TEMP cx1, TEMP cx2, USE_KILL box);
11574   ins_cost(300);
11575   format %{ "fastlock $object,$box\t! kills $box,$tmp,$scr,$cx1,$cx2" %}
11576   ins_encode %{
11577     __ fast_lock($object$$Register, $box$$Register, $tmp$$Register,
11578                  $scr$$Register, $cx1$$Register, $cx2$$Register,
11579                  _counters, _rtm_counters, _stack_rtm_counters,
11580                  ((Method*)(ra_->C->method()->constant_encoding()))->method_data(),
11581                  true, ra_->C->profile_rtm());
11582   %}
11583   ins_pipe(pipe_slow);
11584 %}
11585 
11586 instruct cmpFastLock(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI tmp, rRegP scr) %{
11587   predicate(!Compile::current()->use_rtm());
11588   match(Set cr (FastLock object box));
11589   effect(TEMP tmp, TEMP scr, USE_KILL box);
11590   ins_cost(300);
11591   format %{ "fastlock $object,$box\t! kills $box,$tmp,$scr" %}
11592   ins_encode %{
11593     __ fast_lock($object$$Register, $box$$Register, $tmp$$Register,
11594                  $scr$$Register, noreg, noreg, _counters, NULL, NULL, NULL, false, false);
11595   %}
11596   ins_pipe(pipe_slow);
11597 %}
11598 
11599 instruct cmpFastUnlock(rFlagsReg cr, rRegP object, rax_RegP box, rRegP tmp) %{
11600   match(Set cr (FastUnlock object box));
11601   effect(TEMP tmp, USE_KILL box);
11602   ins_cost(300);
11603   format %{ "fastunlock $object,$box\t! kills $box,$tmp" %}
11604   ins_encode %{
11605     __ fast_unlock($object$$Register, $box$$Register, $tmp$$Register, ra_->C->use_rtm());
11606   %}
11607   ins_pipe(pipe_slow);
11608 %}
11609 
11610 
11611 // ============================================================================
11612 // Safepoint Instructions
11613 instruct safePoint_poll(rFlagsReg cr)
11614 %{
11615   predicate(!Assembler::is_polling_page_far());
11616   match(SafePoint);
11617   effect(KILL cr);
11618 
11619   format %{ "testl  rax, [rip + #offset_to_poll_page]\t"
11620             "# Safepoint: poll for GC" %}
11621   ins_cost(125);
11622   ins_encode %{
11623     AddressLiteral addr(os::get_polling_page(), relocInfo::poll_type);
11624     __ testl(rax, addr);
11625   %}
11626   ins_pipe(ialu_reg_mem);
11627 %}
11628 
11629 instruct safePoint_poll_far(rFlagsReg cr, rRegP poll)
11630 %{
11631   predicate(Assembler::is_polling_page_far());
11632   match(SafePoint poll);
11633   effect(KILL cr, USE poll);
11634 
11635   format %{ "testl  rax, [$poll]\t"
11636             "# Safepoint: poll for GC" %}
11637   ins_cost(125);
11638   ins_encode %{
11639     __ relocate(relocInfo::poll_type);
11640     __ testl(rax, Address($poll$$Register, 0));
11641   %}
11642   ins_pipe(ialu_reg_mem);
11643 %}
11644 
11645 // ============================================================================
11646 // Procedure Call/Return Instructions
11647 // Call Java Static Instruction
11648 // Note: If this code changes, the corresponding ret_addr_offset() and
11649 //       compute_padding() functions will have to be adjusted.
11650 instruct CallStaticJavaDirect(method meth) %{
11651   match(CallStaticJava);
11652   effect(USE meth);
11653 
11654   ins_cost(300);
11655   format %{ "call,static " %}
11656   opcode(0xE8); /* E8 cd */
11657   ins_encode(clear_avx, Java_Static_Call(meth), call_epilog);
11658   ins_pipe(pipe_slow);
11659   ins_alignment(4);
11660 %}
11661 
11662 // Call Java Dynamic Instruction
11663 // Note: If this code changes, the corresponding ret_addr_offset() and
11664 //       compute_padding() functions will have to be adjusted.
11665 instruct CallDynamicJavaDirect(method meth)
11666 %{
11667   match(CallDynamicJava);
11668   effect(USE meth);
11669 
11670   ins_cost(300);
11671   format %{ "movq    rax, #Universe::non_oop_word()\n\t"
11672             "call,dynamic " %}
11673   ins_encode(clear_avx, Java_Dynamic_Call(meth), call_epilog);
11674   ins_pipe(pipe_slow);
11675   ins_alignment(4);
11676 %}
11677 
11678 // Call Runtime Instruction
11679 instruct CallRuntimeDirect(method meth)
11680 %{
11681   match(CallRuntime);
11682   effect(USE meth);
11683 
11684   ins_cost(300);
11685   format %{ "call,runtime " %}
11686   ins_encode(clear_avx, Java_To_Runtime(meth));
11687   ins_pipe(pipe_slow);
11688 %}
11689 
11690 // Call runtime without safepoint
11691 instruct CallLeafDirect(method meth)
11692 %{
11693   match(CallLeaf);
11694   effect(USE meth);
11695 
11696   ins_cost(300);
11697   format %{ "call_leaf,runtime " %}
11698   ins_encode(clear_avx, Java_To_Runtime(meth));
11699   ins_pipe(pipe_slow);
11700 %}
11701 
11702 // Call runtime without safepoint
11703 instruct CallLeafNoFPDirect(method meth)
11704 %{
11705   match(CallLeafNoFP);
11706   effect(USE meth);
11707 
11708   ins_cost(300);
11709   format %{ "call_leaf_nofp,runtime " %}
11710   ins_encode(Java_To_Runtime(meth));
11711   ins_pipe(pipe_slow);
11712 %}
11713 
11714 // Return Instruction
11715 // Remove the return address & jump to it.
11716 // Notice: We always emit a nop after a ret to make sure there is room
11717 // for safepoint patching
11718 instruct Ret()
11719 %{
11720   match(Return);
11721 
11722   format %{ "ret" %}
11723   opcode(0xC3);
11724   ins_encode(OpcP);
11725   ins_pipe(pipe_jmp);
11726 %}
11727 
11728 // Tail Call; Jump from runtime stub to Java code.
11729 // Also known as an 'interprocedural jump'.
11730 // Target of jump will eventually return to caller.
11731 // TailJump below removes the return address.
11732 instruct TailCalljmpInd(no_rbp_RegP jump_target, rbx_RegP method_oop)
11733 %{
11734   match(TailCall jump_target method_oop);
11735 
11736   ins_cost(300);
11737   format %{ "jmp     $jump_target\t# rbx holds method oop" %}
11738   opcode(0xFF, 0x4); /* Opcode FF /4 */
11739   ins_encode(REX_reg(jump_target), OpcP, reg_opc(jump_target));
11740   ins_pipe(pipe_jmp);
11741 %}
11742 
11743 // Tail Jump; remove the return address; jump to target.
11744 // TailCall above leaves the return address around.
11745 instruct tailjmpInd(no_rbp_RegP jump_target, rax_RegP ex_oop)
11746 %{
11747   match(TailJump jump_target ex_oop);
11748 
11749   ins_cost(300);
11750   format %{ "popq    rdx\t# pop return address\n\t"
11751             "jmp     $jump_target" %}
11752   opcode(0xFF, 0x4); /* Opcode FF /4 */
11753   ins_encode(Opcode(0x5a), // popq rdx
11754              REX_reg(jump_target), OpcP, reg_opc(jump_target));
11755   ins_pipe(pipe_jmp);
11756 %}
11757 
11758 // Create exception oop: created by stack-crawling runtime code.
11759 // Created exception is now available to this handler, and is setup
11760 // just prior to jumping to this handler.  No code emitted.
11761 instruct CreateException(rax_RegP ex_oop)
11762 %{
11763   match(Set ex_oop (CreateEx));
11764 
11765   size(0);
11766   // use the following format syntax
11767   format %{ "# exception oop is in rax; no code emitted" %}
11768   ins_encode();
11769   ins_pipe(empty);
11770 %}
11771 
11772 // Rethrow exception:
11773 // The exception oop will come in the first argument position.
11774 // Then JUMP (not call) to the rethrow stub code.
11775 instruct RethrowException()
11776 %{
11777   match(Rethrow);
11778 
11779   // use the following format syntax
11780   format %{ "jmp     rethrow_stub" %}
11781   ins_encode(enc_rethrow);
11782   ins_pipe(pipe_jmp);
11783 %}
11784 
11785 
11786 // ============================================================================
11787 // This name is KNOWN by the ADLC and cannot be changed.
11788 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
11789 // for this guy.
11790 instruct tlsLoadP(r15_RegP dst) %{
11791   match(Set dst (ThreadLocal));
11792   effect(DEF dst);
11793 
11794   size(0);
11795   format %{ "# TLS is in R15" %}
11796   ins_encode( /*empty encoding*/ );
11797   ins_pipe(ialu_reg_reg);
11798 %}
11799 
11800 
11801 //----------PEEPHOLE RULES-----------------------------------------------------
11802 // These must follow all instruction definitions as they use the names
11803 // defined in the instructions definitions.
11804 //
11805 // peepmatch ( root_instr_name [preceding_instruction]* );
11806 //
11807 // peepconstraint %{
11808 // (instruction_number.operand_name relational_op instruction_number.operand_name
11809 //  [, ...] );
11810 // // instruction numbers are zero-based using left to right order in peepmatch
11811 //
11812 // peepreplace ( instr_name  ( [instruction_number.operand_name]* ) );
11813 // // provide an instruction_number.operand_name for each operand that appears
11814 // // in the replacement instruction's match rule
11815 //
11816 // ---------VM FLAGS---------------------------------------------------------
11817 //
11818 // All peephole optimizations can be turned off using -XX:-OptoPeephole
11819 //
11820 // Each peephole rule is given an identifying number starting with zero and
11821 // increasing by one in the order seen by the parser.  An individual peephole
11822 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
11823 // on the command-line.
11824 //
11825 // ---------CURRENT LIMITATIONS----------------------------------------------
11826 //
11827 // Only match adjacent instructions in same basic block
11828 // Only equality constraints
11829 // Only constraints between operands, not (0.dest_reg == RAX_enc)
11830 // Only one replacement instruction
11831 //
11832 // ---------EXAMPLE----------------------------------------------------------
11833 //
11834 // // pertinent parts of existing instructions in architecture description
11835 // instruct movI(rRegI dst, rRegI src)
11836 // %{
11837 //   match(Set dst (CopyI src));
11838 // %}
11839 //
11840 // instruct incI_rReg(rRegI dst, immI1 src, rFlagsReg cr)
11841 // %{
11842 //   match(Set dst (AddI dst src));
11843 //   effect(KILL cr);
11844 // %}
11845 //
11846 // // Change (inc mov) to lea
11847 // peephole %{
11848 //   // increment preceeded by register-register move
11849 //   peepmatch ( incI_rReg movI );
11850 //   // require that the destination register of the increment
11851 //   // match the destination register of the move
11852 //   peepconstraint ( 0.dst == 1.dst );
11853 //   // construct a replacement instruction that sets
11854 //   // the destination to ( move's source register + one )
11855 //   peepreplace ( leaI_rReg_immI( 0.dst 1.src 0.src ) );
11856 // %}
11857 //
11858 
11859 // Implementation no longer uses movX instructions since
11860 // machine-independent system no longer uses CopyX nodes.
11861 //
11862 // peephole
11863 // %{
11864 //   peepmatch (incI_rReg movI);
11865 //   peepconstraint (0.dst == 1.dst);
11866 //   peepreplace (leaI_rReg_immI(0.dst 1.src 0.src));
11867 // %}
11868 
11869 // peephole
11870 // %{
11871 //   peepmatch (decI_rReg movI);
11872 //   peepconstraint (0.dst == 1.dst);
11873 //   peepreplace (leaI_rReg_immI(0.dst 1.src 0.src));
11874 // %}
11875 
11876 // peephole
11877 // %{
11878 //   peepmatch (addI_rReg_imm movI);
11879 //   peepconstraint (0.dst == 1.dst);
11880 //   peepreplace (leaI_rReg_immI(0.dst 1.src 0.src));
11881 // %}
11882 
11883 // peephole
11884 // %{
11885 //   peepmatch (incL_rReg movL);
11886 //   peepconstraint (0.dst == 1.dst);
11887 //   peepreplace (leaL_rReg_immL(0.dst 1.src 0.src));
11888 // %}
11889 
11890 // peephole
11891 // %{
11892 //   peepmatch (decL_rReg movL);
11893 //   peepconstraint (0.dst == 1.dst);
11894 //   peepreplace (leaL_rReg_immL(0.dst 1.src 0.src));
11895 // %}
11896 
11897 // peephole
11898 // %{
11899 //   peepmatch (addL_rReg_imm movL);
11900 //   peepconstraint (0.dst == 1.dst);
11901 //   peepreplace (leaL_rReg_immL(0.dst 1.src 0.src));
11902 // %}
11903 
11904 // peephole
11905 // %{
11906 //   peepmatch (addP_rReg_imm movP);
11907 //   peepconstraint (0.dst == 1.dst);
11908 //   peepreplace (leaP_rReg_imm(0.dst 1.src 0.src));
11909 // %}
11910 
11911 // // Change load of spilled value to only a spill
11912 // instruct storeI(memory mem, rRegI src)
11913 // %{
11914 //   match(Set mem (StoreI mem src));
11915 // %}
11916 //
11917 // instruct loadI(rRegI dst, memory mem)
11918 // %{
11919 //   match(Set dst (LoadI mem));
11920 // %}
11921 //
11922 
11923 peephole
11924 %{
11925   peepmatch (loadI storeI);
11926   peepconstraint (1.src == 0.dst, 1.mem == 0.mem);
11927   peepreplace (storeI(1.mem 1.mem 1.src));
11928 %}
11929 
11930 peephole
11931 %{
11932   peepmatch (loadL storeL);
11933   peepconstraint (1.src == 0.dst, 1.mem == 0.mem);
11934   peepreplace (storeL(1.mem 1.mem 1.src));
11935 %}
11936 
11937 //----------SMARTSPILL RULES---------------------------------------------------
11938 // These must follow all instruction definitions as they use the names
11939 // defined in the instructions definitions.