1 //
   2 // Copyright (c) 1997, 2013, Oracle and/or its affiliates. All rights reserved.
   3 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4 //
   5 // This code is free software; you can redistribute it and/or modify it
   6 // under the terms of the GNU General Public License version 2 only, as
   7 // published by the Free Software Foundation.
   8 //
   9 // This code is distributed in the hope that it will be useful, but WITHOUT
  10 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11 // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12 // version 2 for more details (a copy is included in the LICENSE file that
  13 // accompanied this code).
  14 //
  15 // You should have received a copy of the GNU General Public License version
  16 // 2 along with this work; if not, write to the Free Software Foundation,
  17 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18 //
  19 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20 // or visit www.oracle.com if you need additional information or have any
  21 // questions.
  22 //
  23 //
  24 
  25 // X86 Architecture Description File
  26 
  27 //----------REGISTER DEFINITION BLOCK------------------------------------------
  28 // This information is used by the matcher and the register allocator to
  29 // describe individual registers and classes of registers within the target
  30 // archtecture.
  31 
  32 register %{
  33 //----------Architecture Description Register Definitions----------------------
  34 // General Registers
  35 // "reg_def"  name ( register save type, C convention save type,
  36 //                   ideal register type, encoding );
  37 // Register Save Types:
  38 //
  39 // NS  = No-Save:       The register allocator assumes that these registers
  40 //                      can be used without saving upon entry to the method, &
  41 //                      that they do not need to be saved at call sites.
  42 //
  43 // SOC = Save-On-Call:  The register allocator assumes that these registers
  44 //                      can be used without saving upon entry to the method,
  45 //                      but that they must be saved at call sites.
  46 //
  47 // SOE = Save-On-Entry: The register allocator assumes that these registers
  48 //                      must be saved before using them upon entry to the
  49 //                      method, but they do not need to be saved at call
  50 //                      sites.
  51 //
  52 // AS  = Always-Save:   The register allocator assumes that these registers
  53 //                      must be saved before using them upon entry to the
  54 //                      method, & that they must be saved at call sites.
  55 //
  56 // Ideal Register Type is used to determine how to save & restore a
  57 // register.  Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
  58 // spilled with LoadP/StoreP.  If the register supports both, use Op_RegI.
  59 //
  60 // The encoding number is the actual bit-pattern placed into the opcodes.
  61 
  62 // General Registers
  63 // Previously set EBX, ESI, and EDI as save-on-entry for java code
  64 // Turn off SOE in java-code due to frequent use of uncommon-traps.
  65 // Now that allocator is better, turn on ESI and EDI as SOE registers.
  66 
  67 reg_def EBX(SOC, SOE, Op_RegI, 3, rbx->as_VMReg());
  68 reg_def ECX(SOC, SOC, Op_RegI, 1, rcx->as_VMReg());
  69 reg_def ESI(SOC, SOE, Op_RegI, 6, rsi->as_VMReg());
  70 reg_def EDI(SOC, SOE, Op_RegI, 7, rdi->as_VMReg());
  71 // now that adapter frames are gone EBP is always saved and restored by the prolog/epilog code
  72 reg_def EBP(NS, SOE, Op_RegI, 5, rbp->as_VMReg());
  73 reg_def EDX(SOC, SOC, Op_RegI, 2, rdx->as_VMReg());
  74 reg_def EAX(SOC, SOC, Op_RegI, 0, rax->as_VMReg());
  75 reg_def ESP( NS,  NS, Op_RegI, 4, rsp->as_VMReg());
  76 
  77 // Float registers.  We treat TOS/FPR0 special.  It is invisible to the
  78 // allocator, and only shows up in the encodings.
  79 reg_def FPR0L( SOC, SOC, Op_RegF, 0, VMRegImpl::Bad());
  80 reg_def FPR0H( SOC, SOC, Op_RegF, 0, VMRegImpl::Bad());
  81 // Ok so here's the trick FPR1 is really st(0) except in the midst
  82 // of emission of assembly for a machnode. During the emission the fpu stack
  83 // is pushed making FPR1 == st(1) temporarily. However at any safepoint
  84 // the stack will not have this element so FPR1 == st(0) from the
  85 // oopMap viewpoint. This same weirdness with numbering causes
  86 // instruction encoding to have to play games with the register
  87 // encode to correct for this 0/1 issue. See MachSpillCopyNode::implementation
  88 // where it does flt->flt moves to see an example
  89 //
  90 reg_def FPR1L( SOC, SOC, Op_RegF, 1, as_FloatRegister(0)->as_VMReg());
  91 reg_def FPR1H( SOC, SOC, Op_RegF, 1, as_FloatRegister(0)->as_VMReg()->next());
  92 reg_def FPR2L( SOC, SOC, Op_RegF, 2, as_FloatRegister(1)->as_VMReg());
  93 reg_def FPR2H( SOC, SOC, Op_RegF, 2, as_FloatRegister(1)->as_VMReg()->next());
  94 reg_def FPR3L( SOC, SOC, Op_RegF, 3, as_FloatRegister(2)->as_VMReg());
  95 reg_def FPR3H( SOC, SOC, Op_RegF, 3, as_FloatRegister(2)->as_VMReg()->next());
  96 reg_def FPR4L( SOC, SOC, Op_RegF, 4, as_FloatRegister(3)->as_VMReg());
  97 reg_def FPR4H( SOC, SOC, Op_RegF, 4, as_FloatRegister(3)->as_VMReg()->next());
  98 reg_def FPR5L( SOC, SOC, Op_RegF, 5, as_FloatRegister(4)->as_VMReg());
  99 reg_def FPR5H( SOC, SOC, Op_RegF, 5, as_FloatRegister(4)->as_VMReg()->next());
 100 reg_def FPR6L( SOC, SOC, Op_RegF, 6, as_FloatRegister(5)->as_VMReg());
 101 reg_def FPR6H( SOC, SOC, Op_RegF, 6, as_FloatRegister(5)->as_VMReg()->next());
 102 reg_def FPR7L( SOC, SOC, Op_RegF, 7, as_FloatRegister(6)->as_VMReg());
 103 reg_def FPR7H( SOC, SOC, Op_RegF, 7, as_FloatRegister(6)->as_VMReg()->next());
 104 
 105 // Specify priority of register selection within phases of register
 106 // allocation.  Highest priority is first.  A useful heuristic is to
 107 // give registers a low priority when they are required by machine
 108 // instructions, like EAX and EDX.  Registers which are used as
 109 // pairs must fall on an even boundary (witness the FPR#L's in this list).
 110 // For the Intel integer registers, the equivalent Long pairs are
 111 // EDX:EAX, EBX:ECX, and EDI:EBP.
 112 alloc_class chunk0( ECX,   EBX,   EBP,   EDI,   EAX,   EDX,   ESI, ESP,
 113                     FPR0L, FPR0H, FPR1L, FPR1H, FPR2L, FPR2H,
 114                     FPR3L, FPR3H, FPR4L, FPR4H, FPR5L, FPR5H,
 115                     FPR6L, FPR6H, FPR7L, FPR7H );
 116 
 117 
 118 //----------Architecture Description Register Classes--------------------------
 119 // Several register classes are automatically defined based upon information in
 120 // this architecture description.
 121 // 1) reg_class inline_cache_reg           ( /* as def'd in frame section */ )
 122 // 2) reg_class compiler_method_oop_reg    ( /* as def'd in frame section */ )
 123 // 2) reg_class interpreter_method_oop_reg ( /* as def'd in frame section */ )
 124 // 3) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
 125 //
 126 // Class for no registers (empty set).
 127 reg_class no_reg();
 128 
 129 // Class for all registers
 130 reg_class any_reg_with_ebp(EAX, EDX, EBP, EDI, ESI, ECX, EBX, ESP);
 131 // Class for all registers (excluding EBP)
 132 reg_class any_reg_no_ebp(EAX, EDX, EDI, ESI, ECX, EBX, ESP);
 133 // Dynamic register class that selects at runtime between register classes
 134 // any_reg and any_no_ebp_reg (depending on the value of the flag PreserveFramePointer). 
 135 // Equivalent to: return PreserveFramePointer ? any_no_ebp_reg : any_reg;
 136 reg_class_dynamic any_reg(any_reg_no_ebp, any_reg_with_ebp, %{ PreserveFramePointer %});
 137 
 138 // Class for general registers
 139 reg_class int_reg_with_ebp(EAX, EDX, EBP, EDI, ESI, ECX, EBX);
 140 // Class for general registers (excluding EBP).
 141 // This register class can be used for implicit null checks on win95.
 142 // It is also safe for use by tailjumps (we don't want to allocate in ebp).
 143 // Used also if the PreserveFramePointer flag is true.
 144 reg_class int_reg_no_ebp(EAX, EDX, EDI, ESI, ECX, EBX);
 145 // Dynamic register class that selects between int_reg and int_reg_no_ebp.
 146 reg_class_dynamic int_reg(int_reg_no_ebp, int_reg_with_ebp, %{ PreserveFramePointer %});
 147 
 148 // Class of "X" registers
 149 reg_class int_x_reg(EBX, ECX, EDX, EAX);
 150 
 151 // Class of registers that can appear in an address with no offset.
 152 // EBP and ESP require an extra instruction byte for zero offset.
 153 // Used in fast-unlock
 154 reg_class p_reg(EDX, EDI, ESI, EBX);
 155 
 156 // Class for general registers excluding ECX
 157 reg_class ncx_reg_with_ebp(EAX, EDX, EBP, EDI, ESI, EBX);
 158 // Class for general registers excluding ECX (and EBP)
 159 reg_class ncx_reg_no_ebp(EAX, EDX, EDI, ESI, EBX);
 160 // Dynamic register class that selects between ncx_reg and ncx_reg_no_ebp.
 161 reg_class_dynamic ncx_reg(ncx_reg_no_ebp, ncx_reg_with_ebp, %{ PreserveFramePointer %});
 162 
 163 // Class for general registers excluding EAX
 164 reg_class nax_reg(EDX, EDI, ESI, ECX, EBX);
 165 
 166 // Class for general registers excluding EAX and EBX.
 167 reg_class nabx_reg_with_ebp(EDX, EDI, ESI, ECX, EBP);
 168 // Class for general registers excluding EAX and EBX (and EBP)
 169 reg_class nabx_reg_no_ebp(EDX, EDI, ESI, ECX);
 170 // Dynamic register class that selects between nabx_reg and nabx_reg_no_ebp.
 171 reg_class_dynamic nabx_reg(nabx_reg_no_ebp, nabx_reg_with_ebp, %{ PreserveFramePointer %});
 172 
 173 // Class of EAX (for multiply and divide operations)
 174 reg_class eax_reg(EAX);
 175 
 176 // Class of EBX (for atomic add)
 177 reg_class ebx_reg(EBX);
 178 
 179 // Class of ECX (for shift and JCXZ operations and cmpLTMask)
 180 reg_class ecx_reg(ECX);
 181 
 182 // Class of EDX (for multiply and divide operations)
 183 reg_class edx_reg(EDX);
 184 
 185 // Class of EDI (for synchronization)
 186 reg_class edi_reg(EDI);
 187 
 188 // Class of ESI (for synchronization)
 189 reg_class esi_reg(ESI);
 190 
 191 // Singleton class for stack pointer
 192 reg_class sp_reg(ESP);
 193 
 194 // Singleton class for instruction pointer
 195 // reg_class ip_reg(EIP);
 196 
 197 // Class of integer register pairs
 198 reg_class long_reg_with_ebp( EAX,EDX, ECX,EBX, EBP,EDI );
 199 // Class of integer register pairs (excluding EBP and EDI);
 200 reg_class long_reg_no_ebp( EAX,EDX, ECX,EBX );
 201 // Dynamic register class that selects between long_reg and long_reg_no_ebp.
 202 reg_class_dynamic long_reg(long_reg_no_ebp, long_reg_with_ebp, %{ PreserveFramePointer %});
 203 
 204 // Class of integer register pairs that aligns with calling convention
 205 reg_class eadx_reg( EAX,EDX );
 206 reg_class ebcx_reg( ECX,EBX );
 207 
 208 // Not AX or DX, used in divides
 209 reg_class nadx_reg_with_ebp(EBX, ECX, ESI, EDI, EBP);
 210 // Not AX or DX (and neither EBP), used in divides
 211 reg_class nadx_reg_no_ebp(EBX, ECX, ESI, EDI);
 212 // Dynamic register class that selects between nadx_reg and nadx_reg_no_ebp.
 213 reg_class_dynamic nadx_reg(nadx_reg_no_ebp, nadx_reg_with_ebp, %{ PreserveFramePointer %});
 214 
 215 // Floating point registers.  Notice FPR0 is not a choice.
 216 // FPR0 is not ever allocated; we use clever encodings to fake
 217 // a 2-address instructions out of Intels FP stack.
 218 reg_class fp_flt_reg( FPR1L,FPR2L,FPR3L,FPR4L,FPR5L,FPR6L,FPR7L );
 219 
 220 reg_class fp_dbl_reg( FPR1L,FPR1H, FPR2L,FPR2H, FPR3L,FPR3H,
 221                       FPR4L,FPR4H, FPR5L,FPR5H, FPR6L,FPR6H,
 222                       FPR7L,FPR7H );
 223 
 224 reg_class fp_flt_reg0( FPR1L );
 225 reg_class fp_dbl_reg0( FPR1L,FPR1H );
 226 reg_class fp_dbl_reg1( FPR2L,FPR2H );
 227 reg_class fp_dbl_notreg0( FPR2L,FPR2H, FPR3L,FPR3H, FPR4L,FPR4H,
 228                           FPR5L,FPR5H, FPR6L,FPR6H, FPR7L,FPR7H );
 229 
 230 %}
 231 
 232 source_hpp %{
 233 #if INCLUDE_ALL_GCS
 234 #include "shenandoahBarrierSetAssembler_x86.hpp"
 235 #endif
 236 %}
 237 
 238 //----------SOURCE BLOCK-------------------------------------------------------
 239 // This is a block of C++ code which provides values, functions, and
 240 // definitions necessary in the rest of the architecture description
 241 source_hpp %{
 242 // Must be visible to the DFA in dfa_x86_32.cpp
 243 extern bool is_operand_hi32_zero(Node* n);
 244 %}
 245 
 246 source %{
 247 #define   RELOC_IMM32    Assembler::imm_operand
 248 #define   RELOC_DISP32   Assembler::disp32_operand
 249 
 250 #define __ _masm.
 251 
 252 // How to find the high register of a Long pair, given the low register
 253 #define   HIGH_FROM_LOW(x) ((x)+2)
 254 
 255 // These masks are used to provide 128-bit aligned bitmasks to the XMM
 256 // instructions, to allow sign-masking or sign-bit flipping.  They allow
 257 // fast versions of NegF/NegD and AbsF/AbsD.
 258 
 259 // Note: 'double' and 'long long' have 32-bits alignment on x86.
 260 static jlong* double_quadword(jlong *adr, jlong lo, jlong hi) {
 261   // Use the expression (adr)&(~0xF) to provide 128-bits aligned address
 262   // of 128-bits operands for SSE instructions.
 263   jlong *operand = (jlong*)(((uintptr_t)adr)&((uintptr_t)(~0xF)));
 264   // Store the value to a 128-bits operand.
 265   operand[0] = lo;
 266   operand[1] = hi;
 267   return operand;
 268 }
 269 
 270 // Buffer for 128-bits masks used by SSE instructions.
 271 static jlong fp_signmask_pool[(4+1)*2]; // 4*128bits(data) + 128bits(alignment)
 272 
 273 // Static initialization during VM startup.
 274 static jlong *float_signmask_pool  = double_quadword(&fp_signmask_pool[1*2], CONST64(0x7FFFFFFF7FFFFFFF), CONST64(0x7FFFFFFF7FFFFFFF));
 275 static jlong *double_signmask_pool = double_quadword(&fp_signmask_pool[2*2], CONST64(0x7FFFFFFFFFFFFFFF), CONST64(0x7FFFFFFFFFFFFFFF));
 276 static jlong *float_signflip_pool  = double_quadword(&fp_signmask_pool[3*2], CONST64(0x8000000080000000), CONST64(0x8000000080000000));
 277 static jlong *double_signflip_pool = double_quadword(&fp_signmask_pool[4*2], CONST64(0x8000000000000000), CONST64(0x8000000000000000));
 278 
 279 // Offset hacking within calls.
 280 static int pre_call_resets_size() {
 281   int size = 0;
 282   Compile* C = Compile::current();
 283   if (C->in_24_bit_fp_mode()) {
 284     size += 6; // fldcw
 285   }
 286   if (C->max_vector_size() > 16) {
 287     size += 3; // vzeroupper
 288   }
 289   return size;
 290 }
 291 
 292 // !!!!! Special hack to get all type of calls to specify the byte offset
 293 //       from the start of the call to the point where the return address
 294 //       will point.
 295 int MachCallStaticJavaNode::ret_addr_offset() {
 296   return 5 + pre_call_resets_size();  // 5 bytes from start of call to where return address points  
 297 }
 298 
 299 int MachCallDynamicJavaNode::ret_addr_offset() {
 300   return 10 + pre_call_resets_size();  // 10 bytes from start of call to where return address points
 301 }
 302 
 303 static int sizeof_FFree_Float_Stack_All = -1;
 304 
 305 int MachCallRuntimeNode::ret_addr_offset() {
 306   assert(sizeof_FFree_Float_Stack_All != -1, "must have been emitted already");
 307   return sizeof_FFree_Float_Stack_All + 5 + pre_call_resets_size();
 308 }
 309 
 310 // Indicate if the safepoint node needs the polling page as an input.
 311 // Since x86 does have absolute addressing, it doesn't.
 312 bool SafePointNode::needs_polling_address_input() {
 313   return false;
 314 }
 315 
 316 //
 317 // Compute padding required for nodes which need alignment
 318 //
 319 
 320 // The address of the call instruction needs to be 4-byte aligned to
 321 // ensure that it does not span a cache line so that it can be patched.
 322 int CallStaticJavaDirectNode::compute_padding(int current_offset) const {
 323   current_offset += pre_call_resets_size();  // skip fldcw, if any
 324   current_offset += 1;      // skip call opcode byte
 325   return round_to(current_offset, alignment_required()) - current_offset;
 326 }
 327 
 328 // The address of the call instruction needs to be 4-byte aligned to
 329 // ensure that it does not span a cache line so that it can be patched.
 330 int CallDynamicJavaDirectNode::compute_padding(int current_offset) const {
 331   current_offset += pre_call_resets_size();  // skip fldcw, if any
 332   current_offset += 5;      // skip MOV instruction
 333   current_offset += 1;      // skip call opcode byte
 334   return round_to(current_offset, alignment_required()) - current_offset;
 335 }
 336 
 337 // EMIT_RM()
 338 void emit_rm(CodeBuffer &cbuf, int f1, int f2, int f3) {
 339   unsigned char c = (unsigned char)((f1 << 6) | (f2 << 3) | f3);
 340   cbuf.insts()->emit_int8(c);
 341 }
 342 
 343 // EMIT_CC()
 344 void emit_cc(CodeBuffer &cbuf, int f1, int f2) {
 345   unsigned char c = (unsigned char)( f1 | f2 );
 346   cbuf.insts()->emit_int8(c);
 347 }
 348 
 349 // EMIT_OPCODE()
 350 void emit_opcode(CodeBuffer &cbuf, int code) {
 351   cbuf.insts()->emit_int8((unsigned char) code);
 352 }
 353 
 354 // EMIT_OPCODE() w/ relocation information
 355 void emit_opcode(CodeBuffer &cbuf, int code, relocInfo::relocType reloc, int offset = 0) {
 356   cbuf.relocate(cbuf.insts_mark() + offset, reloc);
 357   emit_opcode(cbuf, code);
 358 }
 359 
 360 // EMIT_D8()
 361 void emit_d8(CodeBuffer &cbuf, int d8) {
 362   cbuf.insts()->emit_int8((unsigned char) d8);
 363 }
 364 
 365 // EMIT_D16()
 366 void emit_d16(CodeBuffer &cbuf, int d16) {
 367   cbuf.insts()->emit_int16(d16);
 368 }
 369 
 370 // EMIT_D32()
 371 void emit_d32(CodeBuffer &cbuf, int d32) {
 372   cbuf.insts()->emit_int32(d32);
 373 }
 374 
 375 // emit 32 bit value and construct relocation entry from relocInfo::relocType
 376 void emit_d32_reloc(CodeBuffer &cbuf, int d32, relocInfo::relocType reloc,
 377         int format) {
 378   cbuf.relocate(cbuf.insts_mark(), reloc, format);
 379   cbuf.insts()->emit_int32(d32);
 380 }
 381 
 382 // emit 32 bit value and construct relocation entry from RelocationHolder
 383 void emit_d32_reloc(CodeBuffer &cbuf, int d32, RelocationHolder const& rspec,
 384         int format) {
 385 #ifdef ASSERT
 386   if (rspec.reloc()->type() == relocInfo::oop_type && d32 != 0 && d32 != (int)Universe::non_oop_word()) {
 387     assert(cast_to_oop(d32)->is_oop() && (ScavengeRootsInCode || !cast_to_oop(d32)->is_scavengable()), "cannot embed scavengable oops in code");
 388   }
 389 #endif
 390   cbuf.relocate(cbuf.insts_mark(), rspec, format);
 391   cbuf.insts()->emit_int32(d32);
 392 }
 393 
 394 // Access stack slot for load or store
 395 void store_to_stackslot(CodeBuffer &cbuf, int opcode, int rm_field, int disp) {
 396   emit_opcode( cbuf, opcode );               // (e.g., FILD   [ESP+src])
 397   if( -128 <= disp && disp <= 127 ) {
 398     emit_rm( cbuf, 0x01, rm_field, ESP_enc );  // R/M byte
 399     emit_rm( cbuf, 0x00, ESP_enc, ESP_enc);    // SIB byte
 400     emit_d8 (cbuf, disp);     // Displacement  // R/M byte
 401   } else {
 402     emit_rm( cbuf, 0x02, rm_field, ESP_enc );  // R/M byte
 403     emit_rm( cbuf, 0x00, ESP_enc, ESP_enc);    // SIB byte
 404     emit_d32(cbuf, disp);     // Displacement  // R/M byte
 405   }
 406 }
 407 
 408    // rRegI ereg, memory mem) %{    // emit_reg_mem
 409 void encode_RegMem( CodeBuffer &cbuf, int reg_encoding, int base, int index, int scale, int displace, relocInfo::relocType disp_reloc ) {
 410   // There is no index & no scale, use form without SIB byte
 411   if ((index == 0x4) &&
 412       (scale == 0) && (base != ESP_enc)) {
 413     // If no displacement, mode is 0x0; unless base is [EBP]
 414     if ( (displace == 0) && (base != EBP_enc) ) {
 415       emit_rm(cbuf, 0x0, reg_encoding, base);
 416     }
 417     else {                    // If 8-bit displacement, mode 0x1
 418       if ((displace >= -128) && (displace <= 127)
 419           && (disp_reloc == relocInfo::none) ) {
 420         emit_rm(cbuf, 0x1, reg_encoding, base);
 421         emit_d8(cbuf, displace);
 422       }
 423       else {                  // If 32-bit displacement
 424         if (base == -1) { // Special flag for absolute address
 425           emit_rm(cbuf, 0x0, reg_encoding, 0x5);
 426           // (manual lies; no SIB needed here)
 427           if ( disp_reloc != relocInfo::none ) {
 428             emit_d32_reloc(cbuf, displace, disp_reloc, 1);
 429           } else {
 430             emit_d32      (cbuf, displace);
 431           }
 432         }
 433         else {                // Normal base + offset
 434           emit_rm(cbuf, 0x2, reg_encoding, base);
 435           if ( disp_reloc != relocInfo::none ) {
 436             emit_d32_reloc(cbuf, displace, disp_reloc, 1);
 437           } else {
 438             emit_d32      (cbuf, displace);
 439           }
 440         }
 441       }
 442     }
 443   }
 444   else {                      // Else, encode with the SIB byte
 445     // If no displacement, mode is 0x0; unless base is [EBP]
 446     if (displace == 0 && (base != EBP_enc)) {  // If no displacement
 447       emit_rm(cbuf, 0x0, reg_encoding, 0x4);
 448       emit_rm(cbuf, scale, index, base);
 449     }
 450     else {                    // If 8-bit displacement, mode 0x1
 451       if ((displace >= -128) && (displace <= 127)
 452           && (disp_reloc == relocInfo::none) ) {
 453         emit_rm(cbuf, 0x1, reg_encoding, 0x4);
 454         emit_rm(cbuf, scale, index, base);
 455         emit_d8(cbuf, displace);
 456       }
 457       else {                  // If 32-bit displacement
 458         if (base == 0x04 ) {
 459           emit_rm(cbuf, 0x2, reg_encoding, 0x4);
 460           emit_rm(cbuf, scale, index, 0x04);
 461         } else {
 462           emit_rm(cbuf, 0x2, reg_encoding, 0x4);
 463           emit_rm(cbuf, scale, index, base);
 464         }
 465         if ( disp_reloc != relocInfo::none ) {
 466           emit_d32_reloc(cbuf, displace, disp_reloc, 1);
 467         } else {
 468           emit_d32      (cbuf, displace);
 469         }
 470       }
 471     }
 472   }
 473 }
 474 
 475 
 476 void encode_Copy( CodeBuffer &cbuf, int dst_encoding, int src_encoding ) {
 477   if( dst_encoding == src_encoding ) {
 478     // reg-reg copy, use an empty encoding
 479   } else {
 480     emit_opcode( cbuf, 0x8B );
 481     emit_rm(cbuf, 0x3, dst_encoding, src_encoding );
 482   }
 483 }
 484 
 485 void emit_cmpfp_fixup(MacroAssembler& _masm) {
 486   Label exit;
 487   __ jccb(Assembler::noParity, exit);
 488   __ pushf();
 489   //
 490   // comiss/ucomiss instructions set ZF,PF,CF flags and
 491   // zero OF,AF,SF for NaN values.
 492   // Fixup flags by zeroing ZF,PF so that compare of NaN
 493   // values returns 'less than' result (CF is set).
 494   // Leave the rest of flags unchanged.
 495   //
 496   //    7 6 5 4 3 2 1 0
 497   //   |S|Z|r|A|r|P|r|C|  (r - reserved bit)
 498   //    0 0 1 0 1 0 1 1   (0x2B)
 499   //
 500   __ andl(Address(rsp, 0), 0xffffff2b);
 501   __ popf();
 502   __ bind(exit);
 503 }
 504 
 505 void emit_cmpfp3(MacroAssembler& _masm, Register dst) {
 506   Label done;
 507   __ movl(dst, -1);
 508   __ jcc(Assembler::parity, done);
 509   __ jcc(Assembler::below, done);
 510   __ setb(Assembler::notEqual, dst);
 511   __ movzbl(dst, dst);
 512   __ bind(done);
 513 }
 514 
 515 
 516 //=============================================================================
 517 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
 518 
 519 int Compile::ConstantTable::calculate_table_base_offset() const {
 520   return 0;  // absolute addressing, no offset
 521 }
 522 
 523 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
 524 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
 525   ShouldNotReachHere();
 526 }
 527 
 528 void MachConstantBaseNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const {
 529   // Empty encoding
 530 }
 531 
 532 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
 533   return 0;
 534 }
 535 
 536 #ifndef PRODUCT
 537 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
 538   st->print("# MachConstantBaseNode (empty encoding)");
 539 }
 540 #endif
 541 
 542 
 543 //=============================================================================
 544 #ifndef PRODUCT
 545 void MachPrologNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
 546   Compile* C = ra_->C;
 547 
 548   int framesize = C->frame_size_in_bytes();
 549   int bangsize = C->bang_size_in_bytes();
 550   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
 551   // Remove wordSize for return addr which is already pushed.
 552   framesize -= wordSize;
 553 
 554   if (C->need_stack_bang(bangsize)) {
 555     framesize -= wordSize;
 556     st->print("# stack bang (%d bytes)", bangsize);
 557     st->print("\n\t");
 558     st->print("PUSH   EBP\t# Save EBP");
 559     if (PreserveFramePointer) {
 560       st->print("\n\t");
 561       st->print("MOV    EBP, ESP\t# Save the caller's SP into EBP");
 562     }
 563     if (framesize) {
 564       st->print("\n\t");
 565       st->print("SUB    ESP, #%d\t# Create frame",framesize);
 566     }
 567   } else {
 568     st->print("SUB    ESP, #%d\t# Create frame",framesize);
 569     st->print("\n\t");
 570     framesize -= wordSize;
 571     st->print("MOV    [ESP + #%d], EBP\t# Save EBP",framesize);
 572     if (PreserveFramePointer) {
 573       st->print("\n\t");
 574       st->print("MOV    EBP, ESP\t# Save the caller's SP into EBP");
 575       if (framesize > 0) {
 576         st->print("\n\t");
 577         st->print("ADD    EBP, #%d", framesize);
 578       }
 579     }
 580   }
 581 
 582   if (VerifyStackAtCalls) {
 583     st->print("\n\t");
 584     framesize -= wordSize;
 585     st->print("MOV    [ESP + #%d], 0xBADB100D\t# Majik cookie for stack depth check",framesize);
 586   }
 587 
 588   if( C->in_24_bit_fp_mode() ) {
 589     st->print("\n\t");
 590     st->print("FLDCW  \t# load 24 bit fpu control word");
 591   }
 592   if (UseSSE >= 2 && VerifyFPU) {
 593     st->print("\n\t");
 594     st->print("# verify FPU stack (must be clean on entry)");
 595   }
 596 
 597 #ifdef ASSERT
 598   if (VerifyStackAtCalls) {
 599     st->print("\n\t");
 600     st->print("# stack alignment check");
 601   }
 602 #endif
 603   st->cr();
 604 }
 605 #endif
 606 
 607 
 608 void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 609   Compile* C = ra_->C;
 610   MacroAssembler _masm(&cbuf);
 611 
 612   int framesize = C->frame_size_in_bytes();
 613   int bangsize = C->bang_size_in_bytes();
 614 
 615   __ verified_entry(framesize, C->need_stack_bang(bangsize)?bangsize:0, C->in_24_bit_fp_mode());
 616 
 617   C->set_frame_complete(cbuf.insts_size());
 618 
 619   if (C->has_mach_constant_base_node()) {
 620     // NOTE: We set the table base offset here because users might be
 621     // emitted before MachConstantBaseNode.
 622     Compile::ConstantTable& constant_table = C->constant_table();
 623     constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
 624   }
 625 }
 626 
 627 uint MachPrologNode::size(PhaseRegAlloc *ra_) const {
 628   return MachNode::size(ra_); // too many variables; just compute it the hard way
 629 }
 630 
 631 int MachPrologNode::reloc() const {
 632   return 0; // a large enough number
 633 }
 634 
 635 //=============================================================================
 636 #ifndef PRODUCT
 637 void MachEpilogNode::format( PhaseRegAlloc *ra_, outputStream* st ) const {
 638   Compile *C = ra_->C;
 639   int framesize = C->frame_size_in_bytes();
 640   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
 641   // Remove two words for return addr and rbp,
 642   framesize -= 2*wordSize;
 643 
 644   if (C->max_vector_size() > 16) {
 645     st->print("VZEROUPPER");
 646     st->cr(); st->print("\t");
 647   }
 648   if (C->in_24_bit_fp_mode()) {
 649     st->print("FLDCW  standard control word");
 650     st->cr(); st->print("\t");
 651   }
 652   if (framesize) {
 653     st->print("ADD    ESP,%d\t# Destroy frame",framesize);
 654     st->cr(); st->print("\t");
 655   }
 656   st->print_cr("POPL   EBP"); st->print("\t");
 657   if (do_polling() && C->is_method_compilation()) {
 658     st->print("TEST   PollPage,EAX\t! Poll Safepoint");
 659     st->cr(); st->print("\t");
 660   }
 661 }
 662 #endif
 663 
 664 void MachEpilogNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 665   Compile *C = ra_->C;
 666 
 667   if (C->max_vector_size() > 16) {
 668     // Clear upper bits of YMM registers when current compiled code uses
 669     // wide vectors to avoid AVX <-> SSE transition penalty during call.
 670     MacroAssembler masm(&cbuf);
 671     masm.vzeroupper();
 672   }
 673   // If method set FPU control word, restore to standard control word
 674   if (C->in_24_bit_fp_mode()) {
 675     MacroAssembler masm(&cbuf);
 676     masm.fldcw(ExternalAddress(StubRoutines::addr_fpu_cntrl_wrd_std()));
 677   }
 678 
 679   int framesize = C->frame_size_in_bytes();
 680   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
 681   // Remove two words for return addr and rbp,
 682   framesize -= 2*wordSize;
 683 
 684   // Note that VerifyStackAtCalls' Majik cookie does not change the frame size popped here
 685 
 686   if (framesize >= 128) {
 687     emit_opcode(cbuf, 0x81); // add  SP, #framesize
 688     emit_rm(cbuf, 0x3, 0x00, ESP_enc);
 689     emit_d32(cbuf, framesize);
 690   } else if (framesize) {
 691     emit_opcode(cbuf, 0x83); // add  SP, #framesize
 692     emit_rm(cbuf, 0x3, 0x00, ESP_enc);
 693     emit_d8(cbuf, framesize);
 694   }
 695 
 696   emit_opcode(cbuf, 0x58 | EBP_enc);
 697 
 698   if (do_polling() && C->is_method_compilation()) {
 699     cbuf.relocate(cbuf.insts_end(), relocInfo::poll_return_type, 0);
 700     emit_opcode(cbuf,0x85);
 701     emit_rm(cbuf, 0x0, EAX_enc, 0x5); // EAX
 702     emit_d32(cbuf, (intptr_t)os::get_polling_page());
 703   }
 704 }
 705 
 706 uint MachEpilogNode::size(PhaseRegAlloc *ra_) const {
 707   Compile *C = ra_->C;
 708   // If method set FPU control word, restore to standard control word
 709   int size = C->in_24_bit_fp_mode() ? 6 : 0;
 710   if (C->max_vector_size() > 16) size += 3; // vzeroupper
 711   if (do_polling() && C->is_method_compilation()) size += 6;
 712 
 713   int framesize = C->frame_size_in_bytes();
 714   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
 715   // Remove two words for return addr and rbp,
 716   framesize -= 2*wordSize;
 717 
 718   size++; // popl rbp,
 719 
 720   if (framesize >= 128) {
 721     size += 6;
 722   } else {
 723     size += framesize ? 3 : 0;
 724   }
 725   return size;
 726 }
 727 
 728 int MachEpilogNode::reloc() const {
 729   return 0; // a large enough number
 730 }
 731 
 732 const Pipeline * MachEpilogNode::pipeline() const {
 733   return MachNode::pipeline_class();
 734 }
 735 
 736 int MachEpilogNode::safepoint_offset() const { return 0; }
 737 
 738 //=============================================================================
 739 
 740 enum RC { rc_bad, rc_int, rc_float, rc_xmm, rc_stack };
 741 static enum RC rc_class( OptoReg::Name reg ) {
 742 
 743   if( !OptoReg::is_valid(reg)  ) return rc_bad;
 744   if (OptoReg::is_stack(reg)) return rc_stack;
 745 
 746   VMReg r = OptoReg::as_VMReg(reg);
 747   if (r->is_Register()) return rc_int;
 748   if (r->is_FloatRegister()) {
 749     assert(UseSSE < 2, "shouldn't be used in SSE2+ mode");
 750     return rc_float;
 751   }
 752   assert(r->is_XMMRegister(), "must be");
 753   return rc_xmm;
 754 }
 755 
 756 static int impl_helper( CodeBuffer *cbuf, bool do_size, bool is_load, int offset, int reg,
 757                         int opcode, const char *op_str, int size, outputStream* st ) {
 758   if( cbuf ) {
 759     emit_opcode  (*cbuf, opcode );
 760     encode_RegMem(*cbuf, Matcher::_regEncode[reg], ESP_enc, 0x4, 0, offset, relocInfo::none);
 761 #ifndef PRODUCT
 762   } else if( !do_size ) {
 763     if( size != 0 ) st->print("\n\t");
 764     if( opcode == 0x8B || opcode == 0x89 ) { // MOV
 765       if( is_load ) st->print("%s   %s,[ESP + #%d]",op_str,Matcher::regName[reg],offset);
 766       else          st->print("%s   [ESP + #%d],%s",op_str,offset,Matcher::regName[reg]);
 767     } else { // FLD, FST, PUSH, POP
 768       st->print("%s [ESP + #%d]",op_str,offset);
 769     }
 770 #endif
 771   }
 772   int offset_size = (offset == 0) ? 0 : ((offset <= 127) ? 1 : 4);
 773   return size+3+offset_size;
 774 }
 775 
 776 // Helper for XMM registers.  Extra opcode bits, limited syntax.
 777 static int impl_x_helper( CodeBuffer *cbuf, bool do_size, bool is_load,
 778                          int offset, int reg_lo, int reg_hi, int size, outputStream* st ) {
 779   if (cbuf) {
 780     MacroAssembler _masm(cbuf);
 781     if (reg_lo+1 == reg_hi) { // double move?
 782       if (is_load) {
 783         __ movdbl(as_XMMRegister(Matcher::_regEncode[reg_lo]), Address(rsp, offset));
 784       } else {
 785         __ movdbl(Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[reg_lo]));
 786       }
 787     } else {
 788       if (is_load) {
 789         __ movflt(as_XMMRegister(Matcher::_regEncode[reg_lo]), Address(rsp, offset));
 790       } else {
 791         __ movflt(Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[reg_lo]));
 792       }
 793     }
 794 #ifndef PRODUCT
 795   } else if (!do_size) {
 796     if (size != 0) st->print("\n\t");
 797     if (reg_lo+1 == reg_hi) { // double move?
 798       if (is_load) st->print("%s %s,[ESP + #%d]",
 799                               UseXmmLoadAndClearUpper ? "MOVSD " : "MOVLPD",
 800                               Matcher::regName[reg_lo], offset);
 801       else         st->print("MOVSD  [ESP + #%d],%s",
 802                               offset, Matcher::regName[reg_lo]);
 803     } else {
 804       if (is_load) st->print("MOVSS  %s,[ESP + #%d]",
 805                               Matcher::regName[reg_lo], offset);
 806       else         st->print("MOVSS  [ESP + #%d],%s",
 807                               offset, Matcher::regName[reg_lo]);
 808     }
 809 #endif
 810   }
 811   int offset_size = (offset == 0) ? 0 : ((offset <= 127) ? 1 : 4);
 812   // VEX_2bytes prefix is used if UseAVX > 0, so it takes the same 2 bytes as SIMD prefix.
 813   return size+5+offset_size;
 814 }
 815 
 816 
 817 static int impl_movx_helper( CodeBuffer *cbuf, bool do_size, int src_lo, int dst_lo,
 818                             int src_hi, int dst_hi, int size, outputStream* st ) {
 819   if (cbuf) {
 820     MacroAssembler _masm(cbuf);
 821     if (src_lo+1 == src_hi && dst_lo+1 == dst_hi) { // double move?
 822       __ movdbl(as_XMMRegister(Matcher::_regEncode[dst_lo]),
 823                 as_XMMRegister(Matcher::_regEncode[src_lo]));
 824     } else {
 825       __ movflt(as_XMMRegister(Matcher::_regEncode[dst_lo]),
 826                 as_XMMRegister(Matcher::_regEncode[src_lo]));
 827     }
 828 #ifndef PRODUCT
 829   } else if (!do_size) {
 830     if (size != 0) st->print("\n\t");
 831     if (UseXmmRegToRegMoveAll) {//Use movaps,movapd to move between xmm registers
 832       if (src_lo+1 == src_hi && dst_lo+1 == dst_hi) { // double move?
 833         st->print("MOVAPD %s,%s",Matcher::regName[dst_lo],Matcher::regName[src_lo]);
 834       } else {
 835         st->print("MOVAPS %s,%s",Matcher::regName[dst_lo],Matcher::regName[src_lo]);
 836       }
 837     } else {
 838       if( src_lo+1 == src_hi && dst_lo+1 == dst_hi ) { // double move?
 839         st->print("MOVSD  %s,%s",Matcher::regName[dst_lo],Matcher::regName[src_lo]);
 840       } else {
 841         st->print("MOVSS  %s,%s",Matcher::regName[dst_lo],Matcher::regName[src_lo]);
 842       }
 843     }
 844 #endif
 845   }
 846   // VEX_2bytes prefix is used if UseAVX > 0, and it takes the same 2 bytes as SIMD prefix.
 847   // Only MOVAPS SSE prefix uses 1 byte.
 848   int sz = 4;
 849   if (!(src_lo+1 == src_hi && dst_lo+1 == dst_hi) &&
 850       UseXmmRegToRegMoveAll && (UseAVX == 0)) sz = 3;
 851   return size + sz;
 852 }
 853 
 854 static int impl_movgpr2x_helper( CodeBuffer *cbuf, bool do_size, int src_lo, int dst_lo,
 855                             int src_hi, int dst_hi, int size, outputStream* st ) {
 856   // 32-bit
 857   if (cbuf) {
 858     MacroAssembler _masm(cbuf);
 859     __ movdl(as_XMMRegister(Matcher::_regEncode[dst_lo]),
 860              as_Register(Matcher::_regEncode[src_lo]));
 861 #ifndef PRODUCT
 862   } else if (!do_size) {
 863     st->print("movdl   %s, %s\t# spill", Matcher::regName[dst_lo], Matcher::regName[src_lo]);
 864 #endif
 865   }
 866   return 4;
 867 }
 868 
 869 
 870 static int impl_movx2gpr_helper( CodeBuffer *cbuf, bool do_size, int src_lo, int dst_lo,
 871                                  int src_hi, int dst_hi, int size, outputStream* st ) {
 872   // 32-bit
 873   if (cbuf) {
 874     MacroAssembler _masm(cbuf);
 875     __ movdl(as_Register(Matcher::_regEncode[dst_lo]),
 876              as_XMMRegister(Matcher::_regEncode[src_lo]));
 877 #ifndef PRODUCT
 878   } else if (!do_size) {
 879     st->print("movdl   %s, %s\t# spill", Matcher::regName[dst_lo], Matcher::regName[src_lo]);
 880 #endif
 881   }
 882   return 4;
 883 }
 884 
 885 static int impl_mov_helper( CodeBuffer *cbuf, bool do_size, int src, int dst, int size, outputStream* st ) {
 886   if( cbuf ) {
 887     emit_opcode(*cbuf, 0x8B );
 888     emit_rm    (*cbuf, 0x3, Matcher::_regEncode[dst], Matcher::_regEncode[src] );
 889 #ifndef PRODUCT
 890   } else if( !do_size ) {
 891     if( size != 0 ) st->print("\n\t");
 892     st->print("MOV    %s,%s",Matcher::regName[dst],Matcher::regName[src]);
 893 #endif
 894   }
 895   return size+2;
 896 }
 897 
 898 static int impl_fp_store_helper( CodeBuffer *cbuf, bool do_size, int src_lo, int src_hi, int dst_lo, int dst_hi,
 899                                  int offset, int size, outputStream* st ) {
 900   if( src_lo != FPR1L_num ) {      // Move value to top of FP stack, if not already there
 901     if( cbuf ) {
 902       emit_opcode( *cbuf, 0xD9 );  // FLD (i.e., push it)
 903       emit_d8( *cbuf, 0xC0-1+Matcher::_regEncode[src_lo] );
 904 #ifndef PRODUCT
 905     } else if( !do_size ) {
 906       if( size != 0 ) st->print("\n\t");
 907       st->print("FLD    %s",Matcher::regName[src_lo]);
 908 #endif
 909     }
 910     size += 2;
 911   }
 912 
 913   int st_op = (src_lo != FPR1L_num) ? EBX_num /*store & pop*/ : EDX_num /*store no pop*/;
 914   const char *op_str;
 915   int op;
 916   if( src_lo+1 == src_hi && dst_lo+1 == dst_hi ) { // double store?
 917     op_str = (src_lo != FPR1L_num) ? "FSTP_D" : "FST_D ";
 918     op = 0xDD;
 919   } else {                   // 32-bit store
 920     op_str = (src_lo != FPR1L_num) ? "FSTP_S" : "FST_S ";
 921     op = 0xD9;
 922     assert( !OptoReg::is_valid(src_hi) && !OptoReg::is_valid(dst_hi), "no non-adjacent float-stores" );
 923   }
 924 
 925   return impl_helper(cbuf,do_size,false,offset,st_op,op,op_str,size, st);
 926 }
 927 
 928 // Next two methods are shared by 32- and 64-bit VM. They are defined in x86.ad.
 929 static int vec_mov_helper(CodeBuffer *cbuf, bool do_size, int src_lo, int dst_lo,
 930                           int src_hi, int dst_hi, uint ireg, outputStream* st);
 931 
 932 static int vec_spill_helper(CodeBuffer *cbuf, bool do_size, bool is_load,
 933                             int stack_offset, int reg, uint ireg, outputStream* st);
 934 
 935 static int vec_stack_to_stack_helper(CodeBuffer *cbuf, bool do_size, int src_offset,
 936                                      int dst_offset, uint ireg, outputStream* st) {
 937   int calc_size = 0;
 938   int src_offset_size = (src_offset == 0) ? 0 : ((src_offset < 0x80) ? 1 : 4);
 939   int dst_offset_size = (dst_offset == 0) ? 0 : ((dst_offset < 0x80) ? 1 : 4);
 940   switch (ireg) {
 941   case Op_VecS:
 942     calc_size = 3+src_offset_size + 3+dst_offset_size;
 943     break;
 944   case Op_VecD:
 945     calc_size = 3+src_offset_size + 3+dst_offset_size;
 946     src_offset += 4;
 947     dst_offset += 4;
 948     src_offset_size = (src_offset == 0) ? 0 : ((src_offset < 0x80) ? 1 : 4);
 949     dst_offset_size = (dst_offset == 0) ? 0 : ((dst_offset < 0x80) ? 1 : 4);
 950     calc_size += 3+src_offset_size + 3+dst_offset_size;
 951     break;
 952   case Op_VecX:
 953     calc_size = 6 + 6 + 5+src_offset_size + 5+dst_offset_size;
 954     break;
 955   case Op_VecY:
 956     calc_size = 6 + 6 + 5+src_offset_size + 5+dst_offset_size;
 957     break;
 958   default:
 959     ShouldNotReachHere();
 960   }
 961   if (cbuf) {
 962     MacroAssembler _masm(cbuf);
 963     int offset = __ offset();
 964     switch (ireg) {
 965     case Op_VecS:
 966       __ pushl(Address(rsp, src_offset));
 967       __ popl (Address(rsp, dst_offset));
 968       break;
 969     case Op_VecD:
 970       __ pushl(Address(rsp, src_offset));
 971       __ popl (Address(rsp, dst_offset));
 972       __ pushl(Address(rsp, src_offset+4));
 973       __ popl (Address(rsp, dst_offset+4));
 974       break;
 975     case Op_VecX:
 976       __ movdqu(Address(rsp, -16), xmm0);
 977       __ movdqu(xmm0, Address(rsp, src_offset));
 978       __ movdqu(Address(rsp, dst_offset), xmm0);
 979       __ movdqu(xmm0, Address(rsp, -16));
 980       break;
 981     case Op_VecY:
 982       __ vmovdqu(Address(rsp, -32), xmm0);
 983       __ vmovdqu(xmm0, Address(rsp, src_offset));
 984       __ vmovdqu(Address(rsp, dst_offset), xmm0);
 985       __ vmovdqu(xmm0, Address(rsp, -32));
 986       break;
 987     default:
 988       ShouldNotReachHere();
 989     }
 990     int size = __ offset() - offset;
 991     assert(size == calc_size, "incorrect size calculattion");
 992     return size;
 993 #ifndef PRODUCT
 994   } else if (!do_size) {
 995     switch (ireg) {
 996     case Op_VecS:
 997       st->print("pushl   [rsp + #%d]\t# 32-bit mem-mem spill\n\t"
 998                 "popl    [rsp + #%d]",
 999                 src_offset, dst_offset);
1000       break;
1001     case Op_VecD:
1002       st->print("pushl   [rsp + #%d]\t# 64-bit mem-mem spill\n\t"
1003                 "popq    [rsp + #%d]\n\t"
1004                 "pushl   [rsp + #%d]\n\t"
1005                 "popq    [rsp + #%d]",
1006                 src_offset, dst_offset, src_offset+4, dst_offset+4);
1007       break;
1008      case Op_VecX:
1009       st->print("movdqu  [rsp - #16], xmm0\t# 128-bit mem-mem spill\n\t"
1010                 "movdqu  xmm0, [rsp + #%d]\n\t"
1011                 "movdqu  [rsp + #%d], xmm0\n\t"
1012                 "movdqu  xmm0, [rsp - #16]",
1013                 src_offset, dst_offset);
1014       break;
1015     case Op_VecY:
1016       st->print("vmovdqu [rsp - #32], xmm0\t# 256-bit mem-mem spill\n\t"
1017                 "vmovdqu xmm0, [rsp + #%d]\n\t"
1018                 "vmovdqu [rsp + #%d], xmm0\n\t"
1019                 "vmovdqu xmm0, [rsp - #32]",
1020                 src_offset, dst_offset);
1021       break;
1022     default:
1023       ShouldNotReachHere();
1024     }
1025 #endif
1026   }
1027   return calc_size;
1028 }
1029 
1030 uint MachSpillCopyNode::implementation( CodeBuffer *cbuf, PhaseRegAlloc *ra_, bool do_size, outputStream* st ) const {
1031   // Get registers to move
1032   OptoReg::Name src_second = ra_->get_reg_second(in(1));
1033   OptoReg::Name src_first = ra_->get_reg_first(in(1));
1034   OptoReg::Name dst_second = ra_->get_reg_second(this );
1035   OptoReg::Name dst_first = ra_->get_reg_first(this );
1036 
1037   enum RC src_second_rc = rc_class(src_second);
1038   enum RC src_first_rc = rc_class(src_first);
1039   enum RC dst_second_rc = rc_class(dst_second);
1040   enum RC dst_first_rc = rc_class(dst_first);
1041 
1042   assert( OptoReg::is_valid(src_first) && OptoReg::is_valid(dst_first), "must move at least 1 register" );
1043 
1044   // Generate spill code!
1045   int size = 0;
1046 
1047   if( src_first == dst_first && src_second == dst_second )
1048     return size;            // Self copy, no move
1049 
1050   if (bottom_type()->isa_vect() != NULL) {
1051     uint ireg = ideal_reg();
1052     assert((src_first_rc != rc_int && dst_first_rc != rc_int), "sanity");
1053     assert((src_first_rc != rc_float && dst_first_rc != rc_float), "sanity");
1054     assert((ireg == Op_VecS || ireg == Op_VecD || ireg == Op_VecX || ireg == Op_VecY), "sanity");
1055     if( src_first_rc == rc_stack && dst_first_rc == rc_stack ) {
1056       // mem -> mem
1057       int src_offset = ra_->reg2offset(src_first);
1058       int dst_offset = ra_->reg2offset(dst_first);
1059       return vec_stack_to_stack_helper(cbuf, do_size, src_offset, dst_offset, ireg, st);
1060     } else if (src_first_rc == rc_xmm && dst_first_rc == rc_xmm ) {
1061       return vec_mov_helper(cbuf, do_size, src_first, dst_first, src_second, dst_second, ireg, st);
1062     } else if (src_first_rc == rc_xmm && dst_first_rc == rc_stack ) {
1063       int stack_offset = ra_->reg2offset(dst_first);
1064       return vec_spill_helper(cbuf, do_size, false, stack_offset, src_first, ireg, st);
1065     } else if (src_first_rc == rc_stack && dst_first_rc == rc_xmm ) {
1066       int stack_offset = ra_->reg2offset(src_first);
1067       return vec_spill_helper(cbuf, do_size, true,  stack_offset, dst_first, ireg, st);
1068     } else {
1069       ShouldNotReachHere();
1070     }
1071   }
1072 
1073   // --------------------------------------
1074   // Check for mem-mem move.  push/pop to move.
1075   if( src_first_rc == rc_stack && dst_first_rc == rc_stack ) {
1076     if( src_second == dst_first ) { // overlapping stack copy ranges
1077       assert( src_second_rc == rc_stack && dst_second_rc == rc_stack, "we only expect a stk-stk copy here" );
1078       size = impl_helper(cbuf,do_size,true ,ra_->reg2offset(src_second),ESI_num,0xFF,"PUSH  ",size, st);
1079       size = impl_helper(cbuf,do_size,false,ra_->reg2offset(dst_second),EAX_num,0x8F,"POP   ",size, st);
1080       src_second_rc = dst_second_rc = rc_bad;  // flag as already moved the second bits
1081     }
1082     // move low bits
1083     size = impl_helper(cbuf,do_size,true ,ra_->reg2offset(src_first),ESI_num,0xFF,"PUSH  ",size, st);
1084     size = impl_helper(cbuf,do_size,false,ra_->reg2offset(dst_first),EAX_num,0x8F,"POP   ",size, st);
1085     if( src_second_rc == rc_stack && dst_second_rc == rc_stack ) { // mov second bits
1086       size = impl_helper(cbuf,do_size,true ,ra_->reg2offset(src_second),ESI_num,0xFF,"PUSH  ",size, st);
1087       size = impl_helper(cbuf,do_size,false,ra_->reg2offset(dst_second),EAX_num,0x8F,"POP   ",size, st);
1088     }
1089     return size;
1090   }
1091 
1092   // --------------------------------------
1093   // Check for integer reg-reg copy
1094   if( src_first_rc == rc_int && dst_first_rc == rc_int )
1095     size = impl_mov_helper(cbuf,do_size,src_first,dst_first,size, st);
1096 
1097   // Check for integer store
1098   if( src_first_rc == rc_int && dst_first_rc == rc_stack )
1099     size = impl_helper(cbuf,do_size,false,ra_->reg2offset(dst_first),src_first,0x89,"MOV ",size, st);
1100 
1101   // Check for integer load
1102   if( dst_first_rc == rc_int && src_first_rc == rc_stack )
1103     size = impl_helper(cbuf,do_size,true ,ra_->reg2offset(src_first),dst_first,0x8B,"MOV ",size, st);
1104 
1105   // Check for integer reg-xmm reg copy
1106   if( src_first_rc == rc_int && dst_first_rc == rc_xmm ) {
1107     assert( (src_second_rc == rc_bad && dst_second_rc == rc_bad),
1108             "no 64 bit integer-float reg moves" );
1109     return impl_movgpr2x_helper(cbuf,do_size,src_first,dst_first,src_second, dst_second, size, st);
1110   }
1111   // --------------------------------------
1112   // Check for float reg-reg copy
1113   if( src_first_rc == rc_float && dst_first_rc == rc_float ) {
1114     assert( (src_second_rc == rc_bad && dst_second_rc == rc_bad) ||
1115             (src_first+1 == src_second && dst_first+1 == dst_second), "no non-adjacent float-moves" );
1116     if( cbuf ) {
1117 
1118       // Note the mucking with the register encode to compensate for the 0/1
1119       // indexing issue mentioned in a comment in the reg_def sections
1120       // for FPR registers many lines above here.
1121 
1122       if( src_first != FPR1L_num ) {
1123         emit_opcode  (*cbuf, 0xD9 );           // FLD    ST(i)
1124         emit_d8      (*cbuf, 0xC0+Matcher::_regEncode[src_first]-1 );
1125         emit_opcode  (*cbuf, 0xDD );           // FSTP   ST(i)
1126         emit_d8      (*cbuf, 0xD8+Matcher::_regEncode[dst_first] );
1127      } else {
1128         emit_opcode  (*cbuf, 0xDD );           // FST    ST(i)
1129         emit_d8      (*cbuf, 0xD0+Matcher::_regEncode[dst_first]-1 );
1130      }
1131 #ifndef PRODUCT
1132     } else if( !do_size ) {
1133       if( size != 0 ) st->print("\n\t");
1134       if( src_first != FPR1L_num ) st->print("FLD    %s\n\tFSTP   %s",Matcher::regName[src_first],Matcher::regName[dst_first]);
1135       else                      st->print(             "FST    %s",                            Matcher::regName[dst_first]);
1136 #endif
1137     }
1138     return size + ((src_first != FPR1L_num) ? 2+2 : 2);
1139   }
1140 
1141   // Check for float store
1142   if( src_first_rc == rc_float && dst_first_rc == rc_stack ) {
1143     return impl_fp_store_helper(cbuf,do_size,src_first,src_second,dst_first,dst_second,ra_->reg2offset(dst_first),size, st);
1144   }
1145 
1146   // Check for float load
1147   if( dst_first_rc == rc_float && src_first_rc == rc_stack ) {
1148     int offset = ra_->reg2offset(src_first);
1149     const char *op_str;
1150     int op;
1151     if( src_first+1 == src_second && dst_first+1 == dst_second ) { // double load?
1152       op_str = "FLD_D";
1153       op = 0xDD;
1154     } else {                   // 32-bit load
1155       op_str = "FLD_S";
1156       op = 0xD9;
1157       assert( src_second_rc == rc_bad && dst_second_rc == rc_bad, "no non-adjacent float-loads" );
1158     }
1159     if( cbuf ) {
1160       emit_opcode  (*cbuf, op );
1161       encode_RegMem(*cbuf, 0x0, ESP_enc, 0x4, 0, offset, relocInfo::none);
1162       emit_opcode  (*cbuf, 0xDD );           // FSTP   ST(i)
1163       emit_d8      (*cbuf, 0xD8+Matcher::_regEncode[dst_first] );
1164 #ifndef PRODUCT
1165     } else if( !do_size ) {
1166       if( size != 0 ) st->print("\n\t");
1167       st->print("%s  ST,[ESP + #%d]\n\tFSTP   %s",op_str, offset,Matcher::regName[dst_first]);
1168 #endif
1169     }
1170     int offset_size = (offset == 0) ? 0 : ((offset <= 127) ? 1 : 4);
1171     return size + 3+offset_size+2;
1172   }
1173 
1174   // Check for xmm reg-reg copy
1175   if( src_first_rc == rc_xmm && dst_first_rc == rc_xmm ) {
1176     assert( (src_second_rc == rc_bad && dst_second_rc == rc_bad) ||
1177             (src_first+1 == src_second && dst_first+1 == dst_second),
1178             "no non-adjacent float-moves" );
1179     return impl_movx_helper(cbuf,do_size,src_first,dst_first,src_second, dst_second, size, st);
1180   }
1181 
1182   // Check for xmm reg-integer reg copy
1183   if( src_first_rc == rc_xmm && dst_first_rc == rc_int ) {
1184     assert( (src_second_rc == rc_bad && dst_second_rc == rc_bad),
1185             "no 64 bit float-integer reg moves" );
1186     return impl_movx2gpr_helper(cbuf,do_size,src_first,dst_first,src_second, dst_second, size, st);
1187   }
1188 
1189   // Check for xmm store
1190   if( src_first_rc == rc_xmm && dst_first_rc == rc_stack ) {
1191     return impl_x_helper(cbuf,do_size,false,ra_->reg2offset(dst_first),src_first, src_second, size, st);
1192   }
1193 
1194   // Check for float xmm load
1195   if( dst_first_rc == rc_xmm && src_first_rc == rc_stack ) {
1196     return impl_x_helper(cbuf,do_size,true ,ra_->reg2offset(src_first),dst_first, dst_second, size, st);
1197   }
1198 
1199   // Copy from float reg to xmm reg
1200   if( dst_first_rc == rc_xmm && src_first_rc == rc_float ) {
1201     // copy to the top of stack from floating point reg
1202     // and use LEA to preserve flags
1203     if( cbuf ) {
1204       emit_opcode(*cbuf,0x8D);  // LEA  ESP,[ESP-8]
1205       emit_rm(*cbuf, 0x1, ESP_enc, 0x04);
1206       emit_rm(*cbuf, 0x0, 0x04, ESP_enc);
1207       emit_d8(*cbuf,0xF8);
1208 #ifndef PRODUCT
1209     } else if( !do_size ) {
1210       if( size != 0 ) st->print("\n\t");
1211       st->print("LEA    ESP,[ESP-8]");
1212 #endif
1213     }
1214     size += 4;
1215 
1216     size = impl_fp_store_helper(cbuf,do_size,src_first,src_second,dst_first,dst_second,0,size, st);
1217 
1218     // Copy from the temp memory to the xmm reg.
1219     size = impl_x_helper(cbuf,do_size,true ,0,dst_first, dst_second, size, st);
1220 
1221     if( cbuf ) {
1222       emit_opcode(*cbuf,0x8D);  // LEA  ESP,[ESP+8]
1223       emit_rm(*cbuf, 0x1, ESP_enc, 0x04);
1224       emit_rm(*cbuf, 0x0, 0x04, ESP_enc);
1225       emit_d8(*cbuf,0x08);
1226 #ifndef PRODUCT
1227     } else if( !do_size ) {
1228       if( size != 0 ) st->print("\n\t");
1229       st->print("LEA    ESP,[ESP+8]");
1230 #endif
1231     }
1232     size += 4;
1233     return size;
1234   }
1235 
1236   assert( size > 0, "missed a case" );
1237 
1238   // --------------------------------------------------------------------
1239   // Check for second bits still needing moving.
1240   if( src_second == dst_second )
1241     return size;               // Self copy; no move
1242   assert( src_second_rc != rc_bad && dst_second_rc != rc_bad, "src_second & dst_second cannot be Bad" );
1243 
1244   // Check for second word int-int move
1245   if( src_second_rc == rc_int && dst_second_rc == rc_int )
1246     return impl_mov_helper(cbuf,do_size,src_second,dst_second,size, st);
1247 
1248   // Check for second word integer store
1249   if( src_second_rc == rc_int && dst_second_rc == rc_stack )
1250     return impl_helper(cbuf,do_size,false,ra_->reg2offset(dst_second),src_second,0x89,"MOV ",size, st);
1251 
1252   // Check for second word integer load
1253   if( dst_second_rc == rc_int && src_second_rc == rc_stack )
1254     return impl_helper(cbuf,do_size,true ,ra_->reg2offset(src_second),dst_second,0x8B,"MOV ",size, st);
1255 
1256 
1257   Unimplemented();
1258 }
1259 
1260 #ifndef PRODUCT
1261 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream* st) const {
1262   implementation( NULL, ra_, false, st );
1263 }
1264 #endif
1265 
1266 void MachSpillCopyNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1267   implementation( &cbuf, ra_, false, NULL );
1268 }
1269 
1270 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
1271   return implementation( NULL, ra_, true, NULL );
1272 }
1273 
1274 
1275 //=============================================================================
1276 #ifndef PRODUCT
1277 void BoxLockNode::format( PhaseRegAlloc *ra_, outputStream* st ) const {
1278   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
1279   int reg = ra_->get_reg_first(this);
1280   st->print("LEA    %s,[ESP + #%d]",Matcher::regName[reg],offset);
1281 }
1282 #endif
1283 
1284 void BoxLockNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1285   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
1286   int reg = ra_->get_encode(this);
1287   if( offset >= 128 ) {
1288     emit_opcode(cbuf, 0x8D);      // LEA  reg,[SP+offset]
1289     emit_rm(cbuf, 0x2, reg, 0x04);
1290     emit_rm(cbuf, 0x0, 0x04, ESP_enc);
1291     emit_d32(cbuf, offset);
1292   }
1293   else {
1294     emit_opcode(cbuf, 0x8D);      // LEA  reg,[SP+offset]
1295     emit_rm(cbuf, 0x1, reg, 0x04);
1296     emit_rm(cbuf, 0x0, 0x04, ESP_enc);
1297     emit_d8(cbuf, offset);
1298   }
1299 }
1300 
1301 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
1302   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
1303   if( offset >= 128 ) {
1304     return 7;
1305   }
1306   else {
1307     return 4;
1308   }
1309 }
1310 
1311 //=============================================================================
1312 #ifndef PRODUCT
1313 void MachUEPNode::format( PhaseRegAlloc *ra_, outputStream* st ) const {
1314   st->print_cr(  "CMP    EAX,[ECX+4]\t# Inline cache check");
1315   st->print_cr("\tJNE    SharedRuntime::handle_ic_miss_stub");
1316   st->print_cr("\tNOP");
1317   st->print_cr("\tNOP");
1318   if( !OptoBreakpoint )
1319     st->print_cr("\tNOP");
1320 }
1321 #endif
1322 
1323 void MachUEPNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1324   MacroAssembler masm(&cbuf);
1325 #ifdef ASSERT
1326   uint insts_size = cbuf.insts_size();
1327 #endif
1328   masm.cmpptr(rax, Address(rcx, oopDesc::klass_offset_in_bytes()));
1329   masm.jump_cc(Assembler::notEqual,
1330                RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
1331   /* WARNING these NOPs are critical so that verified entry point is properly
1332      aligned for patching by NativeJump::patch_verified_entry() */
1333   int nops_cnt = 2;
1334   if( !OptoBreakpoint ) // Leave space for int3
1335      nops_cnt += 1;
1336   masm.nop(nops_cnt);
1337 
1338   assert(cbuf.insts_size() - insts_size == size(ra_), "checking code size of inline cache node");
1339 }
1340 
1341 uint MachUEPNode::size(PhaseRegAlloc *ra_) const {
1342   return OptoBreakpoint ? 11 : 12;
1343 }
1344 
1345 
1346 //=============================================================================
1347 
1348 int Matcher::regnum_to_fpu_offset(int regnum) {
1349   return regnum - 32; // The FP registers are in the second chunk
1350 }
1351 
1352 // This is UltraSparc specific, true just means we have fast l2f conversion
1353 const bool Matcher::convL2FSupported(void) {
1354   return true;
1355 }
1356 
1357 // Is this branch offset short enough that a short branch can be used?
1358 //
1359 // NOTE: If the platform does not provide any short branch variants, then
1360 //       this method should return false for offset 0.
1361 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
1362   // The passed offset is relative to address of the branch.
1363   // On 86 a branch displacement is calculated relative to address
1364   // of a next instruction.
1365   offset -= br_size;
1366 
1367   // the short version of jmpConUCF2 contains multiple branches,
1368   // making the reach slightly less
1369   if (rule == jmpConUCF2_rule)
1370     return (-126 <= offset && offset <= 125);
1371   return (-128 <= offset && offset <= 127);
1372 }
1373 
1374 const bool Matcher::isSimpleConstant64(jlong value) {
1375   // Will one (StoreL ConL) be cheaper than two (StoreI ConI)?.
1376   return false;
1377 }
1378 
1379 // The ecx parameter to rep stos for the ClearArray node is in dwords.
1380 const bool Matcher::init_array_count_is_in_bytes = false;
1381 
1382 // Threshold size for cleararray.
1383 const int Matcher::init_array_short_size = 8 * BytesPerLong;
1384 
1385 // Needs 2 CMOV's for longs.
1386 const int Matcher::long_cmove_cost() { return 1; }
1387 
1388 // No CMOVF/CMOVD with SSE/SSE2
1389 const int Matcher::float_cmove_cost() { return (UseSSE>=1) ? ConditionalMoveLimit : 0; }
1390 
1391 // Does the CPU require late expand (see block.cpp for description of late expand)?
1392 const bool Matcher::require_postalloc_expand = false;
1393 
1394 // Should the Matcher clone shifts on addressing modes, expecting them to
1395 // be subsumed into complex addressing expressions or compute them into
1396 // registers?  True for Intel but false for most RISCs
1397 const bool Matcher::clone_shift_expressions = true;
1398 
1399 // Do we need to mask the count passed to shift instructions or does
1400 // the cpu only look at the lower 5/6 bits anyway?
1401 const bool Matcher::need_masked_shift_count = false;
1402 
1403 bool Matcher::narrow_oop_use_complex_address() {
1404   ShouldNotCallThis();
1405   return true;
1406 }
1407 
1408 bool Matcher::narrow_klass_use_complex_address() {
1409   ShouldNotCallThis();
1410   return true;
1411 }
1412 
1413 
1414 // Is it better to copy float constants, or load them directly from memory?
1415 // Intel can load a float constant from a direct address, requiring no
1416 // extra registers.  Most RISCs will have to materialize an address into a
1417 // register first, so they would do better to copy the constant from stack.
1418 const bool Matcher::rematerialize_float_constants = true;
1419 
1420 // If CPU can load and store mis-aligned doubles directly then no fixup is
1421 // needed.  Else we split the double into 2 integer pieces and move it
1422 // piece-by-piece.  Only happens when passing doubles into C code as the
1423 // Java calling convention forces doubles to be aligned.
1424 const bool Matcher::misaligned_doubles_ok = true;
1425 
1426 
1427 void Matcher::pd_implicit_null_fixup(MachNode *node, uint idx) {
1428   // Get the memory operand from the node
1429   uint numopnds = node->num_opnds();        // Virtual call for number of operands
1430   uint skipped  = node->oper_input_base();  // Sum of leaves skipped so far
1431   assert( idx >= skipped, "idx too low in pd_implicit_null_fixup" );
1432   uint opcnt     = 1;                 // First operand
1433   uint num_edges = node->_opnds[1]->num_edges(); // leaves for first operand
1434   while( idx >= skipped+num_edges ) {
1435     skipped += num_edges;
1436     opcnt++;                          // Bump operand count
1437     assert( opcnt < numopnds, "Accessing non-existent operand" );
1438     num_edges = node->_opnds[opcnt]->num_edges(); // leaves for next operand
1439   }
1440 
1441   MachOper *memory = node->_opnds[opcnt];
1442   MachOper *new_memory = NULL;
1443   switch (memory->opcode()) {
1444   case DIRECT:
1445   case INDOFFSET32X:
1446     // No transformation necessary.
1447     return;
1448   case INDIRECT:
1449     new_memory = new (C) indirect_win95_safeOper( );
1450     break;
1451   case INDOFFSET8:
1452     new_memory = new (C) indOffset8_win95_safeOper(memory->disp(NULL, NULL, 0));
1453     break;
1454   case INDOFFSET32:
1455     new_memory = new (C) indOffset32_win95_safeOper(memory->disp(NULL, NULL, 0));
1456     break;
1457   case INDINDEXOFFSET:
1458     new_memory = new (C) indIndexOffset_win95_safeOper(memory->disp(NULL, NULL, 0));
1459     break;
1460   case INDINDEXSCALE:
1461     new_memory = new (C) indIndexScale_win95_safeOper(memory->scale());
1462     break;
1463   case INDINDEXSCALEOFFSET:
1464     new_memory = new (C) indIndexScaleOffset_win95_safeOper(memory->scale(), memory->disp(NULL, NULL, 0));
1465     break;
1466   case LOAD_LONG_INDIRECT:
1467   case LOAD_LONG_INDOFFSET32:
1468     // Does not use EBP as address register, use { EDX, EBX, EDI, ESI}
1469     return;
1470   default:
1471     assert(false, "unexpected memory operand in pd_implicit_null_fixup()");
1472     return;
1473   }
1474   node->_opnds[opcnt] = new_memory;
1475 }
1476 
1477 // Advertise here if the CPU requires explicit rounding operations
1478 // to implement the UseStrictFP mode.
1479 const bool Matcher::strict_fp_requires_explicit_rounding = true;
1480 
1481 // Are floats conerted to double when stored to stack during deoptimization?
1482 // On x32 it is stored with convertion only when FPU is used for floats.
1483 bool Matcher::float_in_double() { return (UseSSE == 0); }
1484 
1485 // Do ints take an entire long register or just half?
1486 const bool Matcher::int_in_long = false;
1487 
1488 // Return whether or not this register is ever used as an argument.  This
1489 // function is used on startup to build the trampoline stubs in generateOptoStub.
1490 // Registers not mentioned will be killed by the VM call in the trampoline, and
1491 // arguments in those registers not be available to the callee.
1492 bool Matcher::can_be_java_arg( int reg ) {
1493   if(  reg == ECX_num   || reg == EDX_num   ) return true;
1494   if( (reg == XMM0_num  || reg == XMM1_num ) && UseSSE>=1 ) return true;
1495   if( (reg == XMM0b_num || reg == XMM1b_num) && UseSSE>=2 ) return true;
1496   return false;
1497 }
1498 
1499 bool Matcher::is_spillable_arg( int reg ) {
1500   return can_be_java_arg(reg);
1501 }
1502 
1503 bool Matcher::use_asm_for_ldiv_by_con( jlong divisor ) {
1504   // Use hardware integer DIV instruction when
1505   // it is faster than a code which use multiply.
1506   // Only when constant divisor fits into 32 bit
1507   // (min_jint is excluded to get only correct
1508   // positive 32 bit values from negative).
1509   return VM_Version::has_fast_idiv() &&
1510          (divisor == (int)divisor && divisor != min_jint);
1511 }
1512 
1513 // Register for DIVI projection of divmodI
1514 RegMask Matcher::divI_proj_mask() {
1515   return EAX_REG_mask();
1516 }
1517 
1518 // Register for MODI projection of divmodI
1519 RegMask Matcher::modI_proj_mask() {
1520   return EDX_REG_mask();
1521 }
1522 
1523 // Register for DIVL projection of divmodL
1524 RegMask Matcher::divL_proj_mask() {
1525   ShouldNotReachHere();
1526   return RegMask();
1527 }
1528 
1529 // Register for MODL projection of divmodL
1530 RegMask Matcher::modL_proj_mask() {
1531   ShouldNotReachHere();
1532   return RegMask();
1533 }
1534 
1535 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
1536   return NO_REG_mask();
1537 }
1538 
1539 // Returns true if the high 32 bits of the value is known to be zero.
1540 bool is_operand_hi32_zero(Node* n) {
1541   int opc = n->Opcode();
1542   if (opc == Op_AndL) {
1543     Node* o2 = n->in(2);
1544     if (o2->is_Con() && (o2->get_long() & 0xFFFFFFFF00000000LL) == 0LL) {
1545       return true;
1546     }
1547   }
1548   if (opc == Op_ConL && (n->get_long() & 0xFFFFFFFF00000000LL) == 0LL) {
1549     return true;
1550   }
1551   return false;
1552 }
1553 
1554 %}
1555 
1556 //----------ENCODING BLOCK-----------------------------------------------------
1557 // This block specifies the encoding classes used by the compiler to output
1558 // byte streams.  Encoding classes generate functions which are called by
1559 // Machine Instruction Nodes in order to generate the bit encoding of the
1560 // instruction.  Operands specify their base encoding interface with the
1561 // interface keyword.  There are currently supported four interfaces,
1562 // REG_INTER, CONST_INTER, MEMORY_INTER, & COND_INTER.  REG_INTER causes an
1563 // operand to generate a function which returns its register number when
1564 // queried.   CONST_INTER causes an operand to generate a function which
1565 // returns the value of the constant when queried.  MEMORY_INTER causes an
1566 // operand to generate four functions which return the Base Register, the
1567 // Index Register, the Scale Value, and the Offset Value of the operand when
1568 // queried.  COND_INTER causes an operand to generate six functions which
1569 // return the encoding code (ie - encoding bits for the instruction)
1570 // associated with each basic boolean condition for a conditional instruction.
1571 // Instructions specify two basic values for encoding.  They use the
1572 // ins_encode keyword to specify their encoding class (which must be one of
1573 // the class names specified in the encoding block), and they use the
1574 // opcode keyword to specify, in order, their primary, secondary, and
1575 // tertiary opcode.  Only the opcode sections which a particular instruction
1576 // needs for encoding need to be specified.
1577 encode %{
1578   // Build emit functions for each basic byte or larger field in the intel
1579   // encoding scheme (opcode, rm, sib, immediate), and call them from C++
1580   // code in the enc_class source block.  Emit functions will live in the
1581   // main source block for now.  In future, we can generalize this by
1582   // adding a syntax that specifies the sizes of fields in an order,
1583   // so that the adlc can build the emit functions automagically
1584 
1585   // Emit primary opcode
1586   enc_class OpcP %{
1587     emit_opcode(cbuf, $primary);
1588   %}
1589 
1590   // Emit secondary opcode
1591   enc_class OpcS %{
1592     emit_opcode(cbuf, $secondary);
1593   %}
1594 
1595   // Emit opcode directly
1596   enc_class Opcode(immI d8) %{
1597     emit_opcode(cbuf, $d8$$constant);
1598   %}
1599 
1600   enc_class SizePrefix %{
1601     emit_opcode(cbuf,0x66);
1602   %}
1603 
1604   enc_class RegReg (rRegI dst, rRegI src) %{    // RegReg(Many)
1605     emit_rm(cbuf, 0x3, $dst$$reg, $src$$reg);
1606   %}
1607 
1608   enc_class OpcRegReg (immI opcode, rRegI dst, rRegI src) %{    // OpcRegReg(Many)
1609     emit_opcode(cbuf,$opcode$$constant);
1610     emit_rm(cbuf, 0x3, $dst$$reg, $src$$reg);
1611   %}
1612 
1613   enc_class mov_r32_imm0( rRegI dst ) %{
1614     emit_opcode( cbuf, 0xB8 + $dst$$reg ); // 0xB8+ rd   -- MOV r32  ,imm32
1615     emit_d32   ( cbuf, 0x0  );             //                         imm32==0x0
1616   %}
1617 
1618   enc_class cdq_enc %{
1619     // Full implementation of Java idiv and irem; checks for
1620     // special case as described in JVM spec., p.243 & p.271.
1621     //
1622     //         normal case                           special case
1623     //
1624     // input : rax,: dividend                         min_int
1625     //         reg: divisor                          -1
1626     //
1627     // output: rax,: quotient  (= rax, idiv reg)       min_int
1628     //         rdx: remainder (= rax, irem reg)       0
1629     //
1630     //  Code sequnce:
1631     //
1632     //  81 F8 00 00 00 80    cmp         rax,80000000h
1633     //  0F 85 0B 00 00 00    jne         normal_case
1634     //  33 D2                xor         rdx,edx
1635     //  83 F9 FF             cmp         rcx,0FFh
1636     //  0F 84 03 00 00 00    je          done
1637     //                  normal_case:
1638     //  99                   cdq
1639     //  F7 F9                idiv        rax,ecx
1640     //                  done:
1641     //
1642     emit_opcode(cbuf,0x81); emit_d8(cbuf,0xF8);
1643     emit_opcode(cbuf,0x00); emit_d8(cbuf,0x00);
1644     emit_opcode(cbuf,0x00); emit_d8(cbuf,0x80);                     // cmp rax,80000000h
1645     emit_opcode(cbuf,0x0F); emit_d8(cbuf,0x85);
1646     emit_opcode(cbuf,0x0B); emit_d8(cbuf,0x00);
1647     emit_opcode(cbuf,0x00); emit_d8(cbuf,0x00);                     // jne normal_case
1648     emit_opcode(cbuf,0x33); emit_d8(cbuf,0xD2);                     // xor rdx,edx
1649     emit_opcode(cbuf,0x83); emit_d8(cbuf,0xF9); emit_d8(cbuf,0xFF); // cmp rcx,0FFh
1650     emit_opcode(cbuf,0x0F); emit_d8(cbuf,0x84);
1651     emit_opcode(cbuf,0x03); emit_d8(cbuf,0x00);
1652     emit_opcode(cbuf,0x00); emit_d8(cbuf,0x00);                     // je done
1653     // normal_case:
1654     emit_opcode(cbuf,0x99);                                         // cdq
1655     // idiv (note: must be emitted by the user of this rule)
1656     // normal:
1657   %}
1658 
1659   // Dense encoding for older common ops
1660   enc_class Opc_plus(immI opcode, rRegI reg) %{
1661     emit_opcode(cbuf, $opcode$$constant + $reg$$reg);
1662   %}
1663 
1664 
1665   // Opcde enc_class for 8/32 bit immediate instructions with sign-extension
1666   enc_class OpcSE (immI imm) %{ // Emit primary opcode and set sign-extend bit
1667     // Check for 8-bit immediate, and set sign extend bit in opcode
1668     if (($imm$$constant >= -128) && ($imm$$constant <= 127)) {
1669       emit_opcode(cbuf, $primary | 0x02);
1670     }
1671     else {                          // If 32-bit immediate
1672       emit_opcode(cbuf, $primary);
1673     }
1674   %}
1675 
1676   enc_class OpcSErm (rRegI dst, immI imm) %{    // OpcSEr/m
1677     // Emit primary opcode and set sign-extend bit
1678     // Check for 8-bit immediate, and set sign extend bit in opcode
1679     if (($imm$$constant >= -128) && ($imm$$constant <= 127)) {
1680       emit_opcode(cbuf, $primary | 0x02);    }
1681     else {                          // If 32-bit immediate
1682       emit_opcode(cbuf, $primary);
1683     }
1684     // Emit r/m byte with secondary opcode, after primary opcode.
1685     emit_rm(cbuf, 0x3, $secondary, $dst$$reg);
1686   %}
1687 
1688   enc_class Con8or32 (immI imm) %{    // Con8or32(storeImmI), 8 or 32 bits
1689     // Check for 8-bit immediate, and set sign extend bit in opcode
1690     if (($imm$$constant >= -128) && ($imm$$constant <= 127)) {
1691       $$$emit8$imm$$constant;
1692     }
1693     else {                          // If 32-bit immediate
1694       // Output immediate
1695       $$$emit32$imm$$constant;
1696     }
1697   %}
1698 
1699   enc_class Long_OpcSErm_Lo(eRegL dst, immL imm) %{
1700     // Emit primary opcode and set sign-extend bit
1701     // Check for 8-bit immediate, and set sign extend bit in opcode
1702     int con = (int)$imm$$constant; // Throw away top bits
1703     emit_opcode(cbuf, ((con >= -128) && (con <= 127)) ? ($primary | 0x02) : $primary);
1704     // Emit r/m byte with secondary opcode, after primary opcode.
1705     emit_rm(cbuf, 0x3, $secondary, $dst$$reg);
1706     if ((con >= -128) && (con <= 127)) emit_d8 (cbuf,con);
1707     else                               emit_d32(cbuf,con);
1708   %}
1709 
1710   enc_class Long_OpcSErm_Hi(eRegL dst, immL imm) %{
1711     // Emit primary opcode and set sign-extend bit
1712     // Check for 8-bit immediate, and set sign extend bit in opcode
1713     int con = (int)($imm$$constant >> 32); // Throw away bottom bits
1714     emit_opcode(cbuf, ((con >= -128) && (con <= 127)) ? ($primary | 0x02) : $primary);
1715     // Emit r/m byte with tertiary opcode, after primary opcode.
1716     emit_rm(cbuf, 0x3, $tertiary, HIGH_FROM_LOW($dst$$reg));
1717     if ((con >= -128) && (con <= 127)) emit_d8 (cbuf,con);
1718     else                               emit_d32(cbuf,con);
1719   %}
1720 
1721   enc_class OpcSReg (rRegI dst) %{    // BSWAP
1722     emit_cc(cbuf, $secondary, $dst$$reg );
1723   %}
1724 
1725   enc_class bswap_long_bytes(eRegL dst) %{ // BSWAP
1726     int destlo = $dst$$reg;
1727     int desthi = HIGH_FROM_LOW(destlo);
1728     // bswap lo
1729     emit_opcode(cbuf, 0x0F);
1730     emit_cc(cbuf, 0xC8, destlo);
1731     // bswap hi
1732     emit_opcode(cbuf, 0x0F);
1733     emit_cc(cbuf, 0xC8, desthi);
1734     // xchg lo and hi
1735     emit_opcode(cbuf, 0x87);
1736     emit_rm(cbuf, 0x3, destlo, desthi);
1737   %}
1738 
1739   enc_class RegOpc (rRegI div) %{    // IDIV, IMOD, JMP indirect, ...
1740     emit_rm(cbuf, 0x3, $secondary, $div$$reg );
1741   %}
1742 
1743   enc_class enc_cmov(cmpOp cop ) %{ // CMOV
1744     $$$emit8$primary;
1745     emit_cc(cbuf, $secondary, $cop$$cmpcode);
1746   %}
1747 
1748   enc_class enc_cmov_dpr(cmpOp cop, regDPR src ) %{ // CMOV
1749     int op = 0xDA00 + $cop$$cmpcode + ($src$$reg-1);
1750     emit_d8(cbuf, op >> 8 );
1751     emit_d8(cbuf, op & 255);
1752   %}
1753 
1754   // emulate a CMOV with a conditional branch around a MOV
1755   enc_class enc_cmov_branch( cmpOp cop, immI brOffs ) %{ // CMOV
1756     // Invert sense of branch from sense of CMOV
1757     emit_cc( cbuf, 0x70, ($cop$$cmpcode^1) );
1758     emit_d8( cbuf, $brOffs$$constant );
1759   %}
1760 
1761   enc_class enc_PartialSubtypeCheck( ) %{
1762     Register Redi = as_Register(EDI_enc); // result register
1763     Register Reax = as_Register(EAX_enc); // super class
1764     Register Recx = as_Register(ECX_enc); // killed
1765     Register Resi = as_Register(ESI_enc); // sub class
1766     Label miss;
1767 
1768     MacroAssembler _masm(&cbuf);
1769     __ check_klass_subtype_slow_path(Resi, Reax, Recx, Redi,
1770                                      NULL, &miss,
1771                                      /*set_cond_codes:*/ true);
1772     if ($primary) {
1773       __ xorptr(Redi, Redi);
1774     }
1775     __ bind(miss);
1776   %}
1777 
1778   enc_class FFree_Float_Stack_All %{    // Free_Float_Stack_All
1779     MacroAssembler masm(&cbuf);
1780     int start = masm.offset();
1781     if (UseSSE >= 2) {
1782       if (VerifyFPU) {
1783         masm.verify_FPU(0, "must be empty in SSE2+ mode");
1784       }
1785     } else {
1786       // External c_calling_convention expects the FPU stack to be 'clean'.
1787       // Compiled code leaves it dirty.  Do cleanup now.
1788       masm.empty_FPU_stack();
1789     }
1790     if (sizeof_FFree_Float_Stack_All == -1) {
1791       sizeof_FFree_Float_Stack_All = masm.offset() - start;
1792     } else {
1793       assert(masm.offset() - start == sizeof_FFree_Float_Stack_All, "wrong size");
1794     }
1795   %}
1796 
1797   enc_class Verify_FPU_For_Leaf %{
1798     if( VerifyFPU ) {
1799       MacroAssembler masm(&cbuf);
1800       masm.verify_FPU( -3, "Returning from Runtime Leaf call");
1801     }
1802   %}
1803 
1804   enc_class Java_To_Runtime (method meth) %{    // CALL Java_To_Runtime, Java_To_Runtime_Leaf
1805     // This is the instruction starting address for relocation info.
1806     cbuf.set_insts_mark();
1807     $$$emit8$primary;
1808     // CALL directly to the runtime
1809     emit_d32_reloc(cbuf, ($meth$$method - (int)(cbuf.insts_end()) - 4),
1810                 runtime_call_Relocation::spec(), RELOC_IMM32 );
1811 
1812     if (UseSSE >= 2) {
1813       MacroAssembler _masm(&cbuf);
1814       BasicType rt = tf()->return_type();
1815 
1816       if ((rt == T_FLOAT || rt == T_DOUBLE) && !return_value_is_used()) {
1817         // A C runtime call where the return value is unused.  In SSE2+
1818         // mode the result needs to be removed from the FPU stack.  It's
1819         // likely that this function call could be removed by the
1820         // optimizer if the C function is a pure function.
1821         __ ffree(0);
1822       } else if (rt == T_FLOAT) {
1823         __ lea(rsp, Address(rsp, -4));
1824         __ fstp_s(Address(rsp, 0));
1825         __ movflt(xmm0, Address(rsp, 0));
1826         __ lea(rsp, Address(rsp,  4));
1827       } else if (rt == T_DOUBLE) {
1828         __ lea(rsp, Address(rsp, -8));
1829         __ fstp_d(Address(rsp, 0));
1830         __ movdbl(xmm0, Address(rsp, 0));
1831         __ lea(rsp, Address(rsp,  8));
1832       }
1833     }
1834   %}
1835 
1836 
1837   enc_class pre_call_resets %{
1838     // If method sets FPU control word restore it here
1839     debug_only(int off0 = cbuf.insts_size());
1840     if (ra_->C->in_24_bit_fp_mode()) {
1841       MacroAssembler _masm(&cbuf);
1842       __ fldcw(ExternalAddress(StubRoutines::addr_fpu_cntrl_wrd_std()));
1843     }
1844     if (ra_->C->max_vector_size() > 16) {
1845       // Clear upper bits of YMM registers when current compiled code uses
1846       // wide vectors to avoid AVX <-> SSE transition penalty during call.
1847       MacroAssembler _masm(&cbuf);
1848       __ vzeroupper();
1849     }
1850     debug_only(int off1 = cbuf.insts_size());
1851     assert(off1 - off0 == pre_call_resets_size(), "correct size prediction");
1852   %}
1853 
1854   enc_class post_call_FPU %{
1855     // If method sets FPU control word do it here also
1856     if (Compile::current()->in_24_bit_fp_mode()) {
1857       MacroAssembler masm(&cbuf);
1858       masm.fldcw(ExternalAddress(StubRoutines::addr_fpu_cntrl_wrd_24()));
1859     }
1860   %}
1861 
1862   enc_class Java_Static_Call (method meth) %{    // JAVA STATIC CALL
1863     // CALL to fixup routine.  Fixup routine uses ScopeDesc info to determine
1864     // who we intended to call.
1865     cbuf.set_insts_mark();
1866     $$$emit8$primary;
1867     if (!_method) {
1868       emit_d32_reloc(cbuf, ($meth$$method - (int)(cbuf.insts_end()) - 4),
1869                      runtime_call_Relocation::spec(), RELOC_IMM32 );
1870     } else if (_optimized_virtual) {
1871       emit_d32_reloc(cbuf, ($meth$$method - (int)(cbuf.insts_end()) - 4),
1872                      opt_virtual_call_Relocation::spec(), RELOC_IMM32 );
1873     } else {
1874       emit_d32_reloc(cbuf, ($meth$$method - (int)(cbuf.insts_end()) - 4),
1875                      static_call_Relocation::spec(), RELOC_IMM32 );
1876     }
1877     if (_method) {  // Emit stub for static call.
1878       address stub = CompiledStaticCall::emit_to_interp_stub(cbuf);
1879       if (stub == NULL) {
1880         ciEnv::current()->record_failure("CodeCache is full");
1881         return;
1882       } 
1883     }
1884   %}
1885 
1886   enc_class Java_Dynamic_Call (method meth) %{    // JAVA DYNAMIC CALL
1887     MacroAssembler _masm(&cbuf);
1888     __ ic_call((address)$meth$$method);
1889   %}
1890 
1891   enc_class Java_Compiled_Call (method meth) %{    // JAVA COMPILED CALL
1892     int disp = in_bytes(Method::from_compiled_offset());
1893     assert( -128 <= disp && disp <= 127, "compiled_code_offset isn't small");
1894 
1895     // CALL *[EAX+in_bytes(Method::from_compiled_code_entry_point_offset())]
1896     cbuf.set_insts_mark();
1897     $$$emit8$primary;
1898     emit_rm(cbuf, 0x01, $secondary, EAX_enc );  // R/M byte
1899     emit_d8(cbuf, disp);             // Displacement
1900 
1901   %}
1902 
1903 //   Following encoding is no longer used, but may be restored if calling
1904 //   convention changes significantly.
1905 //   Became: Xor_Reg(EBP), Java_To_Runtime( labl )
1906 //
1907 //   enc_class Java_Interpreter_Call (label labl) %{    // JAVA INTERPRETER CALL
1908 //     // int ic_reg     = Matcher::inline_cache_reg();
1909 //     // int ic_encode  = Matcher::_regEncode[ic_reg];
1910 //     // int imo_reg    = Matcher::interpreter_method_oop_reg();
1911 //     // int imo_encode = Matcher::_regEncode[imo_reg];
1912 //
1913 //     // // Interpreter expects method_oop in EBX, currently a callee-saved register,
1914 //     // // so we load it immediately before the call
1915 //     // emit_opcode(cbuf, 0x8B);                     // MOV    imo_reg,ic_reg  # method_oop
1916 //     // emit_rm(cbuf, 0x03, imo_encode, ic_encode ); // R/M byte
1917 //
1918 //     // xor rbp,ebp
1919 //     emit_opcode(cbuf, 0x33);
1920 //     emit_rm(cbuf, 0x3, EBP_enc, EBP_enc);
1921 //
1922 //     // CALL to interpreter.
1923 //     cbuf.set_insts_mark();
1924 //     $$$emit8$primary;
1925 //     emit_d32_reloc(cbuf, ($labl$$label - (int)(cbuf.insts_end()) - 4),
1926 //                 runtime_call_Relocation::spec(), RELOC_IMM32 );
1927 //   %}
1928 
1929   enc_class RegOpcImm (rRegI dst, immI8 shift) %{    // SHL, SAR, SHR
1930     $$$emit8$primary;
1931     emit_rm(cbuf, 0x3, $secondary, $dst$$reg);
1932     $$$emit8$shift$$constant;
1933   %}
1934 
1935   enc_class LdImmI (rRegI dst, immI src) %{    // Load Immediate
1936     // Load immediate does not have a zero or sign extended version
1937     // for 8-bit immediates
1938     emit_opcode(cbuf, 0xB8 + $dst$$reg);
1939     $$$emit32$src$$constant;
1940   %}
1941 
1942   enc_class LdImmP (rRegI dst, immI src) %{    // Load Immediate
1943     // Load immediate does not have a zero or sign extended version
1944     // for 8-bit immediates
1945     emit_opcode(cbuf, $primary + $dst$$reg);
1946     $$$emit32$src$$constant;
1947   %}
1948 
1949   enc_class LdImmL_Lo( eRegL dst, immL src) %{    // Load Immediate
1950     // Load immediate does not have a zero or sign extended version
1951     // for 8-bit immediates
1952     int dst_enc = $dst$$reg;
1953     int src_con = $src$$constant & 0x0FFFFFFFFL;
1954     if (src_con == 0) {
1955       // xor dst, dst
1956       emit_opcode(cbuf, 0x33);
1957       emit_rm(cbuf, 0x3, dst_enc, dst_enc);
1958     } else {
1959       emit_opcode(cbuf, $primary + dst_enc);
1960       emit_d32(cbuf, src_con);
1961     }
1962   %}
1963 
1964   enc_class LdImmL_Hi( eRegL dst, immL src) %{    // Load Immediate
1965     // Load immediate does not have a zero or sign extended version
1966     // for 8-bit immediates
1967     int dst_enc = $dst$$reg + 2;
1968     int src_con = ((julong)($src$$constant)) >> 32;
1969     if (src_con == 0) {
1970       // xor dst, dst
1971       emit_opcode(cbuf, 0x33);
1972       emit_rm(cbuf, 0x3, dst_enc, dst_enc);
1973     } else {
1974       emit_opcode(cbuf, $primary + dst_enc);
1975       emit_d32(cbuf, src_con);
1976     }
1977   %}
1978 
1979 
1980   // Encode a reg-reg copy.  If it is useless, then empty encoding.
1981   enc_class enc_Copy( rRegI dst, rRegI src ) %{
1982     encode_Copy( cbuf, $dst$$reg, $src$$reg );
1983   %}
1984 
1985   enc_class enc_CopyL_Lo( rRegI dst, eRegL src ) %{
1986     encode_Copy( cbuf, $dst$$reg, $src$$reg );
1987   %}
1988 
1989   enc_class RegReg (rRegI dst, rRegI src) %{    // RegReg(Many)
1990     emit_rm(cbuf, 0x3, $dst$$reg, $src$$reg);
1991   %}
1992 
1993   enc_class RegReg_Lo(eRegL dst, eRegL src) %{    // RegReg(Many)
1994     $$$emit8$primary;
1995     emit_rm(cbuf, 0x3, $dst$$reg, $src$$reg);
1996   %}
1997 
1998   enc_class RegReg_Hi(eRegL dst, eRegL src) %{    // RegReg(Many)
1999     $$$emit8$secondary;
2000     emit_rm(cbuf, 0x3, HIGH_FROM_LOW($dst$$reg), HIGH_FROM_LOW($src$$reg));
2001   %}
2002 
2003   enc_class RegReg_Lo2(eRegL dst, eRegL src) %{    // RegReg(Many)
2004     emit_rm(cbuf, 0x3, $dst$$reg, $src$$reg);
2005   %}
2006 
2007   enc_class RegReg_Hi2(eRegL dst, eRegL src) %{    // RegReg(Many)
2008     emit_rm(cbuf, 0x3, HIGH_FROM_LOW($dst$$reg), HIGH_FROM_LOW($src$$reg));
2009   %}
2010 
2011   enc_class RegReg_HiLo( eRegL src, rRegI dst ) %{
2012     emit_rm(cbuf, 0x3, $dst$$reg, HIGH_FROM_LOW($src$$reg));
2013   %}
2014 
2015   enc_class Con32 (immI src) %{    // Con32(storeImmI)
2016     // Output immediate
2017     $$$emit32$src$$constant;
2018   %}
2019 
2020   enc_class Con32FPR_as_bits(immFPR src) %{        // storeF_imm
2021     // Output Float immediate bits
2022     jfloat jf = $src$$constant;
2023     int    jf_as_bits = jint_cast( jf );
2024     emit_d32(cbuf, jf_as_bits);
2025   %}
2026 
2027   enc_class Con32F_as_bits(immF src) %{      // storeX_imm
2028     // Output Float immediate bits
2029     jfloat jf = $src$$constant;
2030     int    jf_as_bits = jint_cast( jf );
2031     emit_d32(cbuf, jf_as_bits);
2032   %}
2033 
2034   enc_class Con16 (immI src) %{    // Con16(storeImmI)
2035     // Output immediate
2036     $$$emit16$src$$constant;
2037   %}
2038 
2039   enc_class Con_d32(immI src) %{
2040     emit_d32(cbuf,$src$$constant);
2041   %}
2042 
2043   enc_class conmemref (eRegP t1) %{    // Con32(storeImmI)
2044     // Output immediate memory reference
2045     emit_rm(cbuf, 0x00, $t1$$reg, 0x05 );
2046     emit_d32(cbuf, 0x00);
2047   %}
2048 
2049   enc_class lock_prefix( ) %{
2050     if( os::is_MP() )
2051       emit_opcode(cbuf,0xF0);         // [Lock]
2052   %}
2053 
2054   // Cmp-xchg long value.
2055   // Note: we need to swap rbx, and rcx before and after the
2056   //       cmpxchg8 instruction because the instruction uses
2057   //       rcx as the high order word of the new value to store but
2058   //       our register encoding uses rbx,.
2059   enc_class enc_cmpxchg8(eSIRegP mem_ptr) %{
2060 
2061     // XCHG  rbx,ecx
2062     emit_opcode(cbuf,0x87);
2063     emit_opcode(cbuf,0xD9);
2064     // [Lock]
2065     if( os::is_MP() )
2066       emit_opcode(cbuf,0xF0);
2067     // CMPXCHG8 [Eptr]
2068     emit_opcode(cbuf,0x0F);
2069     emit_opcode(cbuf,0xC7);
2070     emit_rm( cbuf, 0x0, 1, $mem_ptr$$reg );
2071     // XCHG  rbx,ecx
2072     emit_opcode(cbuf,0x87);
2073     emit_opcode(cbuf,0xD9);
2074   %}
2075 
2076   enc_class enc_cmpxchg(eSIRegP mem_ptr) %{
2077     // [Lock]
2078     if( os::is_MP() )
2079       emit_opcode(cbuf,0xF0);
2080 
2081     // CMPXCHG [Eptr]
2082     emit_opcode(cbuf,0x0F);
2083     emit_opcode(cbuf,0xB1);
2084     emit_rm( cbuf, 0x0, 1, $mem_ptr$$reg );
2085   %}
2086 
2087   enc_class enc_flags_ne_to_boolean( iRegI res ) %{
2088     int res_encoding = $res$$reg;
2089 
2090     // MOV  res,0
2091     emit_opcode( cbuf, 0xB8 + res_encoding);
2092     emit_d32( cbuf, 0 );
2093     // JNE,s  fail
2094     emit_opcode(cbuf,0x75);
2095     emit_d8(cbuf, 5 );
2096     // MOV  res,1
2097     emit_opcode( cbuf, 0xB8 + res_encoding);
2098     emit_d32( cbuf, 1 );
2099     // fail:
2100   %}
2101 
2102   enc_class set_instruction_start( ) %{
2103     cbuf.set_insts_mark();            // Mark start of opcode for reloc info in mem operand
2104   %}
2105 
2106   enc_class RegMem (rRegI ereg, memory mem) %{    // emit_reg_mem
2107     int reg_encoding = $ereg$$reg;
2108     int base  = $mem$$base;
2109     int index = $mem$$index;
2110     int scale = $mem$$scale;
2111     int displace = $mem$$disp;
2112     relocInfo::relocType disp_reloc = $mem->disp_reloc();
2113     encode_RegMem(cbuf, reg_encoding, base, index, scale, displace, disp_reloc);
2114   %}
2115 
2116   enc_class RegMem_Hi(eRegL ereg, memory mem) %{    // emit_reg_mem
2117     int reg_encoding = HIGH_FROM_LOW($ereg$$reg);  // Hi register of pair, computed from lo
2118     int base  = $mem$$base;
2119     int index = $mem$$index;
2120     int scale = $mem$$scale;
2121     int displace = $mem$$disp + 4;      // Offset is 4 further in memory
2122     assert( $mem->disp_reloc() == relocInfo::none, "Cannot add 4 to oop" );
2123     encode_RegMem(cbuf, reg_encoding, base, index, scale, displace, relocInfo::none);
2124   %}
2125 
2126   enc_class move_long_small_shift( eRegL dst, immI_1_31 cnt ) %{
2127     int r1, r2;
2128     if( $tertiary == 0xA4 ) { r1 = $dst$$reg;  r2 = HIGH_FROM_LOW($dst$$reg); }
2129     else                    { r2 = $dst$$reg;  r1 = HIGH_FROM_LOW($dst$$reg); }
2130     emit_opcode(cbuf,0x0F);
2131     emit_opcode(cbuf,$tertiary);
2132     emit_rm(cbuf, 0x3, r1, r2);
2133     emit_d8(cbuf,$cnt$$constant);
2134     emit_d8(cbuf,$primary);
2135     emit_rm(cbuf, 0x3, $secondary, r1);
2136     emit_d8(cbuf,$cnt$$constant);
2137   %}
2138 
2139   enc_class move_long_big_shift_sign( eRegL dst, immI_32_63 cnt ) %{
2140     emit_opcode( cbuf, 0x8B ); // Move
2141     emit_rm(cbuf, 0x3, $dst$$reg, HIGH_FROM_LOW($dst$$reg));
2142     if( $cnt$$constant > 32 ) { // Shift, if not by zero
2143       emit_d8(cbuf,$primary);
2144       emit_rm(cbuf, 0x3, $secondary, $dst$$reg);
2145       emit_d8(cbuf,$cnt$$constant-32);
2146     }
2147     emit_d8(cbuf,$primary);
2148     emit_rm(cbuf, 0x3, $secondary, HIGH_FROM_LOW($dst$$reg));
2149     emit_d8(cbuf,31);
2150   %}
2151 
2152   enc_class move_long_big_shift_clr( eRegL dst, immI_32_63 cnt ) %{
2153     int r1, r2;
2154     if( $secondary == 0x5 ) { r1 = $dst$$reg;  r2 = HIGH_FROM_LOW($dst$$reg); }
2155     else                    { r2 = $dst$$reg;  r1 = HIGH_FROM_LOW($dst$$reg); }
2156 
2157     emit_opcode( cbuf, 0x8B ); // Move r1,r2
2158     emit_rm(cbuf, 0x3, r1, r2);
2159     if( $cnt$$constant > 32 ) { // Shift, if not by zero
2160       emit_opcode(cbuf,$primary);
2161       emit_rm(cbuf, 0x3, $secondary, r1);
2162       emit_d8(cbuf,$cnt$$constant-32);
2163     }
2164     emit_opcode(cbuf,0x33);  // XOR r2,r2
2165     emit_rm(cbuf, 0x3, r2, r2);
2166   %}
2167 
2168   // Clone of RegMem but accepts an extra parameter to access each
2169   // half of a double in memory; it never needs relocation info.
2170   enc_class Mov_MemD_half_to_Reg (immI opcode, memory mem, immI disp_for_half, rRegI rm_reg) %{
2171     emit_opcode(cbuf,$opcode$$constant);
2172     int reg_encoding = $rm_reg$$reg;
2173     int base     = $mem$$base;
2174     int index    = $mem$$index;
2175     int scale    = $mem$$scale;
2176     int displace = $mem$$disp + $disp_for_half$$constant;
2177     relocInfo::relocType disp_reloc = relocInfo::none;
2178     encode_RegMem(cbuf, reg_encoding, base, index, scale, displace, disp_reloc);
2179   %}
2180 
2181   // !!!!! Special Custom Code used by MemMove, and stack access instructions !!!!!
2182   //
2183   // Clone of RegMem except the RM-byte's reg/opcode field is an ADLC-time constant
2184   // and it never needs relocation information.
2185   // Frequently used to move data between FPU's Stack Top and memory.
2186   enc_class RMopc_Mem_no_oop (immI rm_opcode, memory mem) %{
2187     int rm_byte_opcode = $rm_opcode$$constant;
2188     int base     = $mem$$base;
2189     int index    = $mem$$index;
2190     int scale    = $mem$$scale;
2191     int displace = $mem$$disp;
2192     assert( $mem->disp_reloc() == relocInfo::none, "No oops here because no reloc info allowed" );
2193     encode_RegMem(cbuf, rm_byte_opcode, base, index, scale, displace, relocInfo::none);
2194   %}
2195 
2196   enc_class RMopc_Mem (immI rm_opcode, memory mem) %{
2197     int rm_byte_opcode = $rm_opcode$$constant;
2198     int base     = $mem$$base;
2199     int index    = $mem$$index;
2200     int scale    = $mem$$scale;
2201     int displace = $mem$$disp;
2202     relocInfo::relocType disp_reloc = $mem->disp_reloc(); // disp-as-oop when working with static globals
2203     encode_RegMem(cbuf, rm_byte_opcode, base, index, scale, displace, disp_reloc);
2204   %}
2205 
2206   enc_class RegLea (rRegI dst, rRegI src0, immI src1 ) %{    // emit_reg_lea
2207     int reg_encoding = $dst$$reg;
2208     int base         = $src0$$reg;      // 0xFFFFFFFF indicates no base
2209     int index        = 0x04;            // 0x04 indicates no index
2210     int scale        = 0x00;            // 0x00 indicates no scale
2211     int displace     = $src1$$constant; // 0x00 indicates no displacement
2212     relocInfo::relocType disp_reloc = relocInfo::none;
2213     encode_RegMem(cbuf, reg_encoding, base, index, scale, displace, disp_reloc);
2214   %}
2215 
2216   enc_class min_enc (rRegI dst, rRegI src) %{    // MIN
2217     // Compare dst,src
2218     emit_opcode(cbuf,0x3B);
2219     emit_rm(cbuf, 0x3, $dst$$reg, $src$$reg);
2220     // jmp dst < src around move
2221     emit_opcode(cbuf,0x7C);
2222     emit_d8(cbuf,2);
2223     // move dst,src
2224     emit_opcode(cbuf,0x8B);
2225     emit_rm(cbuf, 0x3, $dst$$reg, $src$$reg);
2226   %}
2227 
2228   enc_class max_enc (rRegI dst, rRegI src) %{    // MAX
2229     // Compare dst,src
2230     emit_opcode(cbuf,0x3B);
2231     emit_rm(cbuf, 0x3, $dst$$reg, $src$$reg);
2232     // jmp dst > src around move
2233     emit_opcode(cbuf,0x7F);
2234     emit_d8(cbuf,2);
2235     // move dst,src
2236     emit_opcode(cbuf,0x8B);
2237     emit_rm(cbuf, 0x3, $dst$$reg, $src$$reg);
2238   %}
2239 
2240   enc_class enc_FPR_store(memory mem, regDPR src) %{
2241     // If src is FPR1, we can just FST to store it.
2242     // Else we need to FLD it to FPR1, then FSTP to store/pop it.
2243     int reg_encoding = 0x2; // Just store
2244     int base  = $mem$$base;
2245     int index = $mem$$index;
2246     int scale = $mem$$scale;
2247     int displace = $mem$$disp;
2248     relocInfo::relocType disp_reloc = $mem->disp_reloc(); // disp-as-oop when working with static globals
2249     if( $src$$reg != FPR1L_enc ) {
2250       reg_encoding = 0x3;  // Store & pop
2251       emit_opcode( cbuf, 0xD9 ); // FLD (i.e., push it)
2252       emit_d8( cbuf, 0xC0-1+$src$$reg );
2253     }
2254     cbuf.set_insts_mark();       // Mark start of opcode for reloc info in mem operand
2255     emit_opcode(cbuf,$primary);
2256     encode_RegMem(cbuf, reg_encoding, base, index, scale, displace, disp_reloc);
2257   %}
2258 
2259   enc_class neg_reg(rRegI dst) %{
2260     // NEG $dst
2261     emit_opcode(cbuf,0xF7);
2262     emit_rm(cbuf, 0x3, 0x03, $dst$$reg );
2263   %}
2264 
2265   enc_class setLT_reg(eCXRegI dst) %{
2266     // SETLT $dst
2267     emit_opcode(cbuf,0x0F);
2268     emit_opcode(cbuf,0x9C);
2269     emit_rm( cbuf, 0x3, 0x4, $dst$$reg );
2270   %}
2271 
2272   enc_class enc_cmpLTP(ncxRegI p, ncxRegI q, ncxRegI y, eCXRegI tmp) %{    // cadd_cmpLT
2273     int tmpReg = $tmp$$reg;
2274 
2275     // SUB $p,$q
2276     emit_opcode(cbuf,0x2B);
2277     emit_rm(cbuf, 0x3, $p$$reg, $q$$reg);
2278     // SBB $tmp,$tmp
2279     emit_opcode(cbuf,0x1B);
2280     emit_rm(cbuf, 0x3, tmpReg, tmpReg);
2281     // AND $tmp,$y
2282     emit_opcode(cbuf,0x23);
2283     emit_rm(cbuf, 0x3, tmpReg, $y$$reg);
2284     // ADD $p,$tmp
2285     emit_opcode(cbuf,0x03);
2286     emit_rm(cbuf, 0x3, $p$$reg, tmpReg);
2287   %}
2288 
2289   enc_class shift_left_long( eRegL dst, eCXRegI shift ) %{
2290     // TEST shift,32
2291     emit_opcode(cbuf,0xF7);
2292     emit_rm(cbuf, 0x3, 0, ECX_enc);
2293     emit_d32(cbuf,0x20);
2294     // JEQ,s small
2295     emit_opcode(cbuf, 0x74);
2296     emit_d8(cbuf, 0x04);
2297     // MOV    $dst.hi,$dst.lo
2298     emit_opcode( cbuf, 0x8B );
2299     emit_rm(cbuf, 0x3, HIGH_FROM_LOW($dst$$reg), $dst$$reg );
2300     // CLR    $dst.lo
2301     emit_opcode(cbuf, 0x33);
2302     emit_rm(cbuf, 0x3, $dst$$reg, $dst$$reg);
2303 // small:
2304     // SHLD   $dst.hi,$dst.lo,$shift
2305     emit_opcode(cbuf,0x0F);
2306     emit_opcode(cbuf,0xA5);
2307     emit_rm(cbuf, 0x3, $dst$$reg, HIGH_FROM_LOW($dst$$reg));
2308     // SHL    $dst.lo,$shift"
2309     emit_opcode(cbuf,0xD3);
2310     emit_rm(cbuf, 0x3, 0x4, $dst$$reg );
2311   %}
2312 
2313   enc_class shift_right_long( eRegL dst, eCXRegI shift ) %{
2314     // TEST shift,32
2315     emit_opcode(cbuf,0xF7);
2316     emit_rm(cbuf, 0x3, 0, ECX_enc);
2317     emit_d32(cbuf,0x20);
2318     // JEQ,s small
2319     emit_opcode(cbuf, 0x74);
2320     emit_d8(cbuf, 0x04);
2321     // MOV    $dst.lo,$dst.hi
2322     emit_opcode( cbuf, 0x8B );
2323     emit_rm(cbuf, 0x3, $dst$$reg, HIGH_FROM_LOW($dst$$reg) );
2324     // CLR    $dst.hi
2325     emit_opcode(cbuf, 0x33);
2326     emit_rm(cbuf, 0x3, HIGH_FROM_LOW($dst$$reg), HIGH_FROM_LOW($dst$$reg));
2327 // small:
2328     // SHRD   $dst.lo,$dst.hi,$shift
2329     emit_opcode(cbuf,0x0F);
2330     emit_opcode(cbuf,0xAD);
2331     emit_rm(cbuf, 0x3, HIGH_FROM_LOW($dst$$reg), $dst$$reg);
2332     // SHR    $dst.hi,$shift"
2333     emit_opcode(cbuf,0xD3);
2334     emit_rm(cbuf, 0x3, 0x5, HIGH_FROM_LOW($dst$$reg) );
2335   %}
2336 
2337   enc_class shift_right_arith_long( eRegL dst, eCXRegI shift ) %{
2338     // TEST shift,32
2339     emit_opcode(cbuf,0xF7);
2340     emit_rm(cbuf, 0x3, 0, ECX_enc);
2341     emit_d32(cbuf,0x20);
2342     // JEQ,s small
2343     emit_opcode(cbuf, 0x74);
2344     emit_d8(cbuf, 0x05);
2345     // MOV    $dst.lo,$dst.hi
2346     emit_opcode( cbuf, 0x8B );
2347     emit_rm(cbuf, 0x3, $dst$$reg, HIGH_FROM_LOW($dst$$reg) );
2348     // SAR    $dst.hi,31
2349     emit_opcode(cbuf, 0xC1);
2350     emit_rm(cbuf, 0x3, 7, HIGH_FROM_LOW($dst$$reg) );
2351     emit_d8(cbuf, 0x1F );
2352 // small:
2353     // SHRD   $dst.lo,$dst.hi,$shift
2354     emit_opcode(cbuf,0x0F);
2355     emit_opcode(cbuf,0xAD);
2356     emit_rm(cbuf, 0x3, HIGH_FROM_LOW($dst$$reg), $dst$$reg);
2357     // SAR    $dst.hi,$shift"
2358     emit_opcode(cbuf,0xD3);
2359     emit_rm(cbuf, 0x3, 0x7, HIGH_FROM_LOW($dst$$reg) );
2360   %}
2361 
2362 
2363   // ----------------- Encodings for floating point unit -----------------
2364   // May leave result in FPU-TOS or FPU reg depending on opcodes
2365   enc_class OpcReg_FPR(regFPR src) %{    // FMUL, FDIV
2366     $$$emit8$primary;
2367     emit_rm(cbuf, 0x3, $secondary, $src$$reg );
2368   %}
2369 
2370   // Pop argument in FPR0 with FSTP ST(0)
2371   enc_class PopFPU() %{
2372     emit_opcode( cbuf, 0xDD );
2373     emit_d8( cbuf, 0xD8 );
2374   %}
2375 
2376   // !!!!! equivalent to Pop_Reg_F
2377   enc_class Pop_Reg_DPR( regDPR dst ) %{
2378     emit_opcode( cbuf, 0xDD );           // FSTP   ST(i)
2379     emit_d8( cbuf, 0xD8+$dst$$reg );
2380   %}
2381 
2382   enc_class Push_Reg_DPR( regDPR dst ) %{
2383     emit_opcode( cbuf, 0xD9 );
2384     emit_d8( cbuf, 0xC0-1+$dst$$reg );   // FLD ST(i-1)
2385   %}
2386 
2387   enc_class strictfp_bias1( regDPR dst ) %{
2388     emit_opcode( cbuf, 0xDB );           // FLD m80real
2389     emit_opcode( cbuf, 0x2D );
2390     emit_d32( cbuf, (int)StubRoutines::addr_fpu_subnormal_bias1() );
2391     emit_opcode( cbuf, 0xDE );           // FMULP ST(dst), ST0
2392     emit_opcode( cbuf, 0xC8+$dst$$reg );
2393   %}
2394 
2395   enc_class strictfp_bias2( regDPR dst ) %{
2396     emit_opcode( cbuf, 0xDB );           // FLD m80real
2397     emit_opcode( cbuf, 0x2D );
2398     emit_d32( cbuf, (int)StubRoutines::addr_fpu_subnormal_bias2() );
2399     emit_opcode( cbuf, 0xDE );           // FMULP ST(dst), ST0
2400     emit_opcode( cbuf, 0xC8+$dst$$reg );
2401   %}
2402 
2403   // Special case for moving an integer register to a stack slot.
2404   enc_class OpcPRegSS( stackSlotI dst, rRegI src ) %{ // RegSS
2405     store_to_stackslot( cbuf, $primary, $src$$reg, $dst$$disp );
2406   %}
2407 
2408   // Special case for moving a register to a stack slot.
2409   enc_class RegSS( stackSlotI dst, rRegI src ) %{ // RegSS
2410     // Opcode already emitted
2411     emit_rm( cbuf, 0x02, $src$$reg, ESP_enc );   // R/M byte
2412     emit_rm( cbuf, 0x00, ESP_enc, ESP_enc);          // SIB byte
2413     emit_d32(cbuf, $dst$$disp);   // Displacement
2414   %}
2415 
2416   // Push the integer in stackSlot 'src' onto FP-stack
2417   enc_class Push_Mem_I( memory src ) %{    // FILD   [ESP+src]
2418     store_to_stackslot( cbuf, $primary, $secondary, $src$$disp );
2419   %}
2420 
2421   // Push FPU's TOS float to a stack-slot, and pop FPU-stack
2422   enc_class Pop_Mem_FPR( stackSlotF dst ) %{ // FSTP_S [ESP+dst]
2423     store_to_stackslot( cbuf, 0xD9, 0x03, $dst$$disp );
2424   %}
2425 
2426   // Same as Pop_Mem_F except for opcode
2427   // Push FPU's TOS double to a stack-slot, and pop FPU-stack
2428   enc_class Pop_Mem_DPR( stackSlotD dst ) %{ // FSTP_D [ESP+dst]
2429     store_to_stackslot( cbuf, 0xDD, 0x03, $dst$$disp );
2430   %}
2431 
2432   enc_class Pop_Reg_FPR( regFPR dst ) %{
2433     emit_opcode( cbuf, 0xDD );           // FSTP   ST(i)
2434     emit_d8( cbuf, 0xD8+$dst$$reg );
2435   %}
2436 
2437   enc_class Push_Reg_FPR( regFPR dst ) %{
2438     emit_opcode( cbuf, 0xD9 );           // FLD    ST(i-1)
2439     emit_d8( cbuf, 0xC0-1+$dst$$reg );
2440   %}
2441 
2442   // Push FPU's float to a stack-slot, and pop FPU-stack
2443   enc_class Pop_Mem_Reg_FPR( stackSlotF dst, regFPR src ) %{
2444     int pop = 0x02;
2445     if ($src$$reg != FPR1L_enc) {
2446       emit_opcode( cbuf, 0xD9 );         // FLD    ST(i-1)
2447       emit_d8( cbuf, 0xC0-1+$src$$reg );
2448       pop = 0x03;
2449     }
2450     store_to_stackslot( cbuf, 0xD9, pop, $dst$$disp ); // FST<P>_S  [ESP+dst]
2451   %}
2452 
2453   // Push FPU's double to a stack-slot, and pop FPU-stack
2454   enc_class Pop_Mem_Reg_DPR( stackSlotD dst, regDPR src ) %{
2455     int pop = 0x02;
2456     if ($src$$reg != FPR1L_enc) {
2457       emit_opcode( cbuf, 0xD9 );         // FLD    ST(i-1)
2458       emit_d8( cbuf, 0xC0-1+$src$$reg );
2459       pop = 0x03;
2460     }
2461     store_to_stackslot( cbuf, 0xDD, pop, $dst$$disp ); // FST<P>_D  [ESP+dst]
2462   %}
2463 
2464   // Push FPU's double to a FPU-stack-slot, and pop FPU-stack
2465   enc_class Pop_Reg_Reg_DPR( regDPR dst, regFPR src ) %{
2466     int pop = 0xD0 - 1; // -1 since we skip FLD
2467     if ($src$$reg != FPR1L_enc) {
2468       emit_opcode( cbuf, 0xD9 );         // FLD    ST(src-1)
2469       emit_d8( cbuf, 0xC0-1+$src$$reg );
2470       pop = 0xD8;
2471     }
2472     emit_opcode( cbuf, 0xDD );
2473     emit_d8( cbuf, pop+$dst$$reg );      // FST<P> ST(i)
2474   %}
2475 
2476 
2477   enc_class Push_Reg_Mod_DPR( regDPR dst, regDPR src) %{
2478     // load dst in FPR0
2479     emit_opcode( cbuf, 0xD9 );
2480     emit_d8( cbuf, 0xC0-1+$dst$$reg );
2481     if ($src$$reg != FPR1L_enc) {
2482       // fincstp
2483       emit_opcode (cbuf, 0xD9);
2484       emit_opcode (cbuf, 0xF7);
2485       // swap src with FPR1:
2486       // FXCH FPR1 with src
2487       emit_opcode(cbuf, 0xD9);
2488       emit_d8(cbuf, 0xC8-1+$src$$reg );
2489       // fdecstp
2490       emit_opcode (cbuf, 0xD9);
2491       emit_opcode (cbuf, 0xF6);
2492     }
2493   %}
2494 
2495   enc_class Push_ModD_encoding(regD src0, regD src1) %{
2496     MacroAssembler _masm(&cbuf);
2497     __ subptr(rsp, 8);
2498     __ movdbl(Address(rsp, 0), $src1$$XMMRegister);
2499     __ fld_d(Address(rsp, 0));
2500     __ movdbl(Address(rsp, 0), $src0$$XMMRegister);
2501     __ fld_d(Address(rsp, 0));
2502   %}
2503 
2504   enc_class Push_ModF_encoding(regF src0, regF src1) %{
2505     MacroAssembler _masm(&cbuf);
2506     __ subptr(rsp, 4);
2507     __ movflt(Address(rsp, 0), $src1$$XMMRegister);
2508     __ fld_s(Address(rsp, 0));
2509     __ movflt(Address(rsp, 0), $src0$$XMMRegister);
2510     __ fld_s(Address(rsp, 0));
2511   %}
2512 
2513   enc_class Push_ResultD(regD dst) %{
2514     MacroAssembler _masm(&cbuf);
2515     __ fstp_d(Address(rsp, 0));
2516     __ movdbl($dst$$XMMRegister, Address(rsp, 0));
2517     __ addptr(rsp, 8);
2518   %}
2519 
2520   enc_class Push_ResultF(regF dst, immI d8) %{
2521     MacroAssembler _masm(&cbuf);
2522     __ fstp_s(Address(rsp, 0));
2523     __ movflt($dst$$XMMRegister, Address(rsp, 0));
2524     __ addptr(rsp, $d8$$constant);
2525   %}
2526 
2527   enc_class Push_SrcD(regD src) %{
2528     MacroAssembler _masm(&cbuf);
2529     __ subptr(rsp, 8);
2530     __ movdbl(Address(rsp, 0), $src$$XMMRegister);
2531     __ fld_d(Address(rsp, 0));
2532   %}
2533 
2534   enc_class push_stack_temp_qword() %{
2535     MacroAssembler _masm(&cbuf);
2536     __ subptr(rsp, 8);
2537   %}
2538 
2539   enc_class pop_stack_temp_qword() %{
2540     MacroAssembler _masm(&cbuf);
2541     __ addptr(rsp, 8);
2542   %}
2543 
2544   enc_class push_xmm_to_fpr1(regD src) %{
2545     MacroAssembler _masm(&cbuf);
2546     __ movdbl(Address(rsp, 0), $src$$XMMRegister);
2547     __ fld_d(Address(rsp, 0));
2548   %}
2549 
2550   enc_class Push_Result_Mod_DPR( regDPR src) %{
2551     if ($src$$reg != FPR1L_enc) {
2552       // fincstp
2553       emit_opcode (cbuf, 0xD9);
2554       emit_opcode (cbuf, 0xF7);
2555       // FXCH FPR1 with src
2556       emit_opcode(cbuf, 0xD9);
2557       emit_d8(cbuf, 0xC8-1+$src$$reg );
2558       // fdecstp
2559       emit_opcode (cbuf, 0xD9);
2560       emit_opcode (cbuf, 0xF6);
2561     }
2562     // // following asm replaced with Pop_Reg_F or Pop_Mem_F
2563     // // FSTP   FPR$dst$$reg
2564     // emit_opcode( cbuf, 0xDD );
2565     // emit_d8( cbuf, 0xD8+$dst$$reg );
2566   %}
2567 
2568   enc_class fnstsw_sahf_skip_parity() %{
2569     // fnstsw ax
2570     emit_opcode( cbuf, 0xDF );
2571     emit_opcode( cbuf, 0xE0 );
2572     // sahf
2573     emit_opcode( cbuf, 0x9E );
2574     // jnp  ::skip
2575     emit_opcode( cbuf, 0x7B );
2576     emit_opcode( cbuf, 0x05 );
2577   %}
2578 
2579   enc_class emitModDPR() %{
2580     // fprem must be iterative
2581     // :: loop
2582     // fprem
2583     emit_opcode( cbuf, 0xD9 );
2584     emit_opcode( cbuf, 0xF8 );
2585     // wait
2586     emit_opcode( cbuf, 0x9b );
2587     // fnstsw ax
2588     emit_opcode( cbuf, 0xDF );
2589     emit_opcode( cbuf, 0xE0 );
2590     // sahf
2591     emit_opcode( cbuf, 0x9E );
2592     // jp  ::loop
2593     emit_opcode( cbuf, 0x0F );
2594     emit_opcode( cbuf, 0x8A );
2595     emit_opcode( cbuf, 0xF4 );
2596     emit_opcode( cbuf, 0xFF );
2597     emit_opcode( cbuf, 0xFF );
2598     emit_opcode( cbuf, 0xFF );
2599   %}
2600 
2601   enc_class fpu_flags() %{
2602     // fnstsw_ax
2603     emit_opcode( cbuf, 0xDF);
2604     emit_opcode( cbuf, 0xE0);
2605     // test ax,0x0400
2606     emit_opcode( cbuf, 0x66 );   // operand-size prefix for 16-bit immediate
2607     emit_opcode( cbuf, 0xA9 );
2608     emit_d16   ( cbuf, 0x0400 );
2609     // // // This sequence works, but stalls for 12-16 cycles on PPro
2610     // // test rax,0x0400
2611     // emit_opcode( cbuf, 0xA9 );
2612     // emit_d32   ( cbuf, 0x00000400 );
2613     //
2614     // jz exit (no unordered comparison)
2615     emit_opcode( cbuf, 0x74 );
2616     emit_d8    ( cbuf, 0x02 );
2617     // mov ah,1 - treat as LT case (set carry flag)
2618     emit_opcode( cbuf, 0xB4 );
2619     emit_d8    ( cbuf, 0x01 );
2620     // sahf
2621     emit_opcode( cbuf, 0x9E);
2622   %}
2623 
2624   enc_class cmpF_P6_fixup() %{
2625     // Fixup the integer flags in case comparison involved a NaN
2626     //
2627     // JNP exit (no unordered comparison, P-flag is set by NaN)
2628     emit_opcode( cbuf, 0x7B );
2629     emit_d8    ( cbuf, 0x03 );
2630     // MOV AH,1 - treat as LT case (set carry flag)
2631     emit_opcode( cbuf, 0xB4 );
2632     emit_d8    ( cbuf, 0x01 );
2633     // SAHF
2634     emit_opcode( cbuf, 0x9E);
2635     // NOP     // target for branch to avoid branch to branch
2636     emit_opcode( cbuf, 0x90);
2637   %}
2638 
2639 //     fnstsw_ax();
2640 //     sahf();
2641 //     movl(dst, nan_result);
2642 //     jcc(Assembler::parity, exit);
2643 //     movl(dst, less_result);
2644 //     jcc(Assembler::below, exit);
2645 //     movl(dst, equal_result);
2646 //     jcc(Assembler::equal, exit);
2647 //     movl(dst, greater_result);
2648 
2649 // less_result     =  1;
2650 // greater_result  = -1;
2651 // equal_result    = 0;
2652 // nan_result      = -1;
2653 
2654   enc_class CmpF_Result(rRegI dst) %{
2655     // fnstsw_ax();
2656     emit_opcode( cbuf, 0xDF);
2657     emit_opcode( cbuf, 0xE0);
2658     // sahf
2659     emit_opcode( cbuf, 0x9E);
2660     // movl(dst, nan_result);
2661     emit_opcode( cbuf, 0xB8 + $dst$$reg);
2662     emit_d32( cbuf, -1 );
2663     // jcc(Assembler::parity, exit);
2664     emit_opcode( cbuf, 0x7A );
2665     emit_d8    ( cbuf, 0x13 );
2666     // movl(dst, less_result);
2667     emit_opcode( cbuf, 0xB8 + $dst$$reg);
2668     emit_d32( cbuf, -1 );
2669     // jcc(Assembler::below, exit);
2670     emit_opcode( cbuf, 0x72 );
2671     emit_d8    ( cbuf, 0x0C );
2672     // movl(dst, equal_result);
2673     emit_opcode( cbuf, 0xB8 + $dst$$reg);
2674     emit_d32( cbuf, 0 );
2675     // jcc(Assembler::equal, exit);
2676     emit_opcode( cbuf, 0x74 );
2677     emit_d8    ( cbuf, 0x05 );
2678     // movl(dst, greater_result);
2679     emit_opcode( cbuf, 0xB8 + $dst$$reg);
2680     emit_d32( cbuf, 1 );
2681   %}
2682 
2683 
2684   // Compare the longs and set flags
2685   // BROKEN!  Do Not use as-is
2686   enc_class cmpl_test( eRegL src1, eRegL src2 ) %{
2687     // CMP    $src1.hi,$src2.hi
2688     emit_opcode( cbuf, 0x3B );
2689     emit_rm(cbuf, 0x3, HIGH_FROM_LOW($src1$$reg), HIGH_FROM_LOW($src2$$reg) );
2690     // JNE,s  done
2691     emit_opcode(cbuf,0x75);
2692     emit_d8(cbuf, 2 );
2693     // CMP    $src1.lo,$src2.lo
2694     emit_opcode( cbuf, 0x3B );
2695     emit_rm(cbuf, 0x3, $src1$$reg, $src2$$reg );
2696 // done:
2697   %}
2698 
2699   enc_class convert_int_long( regL dst, rRegI src ) %{
2700     // mov $dst.lo,$src
2701     int dst_encoding = $dst$$reg;
2702     int src_encoding = $src$$reg;
2703     encode_Copy( cbuf, dst_encoding  , src_encoding );
2704     // mov $dst.hi,$src
2705     encode_Copy( cbuf, HIGH_FROM_LOW(dst_encoding), src_encoding );
2706     // sar $dst.hi,31
2707     emit_opcode( cbuf, 0xC1 );
2708     emit_rm(cbuf, 0x3, 7, HIGH_FROM_LOW(dst_encoding) );
2709     emit_d8(cbuf, 0x1F );
2710   %}
2711 
2712   enc_class convert_long_double( eRegL src ) %{
2713     // push $src.hi
2714     emit_opcode(cbuf, 0x50+HIGH_FROM_LOW($src$$reg));
2715     // push $src.lo
2716     emit_opcode(cbuf, 0x50+$src$$reg  );
2717     // fild 64-bits at [SP]
2718     emit_opcode(cbuf,0xdf);
2719     emit_d8(cbuf, 0x6C);
2720     emit_d8(cbuf, 0x24);
2721     emit_d8(cbuf, 0x00);
2722     // pop stack
2723     emit_opcode(cbuf, 0x83); // add  SP, #8
2724     emit_rm(cbuf, 0x3, 0x00, ESP_enc);
2725     emit_d8(cbuf, 0x8);
2726   %}
2727 
2728   enc_class multiply_con_and_shift_high( eDXRegI dst, nadxRegI src1, eADXRegL_low_only src2, immI_32_63 cnt, eFlagsReg cr ) %{
2729     // IMUL   EDX:EAX,$src1
2730     emit_opcode( cbuf, 0xF7 );
2731     emit_rm( cbuf, 0x3, 0x5, $src1$$reg );
2732     // SAR    EDX,$cnt-32
2733     int shift_count = ((int)$cnt$$constant) - 32;
2734     if (shift_count > 0) {
2735       emit_opcode(cbuf, 0xC1);
2736       emit_rm(cbuf, 0x3, 7, $dst$$reg );
2737       emit_d8(cbuf, shift_count);
2738     }
2739   %}
2740 
2741   // this version doesn't have add sp, 8
2742   enc_class convert_long_double2( eRegL src ) %{
2743     // push $src.hi
2744     emit_opcode(cbuf, 0x50+HIGH_FROM_LOW($src$$reg));
2745     // push $src.lo
2746     emit_opcode(cbuf, 0x50+$src$$reg  );
2747     // fild 64-bits at [SP]
2748     emit_opcode(cbuf,0xdf);
2749     emit_d8(cbuf, 0x6C);
2750     emit_d8(cbuf, 0x24);
2751     emit_d8(cbuf, 0x00);
2752   %}
2753 
2754   enc_class long_int_multiply( eADXRegL dst, nadxRegI src) %{
2755     // Basic idea: long = (long)int * (long)int
2756     // IMUL EDX:EAX, src
2757     emit_opcode( cbuf, 0xF7 );
2758     emit_rm( cbuf, 0x3, 0x5, $src$$reg);
2759   %}
2760 
2761   enc_class long_uint_multiply( eADXRegL dst, nadxRegI src) %{
2762     // Basic Idea:  long = (int & 0xffffffffL) * (int & 0xffffffffL)
2763     // MUL EDX:EAX, src
2764     emit_opcode( cbuf, 0xF7 );
2765     emit_rm( cbuf, 0x3, 0x4, $src$$reg);
2766   %}
2767 
2768   enc_class long_multiply( eADXRegL dst, eRegL src, rRegI tmp ) %{
2769     // Basic idea: lo(result) = lo(x_lo * y_lo)
2770     //             hi(result) = hi(x_lo * y_lo) + lo(x_hi * y_lo) + lo(x_lo * y_hi)
2771     // MOV    $tmp,$src.lo
2772     encode_Copy( cbuf, $tmp$$reg, $src$$reg );
2773     // IMUL   $tmp,EDX
2774     emit_opcode( cbuf, 0x0F );
2775     emit_opcode( cbuf, 0xAF );
2776     emit_rm( cbuf, 0x3, $tmp$$reg, HIGH_FROM_LOW($dst$$reg) );
2777     // MOV    EDX,$src.hi
2778     encode_Copy( cbuf, HIGH_FROM_LOW($dst$$reg), HIGH_FROM_LOW($src$$reg) );
2779     // IMUL   EDX,EAX
2780     emit_opcode( cbuf, 0x0F );
2781     emit_opcode( cbuf, 0xAF );
2782     emit_rm( cbuf, 0x3, HIGH_FROM_LOW($dst$$reg), $dst$$reg );
2783     // ADD    $tmp,EDX
2784     emit_opcode( cbuf, 0x03 );
2785     emit_rm( cbuf, 0x3, $tmp$$reg, HIGH_FROM_LOW($dst$$reg) );
2786     // MUL   EDX:EAX,$src.lo
2787     emit_opcode( cbuf, 0xF7 );
2788     emit_rm( cbuf, 0x3, 0x4, $src$$reg );
2789     // ADD    EDX,ESI
2790     emit_opcode( cbuf, 0x03 );
2791     emit_rm( cbuf, 0x3, HIGH_FROM_LOW($dst$$reg), $tmp$$reg );
2792   %}
2793 
2794   enc_class long_multiply_con( eADXRegL dst, immL_127 src, rRegI tmp ) %{
2795     // Basic idea: lo(result) = lo(src * y_lo)
2796     //             hi(result) = hi(src * y_lo) + lo(src * y_hi)
2797     // IMUL   $tmp,EDX,$src
2798     emit_opcode( cbuf, 0x6B );
2799     emit_rm( cbuf, 0x3, $tmp$$reg, HIGH_FROM_LOW($dst$$reg) );
2800     emit_d8( cbuf, (int)$src$$constant );
2801     // MOV    EDX,$src
2802     emit_opcode(cbuf, 0xB8 + EDX_enc);
2803     emit_d32( cbuf, (int)$src$$constant );
2804     // MUL   EDX:EAX,EDX
2805     emit_opcode( cbuf, 0xF7 );
2806     emit_rm( cbuf, 0x3, 0x4, EDX_enc );
2807     // ADD    EDX,ESI
2808     emit_opcode( cbuf, 0x03 );
2809     emit_rm( cbuf, 0x3, EDX_enc, $tmp$$reg );
2810   %}
2811 
2812   enc_class long_div( eRegL src1, eRegL src2 ) %{
2813     // PUSH src1.hi
2814     emit_opcode(cbuf, HIGH_FROM_LOW(0x50+$src1$$reg) );
2815     // PUSH src1.lo
2816     emit_opcode(cbuf,               0x50+$src1$$reg  );
2817     // PUSH src2.hi
2818     emit_opcode(cbuf, HIGH_FROM_LOW(0x50+$src2$$reg) );
2819     // PUSH src2.lo
2820     emit_opcode(cbuf,               0x50+$src2$$reg  );
2821     // CALL directly to the runtime
2822     cbuf.set_insts_mark();
2823     emit_opcode(cbuf,0xE8);       // Call into runtime
2824     emit_d32_reloc(cbuf, (CAST_FROM_FN_PTR(address, SharedRuntime::ldiv) - cbuf.insts_end()) - 4, runtime_call_Relocation::spec(), RELOC_IMM32 );
2825     // Restore stack
2826     emit_opcode(cbuf, 0x83); // add  SP, #framesize
2827     emit_rm(cbuf, 0x3, 0x00, ESP_enc);
2828     emit_d8(cbuf, 4*4);
2829   %}
2830 
2831   enc_class long_mod( eRegL src1, eRegL src2 ) %{
2832     // PUSH src1.hi
2833     emit_opcode(cbuf, HIGH_FROM_LOW(0x50+$src1$$reg) );
2834     // PUSH src1.lo
2835     emit_opcode(cbuf,               0x50+$src1$$reg  );
2836     // PUSH src2.hi
2837     emit_opcode(cbuf, HIGH_FROM_LOW(0x50+$src2$$reg) );
2838     // PUSH src2.lo
2839     emit_opcode(cbuf,               0x50+$src2$$reg  );
2840     // CALL directly to the runtime
2841     cbuf.set_insts_mark();
2842     emit_opcode(cbuf,0xE8);       // Call into runtime
2843     emit_d32_reloc(cbuf, (CAST_FROM_FN_PTR(address, SharedRuntime::lrem ) - cbuf.insts_end()) - 4, runtime_call_Relocation::spec(), RELOC_IMM32 );
2844     // Restore stack
2845     emit_opcode(cbuf, 0x83); // add  SP, #framesize
2846     emit_rm(cbuf, 0x3, 0x00, ESP_enc);
2847     emit_d8(cbuf, 4*4);
2848   %}
2849 
2850   enc_class long_cmp_flags0( eRegL src, rRegI tmp ) %{
2851     // MOV   $tmp,$src.lo
2852     emit_opcode(cbuf, 0x8B);
2853     emit_rm(cbuf, 0x3, $tmp$$reg, $src$$reg);
2854     // OR    $tmp,$src.hi
2855     emit_opcode(cbuf, 0x0B);
2856     emit_rm(cbuf, 0x3, $tmp$$reg, HIGH_FROM_LOW($src$$reg));
2857   %}
2858 
2859   enc_class long_cmp_flags1( eRegL src1, eRegL src2 ) %{
2860     // CMP    $src1.lo,$src2.lo
2861     emit_opcode( cbuf, 0x3B );
2862     emit_rm(cbuf, 0x3, $src1$$reg, $src2$$reg );
2863     // JNE,s  skip
2864     emit_cc(cbuf, 0x70, 0x5);
2865     emit_d8(cbuf,2);
2866     // CMP    $src1.hi,$src2.hi
2867     emit_opcode( cbuf, 0x3B );
2868     emit_rm(cbuf, 0x3, HIGH_FROM_LOW($src1$$reg), HIGH_FROM_LOW($src2$$reg) );
2869   %}
2870 
2871   enc_class long_cmp_flags2( eRegL src1, eRegL src2, rRegI tmp ) %{
2872     // CMP    $src1.lo,$src2.lo\t! Long compare; set flags for low bits
2873     emit_opcode( cbuf, 0x3B );
2874     emit_rm(cbuf, 0x3, $src1$$reg, $src2$$reg );
2875     // MOV    $tmp,$src1.hi
2876     emit_opcode( cbuf, 0x8B );
2877     emit_rm(cbuf, 0x3, $tmp$$reg, HIGH_FROM_LOW($src1$$reg) );
2878     // SBB   $tmp,$src2.hi\t! Compute flags for long compare
2879     emit_opcode( cbuf, 0x1B );
2880     emit_rm(cbuf, 0x3, $tmp$$reg, HIGH_FROM_LOW($src2$$reg) );
2881   %}
2882 
2883   enc_class long_cmp_flags3( eRegL src, rRegI tmp ) %{
2884     // XOR    $tmp,$tmp
2885     emit_opcode(cbuf,0x33);  // XOR
2886     emit_rm(cbuf,0x3, $tmp$$reg, $tmp$$reg);
2887     // CMP    $tmp,$src.lo
2888     emit_opcode( cbuf, 0x3B );
2889     emit_rm(cbuf, 0x3, $tmp$$reg, $src$$reg );
2890     // SBB    $tmp,$src.hi
2891     emit_opcode( cbuf, 0x1B );
2892     emit_rm(cbuf, 0x3, $tmp$$reg, HIGH_FROM_LOW($src$$reg) );
2893   %}
2894 
2895  // Sniff, sniff... smells like Gnu Superoptimizer
2896   enc_class neg_long( eRegL dst ) %{
2897     emit_opcode(cbuf,0xF7);    // NEG hi
2898     emit_rm    (cbuf,0x3, 0x3, HIGH_FROM_LOW($dst$$reg));
2899     emit_opcode(cbuf,0xF7);    // NEG lo
2900     emit_rm    (cbuf,0x3, 0x3,               $dst$$reg );
2901     emit_opcode(cbuf,0x83);    // SBB hi,0
2902     emit_rm    (cbuf,0x3, 0x3, HIGH_FROM_LOW($dst$$reg));
2903     emit_d8    (cbuf,0 );
2904   %}
2905 
2906   enc_class enc_pop_rdx() %{
2907     emit_opcode(cbuf,0x5A);
2908   %}
2909 
2910   enc_class enc_rethrow() %{
2911     cbuf.set_insts_mark();
2912     emit_opcode(cbuf, 0xE9);        // jmp    entry
2913     emit_d32_reloc(cbuf, (int)OptoRuntime::rethrow_stub() - ((int)cbuf.insts_end())-4,
2914                    runtime_call_Relocation::spec(), RELOC_IMM32 );
2915   %}
2916 
2917 
2918   // Convert a double to an int.  Java semantics require we do complex
2919   // manglelations in the corner cases.  So we set the rounding mode to
2920   // 'zero', store the darned double down as an int, and reset the
2921   // rounding mode to 'nearest'.  The hardware throws an exception which
2922   // patches up the correct value directly to the stack.
2923   enc_class DPR2I_encoding( regDPR src ) %{
2924     // Flip to round-to-zero mode.  We attempted to allow invalid-op
2925     // exceptions here, so that a NAN or other corner-case value will
2926     // thrown an exception (but normal values get converted at full speed).
2927     // However, I2C adapters and other float-stack manglers leave pending
2928     // invalid-op exceptions hanging.  We would have to clear them before
2929     // enabling them and that is more expensive than just testing for the
2930     // invalid value Intel stores down in the corner cases.
2931     emit_opcode(cbuf,0xD9);            // FLDCW  trunc
2932     emit_opcode(cbuf,0x2D);
2933     emit_d32(cbuf,(int)StubRoutines::addr_fpu_cntrl_wrd_trunc());
2934     // Allocate a word
2935     emit_opcode(cbuf,0x83);            // SUB ESP,4
2936     emit_opcode(cbuf,0xEC);
2937     emit_d8(cbuf,0x04);
2938     // Encoding assumes a double has been pushed into FPR0.
2939     // Store down the double as an int, popping the FPU stack
2940     emit_opcode(cbuf,0xDB);            // FISTP [ESP]
2941     emit_opcode(cbuf,0x1C);
2942     emit_d8(cbuf,0x24);
2943     // Restore the rounding mode; mask the exception
2944     emit_opcode(cbuf,0xD9);            // FLDCW   std/24-bit mode
2945     emit_opcode(cbuf,0x2D);
2946     emit_d32( cbuf, Compile::current()->in_24_bit_fp_mode()
2947         ? (int)StubRoutines::addr_fpu_cntrl_wrd_24()
2948         : (int)StubRoutines::addr_fpu_cntrl_wrd_std());
2949 
2950     // Load the converted int; adjust CPU stack
2951     emit_opcode(cbuf,0x58);       // POP EAX
2952     emit_opcode(cbuf,0x3D);       // CMP EAX,imm
2953     emit_d32   (cbuf,0x80000000); //         0x80000000
2954     emit_opcode(cbuf,0x75);       // JNE around_slow_call
2955     emit_d8    (cbuf,0x07);       // Size of slow_call
2956     // Push src onto stack slow-path
2957     emit_opcode(cbuf,0xD9 );      // FLD     ST(i)
2958     emit_d8    (cbuf,0xC0-1+$src$$reg );
2959     // CALL directly to the runtime
2960     cbuf.set_insts_mark();
2961     emit_opcode(cbuf,0xE8);       // Call into runtime
2962     emit_d32_reloc(cbuf, (StubRoutines::d2i_wrapper() - cbuf.insts_end()) - 4, runtime_call_Relocation::spec(), RELOC_IMM32 );
2963     // Carry on here...
2964   %}
2965 
2966   enc_class DPR2L_encoding( regDPR src ) %{
2967     emit_opcode(cbuf,0xD9);            // FLDCW  trunc
2968     emit_opcode(cbuf,0x2D);
2969     emit_d32(cbuf,(int)StubRoutines::addr_fpu_cntrl_wrd_trunc());
2970     // Allocate a word
2971     emit_opcode(cbuf,0x83);            // SUB ESP,8
2972     emit_opcode(cbuf,0xEC);
2973     emit_d8(cbuf,0x08);
2974     // Encoding assumes a double has been pushed into FPR0.
2975     // Store down the double as a long, popping the FPU stack
2976     emit_opcode(cbuf,0xDF);            // FISTP [ESP]
2977     emit_opcode(cbuf,0x3C);
2978     emit_d8(cbuf,0x24);
2979     // Restore the rounding mode; mask the exception
2980     emit_opcode(cbuf,0xD9);            // FLDCW   std/24-bit mode
2981     emit_opcode(cbuf,0x2D);
2982     emit_d32( cbuf, Compile::current()->in_24_bit_fp_mode()
2983         ? (int)StubRoutines::addr_fpu_cntrl_wrd_24()
2984         : (int)StubRoutines::addr_fpu_cntrl_wrd_std());
2985 
2986     // Load the converted int; adjust CPU stack
2987     emit_opcode(cbuf,0x58);       // POP EAX
2988     emit_opcode(cbuf,0x5A);       // POP EDX
2989     emit_opcode(cbuf,0x81);       // CMP EDX,imm
2990     emit_d8    (cbuf,0xFA);       // rdx
2991     emit_d32   (cbuf,0x80000000); //         0x80000000
2992     emit_opcode(cbuf,0x75);       // JNE around_slow_call
2993     emit_d8    (cbuf,0x07+4);     // Size of slow_call
2994     emit_opcode(cbuf,0x85);       // TEST EAX,EAX
2995     emit_opcode(cbuf,0xC0);       // 2/rax,/rax,
2996     emit_opcode(cbuf,0x75);       // JNE around_slow_call
2997     emit_d8    (cbuf,0x07);       // Size of slow_call
2998     // Push src onto stack slow-path
2999     emit_opcode(cbuf,0xD9 );      // FLD     ST(i)
3000     emit_d8    (cbuf,0xC0-1+$src$$reg );
3001     // CALL directly to the runtime
3002     cbuf.set_insts_mark();
3003     emit_opcode(cbuf,0xE8);       // Call into runtime
3004     emit_d32_reloc(cbuf, (StubRoutines::d2l_wrapper() - cbuf.insts_end()) - 4, runtime_call_Relocation::spec(), RELOC_IMM32 );
3005     // Carry on here...
3006   %}
3007 
3008   enc_class FMul_ST_reg( eRegFPR src1 ) %{
3009     // Operand was loaded from memory into fp ST (stack top)
3010     // FMUL   ST,$src  /* D8 C8+i */
3011     emit_opcode(cbuf, 0xD8);
3012     emit_opcode(cbuf, 0xC8 + $src1$$reg);
3013   %}
3014 
3015   enc_class FAdd_ST_reg( eRegFPR src2 ) %{
3016     // FADDP  ST,src2  /* D8 C0+i */
3017     emit_opcode(cbuf, 0xD8);
3018     emit_opcode(cbuf, 0xC0 + $src2$$reg);
3019     //could use FADDP  src2,fpST  /* DE C0+i */
3020   %}
3021 
3022   enc_class FAddP_reg_ST( eRegFPR src2 ) %{
3023     // FADDP  src2,ST  /* DE C0+i */
3024     emit_opcode(cbuf, 0xDE);
3025     emit_opcode(cbuf, 0xC0 + $src2$$reg);
3026   %}
3027 
3028   enc_class subFPR_divFPR_encode( eRegFPR src1, eRegFPR src2) %{
3029     // Operand has been loaded into fp ST (stack top)
3030       // FSUB   ST,$src1
3031       emit_opcode(cbuf, 0xD8);
3032       emit_opcode(cbuf, 0xE0 + $src1$$reg);
3033 
3034       // FDIV
3035       emit_opcode(cbuf, 0xD8);
3036       emit_opcode(cbuf, 0xF0 + $src2$$reg);
3037   %}
3038 
3039   enc_class MulFAddF (eRegFPR src1, eRegFPR src2) %{
3040     // Operand was loaded from memory into fp ST (stack top)
3041     // FADD   ST,$src  /* D8 C0+i */
3042     emit_opcode(cbuf, 0xD8);
3043     emit_opcode(cbuf, 0xC0 + $src1$$reg);
3044 
3045     // FMUL  ST,src2  /* D8 C*+i */
3046     emit_opcode(cbuf, 0xD8);
3047     emit_opcode(cbuf, 0xC8 + $src2$$reg);
3048   %}
3049 
3050 
3051   enc_class MulFAddFreverse (eRegFPR src1, eRegFPR src2) %{
3052     // Operand was loaded from memory into fp ST (stack top)
3053     // FADD   ST,$src  /* D8 C0+i */
3054     emit_opcode(cbuf, 0xD8);
3055     emit_opcode(cbuf, 0xC0 + $src1$$reg);
3056 
3057     // FMULP  src2,ST  /* DE C8+i */
3058     emit_opcode(cbuf, 0xDE);
3059     emit_opcode(cbuf, 0xC8 + $src2$$reg);
3060   %}
3061 
3062   // Atomically load the volatile long
3063   enc_class enc_loadL_volatile( memory mem, stackSlotL dst ) %{
3064     emit_opcode(cbuf,0xDF);
3065     int rm_byte_opcode = 0x05;
3066     int base     = $mem$$base;
3067     int index    = $mem$$index;
3068     int scale    = $mem$$scale;
3069     int displace = $mem$$disp;
3070     relocInfo::relocType disp_reloc = $mem->disp_reloc(); // disp-as-oop when working with static globals
3071     encode_RegMem(cbuf, rm_byte_opcode, base, index, scale, displace, disp_reloc);
3072     store_to_stackslot( cbuf, 0x0DF, 0x07, $dst$$disp );
3073   %}
3074 
3075   // Volatile Store Long.  Must be atomic, so move it into
3076   // the FP TOS and then do a 64-bit FIST.  Has to probe the
3077   // target address before the store (for null-ptr checks)
3078   // so the memory operand is used twice in the encoding.
3079   enc_class enc_storeL_volatile( memory mem, stackSlotL src ) %{
3080     store_to_stackslot( cbuf, 0x0DF, 0x05, $src$$disp );
3081     cbuf.set_insts_mark();            // Mark start of FIST in case $mem has an oop
3082     emit_opcode(cbuf,0xDF);
3083     int rm_byte_opcode = 0x07;
3084     int base     = $mem$$base;
3085     int index    = $mem$$index;
3086     int scale    = $mem$$scale;
3087     int displace = $mem$$disp;
3088     relocInfo::relocType disp_reloc = $mem->disp_reloc(); // disp-as-oop when working with static globals
3089     encode_RegMem(cbuf, rm_byte_opcode, base, index, scale, displace, disp_reloc);
3090   %}
3091 
3092   // Safepoint Poll.  This polls the safepoint page, and causes an
3093   // exception if it is not readable. Unfortunately, it kills the condition code
3094   // in the process
3095   // We current use TESTL [spp],EDI
3096   // A better choice might be TESTB [spp + pagesize() - CacheLineSize()],0
3097 
3098   enc_class Safepoint_Poll() %{
3099     cbuf.relocate(cbuf.insts_mark(), relocInfo::poll_type, 0);
3100     emit_opcode(cbuf,0x85);
3101     emit_rm (cbuf, 0x0, 0x7, 0x5);
3102     emit_d32(cbuf, (intptr_t)os::get_polling_page());
3103   %}
3104 %}
3105 
3106 
3107 //----------FRAME--------------------------------------------------------------
3108 // Definition of frame structure and management information.
3109 //
3110 //  S T A C K   L A Y O U T    Allocators stack-slot number
3111 //                             |   (to get allocators register number
3112 //  G  Owned by    |        |  v    add OptoReg::stack0())
3113 //  r   CALLER     |        |
3114 //  o     |        +--------+      pad to even-align allocators stack-slot
3115 //  w     V        |  pad0  |        numbers; owned by CALLER
3116 //  t   -----------+--------+----> Matcher::_in_arg_limit, unaligned
3117 //  h     ^        |   in   |  5
3118 //        |        |  args  |  4   Holes in incoming args owned by SELF
3119 //  |     |        |        |  3
3120 //  |     |        +--------+
3121 //  V     |        | old out|      Empty on Intel, window on Sparc
3122 //        |    old |preserve|      Must be even aligned.
3123 //        |     SP-+--------+----> Matcher::_old_SP, even aligned
3124 //        |        |   in   |  3   area for Intel ret address
3125 //     Owned by    |preserve|      Empty on Sparc.
3126 //       SELF      +--------+
3127 //        |        |  pad2  |  2   pad to align old SP
3128 //        |        +--------+  1
3129 //        |        | locks  |  0
3130 //        |        +--------+----> OptoReg::stack0(), even aligned
3131 //        |        |  pad1  | 11   pad to align new SP
3132 //        |        +--------+
3133 //        |        |        | 10
3134 //        |        | spills |  9   spills
3135 //        V        |        |  8   (pad0 slot for callee)
3136 //      -----------+--------+----> Matcher::_out_arg_limit, unaligned
3137 //        ^        |  out   |  7
3138 //        |        |  args  |  6   Holes in outgoing args owned by CALLEE
3139 //     Owned by    +--------+
3140 //      CALLEE     | new out|  6   Empty on Intel, window on Sparc
3141 //        |    new |preserve|      Must be even-aligned.
3142 //        |     SP-+--------+----> Matcher::_new_SP, even aligned
3143 //        |        |        |
3144 //
3145 // Note 1: Only region 8-11 is determined by the allocator.  Region 0-5 is
3146 //         known from SELF's arguments and the Java calling convention.
3147 //         Region 6-7 is determined per call site.
3148 // Note 2: If the calling convention leaves holes in the incoming argument
3149 //         area, those holes are owned by SELF.  Holes in the outgoing area
3150 //         are owned by the CALLEE.  Holes should not be nessecary in the
3151 //         incoming area, as the Java calling convention is completely under
3152 //         the control of the AD file.  Doubles can be sorted and packed to
3153 //         avoid holes.  Holes in the outgoing arguments may be nessecary for
3154 //         varargs C calling conventions.
3155 // Note 3: Region 0-3 is even aligned, with pad2 as needed.  Region 3-5 is
3156 //         even aligned with pad0 as needed.
3157 //         Region 6 is even aligned.  Region 6-7 is NOT even aligned;
3158 //         region 6-11 is even aligned; it may be padded out more so that
3159 //         the region from SP to FP meets the minimum stack alignment.
3160 
3161 frame %{
3162   // What direction does stack grow in (assumed to be same for C & Java)
3163   stack_direction(TOWARDS_LOW);
3164 
3165   // These three registers define part of the calling convention
3166   // between compiled code and the interpreter.
3167   inline_cache_reg(EAX);                // Inline Cache Register
3168   interpreter_method_oop_reg(EBX);      // Method Oop Register when calling interpreter
3169 
3170   // Optional: name the operand used by cisc-spilling to access [stack_pointer + offset]
3171   cisc_spilling_operand_name(indOffset32);
3172 
3173   // Number of stack slots consumed by locking an object
3174   sync_stack_slots(1);
3175 
3176   // Compiled code's Frame Pointer
3177   frame_pointer(ESP);
3178   // Interpreter stores its frame pointer in a register which is
3179   // stored to the stack by I2CAdaptors.
3180   // I2CAdaptors convert from interpreted java to compiled java.
3181   interpreter_frame_pointer(EBP);
3182 
3183   // Stack alignment requirement
3184   // Alignment size in bytes (128-bit -> 16 bytes)
3185   stack_alignment(StackAlignmentInBytes);
3186 
3187   // Number of stack slots between incoming argument block and the start of
3188   // a new frame.  The PROLOG must add this many slots to the stack.  The
3189   // EPILOG must remove this many slots.  Intel needs one slot for
3190   // return address and one for rbp, (must save rbp)
3191   in_preserve_stack_slots(2+VerifyStackAtCalls);
3192 
3193   // Number of outgoing stack slots killed above the out_preserve_stack_slots
3194   // for calls to C.  Supports the var-args backing area for register parms.
3195   varargs_C_out_slots_killed(0);
3196 
3197   // The after-PROLOG location of the return address.  Location of
3198   // return address specifies a type (REG or STACK) and a number
3199   // representing the register number (i.e. - use a register name) or
3200   // stack slot.
3201   // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
3202   // Otherwise, it is above the locks and verification slot and alignment word
3203   return_addr(STACK - 1 +
3204               round_to((Compile::current()->in_preserve_stack_slots() +
3205                         Compile::current()->fixed_slots()),
3206                        stack_alignment_in_slots()));
3207 
3208   // Body of function which returns an integer array locating
3209   // arguments either in registers or in stack slots.  Passed an array
3210   // of ideal registers called "sig" and a "length" count.  Stack-slot
3211   // offsets are based on outgoing arguments, i.e. a CALLER setting up
3212   // arguments for a CALLEE.  Incoming stack arguments are
3213   // automatically biased by the preserve_stack_slots field above.
3214   calling_convention %{
3215     // No difference between ingoing/outgoing just pass false
3216     SharedRuntime::java_calling_convention(sig_bt, regs, length, false);
3217   %}
3218 
3219 
3220   // Body of function which returns an integer array locating
3221   // arguments either in registers or in stack slots.  Passed an array
3222   // of ideal registers called "sig" and a "length" count.  Stack-slot
3223   // offsets are based on outgoing arguments, i.e. a CALLER setting up
3224   // arguments for a CALLEE.  Incoming stack arguments are
3225   // automatically biased by the preserve_stack_slots field above.
3226   c_calling_convention %{
3227     // This is obviously always outgoing
3228     (void) SharedRuntime::c_calling_convention(sig_bt, regs, /*regs2=*/NULL, length);
3229   %}
3230 
3231   // Location of C & interpreter return values
3232   c_return_value %{
3233     assert( ideal_reg >= Op_RegI && ideal_reg <= Op_RegL, "only return normal values" );
3234     static int lo[Op_RegL+1] = { 0, 0, OptoReg::Bad, EAX_num,      EAX_num,      FPR1L_num,    FPR1L_num, EAX_num };
3235     static int hi[Op_RegL+1] = { 0, 0, OptoReg::Bad, OptoReg::Bad, OptoReg::Bad, OptoReg::Bad, FPR1H_num, EDX_num };
3236 
3237     // in SSE2+ mode we want to keep the FPU stack clean so pretend
3238     // that C functions return float and double results in XMM0.
3239     if( ideal_reg == Op_RegD && UseSSE>=2 )
3240       return OptoRegPair(XMM0b_num,XMM0_num);
3241     if( ideal_reg == Op_RegF && UseSSE>=2 )
3242       return OptoRegPair(OptoReg::Bad,XMM0_num);
3243 
3244     return OptoRegPair(hi[ideal_reg],lo[ideal_reg]);
3245   %}
3246 
3247   // Location of return values
3248   return_value %{
3249     assert( ideal_reg >= Op_RegI && ideal_reg <= Op_RegL, "only return normal values" );
3250     static int lo[Op_RegL+1] = { 0, 0, OptoReg::Bad, EAX_num,      EAX_num,      FPR1L_num,    FPR1L_num, EAX_num };
3251     static int hi[Op_RegL+1] = { 0, 0, OptoReg::Bad, OptoReg::Bad, OptoReg::Bad, OptoReg::Bad, FPR1H_num, EDX_num };
3252     if( ideal_reg == Op_RegD && UseSSE>=2 )
3253       return OptoRegPair(XMM0b_num,XMM0_num);
3254     if( ideal_reg == Op_RegF && UseSSE>=1 )
3255       return OptoRegPair(OptoReg::Bad,XMM0_num);
3256     return OptoRegPair(hi[ideal_reg],lo[ideal_reg]);
3257   %}
3258 
3259 %}
3260 
3261 //----------ATTRIBUTES---------------------------------------------------------
3262 //----------Operand Attributes-------------------------------------------------
3263 op_attrib op_cost(0);        // Required cost attribute
3264 
3265 //----------Instruction Attributes---------------------------------------------
3266 ins_attrib ins_cost(100);       // Required cost attribute
3267 ins_attrib ins_size(8);         // Required size attribute (in bits)
3268 ins_attrib ins_short_branch(0); // Required flag: is this instruction a
3269                                 // non-matching short branch variant of some
3270                                                             // long branch?
3271 ins_attrib ins_alignment(1);    // Required alignment attribute (must be a power of 2)
3272                                 // specifies the alignment that some part of the instruction (not
3273                                 // necessarily the start) requires.  If > 1, a compute_padding()
3274                                 // function must be provided for the instruction
3275 
3276 //----------OPERANDS-----------------------------------------------------------
3277 // Operand definitions must precede instruction definitions for correct parsing
3278 // in the ADLC because operands constitute user defined types which are used in
3279 // instruction definitions.
3280 
3281 //----------Simple Operands----------------------------------------------------
3282 // Immediate Operands
3283 // Integer Immediate
3284 operand immI() %{
3285   match(ConI);
3286 
3287   op_cost(10);
3288   format %{ %}
3289   interface(CONST_INTER);
3290 %}
3291 
3292 // Constant for test vs zero
3293 operand immI0() %{
3294   predicate(n->get_int() == 0);
3295   match(ConI);
3296 
3297   op_cost(0);
3298   format %{ %}
3299   interface(CONST_INTER);
3300 %}
3301 
3302 // Constant for increment
3303 operand immI1() %{
3304   predicate(n->get_int() == 1);
3305   match(ConI);
3306 
3307   op_cost(0);
3308   format %{ %}
3309   interface(CONST_INTER);
3310 %}
3311 
3312 // Constant for decrement
3313 operand immI_M1() %{
3314   predicate(n->get_int() == -1);
3315   match(ConI);
3316 
3317   op_cost(0);
3318   format %{ %}
3319   interface(CONST_INTER);
3320 %}
3321 
3322 // Valid scale values for addressing modes
3323 operand immI2() %{
3324   predicate(0 <= n->get_int() && (n->get_int() <= 3));
3325   match(ConI);
3326 
3327   format %{ %}
3328   interface(CONST_INTER);
3329 %}
3330 
3331 operand immI8() %{
3332   predicate((-128 <= n->get_int()) && (n->get_int() <= 127));
3333   match(ConI);
3334 
3335   op_cost(5);
3336   format %{ %}
3337   interface(CONST_INTER);
3338 %}
3339 
3340 operand immI16() %{
3341   predicate((-32768 <= n->get_int()) && (n->get_int() <= 32767));
3342   match(ConI);
3343 
3344   op_cost(10);
3345   format %{ %}
3346   interface(CONST_INTER);
3347 %}
3348 
3349 // Int Immediate non-negative
3350 operand immU31()
3351 %{
3352   predicate(n->get_int() >= 0);
3353   match(ConI);
3354 
3355   op_cost(0);
3356   format %{ %}
3357   interface(CONST_INTER);
3358 %}
3359 
3360 // Constant for long shifts
3361 operand immI_32() %{
3362   predicate( n->get_int() == 32 );
3363   match(ConI);
3364 
3365   op_cost(0);
3366   format %{ %}
3367   interface(CONST_INTER);
3368 %}
3369 
3370 operand immI_1_31() %{
3371   predicate( n->get_int() >= 1 && n->get_int() <= 31 );
3372   match(ConI);
3373 
3374   op_cost(0);
3375   format %{ %}
3376   interface(CONST_INTER);
3377 %}
3378 
3379 operand immI_32_63() %{
3380   predicate( n->get_int() >= 32 && n->get_int() <= 63 );
3381   match(ConI);
3382   op_cost(0);
3383 
3384   format %{ %}
3385   interface(CONST_INTER);
3386 %}
3387 
3388 operand immI_1() %{
3389   predicate( n->get_int() == 1 );
3390   match(ConI);
3391 
3392   op_cost(0);
3393   format %{ %}
3394   interface(CONST_INTER);
3395 %}
3396 
3397 operand immI_2() %{
3398   predicate( n->get_int() == 2 );
3399   match(ConI);
3400 
3401   op_cost(0);
3402   format %{ %}
3403   interface(CONST_INTER);
3404 %}
3405 
3406 operand immI_3() %{
3407   predicate( n->get_int() == 3 );
3408   match(ConI);
3409 
3410   op_cost(0);
3411   format %{ %}
3412   interface(CONST_INTER);
3413 %}
3414 
3415 // Pointer Immediate
3416 operand immP() %{
3417   match(ConP);
3418 
3419   op_cost(10);
3420   format %{ %}
3421   interface(CONST_INTER);
3422 %}
3423 
3424 // NULL Pointer Immediate
3425 operand immP0() %{
3426   predicate( n->get_ptr() == 0 );
3427   match(ConP);
3428   op_cost(0);
3429 
3430   format %{ %}
3431   interface(CONST_INTER);
3432 %}
3433 
3434 // Long Immediate
3435 operand immL() %{
3436   match(ConL);
3437 
3438   op_cost(20);
3439   format %{ %}
3440   interface(CONST_INTER);
3441 %}
3442 
3443 // Long Immediate zero
3444 operand immL0() %{
3445   predicate( n->get_long() == 0L );
3446   match(ConL);
3447   op_cost(0);
3448 
3449   format %{ %}
3450   interface(CONST_INTER);
3451 %}
3452 
3453 // Long Immediate zero
3454 operand immL_M1() %{
3455   predicate( n->get_long() == -1L );
3456   match(ConL);
3457   op_cost(0);
3458 
3459   format %{ %}
3460   interface(CONST_INTER);
3461 %}
3462 
3463 // Long immediate from 0 to 127.
3464 // Used for a shorter form of long mul by 10.
3465 operand immL_127() %{
3466   predicate((0 <= n->get_long()) && (n->get_long() <= 127));
3467   match(ConL);
3468   op_cost(0);
3469 
3470   format %{ %}
3471   interface(CONST_INTER);
3472 %}
3473 
3474 // Long Immediate: low 32-bit mask
3475 operand immL_32bits() %{
3476   predicate(n->get_long() == 0xFFFFFFFFL);
3477   match(ConL);
3478   op_cost(0);
3479 
3480   format %{ %}
3481   interface(CONST_INTER);
3482 %}
3483 
3484 // Long Immediate: low 32-bit mask
3485 operand immL32() %{
3486   predicate(n->get_long() == (int)(n->get_long()));
3487   match(ConL);
3488   op_cost(20);
3489 
3490   format %{ %}
3491   interface(CONST_INTER);
3492 %}
3493 
3494 //Double Immediate zero
3495 operand immDPR0() %{
3496   // Do additional (and counter-intuitive) test against NaN to work around VC++
3497   // bug that generates code such that NaNs compare equal to 0.0
3498   predicate( UseSSE<=1 && n->getd() == 0.0 && !g_isnan(n->getd()) );
3499   match(ConD);
3500 
3501   op_cost(5);
3502   format %{ %}
3503   interface(CONST_INTER);
3504 %}
3505 
3506 // Double Immediate one
3507 operand immDPR1() %{
3508   predicate( UseSSE<=1 && n->getd() == 1.0 );
3509   match(ConD);
3510 
3511   op_cost(5);
3512   format %{ %}
3513   interface(CONST_INTER);
3514 %}
3515 
3516 // Double Immediate
3517 operand immDPR() %{
3518   predicate(UseSSE<=1);
3519   match(ConD);
3520 
3521   op_cost(5);
3522   format %{ %}
3523   interface(CONST_INTER);
3524 %}
3525 
3526 operand immD() %{
3527   predicate(UseSSE>=2);
3528   match(ConD);
3529 
3530   op_cost(5);
3531   format %{ %}
3532   interface(CONST_INTER);
3533 %}
3534 
3535 // Double Immediate zero
3536 operand immD0() %{
3537   // Do additional (and counter-intuitive) test against NaN to work around VC++
3538   // bug that generates code such that NaNs compare equal to 0.0 AND do not
3539   // compare equal to -0.0.
3540   predicate( UseSSE>=2 && jlong_cast(n->getd()) == 0 );
3541   match(ConD);
3542 
3543   format %{ %}
3544   interface(CONST_INTER);
3545 %}
3546 
3547 // Float Immediate zero
3548 operand immFPR0() %{
3549   predicate(UseSSE == 0 && n->getf() == 0.0F);
3550   match(ConF);
3551 
3552   op_cost(5);
3553   format %{ %}
3554   interface(CONST_INTER);
3555 %}
3556 
3557 // Float Immediate one
3558 operand immFPR1() %{
3559   predicate(UseSSE == 0 && n->getf() == 1.0F);
3560   match(ConF);
3561 
3562   op_cost(5);
3563   format %{ %}
3564   interface(CONST_INTER);
3565 %}
3566 
3567 // Float Immediate
3568 operand immFPR() %{
3569   predicate( UseSSE == 0 );
3570   match(ConF);
3571 
3572   op_cost(5);
3573   format %{ %}
3574   interface(CONST_INTER);
3575 %}
3576 
3577 // Float Immediate
3578 operand immF() %{
3579   predicate(UseSSE >= 1);
3580   match(ConF);
3581 
3582   op_cost(5);
3583   format %{ %}
3584   interface(CONST_INTER);
3585 %}
3586 
3587 // Float Immediate zero.  Zero and not -0.0
3588 operand immF0() %{
3589   predicate( UseSSE >= 1 && jint_cast(n->getf()) == 0 );
3590   match(ConF);
3591 
3592   op_cost(5);
3593   format %{ %}
3594   interface(CONST_INTER);
3595 %}
3596 
3597 // Immediates for special shifts (sign extend)
3598 
3599 // Constants for increment
3600 operand immI_16() %{
3601   predicate( n->get_int() == 16 );
3602   match(ConI);
3603 
3604   format %{ %}
3605   interface(CONST_INTER);
3606 %}
3607 
3608 operand immI_24() %{
3609   predicate( n->get_int() == 24 );
3610   match(ConI);
3611 
3612   format %{ %}
3613   interface(CONST_INTER);
3614 %}
3615 
3616 // Constant for byte-wide masking
3617 operand immI_255() %{
3618   predicate( n->get_int() == 255 );
3619   match(ConI);
3620 
3621   format %{ %}
3622   interface(CONST_INTER);
3623 %}
3624 
3625 // Constant for short-wide masking
3626 operand immI_65535() %{
3627   predicate(n->get_int() == 65535);
3628   match(ConI);
3629 
3630   format %{ %}
3631   interface(CONST_INTER);
3632 %}
3633 
3634 // Register Operands
3635 // Integer Register
3636 operand rRegI() %{
3637   constraint(ALLOC_IN_RC(int_reg));
3638   match(RegI);
3639   match(xRegI);
3640   match(eAXRegI);
3641   match(eBXRegI);
3642   match(eCXRegI);
3643   match(eDXRegI);
3644   match(eDIRegI);
3645   match(eSIRegI);
3646 
3647   format %{ %}
3648   interface(REG_INTER);
3649 %}
3650 
3651 // Subset of Integer Register
3652 operand xRegI(rRegI reg) %{
3653   constraint(ALLOC_IN_RC(int_x_reg));
3654   match(reg);
3655   match(eAXRegI);
3656   match(eBXRegI);
3657   match(eCXRegI);
3658   match(eDXRegI);
3659 
3660   format %{ %}
3661   interface(REG_INTER);
3662 %}
3663 
3664 // Special Registers
3665 operand eAXRegI(xRegI reg) %{
3666   constraint(ALLOC_IN_RC(eax_reg));
3667   match(reg);
3668   match(rRegI);
3669 
3670   format %{ "EAX" %}
3671   interface(REG_INTER);
3672 %}
3673 
3674 // Special Registers
3675 operand eBXRegI(xRegI reg) %{
3676   constraint(ALLOC_IN_RC(ebx_reg));
3677   match(reg);
3678   match(rRegI);
3679 
3680   format %{ "EBX" %}
3681   interface(REG_INTER);
3682 %}
3683 
3684 operand eCXRegI(xRegI reg) %{
3685   constraint(ALLOC_IN_RC(ecx_reg));
3686   match(reg);
3687   match(rRegI);
3688 
3689   format %{ "ECX" %}
3690   interface(REG_INTER);
3691 %}
3692 
3693 operand eDXRegI(xRegI reg) %{
3694   constraint(ALLOC_IN_RC(edx_reg));
3695   match(reg);
3696   match(rRegI);
3697 
3698   format %{ "EDX" %}
3699   interface(REG_INTER);
3700 %}
3701 
3702 operand eDIRegI(xRegI reg) %{
3703   constraint(ALLOC_IN_RC(edi_reg));
3704   match(reg);
3705   match(rRegI);
3706 
3707   format %{ "EDI" %}
3708   interface(REG_INTER);
3709 %}
3710 
3711 operand naxRegI() %{
3712   constraint(ALLOC_IN_RC(nax_reg));
3713   match(RegI);
3714   match(eCXRegI);
3715   match(eDXRegI);
3716   match(eSIRegI);
3717   match(eDIRegI);
3718 
3719   format %{ %}
3720   interface(REG_INTER);
3721 %}
3722 
3723 operand nadxRegI() %{
3724   constraint(ALLOC_IN_RC(nadx_reg));
3725   match(RegI);
3726   match(eBXRegI);
3727   match(eCXRegI);
3728   match(eSIRegI);
3729   match(eDIRegI);
3730 
3731   format %{ %}
3732   interface(REG_INTER);
3733 %}
3734 
3735 operand ncxRegI() %{
3736   constraint(ALLOC_IN_RC(ncx_reg));
3737   match(RegI);
3738   match(eAXRegI);
3739   match(eDXRegI);
3740   match(eSIRegI);
3741   match(eDIRegI);
3742 
3743   format %{ %}
3744   interface(REG_INTER);
3745 %}
3746 
3747 // // This operand was used by cmpFastUnlock, but conflicted with 'object' reg
3748 // //
3749 operand eSIRegI(xRegI reg) %{
3750    constraint(ALLOC_IN_RC(esi_reg));
3751    match(reg);
3752    match(rRegI);
3753 
3754    format %{ "ESI" %}
3755    interface(REG_INTER);
3756 %}
3757 
3758 // Pointer Register
3759 operand anyRegP() %{
3760   constraint(ALLOC_IN_RC(any_reg));
3761   match(RegP);
3762   match(eAXRegP);
3763   match(eBXRegP);
3764   match(eCXRegP);
3765   match(eDIRegP);
3766   match(eRegP);
3767 
3768   format %{ %}
3769   interface(REG_INTER);
3770 %}
3771 
3772 operand eRegP() %{
3773   constraint(ALLOC_IN_RC(int_reg));
3774   match(RegP);
3775   match(eAXRegP);
3776   match(eBXRegP);
3777   match(eCXRegP);
3778   match(eDIRegP);
3779 
3780   format %{ %}
3781   interface(REG_INTER);
3782 %}
3783 
3784 // On windows95, EBP is not safe to use for implicit null tests.
3785 operand eRegP_no_EBP() %{
3786   constraint(ALLOC_IN_RC(int_reg_no_ebp));
3787   match(RegP);
3788   match(eAXRegP);
3789   match(eBXRegP);
3790   match(eCXRegP);
3791   match(eDIRegP);
3792 
3793   op_cost(100);
3794   format %{ %}
3795   interface(REG_INTER);
3796 %}
3797 
3798 operand naxRegP() %{
3799   constraint(ALLOC_IN_RC(nax_reg));
3800   match(RegP);
3801   match(eBXRegP);
3802   match(eDXRegP);
3803   match(eCXRegP);
3804   match(eSIRegP);
3805   match(eDIRegP);
3806 
3807   format %{ %}
3808   interface(REG_INTER);
3809 %}
3810 
3811 operand nabxRegP() %{
3812   constraint(ALLOC_IN_RC(nabx_reg));
3813   match(RegP);
3814   match(eCXRegP);
3815   match(eDXRegP);
3816   match(eSIRegP);
3817   match(eDIRegP);
3818 
3819   format %{ %}
3820   interface(REG_INTER);
3821 %}
3822 
3823 operand pRegP() %{
3824   constraint(ALLOC_IN_RC(p_reg));
3825   match(RegP);
3826   match(eBXRegP);
3827   match(eDXRegP);
3828   match(eSIRegP);
3829   match(eDIRegP);
3830 
3831   format %{ %}
3832   interface(REG_INTER);
3833 %}
3834 
3835 // Special Registers
3836 // Return a pointer value
3837 operand eAXRegP(eRegP reg) %{
3838   constraint(ALLOC_IN_RC(eax_reg));
3839   match(reg);
3840   format %{ "EAX" %}
3841   interface(REG_INTER);
3842 %}
3843 
3844 // Used in AtomicAdd
3845 operand eBXRegP(eRegP reg) %{
3846   constraint(ALLOC_IN_RC(ebx_reg));
3847   match(reg);
3848   format %{ "EBX" %}
3849   interface(REG_INTER);
3850 %}
3851 
3852 // Tail-call (interprocedural jump) to interpreter
3853 operand eCXRegP(eRegP reg) %{
3854   constraint(ALLOC_IN_RC(ecx_reg));
3855   match(reg);
3856   format %{ "ECX" %}
3857   interface(REG_INTER);
3858 %}
3859 
3860 operand eSIRegP(eRegP reg) %{
3861   constraint(ALLOC_IN_RC(esi_reg));
3862   match(reg);
3863   format %{ "ESI" %}
3864   interface(REG_INTER);
3865 %}
3866 
3867 // Used in rep stosw
3868 operand eDIRegP(eRegP reg) %{
3869   constraint(ALLOC_IN_RC(edi_reg));
3870   match(reg);
3871   format %{ "EDI" %}
3872   interface(REG_INTER);
3873 %}
3874 
3875 operand eRegL() %{
3876   constraint(ALLOC_IN_RC(long_reg));
3877   match(RegL);
3878   match(eADXRegL);
3879 
3880   format %{ %}
3881   interface(REG_INTER);
3882 %}
3883 
3884 operand eADXRegL( eRegL reg ) %{
3885   constraint(ALLOC_IN_RC(eadx_reg));
3886   match(reg);
3887 
3888   format %{ "EDX:EAX" %}
3889   interface(REG_INTER);
3890 %}
3891 
3892 operand eBCXRegL( eRegL reg ) %{
3893   constraint(ALLOC_IN_RC(ebcx_reg));
3894   match(reg);
3895 
3896   format %{ "EBX:ECX" %}
3897   interface(REG_INTER);
3898 %}
3899 
3900 // Special case for integer high multiply
3901 operand eADXRegL_low_only() %{
3902   constraint(ALLOC_IN_RC(eadx_reg));
3903   match(RegL);
3904 
3905   format %{ "EAX" %}
3906   interface(REG_INTER);
3907 %}
3908 
3909 // Flags register, used as output of compare instructions
3910 operand eFlagsReg() %{
3911   constraint(ALLOC_IN_RC(int_flags));
3912   match(RegFlags);
3913 
3914   format %{ "EFLAGS" %}
3915   interface(REG_INTER);
3916 %}
3917 
3918 // Flags register, used as output of FLOATING POINT compare instructions
3919 operand eFlagsRegU() %{
3920   constraint(ALLOC_IN_RC(int_flags));
3921   match(RegFlags);
3922 
3923   format %{ "EFLAGS_U" %}
3924   interface(REG_INTER);
3925 %}
3926 
3927 operand eFlagsRegUCF() %{
3928   constraint(ALLOC_IN_RC(int_flags));
3929   match(RegFlags);
3930   predicate(false);
3931 
3932   format %{ "EFLAGS_U_CF" %}
3933   interface(REG_INTER);
3934 %}
3935 
3936 // Condition Code Register used by long compare
3937 operand flagsReg_long_LTGE() %{
3938   constraint(ALLOC_IN_RC(int_flags));
3939   match(RegFlags);
3940   format %{ "FLAGS_LTGE" %}
3941   interface(REG_INTER);
3942 %}
3943 operand flagsReg_long_EQNE() %{
3944   constraint(ALLOC_IN_RC(int_flags));
3945   match(RegFlags);
3946   format %{ "FLAGS_EQNE" %}
3947   interface(REG_INTER);
3948 %}
3949 operand flagsReg_long_LEGT() %{
3950   constraint(ALLOC_IN_RC(int_flags));
3951   match(RegFlags);
3952   format %{ "FLAGS_LEGT" %}
3953   interface(REG_INTER);
3954 %}
3955 
3956 // Condition Code Register used by unsigned long compare
3957 operand flagsReg_ulong_LTGE() %{
3958   constraint(ALLOC_IN_RC(int_flags));
3959   match(RegFlags);
3960   format %{ "FLAGS_U_LTGE" %}
3961   interface(REG_INTER);
3962 %}
3963 operand flagsReg_ulong_EQNE() %{
3964   constraint(ALLOC_IN_RC(int_flags));
3965   match(RegFlags);
3966   format %{ "FLAGS_U_EQNE" %}
3967   interface(REG_INTER);
3968 %}
3969 operand flagsReg_ulong_LEGT() %{
3970   constraint(ALLOC_IN_RC(int_flags));
3971   match(RegFlags);
3972   format %{ "FLAGS_U_LEGT" %}
3973   interface(REG_INTER);
3974 %}
3975 
3976 // Float register operands
3977 operand regDPR() %{
3978   predicate( UseSSE < 2 );
3979   constraint(ALLOC_IN_RC(fp_dbl_reg));
3980   match(RegD);
3981   match(regDPR1);
3982   match(regDPR2);
3983   format %{ %}
3984   interface(REG_INTER);
3985 %}
3986 
3987 operand regDPR1(regDPR reg) %{
3988   predicate( UseSSE < 2 );
3989   constraint(ALLOC_IN_RC(fp_dbl_reg0));
3990   match(reg);
3991   format %{ "FPR1" %}
3992   interface(REG_INTER);
3993 %}
3994 
3995 operand regDPR2(regDPR reg) %{
3996   predicate( UseSSE < 2 );
3997   constraint(ALLOC_IN_RC(fp_dbl_reg1));
3998   match(reg);
3999   format %{ "FPR2" %}
4000   interface(REG_INTER);
4001 %}
4002 
4003 operand regnotDPR1(regDPR reg) %{
4004   predicate( UseSSE < 2 );
4005   constraint(ALLOC_IN_RC(fp_dbl_notreg0));
4006   match(reg);
4007   format %{ %}
4008   interface(REG_INTER);
4009 %}
4010 
4011 // Float register operands
4012 operand regFPR() %{
4013   predicate( UseSSE < 2 );
4014   constraint(ALLOC_IN_RC(fp_flt_reg));
4015   match(RegF);
4016   match(regFPR1);
4017   format %{ %}
4018   interface(REG_INTER);
4019 %}
4020 
4021 // Float register operands
4022 operand regFPR1(regFPR reg) %{
4023   predicate( UseSSE < 2 );
4024   constraint(ALLOC_IN_RC(fp_flt_reg0));
4025   match(reg);
4026   format %{ "FPR1" %}
4027   interface(REG_INTER);
4028 %}
4029 
4030 // XMM Float register operands
4031 operand regF() %{
4032   predicate( UseSSE>=1 );
4033   constraint(ALLOC_IN_RC(float_reg));
4034   match(RegF);
4035   format %{ %}
4036   interface(REG_INTER);
4037 %}
4038 
4039 // XMM Double register operands
4040 operand regD() %{
4041   predicate( UseSSE>=2 );
4042   constraint(ALLOC_IN_RC(double_reg));
4043   match(RegD);
4044   format %{ %}
4045   interface(REG_INTER);
4046 %}
4047 
4048 
4049 //----------Memory Operands----------------------------------------------------
4050 // Direct Memory Operand
4051 operand direct(immP addr) %{
4052   match(addr);
4053 
4054   format %{ "[$addr]" %}
4055   interface(MEMORY_INTER) %{
4056     base(0xFFFFFFFF);
4057     index(0x4);
4058     scale(0x0);
4059     disp($addr);
4060   %}
4061 %}
4062 
4063 // Indirect Memory Operand
4064 operand indirect(eRegP reg) %{
4065   constraint(ALLOC_IN_RC(int_reg));
4066   match(reg);
4067 
4068   format %{ "[$reg]" %}
4069   interface(MEMORY_INTER) %{
4070     base($reg);
4071     index(0x4);
4072     scale(0x0);
4073     disp(0x0);
4074   %}
4075 %}
4076 
4077 // Indirect Memory Plus Short Offset Operand
4078 operand indOffset8(eRegP reg, immI8 off) %{
4079   match(AddP reg off);
4080 
4081   format %{ "[$reg + $off]" %}
4082   interface(MEMORY_INTER) %{
4083     base($reg);
4084     index(0x4);
4085     scale(0x0);
4086     disp($off);
4087   %}
4088 %}
4089 
4090 // Indirect Memory Plus Long Offset Operand
4091 operand indOffset32(eRegP reg, immI off) %{
4092   match(AddP reg off);
4093 
4094   format %{ "[$reg + $off]" %}
4095   interface(MEMORY_INTER) %{
4096     base($reg);
4097     index(0x4);
4098     scale(0x0);
4099     disp($off);
4100   %}
4101 %}
4102 
4103 // Indirect Memory Plus Long Offset Operand
4104 operand indOffset32X(rRegI reg, immP off) %{
4105   match(AddP off reg);
4106 
4107   format %{ "[$reg + $off]" %}
4108   interface(MEMORY_INTER) %{
4109     base($reg);
4110     index(0x4);
4111     scale(0x0);
4112     disp($off);
4113   %}
4114 %}
4115 
4116 // Indirect Memory Plus Index Register Plus Offset Operand
4117 operand indIndexOffset(eRegP reg, rRegI ireg, immI off) %{
4118   match(AddP (AddP reg ireg) off);
4119 
4120   op_cost(10);
4121   format %{"[$reg + $off + $ireg]" %}
4122   interface(MEMORY_INTER) %{
4123     base($reg);
4124     index($ireg);
4125     scale(0x0);
4126     disp($off);
4127   %}
4128 %}
4129 
4130 // Indirect Memory Plus Index Register Plus Offset Operand
4131 operand indIndex(eRegP reg, rRegI ireg) %{
4132   match(AddP reg ireg);
4133 
4134   op_cost(10);
4135   format %{"[$reg + $ireg]" %}
4136   interface(MEMORY_INTER) %{
4137     base($reg);
4138     index($ireg);
4139     scale(0x0);
4140     disp(0x0);
4141   %}
4142 %}
4143 
4144 // // -------------------------------------------------------------------------
4145 // // 486 architecture doesn't support "scale * index + offset" with out a base
4146 // // -------------------------------------------------------------------------
4147 // // Scaled Memory Operands
4148 // // Indirect Memory Times Scale Plus Offset Operand
4149 // operand indScaleOffset(immP off, rRegI ireg, immI2 scale) %{
4150 //   match(AddP off (LShiftI ireg scale));
4151 //
4152 //   op_cost(10);
4153 //   format %{"[$off + $ireg << $scale]" %}
4154 //   interface(MEMORY_INTER) %{
4155 //     base(0x4);
4156 //     index($ireg);
4157 //     scale($scale);
4158 //     disp($off);
4159 //   %}
4160 // %}
4161 
4162 // Indirect Memory Times Scale Plus Index Register
4163 operand indIndexScale(eRegP reg, rRegI ireg, immI2 scale) %{
4164   match(AddP reg (LShiftI ireg scale));
4165 
4166   op_cost(10);
4167   format %{"[$reg + $ireg << $scale]" %}
4168   interface(MEMORY_INTER) %{
4169     base($reg);
4170     index($ireg);
4171     scale($scale);
4172     disp(0x0);
4173   %}
4174 %}
4175 
4176 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
4177 operand indIndexScaleOffset(eRegP reg, immI off, rRegI ireg, immI2 scale) %{
4178   match(AddP (AddP reg (LShiftI ireg scale)) off);
4179 
4180   op_cost(10);
4181   format %{"[$reg + $off + $ireg << $scale]" %}
4182   interface(MEMORY_INTER) %{
4183     base($reg);
4184     index($ireg);
4185     scale($scale);
4186     disp($off);
4187   %}
4188 %}
4189 
4190 //----------Load Long Memory Operands------------------------------------------
4191 // The load-long idiom will use it's address expression again after loading
4192 // the first word of the long.  If the load-long destination overlaps with
4193 // registers used in the addressing expression, the 2nd half will be loaded
4194 // from a clobbered address.  Fix this by requiring that load-long use
4195 // address registers that do not overlap with the load-long target.
4196 
4197 // load-long support
4198 operand load_long_RegP() %{
4199   constraint(ALLOC_IN_RC(esi_reg));
4200   match(RegP);
4201   match(eSIRegP);
4202   op_cost(100);
4203   format %{  %}
4204   interface(REG_INTER);
4205 %}
4206 
4207 // Indirect Memory Operand Long
4208 operand load_long_indirect(load_long_RegP reg) %{
4209   constraint(ALLOC_IN_RC(esi_reg));
4210   match(reg);
4211 
4212   format %{ "[$reg]" %}
4213   interface(MEMORY_INTER) %{
4214     base($reg);
4215     index(0x4);
4216     scale(0x0);
4217     disp(0x0);
4218   %}
4219 %}
4220 
4221 // Indirect Memory Plus Long Offset Operand
4222 operand load_long_indOffset32(load_long_RegP reg, immI off) %{
4223   match(AddP reg off);
4224 
4225   format %{ "[$reg + $off]" %}
4226   interface(MEMORY_INTER) %{
4227     base($reg);
4228     index(0x4);
4229     scale(0x0);
4230     disp($off);
4231   %}
4232 %}
4233 
4234 opclass load_long_memory(load_long_indirect, load_long_indOffset32);
4235 
4236 
4237 //----------Special Memory Operands--------------------------------------------
4238 // Stack Slot Operand - This operand is used for loading and storing temporary
4239 //                      values on the stack where a match requires a value to
4240 //                      flow through memory.
4241 operand stackSlotP(sRegP reg) %{
4242   constraint(ALLOC_IN_RC(stack_slots));
4243   // No match rule because this operand is only generated in matching
4244   format %{ "[$reg]" %}
4245   interface(MEMORY_INTER) %{
4246     base(0x4);   // ESP
4247     index(0x4);  // No Index
4248     scale(0x0);  // No Scale
4249     disp($reg);  // Stack Offset
4250   %}
4251 %}
4252 
4253 operand stackSlotI(sRegI reg) %{
4254   constraint(ALLOC_IN_RC(stack_slots));
4255   // No match rule because this operand is only generated in matching
4256   format %{ "[$reg]" %}
4257   interface(MEMORY_INTER) %{
4258     base(0x4);   // ESP
4259     index(0x4);  // No Index
4260     scale(0x0);  // No Scale
4261     disp($reg);  // Stack Offset
4262   %}
4263 %}
4264 
4265 operand stackSlotF(sRegF reg) %{
4266   constraint(ALLOC_IN_RC(stack_slots));
4267   // No match rule because this operand is only generated in matching
4268   format %{ "[$reg]" %}
4269   interface(MEMORY_INTER) %{
4270     base(0x4);   // ESP
4271     index(0x4);  // No Index
4272     scale(0x0);  // No Scale
4273     disp($reg);  // Stack Offset
4274   %}
4275 %}
4276 
4277 operand stackSlotD(sRegD reg) %{
4278   constraint(ALLOC_IN_RC(stack_slots));
4279   // No match rule because this operand is only generated in matching
4280   format %{ "[$reg]" %}
4281   interface(MEMORY_INTER) %{
4282     base(0x4);   // ESP
4283     index(0x4);  // No Index
4284     scale(0x0);  // No Scale
4285     disp($reg);  // Stack Offset
4286   %}
4287 %}
4288 
4289 operand stackSlotL(sRegL reg) %{
4290   constraint(ALLOC_IN_RC(stack_slots));
4291   // No match rule because this operand is only generated in matching
4292   format %{ "[$reg]" %}
4293   interface(MEMORY_INTER) %{
4294     base(0x4);   // ESP
4295     index(0x4);  // No Index
4296     scale(0x0);  // No Scale
4297     disp($reg);  // Stack Offset
4298   %}
4299 %}
4300 
4301 //----------Memory Operands - Win95 Implicit Null Variants----------------
4302 // Indirect Memory Operand
4303 operand indirect_win95_safe(eRegP_no_EBP reg)
4304 %{
4305   constraint(ALLOC_IN_RC(int_reg));
4306   match(reg);
4307 
4308   op_cost(100);
4309   format %{ "[$reg]" %}
4310   interface(MEMORY_INTER) %{
4311     base($reg);
4312     index(0x4);
4313     scale(0x0);
4314     disp(0x0);
4315   %}
4316 %}
4317 
4318 // Indirect Memory Plus Short Offset Operand
4319 operand indOffset8_win95_safe(eRegP_no_EBP reg, immI8 off)
4320 %{
4321   match(AddP reg off);
4322 
4323   op_cost(100);
4324   format %{ "[$reg + $off]" %}
4325   interface(MEMORY_INTER) %{
4326     base($reg);
4327     index(0x4);
4328     scale(0x0);
4329     disp($off);
4330   %}
4331 %}
4332 
4333 // Indirect Memory Plus Long Offset Operand
4334 operand indOffset32_win95_safe(eRegP_no_EBP reg, immI off)
4335 %{
4336   match(AddP reg off);
4337 
4338   op_cost(100);
4339   format %{ "[$reg + $off]" %}
4340   interface(MEMORY_INTER) %{
4341     base($reg);
4342     index(0x4);
4343     scale(0x0);
4344     disp($off);
4345   %}
4346 %}
4347 
4348 // Indirect Memory Plus Index Register Plus Offset Operand
4349 operand indIndexOffset_win95_safe(eRegP_no_EBP reg, rRegI ireg, immI off)
4350 %{
4351   match(AddP (AddP reg ireg) off);
4352 
4353   op_cost(100);
4354   format %{"[$reg + $off + $ireg]" %}
4355   interface(MEMORY_INTER) %{
4356     base($reg);
4357     index($ireg);
4358     scale(0x0);
4359     disp($off);
4360   %}
4361 %}
4362 
4363 // Indirect Memory Times Scale Plus Index Register
4364 operand indIndexScale_win95_safe(eRegP_no_EBP reg, rRegI ireg, immI2 scale)
4365 %{
4366   match(AddP reg (LShiftI ireg scale));
4367 
4368   op_cost(100);
4369   format %{"[$reg + $ireg << $scale]" %}
4370   interface(MEMORY_INTER) %{
4371     base($reg);
4372     index($ireg);
4373     scale($scale);
4374     disp(0x0);
4375   %}
4376 %}
4377 
4378 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
4379 operand indIndexScaleOffset_win95_safe(eRegP_no_EBP reg, immI off, rRegI ireg, immI2 scale)
4380 %{
4381   match(AddP (AddP reg (LShiftI ireg scale)) off);
4382 
4383   op_cost(100);
4384   format %{"[$reg + $off + $ireg << $scale]" %}
4385   interface(MEMORY_INTER) %{
4386     base($reg);
4387     index($ireg);
4388     scale($scale);
4389     disp($off);
4390   %}
4391 %}
4392 
4393 //----------Conditional Branch Operands----------------------------------------
4394 // Comparison Op  - This is the operation of the comparison, and is limited to
4395 //                  the following set of codes:
4396 //                  L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
4397 //
4398 // Other attributes of the comparison, such as unsignedness, are specified
4399 // by the comparison instruction that sets a condition code flags register.
4400 // That result is represented by a flags operand whose subtype is appropriate
4401 // to the unsignedness (etc.) of the comparison.
4402 //
4403 // Later, the instruction which matches both the Comparison Op (a Bool) and
4404 // the flags (produced by the Cmp) specifies the coding of the comparison op
4405 // by matching a specific subtype of Bool operand below, such as cmpOpU.
4406 
4407 // Comparision Code
4408 operand cmpOp() %{
4409   match(Bool);
4410 
4411   format %{ "" %}
4412   interface(COND_INTER) %{
4413     equal(0x4, "e");
4414     not_equal(0x5, "ne");
4415     less(0xC, "l");
4416     greater_equal(0xD, "ge");
4417     less_equal(0xE, "le");
4418     greater(0xF, "g");
4419     overflow(0x0, "o");
4420     no_overflow(0x1, "no");
4421   %}
4422 %}
4423 
4424 // Comparison Code, unsigned compare.  Used by FP also, with
4425 // C2 (unordered) turned into GT or LT already.  The other bits
4426 // C0 and C3 are turned into Carry & Zero flags.
4427 operand cmpOpU() %{
4428   match(Bool);
4429 
4430   format %{ "" %}
4431   interface(COND_INTER) %{
4432     equal(0x4, "e");
4433     not_equal(0x5, "ne");
4434     less(0x2, "b");
4435     greater_equal(0x3, "nb");
4436     less_equal(0x6, "be");
4437     greater(0x7, "nbe");
4438     overflow(0x0, "o");
4439     no_overflow(0x1, "no");
4440   %}
4441 %}
4442 
4443 // Floating comparisons that don't require any fixup for the unordered case
4444 operand cmpOpUCF() %{
4445   match(Bool);
4446   predicate(n->as_Bool()->_test._test == BoolTest::lt ||
4447             n->as_Bool()->_test._test == BoolTest::ge ||
4448             n->as_Bool()->_test._test == BoolTest::le ||
4449             n->as_Bool()->_test._test == BoolTest::gt);
4450   format %{ "" %}
4451   interface(COND_INTER) %{
4452     equal(0x4, "e");
4453     not_equal(0x5, "ne");
4454     less(0x2, "b");
4455     greater_equal(0x3, "nb");
4456     less_equal(0x6, "be");
4457     greater(0x7, "nbe");
4458     overflow(0x0, "o");
4459     no_overflow(0x1, "no");
4460   %}
4461 %}
4462 
4463 
4464 // Floating comparisons that can be fixed up with extra conditional jumps
4465 operand cmpOpUCF2() %{
4466   match(Bool);
4467   predicate(n->as_Bool()->_test._test == BoolTest::ne ||
4468             n->as_Bool()->_test._test == BoolTest::eq);
4469   format %{ "" %}
4470   interface(COND_INTER) %{
4471     equal(0x4, "e");
4472     not_equal(0x5, "ne");
4473     less(0x2, "b");
4474     greater_equal(0x3, "nb");
4475     less_equal(0x6, "be");
4476     greater(0x7, "nbe");
4477     overflow(0x0, "o");
4478     no_overflow(0x1, "no");
4479   %}
4480 %}
4481 
4482 // Comparison Code for FP conditional move
4483 operand cmpOp_fcmov() %{
4484   match(Bool);
4485 
4486   predicate(n->as_Bool()->_test._test != BoolTest::overflow &&
4487             n->as_Bool()->_test._test != BoolTest::no_overflow);
4488   format %{ "" %}
4489   interface(COND_INTER) %{
4490     equal        (0x0C8);
4491     not_equal    (0x1C8);
4492     less         (0x0C0);
4493     greater_equal(0x1C0);
4494     less_equal   (0x0D0);
4495     greater      (0x1D0);
4496     overflow(0x0, "o"); // not really supported by the instruction
4497     no_overflow(0x1, "no"); // not really supported by the instruction
4498   %}
4499 %}
4500 
4501 // Comparison Code used in long compares
4502 operand cmpOp_commute() %{
4503   match(Bool);
4504 
4505   format %{ "" %}
4506   interface(COND_INTER) %{
4507     equal(0x4, "e");
4508     not_equal(0x5, "ne");
4509     less(0xF, "g");
4510     greater_equal(0xE, "le");
4511     less_equal(0xD, "ge");
4512     greater(0xC, "l");
4513     overflow(0x0, "o");
4514     no_overflow(0x1, "no");
4515   %}
4516 %}
4517 
4518 // Comparison Code used in unsigned long compares
4519 operand cmpOpU_commute() %{
4520   match(Bool);
4521 
4522   format %{ "" %}
4523   interface(COND_INTER) %{
4524     equal(0x4, "e");
4525     not_equal(0x5, "ne");
4526     less(0x7, "nbe");
4527     greater_equal(0x6, "be");
4528     less_equal(0x3, "nb");
4529     greater(0x2, "b");
4530     overflow(0x0, "o");
4531     no_overflow(0x1, "no");
4532   %}
4533 %}
4534 
4535 //----------OPERAND CLASSES----------------------------------------------------
4536 // Operand Classes are groups of operands that are used as to simplify
4537 // instruction definitions by not requiring the AD writer to specify separate
4538 // instructions for every form of operand when the instruction accepts
4539 // multiple operand types with the same basic encoding and format.  The classic
4540 // case of this is memory operands.
4541 
4542 opclass memory(direct, indirect, indOffset8, indOffset32, indOffset32X, indIndexOffset,
4543                indIndex, indIndexScale, indIndexScaleOffset);
4544 
4545 // Long memory operations are encoded in 2 instructions and a +4 offset.
4546 // This means some kind of offset is always required and you cannot use
4547 // an oop as the offset (done when working on static globals).
4548 opclass long_memory(direct, indirect, indOffset8, indOffset32, indIndexOffset,
4549                     indIndex, indIndexScale, indIndexScaleOffset);
4550 
4551 
4552 //----------PIPELINE-----------------------------------------------------------
4553 // Rules which define the behavior of the target architectures pipeline.
4554 pipeline %{
4555 
4556 //----------ATTRIBUTES---------------------------------------------------------
4557 attributes %{
4558   variable_size_instructions;        // Fixed size instructions
4559   max_instructions_per_bundle = 3;   // Up to 3 instructions per bundle
4560   instruction_unit_size = 1;         // An instruction is 1 bytes long
4561   instruction_fetch_unit_size = 16;  // The processor fetches one line
4562   instruction_fetch_units = 1;       // of 16 bytes
4563 
4564   // List of nop instructions
4565   nops( MachNop );
4566 %}
4567 
4568 //----------RESOURCES----------------------------------------------------------
4569 // Resources are the functional units available to the machine
4570 
4571 // Generic P2/P3 pipeline
4572 // 3 decoders, only D0 handles big operands; a "bundle" is the limit of
4573 // 3 instructions decoded per cycle.
4574 // 2 load/store ops per cycle, 1 branch, 1 FPU,
4575 // 2 ALU op, only ALU0 handles mul/div instructions.
4576 resources( D0, D1, D2, DECODE = D0 | D1 | D2,
4577            MS0, MS1, MEM = MS0 | MS1,
4578            BR, FPU,
4579            ALU0, ALU1, ALU = ALU0 | ALU1 );
4580 
4581 //----------PIPELINE DESCRIPTION-----------------------------------------------
4582 // Pipeline Description specifies the stages in the machine's pipeline
4583 
4584 // Generic P2/P3 pipeline
4585 pipe_desc(S0, S1, S2, S3, S4, S5);
4586 
4587 //----------PIPELINE CLASSES---------------------------------------------------
4588 // Pipeline Classes describe the stages in which input and output are
4589 // referenced by the hardware pipeline.
4590 
4591 // Naming convention: ialu or fpu
4592 // Then: _reg
4593 // Then: _reg if there is a 2nd register
4594 // Then: _long if it's a pair of instructions implementing a long
4595 // Then: _fat if it requires the big decoder
4596 //   Or: _mem if it requires the big decoder and a memory unit.
4597 
4598 // Integer ALU reg operation
4599 pipe_class ialu_reg(rRegI dst) %{
4600     single_instruction;
4601     dst    : S4(write);
4602     dst    : S3(read);
4603     DECODE : S0;        // any decoder
4604     ALU    : S3;        // any alu
4605 %}
4606 
4607 // Long ALU reg operation
4608 pipe_class ialu_reg_long(eRegL dst) %{
4609     instruction_count(2);
4610     dst    : S4(write);
4611     dst    : S3(read);
4612     DECODE : S0(2);     // any 2 decoders
4613     ALU    : S3(2);     // both alus
4614 %}
4615 
4616 // Integer ALU reg operation using big decoder
4617 pipe_class ialu_reg_fat(rRegI dst) %{
4618     single_instruction;
4619     dst    : S4(write);
4620     dst    : S3(read);
4621     D0     : S0;        // big decoder only
4622     ALU    : S3;        // any alu
4623 %}
4624 
4625 // Long ALU reg operation using big decoder
4626 pipe_class ialu_reg_long_fat(eRegL dst) %{
4627     instruction_count(2);
4628     dst    : S4(write);
4629     dst    : S3(read);
4630     D0     : S0(2);     // big decoder only; twice
4631     ALU    : S3(2);     // any 2 alus
4632 %}
4633 
4634 // Integer ALU reg-reg operation
4635 pipe_class ialu_reg_reg(rRegI dst, rRegI src) %{
4636     single_instruction;
4637     dst    : S4(write);
4638     src    : S3(read);
4639     DECODE : S0;        // any decoder
4640     ALU    : S3;        // any alu
4641 %}
4642 
4643 // Long ALU reg-reg operation
4644 pipe_class ialu_reg_reg_long(eRegL dst, eRegL src) %{
4645     instruction_count(2);
4646     dst    : S4(write);
4647     src    : S3(read);
4648     DECODE : S0(2);     // any 2 decoders
4649     ALU    : S3(2);     // both alus
4650 %}
4651 
4652 // Integer ALU reg-reg operation
4653 pipe_class ialu_reg_reg_fat(rRegI dst, memory src) %{
4654     single_instruction;
4655     dst    : S4(write);
4656     src    : S3(read);
4657     D0     : S0;        // big decoder only
4658     ALU    : S3;        // any alu
4659 %}
4660 
4661 // Long ALU reg-reg operation
4662 pipe_class ialu_reg_reg_long_fat(eRegL dst, eRegL src) %{
4663     instruction_count(2);
4664     dst    : S4(write);
4665     src    : S3(read);
4666     D0     : S0(2);     // big decoder only; twice
4667     ALU    : S3(2);     // both alus
4668 %}
4669 
4670 // Integer ALU reg-mem operation
4671 pipe_class ialu_reg_mem(rRegI dst, memory mem) %{
4672     single_instruction;
4673     dst    : S5(write);
4674     mem    : S3(read);
4675     D0     : S0;        // big decoder only
4676     ALU    : S4;        // any alu
4677     MEM    : S3;        // any mem
4678 %}
4679 
4680 // Long ALU reg-mem operation
4681 pipe_class ialu_reg_long_mem(eRegL dst, load_long_memory mem) %{
4682     instruction_count(2);
4683     dst    : S5(write);
4684     mem    : S3(read);
4685     D0     : S0(2);     // big decoder only; twice
4686     ALU    : S4(2);     // any 2 alus
4687     MEM    : S3(2);     // both mems
4688 %}
4689 
4690 // Integer mem operation (prefetch)
4691 pipe_class ialu_mem(memory mem)
4692 %{
4693     single_instruction;
4694     mem    : S3(read);
4695     D0     : S0;        // big decoder only
4696     MEM    : S3;        // any mem
4697 %}
4698 
4699 // Integer Store to Memory
4700 pipe_class ialu_mem_reg(memory mem, rRegI src) %{
4701     single_instruction;
4702     mem    : S3(read);
4703     src    : S5(read);
4704     D0     : S0;        // big decoder only
4705     ALU    : S4;        // any alu
4706     MEM    : S3;
4707 %}
4708 
4709 // Long Store to Memory
4710 pipe_class ialu_mem_long_reg(memory mem, eRegL src) %{
4711     instruction_count(2);
4712     mem    : S3(read);
4713     src    : S5(read);
4714     D0     : S0(2);     // big decoder only; twice
4715     ALU    : S4(2);     // any 2 alus
4716     MEM    : S3(2);     // Both mems
4717 %}
4718 
4719 // Integer Store to Memory
4720 pipe_class ialu_mem_imm(memory mem) %{
4721     single_instruction;
4722     mem    : S3(read);
4723     D0     : S0;        // big decoder only
4724     ALU    : S4;        // any alu
4725     MEM    : S3;
4726 %}
4727 
4728 // Integer ALU0 reg-reg operation
4729 pipe_class ialu_reg_reg_alu0(rRegI dst, rRegI src) %{
4730     single_instruction;
4731     dst    : S4(write);
4732     src    : S3(read);
4733     D0     : S0;        // Big decoder only
4734     ALU0   : S3;        // only alu0
4735 %}
4736 
4737 // Integer ALU0 reg-mem operation
4738 pipe_class ialu_reg_mem_alu0(rRegI dst, memory mem) %{
4739     single_instruction;
4740     dst    : S5(write);
4741     mem    : S3(read);
4742     D0     : S0;        // big decoder only
4743     ALU0   : S4;        // ALU0 only
4744     MEM    : S3;        // any mem
4745 %}
4746 
4747 // Integer ALU reg-reg operation
4748 pipe_class ialu_cr_reg_reg(eFlagsReg cr, rRegI src1, rRegI src2) %{
4749     single_instruction;
4750     cr     : S4(write);
4751     src1   : S3(read);
4752     src2   : S3(read);
4753     DECODE : S0;        // any decoder
4754     ALU    : S3;        // any alu
4755 %}
4756 
4757 // Integer ALU reg-imm operation
4758 pipe_class ialu_cr_reg_imm(eFlagsReg cr, rRegI src1) %{
4759     single_instruction;
4760     cr     : S4(write);
4761     src1   : S3(read);
4762     DECODE : S0;        // any decoder
4763     ALU    : S3;        // any alu
4764 %}
4765 
4766 // Integer ALU reg-mem operation
4767 pipe_class ialu_cr_reg_mem(eFlagsReg cr, rRegI src1, memory src2) %{
4768     single_instruction;
4769     cr     : S4(write);
4770     src1   : S3(read);
4771     src2   : S3(read);
4772     D0     : S0;        // big decoder only
4773     ALU    : S4;        // any alu
4774     MEM    : S3;
4775 %}
4776 
4777 // Conditional move reg-reg
4778 pipe_class pipe_cmplt( rRegI p, rRegI q, rRegI y ) %{
4779     instruction_count(4);
4780     y      : S4(read);
4781     q      : S3(read);
4782     p      : S3(read);
4783     DECODE : S0(4);     // any decoder
4784 %}
4785 
4786 // Conditional move reg-reg
4787 pipe_class pipe_cmov_reg( rRegI dst, rRegI src, eFlagsReg cr ) %{
4788     single_instruction;
4789     dst    : S4(write);
4790     src    : S3(read);
4791     cr     : S3(read);
4792     DECODE : S0;        // any decoder
4793 %}
4794 
4795 // Conditional move reg-mem
4796 pipe_class pipe_cmov_mem( eFlagsReg cr, rRegI dst, memory src) %{
4797     single_instruction;
4798     dst    : S4(write);
4799     src    : S3(read);
4800     cr     : S3(read);
4801     DECODE : S0;        // any decoder
4802     MEM    : S3;
4803 %}
4804 
4805 // Conditional move reg-reg long
4806 pipe_class pipe_cmov_reg_long( eFlagsReg cr, eRegL dst, eRegL src) %{
4807     single_instruction;
4808     dst    : S4(write);
4809     src    : S3(read);
4810     cr     : S3(read);
4811     DECODE : S0(2);     // any 2 decoders
4812 %}
4813 
4814 // Conditional move double reg-reg
4815 pipe_class pipe_cmovDPR_reg( eFlagsReg cr, regDPR1 dst, regDPR src) %{
4816     single_instruction;
4817     dst    : S4(write);
4818     src    : S3(read);
4819     cr     : S3(read);
4820     DECODE : S0;        // any decoder
4821 %}
4822 
4823 // Float reg-reg operation
4824 pipe_class fpu_reg(regDPR dst) %{
4825     instruction_count(2);
4826     dst    : S3(read);
4827     DECODE : S0(2);     // any 2 decoders
4828     FPU    : S3;
4829 %}
4830 
4831 // Float reg-reg operation
4832 pipe_class fpu_reg_reg(regDPR dst, regDPR src) %{
4833     instruction_count(2);
4834     dst    : S4(write);
4835     src    : S3(read);
4836     DECODE : S0(2);     // any 2 decoders
4837     FPU    : S3;
4838 %}
4839 
4840 // Float reg-reg operation
4841 pipe_class fpu_reg_reg_reg(regDPR dst, regDPR src1, regDPR src2) %{
4842     instruction_count(3);
4843     dst    : S4(write);
4844     src1   : S3(read);
4845     src2   : S3(read);
4846     DECODE : S0(3);     // any 3 decoders
4847     FPU    : S3(2);
4848 %}
4849 
4850 // Float reg-reg operation
4851 pipe_class fpu_reg_reg_reg_reg(regDPR dst, regDPR src1, regDPR src2, regDPR src3) %{
4852     instruction_count(4);
4853     dst    : S4(write);
4854     src1   : S3(read);
4855     src2   : S3(read);
4856     src3   : S3(read);
4857     DECODE : S0(4);     // any 3 decoders
4858     FPU    : S3(2);
4859 %}
4860 
4861 // Float reg-reg operation
4862 pipe_class fpu_reg_mem_reg_reg(regDPR dst, memory src1, regDPR src2, regDPR src3) %{
4863     instruction_count(4);
4864     dst    : S4(write);
4865     src1   : S3(read);
4866     src2   : S3(read);
4867     src3   : S3(read);
4868     DECODE : S1(3);     // any 3 decoders
4869     D0     : S0;        // Big decoder only
4870     FPU    : S3(2);
4871     MEM    : S3;
4872 %}
4873 
4874 // Float reg-mem operation
4875 pipe_class fpu_reg_mem(regDPR dst, memory mem) %{
4876     instruction_count(2);
4877     dst    : S5(write);
4878     mem    : S3(read);
4879     D0     : S0;        // big decoder only
4880     DECODE : S1;        // any decoder for FPU POP
4881     FPU    : S4;
4882     MEM    : S3;        // any mem
4883 %}
4884 
4885 // Float reg-mem operation
4886 pipe_class fpu_reg_reg_mem(regDPR dst, regDPR src1, memory mem) %{
4887     instruction_count(3);
4888     dst    : S5(write);
4889     src1   : S3(read);
4890     mem    : S3(read);
4891     D0     : S0;        // big decoder only
4892     DECODE : S1(2);     // any decoder for FPU POP
4893     FPU    : S4;
4894     MEM    : S3;        // any mem
4895 %}
4896 
4897 // Float mem-reg operation
4898 pipe_class fpu_mem_reg(memory mem, regDPR src) %{
4899     instruction_count(2);
4900     src    : S5(read);
4901     mem    : S3(read);
4902     DECODE : S0;        // any decoder for FPU PUSH
4903     D0     : S1;        // big decoder only
4904     FPU    : S4;
4905     MEM    : S3;        // any mem
4906 %}
4907 
4908 pipe_class fpu_mem_reg_reg(memory mem, regDPR src1, regDPR src2) %{
4909     instruction_count(3);
4910     src1   : S3(read);
4911     src2   : S3(read);
4912     mem    : S3(read);
4913     DECODE : S0(2);     // any decoder for FPU PUSH
4914     D0     : S1;        // big decoder only
4915     FPU    : S4;
4916     MEM    : S3;        // any mem
4917 %}
4918 
4919 pipe_class fpu_mem_reg_mem(memory mem, regDPR src1, memory src2) %{
4920     instruction_count(3);
4921     src1   : S3(read);
4922     src2   : S3(read);
4923     mem    : S4(read);
4924     DECODE : S0;        // any decoder for FPU PUSH
4925     D0     : S0(2);     // big decoder only
4926     FPU    : S4;
4927     MEM    : S3(2);     // any mem
4928 %}
4929 
4930 pipe_class fpu_mem_mem(memory dst, memory src1) %{
4931     instruction_count(2);
4932     src1   : S3(read);
4933     dst    : S4(read);
4934     D0     : S0(2);     // big decoder only
4935     MEM    : S3(2);     // any mem
4936 %}
4937 
4938 pipe_class fpu_mem_mem_mem(memory dst, memory src1, memory src2) %{
4939     instruction_count(3);
4940     src1   : S3(read);
4941     src2   : S3(read);
4942     dst    : S4(read);
4943     D0     : S0(3);     // big decoder only
4944     FPU    : S4;
4945     MEM    : S3(3);     // any mem
4946 %}
4947 
4948 pipe_class fpu_mem_reg_con(memory mem, regDPR src1) %{
4949     instruction_count(3);
4950     src1   : S4(read);
4951     mem    : S4(read);
4952     DECODE : S0;        // any decoder for FPU PUSH
4953     D0     : S0(2);     // big decoder only
4954     FPU    : S4;
4955     MEM    : S3(2);     // any mem
4956 %}
4957 
4958 // Float load constant
4959 pipe_class fpu_reg_con(regDPR dst) %{
4960     instruction_count(2);
4961     dst    : S5(write);
4962     D0     : S0;        // big decoder only for the load
4963     DECODE : S1;        // any decoder for FPU POP
4964     FPU    : S4;
4965     MEM    : S3;        // any mem
4966 %}
4967 
4968 // Float load constant
4969 pipe_class fpu_reg_reg_con(regDPR dst, regDPR src) %{
4970     instruction_count(3);
4971     dst    : S5(write);
4972     src    : S3(read);
4973     D0     : S0;        // big decoder only for the load
4974     DECODE : S1(2);     // any decoder for FPU POP
4975     FPU    : S4;
4976     MEM    : S3;        // any mem
4977 %}
4978 
4979 // UnConditional branch
4980 pipe_class pipe_jmp( label labl ) %{
4981     single_instruction;
4982     BR   : S3;
4983 %}
4984 
4985 // Conditional branch
4986 pipe_class pipe_jcc( cmpOp cmp, eFlagsReg cr, label labl ) %{
4987     single_instruction;
4988     cr    : S1(read);
4989     BR    : S3;
4990 %}
4991 
4992 // Allocation idiom
4993 pipe_class pipe_cmpxchg( eRegP dst, eRegP heap_ptr ) %{
4994     instruction_count(1); force_serialization;
4995     fixed_latency(6);
4996     heap_ptr : S3(read);
4997     DECODE   : S0(3);
4998     D0       : S2;
4999     MEM      : S3;
5000     ALU      : S3(2);
5001     dst      : S5(write);
5002     BR       : S5;
5003 %}
5004 
5005 // Generic big/slow expanded idiom
5006 pipe_class pipe_slow(  ) %{
5007     instruction_count(10); multiple_bundles; force_serialization;
5008     fixed_latency(100);
5009     D0  : S0(2);
5010     MEM : S3(2);
5011 %}
5012 
5013 // The real do-nothing guy
5014 pipe_class empty( ) %{
5015     instruction_count(0);
5016 %}
5017 
5018 // Define the class for the Nop node
5019 define %{
5020    MachNop = empty;
5021 %}
5022 
5023 %}
5024 
5025 //----------INSTRUCTIONS-------------------------------------------------------
5026 //
5027 // match      -- States which machine-independent subtree may be replaced
5028 //               by this instruction.
5029 // ins_cost   -- The estimated cost of this instruction is used by instruction
5030 //               selection to identify a minimum cost tree of machine
5031 //               instructions that matches a tree of machine-independent
5032 //               instructions.
5033 // format     -- A string providing the disassembly for this instruction.
5034 //               The value of an instruction's operand may be inserted
5035 //               by referring to it with a '$' prefix.
5036 // opcode     -- Three instruction opcodes may be provided.  These are referred
5037 //               to within an encode class as $primary, $secondary, and $tertiary
5038 //               respectively.  The primary opcode is commonly used to
5039 //               indicate the type of machine instruction, while secondary
5040 //               and tertiary are often used for prefix options or addressing
5041 //               modes.
5042 // ins_encode -- A list of encode classes with parameters. The encode class
5043 //               name must have been defined in an 'enc_class' specification
5044 //               in the encode section of the architecture description.
5045 
5046 //----------BSWAP-Instruction--------------------------------------------------
5047 instruct bytes_reverse_int(rRegI dst) %{
5048   match(Set dst (ReverseBytesI dst));
5049 
5050   format %{ "BSWAP  $dst" %}
5051   opcode(0x0F, 0xC8);
5052   ins_encode( OpcP, OpcSReg(dst) );
5053   ins_pipe( ialu_reg );
5054 %}
5055 
5056 instruct bytes_reverse_long(eRegL dst) %{
5057   match(Set dst (ReverseBytesL dst));
5058 
5059   format %{ "BSWAP  $dst.lo\n\t"
5060             "BSWAP  $dst.hi\n\t"
5061             "XCHG   $dst.lo $dst.hi" %}
5062 
5063   ins_cost(125);
5064   ins_encode( bswap_long_bytes(dst) );
5065   ins_pipe( ialu_reg_reg);
5066 %}
5067 
5068 instruct bytes_reverse_unsigned_short(rRegI dst, eFlagsReg cr) %{
5069   match(Set dst (ReverseBytesUS dst));
5070   effect(KILL cr);
5071 
5072   format %{ "BSWAP  $dst\n\t" 
5073             "SHR    $dst,16\n\t" %}
5074   ins_encode %{
5075     __ bswapl($dst$$Register);
5076     __ shrl($dst$$Register, 16); 
5077   %}
5078   ins_pipe( ialu_reg );
5079 %}
5080 
5081 instruct bytes_reverse_short(rRegI dst, eFlagsReg cr) %{
5082   match(Set dst (ReverseBytesS dst));
5083   effect(KILL cr);
5084 
5085   format %{ "BSWAP  $dst\n\t" 
5086             "SAR    $dst,16\n\t" %}
5087   ins_encode %{
5088     __ bswapl($dst$$Register);
5089     __ sarl($dst$$Register, 16); 
5090   %}
5091   ins_pipe( ialu_reg );
5092 %}
5093 
5094 
5095 //---------- Zeros Count Instructions ------------------------------------------
5096 
5097 instruct countLeadingZerosI(rRegI dst, rRegI src, eFlagsReg cr) %{
5098   predicate(UseCountLeadingZerosInstruction);
5099   match(Set dst (CountLeadingZerosI src));
5100   effect(KILL cr);
5101 
5102   format %{ "LZCNT  $dst, $src\t# count leading zeros (int)" %}
5103   ins_encode %{
5104     __ lzcntl($dst$$Register, $src$$Register);
5105   %}
5106   ins_pipe(ialu_reg);
5107 %}
5108 
5109 instruct countLeadingZerosI_bsr(rRegI dst, rRegI src, eFlagsReg cr) %{
5110   predicate(!UseCountLeadingZerosInstruction);
5111   match(Set dst (CountLeadingZerosI src));
5112   effect(KILL cr);
5113 
5114   format %{ "BSR    $dst, $src\t# count leading zeros (int)\n\t"
5115             "JNZ    skip\n\t"
5116             "MOV    $dst, -1\n"
5117       "skip:\n\t"
5118             "NEG    $dst\n\t"
5119             "ADD    $dst, 31" %}
5120   ins_encode %{
5121     Register Rdst = $dst$$Register;
5122     Register Rsrc = $src$$Register;
5123     Label skip;
5124     __ bsrl(Rdst, Rsrc);
5125     __ jccb(Assembler::notZero, skip);
5126     __ movl(Rdst, -1);
5127     __ bind(skip);
5128     __ negl(Rdst);
5129     __ addl(Rdst, BitsPerInt - 1);
5130   %}
5131   ins_pipe(ialu_reg);
5132 %}
5133 
5134 instruct countLeadingZerosL(rRegI dst, eRegL src, eFlagsReg cr) %{
5135   predicate(UseCountLeadingZerosInstruction);
5136   match(Set dst (CountLeadingZerosL src));
5137   effect(TEMP dst, KILL cr);
5138 
5139   format %{ "LZCNT  $dst, $src.hi\t# count leading zeros (long)\n\t"
5140             "JNC    done\n\t"
5141             "LZCNT  $dst, $src.lo\n\t"
5142             "ADD    $dst, 32\n"
5143       "done:" %}
5144   ins_encode %{
5145     Register Rdst = $dst$$Register;
5146     Register Rsrc = $src$$Register;
5147     Label done;
5148     __ lzcntl(Rdst, HIGH_FROM_LOW(Rsrc));
5149     __ jccb(Assembler::carryClear, done);
5150     __ lzcntl(Rdst, Rsrc);
5151     __ addl(Rdst, BitsPerInt);
5152     __ bind(done);
5153   %}
5154   ins_pipe(ialu_reg);
5155 %}
5156 
5157 instruct countLeadingZerosL_bsr(rRegI dst, eRegL src, eFlagsReg cr) %{
5158   predicate(!UseCountLeadingZerosInstruction);
5159   match(Set dst (CountLeadingZerosL src));
5160   effect(TEMP dst, KILL cr);
5161 
5162   format %{ "BSR    $dst, $src.hi\t# count leading zeros (long)\n\t"
5163             "JZ     msw_is_zero\n\t"
5164             "ADD    $dst, 32\n\t"
5165             "JMP    not_zero\n"
5166       "msw_is_zero:\n\t"
5167             "BSR    $dst, $src.lo\n\t"
5168             "JNZ    not_zero\n\t"
5169             "MOV    $dst, -1\n"
5170       "not_zero:\n\t"
5171             "NEG    $dst\n\t"
5172             "ADD    $dst, 63\n" %}
5173  ins_encode %{
5174     Register Rdst = $dst$$Register;
5175     Register Rsrc = $src$$Register;
5176     Label msw_is_zero;
5177     Label not_zero;
5178     __ bsrl(Rdst, HIGH_FROM_LOW(Rsrc));
5179     __ jccb(Assembler::zero, msw_is_zero);
5180     __ addl(Rdst, BitsPerInt);
5181     __ jmpb(not_zero);
5182     __ bind(msw_is_zero);
5183     __ bsrl(Rdst, Rsrc);
5184     __ jccb(Assembler::notZero, not_zero);
5185     __ movl(Rdst, -1);
5186     __ bind(not_zero);
5187     __ negl(Rdst);
5188     __ addl(Rdst, BitsPerLong - 1);
5189   %}
5190   ins_pipe(ialu_reg);
5191 %}
5192 
5193 instruct countTrailingZerosI(rRegI dst, rRegI src, eFlagsReg cr) %{
5194   predicate(UseCountTrailingZerosInstruction);
5195   match(Set dst (CountTrailingZerosI src));
5196   effect(KILL cr);
5197 
5198   format %{ "TZCNT    $dst, $src\t# count trailing zeros (int)" %}
5199   ins_encode %{
5200     __ tzcntl($dst$$Register, $src$$Register);
5201   %}
5202   ins_pipe(ialu_reg);
5203 %}
5204 
5205 instruct countTrailingZerosI_bsf(rRegI dst, rRegI src, eFlagsReg cr) %{
5206   predicate(!UseCountTrailingZerosInstruction);
5207   match(Set dst (CountTrailingZerosI src));
5208   effect(KILL cr);
5209 
5210   format %{ "BSF    $dst, $src\t# count trailing zeros (int)\n\t"
5211             "JNZ    done\n\t"
5212             "MOV    $dst, 32\n"
5213       "done:" %}
5214   ins_encode %{
5215     Register Rdst = $dst$$Register;
5216     Label done;
5217     __ bsfl(Rdst, $src$$Register);
5218     __ jccb(Assembler::notZero, done);
5219     __ movl(Rdst, BitsPerInt);
5220     __ bind(done);
5221   %}
5222   ins_pipe(ialu_reg);
5223 %}
5224 
5225 instruct countTrailingZerosL(rRegI dst, eRegL src, eFlagsReg cr) %{
5226   predicate(UseCountTrailingZerosInstruction);
5227   match(Set dst (CountTrailingZerosL src));
5228   effect(TEMP dst, KILL cr);
5229 
5230   format %{ "TZCNT  $dst, $src.lo\t# count trailing zeros (long) \n\t"
5231             "JNC    done\n\t"
5232             "TZCNT  $dst, $src.hi\n\t"
5233             "ADD    $dst, 32\n"
5234             "done:" %}
5235   ins_encode %{
5236     Register Rdst = $dst$$Register;
5237     Register Rsrc = $src$$Register;
5238     Label done;
5239     __ tzcntl(Rdst, Rsrc);
5240     __ jccb(Assembler::carryClear, done);
5241     __ tzcntl(Rdst, HIGH_FROM_LOW(Rsrc));
5242     __ addl(Rdst, BitsPerInt);
5243     __ bind(done);
5244   %}
5245   ins_pipe(ialu_reg);
5246 %}
5247 
5248 instruct countTrailingZerosL_bsf(rRegI dst, eRegL src, eFlagsReg cr) %{
5249   predicate(!UseCountTrailingZerosInstruction);
5250   match(Set dst (CountTrailingZerosL src));
5251   effect(TEMP dst, KILL cr);
5252 
5253   format %{ "BSF    $dst, $src.lo\t# count trailing zeros (long)\n\t"
5254             "JNZ    done\n\t"
5255             "BSF    $dst, $src.hi\n\t"
5256             "JNZ    msw_not_zero\n\t"
5257             "MOV    $dst, 32\n"
5258       "msw_not_zero:\n\t"
5259             "ADD    $dst, 32\n"
5260       "done:" %}
5261   ins_encode %{
5262     Register Rdst = $dst$$Register;
5263     Register Rsrc = $src$$Register;
5264     Label msw_not_zero;
5265     Label done;
5266     __ bsfl(Rdst, Rsrc);
5267     __ jccb(Assembler::notZero, done);
5268     __ bsfl(Rdst, HIGH_FROM_LOW(Rsrc));
5269     __ jccb(Assembler::notZero, msw_not_zero);
5270     __ movl(Rdst, BitsPerInt);
5271     __ bind(msw_not_zero);
5272     __ addl(Rdst, BitsPerInt);
5273     __ bind(done);
5274   %}
5275   ins_pipe(ialu_reg);
5276 %}
5277 
5278 
5279 //---------- Population Count Instructions -------------------------------------
5280 
5281 instruct popCountI(rRegI dst, rRegI src, eFlagsReg cr) %{
5282   predicate(UsePopCountInstruction);
5283   match(Set dst (PopCountI src));
5284   effect(KILL cr);
5285 
5286   format %{ "POPCNT $dst, $src" %}
5287   ins_encode %{
5288     __ popcntl($dst$$Register, $src$$Register);
5289   %}
5290   ins_pipe(ialu_reg);
5291 %}
5292 
5293 instruct popCountI_mem(rRegI dst, memory mem, eFlagsReg cr) %{
5294   predicate(UsePopCountInstruction);
5295   match(Set dst (PopCountI (LoadI mem)));
5296   effect(KILL cr);
5297 
5298   format %{ "POPCNT $dst, $mem" %}
5299   ins_encode %{
5300     __ popcntl($dst$$Register, $mem$$Address);
5301   %}
5302   ins_pipe(ialu_reg);
5303 %}
5304 
5305 // Note: Long.bitCount(long) returns an int.
5306 instruct popCountL(rRegI dst, eRegL src, rRegI tmp, eFlagsReg cr) %{
5307   predicate(UsePopCountInstruction);
5308   match(Set dst (PopCountL src));
5309   effect(KILL cr, TEMP tmp, TEMP dst);
5310 
5311   format %{ "POPCNT $dst, $src.lo\n\t"
5312             "POPCNT $tmp, $src.hi\n\t"
5313             "ADD    $dst, $tmp" %}
5314   ins_encode %{
5315     __ popcntl($dst$$Register, $src$$Register);
5316     __ popcntl($tmp$$Register, HIGH_FROM_LOW($src$$Register));
5317     __ addl($dst$$Register, $tmp$$Register);
5318   %}
5319   ins_pipe(ialu_reg);
5320 %}
5321 
5322 // Note: Long.bitCount(long) returns an int.
5323 instruct popCountL_mem(rRegI dst, memory mem, rRegI tmp, eFlagsReg cr) %{
5324   predicate(UsePopCountInstruction);
5325   match(Set dst (PopCountL (LoadL mem)));
5326   effect(KILL cr, TEMP tmp, TEMP dst);
5327 
5328   format %{ "POPCNT $dst, $mem\n\t"
5329             "POPCNT $tmp, $mem+4\n\t"
5330             "ADD    $dst, $tmp" %}
5331   ins_encode %{
5332     //__ popcntl($dst$$Register, $mem$$Address$$first);
5333     //__ popcntl($tmp$$Register, $mem$$Address$$second);
5334     __ popcntl($dst$$Register, Address::make_raw($mem$$base, $mem$$index, $mem$$scale, $mem$$disp, relocInfo::none));
5335     __ popcntl($tmp$$Register, Address::make_raw($mem$$base, $mem$$index, $mem$$scale, $mem$$disp + 4, relocInfo::none));
5336     __ addl($dst$$Register, $tmp$$Register);
5337   %}
5338   ins_pipe(ialu_reg);
5339 %}
5340 
5341 
5342 //----------Load/Store/Move Instructions---------------------------------------
5343 //----------Load Instructions--------------------------------------------------
5344 // Load Byte (8bit signed)
5345 instruct loadB(xRegI dst, memory mem) %{
5346   match(Set dst (LoadB mem));
5347 
5348   ins_cost(125);
5349   format %{ "MOVSX8 $dst,$mem\t# byte" %}
5350 
5351   ins_encode %{
5352     __ movsbl($dst$$Register, $mem$$Address);
5353   %}
5354 
5355   ins_pipe(ialu_reg_mem);
5356 %}
5357 
5358 // Load Byte (8bit signed) into Long Register
5359 instruct loadB2L(eRegL dst, memory mem, eFlagsReg cr) %{
5360   match(Set dst (ConvI2L (LoadB mem)));
5361   effect(KILL cr);
5362 
5363   ins_cost(375);
5364   format %{ "MOVSX8 $dst.lo,$mem\t# byte -> long\n\t"
5365             "MOV    $dst.hi,$dst.lo\n\t"
5366             "SAR    $dst.hi,7" %}
5367 
5368   ins_encode %{
5369     __ movsbl($dst$$Register, $mem$$Address);
5370     __ movl(HIGH_FROM_LOW($dst$$Register), $dst$$Register); // This is always a different register.
5371     __ sarl(HIGH_FROM_LOW($dst$$Register), 7); // 24+1 MSB are already signed extended.
5372   %}
5373 
5374   ins_pipe(ialu_reg_mem);
5375 %}
5376 
5377 // Load Unsigned Byte (8bit UNsigned)
5378 instruct loadUB(xRegI dst, memory mem) %{
5379   match(Set dst (LoadUB mem));
5380 
5381   ins_cost(125);
5382   format %{ "MOVZX8 $dst,$mem\t# ubyte -> int" %}
5383 
5384   ins_encode %{
5385     __ movzbl($dst$$Register, $mem$$Address);
5386   %}
5387 
5388   ins_pipe(ialu_reg_mem);
5389 %}
5390 
5391 // Load Unsigned Byte (8 bit UNsigned) into Long Register
5392 instruct loadUB2L(eRegL dst, memory mem, eFlagsReg cr) %{
5393   match(Set dst (ConvI2L (LoadUB mem)));
5394   effect(KILL cr);
5395 
5396   ins_cost(250);
5397   format %{ "MOVZX8 $dst.lo,$mem\t# ubyte -> long\n\t"
5398             "XOR    $dst.hi,$dst.hi" %}
5399 
5400   ins_encode %{
5401     Register Rdst = $dst$$Register;
5402     __ movzbl(Rdst, $mem$$Address);
5403     __ xorl(HIGH_FROM_LOW(Rdst), HIGH_FROM_LOW(Rdst));
5404   %}
5405 
5406   ins_pipe(ialu_reg_mem);
5407 %}
5408 
5409 // Load Unsigned Byte (8 bit UNsigned) with mask into Long Register
5410 instruct loadUB2L_immI8(eRegL dst, memory mem, immI8 mask, eFlagsReg cr) %{
5411   match(Set dst (ConvI2L (AndI (LoadUB mem) mask)));
5412   effect(KILL cr);
5413 
5414   format %{ "MOVZX8 $dst.lo,$mem\t# ubyte & 8-bit mask -> long\n\t"
5415             "XOR    $dst.hi,$dst.hi\n\t"
5416             "AND    $dst.lo,$mask" %}
5417   ins_encode %{
5418     Register Rdst = $dst$$Register;
5419     __ movzbl(Rdst, $mem$$Address);
5420     __ xorl(HIGH_FROM_LOW(Rdst), HIGH_FROM_LOW(Rdst));
5421     __ andl(Rdst, $mask$$constant);
5422   %}
5423   ins_pipe(ialu_reg_mem);
5424 %}
5425 
5426 // Load Short (16bit signed)
5427 instruct loadS(rRegI dst, memory mem) %{
5428   match(Set dst (LoadS mem));
5429 
5430   ins_cost(125);
5431   format %{ "MOVSX  $dst,$mem\t# short" %}
5432 
5433   ins_encode %{
5434     __ movswl($dst$$Register, $mem$$Address);
5435   %}
5436 
5437   ins_pipe(ialu_reg_mem);
5438 %}
5439 
5440 // Load Short (16 bit signed) to Byte (8 bit signed)
5441 instruct loadS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
5442   match(Set dst (RShiftI (LShiftI (LoadS mem) twentyfour) twentyfour));
5443 
5444   ins_cost(125);
5445   format %{ "MOVSX  $dst, $mem\t# short -> byte" %}
5446   ins_encode %{
5447     __ movsbl($dst$$Register, $mem$$Address);
5448   %}
5449   ins_pipe(ialu_reg_mem);
5450 %}
5451 
5452 // Load Short (16bit signed) into Long Register
5453 instruct loadS2L(eRegL dst, memory mem, eFlagsReg cr) %{
5454   match(Set dst (ConvI2L (LoadS mem)));
5455   effect(KILL cr);
5456 
5457   ins_cost(375);
5458   format %{ "MOVSX  $dst.lo,$mem\t# short -> long\n\t"
5459             "MOV    $dst.hi,$dst.lo\n\t"
5460             "SAR    $dst.hi,15" %}
5461 
5462   ins_encode %{
5463     __ movswl($dst$$Register, $mem$$Address);
5464     __ movl(HIGH_FROM_LOW($dst$$Register), $dst$$Register); // This is always a different register.
5465     __ sarl(HIGH_FROM_LOW($dst$$Register), 15); // 16+1 MSB are already signed extended.
5466   %}
5467 
5468   ins_pipe(ialu_reg_mem);
5469 %}
5470 
5471 // Load Unsigned Short/Char (16bit unsigned)
5472 instruct loadUS(rRegI dst, memory mem) %{
5473   match(Set dst (LoadUS mem));
5474 
5475   ins_cost(125);
5476   format %{ "MOVZX  $dst,$mem\t# ushort/char -> int" %}
5477 
5478   ins_encode %{
5479     __ movzwl($dst$$Register, $mem$$Address);
5480   %}
5481 
5482   ins_pipe(ialu_reg_mem);
5483 %}
5484 
5485 // Load Unsigned Short/Char (16 bit UNsigned) to Byte (8 bit signed)
5486 instruct loadUS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
5487   match(Set dst (RShiftI (LShiftI (LoadUS mem) twentyfour) twentyfour));
5488 
5489   ins_cost(125);
5490   format %{ "MOVSX  $dst, $mem\t# ushort -> byte" %}
5491   ins_encode %{
5492     __ movsbl($dst$$Register, $mem$$Address);
5493   %}
5494   ins_pipe(ialu_reg_mem);
5495 %}
5496 
5497 // Load Unsigned Short/Char (16 bit UNsigned) into Long Register
5498 instruct loadUS2L(eRegL dst, memory mem, eFlagsReg cr) %{
5499   match(Set dst (ConvI2L (LoadUS mem)));
5500   effect(KILL cr);
5501 
5502   ins_cost(250);
5503   format %{ "MOVZX  $dst.lo,$mem\t# ushort/char -> long\n\t"
5504             "XOR    $dst.hi,$dst.hi" %}
5505 
5506   ins_encode %{
5507     __ movzwl($dst$$Register, $mem$$Address);
5508     __ xorl(HIGH_FROM_LOW($dst$$Register), HIGH_FROM_LOW($dst$$Register));
5509   %}
5510 
5511   ins_pipe(ialu_reg_mem);
5512 %}
5513 
5514 // Load Unsigned Short/Char (16 bit UNsigned) with mask 0xFF into Long Register
5515 instruct loadUS2L_immI_255(eRegL dst, memory mem, immI_255 mask, eFlagsReg cr) %{
5516   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
5517   effect(KILL cr);
5518 
5519   format %{ "MOVZX8 $dst.lo,$mem\t# ushort/char & 0xFF -> long\n\t"
5520             "XOR    $dst.hi,$dst.hi" %}
5521   ins_encode %{
5522     Register Rdst = $dst$$Register;
5523     __ movzbl(Rdst, $mem$$Address);
5524     __ xorl(HIGH_FROM_LOW(Rdst), HIGH_FROM_LOW(Rdst));
5525   %}
5526   ins_pipe(ialu_reg_mem);
5527 %}
5528 
5529 // Load Unsigned Short/Char (16 bit UNsigned) with a 16-bit mask into Long Register
5530 instruct loadUS2L_immI16(eRegL dst, memory mem, immI16 mask, eFlagsReg cr) %{
5531   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
5532   effect(KILL cr);
5533 
5534   format %{ "MOVZX  $dst.lo, $mem\t# ushort/char & 16-bit mask -> long\n\t"
5535             "XOR    $dst.hi,$dst.hi\n\t"
5536             "AND    $dst.lo,$mask" %}
5537   ins_encode %{
5538     Register Rdst = $dst$$Register;
5539     __ movzwl(Rdst, $mem$$Address);
5540     __ xorl(HIGH_FROM_LOW(Rdst), HIGH_FROM_LOW(Rdst));
5541     __ andl(Rdst, $mask$$constant);
5542   %}
5543   ins_pipe(ialu_reg_mem);
5544 %}
5545 
5546 // Load Integer
5547 instruct loadI(rRegI dst, memory mem) %{
5548   match(Set dst (LoadI mem));
5549 
5550   ins_cost(125);
5551   format %{ "MOV    $dst,$mem\t# int" %}
5552 
5553   ins_encode %{
5554     __ movl($dst$$Register, $mem$$Address);
5555   %}
5556 
5557   ins_pipe(ialu_reg_mem);
5558 %}
5559 
5560 // Load Integer (32 bit signed) to Byte (8 bit signed)
5561 instruct loadI2B(rRegI dst, memory mem, immI_24 twentyfour) %{
5562   match(Set dst (RShiftI (LShiftI (LoadI mem) twentyfour) twentyfour));
5563 
5564   ins_cost(125);
5565   format %{ "MOVSX  $dst, $mem\t# int -> byte" %}
5566   ins_encode %{
5567     __ movsbl($dst$$Register, $mem$$Address);
5568   %}
5569   ins_pipe(ialu_reg_mem);
5570 %}
5571 
5572 // Load Integer (32 bit signed) to Unsigned Byte (8 bit UNsigned)
5573 instruct loadI2UB(rRegI dst, memory mem, immI_255 mask) %{
5574   match(Set dst (AndI (LoadI mem) mask));
5575 
5576   ins_cost(125);
5577   format %{ "MOVZX  $dst, $mem\t# int -> ubyte" %}
5578   ins_encode %{
5579     __ movzbl($dst$$Register, $mem$$Address);
5580   %}
5581   ins_pipe(ialu_reg_mem);
5582 %}
5583 
5584 // Load Integer (32 bit signed) to Short (16 bit signed)
5585 instruct loadI2S(rRegI dst, memory mem, immI_16 sixteen) %{
5586   match(Set dst (RShiftI (LShiftI (LoadI mem) sixteen) sixteen));
5587 
5588   ins_cost(125);
5589   format %{ "MOVSX  $dst, $mem\t# int -> short" %}
5590   ins_encode %{
5591     __ movswl($dst$$Register, $mem$$Address);
5592   %}
5593   ins_pipe(ialu_reg_mem);
5594 %}
5595 
5596 // Load Integer (32 bit signed) to Unsigned Short/Char (16 bit UNsigned)
5597 instruct loadI2US(rRegI dst, memory mem, immI_65535 mask) %{
5598   match(Set dst (AndI (LoadI mem) mask));
5599 
5600   ins_cost(125);
5601   format %{ "MOVZX  $dst, $mem\t# int -> ushort/char" %}
5602   ins_encode %{
5603     __ movzwl($dst$$Register, $mem$$Address);
5604   %}
5605   ins_pipe(ialu_reg_mem);
5606 %}
5607 
5608 // Load Integer into Long Register
5609 instruct loadI2L(eRegL dst, memory mem, eFlagsReg cr) %{
5610   match(Set dst (ConvI2L (LoadI mem)));
5611   effect(KILL cr);
5612 
5613   ins_cost(375);
5614   format %{ "MOV    $dst.lo,$mem\t# int -> long\n\t"
5615             "MOV    $dst.hi,$dst.lo\n\t"
5616             "SAR    $dst.hi,31" %}
5617 
5618   ins_encode %{
5619     __ movl($dst$$Register, $mem$$Address);
5620     __ movl(HIGH_FROM_LOW($dst$$Register), $dst$$Register); // This is always a different register.
5621     __ sarl(HIGH_FROM_LOW($dst$$Register), 31);
5622   %}
5623 
5624   ins_pipe(ialu_reg_mem);
5625 %}
5626 
5627 // Load Integer with mask 0xFF into Long Register
5628 instruct loadI2L_immI_255(eRegL dst, memory mem, immI_255 mask, eFlagsReg cr) %{
5629   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
5630   effect(KILL cr);
5631 
5632   format %{ "MOVZX8 $dst.lo,$mem\t# int & 0xFF -> long\n\t"
5633             "XOR    $dst.hi,$dst.hi" %}
5634   ins_encode %{
5635     Register Rdst = $dst$$Register;
5636     __ movzbl(Rdst, $mem$$Address);
5637     __ xorl(HIGH_FROM_LOW(Rdst), HIGH_FROM_LOW(Rdst));
5638   %}
5639   ins_pipe(ialu_reg_mem);
5640 %}
5641 
5642 // Load Integer with mask 0xFFFF into Long Register
5643 instruct loadI2L_immI_65535(eRegL dst, memory mem, immI_65535 mask, eFlagsReg cr) %{
5644   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
5645   effect(KILL cr);
5646 
5647   format %{ "MOVZX  $dst.lo,$mem\t# int & 0xFFFF -> long\n\t"
5648             "XOR    $dst.hi,$dst.hi" %}
5649   ins_encode %{
5650     Register Rdst = $dst$$Register;
5651     __ movzwl(Rdst, $mem$$Address);
5652     __ xorl(HIGH_FROM_LOW(Rdst), HIGH_FROM_LOW(Rdst));
5653   %}
5654   ins_pipe(ialu_reg_mem);
5655 %}
5656 
5657 // Load Integer with 31-bit mask into Long Register
5658 instruct loadI2L_immU31(eRegL dst, memory mem, immU31 mask, eFlagsReg cr) %{
5659   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
5660   effect(KILL cr);
5661 
5662   format %{ "MOV    $dst.lo,$mem\t# int & 31-bit mask -> long\n\t"
5663             "XOR    $dst.hi,$dst.hi\n\t"
5664             "AND    $dst.lo,$mask" %}
5665   ins_encode %{
5666     Register Rdst = $dst$$Register;
5667     __ movl(Rdst, $mem$$Address);
5668     __ xorl(HIGH_FROM_LOW(Rdst), HIGH_FROM_LOW(Rdst));
5669     __ andl(Rdst, $mask$$constant);
5670   %}
5671   ins_pipe(ialu_reg_mem);
5672 %}
5673 
5674 // Load Unsigned Integer into Long Register
5675 instruct loadUI2L(eRegL dst, memory mem, immL_32bits mask, eFlagsReg cr) %{
5676   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
5677   effect(KILL cr);
5678 
5679   ins_cost(250);
5680   format %{ "MOV    $dst.lo,$mem\t# uint -> long\n\t"
5681             "XOR    $dst.hi,$dst.hi" %}
5682 
5683   ins_encode %{
5684     __ movl($dst$$Register, $mem$$Address);
5685     __ xorl(HIGH_FROM_LOW($dst$$Register), HIGH_FROM_LOW($dst$$Register));
5686   %}
5687 
5688   ins_pipe(ialu_reg_mem);
5689 %}
5690 
5691 // Load Long.  Cannot clobber address while loading, so restrict address
5692 // register to ESI
5693 instruct loadL(eRegL dst, load_long_memory mem) %{
5694   predicate(!((LoadLNode*)n)->require_atomic_access());
5695   match(Set dst (LoadL mem));
5696 
5697   ins_cost(250);
5698   format %{ "MOV    $dst.lo,$mem\t# long\n\t"
5699             "MOV    $dst.hi,$mem+4" %}
5700 
5701   ins_encode %{
5702     Address Amemlo = Address::make_raw($mem$$base, $mem$$index, $mem$$scale, $mem$$disp, relocInfo::none);
5703     Address Amemhi = Address::make_raw($mem$$base, $mem$$index, $mem$$scale, $mem$$disp + 4, relocInfo::none);
5704     __ movl($dst$$Register, Amemlo);
5705     __ movl(HIGH_FROM_LOW($dst$$Register), Amemhi);
5706   %}
5707 
5708   ins_pipe(ialu_reg_long_mem);
5709 %}
5710 
5711 // Volatile Load Long.  Must be atomic, so do 64-bit FILD
5712 // then store it down to the stack and reload on the int
5713 // side.
5714 instruct loadL_volatile(stackSlotL dst, memory mem) %{
5715   predicate(UseSSE<=1 && ((LoadLNode*)n)->require_atomic_access());
5716   match(Set dst (LoadL mem));
5717 
5718   ins_cost(200);
5719   format %{ "FILD   $mem\t# Atomic volatile long load\n\t"
5720             "FISTp  $dst" %}
5721   ins_encode(enc_loadL_volatile(mem,dst));
5722   ins_pipe( fpu_reg_mem );
5723 %}
5724 
5725 instruct loadLX_volatile(stackSlotL dst, memory mem, regD tmp) %{
5726   predicate(UseSSE>=2 && ((LoadLNode*)n)->require_atomic_access());
5727   match(Set dst (LoadL mem));
5728   effect(TEMP tmp);
5729   ins_cost(180);
5730   format %{ "MOVSD  $tmp,$mem\t# Atomic volatile long load\n\t"
5731             "MOVSD  $dst,$tmp" %}
5732   ins_encode %{
5733     __ movdbl($tmp$$XMMRegister, $mem$$Address);
5734     __ movdbl(Address(rsp, $dst$$disp), $tmp$$XMMRegister);
5735   %}
5736   ins_pipe( pipe_slow );
5737 %}
5738 
5739 instruct loadLX_reg_volatile(eRegL dst, memory mem, regD tmp) %{
5740   predicate(UseSSE>=2 && ((LoadLNode*)n)->require_atomic_access());
5741   match(Set dst (LoadL mem));
5742   effect(TEMP tmp);
5743   ins_cost(160);
5744   format %{ "MOVSD  $tmp,$mem\t# Atomic volatile long load\n\t"
5745             "MOVD   $dst.lo,$tmp\n\t"
5746             "PSRLQ  $tmp,32\n\t"
5747             "MOVD   $dst.hi,$tmp" %}
5748   ins_encode %{
5749     __ movdbl($tmp$$XMMRegister, $mem$$Address);
5750     __ movdl($dst$$Register, $tmp$$XMMRegister);
5751     __ psrlq($tmp$$XMMRegister, 32);
5752     __ movdl(HIGH_FROM_LOW($dst$$Register), $tmp$$XMMRegister);
5753   %}
5754   ins_pipe( pipe_slow );
5755 %}
5756 
5757 // Load Range
5758 instruct loadRange(rRegI dst, memory mem) %{
5759   match(Set dst (LoadRange mem));
5760 
5761   ins_cost(125);
5762   format %{ "MOV    $dst,$mem" %}
5763   opcode(0x8B);
5764   ins_encode( OpcP, RegMem(dst,mem));
5765   ins_pipe( ialu_reg_mem );
5766 %}
5767 
5768 
5769 // Load Pointer
5770 instruct loadP(eRegP dst, memory mem) %{
5771   match(Set dst (LoadP mem));
5772 
5773   ins_cost(125);
5774   format %{ "MOV    $dst,$mem" %}
5775   opcode(0x8B);
5776   ins_encode( OpcP, RegMem(dst,mem));
5777   ins_pipe( ialu_reg_mem );
5778 %}
5779 
5780 // Load Klass Pointer
5781 instruct loadKlass(eRegP dst, memory mem) %{
5782   match(Set dst (LoadKlass mem));
5783 
5784   ins_cost(125);
5785   format %{ "MOV    $dst,$mem" %}
5786   opcode(0x8B);
5787   ins_encode( OpcP, RegMem(dst,mem));
5788   ins_pipe( ialu_reg_mem );
5789 %}
5790 
5791 // Load Double
5792 instruct loadDPR(regDPR dst, memory mem) %{
5793   predicate(UseSSE<=1);
5794   match(Set dst (LoadD mem));
5795 
5796   ins_cost(150);
5797   format %{ "FLD_D  ST,$mem\n\t"
5798             "FSTP   $dst" %}
5799   opcode(0xDD);               /* DD /0 */
5800   ins_encode( OpcP, RMopc_Mem(0x00,mem),
5801               Pop_Reg_DPR(dst) );
5802   ins_pipe( fpu_reg_mem );
5803 %}
5804 
5805 // Load Double to XMM
5806 instruct loadD(regD dst, memory mem) %{
5807   predicate(UseSSE>=2 && UseXmmLoadAndClearUpper);
5808   match(Set dst (LoadD mem));
5809   ins_cost(145);
5810   format %{ "MOVSD  $dst,$mem" %}
5811   ins_encode %{
5812     __ movdbl ($dst$$XMMRegister, $mem$$Address);
5813   %}
5814   ins_pipe( pipe_slow );
5815 %}
5816 
5817 instruct loadD_partial(regD dst, memory mem) %{
5818   predicate(UseSSE>=2 && !UseXmmLoadAndClearUpper);
5819   match(Set dst (LoadD mem));
5820   ins_cost(145);
5821   format %{ "MOVLPD $dst,$mem" %}
5822   ins_encode %{
5823     __ movdbl ($dst$$XMMRegister, $mem$$Address);
5824   %}
5825   ins_pipe( pipe_slow );
5826 %}
5827 
5828 // Load to XMM register (single-precision floating point)
5829 // MOVSS instruction
5830 instruct loadF(regF dst, memory mem) %{
5831   predicate(UseSSE>=1);
5832   match(Set dst (LoadF mem));
5833   ins_cost(145);
5834   format %{ "MOVSS  $dst,$mem" %}
5835   ins_encode %{
5836     __ movflt ($dst$$XMMRegister, $mem$$Address);
5837   %}
5838   ins_pipe( pipe_slow );
5839 %}
5840 
5841 // Load Float
5842 instruct loadFPR(regFPR dst, memory mem) %{
5843   predicate(UseSSE==0);
5844   match(Set dst (LoadF mem));
5845 
5846   ins_cost(150);
5847   format %{ "FLD_S  ST,$mem\n\t"
5848             "FSTP   $dst" %}
5849   opcode(0xD9);               /* D9 /0 */
5850   ins_encode( OpcP, RMopc_Mem(0x00,mem),
5851               Pop_Reg_FPR(dst) );
5852   ins_pipe( fpu_reg_mem );
5853 %}
5854 
5855 // Load Effective Address
5856 instruct leaP8(eRegP dst, indOffset8 mem) %{
5857   match(Set dst mem);
5858 
5859   ins_cost(110);
5860   format %{ "LEA    $dst,$mem" %}
5861   opcode(0x8D);
5862   ins_encode( OpcP, RegMem(dst,mem));
5863   ins_pipe( ialu_reg_reg_fat );
5864 %}
5865 
5866 instruct leaP32(eRegP dst, indOffset32 mem) %{
5867   match(Set dst mem);
5868 
5869   ins_cost(110);
5870   format %{ "LEA    $dst,$mem" %}
5871   opcode(0x8D);
5872   ins_encode( OpcP, RegMem(dst,mem));
5873   ins_pipe( ialu_reg_reg_fat );
5874 %}
5875 
5876 instruct leaPIdxOff(eRegP dst, indIndexOffset mem) %{
5877   match(Set dst mem);
5878 
5879   ins_cost(110);
5880   format %{ "LEA    $dst,$mem" %}
5881   opcode(0x8D);
5882   ins_encode( OpcP, RegMem(dst,mem));
5883   ins_pipe( ialu_reg_reg_fat );
5884 %}
5885 
5886 instruct leaPIdxScale(eRegP dst, indIndexScale mem) %{
5887   match(Set dst mem);
5888 
5889   ins_cost(110);
5890   format %{ "LEA    $dst,$mem" %}
5891   opcode(0x8D);
5892   ins_encode( OpcP, RegMem(dst,mem));
5893   ins_pipe( ialu_reg_reg_fat );
5894 %}
5895 
5896 instruct leaPIdxScaleOff(eRegP dst, indIndexScaleOffset mem) %{
5897   match(Set dst mem);
5898 
5899   ins_cost(110);
5900   format %{ "LEA    $dst,$mem" %}
5901   opcode(0x8D);
5902   ins_encode( OpcP, RegMem(dst,mem));
5903   ins_pipe( ialu_reg_reg_fat );
5904 %}
5905 
5906 // Load Constant
5907 instruct loadConI(rRegI dst, immI src) %{
5908   match(Set dst src);
5909 
5910   format %{ "MOV    $dst,$src" %}
5911   ins_encode( LdImmI(dst, src) );
5912   ins_pipe( ialu_reg_fat );
5913 %}
5914 
5915 // Load Constant zero
5916 instruct loadConI0(rRegI dst, immI0 src, eFlagsReg cr) %{
5917   match(Set dst src);
5918   effect(KILL cr);
5919 
5920   ins_cost(50);
5921   format %{ "XOR    $dst,$dst" %}
5922   opcode(0x33);  /* + rd */
5923   ins_encode( OpcP, RegReg( dst, dst ) );
5924   ins_pipe( ialu_reg );
5925 %}
5926 
5927 instruct loadConP(eRegP dst, immP src) %{
5928   match(Set dst src);
5929 
5930   format %{ "MOV    $dst,$src" %}
5931   opcode(0xB8);  /* + rd */
5932   ins_encode( LdImmP(dst, src) );
5933   ins_pipe( ialu_reg_fat );
5934 %}
5935 
5936 instruct loadConL(eRegL dst, immL src, eFlagsReg cr) %{
5937   match(Set dst src);
5938   effect(KILL cr);
5939   ins_cost(200);
5940   format %{ "MOV    $dst.lo,$src.lo\n\t"
5941             "MOV    $dst.hi,$src.hi" %}
5942   opcode(0xB8);
5943   ins_encode( LdImmL_Lo(dst, src), LdImmL_Hi(dst, src) );
5944   ins_pipe( ialu_reg_long_fat );
5945 %}
5946 
5947 instruct loadConL0(eRegL dst, immL0 src, eFlagsReg cr) %{
5948   match(Set dst src);
5949   effect(KILL cr);
5950   ins_cost(150);
5951   format %{ "XOR    $dst.lo,$dst.lo\n\t"
5952             "XOR    $dst.hi,$dst.hi" %}
5953   opcode(0x33,0x33);
5954   ins_encode( RegReg_Lo(dst,dst), RegReg_Hi(dst, dst) );
5955   ins_pipe( ialu_reg_long );
5956 %}
5957 
5958 // The instruction usage is guarded by predicate in operand immFPR().
5959 instruct loadConFPR(regFPR dst, immFPR con) %{
5960   match(Set dst con);
5961   ins_cost(125);
5962   format %{ "FLD_S  ST,[$constantaddress]\t# load from constant table: float=$con\n\t"
5963             "FSTP   $dst" %}
5964   ins_encode %{
5965     __ fld_s($constantaddress($con));
5966     __ fstp_d($dst$$reg);
5967   %}
5968   ins_pipe(fpu_reg_con);
5969 %}
5970 
5971 // The instruction usage is guarded by predicate in operand immFPR0().
5972 instruct loadConFPR0(regFPR dst, immFPR0 con) %{
5973   match(Set dst con);
5974   ins_cost(125);
5975   format %{ "FLDZ   ST\n\t"
5976             "FSTP   $dst" %}
5977   ins_encode %{
5978     __ fldz();
5979     __ fstp_d($dst$$reg);
5980   %}
5981   ins_pipe(fpu_reg_con);
5982 %}
5983 
5984 // The instruction usage is guarded by predicate in operand immFPR1().
5985 instruct loadConFPR1(regFPR dst, immFPR1 con) %{
5986   match(Set dst con);
5987   ins_cost(125);
5988   format %{ "FLD1   ST\n\t"
5989             "FSTP   $dst" %}
5990   ins_encode %{
5991     __ fld1();
5992     __ fstp_d($dst$$reg);
5993   %}
5994   ins_pipe(fpu_reg_con);
5995 %}
5996 
5997 // The instruction usage is guarded by predicate in operand immF().
5998 instruct loadConF(regF dst, immF con) %{
5999   match(Set dst con);
6000   ins_cost(125);
6001   format %{ "MOVSS  $dst,[$constantaddress]\t# load from constant table: float=$con" %}
6002   ins_encode %{
6003     __ movflt($dst$$XMMRegister, $constantaddress($con));
6004   %}
6005   ins_pipe(pipe_slow);
6006 %}
6007 
6008 // The instruction usage is guarded by predicate in operand immF0().
6009 instruct loadConF0(regF dst, immF0 src) %{
6010   match(Set dst src);
6011   ins_cost(100);
6012   format %{ "XORPS  $dst,$dst\t# float 0.0" %}
6013   ins_encode %{
6014     __ xorps($dst$$XMMRegister, $dst$$XMMRegister);
6015   %}
6016   ins_pipe(pipe_slow);
6017 %}
6018 
6019 // The instruction usage is guarded by predicate in operand immDPR().
6020 instruct loadConDPR(regDPR dst, immDPR con) %{
6021   match(Set dst con);
6022   ins_cost(125);
6023 
6024   format %{ "FLD_D  ST,[$constantaddress]\t# load from constant table: double=$con\n\t"
6025             "FSTP   $dst" %}
6026   ins_encode %{
6027     __ fld_d($constantaddress($con));
6028     __ fstp_d($dst$$reg);
6029   %}
6030   ins_pipe(fpu_reg_con);
6031 %}
6032 
6033 // The instruction usage is guarded by predicate in operand immDPR0().
6034 instruct loadConDPR0(regDPR dst, immDPR0 con) %{
6035   match(Set dst con);
6036   ins_cost(125);
6037 
6038   format %{ "FLDZ   ST\n\t"
6039             "FSTP   $dst" %}
6040   ins_encode %{
6041     __ fldz();
6042     __ fstp_d($dst$$reg);
6043   %}
6044   ins_pipe(fpu_reg_con);
6045 %}
6046 
6047 // The instruction usage is guarded by predicate in operand immDPR1().
6048 instruct loadConDPR1(regDPR dst, immDPR1 con) %{
6049   match(Set dst con);
6050   ins_cost(125);
6051 
6052   format %{ "FLD1   ST\n\t"
6053             "FSTP   $dst" %}
6054   ins_encode %{
6055     __ fld1();
6056     __ fstp_d($dst$$reg);
6057   %}
6058   ins_pipe(fpu_reg_con);
6059 %}
6060 
6061 // The instruction usage is guarded by predicate in operand immD().
6062 instruct loadConD(regD dst, immD con) %{
6063   match(Set dst con);
6064   ins_cost(125);
6065   format %{ "MOVSD  $dst,[$constantaddress]\t# load from constant table: double=$con" %}
6066   ins_encode %{
6067     __ movdbl($dst$$XMMRegister, $constantaddress($con));
6068   %}
6069   ins_pipe(pipe_slow);
6070 %}
6071 
6072 // The instruction usage is guarded by predicate in operand immD0().
6073 instruct loadConD0(regD dst, immD0 src) %{
6074   match(Set dst src);
6075   ins_cost(100);
6076   format %{ "XORPD  $dst,$dst\t# double 0.0" %}
6077   ins_encode %{
6078     __ xorpd ($dst$$XMMRegister, $dst$$XMMRegister);
6079   %}
6080   ins_pipe( pipe_slow );
6081 %}
6082 
6083 // Load Stack Slot
6084 instruct loadSSI(rRegI dst, stackSlotI src) %{
6085   match(Set dst src);
6086   ins_cost(125);
6087 
6088   format %{ "MOV    $dst,$src" %}
6089   opcode(0x8B);
6090   ins_encode( OpcP, RegMem(dst,src));
6091   ins_pipe( ialu_reg_mem );
6092 %}
6093 
6094 instruct loadSSL(eRegL dst, stackSlotL src) %{
6095   match(Set dst src);
6096 
6097   ins_cost(200);
6098   format %{ "MOV    $dst,$src.lo\n\t"
6099             "MOV    $dst+4,$src.hi" %}
6100   opcode(0x8B, 0x8B);
6101   ins_encode( OpcP, RegMem( dst, src ), OpcS, RegMem_Hi( dst, src ) );
6102   ins_pipe( ialu_mem_long_reg );
6103 %}
6104 
6105 // Load Stack Slot
6106 instruct loadSSP(eRegP dst, stackSlotP src) %{
6107   match(Set dst src);
6108   ins_cost(125);
6109 
6110   format %{ "MOV    $dst,$src" %}
6111   opcode(0x8B);
6112   ins_encode( OpcP, RegMem(dst,src));
6113   ins_pipe( ialu_reg_mem );
6114 %}
6115 
6116 // Load Stack Slot
6117 instruct loadSSF(regFPR dst, stackSlotF src) %{
6118   match(Set dst src);
6119   ins_cost(125);
6120 
6121   format %{ "FLD_S  $src\n\t"
6122             "FSTP   $dst" %}
6123   opcode(0xD9);               /* D9 /0, FLD m32real */
6124   ins_encode( OpcP, RMopc_Mem_no_oop(0x00,src),
6125               Pop_Reg_FPR(dst) );
6126   ins_pipe( fpu_reg_mem );
6127 %}
6128 
6129 // Load Stack Slot
6130 instruct loadSSD(regDPR dst, stackSlotD src) %{
6131   match(Set dst src);
6132   ins_cost(125);
6133 
6134   format %{ "FLD_D  $src\n\t"
6135             "FSTP   $dst" %}
6136   opcode(0xDD);               /* DD /0, FLD m64real */
6137   ins_encode( OpcP, RMopc_Mem_no_oop(0x00,src),
6138               Pop_Reg_DPR(dst) );
6139   ins_pipe( fpu_reg_mem );
6140 %}
6141 
6142 // Prefetch instructions.
6143 // Must be safe to execute with invalid address (cannot fault).
6144 
6145 instruct prefetchr0( memory mem ) %{
6146   predicate(UseSSE==0 && !VM_Version::supports_3dnow_prefetch());
6147   match(PrefetchRead mem);
6148   ins_cost(0);
6149   size(0);
6150   format %{ "PREFETCHR (non-SSE is empty encoding)" %}
6151   ins_encode();
6152   ins_pipe(empty);
6153 %}
6154 
6155 instruct prefetchr( memory mem ) %{
6156   predicate(UseSSE==0 && VM_Version::supports_3dnow_prefetch() || ReadPrefetchInstr==3);
6157   match(PrefetchRead mem);
6158   ins_cost(100);
6159 
6160   format %{ "PREFETCHR $mem\t! Prefetch into level 1 cache for read" %}
6161   ins_encode %{
6162     __ prefetchr($mem$$Address);
6163   %}
6164   ins_pipe(ialu_mem);
6165 %}
6166 
6167 instruct prefetchrNTA( memory mem ) %{
6168   predicate(UseSSE>=1 && ReadPrefetchInstr==0);
6169   match(PrefetchRead mem);
6170   ins_cost(100);
6171 
6172   format %{ "PREFETCHNTA $mem\t! Prefetch into non-temporal cache for read" %}
6173   ins_encode %{
6174     __ prefetchnta($mem$$Address);
6175   %}
6176   ins_pipe(ialu_mem);
6177 %}
6178 
6179 instruct prefetchrT0( memory mem ) %{
6180   predicate(UseSSE>=1 && ReadPrefetchInstr==1);
6181   match(PrefetchRead mem);
6182   ins_cost(100);
6183 
6184   format %{ "PREFETCHT0 $mem\t! Prefetch into L1 and L2 caches for read" %}
6185   ins_encode %{
6186     __ prefetcht0($mem$$Address);
6187   %}
6188   ins_pipe(ialu_mem);
6189 %}
6190 
6191 instruct prefetchrT2( memory mem ) %{
6192   predicate(UseSSE>=1 && ReadPrefetchInstr==2);
6193   match(PrefetchRead mem);
6194   ins_cost(100);
6195 
6196   format %{ "PREFETCHT2 $mem\t! Prefetch into L2 cache for read" %}
6197   ins_encode %{
6198     __ prefetcht2($mem$$Address);
6199   %}
6200   ins_pipe(ialu_mem);
6201 %}
6202 
6203 instruct prefetchw0( memory mem ) %{
6204   predicate(UseSSE==0 && !VM_Version::supports_3dnow_prefetch());
6205   match(PrefetchWrite mem);
6206   ins_cost(0);
6207   size(0);
6208   format %{ "Prefetch (non-SSE is empty encoding)" %}
6209   ins_encode();
6210   ins_pipe(empty);
6211 %}
6212 
6213 instruct prefetchw( memory mem ) %{
6214   predicate(UseSSE==0 && VM_Version::supports_3dnow_prefetch());
6215   match( PrefetchWrite mem );
6216   ins_cost(100);
6217 
6218   format %{ "PREFETCHW $mem\t! Prefetch into L1 cache and mark modified" %}
6219   ins_encode %{
6220     __ prefetchw($mem$$Address);
6221   %}
6222   ins_pipe(ialu_mem);
6223 %}
6224 
6225 instruct prefetchwNTA( memory mem ) %{
6226   predicate(UseSSE>=1);
6227   match(PrefetchWrite mem);
6228   ins_cost(100);
6229 
6230   format %{ "PREFETCHNTA $mem\t! Prefetch into non-temporal cache for write" %}
6231   ins_encode %{
6232     __ prefetchnta($mem$$Address);
6233   %}
6234   ins_pipe(ialu_mem);
6235 %}
6236 
6237 // Prefetch instructions for allocation.
6238 
6239 instruct prefetchAlloc0( memory mem ) %{
6240   predicate(UseSSE==0 && AllocatePrefetchInstr!=3);
6241   match(PrefetchAllocation mem);
6242   ins_cost(0);
6243   size(0);
6244   format %{ "Prefetch allocation (non-SSE is empty encoding)" %}
6245   ins_encode();
6246   ins_pipe(empty);
6247 %}
6248 
6249 instruct prefetchAlloc( memory mem ) %{
6250   predicate(AllocatePrefetchInstr==3);
6251   match( PrefetchAllocation mem );
6252   ins_cost(100);
6253 
6254   format %{ "PREFETCHW $mem\t! Prefetch allocation into L1 cache and mark modified" %}
6255   ins_encode %{
6256     __ prefetchw($mem$$Address);
6257   %}
6258   ins_pipe(ialu_mem);
6259 %}
6260 
6261 instruct prefetchAllocNTA( memory mem ) %{
6262   predicate(UseSSE>=1 && AllocatePrefetchInstr==0);
6263   match(PrefetchAllocation mem);
6264   ins_cost(100);
6265 
6266   format %{ "PREFETCHNTA $mem\t! Prefetch allocation into non-temporal cache for write" %}
6267   ins_encode %{
6268     __ prefetchnta($mem$$Address);
6269   %}
6270   ins_pipe(ialu_mem);
6271 %}
6272 
6273 instruct prefetchAllocT0( memory mem ) %{
6274   predicate(UseSSE>=1 && AllocatePrefetchInstr==1);
6275   match(PrefetchAllocation mem);
6276   ins_cost(100);
6277 
6278   format %{ "PREFETCHT0 $mem\t! Prefetch allocation into L1 and L2 caches for write" %}
6279   ins_encode %{
6280     __ prefetcht0($mem$$Address);
6281   %}
6282   ins_pipe(ialu_mem);
6283 %}
6284 
6285 instruct prefetchAllocT2( memory mem ) %{
6286   predicate(UseSSE>=1 && AllocatePrefetchInstr==2);
6287   match(PrefetchAllocation mem);
6288   ins_cost(100);
6289 
6290   format %{ "PREFETCHT2 $mem\t! Prefetch allocation into L2 cache for write" %}
6291   ins_encode %{
6292     __ prefetcht2($mem$$Address);
6293   %}
6294   ins_pipe(ialu_mem);
6295 %}
6296 
6297 //----------Store Instructions-------------------------------------------------
6298 
6299 // Store Byte
6300 instruct storeB(memory mem, xRegI src) %{
6301   match(Set mem (StoreB mem src));
6302 
6303   ins_cost(125);
6304   format %{ "MOV8   $mem,$src" %}
6305   opcode(0x88);
6306   ins_encode( OpcP, RegMem( src, mem ) );
6307   ins_pipe( ialu_mem_reg );
6308 %}
6309 
6310 // Store Char/Short
6311 instruct storeC(memory mem, rRegI src) %{
6312   match(Set mem (StoreC mem src));
6313 
6314   ins_cost(125);
6315   format %{ "MOV16  $mem,$src" %}
6316   opcode(0x89, 0x66);
6317   ins_encode( OpcS, OpcP, RegMem( src, mem ) );
6318   ins_pipe( ialu_mem_reg );
6319 %}
6320 
6321 // Store Integer
6322 instruct storeI(memory mem, rRegI src) %{
6323   match(Set mem (StoreI mem src));
6324 
6325   ins_cost(125);
6326   format %{ "MOV    $mem,$src" %}
6327   opcode(0x89);
6328   ins_encode( OpcP, RegMem( src, mem ) );
6329   ins_pipe( ialu_mem_reg );
6330 %}
6331 
6332 // Store Long
6333 instruct storeL(long_memory mem, eRegL src) %{
6334   predicate(!((StoreLNode*)n)->require_atomic_access());
6335   match(Set mem (StoreL mem src));
6336 
6337   ins_cost(200);
6338   format %{ "MOV    $mem,$src.lo\n\t"
6339             "MOV    $mem+4,$src.hi" %}
6340   opcode(0x89, 0x89);
6341   ins_encode( OpcP, RegMem( src, mem ), OpcS, RegMem_Hi( src, mem ) );
6342   ins_pipe( ialu_mem_long_reg );
6343 %}
6344 
6345 // Store Long to Integer
6346 instruct storeL2I(memory mem, eRegL src) %{
6347   match(Set mem (StoreI mem (ConvL2I src)));
6348 
6349   format %{ "MOV    $mem,$src.lo\t# long -> int" %}
6350   ins_encode %{
6351     __ movl($mem$$Address, $src$$Register);
6352   %}
6353   ins_pipe(ialu_mem_reg);
6354 %}
6355 
6356 // Volatile Store Long.  Must be atomic, so move it into
6357 // the FP TOS and then do a 64-bit FIST.  Has to probe the
6358 // target address before the store (for null-ptr checks)
6359 // so the memory operand is used twice in the encoding.
6360 instruct storeL_volatile(memory mem, stackSlotL src, eFlagsReg cr ) %{
6361   predicate(UseSSE<=1 && ((StoreLNode*)n)->require_atomic_access());
6362   match(Set mem (StoreL mem src));
6363   effect( KILL cr );
6364   ins_cost(400);
6365   format %{ "CMP    $mem,EAX\t# Probe address for implicit null check\n\t"
6366             "FILD   $src\n\t"
6367             "FISTp  $mem\t # 64-bit atomic volatile long store" %}
6368   opcode(0x3B);
6369   ins_encode( OpcP, RegMem( EAX, mem ), enc_storeL_volatile(mem,src));
6370   ins_pipe( fpu_reg_mem );
6371 %}
6372 
6373 instruct storeLX_volatile(memory mem, stackSlotL src, regD tmp, eFlagsReg cr) %{
6374   predicate(UseSSE>=2 && ((StoreLNode*)n)->require_atomic_access());
6375   match(Set mem (StoreL mem src));
6376   effect( TEMP tmp, KILL cr );
6377   ins_cost(380);
6378   format %{ "CMP    $mem,EAX\t# Probe address for implicit null check\n\t"
6379             "MOVSD  $tmp,$src\n\t"
6380             "MOVSD  $mem,$tmp\t # 64-bit atomic volatile long store" %}
6381   ins_encode %{
6382     __ cmpl(rax, $mem$$Address);
6383     __ movdbl($tmp$$XMMRegister, Address(rsp, $src$$disp));
6384     __ movdbl($mem$$Address, $tmp$$XMMRegister);
6385   %}
6386   ins_pipe( pipe_slow );
6387 %}
6388 
6389 instruct storeLX_reg_volatile(memory mem, eRegL src, regD tmp2, regD tmp, eFlagsReg cr) %{
6390   predicate(UseSSE>=2 && ((StoreLNode*)n)->require_atomic_access());
6391   match(Set mem (StoreL mem src));
6392   effect( TEMP tmp2 , TEMP tmp, KILL cr );
6393   ins_cost(360);
6394   format %{ "CMP    $mem,EAX\t# Probe address for implicit null check\n\t"
6395             "MOVD   $tmp,$src.lo\n\t"
6396             "MOVD   $tmp2,$src.hi\n\t"
6397             "PUNPCKLDQ $tmp,$tmp2\n\t"
6398             "MOVSD  $mem,$tmp\t # 64-bit atomic volatile long store" %}
6399   ins_encode %{
6400     __ cmpl(rax, $mem$$Address);
6401     __ movdl($tmp$$XMMRegister, $src$$Register);
6402     __ movdl($tmp2$$XMMRegister, HIGH_FROM_LOW($src$$Register));
6403     __ punpckldq($tmp$$XMMRegister, $tmp2$$XMMRegister);
6404     __ movdbl($mem$$Address, $tmp$$XMMRegister);
6405   %}
6406   ins_pipe( pipe_slow );
6407 %}
6408 
6409 // Store Pointer; for storing unknown oops and raw pointers
6410 instruct storeP(memory mem, anyRegP src) %{
6411   match(Set mem (StoreP mem src));
6412 
6413   ins_cost(125);
6414   format %{ "MOV    $mem,$src" %}
6415   opcode(0x89);
6416   ins_encode( OpcP, RegMem( src, mem ) );
6417   ins_pipe( ialu_mem_reg );
6418 %}
6419 
6420 // Store Integer Immediate
6421 instruct storeImmI(memory mem, immI src) %{
6422   match(Set mem (StoreI mem src));
6423 
6424   ins_cost(150);
6425   format %{ "MOV    $mem,$src" %}
6426   opcode(0xC7);               /* C7 /0 */
6427   ins_encode( OpcP, RMopc_Mem(0x00,mem),  Con32( src ));
6428   ins_pipe( ialu_mem_imm );
6429 %}
6430 
6431 // Store Short/Char Immediate
6432 instruct storeImmI16(memory mem, immI16 src) %{
6433   predicate(UseStoreImmI16);
6434   match(Set mem (StoreC mem src));
6435 
6436   ins_cost(150);
6437   format %{ "MOV16  $mem,$src" %}
6438   opcode(0xC7);     /* C7 /0 Same as 32 store immediate with prefix */
6439   ins_encode( SizePrefix, OpcP, RMopc_Mem(0x00,mem),  Con16( src ));
6440   ins_pipe( ialu_mem_imm );
6441 %}
6442 
6443 // Store Pointer Immediate; null pointers or constant oops that do not
6444 // need card-mark barriers.
6445 instruct storeImmP(memory mem, immP src) %{
6446   match(Set mem (StoreP mem src));
6447 
6448   ins_cost(150);
6449   format %{ "MOV    $mem,$src" %}
6450   opcode(0xC7);               /* C7 /0 */
6451   ins_encode( OpcP, RMopc_Mem(0x00,mem),  Con32( src ));
6452   ins_pipe( ialu_mem_imm );
6453 %}
6454 
6455 // Store Byte Immediate
6456 instruct storeImmB(memory mem, immI8 src) %{
6457   match(Set mem (StoreB mem src));
6458 
6459   ins_cost(150);
6460   format %{ "MOV8   $mem,$src" %}
6461   opcode(0xC6);               /* C6 /0 */
6462   ins_encode( OpcP, RMopc_Mem(0x00,mem),  Con8or32( src ));
6463   ins_pipe( ialu_mem_imm );
6464 %}
6465 
6466 // Store CMS card-mark Immediate
6467 instruct storeImmCM(memory mem, immI8 src) %{
6468   match(Set mem (StoreCM mem src));
6469 
6470   ins_cost(150);
6471   format %{ "MOV8   $mem,$src\t! CMS card-mark imm0" %}
6472   opcode(0xC6);               /* C6 /0 */
6473   ins_encode( OpcP, RMopc_Mem(0x00,mem),  Con8or32( src ));
6474   ins_pipe( ialu_mem_imm );
6475 %}
6476 
6477 // Store Double
6478 instruct storeDPR( memory mem, regDPR1 src) %{
6479   predicate(UseSSE<=1);
6480   match(Set mem (StoreD mem src));
6481 
6482   ins_cost(100);
6483   format %{ "FST_D  $mem,$src" %}
6484   opcode(0xDD);       /* DD /2 */
6485   ins_encode( enc_FPR_store(mem,src) );
6486   ins_pipe( fpu_mem_reg );
6487 %}
6488 
6489 // Store double does rounding on x86
6490 instruct storeDPR_rounded( memory mem, regDPR1 src) %{
6491   predicate(UseSSE<=1);
6492   match(Set mem (StoreD mem (RoundDouble src)));
6493 
6494   ins_cost(100);
6495   format %{ "FST_D  $mem,$src\t# round" %}
6496   opcode(0xDD);       /* DD /2 */
6497   ins_encode( enc_FPR_store(mem,src) );
6498   ins_pipe( fpu_mem_reg );
6499 %}
6500 
6501 // Store XMM register to memory (double-precision floating points)
6502 // MOVSD instruction
6503 instruct storeD(memory mem, regD src) %{
6504   predicate(UseSSE>=2);
6505   match(Set mem (StoreD mem src));
6506   ins_cost(95);
6507   format %{ "MOVSD  $mem,$src" %}
6508   ins_encode %{
6509     __ movdbl($mem$$Address, $src$$XMMRegister);
6510   %}
6511   ins_pipe( pipe_slow );
6512 %}
6513 
6514 // Store XMM register to memory (single-precision floating point)
6515 // MOVSS instruction
6516 instruct storeF(memory mem, regF src) %{
6517   predicate(UseSSE>=1);
6518   match(Set mem (StoreF mem src));
6519   ins_cost(95);
6520   format %{ "MOVSS  $mem,$src" %}
6521   ins_encode %{
6522     __ movflt($mem$$Address, $src$$XMMRegister);
6523   %}
6524   ins_pipe( pipe_slow );
6525 %}
6526 
6527 // Store Float
6528 instruct storeFPR( memory mem, regFPR1 src) %{
6529   predicate(UseSSE==0);
6530   match(Set mem (StoreF mem src));
6531 
6532   ins_cost(100);
6533   format %{ "FST_S  $mem,$src" %}
6534   opcode(0xD9);       /* D9 /2 */
6535   ins_encode( enc_FPR_store(mem,src) );
6536   ins_pipe( fpu_mem_reg );
6537 %}
6538 
6539 // Store Float does rounding on x86
6540 instruct storeFPR_rounded( memory mem, regFPR1 src) %{
6541   predicate(UseSSE==0);
6542   match(Set mem (StoreF mem (RoundFloat src)));
6543 
6544   ins_cost(100);
6545   format %{ "FST_S  $mem,$src\t# round" %}
6546   opcode(0xD9);       /* D9 /2 */
6547   ins_encode( enc_FPR_store(mem,src) );
6548   ins_pipe( fpu_mem_reg );
6549 %}
6550 
6551 // Store Float does rounding on x86
6552 instruct storeFPR_Drounded( memory mem, regDPR1 src) %{
6553   predicate(UseSSE<=1);
6554   match(Set mem (StoreF mem (ConvD2F src)));
6555 
6556   ins_cost(100);
6557   format %{ "FST_S  $mem,$src\t# D-round" %}
6558   opcode(0xD9);       /* D9 /2 */
6559   ins_encode( enc_FPR_store(mem,src) );
6560   ins_pipe( fpu_mem_reg );
6561 %}
6562 
6563 // Store immediate Float value (it is faster than store from FPU register)
6564 // The instruction usage is guarded by predicate in operand immFPR().
6565 instruct storeFPR_imm( memory mem, immFPR src) %{
6566   match(Set mem (StoreF mem src));
6567 
6568   ins_cost(50);
6569   format %{ "MOV    $mem,$src\t# store float" %}
6570   opcode(0xC7);               /* C7 /0 */
6571   ins_encode( OpcP, RMopc_Mem(0x00,mem),  Con32FPR_as_bits( src ));
6572   ins_pipe( ialu_mem_imm );
6573 %}
6574 
6575 // Store immediate Float value (it is faster than store from XMM register)
6576 // The instruction usage is guarded by predicate in operand immF().
6577 instruct storeF_imm( memory mem, immF src) %{
6578   match(Set mem (StoreF mem src));
6579 
6580   ins_cost(50);
6581   format %{ "MOV    $mem,$src\t# store float" %}
6582   opcode(0xC7);               /* C7 /0 */
6583   ins_encode( OpcP, RMopc_Mem(0x00,mem),  Con32F_as_bits( src ));
6584   ins_pipe( ialu_mem_imm );
6585 %}
6586 
6587 // Store Integer to stack slot
6588 instruct storeSSI(stackSlotI dst, rRegI src) %{
6589   match(Set dst src);
6590 
6591   ins_cost(100);
6592   format %{ "MOV    $dst,$src" %}
6593   opcode(0x89);
6594   ins_encode( OpcPRegSS( dst, src ) );
6595   ins_pipe( ialu_mem_reg );
6596 %}
6597 
6598 // Store Integer to stack slot
6599 instruct storeSSP(stackSlotP dst, eRegP src) %{
6600   match(Set dst src);
6601 
6602   ins_cost(100);
6603   format %{ "MOV    $dst,$src" %}
6604   opcode(0x89);
6605   ins_encode( OpcPRegSS( dst, src ) );
6606   ins_pipe( ialu_mem_reg );
6607 %}
6608 
6609 // Store Long to stack slot
6610 instruct storeSSL(stackSlotL dst, eRegL src) %{
6611   match(Set dst src);
6612 
6613   ins_cost(200);
6614   format %{ "MOV    $dst,$src.lo\n\t"
6615             "MOV    $dst+4,$src.hi" %}
6616   opcode(0x89, 0x89);
6617   ins_encode( OpcP, RegMem( src, dst ), OpcS, RegMem_Hi( src, dst ) );
6618   ins_pipe( ialu_mem_long_reg );
6619 %}
6620 
6621 //----------MemBar Instructions-----------------------------------------------
6622 // Memory barrier flavors
6623 
6624 instruct membar_acquire() %{
6625   match(MemBarAcquire);
6626   match(LoadFence);
6627   ins_cost(400);
6628 
6629   size(0);
6630   format %{ "MEMBAR-acquire ! (empty encoding)" %}
6631   ins_encode();
6632   ins_pipe(empty);
6633 %}
6634 
6635 instruct membar_acquire_lock() %{
6636   match(MemBarAcquireLock);
6637   ins_cost(0);
6638 
6639   size(0);
6640   format %{ "MEMBAR-acquire (prior CMPXCHG in FastLock so empty encoding)" %}
6641   ins_encode( );
6642   ins_pipe(empty);
6643 %}
6644 
6645 instruct membar_release() %{
6646   match(MemBarRelease);
6647   match(StoreFence);
6648   ins_cost(400);
6649 
6650   size(0);
6651   format %{ "MEMBAR-release ! (empty encoding)" %}
6652   ins_encode( );
6653   ins_pipe(empty);
6654 %}
6655 
6656 instruct membar_release_lock() %{
6657   match(MemBarReleaseLock);
6658   ins_cost(0);
6659 
6660   size(0);
6661   format %{ "MEMBAR-release (a FastUnlock follows so empty encoding)" %}
6662   ins_encode( );
6663   ins_pipe(empty);
6664 %}
6665 
6666 instruct membar_volatile(eFlagsReg cr) %{
6667   match(MemBarVolatile);
6668   effect(KILL cr);
6669   ins_cost(400);
6670 
6671   format %{ 
6672     $$template
6673     if (os::is_MP()) {
6674       $$emit$$"LOCK ADDL [ESP + #0], 0\t! membar_volatile"
6675     } else {
6676       $$emit$$"MEMBAR-volatile ! (empty encoding)"
6677     }
6678   %}
6679   ins_encode %{
6680     __ membar(Assembler::StoreLoad);
6681   %}
6682   ins_pipe(pipe_slow);
6683 %}
6684 
6685 instruct unnecessary_membar_volatile() %{
6686   match(MemBarVolatile);
6687   predicate(Matcher::post_store_load_barrier(n));
6688   ins_cost(0);
6689 
6690   size(0);
6691   format %{ "MEMBAR-volatile (unnecessary so empty encoding)" %}
6692   ins_encode( );
6693   ins_pipe(empty);
6694 %}
6695 
6696 instruct membar_storestore() %{
6697   match(MemBarStoreStore);
6698   ins_cost(0);
6699 
6700   size(0);
6701   format %{ "MEMBAR-storestore (empty encoding)" %}
6702   ins_encode( );
6703   ins_pipe(empty);
6704 %}
6705 
6706 //----------Move Instructions--------------------------------------------------
6707 instruct castX2P(eAXRegP dst, eAXRegI src) %{
6708   match(Set dst (CastX2P src));
6709   format %{ "# X2P  $dst, $src" %}
6710   ins_encode( /*empty encoding*/ );
6711   ins_cost(0);
6712   ins_pipe(empty);
6713 %}
6714 
6715 instruct castP2X(rRegI dst, eRegP src ) %{
6716   match(Set dst (CastP2X src));
6717   ins_cost(50);
6718   format %{ "MOV    $dst, $src\t# CastP2X" %}
6719   ins_encode( enc_Copy( dst, src) );
6720   ins_pipe( ialu_reg_reg );
6721 %}
6722 
6723 //----------Conditional Move---------------------------------------------------
6724 // Conditional move
6725 instruct jmovI_reg(cmpOp cop, eFlagsReg cr, rRegI dst, rRegI src) %{
6726   predicate(!VM_Version::supports_cmov() );
6727   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
6728   ins_cost(200);
6729   format %{ "J$cop,us skip\t# signed cmove\n\t"
6730             "MOV    $dst,$src\n"
6731       "skip:" %}
6732   ins_encode %{
6733     Label Lskip;
6734     // Invert sense of branch from sense of CMOV
6735     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
6736     __ movl($dst$$Register, $src$$Register);
6737     __ bind(Lskip);
6738   %}
6739   ins_pipe( pipe_cmov_reg );
6740 %}
6741 
6742 instruct jmovI_regU(cmpOpU cop, eFlagsRegU cr, rRegI dst, rRegI src) %{
6743   predicate(!VM_Version::supports_cmov() );
6744   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
6745   ins_cost(200);
6746   format %{ "J$cop,us skip\t# unsigned cmove\n\t"
6747             "MOV    $dst,$src\n"
6748       "skip:" %}
6749   ins_encode %{
6750     Label Lskip;
6751     // Invert sense of branch from sense of CMOV
6752     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
6753     __ movl($dst$$Register, $src$$Register);
6754     __ bind(Lskip);
6755   %}
6756   ins_pipe( pipe_cmov_reg );
6757 %}
6758 
6759 instruct cmovI_reg(rRegI dst, rRegI src, eFlagsReg cr, cmpOp cop ) %{
6760   predicate(VM_Version::supports_cmov() );
6761   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
6762   ins_cost(200);
6763   format %{ "CMOV$cop $dst,$src" %}
6764   opcode(0x0F,0x40);
6765   ins_encode( enc_cmov(cop), RegReg( dst, src ) );
6766   ins_pipe( pipe_cmov_reg );
6767 %}
6768 
6769 instruct cmovI_regU( cmpOpU cop, eFlagsRegU cr, rRegI dst, rRegI src ) %{
6770   predicate(VM_Version::supports_cmov() );
6771   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
6772   ins_cost(200);
6773   format %{ "CMOV$cop $dst,$src" %}
6774   opcode(0x0F,0x40);
6775   ins_encode( enc_cmov(cop), RegReg( dst, src ) );
6776   ins_pipe( pipe_cmov_reg );
6777 %}
6778 
6779 instruct cmovI_regUCF( cmpOpUCF cop, eFlagsRegUCF cr, rRegI dst, rRegI src ) %{
6780   predicate(VM_Version::supports_cmov() );
6781   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
6782   ins_cost(200);
6783   expand %{
6784     cmovI_regU(cop, cr, dst, src);
6785   %}
6786 %}
6787 
6788 // Conditional move
6789 instruct cmovI_mem(cmpOp cop, eFlagsReg cr, rRegI dst, memory src) %{
6790   predicate(VM_Version::supports_cmov() );
6791   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
6792   ins_cost(250);
6793   format %{ "CMOV$cop $dst,$src" %}
6794   opcode(0x0F,0x40);
6795   ins_encode( enc_cmov(cop), RegMem( dst, src ) );
6796   ins_pipe( pipe_cmov_mem );
6797 %}
6798 
6799 // Conditional move
6800 instruct cmovI_memU(cmpOpU cop, eFlagsRegU cr, rRegI dst, memory src) %{
6801   predicate(VM_Version::supports_cmov() );
6802   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
6803   ins_cost(250);
6804   format %{ "CMOV$cop $dst,$src" %}
6805   opcode(0x0F,0x40);
6806   ins_encode( enc_cmov(cop), RegMem( dst, src ) );
6807   ins_pipe( pipe_cmov_mem );
6808 %}
6809 
6810 instruct cmovI_memUCF(cmpOpUCF cop, eFlagsRegUCF cr, rRegI dst, memory src) %{
6811   predicate(VM_Version::supports_cmov() );
6812   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
6813   ins_cost(250);
6814   expand %{
6815     cmovI_memU(cop, cr, dst, src);
6816   %}
6817 %}
6818 
6819 // Conditional move
6820 instruct cmovP_reg(eRegP dst, eRegP src, eFlagsReg cr, cmpOp cop ) %{
6821   predicate(VM_Version::supports_cmov() );
6822   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
6823   ins_cost(200);
6824   format %{ "CMOV$cop $dst,$src\t# ptr" %}
6825   opcode(0x0F,0x40);
6826   ins_encode( enc_cmov(cop), RegReg( dst, src ) );
6827   ins_pipe( pipe_cmov_reg );
6828 %}
6829 
6830 // Conditional move (non-P6 version)
6831 // Note:  a CMoveP is generated for  stubs and native wrappers
6832 //        regardless of whether we are on a P6, so we
6833 //        emulate a cmov here
6834 instruct cmovP_reg_nonP6(eRegP dst, eRegP src, eFlagsReg cr, cmpOp cop ) %{
6835   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
6836   ins_cost(300);
6837   format %{ "Jn$cop   skip\n\t"
6838           "MOV    $dst,$src\t# pointer\n"
6839       "skip:" %}
6840   opcode(0x8b);
6841   ins_encode( enc_cmov_branch(cop, 0x2), OpcP, RegReg(dst, src));
6842   ins_pipe( pipe_cmov_reg );
6843 %}
6844 
6845 // Conditional move
6846 instruct cmovP_regU(cmpOpU cop, eFlagsRegU cr, eRegP dst, eRegP src ) %{
6847   predicate(VM_Version::supports_cmov() );
6848   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
6849   ins_cost(200);
6850   format %{ "CMOV$cop $dst,$src\t# ptr" %}
6851   opcode(0x0F,0x40);
6852   ins_encode( enc_cmov(cop), RegReg( dst, src ) );
6853   ins_pipe( pipe_cmov_reg );
6854 %}
6855 
6856 instruct cmovP_regUCF(cmpOpUCF cop, eFlagsRegUCF cr, eRegP dst, eRegP src ) %{
6857   predicate(VM_Version::supports_cmov() );
6858   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
6859   ins_cost(200);
6860   expand %{
6861     cmovP_regU(cop, cr, dst, src);
6862   %}
6863 %}
6864 
6865 // DISABLED: Requires the ADLC to emit a bottom_type call that
6866 // correctly meets the two pointer arguments; one is an incoming
6867 // register but the other is a memory operand.  ALSO appears to
6868 // be buggy with implicit null checks.
6869 //
6870 //// Conditional move
6871 //instruct cmovP_mem(cmpOp cop, eFlagsReg cr, eRegP dst, memory src) %{
6872 //  predicate(VM_Version::supports_cmov() );
6873 //  match(Set dst (CMoveP (Binary cop cr) (Binary dst (LoadP src))));
6874 //  ins_cost(250);
6875 //  format %{ "CMOV$cop $dst,$src\t# ptr" %}
6876 //  opcode(0x0F,0x40);
6877 //  ins_encode( enc_cmov(cop), RegMem( dst, src ) );
6878 //  ins_pipe( pipe_cmov_mem );
6879 //%}
6880 //
6881 //// Conditional move
6882 //instruct cmovP_memU(cmpOpU cop, eFlagsRegU cr, eRegP dst, memory src) %{
6883 //  predicate(VM_Version::supports_cmov() );
6884 //  match(Set dst (CMoveP (Binary cop cr) (Binary dst (LoadP src))));
6885 //  ins_cost(250);
6886 //  format %{ "CMOV$cop $dst,$src\t# ptr" %}
6887 //  opcode(0x0F,0x40);
6888 //  ins_encode( enc_cmov(cop), RegMem( dst, src ) );
6889 //  ins_pipe( pipe_cmov_mem );
6890 //%}
6891 
6892 // Conditional move
6893 instruct fcmovDPR_regU(cmpOp_fcmov cop, eFlagsRegU cr, regDPR1 dst, regDPR src) %{
6894   predicate(UseSSE<=1);
6895   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
6896   ins_cost(200);
6897   format %{ "FCMOV$cop $dst,$src\t# double" %}
6898   opcode(0xDA);
6899   ins_encode( enc_cmov_dpr(cop,src) );
6900   ins_pipe( pipe_cmovDPR_reg );
6901 %}
6902 
6903 // Conditional move
6904 instruct fcmovFPR_regU(cmpOp_fcmov cop, eFlagsRegU cr, regFPR1 dst, regFPR src) %{
6905   predicate(UseSSE==0);
6906   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
6907   ins_cost(200);
6908   format %{ "FCMOV$cop $dst,$src\t# float" %}
6909   opcode(0xDA);
6910   ins_encode( enc_cmov_dpr(cop,src) );
6911   ins_pipe( pipe_cmovDPR_reg );
6912 %}
6913 
6914 // Float CMOV on Intel doesn't handle *signed* compares, only unsigned.
6915 instruct fcmovDPR_regS(cmpOp cop, eFlagsReg cr, regDPR dst, regDPR src) %{
6916   predicate(UseSSE<=1);
6917   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
6918   ins_cost(200);
6919   format %{ "Jn$cop   skip\n\t"
6920             "MOV    $dst,$src\t# double\n"
6921       "skip:" %}
6922   opcode (0xdd, 0x3);     /* DD D8+i or DD /3 */
6923   ins_encode( enc_cmov_branch( cop, 0x4 ), Push_Reg_DPR(src), OpcP, RegOpc(dst) );
6924   ins_pipe( pipe_cmovDPR_reg );
6925 %}
6926 
6927 // Float CMOV on Intel doesn't handle *signed* compares, only unsigned.
6928 instruct fcmovFPR_regS(cmpOp cop, eFlagsReg cr, regFPR dst, regFPR src) %{
6929   predicate(UseSSE==0);
6930   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
6931   ins_cost(200);
6932   format %{ "Jn$cop    skip\n\t"
6933             "MOV    $dst,$src\t# float\n"
6934       "skip:" %}
6935   opcode (0xdd, 0x3);     /* DD D8+i or DD /3 */
6936   ins_encode( enc_cmov_branch( cop, 0x4 ), Push_Reg_FPR(src), OpcP, RegOpc(dst) );
6937   ins_pipe( pipe_cmovDPR_reg );
6938 %}
6939 
6940 // No CMOVE with SSE/SSE2
6941 instruct fcmovF_regS(cmpOp cop, eFlagsReg cr, regF dst, regF src) %{
6942   predicate (UseSSE>=1);
6943   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
6944   ins_cost(200);
6945   format %{ "Jn$cop   skip\n\t"
6946             "MOVSS  $dst,$src\t# float\n"
6947       "skip:" %}
6948   ins_encode %{
6949     Label skip;
6950     // Invert sense of branch from sense of CMOV
6951     __ jccb((Assembler::Condition)($cop$$cmpcode^1), skip);
6952     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
6953     __ bind(skip);
6954   %}
6955   ins_pipe( pipe_slow );
6956 %}
6957 
6958 // No CMOVE with SSE/SSE2
6959 instruct fcmovD_regS(cmpOp cop, eFlagsReg cr, regD dst, regD src) %{
6960   predicate (UseSSE>=2);
6961   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
6962   ins_cost(200);
6963   format %{ "Jn$cop   skip\n\t"
6964             "MOVSD  $dst,$src\t# float\n"
6965       "skip:" %}
6966   ins_encode %{
6967     Label skip;
6968     // Invert sense of branch from sense of CMOV
6969     __ jccb((Assembler::Condition)($cop$$cmpcode^1), skip);
6970     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
6971     __ bind(skip);
6972   %}
6973   ins_pipe( pipe_slow );
6974 %}
6975 
6976 // unsigned version
6977 instruct fcmovF_regU(cmpOpU cop, eFlagsRegU cr, regF dst, regF src) %{
6978   predicate (UseSSE>=1);
6979   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
6980   ins_cost(200);
6981   format %{ "Jn$cop   skip\n\t"
6982             "MOVSS  $dst,$src\t# float\n"
6983       "skip:" %}
6984   ins_encode %{
6985     Label skip;
6986     // Invert sense of branch from sense of CMOV
6987     __ jccb((Assembler::Condition)($cop$$cmpcode^1), skip);
6988     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
6989     __ bind(skip);
6990   %}
6991   ins_pipe( pipe_slow );
6992 %}
6993 
6994 instruct fcmovF_regUCF(cmpOpUCF cop, eFlagsRegUCF cr, regF dst, regF src) %{
6995   predicate (UseSSE>=1);
6996   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
6997   ins_cost(200);
6998   expand %{
6999     fcmovF_regU(cop, cr, dst, src);
7000   %}
7001 %}
7002 
7003 // unsigned version
7004 instruct fcmovD_regU(cmpOpU cop, eFlagsRegU cr, regD dst, regD src) %{
7005   predicate (UseSSE>=2);
7006   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
7007   ins_cost(200);
7008   format %{ "Jn$cop   skip\n\t"
7009             "MOVSD  $dst,$src\t# float\n"
7010       "skip:" %}
7011   ins_encode %{
7012     Label skip;
7013     // Invert sense of branch from sense of CMOV
7014     __ jccb((Assembler::Condition)($cop$$cmpcode^1), skip);
7015     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
7016     __ bind(skip);
7017   %}
7018   ins_pipe( pipe_slow );
7019 %}
7020 
7021 instruct fcmovD_regUCF(cmpOpUCF cop, eFlagsRegUCF cr, regD dst, regD src) %{
7022   predicate (UseSSE>=2);
7023   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
7024   ins_cost(200);
7025   expand %{
7026     fcmovD_regU(cop, cr, dst, src);
7027   %}
7028 %}
7029 
7030 instruct cmovL_reg(cmpOp cop, eFlagsReg cr, eRegL dst, eRegL src) %{
7031   predicate(VM_Version::supports_cmov() );
7032   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
7033   ins_cost(200);
7034   format %{ "CMOV$cop $dst.lo,$src.lo\n\t"
7035             "CMOV$cop $dst.hi,$src.hi" %}
7036   opcode(0x0F,0x40);
7037   ins_encode( enc_cmov(cop), RegReg_Lo2( dst, src ), enc_cmov(cop), RegReg_Hi2( dst, src ) );
7038   ins_pipe( pipe_cmov_reg_long );
7039 %}
7040 
7041 instruct cmovL_regU(cmpOpU cop, eFlagsRegU cr, eRegL dst, eRegL src) %{
7042   predicate(VM_Version::supports_cmov() );
7043   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
7044   ins_cost(200);
7045   format %{ "CMOV$cop $dst.lo,$src.lo\n\t"
7046             "CMOV$cop $dst.hi,$src.hi" %}
7047   opcode(0x0F,0x40);
7048   ins_encode( enc_cmov(cop), RegReg_Lo2( dst, src ), enc_cmov(cop), RegReg_Hi2( dst, src ) );
7049   ins_pipe( pipe_cmov_reg_long );
7050 %}
7051 
7052 instruct cmovL_regUCF(cmpOpUCF cop, eFlagsRegUCF cr, eRegL dst, eRegL src) %{
7053   predicate(VM_Version::supports_cmov() );
7054   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
7055   ins_cost(200);
7056   expand %{
7057     cmovL_regU(cop, cr, dst, src);
7058   %}
7059 %}
7060 
7061 //----------Arithmetic Instructions--------------------------------------------
7062 //----------Addition Instructions----------------------------------------------
7063 
7064 // Integer Addition Instructions
7065 instruct addI_eReg(rRegI dst, rRegI src, eFlagsReg cr) %{
7066   match(Set dst (AddI dst src));
7067   effect(KILL cr);
7068 
7069   size(2);
7070   format %{ "ADD    $dst,$src" %}
7071   opcode(0x03);
7072   ins_encode( OpcP, RegReg( dst, src) );
7073   ins_pipe( ialu_reg_reg );
7074 %}
7075 
7076 instruct addI_eReg_imm(rRegI dst, immI src, eFlagsReg cr) %{
7077   match(Set dst (AddI dst src));
7078   effect(KILL cr);
7079 
7080   format %{ "ADD    $dst,$src" %}
7081   opcode(0x81, 0x00); /* /0 id */
7082   ins_encode( OpcSErm( dst, src ), Con8or32( src ) );
7083   ins_pipe( ialu_reg );
7084 %}
7085 
7086 instruct incI_eReg(rRegI dst, immI1 src, eFlagsReg cr) %{
7087   predicate(UseIncDec);
7088   match(Set dst (AddI dst src));
7089   effect(KILL cr);
7090 
7091   size(1);
7092   format %{ "INC    $dst" %}
7093   opcode(0x40); /*  */
7094   ins_encode( Opc_plus( primary, dst ) );
7095   ins_pipe( ialu_reg );
7096 %}
7097 
7098 instruct leaI_eReg_immI(rRegI dst, rRegI src0, immI src1) %{
7099   match(Set dst (AddI src0 src1));
7100   ins_cost(110);
7101 
7102   format %{ "LEA    $dst,[$src0 + $src1]" %}
7103   opcode(0x8D); /* 0x8D /r */
7104   ins_encode( OpcP, RegLea( dst, src0, src1 ) );
7105   ins_pipe( ialu_reg_reg );
7106 %}
7107 
7108 instruct leaP_eReg_immI(eRegP dst, eRegP src0, immI src1) %{
7109   match(Set dst (AddP src0 src1));
7110   ins_cost(110);
7111 
7112   format %{ "LEA    $dst,[$src0 + $src1]\t# ptr" %}
7113   opcode(0x8D); /* 0x8D /r */
7114   ins_encode( OpcP, RegLea( dst, src0, src1 ) );
7115   ins_pipe( ialu_reg_reg );
7116 %}
7117 
7118 instruct decI_eReg(rRegI dst, immI_M1 src, eFlagsReg cr) %{
7119   predicate(UseIncDec);
7120   match(Set dst (AddI dst src));
7121   effect(KILL cr);
7122 
7123   size(1);
7124   format %{ "DEC    $dst" %}
7125   opcode(0x48); /*  */
7126   ins_encode( Opc_plus( primary, dst ) );
7127   ins_pipe( ialu_reg );
7128 %}
7129 
7130 instruct addP_eReg(eRegP dst, rRegI src, eFlagsReg cr) %{
7131   match(Set dst (AddP dst src));
7132   effect(KILL cr);
7133 
7134   size(2);
7135   format %{ "ADD    $dst,$src" %}
7136   opcode(0x03);
7137   ins_encode( OpcP, RegReg( dst, src) );
7138   ins_pipe( ialu_reg_reg );
7139 %}
7140 
7141 instruct addP_eReg_imm(eRegP dst, immI src, eFlagsReg cr) %{
7142   match(Set dst (AddP dst src));
7143   effect(KILL cr);
7144 
7145   format %{ "ADD    $dst,$src" %}
7146   opcode(0x81,0x00); /* Opcode 81 /0 id */
7147   // ins_encode( RegImm( dst, src) );
7148   ins_encode( OpcSErm( dst, src ), Con8or32( src ) );
7149   ins_pipe( ialu_reg );
7150 %}
7151 
7152 instruct addI_eReg_mem(rRegI dst, memory src, eFlagsReg cr) %{
7153   match(Set dst (AddI dst (LoadI src)));
7154   effect(KILL cr);
7155 
7156   ins_cost(125);
7157   format %{ "ADD    $dst,$src" %}
7158   opcode(0x03);
7159   ins_encode( OpcP, RegMem( dst, src) );
7160   ins_pipe( ialu_reg_mem );
7161 %}
7162 
7163 instruct addI_mem_eReg(memory dst, rRegI src, eFlagsReg cr) %{
7164   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
7165   effect(KILL cr);
7166 
7167   ins_cost(150);
7168   format %{ "ADD    $dst,$src" %}
7169   opcode(0x01);  /* Opcode 01 /r */
7170   ins_encode( OpcP, RegMem( src, dst ) );
7171   ins_pipe( ialu_mem_reg );
7172 %}
7173 
7174 // Add Memory with Immediate
7175 instruct addI_mem_imm(memory dst, immI src, eFlagsReg cr) %{
7176   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
7177   effect(KILL cr);
7178 
7179   ins_cost(125);
7180   format %{ "ADD    $dst,$src" %}
7181   opcode(0x81);               /* Opcode 81 /0 id */
7182   ins_encode( OpcSE( src ), RMopc_Mem(0x00,dst), Con8or32( src ) );
7183   ins_pipe( ialu_mem_imm );
7184 %}
7185 
7186 instruct incI_mem(memory dst, immI1 src, eFlagsReg cr) %{
7187   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
7188   effect(KILL cr);
7189 
7190   ins_cost(125);
7191   format %{ "INC    $dst" %}
7192   opcode(0xFF);               /* Opcode FF /0 */
7193   ins_encode( OpcP, RMopc_Mem(0x00,dst));
7194   ins_pipe( ialu_mem_imm );
7195 %}
7196 
7197 instruct decI_mem(memory dst, immI_M1 src, eFlagsReg cr) %{
7198   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
7199   effect(KILL cr);
7200 
7201   ins_cost(125);
7202   format %{ "DEC    $dst" %}
7203   opcode(0xFF);               /* Opcode FF /1 */
7204   ins_encode( OpcP, RMopc_Mem(0x01,dst));
7205   ins_pipe( ialu_mem_imm );
7206 %}
7207 
7208 
7209 instruct checkCastPP( eRegP dst ) %{
7210   match(Set dst (CheckCastPP dst));
7211 
7212   size(0);
7213   format %{ "#checkcastPP of $dst" %}
7214   ins_encode( /*empty encoding*/ );
7215   ins_pipe( empty );
7216 %}
7217 
7218 instruct castPP( eRegP dst ) %{
7219   match(Set dst (CastPP dst));
7220   format %{ "#castPP of $dst" %}
7221   ins_encode( /*empty encoding*/ );
7222   ins_pipe( empty );
7223 %}
7224 
7225 instruct castII( rRegI dst ) %{
7226   match(Set dst (CastII dst));
7227   format %{ "#castII of $dst" %}
7228   ins_encode( /*empty encoding*/ );
7229   ins_cost(0);
7230   ins_pipe( empty );
7231 %}
7232 
7233 
7234 // Load-locked - same as a regular pointer load when used with compare-swap
7235 instruct loadPLocked(eRegP dst, memory mem) %{
7236   match(Set dst (LoadPLocked mem));
7237 
7238   ins_cost(125);
7239   format %{ "MOV    $dst,$mem\t# Load ptr. locked" %}
7240   opcode(0x8B);
7241   ins_encode( OpcP, RegMem(dst,mem));
7242   ins_pipe( ialu_reg_mem );
7243 %}
7244 
7245 // Conditional-store of the updated heap-top.
7246 // Used during allocation of the shared heap.
7247 // Sets flags (EQ) on success.  Implemented with a CMPXCHG on Intel.
7248 instruct storePConditional( memory heap_top_ptr, eAXRegP oldval, eRegP newval, eFlagsReg cr ) %{
7249   match(Set cr (StorePConditional heap_top_ptr (Binary oldval newval)));
7250   // EAX is killed if there is contention, but then it's also unused.
7251   // In the common case of no contention, EAX holds the new oop address.
7252   format %{ "CMPXCHG $heap_top_ptr,$newval\t# If EAX==$heap_top_ptr Then store $newval into $heap_top_ptr" %}
7253   ins_encode( lock_prefix, Opcode(0x0F), Opcode(0xB1), RegMem(newval,heap_top_ptr) );
7254   ins_pipe( pipe_cmpxchg );
7255 %}
7256 
7257 // Conditional-store of an int value.
7258 // ZF flag is set on success, reset otherwise.  Implemented with a CMPXCHG on Intel.
7259 instruct storeIConditional( memory mem, eAXRegI oldval, rRegI newval, eFlagsReg cr ) %{
7260   match(Set cr (StoreIConditional mem (Binary oldval newval)));
7261   effect(KILL oldval);
7262   format %{ "CMPXCHG $mem,$newval\t# If EAX==$mem Then store $newval into $mem" %}
7263   ins_encode( lock_prefix, Opcode(0x0F), Opcode(0xB1), RegMem(newval, mem) );
7264   ins_pipe( pipe_cmpxchg );
7265 %}
7266 
7267 // Conditional-store of a long value.
7268 // ZF flag is set on success, reset otherwise.  Implemented with a CMPXCHG8 on Intel.
7269 instruct storeLConditional( memory mem, eADXRegL oldval, eBCXRegL newval, eFlagsReg cr ) %{
7270   match(Set cr (StoreLConditional mem (Binary oldval newval)));
7271   effect(KILL oldval);
7272   format %{ "XCHG   EBX,ECX\t# correct order for CMPXCHG8 instruction\n\t"
7273             "CMPXCHG8 $mem,ECX:EBX\t# If EDX:EAX==$mem Then store ECX:EBX into $mem\n\t"
7274             "XCHG   EBX,ECX"
7275   %}
7276   ins_encode %{
7277     // Note: we need to swap rbx, and rcx before and after the
7278     //       cmpxchg8 instruction because the instruction uses
7279     //       rcx as the high order word of the new value to store but
7280     //       our register encoding uses rbx.
7281     __ xchgl(as_Register(EBX_enc), as_Register(ECX_enc));
7282     if( os::is_MP() )
7283       __ lock();
7284     __ cmpxchg8($mem$$Address);
7285     __ xchgl(as_Register(EBX_enc), as_Register(ECX_enc));
7286   %}
7287   ins_pipe( pipe_cmpxchg );
7288 %}
7289 
7290 // No flag versions for CompareAndSwap{P,I,L} because matcher can't match them
7291 
7292 instruct compareAndSwapL( rRegI res, eSIRegP mem_ptr, eADXRegL oldval, eBCXRegL newval, eFlagsReg cr ) %{
7293   predicate(VM_Version::supports_cx8());
7294   match(Set res (CompareAndSwapL mem_ptr (Binary oldval newval)));
7295   effect(KILL cr, KILL oldval);
7296   format %{ "CMPXCHG8 [$mem_ptr],$newval\t# If EDX:EAX==[$mem_ptr] Then store $newval into [$mem_ptr]\n\t"
7297             "MOV    $res,0\n\t"
7298             "JNE,s  fail\n\t"
7299             "MOV    $res,1\n"
7300           "fail:" %}
7301   ins_encode( enc_cmpxchg8(mem_ptr),
7302               enc_flags_ne_to_boolean(res) );
7303   ins_pipe( pipe_cmpxchg );
7304 %}
7305 
7306 instruct compareAndSwapP( rRegI res,  pRegP mem_ptr, eAXRegP oldval, eCXRegP newval, eFlagsReg cr) %{
7307   predicate(!UseShenandoahGC || !ShenandoahCASBarrier || n->in(3)->in(1)->bottom_type() == TypePtr::NULL_PTR);
7308   match(Set res (CompareAndSwapP mem_ptr (Binary oldval newval)));
7309   effect(KILL cr, KILL oldval);
7310   format %{ "CMPXCHG [$mem_ptr],$newval\t# If EAX==[$mem_ptr] Then store $newval into [$mem_ptr]\n\t"
7311             "MOV    $res,0\n\t"
7312             "JNE,s  fail\n\t"
7313             "MOV    $res,1\n"
7314           "fail:" %}
7315   ins_encode( enc_cmpxchg(mem_ptr), enc_flags_ne_to_boolean(res) );
7316   ins_pipe( pipe_cmpxchg );
7317 %}
7318 
7319 instruct compareAndSwapP_shenandoah(rRegI res,
7320                                     memory mem_ptr,
7321                                     eRegP tmp1, eRegP tmp2,
7322                                     eAXRegP oldval, eCXRegP newval,
7323                                     eFlagsReg cr)
7324 %{
7325   predicate(UseShenandoahGC && ShenandoahCASBarrier && n->in(3)->in(1)->bottom_type() != TypePtr::NULL_PTR);
7326   match(Set res (CompareAndSwapP mem_ptr (Binary oldval newval)));
7327   effect(TEMP tmp1, TEMP tmp2, KILL cr, KILL oldval);
7328 
7329   format %{ "shenandoah_cas_oop $mem_ptr,$newval" %}
7330 
7331   ins_encode %{
7332     ShenandoahBarrierSetAssembler::bsasm()->cmpxchg_oop(&_masm,
7333       $res$$Register, $mem_ptr$$Address, $oldval$$Register, $newval$$Register,
7334       false, // swap
7335       $tmp1$$Register, $tmp2$$Register
7336     );
7337   %}
7338   ins_pipe( pipe_cmpxchg );
7339 %}
7340 
7341 instruct compareAndSwapI( rRegI res, pRegP mem_ptr, eAXRegI oldval, eCXRegI newval, eFlagsReg cr) %{
7342   match(Set res (CompareAndSwapI mem_ptr (Binary oldval newval)));
7343   effect(KILL cr, KILL oldval);
7344   format %{ "CMPXCHG [$mem_ptr],$newval\t# If EAX==[$mem_ptr] Then store $newval into [$mem_ptr]\n\t"
7345             "MOV    $res,0\n\t"
7346             "JNE,s  fail\n\t"
7347             "MOV    $res,1\n"
7348           "fail:" %}
7349   ins_encode( enc_cmpxchg(mem_ptr), enc_flags_ne_to_boolean(res) );
7350   ins_pipe( pipe_cmpxchg );
7351 %}
7352 
7353 instruct xaddI_no_res( memory mem, Universe dummy, immI add, eFlagsReg cr) %{
7354   predicate(n->as_LoadStore()->result_not_used());
7355   match(Set dummy (GetAndAddI mem add));
7356   effect(KILL cr);
7357   format %{ "ADDL  [$mem],$add" %}
7358   ins_encode %{
7359     if (os::is_MP()) { __ lock(); }
7360     __ addl($mem$$Address, $add$$constant);
7361   %}
7362   ins_pipe( pipe_cmpxchg );
7363 %}
7364 
7365 instruct xaddI( memory mem, rRegI newval, eFlagsReg cr) %{
7366   match(Set newval (GetAndAddI mem newval));
7367   effect(KILL cr);
7368   format %{ "XADDL  [$mem],$newval" %}
7369   ins_encode %{
7370     if (os::is_MP()) { __ lock(); }
7371     __ xaddl($mem$$Address, $newval$$Register);
7372   %}
7373   ins_pipe( pipe_cmpxchg );
7374 %}
7375 
7376 instruct xchgI( memory mem, rRegI newval) %{
7377   match(Set newval (GetAndSetI mem newval));
7378   format %{ "XCHGL  $newval,[$mem]" %}
7379   ins_encode %{
7380     __ xchgl($newval$$Register, $mem$$Address);
7381   %}
7382   ins_pipe( pipe_cmpxchg );
7383 %}
7384 
7385 instruct xchgP( memory mem, pRegP newval) %{
7386   match(Set newval (GetAndSetP mem newval));
7387   format %{ "XCHGL  $newval,[$mem]" %}
7388   ins_encode %{
7389     __ xchgl($newval$$Register, $mem$$Address);
7390   %}
7391   ins_pipe( pipe_cmpxchg );
7392 %}
7393 
7394 //----------Subtraction Instructions-------------------------------------------
7395 
7396 // Integer Subtraction Instructions
7397 instruct subI_eReg(rRegI dst, rRegI src, eFlagsReg cr) %{
7398   match(Set dst (SubI dst src));
7399   effect(KILL cr);
7400 
7401   size(2);
7402   format %{ "SUB    $dst,$src" %}
7403   opcode(0x2B);
7404   ins_encode( OpcP, RegReg( dst, src) );
7405   ins_pipe( ialu_reg_reg );
7406 %}
7407 
7408 instruct subI_eReg_imm(rRegI dst, immI src, eFlagsReg cr) %{
7409   match(Set dst (SubI dst src));
7410   effect(KILL cr);
7411 
7412   format %{ "SUB    $dst,$src" %}
7413   opcode(0x81,0x05);  /* Opcode 81 /5 */
7414   // ins_encode( RegImm( dst, src) );
7415   ins_encode( OpcSErm( dst, src ), Con8or32( src ) );
7416   ins_pipe( ialu_reg );
7417 %}
7418 
7419 instruct subI_eReg_mem(rRegI dst, memory src, eFlagsReg cr) %{
7420   match(Set dst (SubI dst (LoadI src)));
7421   effect(KILL cr);
7422 
7423   ins_cost(125);
7424   format %{ "SUB    $dst,$src" %}
7425   opcode(0x2B);
7426   ins_encode( OpcP, RegMem( dst, src) );
7427   ins_pipe( ialu_reg_mem );
7428 %}
7429 
7430 instruct subI_mem_eReg(memory dst, rRegI src, eFlagsReg cr) %{
7431   match(Set dst (StoreI dst (SubI (LoadI dst) src)));
7432   effect(KILL cr);
7433 
7434   ins_cost(150);
7435   format %{ "SUB    $dst,$src" %}
7436   opcode(0x29);  /* Opcode 29 /r */
7437   ins_encode( OpcP, RegMem( src, dst ) );
7438   ins_pipe( ialu_mem_reg );
7439 %}
7440 
7441 // Subtract from a pointer
7442 instruct subP_eReg(eRegP dst, rRegI src, immI0 zero, eFlagsReg cr) %{
7443   match(Set dst (AddP dst (SubI zero src)));
7444   effect(KILL cr);
7445 
7446   size(2);
7447   format %{ "SUB    $dst,$src" %}
7448   opcode(0x2B);
7449   ins_encode( OpcP, RegReg( dst, src) );
7450   ins_pipe( ialu_reg_reg );
7451 %}
7452 
7453 instruct negI_eReg(rRegI dst, immI0 zero, eFlagsReg cr) %{
7454   match(Set dst (SubI zero dst));
7455   effect(KILL cr);
7456 
7457   size(2);
7458   format %{ "NEG    $dst" %}
7459   opcode(0xF7,0x03);  // Opcode F7 /3
7460   ins_encode( OpcP, RegOpc( dst ) );
7461   ins_pipe( ialu_reg );
7462 %}
7463 
7464 //----------Multiplication/Division Instructions-------------------------------
7465 // Integer Multiplication Instructions
7466 // Multiply Register
7467 instruct mulI_eReg(rRegI dst, rRegI src, eFlagsReg cr) %{
7468   match(Set dst (MulI dst src));
7469   effect(KILL cr);
7470 
7471   size(3);
7472   ins_cost(300);
7473   format %{ "IMUL   $dst,$src" %}
7474   opcode(0xAF, 0x0F);
7475   ins_encode( OpcS, OpcP, RegReg( dst, src) );
7476   ins_pipe( ialu_reg_reg_alu0 );
7477 %}
7478 
7479 // Multiply 32-bit Immediate
7480 instruct mulI_eReg_imm(rRegI dst, rRegI src, immI imm, eFlagsReg cr) %{
7481   match(Set dst (MulI src imm));
7482   effect(KILL cr);
7483 
7484   ins_cost(300);
7485   format %{ "IMUL   $dst,$src,$imm" %}
7486   opcode(0x69);  /* 69 /r id */
7487   ins_encode( OpcSE(imm), RegReg( dst, src ), Con8or32( imm ) );
7488   ins_pipe( ialu_reg_reg_alu0 );
7489 %}
7490 
7491 instruct loadConL_low_only(eADXRegL_low_only dst, immL32 src, eFlagsReg cr) %{
7492   match(Set dst src);
7493   effect(KILL cr);
7494 
7495   // Note that this is artificially increased to make it more expensive than loadConL
7496   ins_cost(250);
7497   format %{ "MOV    EAX,$src\t// low word only" %}
7498   opcode(0xB8);
7499   ins_encode( LdImmL_Lo(dst, src) );
7500   ins_pipe( ialu_reg_fat );
7501 %}
7502 
7503 // Multiply by 32-bit Immediate, taking the shifted high order results
7504 //  (special case for shift by 32)
7505 instruct mulI_imm_high(eDXRegI dst, nadxRegI src1, eADXRegL_low_only src2, immI_32 cnt, eFlagsReg cr) %{
7506   match(Set dst (ConvL2I (RShiftL (MulL (ConvI2L src1) src2) cnt)));
7507   predicate( _kids[0]->_kids[0]->_kids[1]->_leaf->Opcode() == Op_ConL &&
7508              _kids[0]->_kids[0]->_kids[1]->_leaf->as_Type()->type()->is_long()->get_con() >= min_jint &&
7509              _kids[0]->_kids[0]->_kids[1]->_leaf->as_Type()->type()->is_long()->get_con() <= max_jint );
7510   effect(USE src1, KILL cr);
7511 
7512   // Note that this is adjusted by 150 to compensate for the overcosting of loadConL_low_only
7513   ins_cost(0*100 + 1*400 - 150);
7514   format %{ "IMUL   EDX:EAX,$src1" %}
7515   ins_encode( multiply_con_and_shift_high( dst, src1, src2, cnt, cr ) );
7516   ins_pipe( pipe_slow );
7517 %}
7518 
7519 // Multiply by 32-bit Immediate, taking the shifted high order results
7520 instruct mulI_imm_RShift_high(eDXRegI dst, nadxRegI src1, eADXRegL_low_only src2, immI_32_63 cnt, eFlagsReg cr) %{
7521   match(Set dst (ConvL2I (RShiftL (MulL (ConvI2L src1) src2) cnt)));
7522   predicate( _kids[0]->_kids[0]->_kids[1]->_leaf->Opcode() == Op_ConL &&
7523              _kids[0]->_kids[0]->_kids[1]->_leaf->as_Type()->type()->is_long()->get_con() >= min_jint &&
7524              _kids[0]->_kids[0]->_kids[1]->_leaf->as_Type()->type()->is_long()->get_con() <= max_jint );
7525   effect(USE src1, KILL cr);
7526 
7527   // Note that this is adjusted by 150 to compensate for the overcosting of loadConL_low_only
7528   ins_cost(1*100 + 1*400 - 150);
7529   format %{ "IMUL   EDX:EAX,$src1\n\t"
7530             "SAR    EDX,$cnt-32" %}
7531   ins_encode( multiply_con_and_shift_high( dst, src1, src2, cnt, cr ) );
7532   ins_pipe( pipe_slow );
7533 %}
7534 
7535 // Multiply Memory 32-bit Immediate
7536 instruct mulI_mem_imm(rRegI dst, memory src, immI imm, eFlagsReg cr) %{
7537   match(Set dst (MulI (LoadI src) imm));
7538   effect(KILL cr);
7539 
7540   ins_cost(300);
7541   format %{ "IMUL   $dst,$src,$imm" %}
7542   opcode(0x69);  /* 69 /r id */
7543   ins_encode( OpcSE(imm), RegMem( dst, src ), Con8or32( imm ) );
7544   ins_pipe( ialu_reg_mem_alu0 );
7545 %}
7546 
7547 // Multiply Memory
7548 instruct mulI(rRegI dst, memory src, eFlagsReg cr) %{
7549   match(Set dst (MulI dst (LoadI src)));
7550   effect(KILL cr);
7551 
7552   ins_cost(350);
7553   format %{ "IMUL   $dst,$src" %}
7554   opcode(0xAF, 0x0F);
7555   ins_encode( OpcS, OpcP, RegMem( dst, src) );
7556   ins_pipe( ialu_reg_mem_alu0 );
7557 %}
7558 
7559 // Multiply Register Int to Long
7560 instruct mulI2L(eADXRegL dst, eAXRegI src, nadxRegI src1, eFlagsReg flags) %{
7561   // Basic Idea: long = (long)int * (long)int
7562   match(Set dst (MulL (ConvI2L src) (ConvI2L src1)));
7563   effect(DEF dst, USE src, USE src1, KILL flags);
7564 
7565   ins_cost(300);
7566   format %{ "IMUL   $dst,$src1" %}
7567 
7568   ins_encode( long_int_multiply( dst, src1 ) );
7569   ins_pipe( ialu_reg_reg_alu0 );
7570 %}
7571 
7572 instruct mulIS_eReg(eADXRegL dst, immL_32bits mask, eFlagsReg flags, eAXRegI src, nadxRegI src1) %{
7573   // Basic Idea:  long = (int & 0xffffffffL) * (int & 0xffffffffL)
7574   match(Set dst (MulL (AndL (ConvI2L src) mask) (AndL (ConvI2L src1) mask)));
7575   effect(KILL flags);
7576 
7577   ins_cost(300);
7578   format %{ "MUL    $dst,$src1" %}
7579 
7580   ins_encode( long_uint_multiply(dst, src1) );
7581   ins_pipe( ialu_reg_reg_alu0 );
7582 %}
7583 
7584 // Multiply Register Long
7585 instruct mulL_eReg(eADXRegL dst, eRegL src, rRegI tmp, eFlagsReg cr) %{
7586   match(Set dst (MulL dst src));
7587   effect(KILL cr, TEMP tmp);
7588   ins_cost(4*100+3*400);
7589 // Basic idea: lo(result) = lo(x_lo * y_lo)
7590 //             hi(result) = hi(x_lo * y_lo) + lo(x_hi * y_lo) + lo(x_lo * y_hi)
7591   format %{ "MOV    $tmp,$src.lo\n\t"
7592             "IMUL   $tmp,EDX\n\t"
7593             "MOV    EDX,$src.hi\n\t"
7594             "IMUL   EDX,EAX\n\t"
7595             "ADD    $tmp,EDX\n\t"
7596             "MUL    EDX:EAX,$src.lo\n\t"
7597             "ADD    EDX,$tmp" %}
7598   ins_encode( long_multiply( dst, src, tmp ) );
7599   ins_pipe( pipe_slow );
7600 %}
7601 
7602 // Multiply Register Long where the left operand's high 32 bits are zero
7603 instruct mulL_eReg_lhi0(eADXRegL dst, eRegL src, rRegI tmp, eFlagsReg cr) %{
7604   predicate(is_operand_hi32_zero(n->in(1)));
7605   match(Set dst (MulL dst src));
7606   effect(KILL cr, TEMP tmp);
7607   ins_cost(2*100+2*400);
7608 // Basic idea: lo(result) = lo(x_lo * y_lo)
7609 //             hi(result) = hi(x_lo * y_lo) + lo(x_lo * y_hi) where lo(x_hi * y_lo) = 0 because x_hi = 0
7610   format %{ "MOV    $tmp,$src.hi\n\t"
7611             "IMUL   $tmp,EAX\n\t"
7612             "MUL    EDX:EAX,$src.lo\n\t"
7613             "ADD    EDX,$tmp" %}
7614   ins_encode %{
7615     __ movl($tmp$$Register, HIGH_FROM_LOW($src$$Register));
7616     __ imull($tmp$$Register, rax);
7617     __ mull($src$$Register);
7618     __ addl(rdx, $tmp$$Register);
7619   %}
7620   ins_pipe( pipe_slow );
7621 %}
7622 
7623 // Multiply Register Long where the right operand's high 32 bits are zero
7624 instruct mulL_eReg_rhi0(eADXRegL dst, eRegL src, rRegI tmp, eFlagsReg cr) %{
7625   predicate(is_operand_hi32_zero(n->in(2)));
7626   match(Set dst (MulL dst src));
7627   effect(KILL cr, TEMP tmp);
7628   ins_cost(2*100+2*400);
7629 // Basic idea: lo(result) = lo(x_lo * y_lo)
7630 //             hi(result) = hi(x_lo * y_lo) + lo(x_hi * y_lo) where lo(x_lo * y_hi) = 0 because y_hi = 0
7631   format %{ "MOV    $tmp,$src.lo\n\t"
7632             "IMUL   $tmp,EDX\n\t"
7633             "MUL    EDX:EAX,$src.lo\n\t"
7634             "ADD    EDX,$tmp" %}
7635   ins_encode %{
7636     __ movl($tmp$$Register, $src$$Register);
7637     __ imull($tmp$$Register, rdx);
7638     __ mull($src$$Register);
7639     __ addl(rdx, $tmp$$Register);
7640   %}
7641   ins_pipe( pipe_slow );
7642 %}
7643 
7644 // Multiply Register Long where the left and the right operands' high 32 bits are zero
7645 instruct mulL_eReg_hi0(eADXRegL dst, eRegL src, eFlagsReg cr) %{
7646   predicate(is_operand_hi32_zero(n->in(1)) && is_operand_hi32_zero(n->in(2)));
7647   match(Set dst (MulL dst src));
7648   effect(KILL cr);
7649   ins_cost(1*400);
7650 // Basic idea: lo(result) = lo(x_lo * y_lo)
7651 //             hi(result) = hi(x_lo * y_lo) where lo(x_hi * y_lo) = 0 and lo(x_lo * y_hi) = 0 because x_hi = 0 and y_hi = 0
7652   format %{ "MUL    EDX:EAX,$src.lo\n\t" %}
7653   ins_encode %{
7654     __ mull($src$$Register);
7655   %}
7656   ins_pipe( pipe_slow );
7657 %}
7658 
7659 // Multiply Register Long by small constant
7660 instruct mulL_eReg_con(eADXRegL dst, immL_127 src, rRegI tmp, eFlagsReg cr) %{
7661   match(Set dst (MulL dst src));
7662   effect(KILL cr, TEMP tmp);
7663   ins_cost(2*100+2*400);
7664   size(12);
7665 // Basic idea: lo(result) = lo(src * EAX)
7666 //             hi(result) = hi(src * EAX) + lo(src * EDX)
7667   format %{ "IMUL   $tmp,EDX,$src\n\t"
7668             "MOV    EDX,$src\n\t"
7669             "MUL    EDX\t# EDX*EAX -> EDX:EAX\n\t"
7670             "ADD    EDX,$tmp" %}
7671   ins_encode( long_multiply_con( dst, src, tmp ) );
7672   ins_pipe( pipe_slow );
7673 %}
7674 
7675 // Integer DIV with Register
7676 instruct divI_eReg(eAXRegI rax, eDXRegI rdx, eCXRegI div, eFlagsReg cr) %{
7677   match(Set rax (DivI rax div));
7678   effect(KILL rdx, KILL cr);
7679   size(26);
7680   ins_cost(30*100+10*100);
7681   format %{ "CMP    EAX,0x80000000\n\t"
7682             "JNE,s  normal\n\t"
7683             "XOR    EDX,EDX\n\t"
7684             "CMP    ECX,-1\n\t"
7685             "JE,s   done\n"
7686     "normal: CDQ\n\t"
7687             "IDIV   $div\n\t"
7688     "done:"        %}
7689   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
7690   ins_encode( cdq_enc, OpcP, RegOpc(div) );
7691   ins_pipe( ialu_reg_reg_alu0 );
7692 %}
7693 
7694 // Divide Register Long
7695 instruct divL_eReg( eADXRegL dst, eRegL src1, eRegL src2, eFlagsReg cr, eCXRegI cx, eBXRegI bx ) %{
7696   match(Set dst (DivL src1 src2));
7697   effect( KILL cr, KILL cx, KILL bx );
7698   ins_cost(10000);
7699   format %{ "PUSH   $src1.hi\n\t"
7700             "PUSH   $src1.lo\n\t"
7701             "PUSH   $src2.hi\n\t"
7702             "PUSH   $src2.lo\n\t"
7703             "CALL   SharedRuntime::ldiv\n\t"
7704             "ADD    ESP,16" %}
7705   ins_encode( long_div(src1,src2) );
7706   ins_pipe( pipe_slow );
7707 %}
7708 
7709 // Integer DIVMOD with Register, both quotient and mod results
7710 instruct divModI_eReg_divmod(eAXRegI rax, eDXRegI rdx, eCXRegI div, eFlagsReg cr) %{
7711   match(DivModI rax div);
7712   effect(KILL cr);
7713   size(26);
7714   ins_cost(30*100+10*100);
7715   format %{ "CMP    EAX,0x80000000\n\t"
7716             "JNE,s  normal\n\t"
7717             "XOR    EDX,EDX\n\t"
7718             "CMP    ECX,-1\n\t"
7719             "JE,s   done\n"
7720     "normal: CDQ\n\t"
7721             "IDIV   $div\n\t"
7722     "done:"        %}
7723   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
7724   ins_encode( cdq_enc, OpcP, RegOpc(div) );
7725   ins_pipe( pipe_slow );
7726 %}
7727 
7728 // Integer MOD with Register
7729 instruct modI_eReg(eDXRegI rdx, eAXRegI rax, eCXRegI div, eFlagsReg cr) %{
7730   match(Set rdx (ModI rax div));
7731   effect(KILL rax, KILL cr);
7732 
7733   size(26);
7734   ins_cost(300);
7735   format %{ "CDQ\n\t"
7736             "IDIV   $div" %}
7737   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
7738   ins_encode( cdq_enc, OpcP, RegOpc(div) );
7739   ins_pipe( ialu_reg_reg_alu0 );
7740 %}
7741 
7742 // Remainder Register Long
7743 instruct modL_eReg( eADXRegL dst, eRegL src1, eRegL src2, eFlagsReg cr, eCXRegI cx, eBXRegI bx ) %{
7744   match(Set dst (ModL src1 src2));
7745   effect( KILL cr, KILL cx, KILL bx );
7746   ins_cost(10000);
7747   format %{ "PUSH   $src1.hi\n\t"
7748             "PUSH   $src1.lo\n\t"
7749             "PUSH   $src2.hi\n\t"
7750             "PUSH   $src2.lo\n\t"
7751             "CALL   SharedRuntime::lrem\n\t"
7752             "ADD    ESP,16" %}
7753   ins_encode( long_mod(src1,src2) );
7754   ins_pipe( pipe_slow );
7755 %}
7756 
7757 // Divide Register Long (no special case since divisor != -1)
7758 instruct divL_eReg_imm32( eADXRegL dst, immL32 imm, rRegI tmp, rRegI tmp2, eFlagsReg cr ) %{
7759   match(Set dst (DivL dst imm));
7760   effect( TEMP tmp, TEMP tmp2, KILL cr );
7761   ins_cost(1000);
7762   format %{ "MOV    $tmp,abs($imm) # ldiv EDX:EAX,$imm\n\t"
7763             "XOR    $tmp2,$tmp2\n\t"
7764             "CMP    $tmp,EDX\n\t"
7765             "JA,s   fast\n\t"
7766             "MOV    $tmp2,EAX\n\t"
7767             "MOV    EAX,EDX\n\t"
7768             "MOV    EDX,0\n\t"
7769             "JLE,s  pos\n\t"
7770             "LNEG   EAX : $tmp2\n\t"
7771             "DIV    $tmp # unsigned division\n\t"
7772             "XCHG   EAX,$tmp2\n\t"
7773             "DIV    $tmp\n\t"
7774             "LNEG   $tmp2 : EAX\n\t"
7775             "JMP,s  done\n"
7776     "pos:\n\t"
7777             "DIV    $tmp\n\t"
7778             "XCHG   EAX,$tmp2\n"
7779     "fast:\n\t"
7780             "DIV    $tmp\n"
7781     "done:\n\t"
7782             "MOV    EDX,$tmp2\n\t"
7783             "NEG    EDX:EAX # if $imm < 0" %}
7784   ins_encode %{
7785     int con = (int)$imm$$constant;
7786     assert(con != 0 && con != -1 && con != min_jint, "wrong divisor");
7787     int pcon = (con > 0) ? con : -con;
7788     Label Lfast, Lpos, Ldone;
7789 
7790     __ movl($tmp$$Register, pcon);
7791     __ xorl($tmp2$$Register,$tmp2$$Register);
7792     __ cmpl($tmp$$Register, HIGH_FROM_LOW($dst$$Register));
7793     __ jccb(Assembler::above, Lfast); // result fits into 32 bit
7794 
7795     __ movl($tmp2$$Register, $dst$$Register); // save
7796     __ movl($dst$$Register, HIGH_FROM_LOW($dst$$Register));
7797     __ movl(HIGH_FROM_LOW($dst$$Register),0); // preserve flags
7798     __ jccb(Assembler::lessEqual, Lpos); // result is positive
7799 
7800     // Negative dividend.
7801     // convert value to positive to use unsigned division
7802     __ lneg($dst$$Register, $tmp2$$Register);
7803     __ divl($tmp$$Register);
7804     __ xchgl($dst$$Register, $tmp2$$Register);
7805     __ divl($tmp$$Register);
7806     // revert result back to negative
7807     __ lneg($tmp2$$Register, $dst$$Register);
7808     __ jmpb(Ldone);
7809 
7810     __ bind(Lpos);
7811     __ divl($tmp$$Register); // Use unsigned division
7812     __ xchgl($dst$$Register, $tmp2$$Register);
7813     // Fallthrow for final divide, tmp2 has 32 bit hi result
7814 
7815     __ bind(Lfast);
7816     // fast path: src is positive
7817     __ divl($tmp$$Register); // Use unsigned division
7818 
7819     __ bind(Ldone);
7820     __ movl(HIGH_FROM_LOW($dst$$Register),$tmp2$$Register);
7821     if (con < 0) {
7822       __ lneg(HIGH_FROM_LOW($dst$$Register), $dst$$Register);
7823     }
7824   %}
7825   ins_pipe( pipe_slow );
7826 %}
7827 
7828 // Remainder Register Long (remainder fit into 32 bits)
7829 instruct modL_eReg_imm32( eADXRegL dst, immL32 imm, rRegI tmp, rRegI tmp2, eFlagsReg cr ) %{
7830   match(Set dst (ModL dst imm));
7831   effect( TEMP tmp, TEMP tmp2, KILL cr );
7832   ins_cost(1000);
7833   format %{ "MOV    $tmp,abs($imm) # lrem EDX:EAX,$imm\n\t"
7834             "CMP    $tmp,EDX\n\t"
7835             "JA,s   fast\n\t"
7836             "MOV    $tmp2,EAX\n\t"
7837             "MOV    EAX,EDX\n\t"
7838             "MOV    EDX,0\n\t"
7839             "JLE,s  pos\n\t"
7840             "LNEG   EAX : $tmp2\n\t"
7841             "DIV    $tmp # unsigned division\n\t"
7842             "MOV    EAX,$tmp2\n\t"
7843             "DIV    $tmp\n\t"
7844             "NEG    EDX\n\t"
7845             "JMP,s  done\n"
7846     "pos:\n\t"
7847             "DIV    $tmp\n\t"
7848             "MOV    EAX,$tmp2\n"
7849     "fast:\n\t"
7850             "DIV    $tmp\n"
7851     "done:\n\t"
7852             "MOV    EAX,EDX\n\t"
7853             "SAR    EDX,31\n\t" %}
7854   ins_encode %{
7855     int con = (int)$imm$$constant;
7856     assert(con != 0 && con != -1 && con != min_jint, "wrong divisor");
7857     int pcon = (con > 0) ? con : -con;
7858     Label  Lfast, Lpos, Ldone;
7859 
7860     __ movl($tmp$$Register, pcon);
7861     __ cmpl($tmp$$Register, HIGH_FROM_LOW($dst$$Register));
7862     __ jccb(Assembler::above, Lfast); // src is positive and result fits into 32 bit
7863 
7864     __ movl($tmp2$$Register, $dst$$Register); // save
7865     __ movl($dst$$Register, HIGH_FROM_LOW($dst$$Register));
7866     __ movl(HIGH_FROM_LOW($dst$$Register),0); // preserve flags
7867     __ jccb(Assembler::lessEqual, Lpos); // result is positive
7868 
7869     // Negative dividend.
7870     // convert value to positive to use unsigned division
7871     __ lneg($dst$$Register, $tmp2$$Register);
7872     __ divl($tmp$$Register);
7873     __ movl($dst$$Register, $tmp2$$Register);
7874     __ divl($tmp$$Register);
7875     // revert remainder back to negative
7876     __ negl(HIGH_FROM_LOW($dst$$Register));
7877     __ jmpb(Ldone);
7878 
7879     __ bind(Lpos);
7880     __ divl($tmp$$Register);
7881     __ movl($dst$$Register, $tmp2$$Register);
7882 
7883     __ bind(Lfast);
7884     // fast path: src is positive
7885     __ divl($tmp$$Register);
7886 
7887     __ bind(Ldone);
7888     __ movl($dst$$Register, HIGH_FROM_LOW($dst$$Register));
7889     __ sarl(HIGH_FROM_LOW($dst$$Register), 31); // result sign
7890 
7891   %}
7892   ins_pipe( pipe_slow );
7893 %}
7894 
7895 // Integer Shift Instructions
7896 // Shift Left by one
7897 instruct shlI_eReg_1(rRegI dst, immI1 shift, eFlagsReg cr) %{
7898   match(Set dst (LShiftI dst shift));
7899   effect(KILL cr);
7900 
7901   size(2);
7902   format %{ "SHL    $dst,$shift" %}
7903   opcode(0xD1, 0x4);  /* D1 /4 */
7904   ins_encode( OpcP, RegOpc( dst ) );
7905   ins_pipe( ialu_reg );
7906 %}
7907 
7908 // Shift Left by 8-bit immediate
7909 instruct salI_eReg_imm(rRegI dst, immI8 shift, eFlagsReg cr) %{
7910   match(Set dst (LShiftI dst shift));
7911   effect(KILL cr);
7912 
7913   size(3);
7914   format %{ "SHL    $dst,$shift" %}
7915   opcode(0xC1, 0x4);  /* C1 /4 ib */
7916   ins_encode( RegOpcImm( dst, shift) );
7917   ins_pipe( ialu_reg );
7918 %}
7919 
7920 // Shift Left by variable
7921 instruct salI_eReg_CL(rRegI dst, eCXRegI shift, eFlagsReg cr) %{
7922   match(Set dst (LShiftI dst shift));
7923   effect(KILL cr);
7924 
7925   size(2);
7926   format %{ "SHL    $dst,$shift" %}
7927   opcode(0xD3, 0x4);  /* D3 /4 */
7928   ins_encode( OpcP, RegOpc( dst ) );
7929   ins_pipe( ialu_reg_reg );
7930 %}
7931 
7932 // Arithmetic shift right by one
7933 instruct sarI_eReg_1(rRegI dst, immI1 shift, eFlagsReg cr) %{
7934   match(Set dst (RShiftI dst shift));
7935   effect(KILL cr);
7936 
7937   size(2);
7938   format %{ "SAR    $dst,$shift" %}
7939   opcode(0xD1, 0x7);  /* D1 /7 */
7940   ins_encode( OpcP, RegOpc( dst ) );
7941   ins_pipe( ialu_reg );
7942 %}
7943 
7944 // Arithmetic shift right by one
7945 instruct sarI_mem_1(memory dst, immI1 shift, eFlagsReg cr) %{
7946   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
7947   effect(KILL cr);
7948   format %{ "SAR    $dst,$shift" %}
7949   opcode(0xD1, 0x7);  /* D1 /7 */
7950   ins_encode( OpcP, RMopc_Mem(secondary,dst) );
7951   ins_pipe( ialu_mem_imm );
7952 %}
7953 
7954 // Arithmetic Shift Right by 8-bit immediate
7955 instruct sarI_eReg_imm(rRegI dst, immI8 shift, eFlagsReg cr) %{
7956   match(Set dst (RShiftI dst shift));
7957   effect(KILL cr);
7958 
7959   size(3);
7960   format %{ "SAR    $dst,$shift" %}
7961   opcode(0xC1, 0x7);  /* C1 /7 ib */
7962   ins_encode( RegOpcImm( dst, shift ) );
7963   ins_pipe( ialu_mem_imm );
7964 %}
7965 
7966 // Arithmetic Shift Right by 8-bit immediate
7967 instruct sarI_mem_imm(memory dst, immI8 shift, eFlagsReg cr) %{
7968   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
7969   effect(KILL cr);
7970 
7971   format %{ "SAR    $dst,$shift" %}
7972   opcode(0xC1, 0x7);  /* C1 /7 ib */
7973   ins_encode( OpcP, RMopc_Mem(secondary, dst ), Con8or32( shift ) );
7974   ins_pipe( ialu_mem_imm );
7975 %}
7976 
7977 // Arithmetic Shift Right by variable
7978 instruct sarI_eReg_CL(rRegI dst, eCXRegI shift, eFlagsReg cr) %{
7979   match(Set dst (RShiftI dst shift));
7980   effect(KILL cr);
7981 
7982   size(2);
7983   format %{ "SAR    $dst,$shift" %}
7984   opcode(0xD3, 0x7);  /* D3 /7 */
7985   ins_encode( OpcP, RegOpc( dst ) );
7986   ins_pipe( ialu_reg_reg );
7987 %}
7988 
7989 // Logical shift right by one
7990 instruct shrI_eReg_1(rRegI dst, immI1 shift, eFlagsReg cr) %{
7991   match(Set dst (URShiftI dst shift));
7992   effect(KILL cr);
7993 
7994   size(2);
7995   format %{ "SHR    $dst,$shift" %}
7996   opcode(0xD1, 0x5);  /* D1 /5 */
7997   ins_encode( OpcP, RegOpc( dst ) );
7998   ins_pipe( ialu_reg );
7999 %}
8000 
8001 // Logical Shift Right by 8-bit immediate
8002 instruct shrI_eReg_imm(rRegI dst, immI8 shift, eFlagsReg cr) %{
8003   match(Set dst (URShiftI dst shift));
8004   effect(KILL cr);
8005 
8006   size(3);
8007   format %{ "SHR    $dst,$shift" %}
8008   opcode(0xC1, 0x5);  /* C1 /5 ib */
8009   ins_encode( RegOpcImm( dst, shift) );
8010   ins_pipe( ialu_reg );
8011 %}
8012 
8013 
8014 // Logical Shift Right by 24, followed by Arithmetic Shift Left by 24.
8015 // This idiom is used by the compiler for the i2b bytecode.
8016 instruct i2b(rRegI dst, xRegI src, immI_24 twentyfour) %{
8017   match(Set dst (RShiftI (LShiftI src twentyfour) twentyfour));
8018 
8019   size(3);
8020   format %{ "MOVSX  $dst,$src :8" %}
8021   ins_encode %{
8022     __ movsbl($dst$$Register, $src$$Register);
8023   %}
8024   ins_pipe(ialu_reg_reg);
8025 %}
8026 
8027 // Logical Shift Right by 16, followed by Arithmetic Shift Left by 16.
8028 // This idiom is used by the compiler the i2s bytecode.
8029 instruct i2s(rRegI dst, xRegI src, immI_16 sixteen) %{
8030   match(Set dst (RShiftI (LShiftI src sixteen) sixteen));
8031 
8032   size(3);
8033   format %{ "MOVSX  $dst,$src :16" %}
8034   ins_encode %{
8035     __ movswl($dst$$Register, $src$$Register);
8036   %}
8037   ins_pipe(ialu_reg_reg);
8038 %}
8039 
8040 
8041 // Logical Shift Right by variable
8042 instruct shrI_eReg_CL(rRegI dst, eCXRegI shift, eFlagsReg cr) %{
8043   match(Set dst (URShiftI dst shift));
8044   effect(KILL cr);
8045 
8046   size(2);
8047   format %{ "SHR    $dst,$shift" %}
8048   opcode(0xD3, 0x5);  /* D3 /5 */
8049   ins_encode( OpcP, RegOpc( dst ) );
8050   ins_pipe( ialu_reg_reg );
8051 %}
8052 
8053 
8054 //----------Logical Instructions-----------------------------------------------
8055 //----------Integer Logical Instructions---------------------------------------
8056 // And Instructions
8057 // And Register with Register
8058 instruct andI_eReg(rRegI dst, rRegI src, eFlagsReg cr) %{
8059   match(Set dst (AndI dst src));
8060   effect(KILL cr);
8061 
8062   size(2);
8063   format %{ "AND    $dst,$src" %}
8064   opcode(0x23);
8065   ins_encode( OpcP, RegReg( dst, src) );
8066   ins_pipe( ialu_reg_reg );
8067 %}
8068 
8069 // And Register with Immediate
8070 instruct andI_eReg_imm(rRegI dst, immI src, eFlagsReg cr) %{
8071   match(Set dst (AndI dst src));
8072   effect(KILL cr);
8073 
8074   format %{ "AND    $dst,$src" %}
8075   opcode(0x81,0x04);  /* Opcode 81 /4 */
8076   // ins_encode( RegImm( dst, src) );
8077   ins_encode( OpcSErm( dst, src ), Con8or32( src ) );
8078   ins_pipe( ialu_reg );
8079 %}
8080 
8081 // And Register with Memory
8082 instruct andI_eReg_mem(rRegI dst, memory src, eFlagsReg cr) %{
8083   match(Set dst (AndI dst (LoadI src)));
8084   effect(KILL cr);
8085 
8086   ins_cost(125);
8087   format %{ "AND    $dst,$src" %}
8088   opcode(0x23);
8089   ins_encode( OpcP, RegMem( dst, src) );
8090   ins_pipe( ialu_reg_mem );
8091 %}
8092 
8093 // And Memory with Register
8094 instruct andI_mem_eReg(memory dst, rRegI src, eFlagsReg cr) %{
8095   match(Set dst (StoreI dst (AndI (LoadI dst) src)));
8096   effect(KILL cr);
8097 
8098   ins_cost(150);
8099   format %{ "AND    $dst,$src" %}
8100   opcode(0x21);  /* Opcode 21 /r */
8101   ins_encode( OpcP, RegMem( src, dst ) );
8102   ins_pipe( ialu_mem_reg );
8103 %}
8104 
8105 // And Memory with Immediate
8106 instruct andI_mem_imm(memory dst, immI src, eFlagsReg cr) %{
8107   match(Set dst (StoreI dst (AndI (LoadI dst) src)));
8108   effect(KILL cr);
8109 
8110   ins_cost(125);
8111   format %{ "AND    $dst,$src" %}
8112   opcode(0x81, 0x4);  /* Opcode 81 /4 id */
8113   // ins_encode( MemImm( dst, src) );
8114   ins_encode( OpcSE( src ), RMopc_Mem(secondary, dst ), Con8or32( src ) );
8115   ins_pipe( ialu_mem_imm );
8116 %}
8117 
8118 // BMI1 instructions
8119 instruct andnI_rReg_rReg_rReg(rRegI dst, rRegI src1, rRegI src2, immI_M1 minus_1, eFlagsReg cr) %{
8120   match(Set dst (AndI (XorI src1 minus_1) src2));
8121   predicate(UseBMI1Instructions);
8122   effect(KILL cr);
8123 
8124   format %{ "ANDNL  $dst, $src1, $src2" %}
8125 
8126   ins_encode %{
8127     __ andnl($dst$$Register, $src1$$Register, $src2$$Register);
8128   %}
8129   ins_pipe(ialu_reg);
8130 %}
8131 
8132 instruct andnI_rReg_rReg_mem(rRegI dst, rRegI src1, memory src2, immI_M1 minus_1, eFlagsReg cr) %{
8133   match(Set dst (AndI (XorI src1 minus_1) (LoadI src2) ));
8134   predicate(UseBMI1Instructions);
8135   effect(KILL cr);
8136 
8137   ins_cost(125);
8138   format %{ "ANDNL  $dst, $src1, $src2" %}
8139 
8140   ins_encode %{
8141     __ andnl($dst$$Register, $src1$$Register, $src2$$Address);
8142   %}
8143   ins_pipe(ialu_reg_mem);
8144 %}
8145 
8146 instruct blsiI_rReg_rReg(rRegI dst, rRegI src, immI0 imm_zero, eFlagsReg cr) %{
8147   match(Set dst (AndI (SubI imm_zero src) src));
8148   predicate(UseBMI1Instructions);
8149   effect(KILL cr);
8150 
8151   format %{ "BLSIL  $dst, $src" %}
8152 
8153   ins_encode %{
8154     __ blsil($dst$$Register, $src$$Register);
8155   %}
8156   ins_pipe(ialu_reg);
8157 %}
8158 
8159 instruct blsiI_rReg_mem(rRegI dst, memory src, immI0 imm_zero, eFlagsReg cr) %{
8160   match(Set dst (AndI (SubI imm_zero (LoadI src) ) (LoadI src) ));
8161   predicate(UseBMI1Instructions);
8162   effect(KILL cr);
8163 
8164   ins_cost(125);
8165   format %{ "BLSIL  $dst, $src" %}
8166 
8167   ins_encode %{
8168     __ blsil($dst$$Register, $src$$Address);
8169   %}
8170   ins_pipe(ialu_reg_mem);
8171 %}
8172 
8173 instruct blsmskI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, eFlagsReg cr)
8174 %{
8175   match(Set dst (XorI (AddI src minus_1) src));
8176   predicate(UseBMI1Instructions);
8177   effect(KILL cr);
8178 
8179   format %{ "BLSMSKL $dst, $src" %}
8180 
8181   ins_encode %{
8182     __ blsmskl($dst$$Register, $src$$Register);
8183   %}
8184 
8185   ins_pipe(ialu_reg);
8186 %}
8187 
8188 instruct blsmskI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, eFlagsReg cr)
8189 %{
8190   match(Set dst (XorI (AddI (LoadI src) minus_1) (LoadI src) ));
8191   predicate(UseBMI1Instructions);
8192   effect(KILL cr);
8193 
8194   ins_cost(125);
8195   format %{ "BLSMSKL $dst, $src" %}
8196 
8197   ins_encode %{
8198     __ blsmskl($dst$$Register, $src$$Address);
8199   %}
8200 
8201   ins_pipe(ialu_reg_mem);
8202 %}
8203 
8204 instruct blsrI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, eFlagsReg cr)
8205 %{
8206   match(Set dst (AndI (AddI src minus_1) src) );
8207   predicate(UseBMI1Instructions);
8208   effect(KILL cr);
8209 
8210   format %{ "BLSRL  $dst, $src" %}
8211 
8212   ins_encode %{
8213     __ blsrl($dst$$Register, $src$$Register);
8214   %}
8215 
8216   ins_pipe(ialu_reg);
8217 %}
8218 
8219 instruct blsrI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, eFlagsReg cr)
8220 %{
8221   match(Set dst (AndI (AddI (LoadI src) minus_1) (LoadI src) ));
8222   predicate(UseBMI1Instructions);
8223   effect(KILL cr);
8224 
8225   ins_cost(125);
8226   format %{ "BLSRL  $dst, $src" %}
8227 
8228   ins_encode %{
8229     __ blsrl($dst$$Register, $src$$Address);
8230   %}
8231 
8232   ins_pipe(ialu_reg_mem);
8233 %}
8234 
8235 // Or Instructions
8236 // Or Register with Register
8237 instruct orI_eReg(rRegI dst, rRegI src, eFlagsReg cr) %{
8238   match(Set dst (OrI dst src));
8239   effect(KILL cr);
8240 
8241   size(2);
8242   format %{ "OR     $dst,$src" %}
8243   opcode(0x0B);
8244   ins_encode( OpcP, RegReg( dst, src) );
8245   ins_pipe( ialu_reg_reg );
8246 %}
8247 
8248 instruct orI_eReg_castP2X(rRegI dst, eRegP src, eFlagsReg cr) %{
8249   match(Set dst (OrI dst (CastP2X src)));
8250   effect(KILL cr);
8251 
8252   size(2);
8253   format %{ "OR     $dst,$src" %}
8254   opcode(0x0B);
8255   ins_encode( OpcP, RegReg( dst, src) );
8256   ins_pipe( ialu_reg_reg );
8257 %}
8258 
8259 
8260 // Or Register with Immediate
8261 instruct orI_eReg_imm(rRegI dst, immI src, eFlagsReg cr) %{
8262   match(Set dst (OrI dst src));
8263   effect(KILL cr);
8264 
8265   format %{ "OR     $dst,$src" %}
8266   opcode(0x81,0x01);  /* Opcode 81 /1 id */
8267   // ins_encode( RegImm( dst, src) );
8268   ins_encode( OpcSErm( dst, src ), Con8or32( src ) );
8269   ins_pipe( ialu_reg );
8270 %}
8271 
8272 // Or Register with Memory
8273 instruct orI_eReg_mem(rRegI dst, memory src, eFlagsReg cr) %{
8274   match(Set dst (OrI dst (LoadI src)));
8275   effect(KILL cr);
8276 
8277   ins_cost(125);
8278   format %{ "OR     $dst,$src" %}
8279   opcode(0x0B);
8280   ins_encode( OpcP, RegMem( dst, src) );
8281   ins_pipe( ialu_reg_mem );
8282 %}
8283 
8284 // Or Memory with Register
8285 instruct orI_mem_eReg(memory dst, rRegI src, eFlagsReg cr) %{
8286   match(Set dst (StoreI dst (OrI (LoadI dst) src)));
8287   effect(KILL cr);
8288 
8289   ins_cost(150);
8290   format %{ "OR     $dst,$src" %}
8291   opcode(0x09);  /* Opcode 09 /r */
8292   ins_encode( OpcP, RegMem( src, dst ) );
8293   ins_pipe( ialu_mem_reg );
8294 %}
8295 
8296 // Or Memory with Immediate
8297 instruct orI_mem_imm(memory dst, immI src, eFlagsReg cr) %{
8298   match(Set dst (StoreI dst (OrI (LoadI dst) src)));
8299   effect(KILL cr);
8300 
8301   ins_cost(125);
8302   format %{ "OR     $dst,$src" %}
8303   opcode(0x81,0x1);  /* Opcode 81 /1 id */
8304   // ins_encode( MemImm( dst, src) );
8305   ins_encode( OpcSE( src ), RMopc_Mem(secondary, dst ), Con8or32( src ) );
8306   ins_pipe( ialu_mem_imm );
8307 %}
8308 
8309 // ROL/ROR
8310 // ROL expand
8311 instruct rolI_eReg_imm1(rRegI dst, immI1 shift, eFlagsReg cr) %{
8312   effect(USE_DEF dst, USE shift, KILL cr);
8313 
8314   format %{ "ROL    $dst, $shift" %}
8315   opcode(0xD1, 0x0); /* Opcode D1 /0 */
8316   ins_encode( OpcP, RegOpc( dst ));
8317   ins_pipe( ialu_reg );
8318 %}
8319 
8320 instruct rolI_eReg_imm8(rRegI dst, immI8 shift, eFlagsReg cr) %{
8321   effect(USE_DEF dst, USE shift, KILL cr);
8322 
8323   format %{ "ROL    $dst, $shift" %}
8324   opcode(0xC1, 0x0); /*Opcode /C1  /0  */
8325   ins_encode( RegOpcImm(dst, shift) );
8326   ins_pipe(ialu_reg);
8327 %}
8328 
8329 instruct rolI_eReg_CL(ncxRegI dst, eCXRegI shift, eFlagsReg cr) %{
8330   effect(USE_DEF dst, USE shift, KILL cr);
8331 
8332   format %{ "ROL    $dst, $shift" %}
8333   opcode(0xD3, 0x0);    /* Opcode D3 /0 */
8334   ins_encode(OpcP, RegOpc(dst));
8335   ins_pipe( ialu_reg_reg );
8336 %}
8337 // end of ROL expand
8338 
8339 // ROL 32bit by one once
8340 instruct rolI_eReg_i1(rRegI dst, immI1 lshift, immI_M1 rshift, eFlagsReg cr) %{
8341   match(Set dst ( OrI (LShiftI dst lshift) (URShiftI dst rshift)));
8342 
8343   expand %{
8344     rolI_eReg_imm1(dst, lshift, cr);
8345   %}
8346 %}
8347 
8348 // ROL 32bit var by imm8 once
8349 instruct rolI_eReg_i8(rRegI dst, immI8 lshift, immI8 rshift, eFlagsReg cr) %{
8350   predicate(  0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x1f));
8351   match(Set dst ( OrI (LShiftI dst lshift) (URShiftI dst rshift)));
8352 
8353   expand %{
8354     rolI_eReg_imm8(dst, lshift, cr);
8355   %}
8356 %}
8357 
8358 // ROL 32bit var by var once
8359 instruct rolI_eReg_Var_C0(ncxRegI dst, eCXRegI shift, immI0 zero, eFlagsReg cr) %{
8360   match(Set dst ( OrI (LShiftI dst shift) (URShiftI dst (SubI zero shift))));
8361 
8362   expand %{
8363     rolI_eReg_CL(dst, shift, cr);
8364   %}
8365 %}
8366 
8367 // ROL 32bit var by var once
8368 instruct rolI_eReg_Var_C32(ncxRegI dst, eCXRegI shift, immI_32 c32, eFlagsReg cr) %{
8369   match(Set dst ( OrI (LShiftI dst shift) (URShiftI dst (SubI c32 shift))));
8370 
8371   expand %{
8372     rolI_eReg_CL(dst, shift, cr);
8373   %}
8374 %}
8375 
8376 // ROR expand
8377 instruct rorI_eReg_imm1(rRegI dst, immI1 shift, eFlagsReg cr) %{
8378   effect(USE_DEF dst, USE shift, KILL cr);
8379 
8380   format %{ "ROR    $dst, $shift" %}
8381   opcode(0xD1,0x1);  /* Opcode D1 /1 */
8382   ins_encode( OpcP, RegOpc( dst ) );
8383   ins_pipe( ialu_reg );
8384 %}
8385 
8386 instruct rorI_eReg_imm8(rRegI dst, immI8 shift, eFlagsReg cr) %{
8387   effect (USE_DEF dst, USE shift, KILL cr);
8388 
8389   format %{ "ROR    $dst, $shift" %}
8390   opcode(0xC1, 0x1); /* Opcode /C1 /1 ib */
8391   ins_encode( RegOpcImm(dst, shift) );
8392   ins_pipe( ialu_reg );
8393 %}
8394 
8395 instruct rorI_eReg_CL(ncxRegI dst, eCXRegI shift, eFlagsReg cr)%{
8396   effect(USE_DEF dst, USE shift, KILL cr);
8397 
8398   format %{ "ROR    $dst, $shift" %}
8399   opcode(0xD3, 0x1);    /* Opcode D3 /1 */
8400   ins_encode(OpcP, RegOpc(dst));
8401   ins_pipe( ialu_reg_reg );
8402 %}
8403 // end of ROR expand
8404 
8405 // ROR right once
8406 instruct rorI_eReg_i1(rRegI dst, immI1 rshift, immI_M1 lshift, eFlagsReg cr) %{
8407   match(Set dst ( OrI (URShiftI dst rshift) (LShiftI dst lshift)));
8408 
8409   expand %{
8410     rorI_eReg_imm1(dst, rshift, cr);
8411   %}
8412 %}
8413 
8414 // ROR 32bit by immI8 once
8415 instruct rorI_eReg_i8(rRegI dst, immI8 rshift, immI8 lshift, eFlagsReg cr) %{
8416   predicate(  0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x1f));
8417   match(Set dst ( OrI (URShiftI dst rshift) (LShiftI dst lshift)));
8418 
8419   expand %{
8420     rorI_eReg_imm8(dst, rshift, cr);
8421   %}
8422 %}
8423 
8424 // ROR 32bit var by var once
8425 instruct rorI_eReg_Var_C0(ncxRegI dst, eCXRegI shift, immI0 zero, eFlagsReg cr) %{
8426   match(Set dst ( OrI (URShiftI dst shift) (LShiftI dst (SubI zero shift))));
8427 
8428   expand %{
8429     rorI_eReg_CL(dst, shift, cr);
8430   %}
8431 %}
8432 
8433 // ROR 32bit var by var once
8434 instruct rorI_eReg_Var_C32(ncxRegI dst, eCXRegI shift, immI_32 c32, eFlagsReg cr) %{
8435   match(Set dst ( OrI (URShiftI dst shift) (LShiftI dst (SubI c32 shift))));
8436 
8437   expand %{
8438     rorI_eReg_CL(dst, shift, cr);
8439   %}
8440 %}
8441 
8442 // Xor Instructions
8443 // Xor Register with Register
8444 instruct xorI_eReg(rRegI dst, rRegI src, eFlagsReg cr) %{
8445   match(Set dst (XorI dst src));
8446   effect(KILL cr);
8447 
8448   size(2);
8449   format %{ "XOR    $dst,$src" %}
8450   opcode(0x33);
8451   ins_encode( OpcP, RegReg( dst, src) );
8452   ins_pipe( ialu_reg_reg );
8453 %}
8454 
8455 // Xor Register with Immediate -1
8456 instruct xorI_eReg_im1(rRegI dst, immI_M1 imm) %{
8457   match(Set dst (XorI dst imm));  
8458 
8459   size(2);
8460   format %{ "NOT    $dst" %}  
8461   ins_encode %{
8462      __ notl($dst$$Register);
8463   %}
8464   ins_pipe( ialu_reg );
8465 %}
8466 
8467 // Xor Register with Immediate
8468 instruct xorI_eReg_imm(rRegI dst, immI src, eFlagsReg cr) %{
8469   match(Set dst (XorI dst src));
8470   effect(KILL cr);
8471 
8472   format %{ "XOR    $dst,$src" %}
8473   opcode(0x81,0x06);  /* Opcode 81 /6 id */
8474   // ins_encode( RegImm( dst, src) );
8475   ins_encode( OpcSErm( dst, src ), Con8or32( src ) );
8476   ins_pipe( ialu_reg );
8477 %}
8478 
8479 // Xor Register with Memory
8480 instruct xorI_eReg_mem(rRegI dst, memory src, eFlagsReg cr) %{
8481   match(Set dst (XorI dst (LoadI src)));
8482   effect(KILL cr);
8483 
8484   ins_cost(125);
8485   format %{ "XOR    $dst,$src" %}
8486   opcode(0x33);
8487   ins_encode( OpcP, RegMem(dst, src) );
8488   ins_pipe( ialu_reg_mem );
8489 %}
8490 
8491 // Xor Memory with Register
8492 instruct xorI_mem_eReg(memory dst, rRegI src, eFlagsReg cr) %{
8493   match(Set dst (StoreI dst (XorI (LoadI dst) src)));
8494   effect(KILL cr);
8495 
8496   ins_cost(150);
8497   format %{ "XOR    $dst,$src" %}
8498   opcode(0x31);  /* Opcode 31 /r */
8499   ins_encode( OpcP, RegMem( src, dst ) );
8500   ins_pipe( ialu_mem_reg );
8501 %}
8502 
8503 // Xor Memory with Immediate
8504 instruct xorI_mem_imm(memory dst, immI src, eFlagsReg cr) %{
8505   match(Set dst (StoreI dst (XorI (LoadI dst) src)));
8506   effect(KILL cr);
8507 
8508   ins_cost(125);
8509   format %{ "XOR    $dst,$src" %}
8510   opcode(0x81,0x6);  /* Opcode 81 /6 id */
8511   ins_encode( OpcSE( src ), RMopc_Mem(secondary, dst ), Con8or32( src ) );
8512   ins_pipe( ialu_mem_imm );
8513 %}
8514 
8515 //----------Convert Int to Boolean---------------------------------------------
8516 
8517 instruct movI_nocopy(rRegI dst, rRegI src) %{
8518   effect( DEF dst, USE src );
8519   format %{ "MOV    $dst,$src" %}
8520   ins_encode( enc_Copy( dst, src) );
8521   ins_pipe( ialu_reg_reg );
8522 %}
8523 
8524 instruct ci2b( rRegI dst, rRegI src, eFlagsReg cr ) %{
8525   effect( USE_DEF dst, USE src, KILL cr );
8526 
8527   size(4);
8528   format %{ "NEG    $dst\n\t"
8529             "ADC    $dst,$src" %}
8530   ins_encode( neg_reg(dst),
8531               OpcRegReg(0x13,dst,src) );
8532   ins_pipe( ialu_reg_reg_long );
8533 %}
8534 
8535 instruct convI2B( rRegI dst, rRegI src, eFlagsReg cr ) %{
8536   match(Set dst (Conv2B src));
8537 
8538   expand %{
8539     movI_nocopy(dst,src);
8540     ci2b(dst,src,cr);
8541   %}
8542 %}
8543 
8544 instruct movP_nocopy(rRegI dst, eRegP src) %{
8545   effect( DEF dst, USE src );
8546   format %{ "MOV    $dst,$src" %}
8547   ins_encode( enc_Copy( dst, src) );
8548   ins_pipe( ialu_reg_reg );
8549 %}
8550 
8551 instruct cp2b( rRegI dst, eRegP src, eFlagsReg cr ) %{
8552   effect( USE_DEF dst, USE src, KILL cr );
8553   format %{ "NEG    $dst\n\t"
8554             "ADC    $dst,$src" %}
8555   ins_encode( neg_reg(dst),
8556               OpcRegReg(0x13,dst,src) );
8557   ins_pipe( ialu_reg_reg_long );
8558 %}
8559 
8560 instruct convP2B( rRegI dst, eRegP src, eFlagsReg cr ) %{
8561   match(Set dst (Conv2B src));
8562 
8563   expand %{
8564     movP_nocopy(dst,src);
8565     cp2b(dst,src,cr);
8566   %}
8567 %}
8568 
8569 instruct cmpLTMask(eCXRegI dst, ncxRegI p, ncxRegI q, eFlagsReg cr) %{
8570   match(Set dst (CmpLTMask p q));
8571   effect(KILL cr);
8572   ins_cost(400);
8573 
8574   // SETlt can only use low byte of EAX,EBX, ECX, or EDX as destination
8575   format %{ "XOR    $dst,$dst\n\t"
8576             "CMP    $p,$q\n\t"
8577             "SETlt  $dst\n\t"
8578             "NEG    $dst" %}
8579   ins_encode %{
8580     Register Rp = $p$$Register;
8581     Register Rq = $q$$Register;
8582     Register Rd = $dst$$Register;
8583     Label done;
8584     __ xorl(Rd, Rd);
8585     __ cmpl(Rp, Rq);
8586     __ setb(Assembler::less, Rd);
8587     __ negl(Rd);
8588   %}
8589 
8590   ins_pipe(pipe_slow);
8591 %}
8592 
8593 instruct cmpLTMask0(rRegI dst, immI0 zero, eFlagsReg cr) %{
8594   match(Set dst (CmpLTMask dst zero));
8595   effect(DEF dst, KILL cr);
8596   ins_cost(100);
8597 
8598   format %{ "SAR    $dst,31\t# cmpLTMask0" %}
8599   ins_encode %{
8600   __ sarl($dst$$Register, 31);
8601   %}
8602   ins_pipe(ialu_reg);
8603 %}
8604 
8605 /* better to save a register than avoid a branch */
8606 instruct cadd_cmpLTMask(rRegI p, rRegI q, rRegI y, eFlagsReg cr) %{
8607   match(Set p (AddI (AndI (CmpLTMask p q) y) (SubI p q)));
8608   effect(KILL cr);
8609   ins_cost(400);
8610   format %{ "SUB    $p,$q\t# cadd_cmpLTMask\n\t"
8611             "JGE    done\n\t"
8612             "ADD    $p,$y\n"
8613             "done:  " %}
8614   ins_encode %{
8615     Register Rp = $p$$Register;
8616     Register Rq = $q$$Register;
8617     Register Ry = $y$$Register;
8618     Label done;
8619     __ subl(Rp, Rq);
8620     __ jccb(Assembler::greaterEqual, done);
8621     __ addl(Rp, Ry);
8622     __ bind(done);
8623   %}
8624 
8625   ins_pipe(pipe_cmplt);
8626 %}
8627 
8628 /* better to save a register than avoid a branch */
8629 instruct and_cmpLTMask(rRegI p, rRegI q, rRegI y, eFlagsReg cr) %{
8630   match(Set y (AndI (CmpLTMask p q) y));
8631   effect(KILL cr);
8632 
8633   ins_cost(300);
8634 
8635   format %{ "CMPL     $p, $q\t# and_cmpLTMask\n\t"
8636             "JLT      done\n\t"
8637             "XORL     $y, $y\n"
8638             "done:  " %}
8639   ins_encode %{
8640     Register Rp = $p$$Register;
8641     Register Rq = $q$$Register;
8642     Register Ry = $y$$Register;
8643     Label done;
8644     __ cmpl(Rp, Rq);
8645     __ jccb(Assembler::less, done);
8646     __ xorl(Ry, Ry);
8647     __ bind(done);
8648   %}
8649 
8650   ins_pipe(pipe_cmplt);
8651 %}
8652 
8653 /* If I enable this, I encourage spilling in the inner loop of compress.
8654 instruct cadd_cmpLTMask_mem(ncxRegI p, ncxRegI q, memory y, eCXRegI tmp, eFlagsReg cr) %{
8655   match(Set p (AddI (AndI (CmpLTMask p q) (LoadI y)) (SubI p q)));
8656 */
8657 //----------Overflow Math Instructions-----------------------------------------
8658 
8659 instruct overflowAddI_eReg(eFlagsReg cr, eAXRegI op1, rRegI op2)
8660 %{
8661   match(Set cr (OverflowAddI op1 op2));
8662   effect(DEF cr, USE_KILL op1, USE op2);
8663 
8664   format %{ "ADD    $op1, $op2\t# overflow check int" %}
8665 
8666   ins_encode %{
8667     __ addl($op1$$Register, $op2$$Register);
8668   %}
8669   ins_pipe(ialu_reg_reg);
8670 %}
8671 
8672 instruct overflowAddI_rReg_imm(eFlagsReg cr, eAXRegI op1, immI op2)
8673 %{
8674   match(Set cr (OverflowAddI op1 op2));
8675   effect(DEF cr, USE_KILL op1, USE op2);
8676 
8677   format %{ "ADD    $op1, $op2\t# overflow check int" %}
8678 
8679   ins_encode %{
8680     __ addl($op1$$Register, $op2$$constant);
8681   %}
8682   ins_pipe(ialu_reg_reg);
8683 %}
8684 
8685 instruct overflowSubI_rReg(eFlagsReg cr, rRegI op1, rRegI op2)
8686 %{
8687   match(Set cr (OverflowSubI op1 op2));
8688 
8689   format %{ "CMP    $op1, $op2\t# overflow check int" %}
8690   ins_encode %{
8691     __ cmpl($op1$$Register, $op2$$Register);
8692   %}
8693   ins_pipe(ialu_reg_reg);
8694 %}
8695 
8696 instruct overflowSubI_rReg_imm(eFlagsReg cr, rRegI op1, immI op2)
8697 %{
8698   match(Set cr (OverflowSubI op1 op2));
8699 
8700   format %{ "CMP    $op1, $op2\t# overflow check int" %}
8701   ins_encode %{
8702     __ cmpl($op1$$Register, $op2$$constant);
8703   %}
8704   ins_pipe(ialu_reg_reg);
8705 %}
8706 
8707 instruct overflowNegI_rReg(eFlagsReg cr, immI0 zero, eAXRegI op2)
8708 %{
8709   match(Set cr (OverflowSubI zero op2));
8710   effect(DEF cr, USE_KILL op2);
8711 
8712   format %{ "NEG    $op2\t# overflow check int" %}
8713   ins_encode %{
8714     __ negl($op2$$Register);
8715   %}
8716   ins_pipe(ialu_reg_reg);
8717 %}
8718 
8719 instruct overflowMulI_rReg(eFlagsReg cr, eAXRegI op1, rRegI op2)
8720 %{
8721   match(Set cr (OverflowMulI op1 op2));
8722   effect(DEF cr, USE_KILL op1, USE op2);
8723 
8724   format %{ "IMUL    $op1, $op2\t# overflow check int" %}
8725   ins_encode %{
8726     __ imull($op1$$Register, $op2$$Register);
8727   %}
8728   ins_pipe(ialu_reg_reg_alu0);
8729 %}
8730 
8731 instruct overflowMulI_rReg_imm(eFlagsReg cr, rRegI op1, immI op2, rRegI tmp)
8732 %{
8733   match(Set cr (OverflowMulI op1 op2));
8734   effect(DEF cr, TEMP tmp, USE op1, USE op2);
8735 
8736   format %{ "IMUL    $tmp, $op1, $op2\t# overflow check int" %}
8737   ins_encode %{
8738     __ imull($tmp$$Register, $op1$$Register, $op2$$constant);
8739   %}
8740   ins_pipe(ialu_reg_reg_alu0);
8741 %}
8742 
8743 //----------Long Instructions------------------------------------------------
8744 // Add Long Register with Register
8745 instruct addL_eReg(eRegL dst, eRegL src, eFlagsReg cr) %{
8746   match(Set dst (AddL dst src));
8747   effect(KILL cr);
8748   ins_cost(200);
8749   format %{ "ADD    $dst.lo,$src.lo\n\t"
8750             "ADC    $dst.hi,$src.hi" %}
8751   opcode(0x03, 0x13);
8752   ins_encode( RegReg_Lo(dst, src), RegReg_Hi(dst,src) );
8753   ins_pipe( ialu_reg_reg_long );
8754 %}
8755 
8756 // Add Long Register with Immediate
8757 instruct addL_eReg_imm(eRegL dst, immL src, eFlagsReg cr) %{
8758   match(Set dst (AddL dst src));
8759   effect(KILL cr);
8760   format %{ "ADD    $dst.lo,$src.lo\n\t"
8761             "ADC    $dst.hi,$src.hi" %}
8762   opcode(0x81,0x00,0x02);  /* Opcode 81 /0, 81 /2 */
8763   ins_encode( Long_OpcSErm_Lo( dst, src ), Long_OpcSErm_Hi( dst, src ) );
8764   ins_pipe( ialu_reg_long );
8765 %}
8766 
8767 // Add Long Register with Memory
8768 instruct addL_eReg_mem(eRegL dst, load_long_memory mem, eFlagsReg cr) %{
8769   match(Set dst (AddL dst (LoadL mem)));
8770   effect(KILL cr);
8771   ins_cost(125);
8772   format %{ "ADD    $dst.lo,$mem\n\t"
8773             "ADC    $dst.hi,$mem+4" %}
8774   opcode(0x03, 0x13);
8775   ins_encode( OpcP, RegMem( dst, mem), OpcS, RegMem_Hi(dst,mem) );
8776   ins_pipe( ialu_reg_long_mem );
8777 %}
8778 
8779 // Subtract Long Register with Register.
8780 instruct subL_eReg(eRegL dst, eRegL src, eFlagsReg cr) %{
8781   match(Set dst (SubL dst src));
8782   effect(KILL cr);
8783   ins_cost(200);
8784   format %{ "SUB    $dst.lo,$src.lo\n\t"
8785             "SBB    $dst.hi,$src.hi" %}
8786   opcode(0x2B, 0x1B);
8787   ins_encode( RegReg_Lo(dst, src), RegReg_Hi(dst,src) );
8788   ins_pipe( ialu_reg_reg_long );
8789 %}
8790 
8791 // Subtract Long Register with Immediate
8792 instruct subL_eReg_imm(eRegL dst, immL src, eFlagsReg cr) %{
8793   match(Set dst (SubL dst src));
8794   effect(KILL cr);
8795   format %{ "SUB    $dst.lo,$src.lo\n\t"
8796             "SBB    $dst.hi,$src.hi" %}
8797   opcode(0x81,0x05,0x03);  /* Opcode 81 /5, 81 /3 */
8798   ins_encode( Long_OpcSErm_Lo( dst, src ), Long_OpcSErm_Hi( dst, src ) );
8799   ins_pipe( ialu_reg_long );
8800 %}
8801 
8802 // Subtract Long Register with Memory
8803 instruct subL_eReg_mem(eRegL dst, load_long_memory mem, eFlagsReg cr) %{
8804   match(Set dst (SubL dst (LoadL mem)));
8805   effect(KILL cr);
8806   ins_cost(125);
8807   format %{ "SUB    $dst.lo,$mem\n\t"
8808             "SBB    $dst.hi,$mem+4" %}
8809   opcode(0x2B, 0x1B);
8810   ins_encode( OpcP, RegMem( dst, mem), OpcS, RegMem_Hi(dst,mem) );
8811   ins_pipe( ialu_reg_long_mem );
8812 %}
8813 
8814 instruct negL_eReg(eRegL dst, immL0 zero, eFlagsReg cr) %{
8815   match(Set dst (SubL zero dst));
8816   effect(KILL cr);
8817   ins_cost(300);
8818   format %{ "NEG    $dst.hi\n\tNEG    $dst.lo\n\tSBB    $dst.hi,0" %}
8819   ins_encode( neg_long(dst) );
8820   ins_pipe( ialu_reg_reg_long );
8821 %}
8822 
8823 // And Long Register with Register
8824 instruct andL_eReg(eRegL dst, eRegL src, eFlagsReg cr) %{
8825   match(Set dst (AndL dst src));
8826   effect(KILL cr);
8827   format %{ "AND    $dst.lo,$src.lo\n\t"
8828             "AND    $dst.hi,$src.hi" %}
8829   opcode(0x23,0x23);
8830   ins_encode( RegReg_Lo( dst, src), RegReg_Hi( dst, src) );
8831   ins_pipe( ialu_reg_reg_long );
8832 %}
8833 
8834 // And Long Register with Immediate
8835 instruct andL_eReg_imm(eRegL dst, immL src, eFlagsReg cr) %{
8836   match(Set dst (AndL dst src));
8837   effect(KILL cr);
8838   format %{ "AND    $dst.lo,$src.lo\n\t"
8839             "AND    $dst.hi,$src.hi" %}
8840   opcode(0x81,0x04,0x04);  /* Opcode 81 /4, 81 /4 */
8841   ins_encode( Long_OpcSErm_Lo( dst, src ), Long_OpcSErm_Hi( dst, src ) );
8842   ins_pipe( ialu_reg_long );
8843 %}
8844 
8845 // And Long Register with Memory
8846 instruct andL_eReg_mem(eRegL dst, load_long_memory mem, eFlagsReg cr) %{
8847   match(Set dst (AndL dst (LoadL mem)));
8848   effect(KILL cr);
8849   ins_cost(125);
8850   format %{ "AND    $dst.lo,$mem\n\t"
8851             "AND    $dst.hi,$mem+4" %}
8852   opcode(0x23, 0x23);
8853   ins_encode( OpcP, RegMem( dst, mem), OpcS, RegMem_Hi(dst,mem) );
8854   ins_pipe( ialu_reg_long_mem );
8855 %}
8856 
8857 // BMI1 instructions
8858 instruct andnL_eReg_eReg_eReg(eRegL dst, eRegL src1, eRegL src2, immL_M1 minus_1, eFlagsReg cr) %{
8859   match(Set dst (AndL (XorL src1 minus_1) src2));
8860   predicate(UseBMI1Instructions);
8861   effect(KILL cr, TEMP dst);
8862 
8863   format %{ "ANDNL  $dst.lo, $src1.lo, $src2.lo\n\t"
8864             "ANDNL  $dst.hi, $src1.hi, $src2.hi"
8865          %}
8866 
8867   ins_encode %{
8868     Register Rdst = $dst$$Register;
8869     Register Rsrc1 = $src1$$Register;
8870     Register Rsrc2 = $src2$$Register;
8871     __ andnl(Rdst, Rsrc1, Rsrc2);
8872     __ andnl(HIGH_FROM_LOW(Rdst), HIGH_FROM_LOW(Rsrc1), HIGH_FROM_LOW(Rsrc2));
8873   %}
8874   ins_pipe(ialu_reg_reg_long);
8875 %}
8876 
8877 instruct andnL_eReg_eReg_mem(eRegL dst, eRegL src1, memory src2, immL_M1 minus_1, eFlagsReg cr) %{
8878   match(Set dst (AndL (XorL src1 minus_1) (LoadL src2) ));
8879   predicate(UseBMI1Instructions);
8880   effect(KILL cr, TEMP dst);
8881 
8882   ins_cost(125);
8883   format %{ "ANDNL  $dst.lo, $src1.lo, $src2\n\t"
8884             "ANDNL  $dst.hi, $src1.hi, $src2+4"
8885          %}
8886 
8887   ins_encode %{
8888     Register Rdst = $dst$$Register;
8889     Register Rsrc1 = $src1$$Register;
8890     Address src2_hi = Address::make_raw($src2$$base, $src2$$index, $src2$$scale, $src2$$disp + 4, relocInfo::none);
8891 
8892     __ andnl(Rdst, Rsrc1, $src2$$Address);
8893     __ andnl(HIGH_FROM_LOW(Rdst), HIGH_FROM_LOW(Rsrc1), src2_hi);
8894   %}
8895   ins_pipe(ialu_reg_mem);
8896 %}
8897 
8898 instruct blsiL_eReg_eReg(eRegL dst, eRegL src, immL0 imm_zero, eFlagsReg cr) %{
8899   match(Set dst (AndL (SubL imm_zero src) src));
8900   predicate(UseBMI1Instructions);
8901   effect(KILL cr, TEMP dst);
8902 
8903   format %{ "MOVL   $dst.hi, 0\n\t"
8904             "BLSIL  $dst.lo, $src.lo\n\t"
8905             "JNZ    done\n\t"
8906             "BLSIL  $dst.hi, $src.hi\n"
8907             "done:"
8908          %}
8909 
8910   ins_encode %{
8911     Label done;
8912     Register Rdst = $dst$$Register;
8913     Register Rsrc = $src$$Register;
8914     __ movl(HIGH_FROM_LOW(Rdst), 0);
8915     __ blsil(Rdst, Rsrc);
8916     __ jccb(Assembler::notZero, done);
8917     __ blsil(HIGH_FROM_LOW(Rdst), HIGH_FROM_LOW(Rsrc));
8918     __ bind(done);
8919   %}
8920   ins_pipe(ialu_reg);
8921 %}
8922 
8923 instruct blsiL_eReg_mem(eRegL dst, memory src, immL0 imm_zero, eFlagsReg cr) %{
8924   match(Set dst (AndL (SubL imm_zero (LoadL src) ) (LoadL src) ));
8925   predicate(UseBMI1Instructions);
8926   effect(KILL cr, TEMP dst);
8927 
8928   ins_cost(125);
8929   format %{ "MOVL   $dst.hi, 0\n\t"
8930             "BLSIL  $dst.lo, $src\n\t"
8931             "JNZ    done\n\t"
8932             "BLSIL  $dst.hi, $src+4\n"
8933             "done:"
8934          %}
8935 
8936   ins_encode %{
8937     Label done;
8938     Register Rdst = $dst$$Register;
8939     Address src_hi = Address::make_raw($src$$base, $src$$index, $src$$scale, $src$$disp + 4, relocInfo::none);
8940 
8941     __ movl(HIGH_FROM_LOW(Rdst), 0);
8942     __ blsil(Rdst, $src$$Address);
8943     __ jccb(Assembler::notZero, done);
8944     __ blsil(HIGH_FROM_LOW(Rdst), src_hi);
8945     __ bind(done);
8946   %}
8947   ins_pipe(ialu_reg_mem);
8948 %}
8949 
8950 instruct blsmskL_eReg_eReg(eRegL dst, eRegL src, immL_M1 minus_1, eFlagsReg cr)
8951 %{
8952   match(Set dst (XorL (AddL src minus_1) src));
8953   predicate(UseBMI1Instructions);
8954   effect(KILL cr, TEMP dst);
8955 
8956   format %{ "MOVL    $dst.hi, 0\n\t"
8957             "BLSMSKL $dst.lo, $src.lo\n\t"
8958             "JNC     done\n\t"
8959             "BLSMSKL $dst.hi, $src.hi\n"
8960             "done:"
8961          %}
8962 
8963   ins_encode %{
8964     Label done;
8965     Register Rdst = $dst$$Register;
8966     Register Rsrc = $src$$Register;
8967     __ movl(HIGH_FROM_LOW(Rdst), 0);
8968     __ blsmskl(Rdst, Rsrc);
8969     __ jccb(Assembler::carryClear, done);
8970     __ blsmskl(HIGH_FROM_LOW(Rdst), HIGH_FROM_LOW(Rsrc));
8971     __ bind(done);
8972   %}
8973 
8974   ins_pipe(ialu_reg);
8975 %}
8976 
8977 instruct blsmskL_eReg_mem(eRegL dst, memory src, immL_M1 minus_1, eFlagsReg cr)
8978 %{
8979   match(Set dst (XorL (AddL (LoadL src) minus_1) (LoadL src) ));
8980   predicate(UseBMI1Instructions);
8981   effect(KILL cr, TEMP dst);
8982 
8983   ins_cost(125);
8984   format %{ "MOVL    $dst.hi, 0\n\t"
8985             "BLSMSKL $dst.lo, $src\n\t"
8986             "JNC     done\n\t"
8987             "BLSMSKL $dst.hi, $src+4\n"
8988             "done:"
8989          %}
8990 
8991   ins_encode %{
8992     Label done;
8993     Register Rdst = $dst$$Register;
8994     Address src_hi = Address::make_raw($src$$base, $src$$index, $src$$scale, $src$$disp + 4, relocInfo::none);
8995 
8996     __ movl(HIGH_FROM_LOW(Rdst), 0);
8997     __ blsmskl(Rdst, $src$$Address);
8998     __ jccb(Assembler::carryClear, done);
8999     __ blsmskl(HIGH_FROM_LOW(Rdst), src_hi);
9000     __ bind(done);
9001   %}
9002 
9003   ins_pipe(ialu_reg_mem);
9004 %}
9005 
9006 instruct blsrL_eReg_eReg(eRegL dst, eRegL src, immL_M1 minus_1, eFlagsReg cr)
9007 %{
9008   match(Set dst (AndL (AddL src minus_1) src) );
9009   predicate(UseBMI1Instructions);
9010   effect(KILL cr, TEMP dst);
9011 
9012   format %{ "MOVL   $dst.hi, $src.hi\n\t"
9013             "BLSRL  $dst.lo, $src.lo\n\t"
9014             "JNC    done\n\t"
9015             "BLSRL  $dst.hi, $src.hi\n"
9016             "done:"
9017   %}
9018 
9019   ins_encode %{
9020     Label done;
9021     Register Rdst = $dst$$Register;
9022     Register Rsrc = $src$$Register;
9023     __ movl(HIGH_FROM_LOW(Rdst), HIGH_FROM_LOW(Rsrc));
9024     __ blsrl(Rdst, Rsrc);
9025     __ jccb(Assembler::carryClear, done);
9026     __ blsrl(HIGH_FROM_LOW(Rdst), HIGH_FROM_LOW(Rsrc));
9027     __ bind(done);
9028   %}
9029 
9030   ins_pipe(ialu_reg);
9031 %}
9032 
9033 instruct blsrL_eReg_mem(eRegL dst, memory src, immL_M1 minus_1, eFlagsReg cr)
9034 %{
9035   match(Set dst (AndL (AddL (LoadL src) minus_1) (LoadL src) ));
9036   predicate(UseBMI1Instructions);
9037   effect(KILL cr, TEMP dst);
9038 
9039   ins_cost(125);
9040   format %{ "MOVL   $dst.hi, $src+4\n\t"
9041             "BLSRL  $dst.lo, $src\n\t"
9042             "JNC    done\n\t"
9043             "BLSRL  $dst.hi, $src+4\n"
9044             "done:"
9045   %}
9046 
9047   ins_encode %{
9048     Label done;
9049     Register Rdst = $dst$$Register;
9050     Address src_hi = Address::make_raw($src$$base, $src$$index, $src$$scale, $src$$disp + 4, relocInfo::none);
9051     __ movl(HIGH_FROM_LOW(Rdst), src_hi);
9052     __ blsrl(Rdst, $src$$Address);
9053     __ jccb(Assembler::carryClear, done);
9054     __ blsrl(HIGH_FROM_LOW(Rdst), src_hi);
9055     __ bind(done);
9056   %}
9057 
9058   ins_pipe(ialu_reg_mem);
9059 %}
9060 
9061 // Or Long Register with Register
9062 instruct orl_eReg(eRegL dst, eRegL src, eFlagsReg cr) %{
9063   match(Set dst (OrL dst src));
9064   effect(KILL cr);
9065   format %{ "OR     $dst.lo,$src.lo\n\t"
9066             "OR     $dst.hi,$src.hi" %}
9067   opcode(0x0B,0x0B);
9068   ins_encode( RegReg_Lo( dst, src), RegReg_Hi( dst, src) );
9069   ins_pipe( ialu_reg_reg_long );
9070 %}
9071 
9072 // Or Long Register with Immediate
9073 instruct orl_eReg_imm(eRegL dst, immL src, eFlagsReg cr) %{
9074   match(Set dst (OrL dst src));
9075   effect(KILL cr);
9076   format %{ "OR     $dst.lo,$src.lo\n\t"
9077             "OR     $dst.hi,$src.hi" %}
9078   opcode(0x81,0x01,0x01);  /* Opcode 81 /1, 81 /1 */
9079   ins_encode( Long_OpcSErm_Lo( dst, src ), Long_OpcSErm_Hi( dst, src ) );
9080   ins_pipe( ialu_reg_long );
9081 %}
9082 
9083 // Or Long Register with Memory
9084 instruct orl_eReg_mem(eRegL dst, load_long_memory mem, eFlagsReg cr) %{
9085   match(Set dst (OrL dst (LoadL mem)));
9086   effect(KILL cr);
9087   ins_cost(125);
9088   format %{ "OR     $dst.lo,$mem\n\t"
9089             "OR     $dst.hi,$mem+4" %}
9090   opcode(0x0B,0x0B);
9091   ins_encode( OpcP, RegMem( dst, mem), OpcS, RegMem_Hi(dst,mem) );
9092   ins_pipe( ialu_reg_long_mem );
9093 %}
9094 
9095 // Xor Long Register with Register
9096 instruct xorl_eReg(eRegL dst, eRegL src, eFlagsReg cr) %{
9097   match(Set dst (XorL dst src));
9098   effect(KILL cr);
9099   format %{ "XOR    $dst.lo,$src.lo\n\t"
9100             "XOR    $dst.hi,$src.hi" %}
9101   opcode(0x33,0x33);
9102   ins_encode( RegReg_Lo( dst, src), RegReg_Hi( dst, src) );
9103   ins_pipe( ialu_reg_reg_long );
9104 %}
9105 
9106 // Xor Long Register with Immediate -1
9107 instruct xorl_eReg_im1(eRegL dst, immL_M1 imm) %{
9108   match(Set dst (XorL dst imm));  
9109   format %{ "NOT    $dst.lo\n\t"
9110             "NOT    $dst.hi" %}
9111   ins_encode %{
9112      __ notl($dst$$Register);
9113      __ notl(HIGH_FROM_LOW($dst$$Register));
9114   %}
9115   ins_pipe( ialu_reg_long );
9116 %}
9117 
9118 // Xor Long Register with Immediate
9119 instruct xorl_eReg_imm(eRegL dst, immL src, eFlagsReg cr) %{
9120   match(Set dst (XorL dst src));
9121   effect(KILL cr);
9122   format %{ "XOR    $dst.lo,$src.lo\n\t"
9123             "XOR    $dst.hi,$src.hi" %}
9124   opcode(0x81,0x06,0x06);  /* Opcode 81 /6, 81 /6 */
9125   ins_encode( Long_OpcSErm_Lo( dst, src ), Long_OpcSErm_Hi( dst, src ) );
9126   ins_pipe( ialu_reg_long );
9127 %}
9128 
9129 // Xor Long Register with Memory
9130 instruct xorl_eReg_mem(eRegL dst, load_long_memory mem, eFlagsReg cr) %{
9131   match(Set dst (XorL dst (LoadL mem)));
9132   effect(KILL cr);
9133   ins_cost(125);
9134   format %{ "XOR    $dst.lo,$mem\n\t"
9135             "XOR    $dst.hi,$mem+4" %}
9136   opcode(0x33,0x33);
9137   ins_encode( OpcP, RegMem( dst, mem), OpcS, RegMem_Hi(dst,mem) );
9138   ins_pipe( ialu_reg_long_mem );
9139 %}
9140 
9141 // Shift Left Long by 1
9142 instruct shlL_eReg_1(eRegL dst, immI_1 cnt, eFlagsReg cr) %{
9143   predicate(UseNewLongLShift);
9144   match(Set dst (LShiftL dst cnt));
9145   effect(KILL cr);
9146   ins_cost(100);
9147   format %{ "ADD    $dst.lo,$dst.lo\n\t"
9148             "ADC    $dst.hi,$dst.hi" %}
9149   ins_encode %{
9150     __ addl($dst$$Register,$dst$$Register);
9151     __ adcl(HIGH_FROM_LOW($dst$$Register),HIGH_FROM_LOW($dst$$Register));
9152   %}
9153   ins_pipe( ialu_reg_long );
9154 %}
9155 
9156 // Shift Left Long by 2
9157 instruct shlL_eReg_2(eRegL dst, immI_2 cnt, eFlagsReg cr) %{
9158   predicate(UseNewLongLShift);
9159   match(Set dst (LShiftL dst cnt));
9160   effect(KILL cr);
9161   ins_cost(100);
9162   format %{ "ADD    $dst.lo,$dst.lo\n\t"
9163             "ADC    $dst.hi,$dst.hi\n\t" 
9164             "ADD    $dst.lo,$dst.lo\n\t"
9165             "ADC    $dst.hi,$dst.hi" %}
9166   ins_encode %{
9167     __ addl($dst$$Register,$dst$$Register);
9168     __ adcl(HIGH_FROM_LOW($dst$$Register),HIGH_FROM_LOW($dst$$Register));
9169     __ addl($dst$$Register,$dst$$Register);
9170     __ adcl(HIGH_FROM_LOW($dst$$Register),HIGH_FROM_LOW($dst$$Register));
9171   %}
9172   ins_pipe( ialu_reg_long );
9173 %}
9174 
9175 // Shift Left Long by 3
9176 instruct shlL_eReg_3(eRegL dst, immI_3 cnt, eFlagsReg cr) %{
9177   predicate(UseNewLongLShift);
9178   match(Set dst (LShiftL dst cnt));
9179   effect(KILL cr);
9180   ins_cost(100);
9181   format %{ "ADD    $dst.lo,$dst.lo\n\t"
9182             "ADC    $dst.hi,$dst.hi\n\t" 
9183             "ADD    $dst.lo,$dst.lo\n\t"
9184             "ADC    $dst.hi,$dst.hi\n\t" 
9185             "ADD    $dst.lo,$dst.lo\n\t"
9186             "ADC    $dst.hi,$dst.hi" %}
9187   ins_encode %{
9188     __ addl($dst$$Register,$dst$$Register);
9189     __ adcl(HIGH_FROM_LOW($dst$$Register),HIGH_FROM_LOW($dst$$Register));
9190     __ addl($dst$$Register,$dst$$Register);
9191     __ adcl(HIGH_FROM_LOW($dst$$Register),HIGH_FROM_LOW($dst$$Register));
9192     __ addl($dst$$Register,$dst$$Register);
9193     __ adcl(HIGH_FROM_LOW($dst$$Register),HIGH_FROM_LOW($dst$$Register));
9194   %}
9195   ins_pipe( ialu_reg_long );
9196 %}
9197 
9198 // Shift Left Long by 1-31
9199 instruct shlL_eReg_1_31(eRegL dst, immI_1_31 cnt, eFlagsReg cr) %{
9200   match(Set dst (LShiftL dst cnt));
9201   effect(KILL cr);
9202   ins_cost(200);
9203   format %{ "SHLD   $dst.hi,$dst.lo,$cnt\n\t"
9204             "SHL    $dst.lo,$cnt" %}
9205   opcode(0xC1, 0x4, 0xA4);  /* 0F/A4, then C1 /4 ib */
9206   ins_encode( move_long_small_shift(dst,cnt) );
9207   ins_pipe( ialu_reg_long );
9208 %}
9209 
9210 // Shift Left Long by 32-63
9211 instruct shlL_eReg_32_63(eRegL dst, immI_32_63 cnt, eFlagsReg cr) %{
9212   match(Set dst (LShiftL dst cnt));
9213   effect(KILL cr);
9214   ins_cost(300);
9215   format %{ "MOV    $dst.hi,$dst.lo\n"
9216           "\tSHL    $dst.hi,$cnt-32\n"
9217           "\tXOR    $dst.lo,$dst.lo" %}
9218   opcode(0xC1, 0x4);  /* C1 /4 ib */
9219   ins_encode( move_long_big_shift_clr(dst,cnt) );
9220   ins_pipe( ialu_reg_long );
9221 %}
9222 
9223 // Shift Left Long by variable
9224 instruct salL_eReg_CL(eRegL dst, eCXRegI shift, eFlagsReg cr) %{
9225   match(Set dst (LShiftL dst shift));
9226   effect(KILL cr);
9227   ins_cost(500+200);
9228   size(17);
9229   format %{ "TEST   $shift,32\n\t"
9230             "JEQ,s  small\n\t"
9231             "MOV    $dst.hi,$dst.lo\n\t"
9232             "XOR    $dst.lo,$dst.lo\n"
9233     "small:\tSHLD   $dst.hi,$dst.lo,$shift\n\t"
9234             "SHL    $dst.lo,$shift" %}
9235   ins_encode( shift_left_long( dst, shift ) );
9236   ins_pipe( pipe_slow );
9237 %}
9238 
9239 // Shift Right Long by 1-31
9240 instruct shrL_eReg_1_31(eRegL dst, immI_1_31 cnt, eFlagsReg cr) %{
9241   match(Set dst (URShiftL dst cnt));
9242   effect(KILL cr);
9243   ins_cost(200);
9244   format %{ "SHRD   $dst.lo,$dst.hi,$cnt\n\t"
9245             "SHR    $dst.hi,$cnt" %}
9246   opcode(0xC1, 0x5, 0xAC);  /* 0F/AC, then C1 /5 ib */
9247   ins_encode( move_long_small_shift(dst,cnt) );
9248   ins_pipe( ialu_reg_long );
9249 %}
9250 
9251 // Shift Right Long by 32-63
9252 instruct shrL_eReg_32_63(eRegL dst, immI_32_63 cnt, eFlagsReg cr) %{
9253   match(Set dst (URShiftL dst cnt));
9254   effect(KILL cr);
9255   ins_cost(300);
9256   format %{ "MOV    $dst.lo,$dst.hi\n"
9257           "\tSHR    $dst.lo,$cnt-32\n"
9258           "\tXOR    $dst.hi,$dst.hi" %}
9259   opcode(0xC1, 0x5);  /* C1 /5 ib */
9260   ins_encode( move_long_big_shift_clr(dst,cnt) );
9261   ins_pipe( ialu_reg_long );
9262 %}
9263 
9264 // Shift Right Long by variable
9265 instruct shrL_eReg_CL(eRegL dst, eCXRegI shift, eFlagsReg cr) %{
9266   match(Set dst (URShiftL dst shift));
9267   effect(KILL cr);
9268   ins_cost(600);
9269   size(17);
9270   format %{ "TEST   $shift,32\n\t"
9271             "JEQ,s  small\n\t"
9272             "MOV    $dst.lo,$dst.hi\n\t"
9273             "XOR    $dst.hi,$dst.hi\n"
9274     "small:\tSHRD   $dst.lo,$dst.hi,$shift\n\t"
9275             "SHR    $dst.hi,$shift" %}
9276   ins_encode( shift_right_long( dst, shift ) );
9277   ins_pipe( pipe_slow );
9278 %}
9279 
9280 // Shift Right Long by 1-31
9281 instruct sarL_eReg_1_31(eRegL dst, immI_1_31 cnt, eFlagsReg cr) %{
9282   match(Set dst (RShiftL dst cnt));
9283   effect(KILL cr);
9284   ins_cost(200);
9285   format %{ "SHRD   $dst.lo,$dst.hi,$cnt\n\t"
9286             "SAR    $dst.hi,$cnt" %}
9287   opcode(0xC1, 0x7, 0xAC);  /* 0F/AC, then C1 /7 ib */
9288   ins_encode( move_long_small_shift(dst,cnt) );
9289   ins_pipe( ialu_reg_long );
9290 %}
9291 
9292 // Shift Right Long by 32-63
9293 instruct sarL_eReg_32_63( eRegL dst, immI_32_63 cnt, eFlagsReg cr) %{
9294   match(Set dst (RShiftL dst cnt));
9295   effect(KILL cr);
9296   ins_cost(300);
9297   format %{ "MOV    $dst.lo,$dst.hi\n"
9298           "\tSAR    $dst.lo,$cnt-32\n"
9299           "\tSAR    $dst.hi,31" %}
9300   opcode(0xC1, 0x7);  /* C1 /7 ib */
9301   ins_encode( move_long_big_shift_sign(dst,cnt) );
9302   ins_pipe( ialu_reg_long );
9303 %}
9304 
9305 // Shift Right arithmetic Long by variable
9306 instruct sarL_eReg_CL(eRegL dst, eCXRegI shift, eFlagsReg cr) %{
9307   match(Set dst (RShiftL dst shift));
9308   effect(KILL cr);
9309   ins_cost(600);
9310   size(18);
9311   format %{ "TEST   $shift,32\n\t"
9312             "JEQ,s  small\n\t"
9313             "MOV    $dst.lo,$dst.hi\n\t"
9314             "SAR    $dst.hi,31\n"
9315     "small:\tSHRD   $dst.lo,$dst.hi,$shift\n\t"
9316             "SAR    $dst.hi,$shift" %}
9317   ins_encode( shift_right_arith_long( dst, shift ) );
9318   ins_pipe( pipe_slow );
9319 %}
9320 
9321 
9322 //----------Double Instructions------------------------------------------------
9323 // Double Math
9324 
9325 // Compare & branch
9326 
9327 // P6 version of float compare, sets condition codes in EFLAGS
9328 instruct cmpDPR_cc_P6(eFlagsRegU cr, regDPR src1, regDPR src2, eAXRegI rax) %{
9329   predicate(VM_Version::supports_cmov() && UseSSE <=1);
9330   match(Set cr (CmpD src1 src2));
9331   effect(KILL rax);
9332   ins_cost(150);
9333   format %{ "FLD    $src1\n\t"
9334             "FUCOMIP ST,$src2  // P6 instruction\n\t"
9335             "JNP    exit\n\t"
9336             "MOV    ah,1       // saw a NaN, set CF\n\t"
9337             "SAHF\n"
9338      "exit:\tNOP               // avoid branch to branch" %}
9339   opcode(0xDF, 0x05); /* DF E8+i or DF /5 */
9340   ins_encode( Push_Reg_DPR(src1),
9341               OpcP, RegOpc(src2),
9342               cmpF_P6_fixup );
9343   ins_pipe( pipe_slow );
9344 %}
9345 
9346 instruct cmpDPR_cc_P6CF(eFlagsRegUCF cr, regDPR src1, regDPR src2) %{
9347   predicate(VM_Version::supports_cmov() && UseSSE <=1);
9348   match(Set cr (CmpD src1 src2));
9349   ins_cost(150);
9350   format %{ "FLD    $src1\n\t"
9351             "FUCOMIP ST,$src2  // P6 instruction" %}
9352   opcode(0xDF, 0x05); /* DF E8+i or DF /5 */
9353   ins_encode( Push_Reg_DPR(src1),
9354               OpcP, RegOpc(src2));
9355   ins_pipe( pipe_slow );
9356 %}
9357 
9358 // Compare & branch
9359 instruct cmpDPR_cc(eFlagsRegU cr, regDPR src1, regDPR src2, eAXRegI rax) %{
9360   predicate(UseSSE<=1);
9361   match(Set cr (CmpD src1 src2));
9362   effect(KILL rax);
9363   ins_cost(200);
9364   format %{ "FLD    $src1\n\t"
9365             "FCOMp  $src2\n\t"
9366             "FNSTSW AX\n\t"
9367             "TEST   AX,0x400\n\t"
9368             "JZ,s   flags\n\t"
9369             "MOV    AH,1\t# unordered treat as LT\n"
9370     "flags:\tSAHF" %}
9371   opcode(0xD8, 0x3); /* D8 D8+i or D8 /3 */
9372   ins_encode( Push_Reg_DPR(src1),
9373               OpcP, RegOpc(src2),
9374               fpu_flags);
9375   ins_pipe( pipe_slow );
9376 %}
9377 
9378 // Compare vs zero into -1,0,1
9379 instruct cmpDPR_0(rRegI dst, regDPR src1, immDPR0 zero, eAXRegI rax, eFlagsReg cr) %{
9380   predicate(UseSSE<=1);
9381   match(Set dst (CmpD3 src1 zero));
9382   effect(KILL cr, KILL rax);
9383   ins_cost(280);
9384   format %{ "FTSTD  $dst,$src1" %}
9385   opcode(0xE4, 0xD9);
9386   ins_encode( Push_Reg_DPR(src1),
9387               OpcS, OpcP, PopFPU,
9388               CmpF_Result(dst));
9389   ins_pipe( pipe_slow );
9390 %}
9391 
9392 // Compare into -1,0,1
9393 instruct cmpDPR_reg(rRegI dst, regDPR src1, regDPR src2, eAXRegI rax, eFlagsReg cr) %{
9394   predicate(UseSSE<=1);
9395   match(Set dst (CmpD3 src1 src2));
9396   effect(KILL cr, KILL rax);
9397   ins_cost(300);
9398   format %{ "FCMPD  $dst,$src1,$src2" %}
9399   opcode(0xD8, 0x3); /* D8 D8+i or D8 /3 */
9400   ins_encode( Push_Reg_DPR(src1),
9401               OpcP, RegOpc(src2),
9402               CmpF_Result(dst));
9403   ins_pipe( pipe_slow );
9404 %}
9405 
9406 // float compare and set condition codes in EFLAGS by XMM regs
9407 instruct cmpD_cc(eFlagsRegU cr, regD src1, regD src2) %{
9408   predicate(UseSSE>=2);
9409   match(Set cr (CmpD src1 src2));
9410   ins_cost(145);
9411   format %{ "UCOMISD $src1,$src2\n\t"
9412             "JNP,s   exit\n\t"
9413             "PUSHF\t# saw NaN, set CF\n\t"
9414             "AND     [rsp], #0xffffff2b\n\t"
9415             "POPF\n"
9416     "exit:" %}
9417   ins_encode %{
9418     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
9419     emit_cmpfp_fixup(_masm);
9420   %}
9421   ins_pipe( pipe_slow );
9422 %}
9423 
9424 instruct cmpD_ccCF(eFlagsRegUCF cr, regD src1, regD src2) %{
9425   predicate(UseSSE>=2);
9426   match(Set cr (CmpD src1 src2));
9427   ins_cost(100);
9428   format %{ "UCOMISD $src1,$src2" %}
9429   ins_encode %{
9430     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
9431   %}
9432   ins_pipe( pipe_slow );
9433 %}
9434 
9435 // float compare and set condition codes in EFLAGS by XMM regs
9436 instruct cmpD_ccmem(eFlagsRegU cr, regD src1, memory src2) %{
9437   predicate(UseSSE>=2);
9438   match(Set cr (CmpD src1 (LoadD src2)));
9439   ins_cost(145);
9440   format %{ "UCOMISD $src1,$src2\n\t"
9441             "JNP,s   exit\n\t"
9442             "PUSHF\t# saw NaN, set CF\n\t"
9443             "AND     [rsp], #0xffffff2b\n\t"
9444             "POPF\n"
9445     "exit:" %}
9446   ins_encode %{
9447     __ ucomisd($src1$$XMMRegister, $src2$$Address);
9448     emit_cmpfp_fixup(_masm);
9449   %}
9450   ins_pipe( pipe_slow );
9451 %}
9452 
9453 instruct cmpD_ccmemCF(eFlagsRegUCF cr, regD src1, memory src2) %{
9454   predicate(UseSSE>=2);
9455   match(Set cr (CmpD src1 (LoadD src2)));
9456   ins_cost(100);
9457   format %{ "UCOMISD $src1,$src2" %}
9458   ins_encode %{
9459     __ ucomisd($src1$$XMMRegister, $src2$$Address);
9460   %}
9461   ins_pipe( pipe_slow );
9462 %}
9463 
9464 // Compare into -1,0,1 in XMM
9465 instruct cmpD_reg(xRegI dst, regD src1, regD src2, eFlagsReg cr) %{
9466   predicate(UseSSE>=2);
9467   match(Set dst (CmpD3 src1 src2));
9468   effect(KILL cr);
9469   ins_cost(255);
9470   format %{ "UCOMISD $src1, $src2\n\t"
9471             "MOV     $dst, #-1\n\t"
9472             "JP,s    done\n\t"
9473             "JB,s    done\n\t"
9474             "SETNE   $dst\n\t"
9475             "MOVZB   $dst, $dst\n"
9476     "done:" %}
9477   ins_encode %{
9478     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
9479     emit_cmpfp3(_masm, $dst$$Register);
9480   %}
9481   ins_pipe( pipe_slow );
9482 %}
9483 
9484 // Compare into -1,0,1 in XMM and memory
9485 instruct cmpD_regmem(xRegI dst, regD src1, memory src2, eFlagsReg cr) %{
9486   predicate(UseSSE>=2);
9487   match(Set dst (CmpD3 src1 (LoadD src2)));
9488   effect(KILL cr);
9489   ins_cost(275);
9490   format %{ "UCOMISD $src1, $src2\n\t"
9491             "MOV     $dst, #-1\n\t"
9492             "JP,s    done\n\t"
9493             "JB,s    done\n\t"
9494             "SETNE   $dst\n\t"
9495             "MOVZB   $dst, $dst\n"
9496     "done:" %}
9497   ins_encode %{
9498     __ ucomisd($src1$$XMMRegister, $src2$$Address);
9499     emit_cmpfp3(_masm, $dst$$Register);
9500   %}
9501   ins_pipe( pipe_slow );
9502 %}
9503 
9504 
9505 instruct subDPR_reg(regDPR dst, regDPR src) %{
9506   predicate (UseSSE <=1);
9507   match(Set dst (SubD dst src));
9508 
9509   format %{ "FLD    $src\n\t"
9510             "DSUBp  $dst,ST" %}
9511   opcode(0xDE, 0x5); /* DE E8+i  or DE /5 */
9512   ins_cost(150);
9513   ins_encode( Push_Reg_DPR(src),
9514               OpcP, RegOpc(dst) );
9515   ins_pipe( fpu_reg_reg );
9516 %}
9517 
9518 instruct subDPR_reg_round(stackSlotD dst, regDPR src1, regDPR src2) %{
9519   predicate (UseSSE <=1);
9520   match(Set dst (RoundDouble (SubD src1 src2)));
9521   ins_cost(250);
9522 
9523   format %{ "FLD    $src2\n\t"
9524             "DSUB   ST,$src1\n\t"
9525             "FSTP_D $dst\t# D-round" %}
9526   opcode(0xD8, 0x5);
9527   ins_encode( Push_Reg_DPR(src2),
9528               OpcP, RegOpc(src1), Pop_Mem_DPR(dst) );
9529   ins_pipe( fpu_mem_reg_reg );
9530 %}
9531 
9532 
9533 instruct subDPR_reg_mem(regDPR dst, memory src) %{
9534   predicate (UseSSE <=1);
9535   match(Set dst (SubD dst (LoadD src)));
9536   ins_cost(150);
9537 
9538   format %{ "FLD    $src\n\t"
9539             "DSUBp  $dst,ST" %}
9540   opcode(0xDE, 0x5, 0xDD); /* DE C0+i */  /* LoadD  DD /0 */
9541   ins_encode( Opcode(tertiary), RMopc_Mem(0x00,src),
9542               OpcP, RegOpc(dst) );
9543   ins_pipe( fpu_reg_mem );
9544 %}
9545 
9546 instruct absDPR_reg(regDPR1 dst, regDPR1 src) %{
9547   predicate (UseSSE<=1);
9548   match(Set dst (AbsD src));
9549   ins_cost(100);
9550   format %{ "FABS" %}
9551   opcode(0xE1, 0xD9);
9552   ins_encode( OpcS, OpcP );
9553   ins_pipe( fpu_reg_reg );
9554 %}
9555 
9556 instruct negDPR_reg(regDPR1 dst, regDPR1 src) %{
9557   predicate(UseSSE<=1);
9558   match(Set dst (NegD src));
9559   ins_cost(100);
9560   format %{ "FCHS" %}
9561   opcode(0xE0, 0xD9);
9562   ins_encode( OpcS, OpcP );
9563   ins_pipe( fpu_reg_reg );
9564 %}
9565 
9566 instruct addDPR_reg(regDPR dst, regDPR src) %{
9567   predicate(UseSSE<=1);
9568   match(Set dst (AddD dst src));
9569   format %{ "FLD    $src\n\t"
9570             "DADD   $dst,ST" %}
9571   size(4);
9572   ins_cost(150);
9573   opcode(0xDE, 0x0); /* DE C0+i or DE /0*/
9574   ins_encode( Push_Reg_DPR(src),
9575               OpcP, RegOpc(dst) );
9576   ins_pipe( fpu_reg_reg );
9577 %}
9578 
9579 
9580 instruct addDPR_reg_round(stackSlotD dst, regDPR src1, regDPR src2) %{
9581   predicate(UseSSE<=1);
9582   match(Set dst (RoundDouble (AddD src1 src2)));
9583   ins_cost(250);
9584 
9585   format %{ "FLD    $src2\n\t"
9586             "DADD   ST,$src1\n\t"
9587             "FSTP_D $dst\t# D-round" %}
9588   opcode(0xD8, 0x0); /* D8 C0+i or D8 /0*/
9589   ins_encode( Push_Reg_DPR(src2),
9590               OpcP, RegOpc(src1), Pop_Mem_DPR(dst) );
9591   ins_pipe( fpu_mem_reg_reg );
9592 %}
9593 
9594 
9595 instruct addDPR_reg_mem(regDPR dst, memory src) %{
9596   predicate(UseSSE<=1);
9597   match(Set dst (AddD dst (LoadD src)));
9598   ins_cost(150);
9599 
9600   format %{ "FLD    $src\n\t"
9601             "DADDp  $dst,ST" %}
9602   opcode(0xDE, 0x0, 0xDD); /* DE C0+i */  /* LoadD  DD /0 */
9603   ins_encode( Opcode(tertiary), RMopc_Mem(0x00,src),
9604               OpcP, RegOpc(dst) );
9605   ins_pipe( fpu_reg_mem );
9606 %}
9607 
9608 // add-to-memory
9609 instruct addDPR_mem_reg(memory dst, regDPR src) %{
9610   predicate(UseSSE<=1);
9611   match(Set dst (StoreD dst (RoundDouble (AddD (LoadD dst) src))));
9612   ins_cost(150);
9613 
9614   format %{ "FLD_D  $dst\n\t"
9615             "DADD   ST,$src\n\t"
9616             "FST_D  $dst" %}
9617   opcode(0xDD, 0x0);
9618   ins_encode( Opcode(0xDD), RMopc_Mem(0x00,dst),
9619               Opcode(0xD8), RegOpc(src),
9620               set_instruction_start,
9621               Opcode(0xDD), RMopc_Mem(0x03,dst) );
9622   ins_pipe( fpu_reg_mem );
9623 %}
9624 
9625 instruct addDPR_reg_imm1(regDPR dst, immDPR1 con) %{
9626   predicate(UseSSE<=1);
9627   match(Set dst (AddD dst con));
9628   ins_cost(125);
9629   format %{ "FLD1\n\t"
9630             "DADDp  $dst,ST" %}
9631   ins_encode %{
9632     __ fld1();
9633     __ faddp($dst$$reg);
9634   %}
9635   ins_pipe(fpu_reg);
9636 %}
9637 
9638 instruct addDPR_reg_imm(regDPR dst, immDPR con) %{
9639   predicate(UseSSE<=1 && _kids[1]->_leaf->getd() != 0.0 && _kids[1]->_leaf->getd() != 1.0 );
9640   match(Set dst (AddD dst con));
9641   ins_cost(200);
9642   format %{ "FLD_D  [$constantaddress]\t# load from constant table: double=$con\n\t"
9643             "DADDp  $dst,ST" %}
9644   ins_encode %{
9645     __ fld_d($constantaddress($con));
9646     __ faddp($dst$$reg);
9647   %}
9648   ins_pipe(fpu_reg_mem);
9649 %}
9650 
9651 instruct addDPR_reg_imm_round(stackSlotD dst, regDPR src, immDPR con) %{
9652   predicate(UseSSE<=1 && _kids[0]->_kids[1]->_leaf->getd() != 0.0 && _kids[0]->_kids[1]->_leaf->getd() != 1.0 );
9653   match(Set dst (RoundDouble (AddD src con)));
9654   ins_cost(200);
9655   format %{ "FLD_D  [$constantaddress]\t# load from constant table: double=$con\n\t"
9656             "DADD   ST,$src\n\t"
9657             "FSTP_D $dst\t# D-round" %}
9658   ins_encode %{
9659     __ fld_d($constantaddress($con));
9660     __ fadd($src$$reg);
9661     __ fstp_d(Address(rsp, $dst$$disp));
9662   %}
9663   ins_pipe(fpu_mem_reg_con);
9664 %}
9665 
9666 instruct mulDPR_reg(regDPR dst, regDPR src) %{
9667   predicate(UseSSE<=1);
9668   match(Set dst (MulD dst src));
9669   format %{ "FLD    $src\n\t"
9670             "DMULp  $dst,ST" %}
9671   opcode(0xDE, 0x1); /* DE C8+i or DE /1*/
9672   ins_cost(150);
9673   ins_encode( Push_Reg_DPR(src),
9674               OpcP, RegOpc(dst) );
9675   ins_pipe( fpu_reg_reg );
9676 %}
9677 
9678 // Strict FP instruction biases argument before multiply then
9679 // biases result to avoid double rounding of subnormals.
9680 //
9681 // scale arg1 by multiplying arg1 by 2^(-15360)
9682 // load arg2
9683 // multiply scaled arg1 by arg2
9684 // rescale product by 2^(15360)
9685 //
9686 instruct strictfp_mulDPR_reg(regDPR1 dst, regnotDPR1 src) %{
9687   predicate( UseSSE<=1 && Compile::current()->has_method() && Compile::current()->method()->is_strict() );
9688   match(Set dst (MulD dst src));
9689   ins_cost(1);   // Select this instruction for all strict FP double multiplies
9690 
9691   format %{ "FLD    StubRoutines::_fpu_subnormal_bias1\n\t"
9692             "DMULp  $dst,ST\n\t"
9693             "FLD    $src\n\t"
9694             "DMULp  $dst,ST\n\t"
9695             "FLD    StubRoutines::_fpu_subnormal_bias2\n\t"
9696             "DMULp  $dst,ST\n\t" %}
9697   opcode(0xDE, 0x1); /* DE C8+i or DE /1*/
9698   ins_encode( strictfp_bias1(dst),
9699               Push_Reg_DPR(src),
9700               OpcP, RegOpc(dst),
9701               strictfp_bias2(dst) );
9702   ins_pipe( fpu_reg_reg );
9703 %}
9704 
9705 instruct mulDPR_reg_imm(regDPR dst, immDPR con) %{
9706   predicate( UseSSE<=1 && _kids[1]->_leaf->getd() != 0.0 && _kids[1]->_leaf->getd() != 1.0 );
9707   match(Set dst (MulD dst con));
9708   ins_cost(200);
9709   format %{ "FLD_D  [$constantaddress]\t# load from constant table: double=$con\n\t"
9710             "DMULp  $dst,ST" %}
9711   ins_encode %{
9712     __ fld_d($constantaddress($con));
9713     __ fmulp($dst$$reg);
9714   %}
9715   ins_pipe(fpu_reg_mem);
9716 %}
9717 
9718 
9719 instruct mulDPR_reg_mem(regDPR dst, memory src) %{
9720   predicate( UseSSE<=1 );
9721   match(Set dst (MulD dst (LoadD src)));
9722   ins_cost(200);
9723   format %{ "FLD_D  $src\n\t"
9724             "DMULp  $dst,ST" %}
9725   opcode(0xDE, 0x1, 0xDD); /* DE C8+i or DE /1*/  /* LoadD  DD /0 */
9726   ins_encode( Opcode(tertiary), RMopc_Mem(0x00,src),
9727               OpcP, RegOpc(dst) );
9728   ins_pipe( fpu_reg_mem );
9729 %}
9730 
9731 //
9732 // Cisc-alternate to reg-reg multiply
9733 instruct mulDPR_reg_mem_cisc(regDPR dst, regDPR src, memory mem) %{
9734   predicate( UseSSE<=1 );
9735   match(Set dst (MulD src (LoadD mem)));
9736   ins_cost(250);
9737   format %{ "FLD_D  $mem\n\t"
9738             "DMUL   ST,$src\n\t"
9739             "FSTP_D $dst" %}
9740   opcode(0xD8, 0x1, 0xD9); /* D8 C8+i */  /* LoadD D9 /0 */
9741   ins_encode( Opcode(tertiary), RMopc_Mem(0x00,mem),
9742               OpcReg_FPR(src),
9743               Pop_Reg_DPR(dst) );
9744   ins_pipe( fpu_reg_reg_mem );
9745 %}
9746 
9747 
9748 // MACRO3 -- addDPR a mulDPR
9749 // This instruction is a '2-address' instruction in that the result goes
9750 // back to src2.  This eliminates a move from the macro; possibly the
9751 // register allocator will have to add it back (and maybe not).
9752 instruct addDPR_mulDPR_reg(regDPR src2, regDPR src1, regDPR src0) %{
9753   predicate( UseSSE<=1 );
9754   match(Set src2 (AddD (MulD src0 src1) src2));
9755   format %{ "FLD    $src0\t# ===MACRO3d===\n\t"
9756             "DMUL   ST,$src1\n\t"
9757             "DADDp  $src2,ST" %}
9758   ins_cost(250);
9759   opcode(0xDD); /* LoadD DD /0 */
9760   ins_encode( Push_Reg_FPR(src0),
9761               FMul_ST_reg(src1),
9762               FAddP_reg_ST(src2) );
9763   ins_pipe( fpu_reg_reg_reg );
9764 %}
9765 
9766 
9767 // MACRO3 -- subDPR a mulDPR
9768 instruct subDPR_mulDPR_reg(regDPR src2, regDPR src1, regDPR src0) %{
9769   predicate( UseSSE<=1 );
9770   match(Set src2 (SubD (MulD src0 src1) src2));
9771   format %{ "FLD    $src0\t# ===MACRO3d===\n\t"
9772             "DMUL   ST,$src1\n\t"
9773             "DSUBRp $src2,ST" %}
9774   ins_cost(250);
9775   ins_encode( Push_Reg_FPR(src0),
9776               FMul_ST_reg(src1),
9777               Opcode(0xDE), Opc_plus(0xE0,src2));
9778   ins_pipe( fpu_reg_reg_reg );
9779 %}
9780 
9781 
9782 instruct divDPR_reg(regDPR dst, regDPR src) %{
9783   predicate( UseSSE<=1 );
9784   match(Set dst (DivD dst src));
9785 
9786   format %{ "FLD    $src\n\t"
9787             "FDIVp  $dst,ST" %}
9788   opcode(0xDE, 0x7); /* DE F8+i or DE /7*/
9789   ins_cost(150);
9790   ins_encode( Push_Reg_DPR(src),
9791               OpcP, RegOpc(dst) );
9792   ins_pipe( fpu_reg_reg );
9793 %}
9794 
9795 // Strict FP instruction biases argument before division then
9796 // biases result, to avoid double rounding of subnormals.
9797 //
9798 // scale dividend by multiplying dividend by 2^(-15360)
9799 // load divisor
9800 // divide scaled dividend by divisor
9801 // rescale quotient by 2^(15360)
9802 //
9803 instruct strictfp_divDPR_reg(regDPR1 dst, regnotDPR1 src) %{
9804   predicate (UseSSE<=1);
9805   match(Set dst (DivD dst src));
9806   predicate( UseSSE<=1 && Compile::current()->has_method() && Compile::current()->method()->is_strict() );
9807   ins_cost(01);
9808 
9809   format %{ "FLD    StubRoutines::_fpu_subnormal_bias1\n\t"
9810             "DMULp  $dst,ST\n\t"
9811             "FLD    $src\n\t"
9812             "FDIVp  $dst,ST\n\t"
9813             "FLD    StubRoutines::_fpu_subnormal_bias2\n\t"
9814             "DMULp  $dst,ST\n\t" %}
9815   opcode(0xDE, 0x7); /* DE F8+i or DE /7*/
9816   ins_encode( strictfp_bias1(dst),
9817               Push_Reg_DPR(src),
9818               OpcP, RegOpc(dst),
9819               strictfp_bias2(dst) );
9820   ins_pipe( fpu_reg_reg );
9821 %}
9822 
9823 instruct divDPR_reg_round(stackSlotD dst, regDPR src1, regDPR src2) %{
9824   predicate( UseSSE<=1 && !(Compile::current()->has_method() && Compile::current()->method()->is_strict()) );
9825   match(Set dst (RoundDouble (DivD src1 src2)));
9826 
9827   format %{ "FLD    $src1\n\t"
9828             "FDIV   ST,$src2\n\t"
9829             "FSTP_D $dst\t# D-round" %}
9830   opcode(0xD8, 0x6); /* D8 F0+i or D8 /6 */
9831   ins_encode( Push_Reg_DPR(src1),
9832               OpcP, RegOpc(src2), Pop_Mem_DPR(dst) );
9833   ins_pipe( fpu_mem_reg_reg );
9834 %}
9835 
9836 
9837 instruct modDPR_reg(regDPR dst, regDPR src, eAXRegI rax, eFlagsReg cr) %{
9838   predicate(UseSSE<=1);
9839   match(Set dst (ModD dst src));
9840   effect(KILL rax, KILL cr); // emitModDPR() uses EAX and EFLAGS
9841 
9842   format %{ "DMOD   $dst,$src" %}
9843   ins_cost(250);
9844   ins_encode(Push_Reg_Mod_DPR(dst, src),
9845               emitModDPR(),
9846               Push_Result_Mod_DPR(src),
9847               Pop_Reg_DPR(dst));
9848   ins_pipe( pipe_slow );
9849 %}
9850 
9851 instruct modD_reg(regD dst, regD src0, regD src1, eAXRegI rax, eFlagsReg cr) %{
9852   predicate(UseSSE>=2);
9853   match(Set dst (ModD src0 src1));
9854   effect(KILL rax, KILL cr);
9855 
9856   format %{ "SUB    ESP,8\t # DMOD\n"
9857           "\tMOVSD  [ESP+0],$src1\n"
9858           "\tFLD_D  [ESP+0]\n"
9859           "\tMOVSD  [ESP+0],$src0\n"
9860           "\tFLD_D  [ESP+0]\n"
9861      "loop:\tFPREM\n"
9862           "\tFWAIT\n"
9863           "\tFNSTSW AX\n"
9864           "\tSAHF\n"
9865           "\tJP     loop\n"
9866           "\tFSTP_D [ESP+0]\n"
9867           "\tMOVSD  $dst,[ESP+0]\n"
9868           "\tADD    ESP,8\n"
9869           "\tFSTP   ST0\t # Restore FPU Stack"
9870     %}
9871   ins_cost(250);
9872   ins_encode( Push_ModD_encoding(src0, src1), emitModDPR(), Push_ResultD(dst), PopFPU);
9873   ins_pipe( pipe_slow );
9874 %}
9875 
9876 instruct sinDPR_reg(regDPR1 dst, regDPR1 src) %{
9877   predicate (UseSSE<=1);
9878   match(Set dst (SinD src));
9879   ins_cost(1800);
9880   format %{ "DSIN   $dst" %}
9881   opcode(0xD9, 0xFE);
9882   ins_encode( OpcP, OpcS );
9883   ins_pipe( pipe_slow );
9884 %}
9885 
9886 instruct sinD_reg(regD dst, eFlagsReg cr) %{
9887   predicate (UseSSE>=2);
9888   match(Set dst (SinD dst));
9889   effect(KILL cr); // Push_{Src|Result}D() uses "{SUB|ADD} ESP,8"
9890   ins_cost(1800);
9891   format %{ "DSIN   $dst" %}
9892   opcode(0xD9, 0xFE);
9893   ins_encode( Push_SrcD(dst), OpcP, OpcS, Push_ResultD(dst) );
9894   ins_pipe( pipe_slow );
9895 %}
9896 
9897 instruct cosDPR_reg(regDPR1 dst, regDPR1 src) %{
9898   predicate (UseSSE<=1);
9899   match(Set dst (CosD src));
9900   ins_cost(1800);
9901   format %{ "DCOS   $dst" %}
9902   opcode(0xD9, 0xFF);
9903   ins_encode( OpcP, OpcS );
9904   ins_pipe( pipe_slow );
9905 %}
9906 
9907 instruct cosD_reg(regD dst, eFlagsReg cr) %{
9908   predicate (UseSSE>=2);
9909   match(Set dst (CosD dst));
9910   effect(KILL cr); // Push_{Src|Result}D() uses "{SUB|ADD} ESP,8"
9911   ins_cost(1800);
9912   format %{ "DCOS   $dst" %}
9913   opcode(0xD9, 0xFF);
9914   ins_encode( Push_SrcD(dst), OpcP, OpcS, Push_ResultD(dst) );
9915   ins_pipe( pipe_slow );
9916 %}
9917 
9918 instruct tanDPR_reg(regDPR1 dst, regDPR1 src) %{
9919   predicate (UseSSE<=1);
9920   match(Set dst(TanD src));
9921   format %{ "DTAN   $dst" %}
9922   ins_encode( Opcode(0xD9), Opcode(0xF2),    // fptan
9923               Opcode(0xDD), Opcode(0xD8));   // fstp st
9924   ins_pipe( pipe_slow );
9925 %}
9926 
9927 instruct tanD_reg(regD dst, eFlagsReg cr) %{
9928   predicate (UseSSE>=2);
9929   match(Set dst(TanD dst));
9930   effect(KILL cr); // Push_{Src|Result}D() uses "{SUB|ADD} ESP,8"
9931   format %{ "DTAN   $dst" %}
9932   ins_encode( Push_SrcD(dst),
9933               Opcode(0xD9), Opcode(0xF2),    // fptan
9934               Opcode(0xDD), Opcode(0xD8),   // fstp st
9935               Push_ResultD(dst) );
9936   ins_pipe( pipe_slow );
9937 %}
9938 
9939 instruct atanDPR_reg(regDPR dst, regDPR src) %{
9940   predicate (UseSSE<=1);
9941   match(Set dst(AtanD dst src));
9942   format %{ "DATA   $dst,$src" %}
9943   opcode(0xD9, 0xF3);
9944   ins_encode( Push_Reg_DPR(src),
9945               OpcP, OpcS, RegOpc(dst) );
9946   ins_pipe( pipe_slow );
9947 %}
9948 
9949 instruct atanD_reg(regD dst, regD src, eFlagsReg cr) %{
9950   predicate (UseSSE>=2);
9951   match(Set dst(AtanD dst src));
9952   effect(KILL cr); // Push_{Src|Result}D() uses "{SUB|ADD} ESP,8"
9953   format %{ "DATA   $dst,$src" %}
9954   opcode(0xD9, 0xF3);
9955   ins_encode( Push_SrcD(src),
9956               OpcP, OpcS, Push_ResultD(dst) );
9957   ins_pipe( pipe_slow );
9958 %}
9959 
9960 instruct sqrtDPR_reg(regDPR dst, regDPR src) %{
9961   predicate (UseSSE<=1);
9962   match(Set dst (SqrtD src));
9963   format %{ "DSQRT  $dst,$src" %}
9964   opcode(0xFA, 0xD9);
9965   ins_encode( Push_Reg_DPR(src),
9966               OpcS, OpcP, Pop_Reg_DPR(dst) );
9967   ins_pipe( pipe_slow );
9968 %}
9969 
9970 instruct powDPR_reg(regDPR X, regDPR1 Y, eAXRegI rax, eDXRegI rdx, eCXRegI rcx, eFlagsReg cr) %{
9971   predicate (UseSSE<=1);
9972   match(Set Y (PowD X Y));  // Raise X to the Yth power
9973   effect(KILL rax, KILL rdx, KILL rcx, KILL cr);
9974   format %{ "fast_pow $X $Y -> $Y  // KILL $rax, $rcx, $rdx" %}
9975   ins_encode %{
9976     __ subptr(rsp, 8);
9977     __ fld_s($X$$reg - 1);
9978     __ fast_pow();
9979     __ addptr(rsp, 8);
9980   %}
9981   ins_pipe( pipe_slow );
9982 %}
9983 
9984 instruct powD_reg(regD dst, regD src0, regD src1, eAXRegI rax, eDXRegI rdx, eCXRegI rcx, eFlagsReg cr) %{
9985   predicate (UseSSE>=2);
9986   match(Set dst (PowD src0 src1));  // Raise src0 to the src1'th power
9987   effect(KILL rax, KILL rdx, KILL rcx, KILL cr);
9988   format %{ "fast_pow $src0 $src1 -> $dst  // KILL $rax, $rcx, $rdx" %}
9989   ins_encode %{
9990     __ subptr(rsp, 8);
9991     __ movdbl(Address(rsp, 0), $src1$$XMMRegister);
9992     __ fld_d(Address(rsp, 0));
9993     __ movdbl(Address(rsp, 0), $src0$$XMMRegister);
9994     __ fld_d(Address(rsp, 0));
9995     __ fast_pow();
9996     __ fstp_d(Address(rsp, 0));
9997     __ movdbl($dst$$XMMRegister, Address(rsp, 0));
9998     __ addptr(rsp, 8);
9999   %}
10000   ins_pipe( pipe_slow );
10001 %}
10002 
10003 
10004 instruct expDPR_reg(regDPR1 dpr1, eAXRegI rax, eDXRegI rdx, eCXRegI rcx, eFlagsReg cr) %{
10005   predicate (UseSSE<=1);
10006   match(Set dpr1 (ExpD dpr1));
10007   effect(KILL rax, KILL rcx, KILL rdx, KILL cr);
10008   format %{ "fast_exp $dpr1 -> $dpr1  // KILL $rax, $rcx, $rdx" %}
10009   ins_encode %{
10010     __ fast_exp();
10011   %}
10012   ins_pipe( pipe_slow );
10013 %}
10014 
10015 instruct expD_reg(regD dst, regD src, eAXRegI rax, eDXRegI rdx, eCXRegI rcx, eFlagsReg cr) %{
10016   predicate (UseSSE>=2);
10017   match(Set dst (ExpD src));
10018   effect(KILL rax, KILL rcx, KILL rdx, KILL cr);
10019   format %{ "fast_exp $dst -> $src  // KILL $rax, $rcx, $rdx" %}
10020   ins_encode %{
10021     __ subptr(rsp, 8);
10022     __ movdbl(Address(rsp, 0), $src$$XMMRegister);
10023     __ fld_d(Address(rsp, 0));
10024     __ fast_exp();
10025     __ fstp_d(Address(rsp, 0));
10026     __ movdbl($dst$$XMMRegister, Address(rsp, 0));
10027     __ addptr(rsp, 8);
10028   %}
10029   ins_pipe( pipe_slow );
10030 %}
10031 
10032 instruct log10DPR_reg(regDPR1 dst, regDPR1 src) %{
10033   predicate (UseSSE<=1);
10034   // The source Double operand on FPU stack
10035   match(Set dst (Log10D src));
10036   // fldlg2       ; push log_10(2) on the FPU stack; full 80-bit number
10037   // fxch         ; swap ST(0) with ST(1)
10038   // fyl2x        ; compute log_10(2) * log_2(x)
10039   format %{ "FLDLG2 \t\t\t#Log10\n\t"
10040             "FXCH   \n\t"
10041             "FYL2X  \t\t\t# Q=Log10*Log_2(x)"
10042          %}
10043   ins_encode( Opcode(0xD9), Opcode(0xEC),   // fldlg2
10044               Opcode(0xD9), Opcode(0xC9),   // fxch
10045               Opcode(0xD9), Opcode(0xF1));  // fyl2x
10046 
10047   ins_pipe( pipe_slow );
10048 %}
10049 
10050 instruct log10D_reg(regD dst, regD src, eFlagsReg cr) %{
10051   predicate (UseSSE>=2);
10052   effect(KILL cr);
10053   match(Set dst (Log10D src));
10054   // fldlg2       ; push log_10(2) on the FPU stack; full 80-bit number
10055   // fyl2x        ; compute log_10(2) * log_2(x)
10056   format %{ "FLDLG2 \t\t\t#Log10\n\t"
10057             "FYL2X  \t\t\t# Q=Log10*Log_2(x)"
10058          %}
10059   ins_encode( Opcode(0xD9), Opcode(0xEC),   // fldlg2
10060               Push_SrcD(src),
10061               Opcode(0xD9), Opcode(0xF1),   // fyl2x
10062               Push_ResultD(dst));
10063 
10064   ins_pipe( pipe_slow );
10065 %}
10066 
10067 instruct logDPR_reg(regDPR1 dst, regDPR1 src) %{
10068   predicate (UseSSE<=1);
10069   // The source Double operand on FPU stack
10070   match(Set dst (LogD src));
10071   // fldln2       ; push log_e(2) on the FPU stack; full 80-bit number
10072   // fxch         ; swap ST(0) with ST(1)
10073   // fyl2x        ; compute log_e(2) * log_2(x)
10074   format %{ "FLDLN2 \t\t\t#Log_e\n\t"
10075             "FXCH   \n\t"
10076             "FYL2X  \t\t\t# Q=Log_e*Log_2(x)"
10077          %}
10078   ins_encode( Opcode(0xD9), Opcode(0xED),   // fldln2
10079               Opcode(0xD9), Opcode(0xC9),   // fxch
10080               Opcode(0xD9), Opcode(0xF1));  // fyl2x
10081 
10082   ins_pipe( pipe_slow );
10083 %}
10084 
10085 instruct logD_reg(regD dst, regD src, eFlagsReg cr) %{
10086   predicate (UseSSE>=2);
10087   effect(KILL cr);
10088   // The source and result Double operands in XMM registers
10089   match(Set dst (LogD src));
10090   // fldln2       ; push log_e(2) on the FPU stack; full 80-bit number
10091   // fyl2x        ; compute log_e(2) * log_2(x)
10092   format %{ "FLDLN2 \t\t\t#Log_e\n\t"
10093             "FYL2X  \t\t\t# Q=Log_e*Log_2(x)"
10094          %}
10095   ins_encode( Opcode(0xD9), Opcode(0xED),   // fldln2
10096               Push_SrcD(src),
10097               Opcode(0xD9), Opcode(0xF1),   // fyl2x
10098               Push_ResultD(dst));
10099   ins_pipe( pipe_slow );
10100 %}
10101 
10102 //-------------Float Instructions-------------------------------
10103 // Float Math
10104 
10105 // Code for float compare:
10106 //     fcompp();
10107 //     fwait(); fnstsw_ax();
10108 //     sahf();
10109 //     movl(dst, unordered_result);
10110 //     jcc(Assembler::parity, exit);
10111 //     movl(dst, less_result);
10112 //     jcc(Assembler::below, exit);
10113 //     movl(dst, equal_result);
10114 //     jcc(Assembler::equal, exit);
10115 //     movl(dst, greater_result);
10116 //   exit:
10117 
10118 // P6 version of float compare, sets condition codes in EFLAGS
10119 instruct cmpFPR_cc_P6(eFlagsRegU cr, regFPR src1, regFPR src2, eAXRegI rax) %{
10120   predicate(VM_Version::supports_cmov() && UseSSE == 0);
10121   match(Set cr (CmpF src1 src2));
10122   effect(KILL rax);
10123   ins_cost(150);
10124   format %{ "FLD    $src1\n\t"
10125             "FUCOMIP ST,$src2  // P6 instruction\n\t"
10126             "JNP    exit\n\t"
10127             "MOV    ah,1       // saw a NaN, set CF (treat as LT)\n\t"
10128             "SAHF\n"
10129      "exit:\tNOP               // avoid branch to branch" %}
10130   opcode(0xDF, 0x05); /* DF E8+i or DF /5 */
10131   ins_encode( Push_Reg_DPR(src1),
10132               OpcP, RegOpc(src2),
10133               cmpF_P6_fixup );
10134   ins_pipe( pipe_slow );
10135 %}
10136 
10137 instruct cmpFPR_cc_P6CF(eFlagsRegUCF cr, regFPR src1, regFPR src2) %{
10138   predicate(VM_Version::supports_cmov() && UseSSE == 0);
10139   match(Set cr (CmpF src1 src2));
10140   ins_cost(100);
10141   format %{ "FLD    $src1\n\t"
10142             "FUCOMIP ST,$src2  // P6 instruction" %}
10143   opcode(0xDF, 0x05); /* DF E8+i or DF /5 */
10144   ins_encode( Push_Reg_DPR(src1),
10145               OpcP, RegOpc(src2));
10146   ins_pipe( pipe_slow );
10147 %}
10148 
10149 
10150 // Compare & branch
10151 instruct cmpFPR_cc(eFlagsRegU cr, regFPR src1, regFPR src2, eAXRegI rax) %{
10152   predicate(UseSSE == 0);
10153   match(Set cr (CmpF src1 src2));
10154   effect(KILL rax);
10155   ins_cost(200);
10156   format %{ "FLD    $src1\n\t"
10157             "FCOMp  $src2\n\t"
10158             "FNSTSW AX\n\t"
10159             "TEST   AX,0x400\n\t"
10160             "JZ,s   flags\n\t"
10161             "MOV    AH,1\t# unordered treat as LT\n"
10162     "flags:\tSAHF" %}
10163   opcode(0xD8, 0x3); /* D8 D8+i or D8 /3 */
10164   ins_encode( Push_Reg_DPR(src1),
10165               OpcP, RegOpc(src2),
10166               fpu_flags);
10167   ins_pipe( pipe_slow );
10168 %}
10169 
10170 // Compare vs zero into -1,0,1
10171 instruct cmpFPR_0(rRegI dst, regFPR src1, immFPR0 zero, eAXRegI rax, eFlagsReg cr) %{
10172   predicate(UseSSE == 0);
10173   match(Set dst (CmpF3 src1 zero));
10174   effect(KILL cr, KILL rax);
10175   ins_cost(280);
10176   format %{ "FTSTF  $dst,$src1" %}
10177   opcode(0xE4, 0xD9);
10178   ins_encode( Push_Reg_DPR(src1),
10179               OpcS, OpcP, PopFPU,
10180               CmpF_Result(dst));
10181   ins_pipe( pipe_slow );
10182 %}
10183 
10184 // Compare into -1,0,1
10185 instruct cmpFPR_reg(rRegI dst, regFPR src1, regFPR src2, eAXRegI rax, eFlagsReg cr) %{
10186   predicate(UseSSE == 0);
10187   match(Set dst (CmpF3 src1 src2));
10188   effect(KILL cr, KILL rax);
10189   ins_cost(300);
10190   format %{ "FCMPF  $dst,$src1,$src2" %}
10191   opcode(0xD8, 0x3); /* D8 D8+i or D8 /3 */
10192   ins_encode( Push_Reg_DPR(src1),
10193               OpcP, RegOpc(src2),
10194               CmpF_Result(dst));
10195   ins_pipe( pipe_slow );
10196 %}
10197 
10198 // float compare and set condition codes in EFLAGS by XMM regs
10199 instruct cmpF_cc(eFlagsRegU cr, regF src1, regF src2) %{
10200   predicate(UseSSE>=1);
10201   match(Set cr (CmpF src1 src2));
10202   ins_cost(145);
10203   format %{ "UCOMISS $src1,$src2\n\t"
10204             "JNP,s   exit\n\t"
10205             "PUSHF\t# saw NaN, set CF\n\t"
10206             "AND     [rsp], #0xffffff2b\n\t"
10207             "POPF\n"
10208     "exit:" %}
10209   ins_encode %{
10210     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
10211     emit_cmpfp_fixup(_masm);
10212   %}
10213   ins_pipe( pipe_slow );
10214 %}
10215 
10216 instruct cmpF_ccCF(eFlagsRegUCF cr, regF src1, regF src2) %{
10217   predicate(UseSSE>=1);
10218   match(Set cr (CmpF src1 src2));
10219   ins_cost(100);
10220   format %{ "UCOMISS $src1,$src2" %}
10221   ins_encode %{
10222     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
10223   %}
10224   ins_pipe( pipe_slow );
10225 %}
10226 
10227 // float compare and set condition codes in EFLAGS by XMM regs
10228 instruct cmpF_ccmem(eFlagsRegU cr, regF src1, memory src2) %{
10229   predicate(UseSSE>=1);
10230   match(Set cr (CmpF src1 (LoadF src2)));
10231   ins_cost(165);
10232   format %{ "UCOMISS $src1,$src2\n\t"
10233             "JNP,s   exit\n\t"
10234             "PUSHF\t# saw NaN, set CF\n\t"
10235             "AND     [rsp], #0xffffff2b\n\t"
10236             "POPF\n"
10237     "exit:" %}
10238   ins_encode %{
10239     __ ucomiss($src1$$XMMRegister, $src2$$Address);
10240     emit_cmpfp_fixup(_masm);
10241   %}
10242   ins_pipe( pipe_slow );
10243 %}
10244 
10245 instruct cmpF_ccmemCF(eFlagsRegUCF cr, regF src1, memory src2) %{
10246   predicate(UseSSE>=1);
10247   match(Set cr (CmpF src1 (LoadF src2)));
10248   ins_cost(100);
10249   format %{ "UCOMISS $src1,$src2" %}
10250   ins_encode %{
10251     __ ucomiss($src1$$XMMRegister, $src2$$Address);
10252   %}
10253   ins_pipe( pipe_slow );
10254 %}
10255 
10256 // Compare into -1,0,1 in XMM
10257 instruct cmpF_reg(xRegI dst, regF src1, regF src2, eFlagsReg cr) %{
10258   predicate(UseSSE>=1);
10259   match(Set dst (CmpF3 src1 src2));
10260   effect(KILL cr);
10261   ins_cost(255);
10262   format %{ "UCOMISS $src1, $src2\n\t"
10263             "MOV     $dst, #-1\n\t"
10264             "JP,s    done\n\t"
10265             "JB,s    done\n\t"
10266             "SETNE   $dst\n\t"
10267             "MOVZB   $dst, $dst\n"
10268     "done:" %}
10269   ins_encode %{
10270     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
10271     emit_cmpfp3(_masm, $dst$$Register);
10272   %}
10273   ins_pipe( pipe_slow );
10274 %}
10275 
10276 // Compare into -1,0,1 in XMM and memory
10277 instruct cmpF_regmem(xRegI dst, regF src1, memory src2, eFlagsReg cr) %{
10278   predicate(UseSSE>=1);
10279   match(Set dst (CmpF3 src1 (LoadF src2)));
10280   effect(KILL cr);
10281   ins_cost(275);
10282   format %{ "UCOMISS $src1, $src2\n\t"
10283             "MOV     $dst, #-1\n\t"
10284             "JP,s    done\n\t"
10285             "JB,s    done\n\t"
10286             "SETNE   $dst\n\t"
10287             "MOVZB   $dst, $dst\n"
10288     "done:" %}
10289   ins_encode %{
10290     __ ucomiss($src1$$XMMRegister, $src2$$Address);
10291     emit_cmpfp3(_masm, $dst$$Register);
10292   %}
10293   ins_pipe( pipe_slow );
10294 %}
10295 
10296 // Spill to obtain 24-bit precision
10297 instruct subFPR24_reg(stackSlotF dst, regFPR src1, regFPR src2) %{
10298   predicate(UseSSE==0 && Compile::current()->select_24_bit_instr());
10299   match(Set dst (SubF src1 src2));
10300 
10301   format %{ "FSUB   $dst,$src1 - $src2" %}
10302   opcode(0xD8, 0x4); /* D8 E0+i or D8 /4 mod==0x3 ;; result in TOS */
10303   ins_encode( Push_Reg_FPR(src1),
10304               OpcReg_FPR(src2),
10305               Pop_Mem_FPR(dst) );
10306   ins_pipe( fpu_mem_reg_reg );
10307 %}
10308 //
10309 // This instruction does not round to 24-bits
10310 instruct subFPR_reg(regFPR dst, regFPR src) %{
10311   predicate(UseSSE==0 && !Compile::current()->select_24_bit_instr());
10312   match(Set dst (SubF dst src));
10313 
10314   format %{ "FSUB   $dst,$src" %}
10315   opcode(0xDE, 0x5); /* DE E8+i  or DE /5 */
10316   ins_encode( Push_Reg_FPR(src),
10317               OpcP, RegOpc(dst) );
10318   ins_pipe( fpu_reg_reg );
10319 %}
10320 
10321 // Spill to obtain 24-bit precision
10322 instruct addFPR24_reg(stackSlotF dst, regFPR src1, regFPR src2) %{
10323   predicate(UseSSE==0 && Compile::current()->select_24_bit_instr());
10324   match(Set dst (AddF src1 src2));
10325 
10326   format %{ "FADD   $dst,$src1,$src2" %}
10327   opcode(0xD8, 0x0); /* D8 C0+i */
10328   ins_encode( Push_Reg_FPR(src2),
10329               OpcReg_FPR(src1),
10330               Pop_Mem_FPR(dst) );
10331   ins_pipe( fpu_mem_reg_reg );
10332 %}
10333 //
10334 // This instruction does not round to 24-bits
10335 instruct addFPR_reg(regFPR dst, regFPR src) %{
10336   predicate(UseSSE==0 && !Compile::current()->select_24_bit_instr());
10337   match(Set dst (AddF dst src));
10338 
10339   format %{ "FLD    $src\n\t"
10340             "FADDp  $dst,ST" %}
10341   opcode(0xDE, 0x0); /* DE C0+i or DE /0*/
10342   ins_encode( Push_Reg_FPR(src),
10343               OpcP, RegOpc(dst) );
10344   ins_pipe( fpu_reg_reg );
10345 %}
10346 
10347 instruct absFPR_reg(regFPR1 dst, regFPR1 src) %{
10348   predicate(UseSSE==0);
10349   match(Set dst (AbsF src));
10350   ins_cost(100);
10351   format %{ "FABS" %}
10352   opcode(0xE1, 0xD9);
10353   ins_encode( OpcS, OpcP );
10354   ins_pipe( fpu_reg_reg );
10355 %}
10356 
10357 instruct negFPR_reg(regFPR1 dst, regFPR1 src) %{
10358   predicate(UseSSE==0);
10359   match(Set dst (NegF src));
10360   ins_cost(100);
10361   format %{ "FCHS" %}
10362   opcode(0xE0, 0xD9);
10363   ins_encode( OpcS, OpcP );
10364   ins_pipe( fpu_reg_reg );
10365 %}
10366 
10367 // Cisc-alternate to addFPR_reg
10368 // Spill to obtain 24-bit precision
10369 instruct addFPR24_reg_mem(stackSlotF dst, regFPR src1, memory src2) %{
10370   predicate(UseSSE==0 && Compile::current()->select_24_bit_instr());
10371   match(Set dst (AddF src1 (LoadF src2)));
10372 
10373   format %{ "FLD    $src2\n\t"
10374             "FADD   ST,$src1\n\t"
10375             "FSTP_S $dst" %}
10376   opcode(0xD8, 0x0, 0xD9); /* D8 C0+i */  /* LoadF  D9 /0 */
10377   ins_encode( Opcode(tertiary), RMopc_Mem(0x00,src2),
10378               OpcReg_FPR(src1),
10379               Pop_Mem_FPR(dst) );
10380   ins_pipe( fpu_mem_reg_mem );
10381 %}
10382 //
10383 // Cisc-alternate to addFPR_reg
10384 // This instruction does not round to 24-bits
10385 instruct addFPR_reg_mem(regFPR dst, memory src) %{
10386   predicate(UseSSE==0 && !Compile::current()->select_24_bit_instr());
10387   match(Set dst (AddF dst (LoadF src)));
10388 
10389   format %{ "FADD   $dst,$src" %}
10390   opcode(0xDE, 0x0, 0xD9); /* DE C0+i or DE /0*/  /* LoadF  D9 /0 */
10391   ins_encode( Opcode(tertiary), RMopc_Mem(0x00,src),
10392               OpcP, RegOpc(dst) );
10393   ins_pipe( fpu_reg_mem );
10394 %}
10395 
10396 // // Following two instructions for _222_mpegaudio
10397 // Spill to obtain 24-bit precision
10398 instruct addFPR24_mem_reg(stackSlotF dst, regFPR src2, memory src1 ) %{
10399   predicate(UseSSE==0 && Compile::current()->select_24_bit_instr());
10400   match(Set dst (AddF src1 src2));
10401 
10402   format %{ "FADD   $dst,$src1,$src2" %}
10403   opcode(0xD8, 0x0, 0xD9); /* D8 C0+i */  /* LoadF  D9 /0 */
10404   ins_encode( Opcode(tertiary), RMopc_Mem(0x00,src1),
10405               OpcReg_FPR(src2),
10406               Pop_Mem_FPR(dst) );
10407   ins_pipe( fpu_mem_reg_mem );
10408 %}
10409 
10410 // Cisc-spill variant
10411 // Spill to obtain 24-bit precision
10412 instruct addFPR24_mem_cisc(stackSlotF dst, memory src1, memory src2) %{
10413   predicate(UseSSE==0 && Compile::current()->select_24_bit_instr());
10414   match(Set dst (AddF src1 (LoadF src2)));
10415 
10416   format %{ "FADD   $dst,$src1,$src2 cisc" %}
10417   opcode(0xD8, 0x0, 0xD9); /* D8 C0+i */  /* LoadF  D9 /0 */
10418   ins_encode( Opcode(tertiary), RMopc_Mem(0x00,src2),
10419               set_instruction_start,
10420               OpcP, RMopc_Mem(secondary,src1),
10421               Pop_Mem_FPR(dst) );
10422   ins_pipe( fpu_mem_mem_mem );
10423 %}
10424 
10425 // Spill to obtain 24-bit precision
10426 instruct addFPR24_mem_mem(stackSlotF dst, memory src1, memory src2) %{
10427   predicate(UseSSE==0 && Compile::current()->select_24_bit_instr());
10428   match(Set dst (AddF src1 src2));
10429 
10430   format %{ "FADD   $dst,$src1,$src2" %}
10431   opcode(0xD8, 0x0, 0xD9); /* D8 /0 */  /* LoadF  D9 /0 */
10432   ins_encode( Opcode(tertiary), RMopc_Mem(0x00,src2),
10433               set_instruction_start,
10434               OpcP, RMopc_Mem(secondary,src1),
10435               Pop_Mem_FPR(dst) );
10436   ins_pipe( fpu_mem_mem_mem );
10437 %}
10438 
10439 
10440 // Spill to obtain 24-bit precision
10441 instruct addFPR24_reg_imm(stackSlotF dst, regFPR src, immFPR con) %{
10442   predicate(UseSSE==0 && Compile::current()->select_24_bit_instr());
10443   match(Set dst (AddF src con));
10444   format %{ "FLD    $src\n\t"
10445             "FADD_S [$constantaddress]\t# load from constant table: float=$con\n\t"
10446             "FSTP_S $dst"  %}
10447   ins_encode %{
10448     __ fld_s($src$$reg - 1);  // FLD ST(i-1)
10449     __ fadd_s($constantaddress($con));
10450     __ fstp_s(Address(rsp, $dst$$disp));
10451   %}
10452   ins_pipe(fpu_mem_reg_con);
10453 %}
10454 //
10455 // This instruction does not round to 24-bits
10456 instruct addFPR_reg_imm(regFPR dst, regFPR src, immFPR con) %{
10457   predicate(UseSSE==0 && !Compile::current()->select_24_bit_instr());
10458   match(Set dst (AddF src con));
10459   format %{ "FLD    $src\n\t"
10460             "FADD_S [$constantaddress]\t# load from constant table: float=$con\n\t"
10461             "FSTP   $dst"  %}
10462   ins_encode %{
10463     __ fld_s($src$$reg - 1);  // FLD ST(i-1)
10464     __ fadd_s($constantaddress($con));
10465     __ fstp_d($dst$$reg);
10466   %}
10467   ins_pipe(fpu_reg_reg_con);
10468 %}
10469 
10470 // Spill to obtain 24-bit precision
10471 instruct mulFPR24_reg(stackSlotF dst, regFPR src1, regFPR src2) %{
10472   predicate(UseSSE==0 && Compile::current()->select_24_bit_instr());
10473   match(Set dst (MulF src1 src2));
10474 
10475   format %{ "FLD    $src1\n\t"
10476             "FMUL   $src2\n\t"
10477             "FSTP_S $dst"  %}
10478   opcode(0xD8, 0x1); /* D8 C8+i or D8 /1 ;; result in TOS */
10479   ins_encode( Push_Reg_FPR(src1),
10480               OpcReg_FPR(src2),
10481               Pop_Mem_FPR(dst) );
10482   ins_pipe( fpu_mem_reg_reg );
10483 %}
10484 //
10485 // This instruction does not round to 24-bits
10486 instruct mulFPR_reg(regFPR dst, regFPR src1, regFPR src2) %{
10487   predicate(UseSSE==0 && !Compile::current()->select_24_bit_instr());
10488   match(Set dst (MulF src1 src2));
10489 
10490   format %{ "FLD    $src1\n\t"
10491             "FMUL   $src2\n\t"
10492             "FSTP_S $dst"  %}
10493   opcode(0xD8, 0x1); /* D8 C8+i */
10494   ins_encode( Push_Reg_FPR(src2),
10495               OpcReg_FPR(src1),
10496               Pop_Reg_FPR(dst) );
10497   ins_pipe( fpu_reg_reg_reg );
10498 %}
10499 
10500 
10501 // Spill to obtain 24-bit precision
10502 // Cisc-alternate to reg-reg multiply
10503 instruct mulFPR24_reg_mem(stackSlotF dst, regFPR src1, memory src2) %{
10504   predicate(UseSSE==0 && Compile::current()->select_24_bit_instr());
10505   match(Set dst (MulF src1 (LoadF src2)));
10506 
10507   format %{ "FLD_S  $src2\n\t"
10508             "FMUL   $src1\n\t"
10509             "FSTP_S $dst"  %}
10510   opcode(0xD8, 0x1, 0xD9); /* D8 C8+i or DE /1*/  /* LoadF D9 /0 */
10511   ins_encode( Opcode(tertiary), RMopc_Mem(0x00,src2),
10512               OpcReg_FPR(src1),
10513               Pop_Mem_FPR(dst) );
10514   ins_pipe( fpu_mem_reg_mem );
10515 %}
10516 //
10517 // This instruction does not round to 24-bits
10518 // Cisc-alternate to reg-reg multiply
10519 instruct mulFPR_reg_mem(regFPR dst, regFPR src1, memory src2) %{
10520   predicate(UseSSE==0 && !Compile::current()->select_24_bit_instr());
10521   match(Set dst (MulF src1 (LoadF src2)));
10522 
10523   format %{ "FMUL   $dst,$src1,$src2" %}
10524   opcode(0xD8, 0x1, 0xD9); /* D8 C8+i */  /* LoadF D9 /0 */
10525   ins_encode( Opcode(tertiary), RMopc_Mem(0x00,src2),
10526               OpcReg_FPR(src1),
10527               Pop_Reg_FPR(dst) );
10528   ins_pipe( fpu_reg_reg_mem );
10529 %}
10530 
10531 // Spill to obtain 24-bit precision
10532 instruct mulFPR24_mem_mem(stackSlotF dst, memory src1, memory src2) %{
10533   predicate(UseSSE==0 && Compile::current()->select_24_bit_instr());
10534   match(Set dst (MulF src1 src2));
10535 
10536   format %{ "FMUL   $dst,$src1,$src2" %}
10537   opcode(0xD8, 0x1, 0xD9); /* D8 /1 */  /* LoadF D9 /0 */
10538   ins_encode( Opcode(tertiary), RMopc_Mem(0x00,src2),
10539               set_instruction_start,
10540               OpcP, RMopc_Mem(secondary,src1),
10541               Pop_Mem_FPR(dst) );
10542   ins_pipe( fpu_mem_mem_mem );
10543 %}
10544 
10545 // Spill to obtain 24-bit precision
10546 instruct mulFPR24_reg_imm(stackSlotF dst, regFPR src, immFPR con) %{
10547   predicate(UseSSE==0 && Compile::current()->select_24_bit_instr());
10548   match(Set dst (MulF src con));
10549 
10550   format %{ "FLD    $src\n\t"
10551             "FMUL_S [$constantaddress]\t# load from constant table: float=$con\n\t"
10552             "FSTP_S $dst"  %}
10553   ins_encode %{
10554     __ fld_s($src$$reg - 1);  // FLD ST(i-1)
10555     __ fmul_s($constantaddress($con));
10556     __ fstp_s(Address(rsp, $dst$$disp));
10557   %}
10558   ins_pipe(fpu_mem_reg_con);
10559 %}
10560 //
10561 // This instruction does not round to 24-bits
10562 instruct mulFPR_reg_imm(regFPR dst, regFPR src, immFPR con) %{
10563   predicate(UseSSE==0 && !Compile::current()->select_24_bit_instr());
10564   match(Set dst (MulF src con));
10565 
10566   format %{ "FLD    $src\n\t"
10567             "FMUL_S [$constantaddress]\t# load from constant table: float=$con\n\t"
10568             "FSTP   $dst"  %}
10569   ins_encode %{
10570     __ fld_s($src$$reg - 1);  // FLD ST(i-1)
10571     __ fmul_s($constantaddress($con));
10572     __ fstp_d($dst$$reg);
10573   %}
10574   ins_pipe(fpu_reg_reg_con);
10575 %}
10576 
10577 
10578 //
10579 // MACRO1 -- subsume unshared load into mulFPR
10580 // This instruction does not round to 24-bits
10581 instruct mulFPR_reg_load1(regFPR dst, regFPR src, memory mem1 ) %{
10582   predicate(UseSSE==0 && !Compile::current()->select_24_bit_instr());
10583   match(Set dst (MulF (LoadF mem1) src));
10584 
10585   format %{ "FLD    $mem1    ===MACRO1===\n\t"
10586             "FMUL   ST,$src\n\t"
10587             "FSTP   $dst" %}
10588   opcode(0xD8, 0x1, 0xD9); /* D8 C8+i or D8 /1 */  /* LoadF D9 /0 */
10589   ins_encode( Opcode(tertiary), RMopc_Mem(0x00,mem1),
10590               OpcReg_FPR(src),
10591               Pop_Reg_FPR(dst) );
10592   ins_pipe( fpu_reg_reg_mem );
10593 %}
10594 //
10595 // MACRO2 -- addFPR a mulFPR which subsumed an unshared load
10596 // This instruction does not round to 24-bits
10597 instruct addFPR_mulFPR_reg_load1(regFPR dst, memory mem1, regFPR src1, regFPR src2) %{
10598   predicate(UseSSE==0 && !Compile::current()->select_24_bit_instr());
10599   match(Set dst (AddF (MulF (LoadF mem1) src1) src2));
10600   ins_cost(95);
10601 
10602   format %{ "FLD    $mem1     ===MACRO2===\n\t"
10603             "FMUL   ST,$src1  subsume mulFPR left load\n\t"
10604             "FADD   ST,$src2\n\t"
10605             "FSTP   $dst" %}
10606   opcode(0xD9); /* LoadF D9 /0 */
10607   ins_encode( OpcP, RMopc_Mem(0x00,mem1),
10608               FMul_ST_reg(src1),
10609               FAdd_ST_reg(src2),
10610               Pop_Reg_FPR(dst) );
10611   ins_pipe( fpu_reg_mem_reg_reg );
10612 %}
10613 
10614 // MACRO3 -- addFPR a mulFPR
10615 // This instruction does not round to 24-bits.  It is a '2-address'
10616 // instruction in that the result goes back to src2.  This eliminates
10617 // a move from the macro; possibly the register allocator will have
10618 // to add it back (and maybe not).
10619 instruct addFPR_mulFPR_reg(regFPR src2, regFPR src1, regFPR src0) %{
10620   predicate(UseSSE==0 && !Compile::current()->select_24_bit_instr());
10621   match(Set src2 (AddF (MulF src0 src1) src2));
10622 
10623   format %{ "FLD    $src0     ===MACRO3===\n\t"
10624             "FMUL   ST,$src1\n\t"
10625             "FADDP  $src2,ST" %}
10626   opcode(0xD9); /* LoadF D9 /0 */
10627   ins_encode( Push_Reg_FPR(src0),
10628               FMul_ST_reg(src1),
10629               FAddP_reg_ST(src2) );
10630   ins_pipe( fpu_reg_reg_reg );
10631 %}
10632 
10633 // MACRO4 -- divFPR subFPR
10634 // This instruction does not round to 24-bits
10635 instruct subFPR_divFPR_reg(regFPR dst, regFPR src1, regFPR src2, regFPR src3) %{
10636   predicate(UseSSE==0 && !Compile::current()->select_24_bit_instr());
10637   match(Set dst (DivF (SubF src2 src1) src3));
10638 
10639   format %{ "FLD    $src2   ===MACRO4===\n\t"
10640             "FSUB   ST,$src1\n\t"
10641             "FDIV   ST,$src3\n\t"
10642             "FSTP  $dst" %}
10643   opcode(0xDE, 0x7); /* DE F8+i or DE /7*/
10644   ins_encode( Push_Reg_FPR(src2),
10645               subFPR_divFPR_encode(src1,src3),
10646               Pop_Reg_FPR(dst) );
10647   ins_pipe( fpu_reg_reg_reg_reg );
10648 %}
10649 
10650 // Spill to obtain 24-bit precision
10651 instruct divFPR24_reg(stackSlotF dst, regFPR src1, regFPR src2) %{
10652   predicate(UseSSE==0 && Compile::current()->select_24_bit_instr());
10653   match(Set dst (DivF src1 src2));
10654 
10655   format %{ "FDIV   $dst,$src1,$src2" %}
10656   opcode(0xD8, 0x6); /* D8 F0+i or DE /6*/
10657   ins_encode( Push_Reg_FPR(src1),
10658               OpcReg_FPR(src2),
10659               Pop_Mem_FPR(dst) );
10660   ins_pipe( fpu_mem_reg_reg );
10661 %}
10662 //
10663 // This instruction does not round to 24-bits
10664 instruct divFPR_reg(regFPR dst, regFPR src) %{
10665   predicate(UseSSE==0 && !Compile::current()->select_24_bit_instr());
10666   match(Set dst (DivF dst src));
10667 
10668   format %{ "FDIV   $dst,$src" %}
10669   opcode(0xDE, 0x7); /* DE F8+i or DE /7*/
10670   ins_encode( Push_Reg_FPR(src),
10671               OpcP, RegOpc(dst) );
10672   ins_pipe( fpu_reg_reg );
10673 %}
10674 
10675 
10676 // Spill to obtain 24-bit precision
10677 instruct modFPR24_reg(stackSlotF dst, regFPR src1, regFPR src2, eAXRegI rax, eFlagsReg cr) %{
10678   predicate( UseSSE==0 && Compile::current()->select_24_bit_instr());
10679   match(Set dst (ModF src1 src2));
10680   effect(KILL rax, KILL cr); // emitModDPR() uses EAX and EFLAGS
10681 
10682   format %{ "FMOD   $dst,$src1,$src2" %}
10683   ins_encode( Push_Reg_Mod_DPR(src1, src2),
10684               emitModDPR(),
10685               Push_Result_Mod_DPR(src2),
10686               Pop_Mem_FPR(dst));
10687   ins_pipe( pipe_slow );
10688 %}
10689 //
10690 // This instruction does not round to 24-bits
10691 instruct modFPR_reg(regFPR dst, regFPR src, eAXRegI rax, eFlagsReg cr) %{
10692   predicate( UseSSE==0 && !Compile::current()->select_24_bit_instr());
10693   match(Set dst (ModF dst src));
10694   effect(KILL rax, KILL cr); // emitModDPR() uses EAX and EFLAGS
10695 
10696   format %{ "FMOD   $dst,$src" %}
10697   ins_encode(Push_Reg_Mod_DPR(dst, src),
10698               emitModDPR(),
10699               Push_Result_Mod_DPR(src),
10700               Pop_Reg_FPR(dst));
10701   ins_pipe( pipe_slow );
10702 %}
10703 
10704 instruct modF_reg(regF dst, regF src0, regF src1, eAXRegI rax, eFlagsReg cr) %{
10705   predicate(UseSSE>=1);
10706   match(Set dst (ModF src0 src1));
10707   effect(KILL rax, KILL cr);
10708   format %{ "SUB    ESP,4\t # FMOD\n"
10709           "\tMOVSS  [ESP+0],$src1\n"
10710           "\tFLD_S  [ESP+0]\n"
10711           "\tMOVSS  [ESP+0],$src0\n"
10712           "\tFLD_S  [ESP+0]\n"
10713      "loop:\tFPREM\n"
10714           "\tFWAIT\n"
10715           "\tFNSTSW AX\n"
10716           "\tSAHF\n"
10717           "\tJP     loop\n"
10718           "\tFSTP_S [ESP+0]\n"
10719           "\tMOVSS  $dst,[ESP+0]\n"
10720           "\tADD    ESP,4\n"
10721           "\tFSTP   ST0\t # Restore FPU Stack"
10722     %}
10723   ins_cost(250);
10724   ins_encode( Push_ModF_encoding(src0, src1), emitModDPR(), Push_ResultF(dst,0x4), PopFPU);
10725   ins_pipe( pipe_slow );
10726 %}
10727 
10728 
10729 //----------Arithmetic Conversion Instructions---------------------------------
10730 // The conversions operations are all Alpha sorted.  Please keep it that way!
10731 
10732 instruct roundFloat_mem_reg(stackSlotF dst, regFPR src) %{
10733   predicate(UseSSE==0);
10734   match(Set dst (RoundFloat src));
10735   ins_cost(125);
10736   format %{ "FST_S  $dst,$src\t# F-round" %}
10737   ins_encode( Pop_Mem_Reg_FPR(dst, src) );
10738   ins_pipe( fpu_mem_reg );
10739 %}
10740 
10741 instruct roundDouble_mem_reg(stackSlotD dst, regDPR src) %{
10742   predicate(UseSSE<=1);
10743   match(Set dst (RoundDouble src));
10744   ins_cost(125);
10745   format %{ "FST_D  $dst,$src\t# D-round" %}
10746   ins_encode( Pop_Mem_Reg_DPR(dst, src) );
10747   ins_pipe( fpu_mem_reg );
10748 %}
10749 
10750 // Force rounding to 24-bit precision and 6-bit exponent
10751 instruct convDPR2FPR_reg(stackSlotF dst, regDPR src) %{
10752   predicate(UseSSE==0);
10753   match(Set dst (ConvD2F src));
10754   format %{ "FST_S  $dst,$src\t# F-round" %}
10755   expand %{
10756     roundFloat_mem_reg(dst,src);
10757   %}
10758 %}
10759 
10760 // Force rounding to 24-bit precision and 6-bit exponent
10761 instruct convDPR2F_reg(regF dst, regDPR src, eFlagsReg cr) %{
10762   predicate(UseSSE==1);
10763   match(Set dst (ConvD2F src));
10764   effect( KILL cr );
10765   format %{ "SUB    ESP,4\n\t"
10766             "FST_S  [ESP],$src\t# F-round\n\t"
10767             "MOVSS  $dst,[ESP]\n\t"
10768             "ADD ESP,4" %}
10769   ins_encode %{
10770     __ subptr(rsp, 4);
10771     if ($src$$reg != FPR1L_enc) {
10772       __ fld_s($src$$reg-1);
10773       __ fstp_s(Address(rsp, 0));
10774     } else {
10775       __ fst_s(Address(rsp, 0));
10776     }
10777     __ movflt($dst$$XMMRegister, Address(rsp, 0));
10778     __ addptr(rsp, 4);
10779   %}
10780   ins_pipe( pipe_slow );
10781 %}
10782 
10783 // Force rounding double precision to single precision
10784 instruct convD2F_reg(regF dst, regD src) %{
10785   predicate(UseSSE>=2);
10786   match(Set dst (ConvD2F src));
10787   format %{ "CVTSD2SS $dst,$src\t# F-round" %}
10788   ins_encode %{
10789     __ cvtsd2ss ($dst$$XMMRegister, $src$$XMMRegister);
10790   %}
10791   ins_pipe( pipe_slow );
10792 %}
10793 
10794 instruct convFPR2DPR_reg_reg(regDPR dst, regFPR src) %{
10795   predicate(UseSSE==0);
10796   match(Set dst (ConvF2D src));
10797   format %{ "FST_S  $dst,$src\t# D-round" %}
10798   ins_encode( Pop_Reg_Reg_DPR(dst, src));
10799   ins_pipe( fpu_reg_reg );
10800 %}
10801 
10802 instruct convFPR2D_reg(stackSlotD dst, regFPR src) %{
10803   predicate(UseSSE==1);
10804   match(Set dst (ConvF2D src));
10805   format %{ "FST_D  $dst,$src\t# D-round" %}
10806   expand %{
10807     roundDouble_mem_reg(dst,src);
10808   %}
10809 %}
10810 
10811 instruct convF2DPR_reg(regDPR dst, regF src, eFlagsReg cr) %{
10812   predicate(UseSSE==1);
10813   match(Set dst (ConvF2D src));
10814   effect( KILL cr );
10815   format %{ "SUB    ESP,4\n\t"
10816             "MOVSS  [ESP] $src\n\t"
10817             "FLD_S  [ESP]\n\t"
10818             "ADD    ESP,4\n\t"
10819             "FSTP   $dst\t# D-round" %}
10820   ins_encode %{
10821     __ subptr(rsp, 4);
10822     __ movflt(Address(rsp, 0), $src$$XMMRegister);
10823     __ fld_s(Address(rsp, 0));
10824     __ addptr(rsp, 4);
10825     __ fstp_d($dst$$reg);
10826   %}
10827   ins_pipe( pipe_slow );
10828 %}
10829 
10830 instruct convF2D_reg(regD dst, regF src) %{
10831   predicate(UseSSE>=2);
10832   match(Set dst (ConvF2D src));
10833   format %{ "CVTSS2SD $dst,$src\t# D-round" %}
10834   ins_encode %{
10835     __ cvtss2sd ($dst$$XMMRegister, $src$$XMMRegister);
10836   %}
10837   ins_pipe( pipe_slow );
10838 %}
10839 
10840 // Convert a double to an int.  If the double is a NAN, stuff a zero in instead.
10841 instruct convDPR2I_reg_reg( eAXRegI dst, eDXRegI tmp, regDPR src, eFlagsReg cr ) %{
10842   predicate(UseSSE<=1);
10843   match(Set dst (ConvD2I src));
10844   effect( KILL tmp, KILL cr );
10845   format %{ "FLD    $src\t# Convert double to int \n\t"
10846             "FLDCW  trunc mode\n\t"
10847             "SUB    ESP,4\n\t"
10848             "FISTp  [ESP + #0]\n\t"
10849             "FLDCW  std/24-bit mode\n\t"
10850             "POP    EAX\n\t"
10851             "CMP    EAX,0x80000000\n\t"
10852             "JNE,s  fast\n\t"
10853             "FLD_D  $src\n\t"
10854             "CALL   d2i_wrapper\n"
10855       "fast:" %}
10856   ins_encode( Push_Reg_DPR(src), DPR2I_encoding(src) );
10857   ins_pipe( pipe_slow );
10858 %}
10859 
10860 // Convert a double to an int.  If the double is a NAN, stuff a zero in instead.
10861 instruct convD2I_reg_reg( eAXRegI dst, eDXRegI tmp, regD src, eFlagsReg cr ) %{
10862   predicate(UseSSE>=2);
10863   match(Set dst (ConvD2I src));
10864   effect( KILL tmp, KILL cr );
10865   format %{ "CVTTSD2SI $dst, $src\n\t"
10866             "CMP    $dst,0x80000000\n\t"
10867             "JNE,s  fast\n\t"
10868             "SUB    ESP, 8\n\t"
10869             "MOVSD  [ESP], $src\n\t"
10870             "FLD_D  [ESP]\n\t"
10871             "ADD    ESP, 8\n\t"
10872             "CALL   d2i_wrapper\n"
10873       "fast:" %}
10874   ins_encode %{
10875     Label fast;
10876     __ cvttsd2sil($dst$$Register, $src$$XMMRegister);
10877     __ cmpl($dst$$Register, 0x80000000);
10878     __ jccb(Assembler::notEqual, fast);
10879     __ subptr(rsp, 8);
10880     __ movdbl(Address(rsp, 0), $src$$XMMRegister);
10881     __ fld_d(Address(rsp, 0));
10882     __ addptr(rsp, 8);
10883     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::d2i_wrapper())));
10884     __ bind(fast);
10885   %}
10886   ins_pipe( pipe_slow );
10887 %}
10888 
10889 instruct convDPR2L_reg_reg( eADXRegL dst, regDPR src, eFlagsReg cr ) %{
10890   predicate(UseSSE<=1);
10891   match(Set dst (ConvD2L src));
10892   effect( KILL cr );
10893   format %{ "FLD    $src\t# Convert double to long\n\t"
10894             "FLDCW  trunc mode\n\t"
10895             "SUB    ESP,8\n\t"
10896             "FISTp  [ESP + #0]\n\t"
10897             "FLDCW  std/24-bit mode\n\t"
10898             "POP    EAX\n\t"
10899             "POP    EDX\n\t"
10900             "CMP    EDX,0x80000000\n\t"
10901             "JNE,s  fast\n\t"
10902             "TEST   EAX,EAX\n\t"
10903             "JNE,s  fast\n\t"
10904             "FLD    $src\n\t"
10905             "CALL   d2l_wrapper\n"
10906       "fast:" %}
10907   ins_encode( Push_Reg_DPR(src),  DPR2L_encoding(src) );
10908   ins_pipe( pipe_slow );
10909 %}
10910 
10911 // XMM lacks a float/double->long conversion, so use the old FPU stack.
10912 instruct convD2L_reg_reg( eADXRegL dst, regD src, eFlagsReg cr ) %{
10913   predicate (UseSSE>=2);
10914   match(Set dst (ConvD2L src));
10915   effect( KILL cr );
10916   format %{ "SUB    ESP,8\t# Convert double to long\n\t"
10917             "MOVSD  [ESP],$src\n\t"
10918             "FLD_D  [ESP]\n\t"
10919             "FLDCW  trunc mode\n\t"
10920             "FISTp  [ESP + #0]\n\t"
10921             "FLDCW  std/24-bit mode\n\t"
10922             "POP    EAX\n\t"
10923             "POP    EDX\n\t"
10924             "CMP    EDX,0x80000000\n\t"
10925             "JNE,s  fast\n\t"
10926             "TEST   EAX,EAX\n\t"
10927             "JNE,s  fast\n\t"
10928             "SUB    ESP,8\n\t"
10929             "MOVSD  [ESP],$src\n\t"
10930             "FLD_D  [ESP]\n\t"
10931             "ADD    ESP,8\n\t"
10932             "CALL   d2l_wrapper\n"
10933       "fast:" %}
10934   ins_encode %{
10935     Label fast;
10936     __ subptr(rsp, 8);
10937     __ movdbl(Address(rsp, 0), $src$$XMMRegister);
10938     __ fld_d(Address(rsp, 0));
10939     __ fldcw(ExternalAddress(StubRoutines::addr_fpu_cntrl_wrd_trunc()));
10940     __ fistp_d(Address(rsp, 0));
10941     // Restore the rounding mode, mask the exception
10942     if (Compile::current()->in_24_bit_fp_mode()) {
10943       __ fldcw(ExternalAddress(StubRoutines::addr_fpu_cntrl_wrd_24()));
10944     } else {
10945       __ fldcw(ExternalAddress(StubRoutines::addr_fpu_cntrl_wrd_std()));
10946     }
10947     // Load the converted long, adjust CPU stack
10948     __ pop(rax);
10949     __ pop(rdx);
10950     __ cmpl(rdx, 0x80000000);
10951     __ jccb(Assembler::notEqual, fast);
10952     __ testl(rax, rax);
10953     __ jccb(Assembler::notEqual, fast);
10954     __ subptr(rsp, 8);
10955     __ movdbl(Address(rsp, 0), $src$$XMMRegister);
10956     __ fld_d(Address(rsp, 0));
10957     __ addptr(rsp, 8);
10958     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::d2l_wrapper())));
10959     __ bind(fast);
10960   %}
10961   ins_pipe( pipe_slow );
10962 %}
10963 
10964 // Convert a double to an int.  Java semantics require we do complex
10965 // manglations in the corner cases.  So we set the rounding mode to
10966 // 'zero', store the darned double down as an int, and reset the
10967 // rounding mode to 'nearest'.  The hardware stores a flag value down
10968 // if we would overflow or converted a NAN; we check for this and
10969 // and go the slow path if needed.
10970 instruct convFPR2I_reg_reg(eAXRegI dst, eDXRegI tmp, regFPR src, eFlagsReg cr ) %{
10971   predicate(UseSSE==0);
10972   match(Set dst (ConvF2I src));
10973   effect( KILL tmp, KILL cr );
10974   format %{ "FLD    $src\t# Convert float to int \n\t"
10975             "FLDCW  trunc mode\n\t"
10976             "SUB    ESP,4\n\t"
10977             "FISTp  [ESP + #0]\n\t"
10978             "FLDCW  std/24-bit mode\n\t"
10979             "POP    EAX\n\t"
10980             "CMP    EAX,0x80000000\n\t"
10981             "JNE,s  fast\n\t"
10982             "FLD    $src\n\t"
10983             "CALL   d2i_wrapper\n"
10984       "fast:" %}
10985   // DPR2I_encoding works for FPR2I
10986   ins_encode( Push_Reg_FPR(src), DPR2I_encoding(src) );
10987   ins_pipe( pipe_slow );
10988 %}
10989 
10990 // Convert a float in xmm to an int reg.
10991 instruct convF2I_reg(eAXRegI dst, eDXRegI tmp, regF src, eFlagsReg cr ) %{
10992   predicate(UseSSE>=1);
10993   match(Set dst (ConvF2I src));
10994   effect( KILL tmp, KILL cr );
10995   format %{ "CVTTSS2SI $dst, $src\n\t"
10996             "CMP    $dst,0x80000000\n\t"
10997             "JNE,s  fast\n\t"
10998             "SUB    ESP, 4\n\t"
10999             "MOVSS  [ESP], $src\n\t"
11000             "FLD    [ESP]\n\t"
11001             "ADD    ESP, 4\n\t"
11002             "CALL   d2i_wrapper\n"
11003       "fast:" %}
11004   ins_encode %{
11005     Label fast;
11006     __ cvttss2sil($dst$$Register, $src$$XMMRegister);
11007     __ cmpl($dst$$Register, 0x80000000);
11008     __ jccb(Assembler::notEqual, fast);
11009     __ subptr(rsp, 4);
11010     __ movflt(Address(rsp, 0), $src$$XMMRegister);
11011     __ fld_s(Address(rsp, 0));
11012     __ addptr(rsp, 4);
11013     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::d2i_wrapper())));
11014     __ bind(fast);
11015   %}
11016   ins_pipe( pipe_slow );
11017 %}
11018 
11019 instruct convFPR2L_reg_reg( eADXRegL dst, regFPR src, eFlagsReg cr ) %{
11020   predicate(UseSSE==0);
11021   match(Set dst (ConvF2L src));
11022   effect( KILL cr );
11023   format %{ "FLD    $src\t# Convert float to long\n\t"
11024             "FLDCW  trunc mode\n\t"
11025             "SUB    ESP,8\n\t"
11026             "FISTp  [ESP + #0]\n\t"
11027             "FLDCW  std/24-bit mode\n\t"
11028             "POP    EAX\n\t"
11029             "POP    EDX\n\t"
11030             "CMP    EDX,0x80000000\n\t"
11031             "JNE,s  fast\n\t"
11032             "TEST   EAX,EAX\n\t"
11033             "JNE,s  fast\n\t"
11034             "FLD    $src\n\t"
11035             "CALL   d2l_wrapper\n"
11036       "fast:" %}
11037   // DPR2L_encoding works for FPR2L
11038   ins_encode( Push_Reg_FPR(src), DPR2L_encoding(src) );
11039   ins_pipe( pipe_slow );
11040 %}
11041 
11042 // XMM lacks a float/double->long conversion, so use the old FPU stack.
11043 instruct convF2L_reg_reg( eADXRegL dst, regF src, eFlagsReg cr ) %{
11044   predicate (UseSSE>=1);
11045   match(Set dst (ConvF2L src));
11046   effect( KILL cr );
11047   format %{ "SUB    ESP,8\t# Convert float to long\n\t"
11048             "MOVSS  [ESP],$src\n\t"
11049             "FLD_S  [ESP]\n\t"
11050             "FLDCW  trunc mode\n\t"
11051             "FISTp  [ESP + #0]\n\t"
11052             "FLDCW  std/24-bit mode\n\t"
11053             "POP    EAX\n\t"
11054             "POP    EDX\n\t"
11055             "CMP    EDX,0x80000000\n\t"
11056             "JNE,s  fast\n\t"
11057             "TEST   EAX,EAX\n\t"
11058             "JNE,s  fast\n\t"
11059             "SUB    ESP,4\t# Convert float to long\n\t"
11060             "MOVSS  [ESP],$src\n\t"
11061             "FLD_S  [ESP]\n\t"
11062             "ADD    ESP,4\n\t"
11063             "CALL   d2l_wrapper\n"
11064       "fast:" %}
11065   ins_encode %{
11066     Label fast;
11067     __ subptr(rsp, 8);
11068     __ movflt(Address(rsp, 0), $src$$XMMRegister);
11069     __ fld_s(Address(rsp, 0));
11070     __ fldcw(ExternalAddress(StubRoutines::addr_fpu_cntrl_wrd_trunc()));
11071     __ fistp_d(Address(rsp, 0));
11072     // Restore the rounding mode, mask the exception
11073     if (Compile::current()->in_24_bit_fp_mode()) {
11074       __ fldcw(ExternalAddress(StubRoutines::addr_fpu_cntrl_wrd_24()));
11075     } else {
11076       __ fldcw(ExternalAddress(StubRoutines::addr_fpu_cntrl_wrd_std()));
11077     }
11078     // Load the converted long, adjust CPU stack
11079     __ pop(rax);
11080     __ pop(rdx);
11081     __ cmpl(rdx, 0x80000000);
11082     __ jccb(Assembler::notEqual, fast);
11083     __ testl(rax, rax);
11084     __ jccb(Assembler::notEqual, fast);
11085     __ subptr(rsp, 4);
11086     __ movflt(Address(rsp, 0), $src$$XMMRegister);
11087     __ fld_s(Address(rsp, 0));
11088     __ addptr(rsp, 4);
11089     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::d2l_wrapper())));
11090     __ bind(fast);
11091   %}
11092   ins_pipe( pipe_slow );
11093 %}
11094 
11095 instruct convI2DPR_reg(regDPR dst, stackSlotI src) %{
11096   predicate( UseSSE<=1 );
11097   match(Set dst (ConvI2D src));
11098   format %{ "FILD   $src\n\t"
11099             "FSTP   $dst" %}
11100   opcode(0xDB, 0x0);  /* DB /0 */
11101   ins_encode(Push_Mem_I(src), Pop_Reg_DPR(dst));
11102   ins_pipe( fpu_reg_mem );
11103 %}
11104 
11105 instruct convI2D_reg(regD dst, rRegI src) %{
11106   predicate( UseSSE>=2 && !UseXmmI2D );
11107   match(Set dst (ConvI2D src));
11108   format %{ "CVTSI2SD $dst,$src" %}
11109   ins_encode %{
11110     __ cvtsi2sdl ($dst$$XMMRegister, $src$$Register);
11111   %}
11112   ins_pipe( pipe_slow );
11113 %}
11114 
11115 instruct convI2D_mem(regD dst, memory mem) %{
11116   predicate( UseSSE>=2 );
11117   match(Set dst (ConvI2D (LoadI mem)));
11118   format %{ "CVTSI2SD $dst,$mem" %}
11119   ins_encode %{
11120     __ cvtsi2sdl ($dst$$XMMRegister, $mem$$Address);
11121   %}
11122   ins_pipe( pipe_slow );
11123 %}
11124 
11125 instruct convXI2D_reg(regD dst, rRegI src)
11126 %{
11127   predicate( UseSSE>=2 && UseXmmI2D );
11128   match(Set dst (ConvI2D src));
11129 
11130   format %{ "MOVD  $dst,$src\n\t"
11131             "CVTDQ2PD $dst,$dst\t# i2d" %}
11132   ins_encode %{
11133     __ movdl($dst$$XMMRegister, $src$$Register);
11134     __ cvtdq2pd($dst$$XMMRegister, $dst$$XMMRegister);
11135   %}
11136   ins_pipe(pipe_slow); // XXX
11137 %}
11138 
11139 instruct convI2DPR_mem(regDPR dst, memory mem) %{
11140   predicate( UseSSE<=1 && !Compile::current()->select_24_bit_instr());
11141   match(Set dst (ConvI2D (LoadI mem)));
11142   format %{ "FILD   $mem\n\t"
11143             "FSTP   $dst" %}
11144   opcode(0xDB);      /* DB /0 */
11145   ins_encode( OpcP, RMopc_Mem(0x00,mem),
11146               Pop_Reg_DPR(dst));
11147   ins_pipe( fpu_reg_mem );
11148 %}
11149 
11150 // Convert a byte to a float; no rounding step needed.
11151 instruct conv24I2FPR_reg(regFPR dst, stackSlotI src) %{
11152   predicate( UseSSE==0 && n->in(1)->Opcode() == Op_AndI && n->in(1)->in(2)->is_Con() && n->in(1)->in(2)->get_int() == 255 );
11153   match(Set dst (ConvI2F src));
11154   format %{ "FILD   $src\n\t"
11155             "FSTP   $dst" %}
11156 
11157   opcode(0xDB, 0x0);  /* DB /0 */
11158   ins_encode(Push_Mem_I(src), Pop_Reg_FPR(dst));
11159   ins_pipe( fpu_reg_mem );
11160 %}
11161 
11162 // In 24-bit mode, force exponent rounding by storing back out
11163 instruct convI2FPR_SSF(stackSlotF dst, stackSlotI src) %{
11164   predicate( UseSSE==0 && Compile::current()->select_24_bit_instr());
11165   match(Set dst (ConvI2F src));
11166   ins_cost(200);
11167   format %{ "FILD   $src\n\t"
11168             "FSTP_S $dst" %}
11169   opcode(0xDB, 0x0);  /* DB /0 */
11170   ins_encode( Push_Mem_I(src),
11171               Pop_Mem_FPR(dst));
11172   ins_pipe( fpu_mem_mem );
11173 %}
11174 
11175 // In 24-bit mode, force exponent rounding by storing back out
11176 instruct convI2FPR_SSF_mem(stackSlotF dst, memory mem) %{
11177   predicate( UseSSE==0 && Compile::current()->select_24_bit_instr());
11178   match(Set dst (ConvI2F (LoadI mem)));
11179   ins_cost(200);
11180   format %{ "FILD   $mem\n\t"
11181             "FSTP_S $dst" %}
11182   opcode(0xDB);  /* DB /0 */
11183   ins_encode( OpcP, RMopc_Mem(0x00,mem),
11184               Pop_Mem_FPR(dst));
11185   ins_pipe( fpu_mem_mem );
11186 %}
11187 
11188 // This instruction does not round to 24-bits
11189 instruct convI2FPR_reg(regFPR dst, stackSlotI src) %{
11190   predicate( UseSSE==0 && !Compile::current()->select_24_bit_instr());
11191   match(Set dst (ConvI2F src));
11192   format %{ "FILD   $src\n\t"
11193             "FSTP   $dst" %}
11194   opcode(0xDB, 0x0);  /* DB /0 */
11195   ins_encode( Push_Mem_I(src),
11196               Pop_Reg_FPR(dst));
11197   ins_pipe( fpu_reg_mem );
11198 %}
11199 
11200 // This instruction does not round to 24-bits
11201 instruct convI2FPR_mem(regFPR dst, memory mem) %{
11202   predicate( UseSSE==0 && !Compile::current()->select_24_bit_instr());
11203   match(Set dst (ConvI2F (LoadI mem)));
11204   format %{ "FILD   $mem\n\t"
11205             "FSTP   $dst" %}
11206   opcode(0xDB);      /* DB /0 */
11207   ins_encode( OpcP, RMopc_Mem(0x00,mem),
11208               Pop_Reg_FPR(dst));
11209   ins_pipe( fpu_reg_mem );
11210 %}
11211 
11212 // Convert an int to a float in xmm; no rounding step needed.
11213 instruct convI2F_reg(regF dst, rRegI src) %{
11214   predicate( UseSSE==1 || UseSSE>=2 && !UseXmmI2F );
11215   match(Set dst (ConvI2F src));
11216   format %{ "CVTSI2SS $dst, $src" %}
11217   ins_encode %{
11218     __ cvtsi2ssl ($dst$$XMMRegister, $src$$Register);
11219   %}
11220   ins_pipe( pipe_slow );
11221 %}
11222 
11223  instruct convXI2F_reg(regF dst, rRegI src)
11224 %{
11225   predicate( UseSSE>=2 && UseXmmI2F );
11226   match(Set dst (ConvI2F src));
11227 
11228   format %{ "MOVD  $dst,$src\n\t"
11229             "CVTDQ2PS $dst,$dst\t# i2f" %}
11230   ins_encode %{
11231     __ movdl($dst$$XMMRegister, $src$$Register);
11232     __ cvtdq2ps($dst$$XMMRegister, $dst$$XMMRegister);
11233   %}
11234   ins_pipe(pipe_slow); // XXX
11235 %}
11236 
11237 instruct convI2L_reg( eRegL dst, rRegI src, eFlagsReg cr) %{
11238   match(Set dst (ConvI2L src));
11239   effect(KILL cr);
11240   ins_cost(375);
11241   format %{ "MOV    $dst.lo,$src\n\t"
11242             "MOV    $dst.hi,$src\n\t"
11243             "SAR    $dst.hi,31" %}
11244   ins_encode(convert_int_long(dst,src));
11245   ins_pipe( ialu_reg_reg_long );
11246 %}
11247 
11248 // Zero-extend convert int to long
11249 instruct convI2L_reg_zex(eRegL dst, rRegI src, immL_32bits mask, eFlagsReg flags ) %{
11250   match(Set dst (AndL (ConvI2L src) mask) );
11251   effect( KILL flags );
11252   ins_cost(250);
11253   format %{ "MOV    $dst.lo,$src\n\t"
11254             "XOR    $dst.hi,$dst.hi" %}
11255   opcode(0x33); // XOR
11256   ins_encode(enc_Copy(dst,src), OpcP, RegReg_Hi2(dst,dst) );
11257   ins_pipe( ialu_reg_reg_long );
11258 %}
11259 
11260 // Zero-extend long
11261 instruct zerox_long(eRegL dst, eRegL src, immL_32bits mask, eFlagsReg flags ) %{
11262   match(Set dst (AndL src mask) );
11263   effect( KILL flags );
11264   ins_cost(250);
11265   format %{ "MOV    $dst.lo,$src.lo\n\t"
11266             "XOR    $dst.hi,$dst.hi\n\t" %}
11267   opcode(0x33); // XOR
11268   ins_encode(enc_Copy(dst,src), OpcP, RegReg_Hi2(dst,dst) );
11269   ins_pipe( ialu_reg_reg_long );
11270 %}
11271 
11272 instruct convL2DPR_reg( stackSlotD dst, eRegL src, eFlagsReg cr) %{
11273   predicate (UseSSE<=1);
11274   match(Set dst (ConvL2D src));
11275   effect( KILL cr );
11276   format %{ "PUSH   $src.hi\t# Convert long to double\n\t"
11277             "PUSH   $src.lo\n\t"
11278             "FILD   ST,[ESP + #0]\n\t"
11279             "ADD    ESP,8\n\t"
11280             "FSTP_D $dst\t# D-round" %}
11281   opcode(0xDF, 0x5);  /* DF /5 */
11282   ins_encode(convert_long_double(src), Pop_Mem_DPR(dst));
11283   ins_pipe( pipe_slow );
11284 %}
11285 
11286 instruct convL2D_reg( regD dst, eRegL src, eFlagsReg cr) %{
11287   predicate (UseSSE>=2);
11288   match(Set dst (ConvL2D src));
11289   effect( KILL cr );
11290   format %{ "PUSH   $src.hi\t# Convert long to double\n\t"
11291             "PUSH   $src.lo\n\t"
11292             "FILD_D [ESP]\n\t"
11293             "FSTP_D [ESP]\n\t"
11294             "MOVSD  $dst,[ESP]\n\t"
11295             "ADD    ESP,8" %}
11296   opcode(0xDF, 0x5);  /* DF /5 */
11297   ins_encode(convert_long_double2(src), Push_ResultD(dst));
11298   ins_pipe( pipe_slow );
11299 %}
11300 
11301 instruct convL2F_reg( regF dst, eRegL src, eFlagsReg cr) %{
11302   predicate (UseSSE>=1);
11303   match(Set dst (ConvL2F src));
11304   effect( KILL cr );
11305   format %{ "PUSH   $src.hi\t# Convert long to single float\n\t"
11306             "PUSH   $src.lo\n\t"
11307             "FILD_D [ESP]\n\t"
11308             "FSTP_S [ESP]\n\t"
11309             "MOVSS  $dst,[ESP]\n\t"
11310             "ADD    ESP,8" %}
11311   opcode(0xDF, 0x5);  /* DF /5 */
11312   ins_encode(convert_long_double2(src), Push_ResultF(dst,0x8));
11313   ins_pipe( pipe_slow );
11314 %}
11315 
11316 instruct convL2FPR_reg( stackSlotF dst, eRegL src, eFlagsReg cr) %{
11317   match(Set dst (ConvL2F src));
11318   effect( KILL cr );
11319   format %{ "PUSH   $src.hi\t# Convert long to single float\n\t"
11320             "PUSH   $src.lo\n\t"
11321             "FILD   ST,[ESP + #0]\n\t"
11322             "ADD    ESP,8\n\t"
11323             "FSTP_S $dst\t# F-round" %}
11324   opcode(0xDF, 0x5);  /* DF /5 */
11325   ins_encode(convert_long_double(src), Pop_Mem_FPR(dst));
11326   ins_pipe( pipe_slow );
11327 %}
11328 
11329 instruct convL2I_reg( rRegI dst, eRegL src ) %{
11330   match(Set dst (ConvL2I src));
11331   effect( DEF dst, USE src );
11332   format %{ "MOV    $dst,$src.lo" %}
11333   ins_encode(enc_CopyL_Lo(dst,src));
11334   ins_pipe( ialu_reg_reg );
11335 %}
11336 
11337 
11338 instruct MoveF2I_stack_reg(rRegI dst, stackSlotF src) %{
11339   match(Set dst (MoveF2I src));
11340   effect( DEF dst, USE src );
11341   ins_cost(100);
11342   format %{ "MOV    $dst,$src\t# MoveF2I_stack_reg" %}
11343   ins_encode %{
11344     __ movl($dst$$Register, Address(rsp, $src$$disp));
11345   %}
11346   ins_pipe( ialu_reg_mem );
11347 %}
11348 
11349 instruct MoveFPR2I_reg_stack(stackSlotI dst, regFPR src) %{
11350   predicate(UseSSE==0);
11351   match(Set dst (MoveF2I src));
11352   effect( DEF dst, USE src );
11353 
11354   ins_cost(125);
11355   format %{ "FST_S  $dst,$src\t# MoveF2I_reg_stack" %}
11356   ins_encode( Pop_Mem_Reg_FPR(dst, src) );
11357   ins_pipe( fpu_mem_reg );
11358 %}
11359 
11360 instruct MoveF2I_reg_stack_sse(stackSlotI dst, regF src) %{
11361   predicate(UseSSE>=1);
11362   match(Set dst (MoveF2I src));
11363   effect( DEF dst, USE src );
11364 
11365   ins_cost(95);
11366   format %{ "MOVSS  $dst,$src\t# MoveF2I_reg_stack_sse" %}
11367   ins_encode %{
11368     __ movflt(Address(rsp, $dst$$disp), $src$$XMMRegister);
11369   %}
11370   ins_pipe( pipe_slow );
11371 %}
11372 
11373 instruct MoveF2I_reg_reg_sse(rRegI dst, regF src) %{
11374   predicate(UseSSE>=2);
11375   match(Set dst (MoveF2I src));
11376   effect( DEF dst, USE src );
11377   ins_cost(85);
11378   format %{ "MOVD   $dst,$src\t# MoveF2I_reg_reg_sse" %}
11379   ins_encode %{
11380     __ movdl($dst$$Register, $src$$XMMRegister);
11381   %}
11382   ins_pipe( pipe_slow );
11383 %}
11384 
11385 instruct MoveI2F_reg_stack(stackSlotF dst, rRegI src) %{
11386   match(Set dst (MoveI2F src));
11387   effect( DEF dst, USE src );
11388 
11389   ins_cost(100);
11390   format %{ "MOV    $dst,$src\t# MoveI2F_reg_stack" %}
11391   ins_encode %{
11392     __ movl(Address(rsp, $dst$$disp), $src$$Register);
11393   %}
11394   ins_pipe( ialu_mem_reg );
11395 %}
11396 
11397 
11398 instruct MoveI2FPR_stack_reg(regFPR dst, stackSlotI src) %{
11399   predicate(UseSSE==0);
11400   match(Set dst (MoveI2F src));
11401   effect(DEF dst, USE src);
11402 
11403   ins_cost(125);
11404   format %{ "FLD_S  $src\n\t"
11405             "FSTP   $dst\t# MoveI2F_stack_reg" %}
11406   opcode(0xD9);               /* D9 /0, FLD m32real */
11407   ins_encode( OpcP, RMopc_Mem_no_oop(0x00,src),
11408               Pop_Reg_FPR(dst) );
11409   ins_pipe( fpu_reg_mem );
11410 %}
11411 
11412 instruct MoveI2F_stack_reg_sse(regF dst, stackSlotI src) %{
11413   predicate(UseSSE>=1);
11414   match(Set dst (MoveI2F src));
11415   effect( DEF dst, USE src );
11416 
11417   ins_cost(95);
11418   format %{ "MOVSS  $dst,$src\t# MoveI2F_stack_reg_sse" %}
11419   ins_encode %{
11420     __ movflt($dst$$XMMRegister, Address(rsp, $src$$disp));
11421   %}
11422   ins_pipe( pipe_slow );
11423 %}
11424 
11425 instruct MoveI2F_reg_reg_sse(regF dst, rRegI src) %{
11426   predicate(UseSSE>=2);
11427   match(Set dst (MoveI2F src));
11428   effect( DEF dst, USE src );
11429 
11430   ins_cost(85);
11431   format %{ "MOVD   $dst,$src\t# MoveI2F_reg_reg_sse" %}
11432   ins_encode %{
11433     __ movdl($dst$$XMMRegister, $src$$Register);
11434   %}
11435   ins_pipe( pipe_slow );
11436 %}
11437 
11438 instruct MoveD2L_stack_reg(eRegL dst, stackSlotD src) %{
11439   match(Set dst (MoveD2L src));
11440   effect(DEF dst, USE src);
11441 
11442   ins_cost(250);
11443   format %{ "MOV    $dst.lo,$src\n\t"
11444             "MOV    $dst.hi,$src+4\t# MoveD2L_stack_reg" %}
11445   opcode(0x8B, 0x8B);
11446   ins_encode( OpcP, RegMem(dst,src), OpcS, RegMem_Hi(dst,src));
11447   ins_pipe( ialu_mem_long_reg );
11448 %}
11449 
11450 instruct MoveDPR2L_reg_stack(stackSlotL dst, regDPR src) %{
11451   predicate(UseSSE<=1);
11452   match(Set dst (MoveD2L src));
11453   effect(DEF dst, USE src);
11454 
11455   ins_cost(125);
11456   format %{ "FST_D  $dst,$src\t# MoveD2L_reg_stack" %}
11457   ins_encode( Pop_Mem_Reg_DPR(dst, src) );
11458   ins_pipe( fpu_mem_reg );
11459 %}
11460 
11461 instruct MoveD2L_reg_stack_sse(stackSlotL dst, regD src) %{
11462   predicate(UseSSE>=2);
11463   match(Set dst (MoveD2L src));
11464   effect(DEF dst, USE src);
11465   ins_cost(95);
11466   format %{ "MOVSD  $dst,$src\t# MoveD2L_reg_stack_sse" %}
11467   ins_encode %{
11468     __ movdbl(Address(rsp, $dst$$disp), $src$$XMMRegister);
11469   %}
11470   ins_pipe( pipe_slow );
11471 %}
11472 
11473 instruct MoveD2L_reg_reg_sse(eRegL dst, regD src, regD tmp) %{
11474   predicate(UseSSE>=2);
11475   match(Set dst (MoveD2L src));
11476   effect(DEF dst, USE src, TEMP tmp);
11477   ins_cost(85);
11478   format %{ "MOVD   $dst.lo,$src\n\t"
11479             "PSHUFLW $tmp,$src,0x4E\n\t"
11480             "MOVD   $dst.hi,$tmp\t# MoveD2L_reg_reg_sse" %}
11481   ins_encode %{
11482     __ movdl($dst$$Register, $src$$XMMRegister);
11483     __ pshuflw($tmp$$XMMRegister, $src$$XMMRegister, 0x4e);
11484     __ movdl(HIGH_FROM_LOW($dst$$Register), $tmp$$XMMRegister);
11485   %}
11486   ins_pipe( pipe_slow );
11487 %}
11488 
11489 instruct MoveL2D_reg_stack(stackSlotD dst, eRegL src) %{
11490   match(Set dst (MoveL2D src));
11491   effect(DEF dst, USE src);
11492 
11493   ins_cost(200);
11494   format %{ "MOV    $dst,$src.lo\n\t"
11495             "MOV    $dst+4,$src.hi\t# MoveL2D_reg_stack" %}
11496   opcode(0x89, 0x89);
11497   ins_encode( OpcP, RegMem( src, dst ), OpcS, RegMem_Hi( src, dst ) );
11498   ins_pipe( ialu_mem_long_reg );
11499 %}
11500 
11501 
11502 instruct MoveL2DPR_stack_reg(regDPR dst, stackSlotL src) %{
11503   predicate(UseSSE<=1);
11504   match(Set dst (MoveL2D src));
11505   effect(DEF dst, USE src);
11506   ins_cost(125);
11507 
11508   format %{ "FLD_D  $src\n\t"
11509             "FSTP   $dst\t# MoveL2D_stack_reg" %}
11510   opcode(0xDD);               /* DD /0, FLD m64real */
11511   ins_encode( OpcP, RMopc_Mem_no_oop(0x00,src),
11512               Pop_Reg_DPR(dst) );
11513   ins_pipe( fpu_reg_mem );
11514 %}
11515 
11516 
11517 instruct MoveL2D_stack_reg_sse(regD dst, stackSlotL src) %{
11518   predicate(UseSSE>=2 && UseXmmLoadAndClearUpper);
11519   match(Set dst (MoveL2D src));
11520   effect(DEF dst, USE src);
11521 
11522   ins_cost(95);
11523   format %{ "MOVSD  $dst,$src\t# MoveL2D_stack_reg_sse" %}
11524   ins_encode %{
11525     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
11526   %}
11527   ins_pipe( pipe_slow );
11528 %}
11529 
11530 instruct MoveL2D_stack_reg_sse_partial(regD dst, stackSlotL src) %{
11531   predicate(UseSSE>=2 && !UseXmmLoadAndClearUpper);
11532   match(Set dst (MoveL2D src));
11533   effect(DEF dst, USE src);
11534 
11535   ins_cost(95);
11536   format %{ "MOVLPD $dst,$src\t# MoveL2D_stack_reg_sse" %}
11537   ins_encode %{
11538     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
11539   %}
11540   ins_pipe( pipe_slow );
11541 %}
11542 
11543 instruct MoveL2D_reg_reg_sse(regD dst, eRegL src, regD tmp) %{
11544   predicate(UseSSE>=2);
11545   match(Set dst (MoveL2D src));
11546   effect(TEMP dst, USE src, TEMP tmp);
11547   ins_cost(85);
11548   format %{ "MOVD   $dst,$src.lo\n\t"
11549             "MOVD   $tmp,$src.hi\n\t"
11550             "PUNPCKLDQ $dst,$tmp\t# MoveL2D_reg_reg_sse" %}
11551   ins_encode %{
11552     __ movdl($dst$$XMMRegister, $src$$Register);
11553     __ movdl($tmp$$XMMRegister, HIGH_FROM_LOW($src$$Register));
11554     __ punpckldq($dst$$XMMRegister, $tmp$$XMMRegister);
11555   %}
11556   ins_pipe( pipe_slow );
11557 %}
11558 
11559 
11560 // =======================================================================
11561 // fast clearing of an array
11562 instruct rep_stos(eCXRegI cnt, eDIRegP base, eAXRegI zero, Universe dummy, eFlagsReg cr) %{
11563   predicate(!UseFastStosb);
11564   match(Set dummy (ClearArray cnt base));
11565   effect(USE_KILL cnt, USE_KILL base, KILL zero, KILL cr);
11566   format %{ "XOR    EAX,EAX\t# ClearArray:\n\t"
11567             "SHL    ECX,1\t# Convert doublewords to words\n\t"
11568             "REP STOS\t# store EAX into [EDI++] while ECX--" %}
11569   ins_encode %{ 
11570     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register);
11571   %}
11572   ins_pipe( pipe_slow );
11573 %}
11574 
11575 instruct rep_fast_stosb(eCXRegI cnt, eDIRegP base, eAXRegI zero, Universe dummy, eFlagsReg cr) %{
11576   predicate(UseFastStosb);
11577   match(Set dummy (ClearArray cnt base));
11578   effect(USE_KILL cnt, USE_KILL base, KILL zero, KILL cr);
11579   format %{ "XOR    EAX,EAX\t# ClearArray:\n\t"
11580             "SHL    ECX,3\t# Convert doublewords to bytes\n\t"
11581             "REP STOSB\t# store EAX into [EDI++] while ECX--" %}
11582   ins_encode %{ 
11583     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register);
11584   %}
11585   ins_pipe( pipe_slow );
11586 %}
11587 
11588 instruct string_compare(eDIRegP str1, eCXRegI cnt1, eSIRegP str2, eDXRegI cnt2,
11589                         eAXRegI result, regD tmp1, eFlagsReg cr) %{
11590   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11591   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11592 
11593   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11594   ins_encode %{
11595     __ string_compare($str1$$Register, $str2$$Register,
11596                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
11597                       $tmp1$$XMMRegister);
11598   %}
11599   ins_pipe( pipe_slow );
11600 %}
11601 
11602 // fast string equals
11603 instruct string_equals(eDIRegP str1, eSIRegP str2, eCXRegI cnt, eAXRegI result,
11604                        regD tmp1, regD tmp2, eBXRegI tmp3, eFlagsReg cr) %{
11605   match(Set result (StrEquals (Binary str1 str2) cnt));
11606   effect(TEMP tmp1, TEMP tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL tmp3, KILL cr);
11607 
11608   format %{ "String Equals $str1,$str2,$cnt -> $result    // KILL $tmp1, $tmp2, $tmp3" %}
11609   ins_encode %{
11610     __ char_arrays_equals(false, $str1$$Register, $str2$$Register,
11611                           $cnt$$Register, $result$$Register, $tmp3$$Register,
11612                           $tmp1$$XMMRegister, $tmp2$$XMMRegister);
11613   %}
11614   ins_pipe( pipe_slow );
11615 %}
11616 
11617 // fast search of substring with known size.
11618 instruct string_indexof_con(eDIRegP str1, eDXRegI cnt1, eSIRegP str2, immI int_cnt2,
11619                             eBXRegI result, regD vec, eAXRegI cnt2, eCXRegI tmp, eFlagsReg cr) %{
11620   predicate(UseSSE42Intrinsics);
11621   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
11622   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
11623 
11624   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $vec, $cnt1, $cnt2, $tmp" %}
11625   ins_encode %{
11626     int icnt2 = (int)$int_cnt2$$constant;
11627     if (icnt2 >= 8) {
11628       // IndexOf for constant substrings with size >= 8 elements
11629       // which don't need to be loaded through stack.
11630       __ string_indexofC8($str1$$Register, $str2$$Register,
11631                           $cnt1$$Register, $cnt2$$Register,
11632                           icnt2, $result$$Register,
11633                           $vec$$XMMRegister, $tmp$$Register);
11634     } else {
11635       // Small strings are loaded through stack if they cross page boundary.
11636       __ string_indexof($str1$$Register, $str2$$Register,
11637                         $cnt1$$Register, $cnt2$$Register,
11638                         icnt2, $result$$Register,
11639                         $vec$$XMMRegister, $tmp$$Register);
11640     }
11641   %}
11642   ins_pipe( pipe_slow );
11643 %}
11644 
11645 instruct string_indexof(eDIRegP str1, eDXRegI cnt1, eSIRegP str2, eAXRegI cnt2,
11646                         eBXRegI result, regD vec, eCXRegI tmp, eFlagsReg cr) %{
11647   predicate(UseSSE42Intrinsics);
11648   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
11649   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
11650 
11651   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
11652   ins_encode %{
11653     __ string_indexof($str1$$Register, $str2$$Register,
11654                       $cnt1$$Register, $cnt2$$Register,
11655                       (-1), $result$$Register,
11656                       $vec$$XMMRegister, $tmp$$Register);
11657   %}
11658   ins_pipe( pipe_slow );
11659 %}
11660 
11661 // fast array equals
11662 instruct array_equals(eDIRegP ary1, eSIRegP ary2, eAXRegI result,
11663                       regD tmp1, regD tmp2, eCXRegI tmp3, eBXRegI tmp4, eFlagsReg cr)
11664 %{
11665   match(Set result (AryEq ary1 ary2));
11666   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
11667   //ins_cost(300);
11668 
11669   format %{ "Array Equals $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
11670   ins_encode %{
11671     __ char_arrays_equals(true, $ary1$$Register, $ary2$$Register,
11672                           $tmp3$$Register, $result$$Register, $tmp4$$Register,
11673                           $tmp1$$XMMRegister, $tmp2$$XMMRegister);
11674   %}
11675   ins_pipe( pipe_slow );
11676 %}
11677 
11678 // encode char[] to byte[] in ISO_8859_1
11679 instruct encode_iso_array(eSIRegP src, eDIRegP dst, eDXRegI len,
11680                           regD tmp1, regD tmp2, regD tmp3, regD tmp4,
11681                           eCXRegI tmp5, eAXRegI result, eFlagsReg cr) %{
11682   match(Set result (EncodeISOArray src (Binary dst len)));
11683   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);
11684 
11685   format %{ "Encode array $src,$dst,$len -> $result    // KILL ECX, EDX, $tmp1, $tmp2, $tmp3, $tmp4, ESI, EDI " %}
11686   ins_encode %{
11687     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
11688                         $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
11689                         $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register);
11690   %}
11691   ins_pipe( pipe_slow );
11692 %}
11693 
11694 
11695 //----------Control Flow Instructions------------------------------------------
11696 // Signed compare Instructions
11697 instruct compI_eReg(eFlagsReg cr, rRegI op1, rRegI op2) %{
11698   match(Set cr (CmpI op1 op2));
11699   effect( DEF cr, USE op1, USE op2 );
11700   format %{ "CMP    $op1,$op2" %}
11701   opcode(0x3B);  /* Opcode 3B /r */
11702   ins_encode( OpcP, RegReg( op1, op2) );
11703   ins_pipe( ialu_cr_reg_reg );
11704 %}
11705 
11706 instruct compI_eReg_imm(eFlagsReg cr, rRegI op1, immI op2) %{
11707   match(Set cr (CmpI op1 op2));
11708   effect( DEF cr, USE op1 );
11709   format %{ "CMP    $op1,$op2" %}
11710   opcode(0x81,0x07);  /* Opcode 81 /7 */
11711   // ins_encode( RegImm( op1, op2) );  /* Was CmpImm */
11712   ins_encode( OpcSErm( op1, op2 ), Con8or32( op2 ) );
11713   ins_pipe( ialu_cr_reg_imm );
11714 %}
11715 
11716 // Cisc-spilled version of cmpI_eReg
11717 instruct compI_eReg_mem(eFlagsReg cr, rRegI op1, memory op2) %{
11718   match(Set cr (CmpI op1 (LoadI op2)));
11719 
11720   format %{ "CMP    $op1,$op2" %}
11721   ins_cost(500);
11722   opcode(0x3B);  /* Opcode 3B /r */
11723   ins_encode( OpcP, RegMem( op1, op2) );
11724   ins_pipe( ialu_cr_reg_mem );
11725 %}
11726 
11727 instruct testI_reg( eFlagsReg cr, rRegI src, immI0 zero ) %{
11728   match(Set cr (CmpI src zero));
11729   effect( DEF cr, USE src );
11730 
11731   format %{ "TEST   $src,$src" %}
11732   opcode(0x85);
11733   ins_encode( OpcP, RegReg( src, src ) );
11734   ins_pipe( ialu_cr_reg_imm );
11735 %}
11736 
11737 instruct testI_reg_imm( eFlagsReg cr, rRegI src, immI con, immI0 zero ) %{
11738   match(Set cr (CmpI (AndI src con) zero));
11739 
11740   format %{ "TEST   $src,$con" %}
11741   opcode(0xF7,0x00);
11742   ins_encode( OpcP, RegOpc(src), Con32(con) );
11743   ins_pipe( ialu_cr_reg_imm );
11744 %}
11745 
11746 instruct testI_reg_mem( eFlagsReg cr, rRegI src, memory mem, immI0 zero ) %{
11747   match(Set cr (CmpI (AndI src mem) zero));
11748 
11749   format %{ "TEST   $src,$mem" %}
11750   opcode(0x85);
11751   ins_encode( OpcP, RegMem( src, mem ) );
11752   ins_pipe( ialu_cr_reg_mem );
11753 %}
11754 
11755 // Unsigned compare Instructions; really, same as signed except they
11756 // produce an eFlagsRegU instead of eFlagsReg.
11757 instruct compU_eReg(eFlagsRegU cr, rRegI op1, rRegI op2) %{
11758   match(Set cr (CmpU op1 op2));
11759 
11760   format %{ "CMPu   $op1,$op2" %}
11761   opcode(0x3B);  /* Opcode 3B /r */
11762   ins_encode( OpcP, RegReg( op1, op2) );
11763   ins_pipe( ialu_cr_reg_reg );
11764 %}
11765 
11766 instruct compU_eReg_imm(eFlagsRegU cr, rRegI op1, immI op2) %{
11767   match(Set cr (CmpU op1 op2));
11768 
11769   format %{ "CMPu   $op1,$op2" %}
11770   opcode(0x81,0x07);  /* Opcode 81 /7 */
11771   ins_encode( OpcSErm( op1, op2 ), Con8or32( op2 ) );
11772   ins_pipe( ialu_cr_reg_imm );
11773 %}
11774 
11775 // // Cisc-spilled version of cmpU_eReg
11776 instruct compU_eReg_mem(eFlagsRegU cr, rRegI op1, memory op2) %{
11777   match(Set cr (CmpU op1 (LoadI op2)));
11778 
11779   format %{ "CMPu   $op1,$op2" %}
11780   ins_cost(500);
11781   opcode(0x3B);  /* Opcode 3B /r */
11782   ins_encode( OpcP, RegMem( op1, op2) );
11783   ins_pipe( ialu_cr_reg_mem );
11784 %}
11785 
11786 // // Cisc-spilled version of cmpU_eReg
11787 //instruct compU_mem_eReg(eFlagsRegU cr, memory op1, rRegI op2) %{
11788 //  match(Set cr (CmpU (LoadI op1) op2));
11789 //
11790 //  format %{ "CMPu   $op1,$op2" %}
11791 //  ins_cost(500);
11792 //  opcode(0x39);  /* Opcode 39 /r */
11793 //  ins_encode( OpcP, RegMem( op1, op2) );
11794 //%}
11795 
11796 instruct testU_reg( eFlagsRegU cr, rRegI src, immI0 zero ) %{
11797   match(Set cr (CmpU src zero));
11798 
11799   format %{ "TESTu  $src,$src" %}
11800   opcode(0x85);
11801   ins_encode( OpcP, RegReg( src, src ) );
11802   ins_pipe( ialu_cr_reg_imm );
11803 %}
11804 
11805 // Unsigned pointer compare Instructions
11806 instruct compP_eReg(eFlagsRegU cr, eRegP op1, eRegP op2) %{
11807   match(Set cr (CmpP op1 op2));
11808 
11809   format %{ "CMPu   $op1,$op2" %}
11810   opcode(0x3B);  /* Opcode 3B /r */
11811   ins_encode( OpcP, RegReg( op1, op2) );
11812   ins_pipe( ialu_cr_reg_reg );
11813 %}
11814 
11815 instruct compP_eReg_imm(eFlagsRegU cr, eRegP op1, immP op2) %{
11816   match(Set cr (CmpP op1 op2));
11817 
11818   format %{ "CMPu   $op1,$op2" %}
11819   opcode(0x81,0x07);  /* Opcode 81 /7 */
11820   ins_encode( OpcSErm( op1, op2 ), Con8or32( op2 ) );
11821   ins_pipe( ialu_cr_reg_imm );
11822 %}
11823 
11824 // // Cisc-spilled version of cmpP_eReg
11825 instruct compP_eReg_mem(eFlagsRegU cr, eRegP op1, memory op2) %{
11826   match(Set cr (CmpP op1 (LoadP op2)));
11827 
11828   format %{ "CMPu   $op1,$op2" %}
11829   ins_cost(500);
11830   opcode(0x3B);  /* Opcode 3B /r */
11831   ins_encode( OpcP, RegMem( op1, op2) );
11832   ins_pipe( ialu_cr_reg_mem );
11833 %}
11834 
11835 // // Cisc-spilled version of cmpP_eReg
11836 //instruct compP_mem_eReg(eFlagsRegU cr, memory op1, eRegP op2) %{
11837 //  match(Set cr (CmpP (LoadP op1) op2));
11838 //
11839 //  format %{ "CMPu   $op1,$op2" %}
11840 //  ins_cost(500);
11841 //  opcode(0x39);  /* Opcode 39 /r */
11842 //  ins_encode( OpcP, RegMem( op1, op2) );
11843 //%}
11844 
11845 // Compare raw pointer (used in out-of-heap check).
11846 // Only works because non-oop pointers must be raw pointers
11847 // and raw pointers have no anti-dependencies.
11848 instruct compP_mem_eReg( eFlagsRegU cr, eRegP op1, memory op2 ) %{
11849   predicate( n->in(2)->in(2)->bottom_type()->reloc() == relocInfo::none );
11850   match(Set cr (CmpP op1 (LoadP op2)));
11851 
11852   format %{ "CMPu   $op1,$op2" %}
11853   opcode(0x3B);  /* Opcode 3B /r */
11854   ins_encode( OpcP, RegMem( op1, op2) );
11855   ins_pipe( ialu_cr_reg_mem );
11856 %}
11857 
11858 //
11859 // This will generate a signed flags result. This should be ok
11860 // since any compare to a zero should be eq/neq.
11861 instruct testP_reg( eFlagsReg cr, eRegP src, immP0 zero ) %{
11862   match(Set cr (CmpP src zero));
11863 
11864   format %{ "TEST   $src,$src" %}
11865   opcode(0x85);
11866   ins_encode( OpcP, RegReg( src, src ) );
11867   ins_pipe( ialu_cr_reg_imm );
11868 %}
11869 
11870 // Cisc-spilled version of testP_reg
11871 // This will generate a signed flags result. This should be ok
11872 // since any compare to a zero should be eq/neq.
11873 instruct testP_Reg_mem( eFlagsReg cr, memory op, immI0 zero ) %{
11874   match(Set cr (CmpP (LoadP op) zero));
11875 
11876   format %{ "TEST   $op,0xFFFFFFFF" %}
11877   ins_cost(500);
11878   opcode(0xF7);               /* Opcode F7 /0 */
11879   ins_encode( OpcP, RMopc_Mem(0x00,op), Con_d32(0xFFFFFFFF) );
11880   ins_pipe( ialu_cr_reg_imm );
11881 %}
11882 
11883 // Yanked all unsigned pointer compare operations.
11884 // Pointer compares are done with CmpP which is already unsigned.
11885 
11886 //----------Max and Min--------------------------------------------------------
11887 // Min Instructions
11888 ////
11889 //   *** Min and Max using the conditional move are slower than the
11890 //   *** branch version on a Pentium III.
11891 // // Conditional move for min
11892 //instruct cmovI_reg_lt( rRegI op2, rRegI op1, eFlagsReg cr ) %{
11893 //  effect( USE_DEF op2, USE op1, USE cr );
11894 //  format %{ "CMOVlt $op2,$op1\t! min" %}
11895 //  opcode(0x4C,0x0F);
11896 //  ins_encode( OpcS, OpcP, RegReg( op2, op1 ) );
11897 //  ins_pipe( pipe_cmov_reg );
11898 //%}
11899 //
11900 //// Min Register with Register (P6 version)
11901 //instruct minI_eReg_p6( rRegI op1, rRegI op2 ) %{
11902 //  predicate(VM_Version::supports_cmov() );
11903 //  match(Set op2 (MinI op1 op2));
11904 //  ins_cost(200);
11905 //  expand %{
11906 //    eFlagsReg cr;
11907 //    compI_eReg(cr,op1,op2);
11908 //    cmovI_reg_lt(op2,op1,cr);
11909 //  %}
11910 //%}
11911 
11912 // Min Register with Register (generic version)
11913 instruct minI_eReg(rRegI dst, rRegI src, eFlagsReg flags) %{
11914   match(Set dst (MinI dst src));
11915   effect(KILL flags);
11916   ins_cost(300);
11917 
11918   format %{ "MIN    $dst,$src" %}
11919   opcode(0xCC);
11920   ins_encode( min_enc(dst,src) );
11921   ins_pipe( pipe_slow );
11922 %}
11923 
11924 // Max Register with Register
11925 //   *** Min and Max using the conditional move are slower than the
11926 //   *** branch version on a Pentium III.
11927 // // Conditional move for max
11928 //instruct cmovI_reg_gt( rRegI op2, rRegI op1, eFlagsReg cr ) %{
11929 //  effect( USE_DEF op2, USE op1, USE cr );
11930 //  format %{ "CMOVgt $op2,$op1\t! max" %}
11931 //  opcode(0x4F,0x0F);
11932 //  ins_encode( OpcS, OpcP, RegReg( op2, op1 ) );
11933 //  ins_pipe( pipe_cmov_reg );
11934 //%}
11935 //
11936 // // Max Register with Register (P6 version)
11937 //instruct maxI_eReg_p6( rRegI op1, rRegI op2 ) %{
11938 //  predicate(VM_Version::supports_cmov() );
11939 //  match(Set op2 (MaxI op1 op2));
11940 //  ins_cost(200);
11941 //  expand %{
11942 //    eFlagsReg cr;
11943 //    compI_eReg(cr,op1,op2);
11944 //    cmovI_reg_gt(op2,op1,cr);
11945 //  %}
11946 //%}
11947 
11948 // Max Register with Register (generic version)
11949 instruct maxI_eReg(rRegI dst, rRegI src, eFlagsReg flags) %{
11950   match(Set dst (MaxI dst src));
11951   effect(KILL flags);
11952   ins_cost(300);
11953 
11954   format %{ "MAX    $dst,$src" %}
11955   opcode(0xCC);
11956   ins_encode( max_enc(dst,src) );
11957   ins_pipe( pipe_slow );
11958 %}
11959 
11960 // ============================================================================
11961 // Counted Loop limit node which represents exact final iterator value.
11962 // Note: the resulting value should fit into integer range since
11963 // counted loops have limit check on overflow.
11964 instruct loopLimit_eReg(eAXRegI limit, nadxRegI init, immI stride, eDXRegI limit_hi, nadxRegI tmp, eFlagsReg flags) %{
11965   match(Set limit (LoopLimit (Binary init limit) stride));
11966   effect(TEMP limit_hi, TEMP tmp, KILL flags);
11967   ins_cost(300);
11968 
11969   format %{ "loopLimit $init,$limit,$stride  # $limit = $init + $stride *( $limit - $init + $stride -1)/ $stride, kills $limit_hi" %}
11970   ins_encode %{
11971     int strd = (int)$stride$$constant;
11972     assert(strd != 1 && strd != -1, "sanity");
11973     int m1 = (strd > 0) ? 1 : -1;
11974     // Convert limit to long (EAX:EDX)
11975     __ cdql();
11976     // Convert init to long (init:tmp)
11977     __ movl($tmp$$Register, $init$$Register);
11978     __ sarl($tmp$$Register, 31);
11979     // $limit - $init
11980     __ subl($limit$$Register, $init$$Register);
11981     __ sbbl($limit_hi$$Register, $tmp$$Register);
11982     // + ($stride - 1)
11983     if (strd > 0) {
11984       __ addl($limit$$Register, (strd - 1));
11985       __ adcl($limit_hi$$Register, 0);
11986       __ movl($tmp$$Register, strd);
11987     } else {
11988       __ addl($limit$$Register, (strd + 1));
11989       __ adcl($limit_hi$$Register, -1);
11990       __ lneg($limit_hi$$Register, $limit$$Register);
11991       __ movl($tmp$$Register, -strd);
11992     }
11993     // signed devision: (EAX:EDX) / pos_stride
11994     __ idivl($tmp$$Register);
11995     if (strd < 0) {
11996       // restore sign
11997       __ negl($tmp$$Register);
11998     }
11999     // (EAX) * stride
12000     __ mull($tmp$$Register);
12001     // + init (ignore upper bits)
12002     __ addl($limit$$Register, $init$$Register);
12003   %}
12004   ins_pipe( pipe_slow );
12005 %}
12006 
12007 // ============================================================================
12008 // Branch Instructions
12009 // Jump Table
12010 instruct jumpXtnd(rRegI switch_val) %{
12011   match(Jump switch_val);
12012   ins_cost(350);
12013   format %{  "JMP    [$constantaddress](,$switch_val,1)\n\t" %}
12014   ins_encode %{
12015     // Jump to Address(table_base + switch_reg)
12016     Address index(noreg, $switch_val$$Register, Address::times_1);
12017     __ jump(ArrayAddress($constantaddress, index));
12018   %}
12019   ins_pipe(pipe_jmp);
12020 %}
12021 
12022 // Jump Direct - Label defines a relative address from JMP+1
12023 instruct jmpDir(label labl) %{
12024   match(Goto);
12025   effect(USE labl);
12026 
12027   ins_cost(300);
12028   format %{ "JMP    $labl" %}
12029   size(5);
12030   ins_encode %{
12031     Label* L = $labl$$label;
12032     __ jmp(*L, false); // Always long jump
12033   %}
12034   ins_pipe( pipe_jmp );
12035 %}
12036 
12037 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12038 instruct jmpCon(cmpOp cop, eFlagsReg cr, label labl) %{
12039   match(If cop cr);
12040   effect(USE labl);
12041 
12042   ins_cost(300);
12043   format %{ "J$cop    $labl" %}
12044   size(6);
12045   ins_encode %{
12046     Label* L = $labl$$label;
12047     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12048   %}
12049   ins_pipe( pipe_jcc );
12050 %}
12051 
12052 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12053 instruct jmpLoopEnd(cmpOp cop, eFlagsReg cr, label labl) %{
12054   match(CountedLoopEnd cop cr);
12055   effect(USE labl);
12056 
12057   ins_cost(300);
12058   format %{ "J$cop    $labl\t# Loop end" %}
12059   size(6);
12060   ins_encode %{
12061     Label* L = $labl$$label;
12062     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12063   %}
12064   ins_pipe( pipe_jcc );
12065 %}
12066 
12067 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12068 instruct jmpLoopEndU(cmpOpU cop, eFlagsRegU cmp, label labl) %{
12069   match(CountedLoopEnd cop cmp);
12070   effect(USE labl);
12071 
12072   ins_cost(300);
12073   format %{ "J$cop,u  $labl\t# Loop end" %}
12074   size(6);
12075   ins_encode %{
12076     Label* L = $labl$$label;
12077     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12078   %}
12079   ins_pipe( pipe_jcc );
12080 %}
12081 
12082 instruct jmpLoopEndUCF(cmpOpUCF cop, eFlagsRegUCF cmp, label labl) %{
12083   match(CountedLoopEnd cop cmp);
12084   effect(USE labl);
12085 
12086   ins_cost(200);
12087   format %{ "J$cop,u  $labl\t# Loop end" %}
12088   size(6);
12089   ins_encode %{
12090     Label* L = $labl$$label;
12091     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12092   %}
12093   ins_pipe( pipe_jcc );
12094 %}
12095 
12096 // Jump Direct Conditional - using unsigned comparison
12097 instruct jmpConU(cmpOpU cop, eFlagsRegU cmp, label labl) %{
12098   match(If cop cmp);
12099   effect(USE labl);
12100 
12101   ins_cost(300);
12102   format %{ "J$cop,u  $labl" %}
12103   size(6);
12104   ins_encode %{
12105     Label* L = $labl$$label;
12106     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12107   %}
12108   ins_pipe(pipe_jcc);
12109 %}
12110 
12111 instruct jmpConUCF(cmpOpUCF cop, eFlagsRegUCF cmp, label labl) %{
12112   match(If cop cmp);
12113   effect(USE labl);
12114 
12115   ins_cost(200);
12116   format %{ "J$cop,u  $labl" %}
12117   size(6);
12118   ins_encode %{
12119     Label* L = $labl$$label;
12120     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12121   %}
12122   ins_pipe(pipe_jcc);
12123 %}
12124 
12125 instruct jmpConUCF2(cmpOpUCF2 cop, eFlagsRegUCF cmp, label labl) %{
12126   match(If cop cmp);
12127   effect(USE labl);
12128 
12129   ins_cost(200);
12130   format %{ $$template
12131     if ($cop$$cmpcode == Assembler::notEqual) {
12132       $$emit$$"JP,u   $labl\n\t"
12133       $$emit$$"J$cop,u   $labl"
12134     } else {
12135       $$emit$$"JP,u   done\n\t"
12136       $$emit$$"J$cop,u   $labl\n\t"
12137       $$emit$$"done:"
12138     }
12139   %}
12140   ins_encode %{
12141     Label* l = $labl$$label;
12142     if ($cop$$cmpcode == Assembler::notEqual) {
12143       __ jcc(Assembler::parity, *l, false);
12144       __ jcc(Assembler::notEqual, *l, false);
12145     } else if ($cop$$cmpcode == Assembler::equal) {
12146       Label done;
12147       __ jccb(Assembler::parity, done);
12148       __ jcc(Assembler::equal, *l, false);
12149       __ bind(done);
12150     } else {
12151        ShouldNotReachHere();
12152     }
12153   %}
12154   ins_pipe(pipe_jcc);
12155 %}
12156 
12157 // ============================================================================
12158 // The 2nd slow-half of a subtype check.  Scan the subklass's 2ndary superklass
12159 // array for an instance of the superklass.  Set a hidden internal cache on a
12160 // hit (cache is checked with exposed code in gen_subtype_check()).  Return
12161 // NZ for a miss or zero for a hit.  The encoding ALSO sets flags.
12162 instruct partialSubtypeCheck( eDIRegP result, eSIRegP sub, eAXRegP super, eCXRegI rcx, eFlagsReg cr ) %{
12163   match(Set result (PartialSubtypeCheck sub super));
12164   effect( KILL rcx, KILL cr );
12165 
12166   ins_cost(1100);  // slightly larger than the next version
12167   format %{ "MOV    EDI,[$sub+Klass::secondary_supers]\n\t"
12168             "MOV    ECX,[EDI+ArrayKlass::length]\t# length to scan\n\t"
12169             "ADD    EDI,ArrayKlass::base_offset\t# Skip to start of data; set NZ in case count is zero\n\t"
12170             "REPNE SCASD\t# Scan *EDI++ for a match with EAX while CX-- != 0\n\t"
12171             "JNE,s  miss\t\t# Missed: EDI not-zero\n\t"
12172             "MOV    [$sub+Klass::secondary_super_cache],$super\t# Hit: update cache\n\t"
12173             "XOR    $result,$result\t\t Hit: EDI zero\n\t"
12174      "miss:\t" %}
12175 
12176   opcode(0x1); // Force a XOR of EDI
12177   ins_encode( enc_PartialSubtypeCheck() );
12178   ins_pipe( pipe_slow );
12179 %}
12180 
12181 instruct partialSubtypeCheck_vs_Zero( eFlagsReg cr, eSIRegP sub, eAXRegP super, eCXRegI rcx, eDIRegP result, immP0 zero ) %{
12182   match(Set cr (CmpP (PartialSubtypeCheck sub super) zero));
12183   effect( KILL rcx, KILL result );
12184 
12185   ins_cost(1000);
12186   format %{ "MOV    EDI,[$sub+Klass::secondary_supers]\n\t"
12187             "MOV    ECX,[EDI+ArrayKlass::length]\t# length to scan\n\t"
12188             "ADD    EDI,ArrayKlass::base_offset\t# Skip to start of data; set NZ in case count is zero\n\t"
12189             "REPNE SCASD\t# Scan *EDI++ for a match with EAX while CX-- != 0\n\t"
12190             "JNE,s  miss\t\t# Missed: flags NZ\n\t"
12191             "MOV    [$sub+Klass::secondary_super_cache],$super\t# Hit: update cache, flags Z\n\t"
12192      "miss:\t" %}
12193 
12194   opcode(0x0);  // No need to XOR EDI
12195   ins_encode( enc_PartialSubtypeCheck() );
12196   ins_pipe( pipe_slow );
12197 %}
12198 
12199 // ============================================================================
12200 // Branch Instructions -- short offset versions
12201 //
12202 // These instructions are used to replace jumps of a long offset (the default
12203 // match) with jumps of a shorter offset.  These instructions are all tagged
12204 // with the ins_short_branch attribute, which causes the ADLC to suppress the
12205 // match rules in general matching.  Instead, the ADLC generates a conversion
12206 // method in the MachNode which can be used to do in-place replacement of the
12207 // long variant with the shorter variant.  The compiler will determine if a
12208 // branch can be taken by the is_short_branch_offset() predicate in the machine
12209 // specific code section of the file.
12210 
12211 // Jump Direct - Label defines a relative address from JMP+1
12212 instruct jmpDir_short(label labl) %{
12213   match(Goto);
12214   effect(USE labl);
12215 
12216   ins_cost(300);
12217   format %{ "JMP,s  $labl" %}
12218   size(2);
12219   ins_encode %{
12220     Label* L = $labl$$label;
12221     __ jmpb(*L);
12222   %}
12223   ins_pipe( pipe_jmp );
12224   ins_short_branch(1);
12225 %}
12226 
12227 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12228 instruct jmpCon_short(cmpOp cop, eFlagsReg cr, label labl) %{
12229   match(If cop cr);
12230   effect(USE labl);
12231 
12232   ins_cost(300);
12233   format %{ "J$cop,s  $labl" %}
12234   size(2);
12235   ins_encode %{
12236     Label* L = $labl$$label;
12237     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12238   %}
12239   ins_pipe( pipe_jcc );
12240   ins_short_branch(1);
12241 %}
12242 
12243 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12244 instruct jmpLoopEnd_short(cmpOp cop, eFlagsReg cr, label labl) %{
12245   match(CountedLoopEnd cop cr);
12246   effect(USE labl);
12247 
12248   ins_cost(300);
12249   format %{ "J$cop,s  $labl\t# Loop end" %}
12250   size(2);
12251   ins_encode %{
12252     Label* L = $labl$$label;
12253     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12254   %}
12255   ins_pipe( pipe_jcc );
12256   ins_short_branch(1);
12257 %}
12258 
12259 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12260 instruct jmpLoopEndU_short(cmpOpU cop, eFlagsRegU cmp, label labl) %{
12261   match(CountedLoopEnd cop cmp);
12262   effect(USE labl);
12263 
12264   ins_cost(300);
12265   format %{ "J$cop,us $labl\t# Loop end" %}
12266   size(2);
12267   ins_encode %{
12268     Label* L = $labl$$label;
12269     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12270   %}
12271   ins_pipe( pipe_jcc );
12272   ins_short_branch(1);
12273 %}
12274 
12275 instruct jmpLoopEndUCF_short(cmpOpUCF cop, eFlagsRegUCF cmp, label labl) %{
12276   match(CountedLoopEnd cop cmp);
12277   effect(USE labl);
12278 
12279   ins_cost(300);
12280   format %{ "J$cop,us $labl\t# Loop end" %}
12281   size(2);
12282   ins_encode %{
12283     Label* L = $labl$$label;
12284     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12285   %}
12286   ins_pipe( pipe_jcc );
12287   ins_short_branch(1);
12288 %}
12289 
12290 // Jump Direct Conditional - using unsigned comparison
12291 instruct jmpConU_short(cmpOpU cop, eFlagsRegU cmp, label labl) %{
12292   match(If cop cmp);
12293   effect(USE labl);
12294 
12295   ins_cost(300);
12296   format %{ "J$cop,us $labl" %}
12297   size(2);
12298   ins_encode %{
12299     Label* L = $labl$$label;
12300     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12301   %}
12302   ins_pipe( pipe_jcc );
12303   ins_short_branch(1);
12304 %}
12305 
12306 instruct jmpConUCF_short(cmpOpUCF cop, eFlagsRegUCF cmp, label labl) %{
12307   match(If cop cmp);
12308   effect(USE labl);
12309 
12310   ins_cost(300);
12311   format %{ "J$cop,us $labl" %}
12312   size(2);
12313   ins_encode %{
12314     Label* L = $labl$$label;
12315     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12316   %}
12317   ins_pipe( pipe_jcc );
12318   ins_short_branch(1);
12319 %}
12320 
12321 instruct jmpConUCF2_short(cmpOpUCF2 cop, eFlagsRegUCF cmp, label labl) %{
12322   match(If cop cmp);
12323   effect(USE labl);
12324 
12325   ins_cost(300);
12326   format %{ $$template
12327     if ($cop$$cmpcode == Assembler::notEqual) {
12328       $$emit$$"JP,u,s   $labl\n\t"
12329       $$emit$$"J$cop,u,s   $labl"
12330     } else {
12331       $$emit$$"JP,u,s   done\n\t"
12332       $$emit$$"J$cop,u,s  $labl\n\t"
12333       $$emit$$"done:"
12334     }
12335   %}
12336   size(4);
12337   ins_encode %{
12338     Label* l = $labl$$label;
12339     if ($cop$$cmpcode == Assembler::notEqual) {
12340       __ jccb(Assembler::parity, *l);
12341       __ jccb(Assembler::notEqual, *l);
12342     } else if ($cop$$cmpcode == Assembler::equal) {
12343       Label done;
12344       __ jccb(Assembler::parity, done);
12345       __ jccb(Assembler::equal, *l);
12346       __ bind(done);
12347     } else {
12348        ShouldNotReachHere();
12349     }
12350   %}
12351   ins_pipe(pipe_jcc);
12352   ins_short_branch(1);
12353 %}
12354 
12355 // ============================================================================
12356 // Long Compare
12357 //
12358 // Currently we hold longs in 2 registers.  Comparing such values efficiently
12359 // is tricky.  The flavor of compare used depends on whether we are testing
12360 // for LT, LE, or EQ.  For a simple LT test we can check just the sign bit.
12361 // The GE test is the negated LT test.  The LE test can be had by commuting
12362 // the operands (yielding a GE test) and then negating; negate again for the
12363 // GT test.  The EQ test is done by ORcc'ing the high and low halves, and the
12364 // NE test is negated from that.
12365 
12366 // Due to a shortcoming in the ADLC, it mixes up expressions like:
12367 // (foo (CmpI (CmpL X Y) 0)) and (bar (CmpI (CmpL X 0L) 0)).  Note the
12368 // difference between 'Y' and '0L'.  The tree-matches for the CmpI sections
12369 // are collapsed internally in the ADLC's dfa-gen code.  The match for
12370 // (CmpI (CmpL X Y) 0) is silently replaced with (CmpI (CmpL X 0L) 0) and the
12371 // foo match ends up with the wrong leaf.  One fix is to not match both
12372 // reg-reg and reg-zero forms of long-compare.  This is unfortunate because
12373 // both forms beat the trinary form of long-compare and both are very useful
12374 // on Intel which has so few registers.
12375 
12376 // Manifest a CmpL result in an integer register.  Very painful.
12377 // This is the test to avoid.
12378 instruct cmpL3_reg_reg(eSIRegI dst, eRegL src1, eRegL src2, eFlagsReg flags ) %{
12379   match(Set dst (CmpL3 src1 src2));
12380   effect( KILL flags );
12381   ins_cost(1000);
12382   format %{ "XOR    $dst,$dst\n\t"
12383             "CMP    $src1.hi,$src2.hi\n\t"
12384             "JLT,s  m_one\n\t"
12385             "JGT,s  p_one\n\t"
12386             "CMP    $src1.lo,$src2.lo\n\t"
12387             "JB,s   m_one\n\t"
12388             "JEQ,s  done\n"
12389     "p_one:\tINC    $dst\n\t"
12390             "JMP,s  done\n"
12391     "m_one:\tDEC    $dst\n"
12392      "done:" %}
12393   ins_encode %{
12394     Label p_one, m_one, done;
12395     __ xorptr($dst$$Register, $dst$$Register);
12396     __ cmpl(HIGH_FROM_LOW($src1$$Register), HIGH_FROM_LOW($src2$$Register));
12397     __ jccb(Assembler::less,    m_one);
12398     __ jccb(Assembler::greater, p_one);
12399     __ cmpl($src1$$Register, $src2$$Register);
12400     __ jccb(Assembler::below,   m_one);
12401     __ jccb(Assembler::equal,   done);
12402     __ bind(p_one);
12403     __ incrementl($dst$$Register);
12404     __ jmpb(done);
12405     __ bind(m_one);
12406     __ decrementl($dst$$Register);
12407     __ bind(done);
12408   %}
12409   ins_pipe( pipe_slow );
12410 %}
12411 
12412 //======
12413 // Manifest a CmpL result in the normal flags.  Only good for LT or GE
12414 // compares.  Can be used for LE or GT compares by reversing arguments.
12415 // NOT GOOD FOR EQ/NE tests.
12416 instruct cmpL_zero_flags_LTGE( flagsReg_long_LTGE flags, eRegL src, immL0 zero ) %{
12417   match( Set flags (CmpL src zero ));
12418   ins_cost(100);
12419   format %{ "TEST   $src.hi,$src.hi" %}
12420   opcode(0x85);
12421   ins_encode( OpcP, RegReg_Hi2( src, src ) );
12422   ins_pipe( ialu_cr_reg_reg );
12423 %}
12424 
12425 // Manifest a CmpL result in the normal flags.  Only good for LT or GE
12426 // compares.  Can be used for LE or GT compares by reversing arguments.
12427 // NOT GOOD FOR EQ/NE tests.
12428 instruct cmpL_reg_flags_LTGE( flagsReg_long_LTGE flags, eRegL src1, eRegL src2, rRegI tmp ) %{
12429   match( Set flags (CmpL src1 src2 ));
12430   effect( TEMP tmp );
12431   ins_cost(300);
12432   format %{ "CMP    $src1.lo,$src2.lo\t! Long compare; set flags for low bits\n\t"
12433             "MOV    $tmp,$src1.hi\n\t"
12434             "SBB    $tmp,$src2.hi\t! Compute flags for long compare" %}
12435   ins_encode( long_cmp_flags2( src1, src2, tmp ) );
12436   ins_pipe( ialu_cr_reg_reg );
12437 %}
12438 
12439 // Long compares reg < zero/req OR reg >= zero/req.
12440 // Just a wrapper for a normal branch, plus the predicate test.
12441 instruct cmpL_LTGE(cmpOp cmp, flagsReg_long_LTGE flags, label labl) %{
12442   match(If cmp flags);
12443   effect(USE labl);
12444   predicate( _kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt || _kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge );
12445   expand %{
12446     jmpCon(cmp,flags,labl);    // JLT or JGE...
12447   %}
12448 %}
12449 
12450 //======
12451 // Manifest a CmpUL result in the normal flags.  Only good for LT or GE
12452 // compares.  Can be used for LE or GT compares by reversing arguments.
12453 // NOT GOOD FOR EQ/NE tests.
12454 instruct cmpUL_zero_flags_LTGE(flagsReg_ulong_LTGE flags, eRegL src, immL0 zero) %{
12455   match(Set flags (CmpUL src zero));
12456   ins_cost(100);
12457   format %{ "TEST   $src.hi,$src.hi" %}
12458   opcode(0x85);
12459   ins_encode(OpcP, RegReg_Hi2(src, src));
12460   ins_pipe(ialu_cr_reg_reg);
12461 %}
12462 
12463 // Manifest a CmpUL result in the normal flags.  Only good for LT or GE
12464 // compares.  Can be used for LE or GT compares by reversing arguments.
12465 // NOT GOOD FOR EQ/NE tests.
12466 instruct cmpUL_reg_flags_LTGE(flagsReg_ulong_LTGE flags, eRegL src1, eRegL src2, rRegI tmp) %{
12467   match(Set flags (CmpUL src1 src2));
12468   effect(TEMP tmp);
12469   ins_cost(300);
12470   format %{ "CMP    $src1.lo,$src2.lo\t! Unsigned long compare; set flags for low bits\n\t"
12471             "MOV    $tmp,$src1.hi\n\t"
12472             "SBB    $tmp,$src2.hi\t! Compute flags for unsigned long compare" %}
12473   ins_encode(long_cmp_flags2(src1, src2, tmp));
12474   ins_pipe(ialu_cr_reg_reg);
12475 %}
12476 
12477 // Unsigned long compares reg < zero/req OR reg >= zero/req.
12478 // Just a wrapper for a normal branch, plus the predicate test.
12479 instruct cmpUL_LTGE(cmpOpU cmp, flagsReg_ulong_LTGE flags, label labl) %{
12480   match(If cmp flags);
12481   effect(USE labl);
12482   predicate(_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt || _kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge);
12483   expand %{
12484     jmpCon(cmp, flags, labl);    // JLT or JGE...
12485   %}
12486 %}
12487 
12488 // Compare 2 longs and CMOVE longs.
12489 instruct cmovLL_reg_LTGE(cmpOp cmp, flagsReg_long_LTGE flags, eRegL dst, eRegL src) %{
12490   match(Set dst (CMoveL (Binary cmp flags) (Binary dst src)));
12491   predicate(VM_Version::supports_cmov() && ( _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge ));
12492   ins_cost(400);
12493   format %{ "CMOV$cmp $dst.lo,$src.lo\n\t"
12494             "CMOV$cmp $dst.hi,$src.hi" %}
12495   opcode(0x0F,0x40);
12496   ins_encode( enc_cmov(cmp), RegReg_Lo2( dst, src ), enc_cmov(cmp), RegReg_Hi2( dst, src ) );
12497   ins_pipe( pipe_cmov_reg_long );
12498 %}
12499 
12500 instruct cmovLL_mem_LTGE(cmpOp cmp, flagsReg_long_LTGE flags, eRegL dst, load_long_memory src) %{
12501   match(Set dst (CMoveL (Binary cmp flags) (Binary dst (LoadL src))));
12502   predicate(VM_Version::supports_cmov() && ( _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge ));
12503   ins_cost(500);
12504   format %{ "CMOV$cmp $dst.lo,$src.lo\n\t"
12505             "CMOV$cmp $dst.hi,$src.hi" %}
12506   opcode(0x0F,0x40);
12507   ins_encode( enc_cmov(cmp), RegMem(dst, src), enc_cmov(cmp), RegMem_Hi(dst, src) );
12508   ins_pipe( pipe_cmov_reg_long );
12509 %}
12510 
12511 // Compare 2 longs and CMOVE ints.
12512 instruct cmovII_reg_LTGE(cmpOp cmp, flagsReg_long_LTGE flags, rRegI dst, rRegI src) %{
12513   predicate(VM_Version::supports_cmov() && ( _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge ));
12514   match(Set dst (CMoveI (Binary cmp flags) (Binary dst src)));
12515   ins_cost(200);
12516   format %{ "CMOV$cmp $dst,$src" %}
12517   opcode(0x0F,0x40);
12518   ins_encode( enc_cmov(cmp), RegReg( dst, src ) );
12519   ins_pipe( pipe_cmov_reg );
12520 %}
12521 
12522 instruct cmovII_mem_LTGE(cmpOp cmp, flagsReg_long_LTGE flags, rRegI dst, memory src) %{
12523   predicate(VM_Version::supports_cmov() && ( _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge ));
12524   match(Set dst (CMoveI (Binary cmp flags) (Binary dst (LoadI src))));
12525   ins_cost(250);
12526   format %{ "CMOV$cmp $dst,$src" %}
12527   opcode(0x0F,0x40);
12528   ins_encode( enc_cmov(cmp), RegMem( dst, src ) );
12529   ins_pipe( pipe_cmov_mem );
12530 %}
12531 
12532 // Compare 2 longs and CMOVE ints.
12533 instruct cmovPP_reg_LTGE(cmpOp cmp, flagsReg_long_LTGE flags, eRegP dst, eRegP src) %{
12534   predicate(VM_Version::supports_cmov() && ( _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge ));
12535   match(Set dst (CMoveP (Binary cmp flags) (Binary dst src)));
12536   ins_cost(200);
12537   format %{ "CMOV$cmp $dst,$src" %}
12538   opcode(0x0F,0x40);
12539   ins_encode( enc_cmov(cmp), RegReg( dst, src ) );
12540   ins_pipe( pipe_cmov_reg );
12541 %}
12542 
12543 // Compare 2 longs and CMOVE doubles
12544 instruct cmovDDPR_reg_LTGE(cmpOp cmp, flagsReg_long_LTGE flags, regDPR dst, regDPR src) %{
12545   predicate( UseSSE<=1 && _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge );
12546   match(Set dst (CMoveD (Binary cmp flags) (Binary dst src)));
12547   ins_cost(200);
12548   expand %{
12549     fcmovDPR_regS(cmp,flags,dst,src);
12550   %}
12551 %}
12552 
12553 // Compare 2 longs and CMOVE doubles
12554 instruct cmovDD_reg_LTGE(cmpOp cmp, flagsReg_long_LTGE flags, regD dst, regD src) %{
12555   predicate( UseSSE>=2 && _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge );
12556   match(Set dst (CMoveD (Binary cmp flags) (Binary dst src)));
12557   ins_cost(200);
12558   expand %{
12559     fcmovD_regS(cmp,flags,dst,src);
12560   %}
12561 %}
12562 
12563 instruct cmovFFPR_reg_LTGE(cmpOp cmp, flagsReg_long_LTGE flags, regFPR dst, regFPR src) %{
12564   predicate( UseSSE==0 && _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge );
12565   match(Set dst (CMoveF (Binary cmp flags) (Binary dst src)));
12566   ins_cost(200);
12567   expand %{
12568     fcmovFPR_regS(cmp,flags,dst,src);
12569   %}
12570 %}
12571 
12572 instruct cmovFF_reg_LTGE(cmpOp cmp, flagsReg_long_LTGE flags, regF dst, regF src) %{
12573   predicate( UseSSE>=1 && _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge );
12574   match(Set dst (CMoveF (Binary cmp flags) (Binary dst src)));
12575   ins_cost(200);
12576   expand %{
12577     fcmovF_regS(cmp,flags,dst,src);
12578   %}
12579 %}
12580 
12581 //======
12582 // Manifest a CmpL result in the normal flags.  Only good for EQ/NE compares.
12583 instruct cmpL_zero_flags_EQNE( flagsReg_long_EQNE flags, eRegL src, immL0 zero, rRegI tmp ) %{
12584   match( Set flags (CmpL src zero ));
12585   effect(TEMP tmp);
12586   ins_cost(200);
12587   format %{ "MOV    $tmp,$src.lo\n\t"
12588             "OR     $tmp,$src.hi\t! Long is EQ/NE 0?" %}
12589   ins_encode( long_cmp_flags0( src, tmp ) );
12590   ins_pipe( ialu_reg_reg_long );
12591 %}
12592 
12593 // Manifest a CmpL result in the normal flags.  Only good for EQ/NE compares.
12594 instruct cmpL_reg_flags_EQNE( flagsReg_long_EQNE flags, eRegL src1, eRegL src2 ) %{
12595   match( Set flags (CmpL src1 src2 ));
12596   ins_cost(200+300);
12597   format %{ "CMP    $src1.lo,$src2.lo\t! Long compare; set flags for low bits\n\t"
12598             "JNE,s  skip\n\t"
12599             "CMP    $src1.hi,$src2.hi\n\t"
12600      "skip:\t" %}
12601   ins_encode( long_cmp_flags1( src1, src2 ) );
12602   ins_pipe( ialu_cr_reg_reg );
12603 %}
12604 
12605 // Long compare reg == zero/reg OR reg != zero/reg
12606 // Just a wrapper for a normal branch, plus the predicate test.
12607 instruct cmpL_EQNE(cmpOp cmp, flagsReg_long_EQNE flags, label labl) %{
12608   match(If cmp flags);
12609   effect(USE labl);
12610   predicate( _kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq || _kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne );
12611   expand %{
12612     jmpCon(cmp,flags,labl);    // JEQ or JNE...
12613   %}
12614 %}
12615 
12616 //======
12617 // Manifest a CmpUL result in the normal flags.  Only good for EQ/NE compares.
12618 instruct cmpUL_zero_flags_EQNE(flagsReg_ulong_EQNE flags, eRegL src, immL0 zero, rRegI tmp) %{
12619   match(Set flags (CmpUL src zero));
12620   effect(TEMP tmp);
12621   ins_cost(200);
12622   format %{ "MOV    $tmp,$src.lo\n\t"
12623             "OR     $tmp,$src.hi\t! Unsigned long is EQ/NE 0?" %}
12624   ins_encode(long_cmp_flags0(src, tmp));
12625   ins_pipe(ialu_reg_reg_long);
12626 %}
12627 
12628 // Manifest a CmpUL result in the normal flags.  Only good for EQ/NE compares.
12629 instruct cmpUL_reg_flags_EQNE(flagsReg_ulong_EQNE flags, eRegL src1, eRegL src2) %{
12630   match(Set flags (CmpUL src1 src2));
12631   ins_cost(200+300);
12632   format %{ "CMP    $src1.lo,$src2.lo\t! Unsigned long compare; set flags for low bits\n\t"
12633             "JNE,s  skip\n\t"
12634             "CMP    $src1.hi,$src2.hi\n\t"
12635      "skip:\t" %}
12636   ins_encode(long_cmp_flags1(src1, src2));
12637   ins_pipe(ialu_cr_reg_reg);
12638 %}
12639 
12640 // Unsigned long compare reg == zero/reg OR reg != zero/reg
12641 // Just a wrapper for a normal branch, plus the predicate test.
12642 instruct cmpUL_EQNE(cmpOpU cmp, flagsReg_ulong_EQNE flags, label labl) %{
12643   match(If cmp flags);
12644   effect(USE labl);
12645   predicate(_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq || _kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne);
12646   expand %{
12647     jmpCon(cmp, flags, labl);    // JEQ or JNE...
12648   %}
12649 %}
12650 
12651 // Compare 2 longs and CMOVE longs.
12652 instruct cmovLL_reg_EQNE(cmpOp cmp, flagsReg_long_EQNE flags, eRegL dst, eRegL src) %{
12653   match(Set dst (CMoveL (Binary cmp flags) (Binary dst src)));
12654   predicate(VM_Version::supports_cmov() && ( _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne ));
12655   ins_cost(400);
12656   format %{ "CMOV$cmp $dst.lo,$src.lo\n\t"
12657             "CMOV$cmp $dst.hi,$src.hi" %}
12658   opcode(0x0F,0x40);
12659   ins_encode( enc_cmov(cmp), RegReg_Lo2( dst, src ), enc_cmov(cmp), RegReg_Hi2( dst, src ) );
12660   ins_pipe( pipe_cmov_reg_long );
12661 %}
12662 
12663 instruct cmovLL_mem_EQNE(cmpOp cmp, flagsReg_long_EQNE flags, eRegL dst, load_long_memory src) %{
12664   match(Set dst (CMoveL (Binary cmp flags) (Binary dst (LoadL src))));
12665   predicate(VM_Version::supports_cmov() && ( _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne ));
12666   ins_cost(500);
12667   format %{ "CMOV$cmp $dst.lo,$src.lo\n\t"
12668             "CMOV$cmp $dst.hi,$src.hi" %}
12669   opcode(0x0F,0x40);
12670   ins_encode( enc_cmov(cmp), RegMem(dst, src), enc_cmov(cmp), RegMem_Hi(dst, src) );
12671   ins_pipe( pipe_cmov_reg_long );
12672 %}
12673 
12674 // Compare 2 longs and CMOVE ints.
12675 instruct cmovII_reg_EQNE(cmpOp cmp, flagsReg_long_EQNE flags, rRegI dst, rRegI src) %{
12676   predicate(VM_Version::supports_cmov() && ( _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne ));
12677   match(Set dst (CMoveI (Binary cmp flags) (Binary dst src)));
12678   ins_cost(200);
12679   format %{ "CMOV$cmp $dst,$src" %}
12680   opcode(0x0F,0x40);
12681   ins_encode( enc_cmov(cmp), RegReg( dst, src ) );
12682   ins_pipe( pipe_cmov_reg );
12683 %}
12684 
12685 instruct cmovII_mem_EQNE(cmpOp cmp, flagsReg_long_EQNE flags, rRegI dst, memory src) %{
12686   predicate(VM_Version::supports_cmov() && ( _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne ));
12687   match(Set dst (CMoveI (Binary cmp flags) (Binary dst (LoadI src))));
12688   ins_cost(250);
12689   format %{ "CMOV$cmp $dst,$src" %}
12690   opcode(0x0F,0x40);
12691   ins_encode( enc_cmov(cmp), RegMem( dst, src ) );
12692   ins_pipe( pipe_cmov_mem );
12693 %}
12694 
12695 // Compare 2 longs and CMOVE ints.
12696 instruct cmovPP_reg_EQNE(cmpOp cmp, flagsReg_long_EQNE flags, eRegP dst, eRegP src) %{
12697   predicate(VM_Version::supports_cmov() && ( _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne ));
12698   match(Set dst (CMoveP (Binary cmp flags) (Binary dst src)));
12699   ins_cost(200);
12700   format %{ "CMOV$cmp $dst,$src" %}
12701   opcode(0x0F,0x40);
12702   ins_encode( enc_cmov(cmp), RegReg( dst, src ) );
12703   ins_pipe( pipe_cmov_reg );
12704 %}
12705 
12706 // Compare 2 longs and CMOVE doubles
12707 instruct cmovDDPR_reg_EQNE(cmpOp cmp, flagsReg_long_EQNE flags, regDPR dst, regDPR src) %{
12708   predicate( UseSSE<=1 && _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne );
12709   match(Set dst (CMoveD (Binary cmp flags) (Binary dst src)));
12710   ins_cost(200);
12711   expand %{
12712     fcmovDPR_regS(cmp,flags,dst,src);
12713   %}
12714 %}
12715 
12716 // Compare 2 longs and CMOVE doubles
12717 instruct cmovDD_reg_EQNE(cmpOp cmp, flagsReg_long_EQNE flags, regD dst, regD src) %{
12718   predicate( UseSSE>=2 && _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne );
12719   match(Set dst (CMoveD (Binary cmp flags) (Binary dst src)));
12720   ins_cost(200);
12721   expand %{
12722     fcmovD_regS(cmp,flags,dst,src);
12723   %}
12724 %}
12725 
12726 instruct cmovFFPR_reg_EQNE(cmpOp cmp, flagsReg_long_EQNE flags, regFPR dst, regFPR src) %{
12727   predicate( UseSSE==0 && _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne );
12728   match(Set dst (CMoveF (Binary cmp flags) (Binary dst src)));
12729   ins_cost(200);
12730   expand %{
12731     fcmovFPR_regS(cmp,flags,dst,src);
12732   %}
12733 %}
12734 
12735 instruct cmovFF_reg_EQNE(cmpOp cmp, flagsReg_long_EQNE flags, regF dst, regF src) %{
12736   predicate( UseSSE>=1 && _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne );
12737   match(Set dst (CMoveF (Binary cmp flags) (Binary dst src)));
12738   ins_cost(200);
12739   expand %{
12740     fcmovF_regS(cmp,flags,dst,src);
12741   %}
12742 %}
12743 
12744 //======
12745 // Manifest a CmpL result in the normal flags.  Only good for LE or GT compares.
12746 // Same as cmpL_reg_flags_LEGT except must negate src
12747 instruct cmpL_zero_flags_LEGT( flagsReg_long_LEGT flags, eRegL src, immL0 zero, rRegI tmp ) %{
12748   match( Set flags (CmpL src zero ));
12749   effect( TEMP tmp );
12750   ins_cost(300);
12751   format %{ "XOR    $tmp,$tmp\t# Long compare for -$src < 0, use commuted test\n\t"
12752             "CMP    $tmp,$src.lo\n\t"
12753             "SBB    $tmp,$src.hi\n\t" %}
12754   ins_encode( long_cmp_flags3(src, tmp) );
12755   ins_pipe( ialu_reg_reg_long );
12756 %}
12757 
12758 // Manifest a CmpL result in the normal flags.  Only good for LE or GT compares.
12759 // Same as cmpL_reg_flags_LTGE except operands swapped.  Swapping operands
12760 // requires a commuted test to get the same result.
12761 instruct cmpL_reg_flags_LEGT( flagsReg_long_LEGT flags, eRegL src1, eRegL src2, rRegI tmp ) %{
12762   match( Set flags (CmpL src1 src2 ));
12763   effect( TEMP tmp );
12764   ins_cost(300);
12765   format %{ "CMP    $src2.lo,$src1.lo\t! Long compare, swapped operands, use with commuted test\n\t"
12766             "MOV    $tmp,$src2.hi\n\t"
12767             "SBB    $tmp,$src1.hi\t! Compute flags for long compare" %}
12768   ins_encode( long_cmp_flags2( src2, src1, tmp ) );
12769   ins_pipe( ialu_cr_reg_reg );
12770 %}
12771 
12772 // Long compares reg < zero/req OR reg >= zero/req.
12773 // Just a wrapper for a normal branch, plus the predicate test
12774 instruct cmpL_LEGT(cmpOp_commute cmp, flagsReg_long_LEGT flags, label labl) %{
12775   match(If cmp flags);
12776   effect(USE labl);
12777   predicate( _kids[0]->_leaf->as_Bool()->_test._test == BoolTest::gt || _kids[0]->_leaf->as_Bool()->_test._test == BoolTest::le );
12778   ins_cost(300);
12779   expand %{
12780     jmpCon(cmp,flags,labl);    // JGT or JLE...
12781   %}
12782 %}
12783 
12784 //======
12785 // Manifest a CmpUL result in the normal flags.  Only good for LE or GT compares.
12786 // Same as cmpUL_reg_flags_LEGT except must negate src
12787 instruct cmpUL_zero_flags_LEGT(flagsReg_ulong_LEGT flags, eRegL src, immL0 zero, rRegI tmp) %{
12788   match(Set flags (CmpUL src zero));
12789   effect(TEMP tmp);
12790   ins_cost(300);
12791   format %{ "XOR    $tmp,$tmp\t# Unsigned long compare for -$src < 0, use commuted test\n\t"
12792             "CMP    $tmp,$src.lo\n\t"
12793             "SBB    $tmp,$src.hi\n\t" %}
12794   ins_encode(long_cmp_flags3(src, tmp));
12795   ins_pipe(ialu_reg_reg_long);
12796 %}
12797 
12798 // Manifest a CmpUL result in the normal flags.  Only good for LE or GT compares.
12799 // Same as cmpUL_reg_flags_LTGE except operands swapped.  Swapping operands
12800 // requires a commuted test to get the same result.
12801 instruct cmpUL_reg_flags_LEGT(flagsReg_ulong_LEGT flags, eRegL src1, eRegL src2, rRegI tmp) %{
12802   match(Set flags (CmpUL src1 src2));
12803   effect(TEMP tmp);
12804   ins_cost(300);
12805   format %{ "CMP    $src2.lo,$src1.lo\t! Unsigned long compare, swapped operands, use with commuted test\n\t"
12806             "MOV    $tmp,$src2.hi\n\t"
12807             "SBB    $tmp,$src1.hi\t! Compute flags for unsigned long compare" %}
12808   ins_encode(long_cmp_flags2( src2, src1, tmp));
12809   ins_pipe(ialu_cr_reg_reg);
12810 %}
12811 
12812 // Unsigned long compares reg < zero/req OR reg >= zero/req.
12813 // Just a wrapper for a normal branch, plus the predicate test
12814 instruct cmpUL_LEGT(cmpOpU_commute cmp, flagsReg_ulong_LEGT flags, label labl) %{
12815   match(If cmp flags);
12816   effect(USE labl);
12817   predicate(_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::gt || _kids[0]->_leaf->as_Bool()->_test._test == BoolTest::le);
12818   ins_cost(300);
12819   expand %{
12820     jmpCon(cmp, flags, labl);    // JGT or JLE...
12821   %}
12822 %}
12823 
12824 // Compare 2 longs and CMOVE longs.
12825 instruct cmovLL_reg_LEGT(cmpOp_commute cmp, flagsReg_long_LEGT flags, eRegL dst, eRegL src) %{
12826   match(Set dst (CMoveL (Binary cmp flags) (Binary dst src)));
12827   predicate(VM_Version::supports_cmov() && ( _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::le || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::gt ));
12828   ins_cost(400);
12829   format %{ "CMOV$cmp $dst.lo,$src.lo\n\t"
12830             "CMOV$cmp $dst.hi,$src.hi" %}
12831   opcode(0x0F,0x40);
12832   ins_encode( enc_cmov(cmp), RegReg_Lo2( dst, src ), enc_cmov(cmp), RegReg_Hi2( dst, src ) );
12833   ins_pipe( pipe_cmov_reg_long );
12834 %}
12835 
12836 instruct cmovLL_mem_LEGT(cmpOp_commute cmp, flagsReg_long_LEGT flags, eRegL dst, load_long_memory src) %{
12837   match(Set dst (CMoveL (Binary cmp flags) (Binary dst (LoadL src))));
12838   predicate(VM_Version::supports_cmov() && ( _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::le || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::gt ));
12839   ins_cost(500);
12840   format %{ "CMOV$cmp $dst.lo,$src.lo\n\t"
12841             "CMOV$cmp $dst.hi,$src.hi+4" %}
12842   opcode(0x0F,0x40);
12843   ins_encode( enc_cmov(cmp), RegMem(dst, src), enc_cmov(cmp), RegMem_Hi(dst, src) );
12844   ins_pipe( pipe_cmov_reg_long );
12845 %}
12846 
12847 // Compare 2 longs and CMOVE ints.
12848 instruct cmovII_reg_LEGT(cmpOp_commute cmp, flagsReg_long_LEGT flags, rRegI dst, rRegI src) %{
12849   predicate(VM_Version::supports_cmov() && ( _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::le || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::gt ));
12850   match(Set dst (CMoveI (Binary cmp flags) (Binary dst src)));
12851   ins_cost(200);
12852   format %{ "CMOV$cmp $dst,$src" %}
12853   opcode(0x0F,0x40);
12854   ins_encode( enc_cmov(cmp), RegReg( dst, src ) );
12855   ins_pipe( pipe_cmov_reg );
12856 %}
12857 
12858 instruct cmovII_mem_LEGT(cmpOp_commute cmp, flagsReg_long_LEGT flags, rRegI dst, memory src) %{
12859   predicate(VM_Version::supports_cmov() && ( _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::le || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::gt ));
12860   match(Set dst (CMoveI (Binary cmp flags) (Binary dst (LoadI src))));
12861   ins_cost(250);
12862   format %{ "CMOV$cmp $dst,$src" %}
12863   opcode(0x0F,0x40);
12864   ins_encode( enc_cmov(cmp), RegMem( dst, src ) );
12865   ins_pipe( pipe_cmov_mem );
12866 %}
12867 
12868 // Compare 2 longs and CMOVE ptrs.
12869 instruct cmovPP_reg_LEGT(cmpOp_commute cmp, flagsReg_long_LEGT flags, eRegP dst, eRegP src) %{
12870   predicate(VM_Version::supports_cmov() && ( _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::le || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::gt ));
12871   match(Set dst (CMoveP (Binary cmp flags) (Binary dst src)));
12872   ins_cost(200);
12873   format %{ "CMOV$cmp $dst,$src" %}
12874   opcode(0x0F,0x40);
12875   ins_encode( enc_cmov(cmp), RegReg( dst, src ) );
12876   ins_pipe( pipe_cmov_reg );
12877 %}
12878 
12879 // Compare 2 longs and CMOVE doubles
12880 instruct cmovDDPR_reg_LEGT(cmpOp_commute cmp, flagsReg_long_LEGT flags, regDPR dst, regDPR src) %{
12881   predicate( UseSSE<=1 && _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::le || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::gt );
12882   match(Set dst (CMoveD (Binary cmp flags) (Binary dst src)));
12883   ins_cost(200);
12884   expand %{
12885     fcmovDPR_regS(cmp,flags,dst,src);
12886   %}
12887 %}
12888 
12889 // Compare 2 longs and CMOVE doubles
12890 instruct cmovDD_reg_LEGT(cmpOp_commute cmp, flagsReg_long_LEGT flags, regD dst, regD src) %{
12891   predicate( UseSSE>=2 && _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::le || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::gt );
12892   match(Set dst (CMoveD (Binary cmp flags) (Binary dst src)));
12893   ins_cost(200);
12894   expand %{
12895     fcmovD_regS(cmp,flags,dst,src);
12896   %}
12897 %}
12898 
12899 instruct cmovFFPR_reg_LEGT(cmpOp_commute cmp, flagsReg_long_LEGT flags, regFPR dst, regFPR src) %{
12900   predicate( UseSSE==0 && _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::le || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::gt );
12901   match(Set dst (CMoveF (Binary cmp flags) (Binary dst src)));
12902   ins_cost(200);
12903   expand %{
12904     fcmovFPR_regS(cmp,flags,dst,src);
12905   %}
12906 %}
12907 
12908 
12909 instruct cmovFF_reg_LEGT(cmpOp_commute cmp, flagsReg_long_LEGT flags, regF dst, regF src) %{
12910   predicate( UseSSE>=1 && _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::le || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::gt );
12911   match(Set dst (CMoveF (Binary cmp flags) (Binary dst src)));
12912   ins_cost(200);
12913   expand %{
12914     fcmovF_regS(cmp,flags,dst,src);
12915   %}
12916 %}
12917 
12918 
12919 // ============================================================================
12920 // Procedure Call/Return Instructions
12921 // Call Java Static Instruction
12922 // Note: If this code changes, the corresponding ret_addr_offset() and
12923 //       compute_padding() functions will have to be adjusted.
12924 instruct CallStaticJavaDirect(method meth) %{
12925   match(CallStaticJava);
12926   effect(USE meth);
12927 
12928   ins_cost(300);
12929   format %{ "CALL,static " %}
12930   opcode(0xE8); /* E8 cd */
12931   ins_encode( pre_call_resets,
12932               Java_Static_Call( meth ),
12933               call_epilog,
12934               post_call_FPU );
12935   ins_pipe( pipe_slow );
12936   ins_alignment(4);
12937 %}
12938 
12939 // Call Java Dynamic Instruction
12940 // Note: If this code changes, the corresponding ret_addr_offset() and
12941 //       compute_padding() functions will have to be adjusted.
12942 instruct CallDynamicJavaDirect(method meth) %{
12943   match(CallDynamicJava);
12944   effect(USE meth);
12945 
12946   ins_cost(300);
12947   format %{ "MOV    EAX,(oop)-1\n\t"
12948             "CALL,dynamic" %}
12949   opcode(0xE8); /* E8 cd */
12950   ins_encode( pre_call_resets,
12951               Java_Dynamic_Call( meth ),
12952               call_epilog,
12953               post_call_FPU );
12954   ins_pipe( pipe_slow );
12955   ins_alignment(4);
12956 %}
12957 
12958 // Call Runtime Instruction
12959 instruct CallRuntimeDirect(method meth) %{
12960   match(CallRuntime );
12961   effect(USE meth);
12962 
12963   ins_cost(300);
12964   format %{ "CALL,runtime " %}
12965   opcode(0xE8); /* E8 cd */
12966   // Use FFREEs to clear entries in float stack
12967   ins_encode( pre_call_resets,
12968               FFree_Float_Stack_All,
12969               Java_To_Runtime( meth ),
12970               post_call_FPU );
12971   ins_pipe( pipe_slow );
12972 %}
12973 
12974 // Call runtime without safepoint
12975 instruct CallLeafDirect(method meth) %{
12976   match(CallLeaf);
12977   effect(USE meth);
12978 
12979   ins_cost(300);
12980   format %{ "CALL_LEAF,runtime " %}
12981   opcode(0xE8); /* E8 cd */
12982   ins_encode( pre_call_resets,
12983               FFree_Float_Stack_All,
12984               Java_To_Runtime( meth ),
12985               Verify_FPU_For_Leaf, post_call_FPU );
12986   ins_pipe( pipe_slow );
12987 %}
12988 
12989 instruct CallLeafNoFPDirect(method meth) %{
12990   match(CallLeafNoFP);
12991   effect(USE meth);
12992 
12993   ins_cost(300);
12994   format %{ "CALL_LEAF_NOFP,runtime " %}
12995   opcode(0xE8); /* E8 cd */
12996   ins_encode(Java_To_Runtime(meth));
12997   ins_pipe( pipe_slow );
12998 %}
12999 
13000 
13001 // Return Instruction
13002 // Remove the return address & jump to it.
13003 instruct Ret() %{
13004   match(Return);
13005   format %{ "RET" %}
13006   opcode(0xC3);
13007   ins_encode(OpcP);
13008   ins_pipe( pipe_jmp );
13009 %}
13010 
13011 // Tail Call; Jump from runtime stub to Java code.
13012 // Also known as an 'interprocedural jump'.
13013 // Target of jump will eventually return to caller.
13014 // TailJump below removes the return address.
13015 instruct TailCalljmpInd(eRegP_no_EBP jump_target, eBXRegP method_oop) %{
13016   match(TailCall jump_target method_oop );
13017   ins_cost(300);
13018   format %{ "JMP    $jump_target \t# EBX holds method oop" %}
13019   opcode(0xFF, 0x4);  /* Opcode FF /4 */
13020   ins_encode( OpcP, RegOpc(jump_target) );
13021   ins_pipe( pipe_jmp );
13022 %}
13023 
13024 
13025 // Tail Jump; remove the return address; jump to target.
13026 // TailCall above leaves the return address around.
13027 instruct tailjmpInd(eRegP_no_EBP jump_target, eAXRegP ex_oop) %{
13028   match( TailJump jump_target ex_oop );
13029   ins_cost(300);
13030   format %{ "POP    EDX\t# pop return address into dummy\n\t"
13031             "JMP    $jump_target " %}
13032   opcode(0xFF, 0x4);  /* Opcode FF /4 */
13033   ins_encode( enc_pop_rdx,
13034               OpcP, RegOpc(jump_target) );
13035   ins_pipe( pipe_jmp );
13036 %}
13037 
13038 // Create exception oop: created by stack-crawling runtime code.
13039 // Created exception is now available to this handler, and is setup
13040 // just prior to jumping to this handler.  No code emitted.
13041 instruct CreateException( eAXRegP ex_oop )
13042 %{
13043   match(Set ex_oop (CreateEx));
13044 
13045   size(0);
13046   // use the following format syntax
13047   format %{ "# exception oop is in EAX; no code emitted" %}
13048   ins_encode();
13049   ins_pipe( empty );
13050 %}
13051 
13052 
13053 // Rethrow exception:
13054 // The exception oop will come in the first argument position.
13055 // Then JUMP (not call) to the rethrow stub code.
13056 instruct RethrowException()
13057 %{
13058   match(Rethrow);
13059 
13060   // use the following format syntax
13061   format %{ "JMP    rethrow_stub" %}
13062   ins_encode(enc_rethrow);
13063   ins_pipe( pipe_jmp );
13064 %}
13065 
13066 // inlined locking and unlocking
13067 
13068 instruct cmpFastLockRTM(eFlagsReg cr, eRegP object, eBXRegP box, eAXRegI tmp, eDXRegI scr, rRegI cx1, rRegI cx2) %{
13069   predicate(Compile::current()->use_rtm());
13070   match(Set cr (FastLock object box));
13071   effect(TEMP tmp, TEMP scr, TEMP cx1, TEMP cx2, USE_KILL box);
13072   ins_cost(300);
13073   format %{ "FASTLOCK $object,$box\t! kills $box,$tmp,$scr,$cx1,$cx2" %}
13074   ins_encode %{
13075     __ fast_lock($object$$Register, $box$$Register, $tmp$$Register,
13076                  $scr$$Register, $cx1$$Register, $cx2$$Register,
13077                  _counters, _rtm_counters, _stack_rtm_counters,
13078                  ((Method*)(ra_->C->method()->constant_encoding()))->method_data(),
13079                  true, ra_->C->profile_rtm());
13080   %}
13081   ins_pipe(pipe_slow);
13082 %}
13083 
13084 instruct cmpFastLock(eFlagsReg cr, eRegP object, eBXRegP box, eAXRegI tmp, eRegP scr) %{
13085   predicate(!Compile::current()->use_rtm());
13086   match(Set cr (FastLock object box));
13087   effect(TEMP tmp, TEMP scr, USE_KILL box);
13088   ins_cost(300);
13089   format %{ "FASTLOCK $object,$box\t! kills $box,$tmp,$scr" %}
13090   ins_encode %{
13091     __ fast_lock($object$$Register, $box$$Register, $tmp$$Register,
13092                  $scr$$Register, noreg, noreg, _counters, NULL, NULL, NULL, false, false);
13093   %}
13094   ins_pipe(pipe_slow);
13095 %}
13096 
13097 instruct cmpFastUnlock(eFlagsReg cr, eRegP object, eAXRegP box, eRegP tmp ) %{
13098   match(Set cr (FastUnlock object box));
13099   effect(TEMP tmp, USE_KILL box);
13100   ins_cost(300);
13101   format %{ "FASTUNLOCK $object,$box\t! kills $box,$tmp" %}
13102   ins_encode %{
13103     __ fast_unlock($object$$Register, $box$$Register, $tmp$$Register, ra_->C->use_rtm());
13104   %}
13105   ins_pipe(pipe_slow);
13106 %}
13107 
13108 
13109 
13110 // ============================================================================
13111 // Safepoint Instruction
13112 instruct safePoint_poll(eFlagsReg cr) %{
13113   match(SafePoint);
13114   effect(KILL cr);
13115 
13116   // TODO-FIXME: we currently poll at offset 0 of the safepoint polling page.
13117   // On SPARC that might be acceptable as we can generate the address with
13118   // just a sethi, saving an or.  By polling at offset 0 we can end up
13119   // putting additional pressure on the index-0 in the D$.  Because of
13120   // alignment (just like the situation at hand) the lower indices tend
13121   // to see more traffic.  It'd be better to change the polling address
13122   // to offset 0 of the last $line in the polling page.
13123 
13124   format %{ "TSTL   #polladdr,EAX\t! Safepoint: poll for GC" %}
13125   ins_cost(125);
13126   size(6) ;
13127   ins_encode( Safepoint_Poll() );
13128   ins_pipe( ialu_reg_mem );
13129 %}
13130 
13131 
13132 // ============================================================================
13133 // This name is KNOWN by the ADLC and cannot be changed.
13134 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
13135 // for this guy.
13136 instruct tlsLoadP(eRegP dst, eFlagsReg cr) %{
13137   match(Set dst (ThreadLocal));
13138   effect(DEF dst, KILL cr);
13139 
13140   format %{ "MOV    $dst, Thread::current()" %}
13141   ins_encode %{
13142     Register dstReg = as_Register($dst$$reg);
13143     __ get_thread(dstReg);
13144   %}
13145   ins_pipe( ialu_reg_fat );
13146 %}
13147 
13148 
13149 
13150 //----------PEEPHOLE RULES-----------------------------------------------------
13151 // These must follow all instruction definitions as they use the names
13152 // defined in the instructions definitions.
13153 //
13154 // peepmatch ( root_instr_name [preceding_instruction]* );
13155 //
13156 // peepconstraint %{
13157 // (instruction_number.operand_name relational_op instruction_number.operand_name
13158 //  [, ...] );
13159 // // instruction numbers are zero-based using left to right order in peepmatch
13160 //
13161 // peepreplace ( instr_name  ( [instruction_number.operand_name]* ) );
13162 // // provide an instruction_number.operand_name for each operand that appears
13163 // // in the replacement instruction's match rule
13164 //
13165 // ---------VM FLAGS---------------------------------------------------------
13166 //
13167 // All peephole optimizations can be turned off using -XX:-OptoPeephole
13168 //
13169 // Each peephole rule is given an identifying number starting with zero and
13170 // increasing by one in the order seen by the parser.  An individual peephole
13171 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
13172 // on the command-line.
13173 //
13174 // ---------CURRENT LIMITATIONS----------------------------------------------
13175 //
13176 // Only match adjacent instructions in same basic block
13177 // Only equality constraints
13178 // Only constraints between operands, not (0.dest_reg == EAX_enc)
13179 // Only one replacement instruction
13180 //
13181 // ---------EXAMPLE----------------------------------------------------------
13182 //
13183 // // pertinent parts of existing instructions in architecture description
13184 // instruct movI(rRegI dst, rRegI src) %{
13185 //   match(Set dst (CopyI src));
13186 // %}
13187 //
13188 // instruct incI_eReg(rRegI dst, immI1 src, eFlagsReg cr) %{
13189 //   match(Set dst (AddI dst src));
13190 //   effect(KILL cr);
13191 // %}
13192 //
13193 // // Change (inc mov) to lea
13194 // peephole %{
13195 //   // increment preceeded by register-register move
13196 //   peepmatch ( incI_eReg movI );
13197 //   // require that the destination register of the increment
13198 //   // match the destination register of the move
13199 //   peepconstraint ( 0.dst == 1.dst );
13200 //   // construct a replacement instruction that sets
13201 //   // the destination to ( move's source register + one )
13202 //   peepreplace ( leaI_eReg_immI( 0.dst 1.src 0.src ) );
13203 // %}
13204 //
13205 // Implementation no longer uses movX instructions since
13206 // machine-independent system no longer uses CopyX nodes.
13207 //
13208 // peephole %{
13209 //   peepmatch ( incI_eReg movI );
13210 //   peepconstraint ( 0.dst == 1.dst );
13211 //   peepreplace ( leaI_eReg_immI( 0.dst 1.src 0.src ) );
13212 // %}
13213 //
13214 // peephole %{
13215 //   peepmatch ( decI_eReg movI );
13216 //   peepconstraint ( 0.dst == 1.dst );
13217 //   peepreplace ( leaI_eReg_immI( 0.dst 1.src 0.src ) );
13218 // %}
13219 //
13220 // peephole %{
13221 //   peepmatch ( addI_eReg_imm movI );
13222 //   peepconstraint ( 0.dst == 1.dst );
13223 //   peepreplace ( leaI_eReg_immI( 0.dst 1.src 0.src ) );
13224 // %}
13225 //
13226 // peephole %{
13227 //   peepmatch ( addP_eReg_imm movP );
13228 //   peepconstraint ( 0.dst == 1.dst );
13229 //   peepreplace ( leaP_eReg_immI( 0.dst 1.src 0.src ) );
13230 // %}
13231 
13232 // // Change load of spilled value to only a spill
13233 // instruct storeI(memory mem, rRegI src) %{
13234 //   match(Set mem (StoreI mem src));
13235 // %}
13236 //
13237 // instruct loadI(rRegI dst, memory mem) %{
13238 //   match(Set dst (LoadI mem));
13239 // %}
13240 //
13241 peephole %{
13242   peepmatch ( loadI storeI );
13243   peepconstraint ( 1.src == 0.dst, 1.mem == 0.mem );
13244   peepreplace ( storeI( 1.mem 1.mem 1.src ) );
13245 %}
13246 
13247 //----------SMARTSPILL RULES---------------------------------------------------
13248 // These must follow all instruction definitions as they use the names
13249 // defined in the instructions definitions.