1 /*
   2  * Copyright (c) 1997, 2021, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #ifndef SHARE_OPTO_MEMNODE_HPP
  26 #define SHARE_OPTO_MEMNODE_HPP
  27 
  28 #include "opto/multnode.hpp"
  29 #include "opto/node.hpp"
  30 #include "opto/opcodes.hpp"
  31 #include "opto/type.hpp"
  32 
  33 // Portions of code courtesy of Clifford Click
  34 
  35 class MultiNode;
  36 class PhaseCCP;
  37 class PhaseTransform;
  38 
  39 //------------------------------MemNode----------------------------------------
  40 // Load or Store, possibly throwing a NULL pointer exception
  41 class MemNode : public Node {
  42 private:
  43   bool _unaligned_access; // Unaligned access from unsafe
  44   bool _mismatched_access; // Mismatched access from unsafe: byte read in integer array for instance
  45   bool _unsafe_access;     // Access of unsafe origin.
  46   uint8_t _barrier_data;   // Bit field with barrier information
  47 
  48 protected:
  49 #ifdef ASSERT
  50   const TypePtr* _adr_type;     // What kind of memory is being addressed?
  51 #endif
  52   virtual uint size_of() const;
  53 public:
  54   enum { Control,               // When is it safe to do this load?
  55          Memory,                // Chunk of memory is being loaded from
  56          Address,               // Actually address, derived from base
  57          ValueIn,               // Value to store
  58          OopStore               // Preceeding oop store, only in StoreCM
  59   };
  60   typedef enum { unordered = 0,
  61                  acquire,       // Load has to acquire or be succeeded by MemBarAcquire.
  62                  release,       // Store has to release or be preceded by MemBarRelease.
  63                  seqcst,        // LoadStore has to have both acquire and release semantics.
  64                  unset          // The memory ordering is not set (used for testing)
  65   } MemOrd;
  66 protected:
  67   MemNode( Node *c0, Node *c1, Node *c2, const TypePtr* at ) :
  68       Node(c0,c1,c2),
  69       _unaligned_access(false),
  70       _mismatched_access(false),
  71       _unsafe_access(false),
  72       _barrier_data(0) {
  73     init_class_id(Class_Mem);
  74     debug_only(_adr_type=at; adr_type();)
  75   }
  76   MemNode( Node *c0, Node *c1, Node *c2, const TypePtr* at, Node *c3 ) :
  77       Node(c0,c1,c2,c3),
  78       _unaligned_access(false),
  79       _mismatched_access(false),
  80       _unsafe_access(false),
  81       _barrier_data(0) {
  82     init_class_id(Class_Mem);
  83     debug_only(_adr_type=at; adr_type();)
  84   }
  85   MemNode( Node *c0, Node *c1, Node *c2, const TypePtr* at, Node *c3, Node *c4) :
  86       Node(c0,c1,c2,c3,c4),
  87       _unaligned_access(false),
  88       _mismatched_access(false),
  89       _unsafe_access(false),
  90       _barrier_data(0) {
  91     init_class_id(Class_Mem);
  92     debug_only(_adr_type=at; adr_type();)
  93   }
  94 
  95   virtual Node* find_previous_arraycopy(PhaseTransform* phase, Node* ld_alloc, Node*& mem, bool can_see_stored_value) const { return NULL; }
  96   ArrayCopyNode* find_array_copy_clone(PhaseTransform* phase, Node* ld_alloc, Node* mem) const;
  97   static bool check_if_adr_maybe_raw(Node* adr);
  98 
  99 public:
 100   // Helpers for the optimizer.  Documented in memnode.cpp.
 101   static bool detect_ptr_independence(Node* p1, AllocateNode* a1,
 102                                       Node* p2, AllocateNode* a2,
 103                                       PhaseTransform* phase);
 104   static bool adr_phi_is_loop_invariant(Node* adr_phi, Node* cast);
 105 
 106   static Node *optimize_simple_memory_chain(Node *mchain, const TypeOopPtr *t_oop, Node *load, PhaseGVN *phase);
 107   static Node *optimize_memory_chain(Node *mchain, const TypePtr *t_adr, Node *load, PhaseGVN *phase);
 108   // This one should probably be a phase-specific function:
 109   static bool all_controls_dominate(Node* dom, Node* sub);
 110 
 111   virtual const class TypePtr *adr_type() const;  // returns bottom_type of address
 112 
 113   // Shared code for Ideal methods:
 114   Node *Ideal_common(PhaseGVN *phase, bool can_reshape);  // Return -1 for short-circuit NULL.
 115 
 116   // Helper function for adr_type() implementations.
 117   static const TypePtr* calculate_adr_type(const Type* t, const TypePtr* cross_check = NULL);
 118 
 119   // Raw access function, to allow copying of adr_type efficiently in
 120   // product builds and retain the debug info for debug builds.
 121   const TypePtr *raw_adr_type() const {
 122 #ifdef ASSERT
 123     return _adr_type;
 124 #else
 125     return 0;
 126 #endif
 127   }
 128 
 129 #ifdef ASSERT
 130   void set_adr_type(const TypePtr* adr_type) { _adr_type = adr_type; }
 131 #endif
 132 
 133   // Map a load or store opcode to its corresponding store opcode.
 134   // (Return -1 if unknown.)
 135   virtual int store_Opcode() const { return -1; }
 136 
 137   // What is the type of the value in memory?  (T_VOID mean "unspecified".)
 138   virtual BasicType memory_type() const = 0;
 139   virtual int memory_size() const {
 140 #ifdef ASSERT
 141     return type2aelembytes(memory_type(), true);
 142 #else
 143     return type2aelembytes(memory_type());
 144 #endif
 145   }
 146 
 147   uint8_t barrier_data() { return _barrier_data; }
 148   void set_barrier_data(uint8_t barrier_data) { _barrier_data = barrier_data; }
 149 
 150   // Search through memory states which precede this node (load or store).
 151   // Look for an exact match for the address, with no intervening
 152   // aliased stores.
 153   Node* find_previous_store(PhaseTransform* phase);
 154 
 155   // Can this node (load or store) accurately see a stored value in
 156   // the given memory state?  (The state may or may not be in(Memory).)
 157   Node* can_see_stored_value(Node* st, PhaseTransform* phase) const;
 158 
 159   void set_unaligned_access() { _unaligned_access = true; }
 160   bool is_unaligned_access() const { return _unaligned_access; }
 161   void set_mismatched_access() { _mismatched_access = true; }
 162   bool is_mismatched_access() const { return _mismatched_access; }
 163   void set_unsafe_access() { _unsafe_access = true; }
 164   bool is_unsafe_access() const { return _unsafe_access; }
 165 
 166 #ifndef PRODUCT
 167   static void dump_adr_type(const Node* mem, const TypePtr* adr_type, outputStream *st);
 168   virtual void dump_spec(outputStream *st) const;
 169 #endif
 170 };
 171 
 172 //------------------------------LoadNode---------------------------------------
 173 // Load value; requires Memory and Address
 174 class LoadNode : public MemNode {
 175 public:
 176   // Some loads (from unsafe) should be pinned: they don't depend only
 177   // on the dominating test.  The field _control_dependency below records
 178   // whether that node depends only on the dominating test.
 179   // Pinned and UnknownControl are similar, but differ in that Pinned
 180   // loads are not allowed to float across safepoints, whereas UnknownControl
 181   // loads are allowed to do that. Therefore, Pinned is stricter.
 182   enum ControlDependency {
 183     Pinned,
 184     UnknownControl,
 185     DependsOnlyOnTest
 186   };
 187 
 188 private:
 189   // LoadNode::hash() doesn't take the _control_dependency field
 190   // into account: If the graph already has a non-pinned LoadNode and
 191   // we add a pinned LoadNode with the same inputs, it's safe for GVN
 192   // to replace the pinned LoadNode with the non-pinned LoadNode,
 193   // otherwise it wouldn't be safe to have a non pinned LoadNode with
 194   // those inputs in the first place. If the graph already has a
 195   // pinned LoadNode and we add a non pinned LoadNode with the same
 196   // inputs, it's safe (but suboptimal) for GVN to replace the
 197   // non-pinned LoadNode by the pinned LoadNode.
 198   ControlDependency _control_dependency;
 199 
 200   // On platforms with weak memory ordering (e.g., PPC, Ia64) we distinguish
 201   // loads that can be reordered, and such requiring acquire semantics to
 202   // adhere to the Java specification.  The required behaviour is stored in
 203   // this field.
 204   const MemOrd _mo;
 205 
 206   AllocateNode* is_new_object_mark_load(PhaseGVN *phase) const;
 207 
 208 protected:
 209   virtual bool cmp(const Node &n) const;
 210   virtual uint size_of() const; // Size is bigger
 211   // Should LoadNode::Ideal() attempt to remove control edges?
 212   virtual bool can_remove_control() const;
 213   const Type* const _type;      // What kind of value is loaded?
 214 
 215   virtual Node* find_previous_arraycopy(PhaseTransform* phase, Node* ld_alloc, Node*& mem, bool can_see_stored_value) const;
 216 public:
 217 
 218   LoadNode(Node *c, Node *mem, Node *adr, const TypePtr* at, const Type *rt, MemOrd mo, ControlDependency control_dependency)
 219     : MemNode(c,mem,adr,at), _control_dependency(control_dependency), _mo(mo), _type(rt) {
 220     init_class_id(Class_Load);
 221   }
 222   inline bool is_unordered() const { return !is_acquire(); }
 223   inline bool is_acquire() const {
 224     assert(_mo == unordered || _mo == acquire, "unexpected");
 225     return _mo == acquire;
 226   }
 227   inline bool is_unsigned() const {
 228     int lop = Opcode();
 229     return (lop == Op_LoadUB) || (lop == Op_LoadUS);
 230   }
 231 
 232   // Polymorphic factory method:
 233   static Node* make(PhaseGVN& gvn, Node *c, Node *mem, Node *adr,
 234                     const TypePtr* at, const Type *rt, BasicType bt,
 235                     MemOrd mo, ControlDependency control_dependency = DependsOnlyOnTest,
 236                     bool unaligned = false, bool mismatched = false, bool unsafe = false,
 237                     uint8_t barrier_data = 0);
 238 
 239   virtual uint hash()   const;  // Check the type
 240 
 241   // Handle algebraic identities here.  If we have an identity, return the Node
 242   // we are equivalent to.  We look for Load of a Store.
 243   virtual Node* Identity(PhaseGVN* phase);
 244 
 245   // If the load is from Field memory and the pointer is non-null, it might be possible to
 246   // zero out the control input.
 247   // If the offset is constant and the base is an object allocation,
 248   // try to hook me up to the exact initializing store.
 249   virtual Node *Ideal(PhaseGVN *phase, bool can_reshape);
 250 
 251   // Split instance field load through Phi.
 252   Node* split_through_phi(PhaseGVN *phase);
 253 
 254   // Recover original value from boxed values
 255   Node *eliminate_autobox(PhaseIterGVN *igvn);
 256 
 257   // Compute a new Type for this node.  Basically we just do the pre-check,
 258   // then call the virtual add() to set the type.
 259   virtual const Type* Value(PhaseGVN* phase) const;
 260 
 261   // Common methods for LoadKlass and LoadNKlass nodes.
 262   const Type* klass_value_common(PhaseGVN* phase) const;
 263   Node* klass_identity_common(PhaseGVN* phase);
 264 
 265   virtual uint ideal_reg() const;
 266   virtual const Type *bottom_type() const;
 267   // Following method is copied from TypeNode:
 268   void set_type(const Type* t) {
 269     assert(t != NULL, "sanity");
 270     debug_only(uint check_hash = (VerifyHashTableKeys && _hash_lock) ? hash() : NO_HASH);
 271     *(const Type**)&_type = t;   // cast away const-ness
 272     // If this node is in the hash table, make sure it doesn't need a rehash.
 273     assert(check_hash == NO_HASH || check_hash == hash(), "type change must preserve hash code");
 274   }
 275   const Type* type() const { assert(_type != NULL, "sanity"); return _type; };
 276 
 277   // Do not match memory edge
 278   virtual uint match_edge(uint idx) const;
 279 
 280   // Map a load opcode to its corresponding store opcode.
 281   virtual int store_Opcode() const = 0;
 282 
 283   // Check if the load's memory input is a Phi node with the same control.
 284   bool is_instance_field_load_with_local_phi(Node* ctrl);
 285 
 286   Node* convert_to_unsigned_load(PhaseGVN& gvn);
 287   Node* convert_to_signed_load(PhaseGVN& gvn);
 288 
 289   bool  has_reinterpret_variant(const Type* rt);
 290   Node* convert_to_reinterpret_load(PhaseGVN& gvn, const Type* rt);
 291 
 292   bool has_unknown_control_dependency() const { return _control_dependency == UnknownControl; }
 293 
 294 #ifndef PRODUCT
 295   virtual void dump_spec(outputStream *st) const;
 296 #endif
 297 #ifdef ASSERT
 298   // Helper function to allow a raw load without control edge for some cases
 299   static bool is_immutable_value(Node* adr);
 300 #endif
 301 protected:
 302   const Type* load_array_final_field(const TypeKlassPtr *tkls,
 303                                      ciKlass* klass) const;
 304 
 305   Node* can_see_arraycopy_value(Node* st, PhaseGVN* phase) const;
 306 
 307   // depends_only_on_test is almost always true, and needs to be almost always
 308   // true to enable key hoisting & commoning optimizations.  However, for the
 309   // special case of RawPtr loads from TLS top & end, and other loads performed by
 310   // GC barriers, the control edge carries the dependence preventing hoisting past
 311   // a Safepoint instead of the memory edge.  (An unfortunate consequence of having
 312   // Safepoints not set Raw Memory; itself an unfortunate consequence of having Nodes
 313   // which produce results (new raw memory state) inside of loops preventing all
 314   // manner of other optimizations).  Basically, it's ugly but so is the alternative.
 315   // See comment in macro.cpp, around line 125 expand_allocate_common().
 316   virtual bool depends_only_on_test() const {
 317     return adr_type() != TypeRawPtr::BOTTOM && _control_dependency == DependsOnlyOnTest;
 318   }
 319 };
 320 
 321 //------------------------------LoadBNode--------------------------------------
 322 // Load a byte (8bits signed) from memory
 323 class LoadBNode : public LoadNode {
 324 public:
 325   LoadBNode(Node *c, Node *mem, Node *adr, const TypePtr* at, const TypeInt *ti, MemOrd mo, ControlDependency control_dependency = DependsOnlyOnTest)
 326     : LoadNode(c, mem, adr, at, ti, mo, control_dependency) {}
 327   virtual int Opcode() const;
 328   virtual uint ideal_reg() const { return Op_RegI; }
 329   virtual Node *Ideal(PhaseGVN *phase, bool can_reshape);
 330   virtual const Type* Value(PhaseGVN* phase) const;
 331   virtual int store_Opcode() const { return Op_StoreB; }
 332   virtual BasicType memory_type() const { return T_BYTE; }
 333 };
 334 
 335 //------------------------------LoadUBNode-------------------------------------
 336 // Load a unsigned byte (8bits unsigned) from memory
 337 class LoadUBNode : public LoadNode {
 338 public:
 339   LoadUBNode(Node* c, Node* mem, Node* adr, const TypePtr* at, const TypeInt* ti, MemOrd mo, ControlDependency control_dependency = DependsOnlyOnTest)
 340     : LoadNode(c, mem, adr, at, ti, mo, control_dependency) {}
 341   virtual int Opcode() const;
 342   virtual uint ideal_reg() const { return Op_RegI; }
 343   virtual Node* Ideal(PhaseGVN *phase, bool can_reshape);
 344   virtual const Type* Value(PhaseGVN* phase) const;
 345   virtual int store_Opcode() const { return Op_StoreB; }
 346   virtual BasicType memory_type() const { return T_BYTE; }
 347 };
 348 
 349 //------------------------------LoadUSNode-------------------------------------
 350 // Load an unsigned short/char (16bits unsigned) from memory
 351 class LoadUSNode : public LoadNode {
 352 public:
 353   LoadUSNode(Node *c, Node *mem, Node *adr, const TypePtr* at, const TypeInt *ti, MemOrd mo, ControlDependency control_dependency = DependsOnlyOnTest)
 354     : LoadNode(c, mem, adr, at, ti, mo, control_dependency) {}
 355   virtual int Opcode() const;
 356   virtual uint ideal_reg() const { return Op_RegI; }
 357   virtual Node *Ideal(PhaseGVN *phase, bool can_reshape);
 358   virtual const Type* Value(PhaseGVN* phase) const;
 359   virtual int store_Opcode() const { return Op_StoreC; }
 360   virtual BasicType memory_type() const { return T_CHAR; }
 361 };
 362 
 363 //------------------------------LoadSNode--------------------------------------
 364 // Load a short (16bits signed) from memory
 365 class LoadSNode : public LoadNode {
 366 public:
 367   LoadSNode(Node *c, Node *mem, Node *adr, const TypePtr* at, const TypeInt *ti, MemOrd mo, ControlDependency control_dependency = DependsOnlyOnTest)
 368     : LoadNode(c, mem, adr, at, ti, mo, control_dependency) {}
 369   virtual int Opcode() const;
 370   virtual uint ideal_reg() const { return Op_RegI; }
 371   virtual Node *Ideal(PhaseGVN *phase, bool can_reshape);
 372   virtual const Type* Value(PhaseGVN* phase) const;
 373   virtual int store_Opcode() const { return Op_StoreC; }
 374   virtual BasicType memory_type() const { return T_SHORT; }
 375 };
 376 
 377 //------------------------------LoadINode--------------------------------------
 378 // Load an integer from memory
 379 class LoadINode : public LoadNode {
 380 public:
 381   LoadINode(Node *c, Node *mem, Node *adr, const TypePtr* at, const TypeInt *ti, MemOrd mo, ControlDependency control_dependency = DependsOnlyOnTest)
 382     : LoadNode(c, mem, adr, at, ti, mo, control_dependency) {}
 383   virtual int Opcode() const;
 384   virtual uint ideal_reg() const { return Op_RegI; }
 385   virtual int store_Opcode() const { return Op_StoreI; }
 386   virtual BasicType memory_type() const { return T_INT; }
 387 };
 388 
 389 //------------------------------LoadRangeNode----------------------------------
 390 // Load an array length from the array
 391 class LoadRangeNode : public LoadINode {
 392 public:
 393   LoadRangeNode(Node *c, Node *mem, Node *adr, const TypeInt *ti = TypeInt::POS)
 394     : LoadINode(c, mem, adr, TypeAryPtr::RANGE, ti, MemNode::unordered) {}
 395   virtual int Opcode() const;
 396   virtual const Type* Value(PhaseGVN* phase) const;
 397   virtual Node* Identity(PhaseGVN* phase);
 398   virtual Node *Ideal(PhaseGVN *phase, bool can_reshape);
 399 };
 400 
 401 //------------------------------LoadLNode--------------------------------------
 402 // Load a long from memory
 403 class LoadLNode : public LoadNode {
 404   virtual uint hash() const { return LoadNode::hash() + _require_atomic_access; }
 405   virtual bool cmp( const Node &n ) const {
 406     return _require_atomic_access == ((LoadLNode&)n)._require_atomic_access
 407       && LoadNode::cmp(n);
 408   }
 409   virtual uint size_of() const { return sizeof(*this); }
 410   const bool _require_atomic_access;  // is piecewise load forbidden?
 411 
 412 public:
 413   LoadLNode(Node *c, Node *mem, Node *adr, const TypePtr* at, const TypeLong *tl,
 414             MemOrd mo, ControlDependency control_dependency = DependsOnlyOnTest, bool require_atomic_access = false)
 415     : LoadNode(c, mem, adr, at, tl, mo, control_dependency), _require_atomic_access(require_atomic_access) {}
 416   virtual int Opcode() const;
 417   virtual uint ideal_reg() const { return Op_RegL; }
 418   virtual int store_Opcode() const { return Op_StoreL; }
 419   virtual BasicType memory_type() const { return T_LONG; }
 420   bool require_atomic_access() const { return _require_atomic_access; }
 421   static LoadLNode* make_atomic(Node* ctl, Node* mem, Node* adr, const TypePtr* adr_type,
 422                                 const Type* rt, MemOrd mo, ControlDependency control_dependency = DependsOnlyOnTest,
 423                                 bool unaligned = false, bool mismatched = false, bool unsafe = false, uint8_t barrier_data = 0);
 424 #ifndef PRODUCT
 425   virtual void dump_spec(outputStream *st) const {
 426     LoadNode::dump_spec(st);
 427     if (_require_atomic_access)  st->print(" Atomic!");
 428   }
 429 #endif
 430 };
 431 
 432 //------------------------------LoadL_unalignedNode----------------------------
 433 // Load a long from unaligned memory
 434 class LoadL_unalignedNode : public LoadLNode {
 435 public:
 436   LoadL_unalignedNode(Node *c, Node *mem, Node *adr, const TypePtr* at, MemOrd mo, ControlDependency control_dependency = DependsOnlyOnTest)
 437     : LoadLNode(c, mem, adr, at, TypeLong::LONG, mo, control_dependency) {}
 438   virtual int Opcode() const;
 439 };
 440 
 441 //------------------------------LoadFNode--------------------------------------
 442 // Load a float (64 bits) from memory
 443 class LoadFNode : public LoadNode {
 444 public:
 445   LoadFNode(Node *c, Node *mem, Node *adr, const TypePtr* at, const Type *t, MemOrd mo, ControlDependency control_dependency = DependsOnlyOnTest)
 446     : LoadNode(c, mem, adr, at, t, mo, control_dependency) {}
 447   virtual int Opcode() const;
 448   virtual uint ideal_reg() const { return Op_RegF; }
 449   virtual int store_Opcode() const { return Op_StoreF; }
 450   virtual BasicType memory_type() const { return T_FLOAT; }
 451 };
 452 
 453 //------------------------------LoadDNode--------------------------------------
 454 // Load a double (64 bits) from memory
 455 class LoadDNode : public LoadNode {
 456   virtual uint hash() const { return LoadNode::hash() + _require_atomic_access; }
 457   virtual bool cmp( const Node &n ) const {
 458     return _require_atomic_access == ((LoadDNode&)n)._require_atomic_access
 459       && LoadNode::cmp(n);
 460   }
 461   virtual uint size_of() const { return sizeof(*this); }
 462   const bool _require_atomic_access;  // is piecewise load forbidden?
 463 
 464 public:
 465   LoadDNode(Node *c, Node *mem, Node *adr, const TypePtr* at, const Type *t,
 466             MemOrd mo, ControlDependency control_dependency = DependsOnlyOnTest, bool require_atomic_access = false)
 467     : LoadNode(c, mem, adr, at, t, mo, control_dependency), _require_atomic_access(require_atomic_access) {}
 468   virtual int Opcode() const;
 469   virtual uint ideal_reg() const { return Op_RegD; }
 470   virtual int store_Opcode() const { return Op_StoreD; }
 471   virtual BasicType memory_type() const { return T_DOUBLE; }
 472   bool require_atomic_access() const { return _require_atomic_access; }
 473   static LoadDNode* make_atomic(Node* ctl, Node* mem, Node* adr, const TypePtr* adr_type,
 474                                 const Type* rt, MemOrd mo, ControlDependency control_dependency = DependsOnlyOnTest,
 475                                 bool unaligned = false, bool mismatched = false, bool unsafe = false, uint8_t barrier_data = 0);
 476 #ifndef PRODUCT
 477   virtual void dump_spec(outputStream *st) const {
 478     LoadNode::dump_spec(st);
 479     if (_require_atomic_access)  st->print(" Atomic!");
 480   }
 481 #endif
 482 };
 483 
 484 //------------------------------LoadD_unalignedNode----------------------------
 485 // Load a double from unaligned memory
 486 class LoadD_unalignedNode : public LoadDNode {
 487 public:
 488   LoadD_unalignedNode(Node *c, Node *mem, Node *adr, const TypePtr* at, MemOrd mo, ControlDependency control_dependency = DependsOnlyOnTest)
 489     : LoadDNode(c, mem, adr, at, Type::DOUBLE, mo, control_dependency) {}
 490   virtual int Opcode() const;
 491 };
 492 
 493 //------------------------------LoadPNode--------------------------------------
 494 // Load a pointer from memory (either object or array)
 495 class LoadPNode : public LoadNode {
 496 public:
 497   LoadPNode(Node *c, Node *mem, Node *adr, const TypePtr *at, const TypePtr* t, MemOrd mo, ControlDependency control_dependency = DependsOnlyOnTest)
 498     : LoadNode(c, mem, adr, at, t, mo, control_dependency) {}
 499   virtual int Opcode() const;
 500   virtual uint ideal_reg() const { return Op_RegP; }
 501   virtual int store_Opcode() const { return Op_StoreP; }
 502   virtual BasicType memory_type() const { return T_ADDRESS; }
 503 };
 504 
 505 
 506 //------------------------------LoadNNode--------------------------------------
 507 // Load a narrow oop from memory (either object or array)
 508 class LoadNNode : public LoadNode {
 509 public:
 510   LoadNNode(Node *c, Node *mem, Node *adr, const TypePtr *at, const Type* t, MemOrd mo, ControlDependency control_dependency = DependsOnlyOnTest)
 511     : LoadNode(c, mem, adr, at, t, mo, control_dependency) {}
 512   virtual int Opcode() const;
 513   virtual uint ideal_reg() const { return Op_RegN; }
 514   virtual int store_Opcode() const { return Op_StoreN; }
 515   virtual BasicType memory_type() const { return T_NARROWOOP; }
 516 };
 517 
 518 //------------------------------LoadKlassNode----------------------------------
 519 // Load a Klass from an object
 520 class LoadKlassNode : public LoadPNode {
 521 protected:
 522   // In most cases, LoadKlassNode does not have the control input set. If the control
 523   // input is set, it must not be removed (by LoadNode::Ideal()).
 524   virtual bool can_remove_control() const;
 525 public:
 526   LoadKlassNode(Node *c, Node *mem, Node *adr, const TypePtr *at, const TypeKlassPtr *tk, MemOrd mo)
 527     : LoadPNode(c, mem, adr, at, tk, mo) {}
 528   virtual int Opcode() const;
 529   virtual const Type* Value(PhaseGVN* phase) const;
 530   virtual Node* Identity(PhaseGVN* phase);
 531   virtual bool depends_only_on_test() const { return true; }
 532 
 533   // Polymorphic factory method:
 534   static Node* make(PhaseGVN& gvn, Node* ctl, Node* mem, Node* adr, const TypePtr* at,
 535                     const TypeKlassPtr* tk = TypeInstKlassPtr::OBJECT);
 536 };
 537 
 538 //------------------------------LoadNKlassNode---------------------------------
 539 // Load a narrow Klass from an object.
 540 class LoadNKlassNode : public LoadNNode {
 541 public:
 542   LoadNKlassNode(Node *c, Node *mem, Node *adr, const TypePtr *at, const TypeNarrowKlass *tk, MemOrd mo)
 543     : LoadNNode(c, mem, adr, at, tk, mo) {}
 544   virtual int Opcode() const;
 545   virtual uint ideal_reg() const { return Op_RegN; }
 546   virtual int store_Opcode() const { return Op_StoreNKlass; }
 547   virtual BasicType memory_type() const { return T_NARROWKLASS; }
 548 
 549   virtual const Type* Value(PhaseGVN* phase) const;
 550   virtual Node* Identity(PhaseGVN* phase);
 551   virtual bool depends_only_on_test() const { return true; }
 552 };
 553 

 554 //------------------------------StoreNode--------------------------------------
 555 // Store value; requires Store, Address and Value
 556 class StoreNode : public MemNode {
 557 private:
 558   // On platforms with weak memory ordering (e.g., PPC, Ia64) we distinguish
 559   // stores that can be reordered, and such requiring release semantics to
 560   // adhere to the Java specification.  The required behaviour is stored in
 561   // this field.
 562   const MemOrd _mo;
 563   // Needed for proper cloning.
 564   virtual uint size_of() const { return sizeof(*this); }
 565 protected:
 566   virtual bool cmp( const Node &n ) const;
 567   virtual bool depends_only_on_test() const { return false; }
 568 
 569   Node *Ideal_masked_input       (PhaseGVN *phase, uint mask);
 570   Node *Ideal_sign_extended_input(PhaseGVN *phase, int  num_bits);
 571 
 572 public:
 573   // We must ensure that stores of object references will be visible
 574   // only after the object's initialization. So the callers of this
 575   // procedure must indicate that the store requires `release'
 576   // semantics, if the stored value is an object reference that might
 577   // point to a new object and may become externally visible.
 578   StoreNode(Node *c, Node *mem, Node *adr, const TypePtr* at, Node *val, MemOrd mo)
 579     : MemNode(c, mem, adr, at, val), _mo(mo) {
 580     init_class_id(Class_Store);
 581   }
 582   StoreNode(Node *c, Node *mem, Node *adr, const TypePtr* at, Node *val, Node *oop_store, MemOrd mo)
 583     : MemNode(c, mem, adr, at, val, oop_store), _mo(mo) {
 584     init_class_id(Class_Store);
 585   }
 586 
 587   inline bool is_unordered() const { return !is_release(); }
 588   inline bool is_release() const {
 589     assert((_mo == unordered || _mo == release), "unexpected");
 590     return _mo == release;
 591   }
 592 
 593   // Conservatively release stores of object references in order to
 594   // ensure visibility of object initialization.
 595   static inline MemOrd release_if_reference(const BasicType t) {
 596 #ifdef AARCH64
 597     // AArch64 doesn't need a release store here because object
 598     // initialization contains the necessary barriers.
 599     return unordered;
 600 #else
 601     const MemOrd mo = (t == T_ARRAY ||
 602                        t == T_ADDRESS || // Might be the address of an object reference (`boxing').
 603                        t == T_OBJECT) ? release : unordered;
 604     return mo;
 605 #endif
 606   }
 607 
 608   // Polymorphic factory method
 609   //
 610   // We must ensure that stores of object references will be visible
 611   // only after the object's initialization. So the callers of this
 612   // procedure must indicate that the store requires `release'
 613   // semantics, if the stored value is an object reference that might
 614   // point to a new object and may become externally visible.
 615   static StoreNode* make(PhaseGVN& gvn, Node *c, Node *mem, Node *adr,
 616                          const TypePtr* at, Node *val, BasicType bt, MemOrd mo);
 617 
 618   virtual uint hash() const;    // Check the type
 619 
 620   // If the store is to Field memory and the pointer is non-null, we can
 621   // zero out the control input.
 622   virtual Node *Ideal(PhaseGVN *phase, bool can_reshape);
 623 
 624   // Compute a new Type for this node.  Basically we just do the pre-check,
 625   // then call the virtual add() to set the type.
 626   virtual const Type* Value(PhaseGVN* phase) const;
 627 
 628   // Check for identity function on memory (Load then Store at same address)
 629   virtual Node* Identity(PhaseGVN* phase);
 630 
 631   // Do not match memory edge
 632   virtual uint match_edge(uint idx) const;
 633 
 634   virtual const Type *bottom_type() const;  // returns Type::MEMORY
 635 
 636   // Map a store opcode to its corresponding own opcode, trivially.
 637   virtual int store_Opcode() const { return Opcode(); }
 638 
 639   // have all possible loads of the value stored been optimized away?
 640   bool value_never_loaded(PhaseTransform *phase) const;
 641 
 642   bool  has_reinterpret_variant(const Type* vt);
 643   Node* convert_to_reinterpret_store(PhaseGVN& gvn, Node* val, const Type* vt);
 644 
 645   MemBarNode* trailing_membar() const;
 646 };
 647 
 648 //------------------------------StoreBNode-------------------------------------
 649 // Store byte to memory
 650 class StoreBNode : public StoreNode {
 651 public:
 652   StoreBNode(Node *c, Node *mem, Node *adr, const TypePtr* at, Node *val, MemOrd mo)
 653     : StoreNode(c, mem, adr, at, val, mo) {}
 654   virtual int Opcode() const;
 655   virtual Node *Ideal(PhaseGVN *phase, bool can_reshape);
 656   virtual BasicType memory_type() const { return T_BYTE; }
 657 };
 658 
 659 //------------------------------StoreCNode-------------------------------------
 660 // Store char/short to memory
 661 class StoreCNode : public StoreNode {
 662 public:
 663   StoreCNode(Node *c, Node *mem, Node *adr, const TypePtr* at, Node *val, MemOrd mo)
 664     : StoreNode(c, mem, adr, at, val, mo) {}
 665   virtual int Opcode() const;
 666   virtual Node *Ideal(PhaseGVN *phase, bool can_reshape);
 667   virtual BasicType memory_type() const { return T_CHAR; }
 668 };
 669 
 670 //------------------------------StoreINode-------------------------------------
 671 // Store int to memory
 672 class StoreINode : public StoreNode {
 673 public:
 674   StoreINode(Node *c, Node *mem, Node *adr, const TypePtr* at, Node *val, MemOrd mo)
 675     : StoreNode(c, mem, adr, at, val, mo) {}
 676   virtual int Opcode() const;
 677   virtual BasicType memory_type() const { return T_INT; }
 678 };
 679 
 680 //------------------------------StoreLNode-------------------------------------
 681 // Store long to memory
 682 class StoreLNode : public StoreNode {
 683   virtual uint hash() const { return StoreNode::hash() + _require_atomic_access; }
 684   virtual bool cmp( const Node &n ) const {
 685     return _require_atomic_access == ((StoreLNode&)n)._require_atomic_access
 686       && StoreNode::cmp(n);
 687   }
 688   virtual uint size_of() const { return sizeof(*this); }
 689   const bool _require_atomic_access;  // is piecewise store forbidden?
 690 
 691 public:
 692   StoreLNode(Node *c, Node *mem, Node *adr, const TypePtr* at, Node *val, MemOrd mo, bool require_atomic_access = false)
 693     : StoreNode(c, mem, adr, at, val, mo), _require_atomic_access(require_atomic_access) {}
 694   virtual int Opcode() const;
 695   virtual BasicType memory_type() const { return T_LONG; }
 696   bool require_atomic_access() const { return _require_atomic_access; }
 697   static StoreLNode* make_atomic(Node* ctl, Node* mem, Node* adr, const TypePtr* adr_type, Node* val, MemOrd mo);
 698 #ifndef PRODUCT
 699   virtual void dump_spec(outputStream *st) const {
 700     StoreNode::dump_spec(st);
 701     if (_require_atomic_access)  st->print(" Atomic!");
 702   }
 703 #endif
 704 };
 705 
 706 //------------------------------StoreFNode-------------------------------------
 707 // Store float to memory
 708 class StoreFNode : public StoreNode {
 709 public:
 710   StoreFNode(Node *c, Node *mem, Node *adr, const TypePtr* at, Node *val, MemOrd mo)
 711     : StoreNode(c, mem, adr, at, val, mo) {}
 712   virtual int Opcode() const;
 713   virtual BasicType memory_type() const { return T_FLOAT; }
 714 };
 715 
 716 //------------------------------StoreDNode-------------------------------------
 717 // Store double to memory
 718 class StoreDNode : public StoreNode {
 719   virtual uint hash() const { return StoreNode::hash() + _require_atomic_access; }
 720   virtual bool cmp( const Node &n ) const {
 721     return _require_atomic_access == ((StoreDNode&)n)._require_atomic_access
 722       && StoreNode::cmp(n);
 723   }
 724   virtual uint size_of() const { return sizeof(*this); }
 725   const bool _require_atomic_access;  // is piecewise store forbidden?
 726 public:
 727   StoreDNode(Node *c, Node *mem, Node *adr, const TypePtr* at, Node *val,
 728              MemOrd mo, bool require_atomic_access = false)
 729     : StoreNode(c, mem, adr, at, val, mo), _require_atomic_access(require_atomic_access) {}
 730   virtual int Opcode() const;
 731   virtual BasicType memory_type() const { return T_DOUBLE; }
 732   bool require_atomic_access() const { return _require_atomic_access; }
 733   static StoreDNode* make_atomic(Node* ctl, Node* mem, Node* adr, const TypePtr* adr_type, Node* val, MemOrd mo);
 734 #ifndef PRODUCT
 735   virtual void dump_spec(outputStream *st) const {
 736     StoreNode::dump_spec(st);
 737     if (_require_atomic_access)  st->print(" Atomic!");
 738   }
 739 #endif
 740 
 741 };
 742 
 743 //------------------------------StorePNode-------------------------------------
 744 // Store pointer to memory
 745 class StorePNode : public StoreNode {
 746 public:
 747   StorePNode(Node *c, Node *mem, Node *adr, const TypePtr* at, Node *val, MemOrd mo)
 748     : StoreNode(c, mem, adr, at, val, mo) {}
 749   virtual int Opcode() const;
 750   virtual BasicType memory_type() const { return T_ADDRESS; }
 751 };
 752 
 753 //------------------------------StoreNNode-------------------------------------
 754 // Store narrow oop to memory
 755 class StoreNNode : public StoreNode {
 756 public:
 757   StoreNNode(Node *c, Node *mem, Node *adr, const TypePtr* at, Node *val, MemOrd mo)
 758     : StoreNode(c, mem, adr, at, val, mo) {}
 759   virtual int Opcode() const;
 760   virtual BasicType memory_type() const { return T_NARROWOOP; }
 761 };
 762 
 763 //------------------------------StoreNKlassNode--------------------------------------
 764 // Store narrow klass to memory
 765 class StoreNKlassNode : public StoreNNode {
 766 public:
 767   StoreNKlassNode(Node *c, Node *mem, Node *adr, const TypePtr* at, Node *val, MemOrd mo)
 768     : StoreNNode(c, mem, adr, at, val, mo) {}
 769   virtual int Opcode() const;
 770   virtual BasicType memory_type() const { return T_NARROWKLASS; }
 771 };
 772 
 773 //------------------------------StoreCMNode-----------------------------------
 774 // Store card-mark byte to memory for CM
 775 // The last StoreCM before a SafePoint must be preserved and occur after its "oop" store
 776 // Preceeding equivalent StoreCMs may be eliminated.
 777 class StoreCMNode : public StoreNode {
 778  private:
 779   virtual uint hash() const { return StoreNode::hash() + _oop_alias_idx; }
 780   virtual bool cmp( const Node &n ) const {
 781     return _oop_alias_idx == ((StoreCMNode&)n)._oop_alias_idx
 782       && StoreNode::cmp(n);
 783   }
 784   virtual uint size_of() const { return sizeof(*this); }
 785   int _oop_alias_idx;   // The alias_idx of OopStore
 786 
 787 public:
 788   StoreCMNode( Node *c, Node *mem, Node *adr, const TypePtr* at, Node *val, Node *oop_store, int oop_alias_idx ) :
 789     StoreNode(c, mem, adr, at, val, oop_store, MemNode::release),
 790     _oop_alias_idx(oop_alias_idx) {
 791     assert(_oop_alias_idx >= Compile::AliasIdxRaw ||
 792            _oop_alias_idx == Compile::AliasIdxBot && Compile::current()->AliasLevel() == 0,
 793            "bad oop alias idx");
 794   }
 795   virtual int Opcode() const;
 796   virtual Node* Identity(PhaseGVN* phase);
 797   virtual Node *Ideal(PhaseGVN *phase, bool can_reshape);
 798   virtual const Type* Value(PhaseGVN* phase) const;
 799   virtual BasicType memory_type() const { return T_VOID; } // unspecific
 800   int oop_alias_idx() const { return _oop_alias_idx; }
 801 };
 802 
 803 //------------------------------LoadPLockedNode---------------------------------
 804 // Load-locked a pointer from memory (either object or array).
 805 // On Sparc & Intel this is implemented as a normal pointer load.
 806 // On PowerPC and friends it's a real load-locked.
 807 class LoadPLockedNode : public LoadPNode {
 808 public:
 809   LoadPLockedNode(Node *c, Node *mem, Node *adr, MemOrd mo)
 810     : LoadPNode(c, mem, adr, TypeRawPtr::BOTTOM, TypeRawPtr::BOTTOM, mo) {}
 811   virtual int Opcode() const;
 812   virtual int store_Opcode() const { return Op_StorePConditional; }
 813   virtual bool depends_only_on_test() const { return true; }
 814 };
 815 
 816 //------------------------------SCMemProjNode---------------------------------------
 817 // This class defines a projection of the memory  state of a store conditional node.
 818 // These nodes return a value, but also update memory.
 819 class SCMemProjNode : public ProjNode {
 820 public:
 821   enum {SCMEMPROJCON = (uint)-2};
 822   SCMemProjNode( Node *src) : ProjNode( src, SCMEMPROJCON) { }
 823   virtual int Opcode() const;
 824   virtual bool      is_CFG() const  { return false; }
 825   virtual const Type *bottom_type() const {return Type::MEMORY;}
 826   virtual const TypePtr *adr_type() const {
 827     Node* ctrl = in(0);
 828     if (ctrl == NULL)  return NULL; // node is dead
 829     return ctrl->in(MemNode::Memory)->adr_type();
 830   }
 831   virtual uint ideal_reg() const { return 0;} // memory projections don't have a register
 832   virtual const Type* Value(PhaseGVN* phase) const;
 833 #ifndef PRODUCT
 834   virtual void dump_spec(outputStream *st) const {};
 835 #endif
 836 };
 837 
 838 //------------------------------LoadStoreNode---------------------------
 839 // Note: is_Mem() method returns 'true' for this class.
 840 class LoadStoreNode : public Node {
 841 private:
 842   const Type* const _type;      // What kind of value is loaded?
 843   const TypePtr* _adr_type;     // What kind of memory is being addressed?
 844   uint8_t _barrier_data;        // Bit field with barrier information
 845   virtual uint size_of() const; // Size is bigger
 846 public:
 847   LoadStoreNode( Node *c, Node *mem, Node *adr, Node *val, const TypePtr* at, const Type* rt, uint required );
 848   virtual bool depends_only_on_test() const { return false; }
 849   virtual uint match_edge(uint idx) const { return idx == MemNode::Address || idx == MemNode::ValueIn; }
 850 
 851   virtual const Type *bottom_type() const { return _type; }
 852   virtual uint ideal_reg() const;
 853   virtual const class TypePtr *adr_type() const { return _adr_type; }  // returns bottom_type of address
 854   virtual const Type* Value(PhaseGVN* phase) const;
 855 
 856   bool result_not_used() const;
 857   MemBarNode* trailing_membar() const;
 858 
 859   uint8_t barrier_data() { return _barrier_data; }
 860   void set_barrier_data(uint8_t barrier_data) { _barrier_data = barrier_data; }
 861 };
 862 
 863 class LoadStoreConditionalNode : public LoadStoreNode {
 864 public:
 865   enum {
 866     ExpectedIn = MemNode::ValueIn+1 // One more input than MemNode
 867   };
 868   LoadStoreConditionalNode(Node *c, Node *mem, Node *adr, Node *val, Node *ex);
 869   virtual const Type* Value(PhaseGVN* phase) const;
 870 };
 871 
 872 //------------------------------StorePConditionalNode---------------------------
 873 // Conditionally store pointer to memory, if no change since prior
 874 // load-locked.  Sets flags for success or failure of the store.
 875 class StorePConditionalNode : public LoadStoreConditionalNode {
 876 public:
 877   StorePConditionalNode( Node *c, Node *mem, Node *adr, Node *val, Node *ll ) : LoadStoreConditionalNode(c, mem, adr, val, ll) { }
 878   virtual int Opcode() const;
 879   // Produces flags
 880   virtual uint ideal_reg() const { return Op_RegFlags; }
 881 };
 882 
 883 //------------------------------StoreIConditionalNode---------------------------
 884 // Conditionally store int to memory, if no change since prior
 885 // load-locked.  Sets flags for success or failure of the store.
 886 class StoreIConditionalNode : public LoadStoreConditionalNode {
 887 public:
 888   StoreIConditionalNode( Node *c, Node *mem, Node *adr, Node *val, Node *ii ) : LoadStoreConditionalNode(c, mem, adr, val, ii) { }
 889   virtual int Opcode() const;
 890   // Produces flags
 891   virtual uint ideal_reg() const { return Op_RegFlags; }
 892 };
 893 
 894 //------------------------------StoreLConditionalNode---------------------------
 895 // Conditionally store long to memory, if no change since prior
 896 // load-locked.  Sets flags for success or failure of the store.
 897 class StoreLConditionalNode : public LoadStoreConditionalNode {
 898 public:
 899   StoreLConditionalNode( Node *c, Node *mem, Node *adr, Node *val, Node *ll ) : LoadStoreConditionalNode(c, mem, adr, val, ll) { }
 900   virtual int Opcode() const;
 901   // Produces flags
 902   virtual uint ideal_reg() const { return Op_RegFlags; }
 903 };
 904 
 905 class CompareAndSwapNode : public LoadStoreConditionalNode {
 906 private:
 907   const MemNode::MemOrd _mem_ord;
 908 public:
 909   CompareAndSwapNode( Node *c, Node *mem, Node *adr, Node *val, Node *ex, MemNode::MemOrd mem_ord) : LoadStoreConditionalNode(c, mem, adr, val, ex), _mem_ord(mem_ord) {}
 910   MemNode::MemOrd order() const {
 911     return _mem_ord;
 912   }
 913   virtual uint size_of() const { return sizeof(*this); }
 914 };
 915 
 916 class CompareAndExchangeNode : public LoadStoreNode {
 917 private:
 918   const MemNode::MemOrd _mem_ord;
 919 public:
 920   enum {
 921     ExpectedIn = MemNode::ValueIn+1 // One more input than MemNode
 922   };
 923   CompareAndExchangeNode( Node *c, Node *mem, Node *adr, Node *val, Node *ex, MemNode::MemOrd mem_ord, const TypePtr* at, const Type* t) :
 924     LoadStoreNode(c, mem, adr, val, at, t, 5), _mem_ord(mem_ord) {
 925      init_req(ExpectedIn, ex );
 926   }
 927 
 928   MemNode::MemOrd order() const {
 929     return _mem_ord;
 930   }
 931   virtual uint size_of() const { return sizeof(*this); }
 932 };
 933 
 934 //------------------------------CompareAndSwapBNode---------------------------
 935 class CompareAndSwapBNode : public CompareAndSwapNode {
 936 public:
 937   CompareAndSwapBNode( Node *c, Node *mem, Node *adr, Node *val, Node *ex, MemNode::MemOrd mem_ord) : CompareAndSwapNode(c, mem, adr, val, ex, mem_ord) { }
 938   virtual int Opcode() const;
 939 };
 940 
 941 //------------------------------CompareAndSwapSNode---------------------------
 942 class CompareAndSwapSNode : public CompareAndSwapNode {
 943 public:
 944   CompareAndSwapSNode( Node *c, Node *mem, Node *adr, Node *val, Node *ex, MemNode::MemOrd mem_ord) : CompareAndSwapNode(c, mem, adr, val, ex, mem_ord) { }
 945   virtual int Opcode() const;
 946 };
 947 
 948 //------------------------------CompareAndSwapINode---------------------------
 949 class CompareAndSwapINode : public CompareAndSwapNode {
 950 public:
 951   CompareAndSwapINode( Node *c, Node *mem, Node *adr, Node *val, Node *ex, MemNode::MemOrd mem_ord) : CompareAndSwapNode(c, mem, adr, val, ex, mem_ord) { }
 952   virtual int Opcode() const;
 953 };
 954 
 955 //------------------------------CompareAndSwapLNode---------------------------
 956 class CompareAndSwapLNode : public CompareAndSwapNode {
 957 public:
 958   CompareAndSwapLNode( Node *c, Node *mem, Node *adr, Node *val, Node *ex, MemNode::MemOrd mem_ord) : CompareAndSwapNode(c, mem, adr, val, ex, mem_ord) { }
 959   virtual int Opcode() const;
 960 };
 961 
 962 //------------------------------CompareAndSwapPNode---------------------------
 963 class CompareAndSwapPNode : public CompareAndSwapNode {
 964 public:
 965   CompareAndSwapPNode( Node *c, Node *mem, Node *adr, Node *val, Node *ex, MemNode::MemOrd mem_ord) : CompareAndSwapNode(c, mem, adr, val, ex, mem_ord) { }
 966   virtual int Opcode() const;
 967 };
 968 
 969 //------------------------------CompareAndSwapNNode---------------------------
 970 class CompareAndSwapNNode : public CompareAndSwapNode {
 971 public:
 972   CompareAndSwapNNode( Node *c, Node *mem, Node *adr, Node *val, Node *ex, MemNode::MemOrd mem_ord) : CompareAndSwapNode(c, mem, adr, val, ex, mem_ord) { }
 973   virtual int Opcode() const;
 974 };
 975 
 976 //------------------------------WeakCompareAndSwapBNode---------------------------
 977 class WeakCompareAndSwapBNode : public CompareAndSwapNode {
 978 public:
 979   WeakCompareAndSwapBNode( Node *c, Node *mem, Node *adr, Node *val, Node *ex, MemNode::MemOrd mem_ord) : CompareAndSwapNode(c, mem, adr, val, ex, mem_ord) { }
 980   virtual int Opcode() const;
 981 };
 982 
 983 //------------------------------WeakCompareAndSwapSNode---------------------------
 984 class WeakCompareAndSwapSNode : public CompareAndSwapNode {
 985 public:
 986   WeakCompareAndSwapSNode( Node *c, Node *mem, Node *adr, Node *val, Node *ex, MemNode::MemOrd mem_ord) : CompareAndSwapNode(c, mem, adr, val, ex, mem_ord) { }
 987   virtual int Opcode() const;
 988 };
 989 
 990 //------------------------------WeakCompareAndSwapINode---------------------------
 991 class WeakCompareAndSwapINode : public CompareAndSwapNode {
 992 public:
 993   WeakCompareAndSwapINode( Node *c, Node *mem, Node *adr, Node *val, Node *ex, MemNode::MemOrd mem_ord) : CompareAndSwapNode(c, mem, adr, val, ex, mem_ord) { }
 994   virtual int Opcode() const;
 995 };
 996 
 997 //------------------------------WeakCompareAndSwapLNode---------------------------
 998 class WeakCompareAndSwapLNode : public CompareAndSwapNode {
 999 public:
1000   WeakCompareAndSwapLNode( Node *c, Node *mem, Node *adr, Node *val, Node *ex, MemNode::MemOrd mem_ord) : CompareAndSwapNode(c, mem, adr, val, ex, mem_ord) { }
1001   virtual int Opcode() const;
1002 };
1003 
1004 //------------------------------WeakCompareAndSwapPNode---------------------------
1005 class WeakCompareAndSwapPNode : public CompareAndSwapNode {
1006 public:
1007   WeakCompareAndSwapPNode( Node *c, Node *mem, Node *adr, Node *val, Node *ex, MemNode::MemOrd mem_ord) : CompareAndSwapNode(c, mem, adr, val, ex, mem_ord) { }
1008   virtual int Opcode() const;
1009 };
1010 
1011 //------------------------------WeakCompareAndSwapNNode---------------------------
1012 class WeakCompareAndSwapNNode : public CompareAndSwapNode {
1013 public:
1014   WeakCompareAndSwapNNode( Node *c, Node *mem, Node *adr, Node *val, Node *ex, MemNode::MemOrd mem_ord) : CompareAndSwapNode(c, mem, adr, val, ex, mem_ord) { }
1015   virtual int Opcode() const;
1016 };
1017 
1018 //------------------------------CompareAndExchangeBNode---------------------------
1019 class CompareAndExchangeBNode : public CompareAndExchangeNode {
1020 public:
1021   CompareAndExchangeBNode( Node *c, Node *mem, Node *adr, Node *val, Node *ex, const TypePtr* at, MemNode::MemOrd mem_ord) : CompareAndExchangeNode(c, mem, adr, val, ex, mem_ord, at, TypeInt::BYTE) { }
1022   virtual int Opcode() const;
1023 };
1024 
1025 
1026 //------------------------------CompareAndExchangeSNode---------------------------
1027 class CompareAndExchangeSNode : public CompareAndExchangeNode {
1028 public:
1029   CompareAndExchangeSNode( Node *c, Node *mem, Node *adr, Node *val, Node *ex, const TypePtr* at, MemNode::MemOrd mem_ord) : CompareAndExchangeNode(c, mem, adr, val, ex, mem_ord, at, TypeInt::SHORT) { }
1030   virtual int Opcode() const;
1031 };
1032 
1033 //------------------------------CompareAndExchangeLNode---------------------------
1034 class CompareAndExchangeLNode : public CompareAndExchangeNode {
1035 public:
1036   CompareAndExchangeLNode( Node *c, Node *mem, Node *adr, Node *val, Node *ex, const TypePtr* at, MemNode::MemOrd mem_ord) : CompareAndExchangeNode(c, mem, adr, val, ex, mem_ord, at, TypeLong::LONG) { }
1037   virtual int Opcode() const;
1038 };
1039 
1040 
1041 //------------------------------CompareAndExchangeINode---------------------------
1042 class CompareAndExchangeINode : public CompareAndExchangeNode {
1043 public:
1044   CompareAndExchangeINode( Node *c, Node *mem, Node *adr, Node *val, Node *ex, const TypePtr* at, MemNode::MemOrd mem_ord) : CompareAndExchangeNode(c, mem, adr, val, ex, mem_ord, at, TypeInt::INT) { }
1045   virtual int Opcode() const;
1046 };
1047 
1048 
1049 //------------------------------CompareAndExchangePNode---------------------------
1050 class CompareAndExchangePNode : public CompareAndExchangeNode {
1051 public:
1052   CompareAndExchangePNode( Node *c, Node *mem, Node *adr, Node *val, Node *ex, const TypePtr* at, const Type* t, MemNode::MemOrd mem_ord) : CompareAndExchangeNode(c, mem, adr, val, ex, mem_ord, at, t) { }
1053   virtual int Opcode() const;
1054 };
1055 
1056 //------------------------------CompareAndExchangeNNode---------------------------
1057 class CompareAndExchangeNNode : public CompareAndExchangeNode {
1058 public:
1059   CompareAndExchangeNNode( Node *c, Node *mem, Node *adr, Node *val, Node *ex, const TypePtr* at, const Type* t, MemNode::MemOrd mem_ord) : CompareAndExchangeNode(c, mem, adr, val, ex, mem_ord, at, t) { }
1060   virtual int Opcode() const;
1061 };
1062 
1063 //------------------------------GetAndAddBNode---------------------------
1064 class GetAndAddBNode : public LoadStoreNode {
1065 public:
1066   GetAndAddBNode( Node *c, Node *mem, Node *adr, Node *val, const TypePtr* at ) : LoadStoreNode(c, mem, adr, val, at, TypeInt::BYTE, 4) { }
1067   virtual int Opcode() const;
1068 };
1069 
1070 //------------------------------GetAndAddSNode---------------------------
1071 class GetAndAddSNode : public LoadStoreNode {
1072 public:
1073   GetAndAddSNode( Node *c, Node *mem, Node *adr, Node *val, const TypePtr* at ) : LoadStoreNode(c, mem, adr, val, at, TypeInt::SHORT, 4) { }
1074   virtual int Opcode() const;
1075 };
1076 
1077 //------------------------------GetAndAddINode---------------------------
1078 class GetAndAddINode : public LoadStoreNode {
1079 public:
1080   GetAndAddINode( Node *c, Node *mem, Node *adr, Node *val, const TypePtr* at ) : LoadStoreNode(c, mem, adr, val, at, TypeInt::INT, 4) { }
1081   virtual int Opcode() const;
1082 };
1083 
1084 //------------------------------GetAndAddLNode---------------------------
1085 class GetAndAddLNode : public LoadStoreNode {
1086 public:
1087   GetAndAddLNode( Node *c, Node *mem, Node *adr, Node *val, const TypePtr* at ) : LoadStoreNode(c, mem, adr, val, at, TypeLong::LONG, 4) { }
1088   virtual int Opcode() const;
1089 };
1090 
1091 //------------------------------GetAndSetBNode---------------------------
1092 class GetAndSetBNode : public LoadStoreNode {
1093 public:
1094   GetAndSetBNode( Node *c, Node *mem, Node *adr, Node *val, const TypePtr* at ) : LoadStoreNode(c, mem, adr, val, at, TypeInt::BYTE, 4) { }
1095   virtual int Opcode() const;
1096 };
1097 
1098 //------------------------------GetAndSetSNode---------------------------
1099 class GetAndSetSNode : public LoadStoreNode {
1100 public:
1101   GetAndSetSNode( Node *c, Node *mem, Node *adr, Node *val, const TypePtr* at ) : LoadStoreNode(c, mem, adr, val, at, TypeInt::SHORT, 4) { }
1102   virtual int Opcode() const;
1103 };
1104 
1105 //------------------------------GetAndSetINode---------------------------
1106 class GetAndSetINode : public LoadStoreNode {
1107 public:
1108   GetAndSetINode( Node *c, Node *mem, Node *adr, Node *val, const TypePtr* at ) : LoadStoreNode(c, mem, adr, val, at, TypeInt::INT, 4) { }
1109   virtual int Opcode() const;
1110 };
1111 
1112 //------------------------------GetAndSetLNode---------------------------
1113 class GetAndSetLNode : public LoadStoreNode {
1114 public:
1115   GetAndSetLNode( Node *c, Node *mem, Node *adr, Node *val, const TypePtr* at ) : LoadStoreNode(c, mem, adr, val, at, TypeLong::LONG, 4) { }
1116   virtual int Opcode() const;
1117 };
1118 
1119 //------------------------------GetAndSetPNode---------------------------
1120 class GetAndSetPNode : public LoadStoreNode {
1121 public:
1122   GetAndSetPNode( Node *c, Node *mem, Node *adr, Node *val, const TypePtr* at, const Type* t ) : LoadStoreNode(c, mem, adr, val, at, t, 4) { }
1123   virtual int Opcode() const;
1124 };
1125 
1126 //------------------------------GetAndSetNNode---------------------------
1127 class GetAndSetNNode : public LoadStoreNode {
1128 public:
1129   GetAndSetNNode( Node *c, Node *mem, Node *adr, Node *val, const TypePtr* at, const Type* t ) : LoadStoreNode(c, mem, adr, val, at, t, 4) { }
1130   virtual int Opcode() const;
1131 };
1132 
1133 //------------------------------ClearArray-------------------------------------
1134 class ClearArrayNode: public Node {
1135 private:
1136   bool _is_large;
1137   bool _word_copy_only;
1138 public:
1139   ClearArrayNode( Node *ctrl, Node *arymem, Node *word_cnt, Node *base, Node* val, bool is_large)
1140     : Node(ctrl, arymem, word_cnt, base, val), _is_large(is_large),
1141       _word_copy_only(val->bottom_type()->isa_long() && (!val->bottom_type()->is_long()->is_con() || val->bottom_type()->is_long()->get_con() != 0)) {
1142     init_class_id(Class_ClearArray);
1143   }
1144   virtual int         Opcode() const;
1145   virtual const Type *bottom_type() const { return Type::MEMORY; }
1146   // ClearArray modifies array elements, and so affects only the
1147   // array memory addressed by the bottom_type of its base address.
1148   virtual const class TypePtr *adr_type() const;
1149   virtual Node* Identity(PhaseGVN* phase);
1150   virtual Node *Ideal(PhaseGVN *phase, bool can_reshape);
1151   virtual uint match_edge(uint idx) const;
1152   bool is_large() const { return _is_large; }
1153   bool word_copy_only() const { return _word_copy_only; }
1154 
1155   // Clear the given area of an object or array.
1156   // The start offset must always be aligned mod BytesPerInt.
1157   // The end offset must always be aligned mod BytesPerLong.
1158   // Return the new memory.
1159   static Node* clear_memory(Node* control, Node* mem, Node* dest,
1160                             Node* val,
1161                             Node* raw_val,
1162                             intptr_t start_offset,
1163                             intptr_t end_offset,
1164                             PhaseGVN* phase);
1165   static Node* clear_memory(Node* control, Node* mem, Node* dest,
1166                             Node* val,
1167                             Node* raw_val,
1168                             intptr_t start_offset,
1169                             Node* end_offset,
1170                             PhaseGVN* phase);
1171   static Node* clear_memory(Node* control, Node* mem, Node* dest,
1172                             Node* raw_val,
1173                             Node* start_offset,
1174                             Node* end_offset,
1175                             PhaseGVN* phase);
1176   // Return allocation input memory edge if it is different instance
1177   // or itself if it is the one we are looking for.
1178   static bool step_through(Node** np, uint instance_id, PhaseTransform* phase);
1179 };
1180 
1181 //------------------------------MemBar-----------------------------------------
1182 // There are different flavors of Memory Barriers to match the Java Memory
1183 // Model.  Monitor-enter and volatile-load act as Aquires: no following ref
1184 // can be moved to before them.  We insert a MemBar-Acquire after a FastLock or
1185 // volatile-load.  Monitor-exit and volatile-store act as Release: no
1186 // preceding ref can be moved to after them.  We insert a MemBar-Release
1187 // before a FastUnlock or volatile-store.  All volatiles need to be
1188 // serialized, so we follow all volatile-stores with a MemBar-Volatile to
1189 // separate it from any following volatile-load.
1190 class MemBarNode: public MultiNode {
1191   virtual uint hash() const ;                  // { return NO_HASH; }
1192   virtual bool cmp( const Node &n ) const ;    // Always fail, except on self
1193 
1194   virtual uint size_of() const { return sizeof(*this); }
1195   // Memory type this node is serializing.  Usually either rawptr or bottom.
1196   const TypePtr* _adr_type;
1197 
1198   // How is this membar related to a nearby memory access?
1199   enum {
1200     Standalone,
1201     TrailingLoad,
1202     TrailingStore,
1203     LeadingStore,
1204     TrailingLoadStore,
1205     LeadingLoadStore,
1206     TrailingPartialArrayCopy
1207   } _kind;
1208 
1209 #ifdef ASSERT
1210   uint _pair_idx;
1211 #endif
1212 
1213 public:
1214   enum {
1215     Precedent = TypeFunc::Parms  // optional edge to force precedence
1216   };
1217   MemBarNode(Compile* C, int alias_idx, Node* precedent);
1218   virtual int Opcode() const = 0;
1219   virtual const class TypePtr *adr_type() const { return _adr_type; }
1220   virtual const Type* Value(PhaseGVN* phase) const;
1221   virtual Node *Ideal(PhaseGVN *phase, bool can_reshape);
1222   virtual uint match_edge(uint idx) const { return 0; }
1223   virtual const Type *bottom_type() const { return TypeTuple::MEMBAR; }
1224   virtual Node *match(const ProjNode *proj, const Matcher *m, const RegMask* mask);
1225   // Factory method.  Builds a wide or narrow membar.
1226   // Optional 'precedent' becomes an extra edge if not null.
1227   static MemBarNode* make(Compile* C, int opcode,
1228                           int alias_idx = Compile::AliasIdxBot,
1229                           Node* precedent = NULL);
1230 
1231   MemBarNode* trailing_membar() const;
1232   MemBarNode* leading_membar() const;
1233 
1234   void set_trailing_load() { _kind = TrailingLoad; }
1235   bool trailing_load() const { return _kind == TrailingLoad; }
1236   bool trailing_store() const { return _kind == TrailingStore; }
1237   bool leading_store() const { return _kind == LeadingStore; }
1238   bool trailing_load_store() const { return _kind == TrailingLoadStore; }
1239   bool leading_load_store() const { return _kind == LeadingLoadStore; }
1240   bool trailing() const { return _kind == TrailingLoad || _kind == TrailingStore || _kind == TrailingLoadStore; }
1241   bool leading() const { return _kind == LeadingStore || _kind == LeadingLoadStore; }
1242   bool standalone() const { return _kind == Standalone; }
1243   void set_trailing_partial_array_copy() { _kind = TrailingPartialArrayCopy; }
1244   bool trailing_partial_array_copy() const { return _kind == TrailingPartialArrayCopy; }
1245 
1246   static void set_store_pair(MemBarNode* leading, MemBarNode* trailing);
1247   static void set_load_store_pair(MemBarNode* leading, MemBarNode* trailing);
1248 
1249   void remove(PhaseIterGVN *igvn);
1250 };
1251 
1252 // "Acquire" - no following ref can move before (but earlier refs can
1253 // follow, like an early Load stalled in cache).  Requires multi-cpu
1254 // visibility.  Inserted after a volatile load.
1255 class MemBarAcquireNode: public MemBarNode {
1256 public:
1257   MemBarAcquireNode(Compile* C, int alias_idx, Node* precedent)
1258     : MemBarNode(C, alias_idx, precedent) {}
1259   virtual int Opcode() const;
1260 };
1261 
1262 // "Acquire" - no following ref can move before (but earlier refs can
1263 // follow, like an early Load stalled in cache).  Requires multi-cpu
1264 // visibility.  Inserted independ of any load, as required
1265 // for intrinsic Unsafe.loadFence().
1266 class LoadFenceNode: public MemBarNode {
1267 public:
1268   LoadFenceNode(Compile* C, int alias_idx, Node* precedent)
1269     : MemBarNode(C, alias_idx, precedent) {}
1270   virtual int Opcode() const;
1271 };
1272 
1273 // "Release" - no earlier ref can move after (but later refs can move
1274 // up, like a speculative pipelined cache-hitting Load).  Requires
1275 // multi-cpu visibility.  Inserted before a volatile store.
1276 class MemBarReleaseNode: public MemBarNode {
1277 public:
1278   MemBarReleaseNode(Compile* C, int alias_idx, Node* precedent)
1279     : MemBarNode(C, alias_idx, precedent) {}
1280   virtual int Opcode() const;
1281 };
1282 
1283 // "Release" - no earlier ref can move after (but later refs can move
1284 // up, like a speculative pipelined cache-hitting Load).  Requires
1285 // multi-cpu visibility.  Inserted independent of any store, as required
1286 // for intrinsic Unsafe.storeFence().
1287 class StoreFenceNode: public MemBarNode {
1288 public:
1289   StoreFenceNode(Compile* C, int alias_idx, Node* precedent)
1290     : MemBarNode(C, alias_idx, precedent) {}
1291   virtual int Opcode() const;
1292 };
1293 
1294 // "Acquire" - no following ref can move before (but earlier refs can
1295 // follow, like an early Load stalled in cache).  Requires multi-cpu
1296 // visibility.  Inserted after a FastLock.
1297 class MemBarAcquireLockNode: public MemBarNode {
1298 public:
1299   MemBarAcquireLockNode(Compile* C, int alias_idx, Node* precedent)
1300     : MemBarNode(C, alias_idx, precedent) {}
1301   virtual int Opcode() const;
1302 };
1303 
1304 // "Release" - no earlier ref can move after (but later refs can move
1305 // up, like a speculative pipelined cache-hitting Load).  Requires
1306 // multi-cpu visibility.  Inserted before a FastUnLock.
1307 class MemBarReleaseLockNode: public MemBarNode {
1308 public:
1309   MemBarReleaseLockNode(Compile* C, int alias_idx, Node* precedent)
1310     : MemBarNode(C, alias_idx, precedent) {}
1311   virtual int Opcode() const;
1312 };
1313 
1314 class MemBarStoreStoreNode: public MemBarNode {
1315 public:
1316   MemBarStoreStoreNode(Compile* C, int alias_idx, Node* precedent)
1317     : MemBarNode(C, alias_idx, precedent) {
1318     init_class_id(Class_MemBarStoreStore);
1319   }
1320   virtual int Opcode() const;
1321 };
1322 
1323 class StoreStoreFenceNode: public MemBarNode {
1324 public:
1325   StoreStoreFenceNode(Compile* C, int alias_idx, Node* precedent)
1326     : MemBarNode(C, alias_idx, precedent) {}
1327   virtual int Opcode() const;
1328 };
1329 
1330 // Ordering between a volatile store and a following volatile load.
1331 // Requires multi-CPU visibility?
1332 class MemBarVolatileNode: public MemBarNode {
1333 public:
1334   MemBarVolatileNode(Compile* C, int alias_idx, Node* precedent)
1335     : MemBarNode(C, alias_idx, precedent) {}
1336   virtual int Opcode() const;
1337 };
1338 
1339 // Ordering within the same CPU.  Used to order unsafe memory references
1340 // inside the compiler when we lack alias info.  Not needed "outside" the
1341 // compiler because the CPU does all the ordering for us.
1342 class MemBarCPUOrderNode: public MemBarNode {
1343 public:
1344   MemBarCPUOrderNode(Compile* C, int alias_idx, Node* precedent)
1345     : MemBarNode(C, alias_idx, precedent) {}
1346   virtual int Opcode() const;
1347   virtual uint ideal_reg() const { return 0; } // not matched in the AD file
1348 };
1349 
1350 class OnSpinWaitNode: public MemBarNode {
1351 public:
1352   OnSpinWaitNode(Compile* C, int alias_idx, Node* precedent)
1353     : MemBarNode(C, alias_idx, precedent) {}
1354   virtual int Opcode() const;
1355 };
1356 
1357 //------------------------------BlackholeNode----------------------------
1358 // Blackhole all arguments. This node would survive through the compiler
1359 // the effects on its arguments, and would be finally matched to nothing.
1360 class BlackholeNode : public MemBarNode {
1361 public:
1362   BlackholeNode(Compile* C, int alias_idx, Node* precedent)
1363     : MemBarNode(C, alias_idx, precedent) {
1364     init_class_id(Class_Blackhole);
1365   }
1366   virtual int   Opcode() const;
1367   virtual uint ideal_reg() const { return 0; } // not matched in the AD file
1368   const RegMask &in_RegMask(uint idx) const {
1369     // Fake the incoming arguments mask for blackholes: accept all registers
1370     // and all stack slots. This would avoid any redundant register moves
1371     // for blackhole inputs.
1372     return RegMask::All;
1373   }
1374 #ifndef PRODUCT
1375   virtual void format(PhaseRegAlloc* ra, outputStream* st) const;
1376 #endif
1377 };
1378 
1379 // Isolation of object setup after an AllocateNode and before next safepoint.
1380 // (See comment in memnode.cpp near InitializeNode::InitializeNode for semantics.)
1381 class InitializeNode: public MemBarNode {
1382   friend class AllocateNode;
1383 
1384   enum {
1385     Incomplete    = 0,
1386     Complete      = 1,
1387     WithArraycopy = 2
1388   };
1389   int _is_complete;
1390 
1391   bool _does_not_escape;
1392 
1393 public:
1394   enum {
1395     Control    = TypeFunc::Control,
1396     Memory     = TypeFunc::Memory,     // MergeMem for states affected by this op
1397     RawAddress = TypeFunc::Parms+0,    // the newly-allocated raw address
1398     RawStores  = TypeFunc::Parms+1     // zero or more stores (or TOP)
1399   };
1400 
1401   InitializeNode(Compile* C, int adr_type, Node* rawoop);
1402   virtual int Opcode() const;
1403   virtual uint size_of() const { return sizeof(*this); }
1404   virtual uint ideal_reg() const { return 0; } // not matched in the AD file
1405   virtual const RegMask &in_RegMask(uint) const;  // mask for RawAddress
1406 
1407   // Manage incoming memory edges via a MergeMem on in(Memory):
1408   Node* memory(uint alias_idx);
1409 
1410   // The raw memory edge coming directly from the Allocation.
1411   // The contents of this memory are *always* all-zero-bits.
1412   Node* zero_memory() { return memory(Compile::AliasIdxRaw); }
1413 
1414   // Return the corresponding allocation for this initialization (or null if none).
1415   // (Note: Both InitializeNode::allocation and AllocateNode::initialization
1416   // are defined in graphKit.cpp, which sets up the bidirectional relation.)
1417   AllocateNode* allocation();
1418 
1419   // Anything other than zeroing in this init?
1420   bool is_non_zero();
1421 
1422   // An InitializeNode must completed before macro expansion is done.
1423   // Completion requires that the AllocateNode must be followed by
1424   // initialization of the new memory to zero, then to any initializers.
1425   bool is_complete() { return _is_complete != Incomplete; }
1426   bool is_complete_with_arraycopy() { return (_is_complete & WithArraycopy) != 0; }
1427 
1428   // Mark complete.  (Must not yet be complete.)
1429   void set_complete(PhaseGVN* phase);
1430   void set_complete_with_arraycopy() { _is_complete = Complete | WithArraycopy; }
1431 
1432   bool does_not_escape() { return _does_not_escape; }
1433   void set_does_not_escape() { _does_not_escape = true; }
1434 
1435 #ifdef ASSERT
1436   // ensure all non-degenerate stores are ordered and non-overlapping
1437   bool stores_are_sane(PhaseTransform* phase);
1438 #endif //ASSERT
1439 
1440   // See if this store can be captured; return offset where it initializes.
1441   // Return 0 if the store cannot be moved (any sort of problem).
1442   intptr_t can_capture_store(StoreNode* st, PhaseGVN* phase, bool can_reshape);
1443 
1444   // Capture another store; reformat it to write my internal raw memory.
1445   // Return the captured copy, else NULL if there is some sort of problem.
1446   Node* capture_store(StoreNode* st, intptr_t start, PhaseGVN* phase, bool can_reshape);
1447 
1448   // Find captured store which corresponds to the range [start..start+size).
1449   // Return my own memory projection (meaning the initial zero bits)
1450   // if there is no such store.  Return NULL if there is a problem.
1451   Node* find_captured_store(intptr_t start, int size_in_bytes, PhaseTransform* phase);
1452 
1453   // Called when the associated AllocateNode is expanded into CFG.
1454   Node* complete_stores(Node* rawctl, Node* rawmem, Node* rawptr,
1455                         intptr_t header_size, Node* size_in_bytes,
1456                         PhaseIterGVN* phase);
1457 
1458  private:
1459   void remove_extra_zeroes();
1460 
1461   // Find out where a captured store should be placed (or already is placed).
1462   int captured_store_insertion_point(intptr_t start, int size_in_bytes,
1463                                      PhaseTransform* phase);
1464 
1465   static intptr_t get_store_offset(Node* st, PhaseTransform* phase);
1466 
1467   Node* make_raw_address(intptr_t offset, PhaseTransform* phase);
1468 
1469   bool detect_init_independence(Node* value, PhaseGVN* phase);
1470 
1471   void coalesce_subword_stores(intptr_t header_size, Node* size_in_bytes,
1472                                PhaseGVN* phase);
1473 
1474   intptr_t find_next_fullword_store(uint i, PhaseGVN* phase);
1475 };
1476 
1477 //------------------------------MergeMem---------------------------------------
1478 // (See comment in memnode.cpp near MergeMemNode::MergeMemNode for semantics.)
1479 class MergeMemNode: public Node {
1480   virtual uint hash() const ;                  // { return NO_HASH; }
1481   virtual bool cmp( const Node &n ) const ;    // Always fail, except on self
1482   friend class MergeMemStream;
1483   MergeMemNode(Node* def);  // clients use MergeMemNode::make
1484 
1485 public:
1486   // If the input is a whole memory state, clone it with all its slices intact.
1487   // Otherwise, make a new memory state with just that base memory input.
1488   // In either case, the result is a newly created MergeMem.
1489   static MergeMemNode* make(Node* base_memory);
1490 
1491   virtual int Opcode() const;
1492   virtual Node* Identity(PhaseGVN* phase);
1493   virtual Node *Ideal(PhaseGVN *phase, bool can_reshape);
1494   virtual uint ideal_reg() const { return NotAMachineReg; }
1495   virtual uint match_edge(uint idx) const { return 0; }
1496   virtual const RegMask &out_RegMask() const;
1497   virtual const Type *bottom_type() const { return Type::MEMORY; }
1498   virtual const TypePtr *adr_type() const { return TypePtr::BOTTOM; }
1499   // sparse accessors
1500   // Fetch the previously stored "set_memory_at", or else the base memory.
1501   // (Caller should clone it if it is a phi-nest.)
1502   Node* memory_at(uint alias_idx) const;
1503   // set the memory, regardless of its previous value
1504   void set_memory_at(uint alias_idx, Node* n);
1505   // the "base" is the memory that provides the non-finite support
1506   Node* base_memory() const       { return in(Compile::AliasIdxBot); }
1507   // warning: setting the base can implicitly set any of the other slices too
1508   void set_base_memory(Node* def);
1509   // sentinel value which denotes a copy of the base memory:
1510   Node*   empty_memory() const    { return in(Compile::AliasIdxTop); }
1511   static Node* make_empty_memory(); // where the sentinel comes from
1512   bool is_empty_memory(Node* n) const { assert((n == empty_memory()) == n->is_top(), "sanity"); return n->is_top(); }
1513   // hook for the iterator, to perform any necessary setup
1514   void iteration_setup(const MergeMemNode* other = NULL);
1515   // push sentinels until I am at least as long as the other (semantic no-op)
1516   void grow_to_match(const MergeMemNode* other);
1517   bool verify_sparse() const PRODUCT_RETURN0;
1518 #ifndef PRODUCT
1519   virtual void dump_spec(outputStream *st) const;
1520 #endif
1521 };
1522 
1523 class MergeMemStream : public StackObj {
1524  private:
1525   MergeMemNode*       _mm;
1526   const MergeMemNode* _mm2;  // optional second guy, contributes non-empty iterations
1527   Node*               _mm_base;  // loop-invariant base memory of _mm
1528   int                 _idx;
1529   int                 _cnt;
1530   Node*               _mem;
1531   Node*               _mem2;
1532   int                 _cnt2;
1533 
1534   void init(MergeMemNode* mm, const MergeMemNode* mm2 = NULL) {
1535     // subsume_node will break sparseness at times, whenever a memory slice
1536     // folds down to a copy of the base ("fat") memory.  In such a case,
1537     // the raw edge will update to base, although it should be top.
1538     // This iterator will recognize either top or base_memory as an
1539     // "empty" slice.  See is_empty, is_empty2, and next below.
1540     //
1541     // The sparseness property is repaired in MergeMemNode::Ideal.
1542     // As long as access to a MergeMem goes through this iterator
1543     // or the memory_at accessor, flaws in the sparseness will
1544     // never be observed.
1545     //
1546     // Also, iteration_setup repairs sparseness.
1547     assert(mm->verify_sparse(), "please, no dups of base");
1548     assert(mm2==NULL || mm2->verify_sparse(), "please, no dups of base");
1549 
1550     _mm  = mm;
1551     _mm_base = mm->base_memory();
1552     _mm2 = mm2;
1553     _cnt = mm->req();
1554     _idx = Compile::AliasIdxBot-1; // start at the base memory
1555     _mem = NULL;
1556     _mem2 = NULL;
1557   }
1558 
1559 #ifdef ASSERT
1560   Node* check_memory() const {
1561     if (at_base_memory())
1562       return _mm->base_memory();
1563     else if ((uint)_idx < _mm->req() && !_mm->in(_idx)->is_top())
1564       return _mm->memory_at(_idx);
1565     else
1566       return _mm_base;
1567   }
1568   Node* check_memory2() const {
1569     return at_base_memory()? _mm2->base_memory(): _mm2->memory_at(_idx);
1570   }
1571 #endif
1572 
1573   static bool match_memory(Node* mem, const MergeMemNode* mm, int idx) PRODUCT_RETURN0;
1574   void assert_synch() const {
1575     assert(!_mem || _idx >= _cnt || match_memory(_mem, _mm, _idx),
1576            "no side-effects except through the stream");
1577   }
1578 
1579  public:
1580 
1581   // expected usages:
1582   // for (MergeMemStream mms(mem->is_MergeMem()); next_non_empty(); ) { ... }
1583   // for (MergeMemStream mms(mem1, mem2); next_non_empty2(); ) { ... }
1584 
1585   // iterate over one merge
1586   MergeMemStream(MergeMemNode* mm) {
1587     mm->iteration_setup();
1588     init(mm);
1589     debug_only(_cnt2 = 999);
1590   }
1591   // iterate in parallel over two merges
1592   // only iterates through non-empty elements of mm2
1593   MergeMemStream(MergeMemNode* mm, const MergeMemNode* mm2) {
1594     assert(mm2, "second argument must be a MergeMem also");
1595     ((MergeMemNode*)mm2)->iteration_setup();  // update hidden state
1596     mm->iteration_setup(mm2);
1597     init(mm, mm2);
1598     _cnt2 = mm2->req();
1599   }
1600 #ifdef ASSERT
1601   ~MergeMemStream() {
1602     assert_synch();
1603   }
1604 #endif
1605 
1606   MergeMemNode* all_memory() const {
1607     return _mm;
1608   }
1609   Node* base_memory() const {
1610     assert(_mm_base == _mm->base_memory(), "no update to base memory, please");
1611     return _mm_base;
1612   }
1613   const MergeMemNode* all_memory2() const {
1614     assert(_mm2 != NULL, "");
1615     return _mm2;
1616   }
1617   bool at_base_memory() const {
1618     return _idx == Compile::AliasIdxBot;
1619   }
1620   int alias_idx() const {
1621     assert(_mem, "must call next 1st");
1622     return _idx;
1623   }
1624 
1625   const TypePtr* adr_type() const {
1626     return Compile::current()->get_adr_type(alias_idx());
1627   }
1628 
1629   const TypePtr* adr_type(Compile* C) const {
1630     return C->get_adr_type(alias_idx());
1631   }
1632   bool is_empty() const {
1633     assert(_mem, "must call next 1st");
1634     assert(_mem->is_top() == (_mem==_mm->empty_memory()), "correct sentinel");
1635     return _mem->is_top();
1636   }
1637   bool is_empty2() const {
1638     assert(_mem2, "must call next 1st");
1639     assert(_mem2->is_top() == (_mem2==_mm2->empty_memory()), "correct sentinel");
1640     return _mem2->is_top();
1641   }
1642   Node* memory() const {
1643     assert(!is_empty(), "must not be empty");
1644     assert_synch();
1645     return _mem;
1646   }
1647   // get the current memory, regardless of empty or non-empty status
1648   Node* force_memory() const {
1649     assert(!is_empty() || !at_base_memory(), "");
1650     // Use _mm_base to defend against updates to _mem->base_memory().
1651     Node *mem = _mem->is_top() ? _mm_base : _mem;
1652     assert(mem == check_memory(), "");
1653     return mem;
1654   }
1655   Node* memory2() const {
1656     assert(_mem2 == check_memory2(), "");
1657     return _mem2;
1658   }
1659   void set_memory(Node* mem) {
1660     if (at_base_memory()) {
1661       // Note that this does not change the invariant _mm_base.
1662       _mm->set_base_memory(mem);
1663     } else {
1664       _mm->set_memory_at(_idx, mem);
1665     }
1666     _mem = mem;
1667     assert_synch();
1668   }
1669 
1670   // Recover from a side effect to the MergeMemNode.
1671   void set_memory() {
1672     _mem = _mm->in(_idx);
1673   }
1674 
1675   bool next()  { return next(false); }
1676   bool next2() { return next(true); }
1677 
1678   bool next_non_empty()  { return next_non_empty(false); }
1679   bool next_non_empty2() { return next_non_empty(true); }
1680   // next_non_empty2 can yield states where is_empty() is true
1681 
1682  private:
1683   // find the next item, which might be empty
1684   bool next(bool have_mm2) {
1685     assert((_mm2 != NULL) == have_mm2, "use other next");
1686     assert_synch();
1687     if (++_idx < _cnt) {
1688       // Note:  This iterator allows _mm to be non-sparse.
1689       // It behaves the same whether _mem is top or base_memory.
1690       _mem = _mm->in(_idx);
1691       if (have_mm2)
1692         _mem2 = _mm2->in((_idx < _cnt2) ? _idx : Compile::AliasIdxTop);
1693       return true;
1694     }
1695     return false;
1696   }
1697 
1698   // find the next non-empty item
1699   bool next_non_empty(bool have_mm2) {
1700     while (next(have_mm2)) {
1701       if (!is_empty()) {
1702         // make sure _mem2 is filled in sensibly
1703         if (have_mm2 && _mem2->is_top())  _mem2 = _mm2->base_memory();
1704         return true;
1705       } else if (have_mm2 && !is_empty2()) {
1706         return true;   // is_empty() == true
1707       }
1708     }
1709     return false;
1710   }
1711 };
1712 
1713 // cachewb node for guaranteeing writeback of the cache line at a
1714 // given address to (non-volatile) RAM
1715 class CacheWBNode : public Node {
1716 public:
1717   CacheWBNode(Node *ctrl, Node *mem, Node *addr) : Node(ctrl, mem, addr) {}
1718   virtual int Opcode() const;
1719   virtual uint ideal_reg() const { return NotAMachineReg; }
1720   virtual uint match_edge(uint idx) const { return (idx == 2); }
1721   virtual const TypePtr *adr_type() const { return TypePtr::BOTTOM; }
1722   virtual const Type *bottom_type() const { return Type::MEMORY; }
1723 };
1724 
1725 // cachewb pre sync node for ensuring that writebacks are serialised
1726 // relative to preceding or following stores
1727 class CacheWBPreSyncNode : public Node {
1728 public:
1729   CacheWBPreSyncNode(Node *ctrl, Node *mem) : Node(ctrl, mem) {}
1730   virtual int Opcode() const;
1731   virtual uint ideal_reg() const { return NotAMachineReg; }
1732   virtual uint match_edge(uint idx) const { return false; }
1733   virtual const TypePtr *adr_type() const { return TypePtr::BOTTOM; }
1734   virtual const Type *bottom_type() const { return Type::MEMORY; }
1735 };
1736 
1737 // cachewb pre sync node for ensuring that writebacks are serialised
1738 // relative to preceding or following stores
1739 class CacheWBPostSyncNode : public Node {
1740 public:
1741   CacheWBPostSyncNode(Node *ctrl, Node *mem) : Node(ctrl, mem) {}
1742   virtual int Opcode() const;
1743   virtual uint ideal_reg() const { return NotAMachineReg; }
1744   virtual uint match_edge(uint idx) const { return false; }
1745   virtual const TypePtr *adr_type() const { return TypePtr::BOTTOM; }
1746   virtual const Type *bottom_type() const { return Type::MEMORY; }
1747 };
1748 
1749 //------------------------------Prefetch---------------------------------------
1750 
1751 // Allocation prefetch which may fault, TLAB size have to be adjusted.
1752 class PrefetchAllocationNode : public Node {
1753 public:
1754   PrefetchAllocationNode(Node *mem, Node *adr) : Node(0,mem,adr) {}
1755   virtual int Opcode() const;
1756   virtual uint ideal_reg() const { return NotAMachineReg; }
1757   virtual uint match_edge(uint idx) const { return idx==2; }
1758   virtual const Type *bottom_type() const { return ( AllocatePrefetchStyle == 3 ) ? Type::MEMORY : Type::ABIO; }
1759 };
1760 
1761 #endif // SHARE_OPTO_MEMNODE_HPP
--- EOF ---