1 /* 2 * Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved. 3 * Copyright (c) 2024, Alibaba Group Holding Limited. All rights reserved. 4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 5 * 6 * This code is free software; you can redistribute it and/or modify it 7 * under the terms of the GNU General Public License version 2 only, as 8 * published by the Free Software Foundation. 9 * 10 * This code is distributed in the hope that it will be useful, but WITHOUT 11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 13 * version 2 for more details (a copy is included in the LICENSE file that 14 * accompanied this code). 15 * 16 * You should have received a copy of the GNU General Public License version 17 * 2 along with this work; if not, write to the Free Software Foundation, 18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 19 * 20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 21 * or visit www.oracle.com if you need additional information or have any 22 * questions. 23 * 24 */ 25 26 #ifndef SHARE_OPTO_MEMNODE_HPP 27 #define SHARE_OPTO_MEMNODE_HPP 28 29 #include "opto/multnode.hpp" 30 #include "opto/node.hpp" 31 #include "opto/opcodes.hpp" 32 #include "opto/type.hpp" 33 34 // Portions of code courtesy of Clifford Click 35 36 class MultiNode; 37 class PhaseCCP; 38 class PhaseTransform; 39 40 //------------------------------MemNode---------------------------------------- 41 // Load or Store, possibly throwing a null pointer exception 42 class MemNode : public Node { 43 private: 44 bool _unaligned_access; // Unaligned access from unsafe 45 bool _mismatched_access; // Mismatched access from unsafe: byte read in integer array for instance 46 bool _unsafe_access; // Access of unsafe origin. 47 uint8_t _barrier_data; // Bit field with barrier information 48 49 protected: 50 #ifdef ASSERT 51 const TypePtr* _adr_type; // What kind of memory is being addressed? 52 #endif 53 virtual uint size_of() const; 54 public: 55 enum { Control, // When is it safe to do this load? 56 Memory, // Chunk of memory is being loaded from 57 Address, // Actually address, derived from base 58 ValueIn // Value to store 59 }; 60 typedef enum { unordered = 0, 61 acquire, // Load has to acquire or be succeeded by MemBarAcquire. 62 release, // Store has to release or be preceded by MemBarRelease. 63 seqcst, // LoadStore has to have both acquire and release semantics. 64 unset // The memory ordering is not set (used for testing) 65 } MemOrd; 66 protected: 67 MemNode( Node *c0, Node *c1, Node *c2, const TypePtr* at ) : 68 Node(c0,c1,c2), 69 _unaligned_access(false), 70 _mismatched_access(false), 71 _unsafe_access(false), 72 _barrier_data(0) { 73 init_class_id(Class_Mem); 74 DEBUG_ONLY(_adr_type=at; adr_type();) 75 } 76 MemNode( Node *c0, Node *c1, Node *c2, const TypePtr* at, Node *c3 ) : 77 Node(c0,c1,c2,c3), 78 _unaligned_access(false), 79 _mismatched_access(false), 80 _unsafe_access(false), 81 _barrier_data(0) { 82 init_class_id(Class_Mem); 83 DEBUG_ONLY(_adr_type=at; adr_type();) 84 } 85 MemNode( Node *c0, Node *c1, Node *c2, const TypePtr* at, Node *c3, Node *c4) : 86 Node(c0,c1,c2,c3,c4), 87 _unaligned_access(false), 88 _mismatched_access(false), 89 _unsafe_access(false), 90 _barrier_data(0) { 91 init_class_id(Class_Mem); 92 DEBUG_ONLY(_adr_type=at; adr_type();) 93 } 94 95 virtual Node* find_previous_arraycopy(PhaseValues* phase, Node* ld_alloc, Node*& mem, bool can_see_stored_value) const { return nullptr; } 96 ArrayCopyNode* find_array_copy_clone(Node* ld_alloc, Node* mem) const; 97 static bool check_if_adr_maybe_raw(Node* adr); 98 99 public: 100 // Helpers for the optimizer. Documented in memnode.cpp. 101 static bool detect_ptr_independence(Node* p1, AllocateNode* a1, 102 Node* p2, AllocateNode* a2, 103 PhaseTransform* phase); 104 static bool adr_phi_is_loop_invariant(Node* adr_phi, Node* cast); 105 106 static Node *optimize_simple_memory_chain(Node *mchain, const TypeOopPtr *t_oop, Node *load, PhaseGVN *phase); 107 static Node *optimize_memory_chain(Node *mchain, const TypePtr *t_adr, Node *load, PhaseGVN *phase); 108 // The following two should probably be phase-specific functions: 109 static DomResult maybe_all_controls_dominate(Node* dom, Node* sub); 110 static bool all_controls_dominate(Node* dom, Node* sub) { 111 DomResult dom_result = maybe_all_controls_dominate(dom, sub); 112 return dom_result == DomResult::Dominate; 113 } 114 115 virtual const class TypePtr *adr_type() const; // returns bottom_type of address 116 117 // Shared code for Ideal methods: 118 Node *Ideal_common(PhaseGVN *phase, bool can_reshape); // Return -1 for short-circuit null. 119 120 // Helper function for adr_type() implementations. 121 static const TypePtr* calculate_adr_type(const Type* t, const TypePtr* cross_check = nullptr); 122 123 // Raw access function, to allow copying of adr_type efficiently in 124 // product builds and retain the debug info for debug builds. 125 const TypePtr *raw_adr_type() const { 126 return DEBUG_ONLY(_adr_type) NOT_DEBUG(nullptr); 127 } 128 129 #ifdef ASSERT 130 void set_adr_type(const TypePtr* adr_type) { _adr_type = adr_type; } 131 #endif 132 133 // Return the barrier data of n, if available, or 0 otherwise. 134 static uint8_t barrier_data(const Node* n); 135 136 // Map a load or store opcode to its corresponding store opcode. 137 // (Return -1 if unknown.) 138 virtual int store_Opcode() const { return -1; } 139 140 // What is the type of the value in memory? (T_VOID mean "unspecified".) 141 // The returned type is a property of the value that is loaded/stored and 142 // not the memory that is accessed. For mismatched memory accesses 143 // they might differ. For instance, a value of type 'short' may be stored 144 // into an array of elements of type 'long'. 145 virtual BasicType value_basic_type() const = 0; 146 virtual int memory_size() const { 147 #ifdef ASSERT 148 return type2aelembytes(value_basic_type(), true); 149 #else 150 return type2aelembytes(value_basic_type()); 151 #endif 152 } 153 154 uint8_t barrier_data() { return _barrier_data; } 155 void set_barrier_data(uint8_t barrier_data) { _barrier_data = barrier_data; } 156 157 // Search through memory states which precede this node (load or store). 158 // Look for an exact match for the address, with no intervening 159 // aliased stores. 160 Node* find_previous_store(PhaseValues* phase); 161 162 // Can this node (load or store) accurately see a stored value in 163 // the given memory state? (The state may or may not be in(Memory).) 164 Node* can_see_stored_value(Node* st, PhaseValues* phase) const; 165 166 void set_unaligned_access() { _unaligned_access = true; } 167 bool is_unaligned_access() const { return _unaligned_access; } 168 void set_mismatched_access() { _mismatched_access = true; } 169 bool is_mismatched_access() const { return _mismatched_access; } 170 void set_unsafe_access() { _unsafe_access = true; } 171 bool is_unsafe_access() const { return _unsafe_access; } 172 173 #ifndef PRODUCT 174 static void dump_adr_type(const Node* mem, const TypePtr* adr_type, outputStream *st); 175 virtual void dump_spec(outputStream *st) const; 176 #endif 177 }; 178 179 //------------------------------LoadNode--------------------------------------- 180 // Load value; requires Memory and Address 181 class LoadNode : public MemNode { 182 public: 183 // Some loads (from unsafe) should be pinned: they don't depend only 184 // on the dominating test. The field _control_dependency below records 185 // whether that node depends only on the dominating test. 186 // Pinned and UnknownControl are similar, but differ in that Pinned 187 // loads are not allowed to float across safepoints, whereas UnknownControl 188 // loads are allowed to do that. Therefore, Pinned is stricter. 189 enum ControlDependency { 190 Pinned, 191 UnknownControl, 192 DependsOnlyOnTest 193 }; 194 195 private: 196 // LoadNode::hash() doesn't take the _control_dependency field 197 // into account: If the graph already has a non-pinned LoadNode and 198 // we add a pinned LoadNode with the same inputs, it's safe for GVN 199 // to replace the pinned LoadNode with the non-pinned LoadNode, 200 // otherwise it wouldn't be safe to have a non pinned LoadNode with 201 // those inputs in the first place. If the graph already has a 202 // pinned LoadNode and we add a non pinned LoadNode with the same 203 // inputs, it's safe (but suboptimal) for GVN to replace the 204 // non-pinned LoadNode by the pinned LoadNode. 205 ControlDependency _control_dependency; 206 207 // On platforms with weak memory ordering (e.g., PPC) we distinguish 208 // loads that can be reordered, and such requiring acquire semantics to 209 // adhere to the Java specification. The required behaviour is stored in 210 // this field. 211 const MemOrd _mo; 212 213 AllocateNode* is_new_object_mark_load() const; 214 215 protected: 216 virtual bool cmp(const Node &n) const; 217 virtual uint size_of() const; // Size is bigger 218 // Should LoadNode::Ideal() attempt to remove control edges? 219 virtual bool can_remove_control() const; 220 const Type* const _type; // What kind of value is loaded? 221 222 virtual Node* find_previous_arraycopy(PhaseValues* phase, Node* ld_alloc, Node*& mem, bool can_see_stored_value) const; 223 public: 224 225 LoadNode(Node *c, Node *mem, Node *adr, const TypePtr* at, const Type *rt, MemOrd mo, ControlDependency control_dependency) 226 : MemNode(c,mem,adr,at), _control_dependency(control_dependency), _mo(mo), _type(rt) { 227 init_class_id(Class_Load); 228 } 229 inline bool is_unordered() const { return !is_acquire(); } 230 inline bool is_acquire() const { 231 assert(_mo == unordered || _mo == acquire, "unexpected"); 232 return _mo == acquire; 233 } 234 inline bool is_unsigned() const { 235 int lop = Opcode(); 236 return (lop == Op_LoadUB) || (lop == Op_LoadUS); 237 } 238 239 // Polymorphic factory method: 240 static Node* make(PhaseGVN& gvn, Node* c, Node* mem, Node* adr, 241 const TypePtr* at, const Type* rt, BasicType bt, 242 MemOrd mo, ControlDependency control_dependency = DependsOnlyOnTest, 243 bool require_atomic_access = false, bool unaligned = false, bool mismatched = false, bool unsafe = false, 244 uint8_t barrier_data = 0); 245 246 virtual uint hash() const; // Check the type 247 248 // Handle algebraic identities here. If we have an identity, return the Node 249 // we are equivalent to. We look for Load of a Store. 250 virtual Node* Identity(PhaseGVN* phase); 251 252 // If the load is from Field memory and the pointer is non-null, it might be possible to 253 // zero out the control input. 254 // If the offset is constant and the base is an object allocation, 255 // try to hook me up to the exact initializing store. 256 virtual Node *Ideal(PhaseGVN *phase, bool can_reshape); 257 258 // Return true if it's possible to split the Load through a Phi merging the bases 259 bool can_split_through_phi_base(PhaseGVN *phase); 260 261 // Split instance field load through Phi. 262 Node* split_through_phi(PhaseGVN *phase, bool ignore_missing_instance_id = false); 263 264 // Recover original value from boxed values 265 Node *eliminate_autobox(PhaseIterGVN *igvn); 266 267 // Compute a new Type for this node. Basically we just do the pre-check, 268 // then call the virtual add() to set the type. 269 virtual const Type* Value(PhaseGVN* phase) const; 270 271 // Common methods for LoadKlass and LoadNKlass nodes. 272 const Type* klass_value_common(PhaseGVN* phase, bool fold_for_arrays) const; 273 Node* klass_identity_common(PhaseGVN* phase); 274 275 virtual uint ideal_reg() const; 276 virtual const Type *bottom_type() const; 277 // Following method is copied from TypeNode: 278 void set_type(const Type* t) { 279 assert(t != nullptr, "sanity"); 280 DEBUG_ONLY(uint check_hash = (VerifyHashTableKeys && _hash_lock) ? hash() : NO_HASH); 281 *(const Type**)&_type = t; // cast away const-ness 282 // If this node is in the hash table, make sure it doesn't need a rehash. 283 assert(check_hash == NO_HASH || check_hash == hash(), "type change must preserve hash code"); 284 } 285 const Type* type() const { assert(_type != nullptr, "sanity"); return _type; }; 286 287 // Do not match memory edge 288 virtual uint match_edge(uint idx) const; 289 290 // Map a load opcode to its corresponding store opcode. 291 virtual int store_Opcode() const = 0; 292 293 // Check if the load's memory input is a Phi node with the same control. 294 bool is_instance_field_load_with_local_phi(Node* ctrl); 295 296 Node* convert_to_unsigned_load(PhaseGVN& gvn); 297 Node* convert_to_signed_load(PhaseGVN& gvn); 298 299 bool has_reinterpret_variant(const Type* rt); 300 Node* convert_to_reinterpret_load(PhaseGVN& gvn, const Type* rt); 301 302 ControlDependency control_dependency() const { return _control_dependency; } 303 bool has_unknown_control_dependency() const { return _control_dependency == UnknownControl; } 304 bool has_pinned_control_dependency() const { return _control_dependency == Pinned; } 305 306 LoadNode* pin_array_access_node() const; 307 308 #ifndef PRODUCT 309 virtual void dump_spec(outputStream *st) const; 310 #endif 311 #ifdef ASSERT 312 // Helper function to allow a raw load without control edge for some cases 313 static bool is_immutable_value(Node* adr); 314 #endif 315 protected: 316 const Type* load_array_final_field(const TypeKlassPtr *tkls, 317 ciKlass* klass) const; 318 319 Node* can_see_arraycopy_value(Node* st, PhaseGVN* phase) const; 320 321 // depends_only_on_test is almost always true, and needs to be almost always 322 // true to enable key hoisting & commoning optimizations. However, for the 323 // special case of RawPtr loads from TLS top & end, and other loads performed by 324 // GC barriers, the control edge carries the dependence preventing hoisting past 325 // a Safepoint instead of the memory edge. (An unfortunate consequence of having 326 // Safepoints not set Raw Memory; itself an unfortunate consequence of having Nodes 327 // which produce results (new raw memory state) inside of loops preventing all 328 // manner of other optimizations). Basically, it's ugly but so is the alternative. 329 // See comment in macro.cpp, around line 125 expand_allocate_common(). 330 virtual bool depends_only_on_test() const { 331 return adr_type() != TypeRawPtr::BOTTOM && _control_dependency == DependsOnlyOnTest; 332 } 333 334 LoadNode* clone_pinned() const; 335 }; 336 337 //------------------------------LoadBNode-------------------------------------- 338 // Load a byte (8bits signed) from memory 339 class LoadBNode : public LoadNode { 340 public: 341 LoadBNode(Node *c, Node *mem, Node *adr, const TypePtr* at, const TypeInt *ti, MemOrd mo, ControlDependency control_dependency = DependsOnlyOnTest) 342 : LoadNode(c, mem, adr, at, ti, mo, control_dependency) {} 343 virtual int Opcode() const; 344 virtual uint ideal_reg() const { return Op_RegI; } 345 virtual Node *Ideal(PhaseGVN *phase, bool can_reshape); 346 virtual const Type* Value(PhaseGVN* phase) const; 347 virtual int store_Opcode() const { return Op_StoreB; } 348 virtual BasicType value_basic_type() const { return T_BYTE; } 349 }; 350 351 //------------------------------LoadUBNode------------------------------------- 352 // Load a unsigned byte (8bits unsigned) from memory 353 class LoadUBNode : public LoadNode { 354 public: 355 LoadUBNode(Node* c, Node* mem, Node* adr, const TypePtr* at, const TypeInt* ti, MemOrd mo, ControlDependency control_dependency = DependsOnlyOnTest) 356 : LoadNode(c, mem, adr, at, ti, mo, control_dependency) {} 357 virtual int Opcode() const; 358 virtual uint ideal_reg() const { return Op_RegI; } 359 virtual Node* Ideal(PhaseGVN *phase, bool can_reshape); 360 virtual const Type* Value(PhaseGVN* phase) const; 361 virtual int store_Opcode() const { return Op_StoreB; } 362 virtual BasicType value_basic_type() const { return T_BYTE; } 363 }; 364 365 //------------------------------LoadUSNode------------------------------------- 366 // Load an unsigned short/char (16bits unsigned) from memory 367 class LoadUSNode : public LoadNode { 368 public: 369 LoadUSNode(Node *c, Node *mem, Node *adr, const TypePtr* at, const TypeInt *ti, MemOrd mo, ControlDependency control_dependency = DependsOnlyOnTest) 370 : LoadNode(c, mem, adr, at, ti, mo, control_dependency) {} 371 virtual int Opcode() const; 372 virtual uint ideal_reg() const { return Op_RegI; } 373 virtual Node *Ideal(PhaseGVN *phase, bool can_reshape); 374 virtual const Type* Value(PhaseGVN* phase) const; 375 virtual int store_Opcode() const { return Op_StoreC; } 376 virtual BasicType value_basic_type() const { return T_CHAR; } 377 }; 378 379 //------------------------------LoadSNode-------------------------------------- 380 // Load a short (16bits signed) from memory 381 class LoadSNode : public LoadNode { 382 public: 383 LoadSNode(Node *c, Node *mem, Node *adr, const TypePtr* at, const TypeInt *ti, MemOrd mo, ControlDependency control_dependency = DependsOnlyOnTest) 384 : LoadNode(c, mem, adr, at, ti, mo, control_dependency) {} 385 virtual int Opcode() const; 386 virtual uint ideal_reg() const { return Op_RegI; } 387 virtual Node *Ideal(PhaseGVN *phase, bool can_reshape); 388 virtual const Type* Value(PhaseGVN* phase) const; 389 virtual int store_Opcode() const { return Op_StoreC; } 390 virtual BasicType value_basic_type() const { return T_SHORT; } 391 }; 392 393 //------------------------------LoadINode-------------------------------------- 394 // Load an integer from memory 395 class LoadINode : public LoadNode { 396 public: 397 LoadINode(Node *c, Node *mem, Node *adr, const TypePtr* at, const TypeInt *ti, MemOrd mo, ControlDependency control_dependency = DependsOnlyOnTest) 398 : LoadNode(c, mem, adr, at, ti, mo, control_dependency) {} 399 virtual int Opcode() const; 400 virtual uint ideal_reg() const { return Op_RegI; } 401 virtual int store_Opcode() const { return Op_StoreI; } 402 virtual BasicType value_basic_type() const { return T_INT; } 403 }; 404 405 //------------------------------LoadRangeNode---------------------------------- 406 // Load an array length from the array 407 class LoadRangeNode : public LoadINode { 408 public: 409 LoadRangeNode(Node *c, Node *mem, Node *adr, const TypeInt *ti = TypeInt::POS) 410 : LoadINode(c, mem, adr, TypeAryPtr::RANGE, ti, MemNode::unordered) {} 411 virtual int Opcode() const; 412 virtual const Type* Value(PhaseGVN* phase) const; 413 virtual Node* Identity(PhaseGVN* phase); 414 virtual Node *Ideal(PhaseGVN *phase, bool can_reshape); 415 }; 416 417 //------------------------------LoadLNode-------------------------------------- 418 // Load a long from memory 419 class LoadLNode : public LoadNode { 420 virtual uint hash() const { return LoadNode::hash() + _require_atomic_access; } 421 virtual bool cmp( const Node &n ) const { 422 return _require_atomic_access == ((LoadLNode&)n)._require_atomic_access 423 && LoadNode::cmp(n); 424 } 425 virtual uint size_of() const { return sizeof(*this); } 426 const bool _require_atomic_access; // is piecewise load forbidden? 427 428 public: 429 LoadLNode(Node *c, Node *mem, Node *adr, const TypePtr* at, const TypeLong *tl, 430 MemOrd mo, ControlDependency control_dependency = DependsOnlyOnTest, bool require_atomic_access = false) 431 : LoadNode(c, mem, adr, at, tl, mo, control_dependency), _require_atomic_access(require_atomic_access) {} 432 virtual int Opcode() const; 433 virtual uint ideal_reg() const { return Op_RegL; } 434 virtual int store_Opcode() const { return Op_StoreL; } 435 virtual BasicType value_basic_type() const { return T_LONG; } 436 bool require_atomic_access() const { return _require_atomic_access; } 437 438 #ifndef PRODUCT 439 virtual void dump_spec(outputStream *st) const { 440 LoadNode::dump_spec(st); 441 if (_require_atomic_access) st->print(" Atomic!"); 442 } 443 #endif 444 }; 445 446 //------------------------------LoadL_unalignedNode---------------------------- 447 // Load a long from unaligned memory 448 class LoadL_unalignedNode : public LoadLNode { 449 public: 450 LoadL_unalignedNode(Node *c, Node *mem, Node *adr, const TypePtr* at, MemOrd mo, ControlDependency control_dependency = DependsOnlyOnTest) 451 : LoadLNode(c, mem, adr, at, TypeLong::LONG, mo, control_dependency) {} 452 virtual int Opcode() const; 453 }; 454 455 //------------------------------LoadFNode-------------------------------------- 456 // Load a float (64 bits) from memory 457 class LoadFNode : public LoadNode { 458 public: 459 LoadFNode(Node *c, Node *mem, Node *adr, const TypePtr* at, const Type *t, MemOrd mo, ControlDependency control_dependency = DependsOnlyOnTest) 460 : LoadNode(c, mem, adr, at, t, mo, control_dependency) {} 461 virtual int Opcode() const; 462 virtual uint ideal_reg() const { return Op_RegF; } 463 virtual int store_Opcode() const { return Op_StoreF; } 464 virtual BasicType value_basic_type() const { return T_FLOAT; } 465 }; 466 467 //------------------------------LoadDNode-------------------------------------- 468 // Load a double (64 bits) from memory 469 class LoadDNode : public LoadNode { 470 virtual uint hash() const { return LoadNode::hash() + _require_atomic_access; } 471 virtual bool cmp( const Node &n ) const { 472 return _require_atomic_access == ((LoadDNode&)n)._require_atomic_access 473 && LoadNode::cmp(n); 474 } 475 virtual uint size_of() const { return sizeof(*this); } 476 const bool _require_atomic_access; // is piecewise load forbidden? 477 478 public: 479 LoadDNode(Node *c, Node *mem, Node *adr, const TypePtr* at, const Type *t, 480 MemOrd mo, ControlDependency control_dependency = DependsOnlyOnTest, bool require_atomic_access = false) 481 : LoadNode(c, mem, adr, at, t, mo, control_dependency), _require_atomic_access(require_atomic_access) {} 482 virtual int Opcode() const; 483 virtual uint ideal_reg() const { return Op_RegD; } 484 virtual int store_Opcode() const { return Op_StoreD; } 485 virtual BasicType value_basic_type() const { return T_DOUBLE; } 486 bool require_atomic_access() const { return _require_atomic_access; } 487 488 #ifndef PRODUCT 489 virtual void dump_spec(outputStream *st) const { 490 LoadNode::dump_spec(st); 491 if (_require_atomic_access) st->print(" Atomic!"); 492 } 493 #endif 494 }; 495 496 //------------------------------LoadD_unalignedNode---------------------------- 497 // Load a double from unaligned memory 498 class LoadD_unalignedNode : public LoadDNode { 499 public: 500 LoadD_unalignedNode(Node *c, Node *mem, Node *adr, const TypePtr* at, MemOrd mo, ControlDependency control_dependency = DependsOnlyOnTest) 501 : LoadDNode(c, mem, adr, at, Type::DOUBLE, mo, control_dependency) {} 502 virtual int Opcode() const; 503 }; 504 505 //------------------------------LoadPNode-------------------------------------- 506 // Load a pointer from memory (either object or array) 507 class LoadPNode : public LoadNode { 508 public: 509 LoadPNode(Node *c, Node *mem, Node *adr, const TypePtr *at, const TypePtr* t, MemOrd mo, ControlDependency control_dependency = DependsOnlyOnTest) 510 : LoadNode(c, mem, adr, at, t, mo, control_dependency) {} 511 virtual int Opcode() const; 512 virtual uint ideal_reg() const { return Op_RegP; } 513 virtual int store_Opcode() const { return Op_StoreP; } 514 virtual BasicType value_basic_type() const { return T_ADDRESS; } 515 }; 516 517 518 //------------------------------LoadNNode-------------------------------------- 519 // Load a narrow oop from memory (either object or array) 520 class LoadNNode : public LoadNode { 521 public: 522 LoadNNode(Node *c, Node *mem, Node *adr, const TypePtr *at, const Type* t, MemOrd mo, ControlDependency control_dependency = DependsOnlyOnTest) 523 : LoadNode(c, mem, adr, at, t, mo, control_dependency) {} 524 virtual Node* Ideal(PhaseGVN* phase, bool can_reshape); 525 virtual int Opcode() const; 526 virtual uint ideal_reg() const { return Op_RegN; } 527 virtual int store_Opcode() const { return Op_StoreN; } 528 virtual BasicType value_basic_type() const { return T_NARROWOOP; } 529 }; 530 531 //------------------------------LoadKlassNode---------------------------------- 532 // Load a Klass from an object 533 class LoadKlassNode : public LoadPNode { 534 bool _fold_for_arrays; 535 536 virtual uint size_of() const { return sizeof(*this); } 537 virtual uint hash() const { return LoadNode::hash() + _fold_for_arrays; } 538 virtual bool cmp( const Node &n ) const { 539 return _fold_for_arrays == ((LoadKlassNode&)n)._fold_for_arrays && LoadNode::cmp(n); 540 } 541 542 private: 543 LoadKlassNode(Node* mem, Node* adr, const TypePtr* at, const TypeKlassPtr* tk, MemOrd mo, bool fold_for_arrays) 544 : LoadPNode(nullptr, mem, adr, at, tk, mo), _fold_for_arrays(fold_for_arrays) {} 545 546 public: 547 virtual int Opcode() const; 548 virtual const Type* Value(PhaseGVN* phase) const; 549 virtual Node* Identity(PhaseGVN* phase); 550 virtual bool depends_only_on_test() const { return true; } 551 552 // Polymorphic factory method: 553 static Node* make(PhaseGVN& gvn, Node* mem, Node* adr, const TypePtr* at, 554 const TypeKlassPtr* tk = TypeInstKlassPtr::OBJECT, bool fold_for_arrays = true); 555 }; 556 557 //------------------------------LoadNKlassNode--------------------------------- 558 // Load a narrow Klass from an object. 559 // With compact headers, the input address (adr) does not point at the exact 560 // header position where the (narrow) class pointer is located, but into the 561 // middle of the mark word (see oopDesc::klass_offset_in_bytes()). This node 562 // implicitly shifts the loaded value (markWord::klass_shift_at_offset bits) to 563 // extract the actual class pointer. C2's type system is agnostic on whether the 564 // input address directly points into the class pointer. 565 class LoadNKlassNode : public LoadNNode { 566 bool _fold_for_arrays; 567 568 virtual uint size_of() const { return sizeof(*this); } 569 virtual uint hash() const { return LoadNode::hash() + _fold_for_arrays; } 570 virtual bool cmp( const Node &n ) const { 571 return _fold_for_arrays == ((LoadNKlassNode&)n)._fold_for_arrays && LoadNode::cmp(n); 572 } 573 574 private: 575 friend Node* LoadKlassNode::make(PhaseGVN&, Node*, Node*, const TypePtr*, const TypeKlassPtr*, bool fold_for_arrays); 576 LoadNKlassNode(Node* mem, Node* adr, const TypePtr* at, const TypeNarrowKlass* tk, MemOrd mo, bool fold_for_arrays) 577 : LoadNNode(nullptr, mem, adr, at, tk, mo), _fold_for_arrays(fold_for_arrays) {} 578 579 public: 580 virtual int Opcode() const; 581 virtual uint ideal_reg() const { return Op_RegN; } 582 virtual int store_Opcode() const { return Op_StoreNKlass; } 583 virtual BasicType value_basic_type() const { return T_NARROWKLASS; } 584 585 virtual const Type* Value(PhaseGVN* phase) const; 586 virtual Node* Identity(PhaseGVN* phase); 587 virtual bool depends_only_on_test() const { return true; } 588 }; 589 590 //------------------------------StoreNode-------------------------------------- 591 // Store value; requires Store, Address and Value 592 class StoreNode : public MemNode { 593 private: 594 // On platforms with weak memory ordering (e.g., PPC) we distinguish 595 // stores that can be reordered, and such requiring release semantics to 596 // adhere to the Java specification. The required behaviour is stored in 597 // this field. 598 const MemOrd _mo; 599 // Needed for proper cloning. 600 virtual uint size_of() const { return sizeof(*this); } 601 protected: 602 virtual bool cmp( const Node &n ) const; 603 virtual bool depends_only_on_test() const { return false; } 604 605 Node *Ideal_masked_input (PhaseGVN *phase, uint mask); 606 Node* Ideal_sign_extended_input(PhaseGVN* phase, int num_rejected_bits); 607 608 public: 609 // We must ensure that stores of object references will be visible 610 // only after the object's initialization. So the callers of this 611 // procedure must indicate that the store requires `release' 612 // semantics, if the stored value is an object reference that might 613 // point to a new object and may become externally visible. 614 StoreNode(Node *c, Node *mem, Node *adr, const TypePtr* at, Node *val, MemOrd mo) 615 : MemNode(c, mem, adr, at, val), _mo(mo) { 616 init_class_id(Class_Store); 617 } 618 StoreNode(Node *c, Node *mem, Node *adr, const TypePtr* at, Node *val, Node *oop_store, MemOrd mo) 619 : MemNode(c, mem, adr, at, val, oop_store), _mo(mo) { 620 init_class_id(Class_Store); 621 } 622 623 inline bool is_unordered() const { return !is_release(); } 624 inline bool is_release() const { 625 assert((_mo == unordered || _mo == release), "unexpected"); 626 return _mo == release; 627 } 628 629 // Conservatively release stores of object references in order to 630 // ensure visibility of object initialization. 631 static inline MemOrd release_if_reference(const BasicType t) { 632 #ifdef AARCH64 633 // AArch64 doesn't need a release store here because object 634 // initialization contains the necessary barriers. 635 return unordered; 636 #else 637 const MemOrd mo = (t == T_ARRAY || 638 t == T_ADDRESS || // Might be the address of an object reference (`boxing'). 639 t == T_OBJECT) ? release : unordered; 640 return mo; 641 #endif 642 } 643 644 // Polymorphic factory method 645 // 646 // We must ensure that stores of object references will be visible 647 // only after the object's initialization. So the callers of this 648 // procedure must indicate that the store requires `release' 649 // semantics, if the stored value is an object reference that might 650 // point to a new object and may become externally visible. 651 static StoreNode* make(PhaseGVN& gvn, Node* c, Node* mem, Node* adr, 652 const TypePtr* at, Node* val, BasicType bt, 653 MemOrd mo, bool require_atomic_access = false); 654 655 virtual uint hash() const; // Check the type 656 657 // If the store is to Field memory and the pointer is non-null, we can 658 // zero out the control input. 659 virtual Node *Ideal(PhaseGVN *phase, bool can_reshape); 660 661 // Compute a new Type for this node. Basically we just do the pre-check, 662 // then call the virtual add() to set the type. 663 virtual const Type* Value(PhaseGVN* phase) const; 664 665 // Check for identity function on memory (Load then Store at same address) 666 virtual Node* Identity(PhaseGVN* phase); 667 668 // Do not match memory edge 669 virtual uint match_edge(uint idx) const; 670 671 virtual const Type *bottom_type() const; // returns Type::MEMORY 672 673 // Map a store opcode to its corresponding own opcode, trivially. 674 virtual int store_Opcode() const { return Opcode(); } 675 676 // have all possible loads of the value stored been optimized away? 677 bool value_never_loaded(PhaseValues* phase) const; 678 679 bool has_reinterpret_variant(const Type* vt); 680 Node* convert_to_reinterpret_store(PhaseGVN& gvn, Node* val, const Type* vt); 681 682 MemBarNode* trailing_membar() const; 683 }; 684 685 //------------------------------StoreBNode------------------------------------- 686 // Store byte to memory 687 class StoreBNode : public StoreNode { 688 public: 689 StoreBNode(Node *c, Node *mem, Node *adr, const TypePtr* at, Node *val, MemOrd mo) 690 : StoreNode(c, mem, adr, at, val, mo) {} 691 virtual int Opcode() const; 692 virtual Node *Ideal(PhaseGVN *phase, bool can_reshape); 693 virtual BasicType value_basic_type() const { return T_BYTE; } 694 }; 695 696 //------------------------------StoreCNode------------------------------------- 697 // Store char/short to memory 698 class StoreCNode : public StoreNode { 699 public: 700 StoreCNode(Node *c, Node *mem, Node *adr, const TypePtr* at, Node *val, MemOrd mo) 701 : StoreNode(c, mem, adr, at, val, mo) {} 702 virtual int Opcode() const; 703 virtual Node *Ideal(PhaseGVN *phase, bool can_reshape); 704 virtual BasicType value_basic_type() const { return T_CHAR; } 705 }; 706 707 //------------------------------StoreINode------------------------------------- 708 // Store int to memory 709 class StoreINode : public StoreNode { 710 public: 711 StoreINode(Node *c, Node *mem, Node *adr, const TypePtr* at, Node *val, MemOrd mo) 712 : StoreNode(c, mem, adr, at, val, mo) {} 713 virtual int Opcode() const; 714 virtual BasicType value_basic_type() const { return T_INT; } 715 }; 716 717 //------------------------------StoreLNode------------------------------------- 718 // Store long to memory 719 class StoreLNode : public StoreNode { 720 virtual uint hash() const { return StoreNode::hash() + _require_atomic_access; } 721 virtual bool cmp( const Node &n ) const { 722 return _require_atomic_access == ((StoreLNode&)n)._require_atomic_access 723 && StoreNode::cmp(n); 724 } 725 virtual uint size_of() const { return sizeof(*this); } 726 const bool _require_atomic_access; // is piecewise store forbidden? 727 728 public: 729 StoreLNode(Node *c, Node *mem, Node *adr, const TypePtr* at, Node *val, MemOrd mo, bool require_atomic_access = false) 730 : StoreNode(c, mem, adr, at, val, mo), _require_atomic_access(require_atomic_access) {} 731 virtual int Opcode() const; 732 virtual BasicType value_basic_type() const { return T_LONG; } 733 bool require_atomic_access() const { return _require_atomic_access; } 734 735 #ifndef PRODUCT 736 virtual void dump_spec(outputStream *st) const { 737 StoreNode::dump_spec(st); 738 if (_require_atomic_access) st->print(" Atomic!"); 739 } 740 #endif 741 }; 742 743 // Special StoreL for flat stores that emits GC barriers for field at 'oop_off' in the backend 744 class StoreLSpecialNode : public StoreNode { 745 746 public: 747 StoreLSpecialNode(Node* c, Node* mem, Node* adr, const TypePtr* at, Node* val, Node* oop_off, MemOrd mo) 748 : StoreNode(c, mem, adr, at, val, mo) { 749 set_mismatched_access(); 750 if (oop_off != nullptr) { 751 add_req(oop_off); 752 } 753 } 754 virtual int Opcode() const; 755 virtual BasicType value_basic_type() const { return T_LONG; } 756 757 virtual uint match_edge(uint idx) const { return idx == MemNode::Address || 758 idx == MemNode::ValueIn || 759 idx == MemNode::ValueIn + 1; } 760 }; 761 762 //------------------------------StoreFNode------------------------------------- 763 // Store float to memory 764 class StoreFNode : public StoreNode { 765 public: 766 StoreFNode(Node *c, Node *mem, Node *adr, const TypePtr* at, Node *val, MemOrd mo) 767 : StoreNode(c, mem, adr, at, val, mo) {} 768 virtual int Opcode() const; 769 virtual BasicType value_basic_type() const { return T_FLOAT; } 770 }; 771 772 //------------------------------StoreDNode------------------------------------- 773 // Store double to memory 774 class StoreDNode : public StoreNode { 775 virtual uint hash() const { return StoreNode::hash() + _require_atomic_access; } 776 virtual bool cmp( const Node &n ) const { 777 return _require_atomic_access == ((StoreDNode&)n)._require_atomic_access 778 && StoreNode::cmp(n); 779 } 780 virtual uint size_of() const { return sizeof(*this); } 781 const bool _require_atomic_access; // is piecewise store forbidden? 782 public: 783 StoreDNode(Node *c, Node *mem, Node *adr, const TypePtr* at, Node *val, 784 MemOrd mo, bool require_atomic_access = false) 785 : StoreNode(c, mem, adr, at, val, mo), _require_atomic_access(require_atomic_access) {} 786 virtual int Opcode() const; 787 virtual BasicType value_basic_type() const { return T_DOUBLE; } 788 bool require_atomic_access() const { return _require_atomic_access; } 789 790 #ifndef PRODUCT 791 virtual void dump_spec(outputStream *st) const { 792 StoreNode::dump_spec(st); 793 if (_require_atomic_access) st->print(" Atomic!"); 794 } 795 #endif 796 797 }; 798 799 //------------------------------StorePNode------------------------------------- 800 // Store pointer to memory 801 class StorePNode : public StoreNode { 802 public: 803 StorePNode(Node *c, Node *mem, Node *adr, const TypePtr* at, Node *val, MemOrd mo) 804 : StoreNode(c, mem, adr, at, val, mo) {} 805 virtual int Opcode() const; 806 virtual BasicType value_basic_type() const { return T_ADDRESS; } 807 }; 808 809 //------------------------------StoreNNode------------------------------------- 810 // Store narrow oop to memory 811 class StoreNNode : public StoreNode { 812 public: 813 StoreNNode(Node *c, Node *mem, Node *adr, const TypePtr* at, Node *val, MemOrd mo) 814 : StoreNode(c, mem, adr, at, val, mo) {} 815 virtual int Opcode() const; 816 virtual BasicType value_basic_type() const { return T_NARROWOOP; } 817 }; 818 819 //------------------------------StoreNKlassNode-------------------------------------- 820 // Store narrow klass to memory 821 class StoreNKlassNode : public StoreNNode { 822 public: 823 StoreNKlassNode(Node *c, Node *mem, Node *adr, const TypePtr* at, Node *val, MemOrd mo) 824 : StoreNNode(c, mem, adr, at, val, mo) {} 825 virtual int Opcode() const; 826 virtual BasicType value_basic_type() const { return T_NARROWKLASS; } 827 }; 828 829 //------------------------------SCMemProjNode--------------------------------------- 830 // This class defines a projection of the memory state of a store conditional node. 831 // These nodes return a value, but also update memory. 832 class SCMemProjNode : public ProjNode { 833 public: 834 enum {SCMEMPROJCON = (uint)-2}; 835 SCMemProjNode( Node *src) : ProjNode( src, SCMEMPROJCON) { } 836 virtual int Opcode() const; 837 virtual bool is_CFG() const { return false; } 838 virtual const Type *bottom_type() const {return Type::MEMORY;} 839 virtual const TypePtr *adr_type() const { 840 Node* ctrl = in(0); 841 if (ctrl == nullptr) return nullptr; // node is dead 842 return ctrl->in(MemNode::Memory)->adr_type(); 843 } 844 virtual uint ideal_reg() const { return 0;} // memory projections don't have a register 845 virtual const Type* Value(PhaseGVN* phase) const; 846 #ifndef PRODUCT 847 virtual void dump_spec(outputStream *st) const {}; 848 #endif 849 }; 850 851 //------------------------------LoadStoreNode--------------------------- 852 // Note: is_Mem() method returns 'true' for this class. 853 class LoadStoreNode : public Node { 854 private: 855 const Type* const _type; // What kind of value is loaded? 856 const TypePtr* _adr_type; // What kind of memory is being addressed? 857 uint8_t _barrier_data; // Bit field with barrier information 858 virtual uint size_of() const; // Size is bigger 859 public: 860 LoadStoreNode( Node *c, Node *mem, Node *adr, Node *val, const TypePtr* at, const Type* rt, uint required ); 861 virtual bool depends_only_on_test() const { return false; } 862 virtual uint match_edge(uint idx) const { return idx == MemNode::Address || idx == MemNode::ValueIn; } 863 864 virtual const Type *bottom_type() const { return _type; } 865 virtual uint ideal_reg() const; 866 virtual const class TypePtr *adr_type() const { return _adr_type; } // returns bottom_type of address 867 virtual const Type* Value(PhaseGVN* phase) const; 868 869 bool result_not_used() const; 870 MemBarNode* trailing_membar() const; 871 872 uint8_t barrier_data() { return _barrier_data; } 873 void set_barrier_data(uint8_t barrier_data) { _barrier_data = barrier_data; } 874 }; 875 876 class LoadStoreConditionalNode : public LoadStoreNode { 877 public: 878 enum { 879 ExpectedIn = MemNode::ValueIn+1 // One more input than MemNode 880 }; 881 LoadStoreConditionalNode(Node *c, Node *mem, Node *adr, Node *val, Node *ex); 882 virtual const Type* Value(PhaseGVN* phase) const; 883 }; 884 885 class CompareAndSwapNode : public LoadStoreConditionalNode { 886 private: 887 const MemNode::MemOrd _mem_ord; 888 public: 889 CompareAndSwapNode( Node *c, Node *mem, Node *adr, Node *val, Node *ex, MemNode::MemOrd mem_ord) : LoadStoreConditionalNode(c, mem, adr, val, ex), _mem_ord(mem_ord) {} 890 MemNode::MemOrd order() const { 891 return _mem_ord; 892 } 893 virtual uint size_of() const { return sizeof(*this); } 894 }; 895 896 class CompareAndExchangeNode : public LoadStoreNode { 897 private: 898 const MemNode::MemOrd _mem_ord; 899 public: 900 enum { 901 ExpectedIn = MemNode::ValueIn+1 // One more input than MemNode 902 }; 903 CompareAndExchangeNode( Node *c, Node *mem, Node *adr, Node *val, Node *ex, MemNode::MemOrd mem_ord, const TypePtr* at, const Type* t) : 904 LoadStoreNode(c, mem, adr, val, at, t, 5), _mem_ord(mem_ord) { 905 init_req(ExpectedIn, ex ); 906 } 907 908 MemNode::MemOrd order() const { 909 return _mem_ord; 910 } 911 virtual uint size_of() const { return sizeof(*this); } 912 }; 913 914 //------------------------------CompareAndSwapBNode--------------------------- 915 class CompareAndSwapBNode : public CompareAndSwapNode { 916 public: 917 CompareAndSwapBNode( Node *c, Node *mem, Node *adr, Node *val, Node *ex, MemNode::MemOrd mem_ord) : CompareAndSwapNode(c, mem, adr, val, ex, mem_ord) { } 918 virtual int Opcode() const; 919 }; 920 921 //------------------------------CompareAndSwapSNode--------------------------- 922 class CompareAndSwapSNode : public CompareAndSwapNode { 923 public: 924 CompareAndSwapSNode( Node *c, Node *mem, Node *adr, Node *val, Node *ex, MemNode::MemOrd mem_ord) : CompareAndSwapNode(c, mem, adr, val, ex, mem_ord) { } 925 virtual int Opcode() const; 926 }; 927 928 //------------------------------CompareAndSwapINode--------------------------- 929 class CompareAndSwapINode : public CompareAndSwapNode { 930 public: 931 CompareAndSwapINode( Node *c, Node *mem, Node *adr, Node *val, Node *ex, MemNode::MemOrd mem_ord) : CompareAndSwapNode(c, mem, adr, val, ex, mem_ord) { } 932 virtual int Opcode() const; 933 }; 934 935 //------------------------------CompareAndSwapLNode--------------------------- 936 class CompareAndSwapLNode : public CompareAndSwapNode { 937 public: 938 CompareAndSwapLNode( Node *c, Node *mem, Node *adr, Node *val, Node *ex, MemNode::MemOrd mem_ord) : CompareAndSwapNode(c, mem, adr, val, ex, mem_ord) { } 939 virtual int Opcode() const; 940 }; 941 942 //------------------------------CompareAndSwapPNode--------------------------- 943 class CompareAndSwapPNode : public CompareAndSwapNode { 944 public: 945 CompareAndSwapPNode( Node *c, Node *mem, Node *adr, Node *val, Node *ex, MemNode::MemOrd mem_ord) : CompareAndSwapNode(c, mem, adr, val, ex, mem_ord) { } 946 virtual int Opcode() const; 947 }; 948 949 //------------------------------CompareAndSwapNNode--------------------------- 950 class CompareAndSwapNNode : public CompareAndSwapNode { 951 public: 952 CompareAndSwapNNode( Node *c, Node *mem, Node *adr, Node *val, Node *ex, MemNode::MemOrd mem_ord) : CompareAndSwapNode(c, mem, adr, val, ex, mem_ord) { } 953 virtual int Opcode() const; 954 }; 955 956 //------------------------------WeakCompareAndSwapBNode--------------------------- 957 class WeakCompareAndSwapBNode : public CompareAndSwapNode { 958 public: 959 WeakCompareAndSwapBNode( Node *c, Node *mem, Node *adr, Node *val, Node *ex, MemNode::MemOrd mem_ord) : CompareAndSwapNode(c, mem, adr, val, ex, mem_ord) { } 960 virtual int Opcode() const; 961 }; 962 963 //------------------------------WeakCompareAndSwapSNode--------------------------- 964 class WeakCompareAndSwapSNode : public CompareAndSwapNode { 965 public: 966 WeakCompareAndSwapSNode( Node *c, Node *mem, Node *adr, Node *val, Node *ex, MemNode::MemOrd mem_ord) : CompareAndSwapNode(c, mem, adr, val, ex, mem_ord) { } 967 virtual int Opcode() const; 968 }; 969 970 //------------------------------WeakCompareAndSwapINode--------------------------- 971 class WeakCompareAndSwapINode : public CompareAndSwapNode { 972 public: 973 WeakCompareAndSwapINode( Node *c, Node *mem, Node *adr, Node *val, Node *ex, MemNode::MemOrd mem_ord) : CompareAndSwapNode(c, mem, adr, val, ex, mem_ord) { } 974 virtual int Opcode() const; 975 }; 976 977 //------------------------------WeakCompareAndSwapLNode--------------------------- 978 class WeakCompareAndSwapLNode : public CompareAndSwapNode { 979 public: 980 WeakCompareAndSwapLNode( Node *c, Node *mem, Node *adr, Node *val, Node *ex, MemNode::MemOrd mem_ord) : CompareAndSwapNode(c, mem, adr, val, ex, mem_ord) { } 981 virtual int Opcode() const; 982 }; 983 984 //------------------------------WeakCompareAndSwapPNode--------------------------- 985 class WeakCompareAndSwapPNode : public CompareAndSwapNode { 986 public: 987 WeakCompareAndSwapPNode( Node *c, Node *mem, Node *adr, Node *val, Node *ex, MemNode::MemOrd mem_ord) : CompareAndSwapNode(c, mem, adr, val, ex, mem_ord) { } 988 virtual int Opcode() const; 989 }; 990 991 //------------------------------WeakCompareAndSwapNNode--------------------------- 992 class WeakCompareAndSwapNNode : public CompareAndSwapNode { 993 public: 994 WeakCompareAndSwapNNode( Node *c, Node *mem, Node *adr, Node *val, Node *ex, MemNode::MemOrd mem_ord) : CompareAndSwapNode(c, mem, adr, val, ex, mem_ord) { } 995 virtual int Opcode() const; 996 }; 997 998 //------------------------------CompareAndExchangeBNode--------------------------- 999 class CompareAndExchangeBNode : public CompareAndExchangeNode { 1000 public: 1001 CompareAndExchangeBNode( Node *c, Node *mem, Node *adr, Node *val, Node *ex, const TypePtr* at, MemNode::MemOrd mem_ord) : CompareAndExchangeNode(c, mem, adr, val, ex, mem_ord, at, TypeInt::BYTE) { } 1002 virtual int Opcode() const; 1003 }; 1004 1005 1006 //------------------------------CompareAndExchangeSNode--------------------------- 1007 class CompareAndExchangeSNode : public CompareAndExchangeNode { 1008 public: 1009 CompareAndExchangeSNode( Node *c, Node *mem, Node *adr, Node *val, Node *ex, const TypePtr* at, MemNode::MemOrd mem_ord) : CompareAndExchangeNode(c, mem, adr, val, ex, mem_ord, at, TypeInt::SHORT) { } 1010 virtual int Opcode() const; 1011 }; 1012 1013 //------------------------------CompareAndExchangeLNode--------------------------- 1014 class CompareAndExchangeLNode : public CompareAndExchangeNode { 1015 public: 1016 CompareAndExchangeLNode( Node *c, Node *mem, Node *adr, Node *val, Node *ex, const TypePtr* at, MemNode::MemOrd mem_ord) : CompareAndExchangeNode(c, mem, adr, val, ex, mem_ord, at, TypeLong::LONG) { } 1017 virtual int Opcode() const; 1018 }; 1019 1020 1021 //------------------------------CompareAndExchangeINode--------------------------- 1022 class CompareAndExchangeINode : public CompareAndExchangeNode { 1023 public: 1024 CompareAndExchangeINode( Node *c, Node *mem, Node *adr, Node *val, Node *ex, const TypePtr* at, MemNode::MemOrd mem_ord) : CompareAndExchangeNode(c, mem, adr, val, ex, mem_ord, at, TypeInt::INT) { } 1025 virtual int Opcode() const; 1026 }; 1027 1028 1029 //------------------------------CompareAndExchangePNode--------------------------- 1030 class CompareAndExchangePNode : public CompareAndExchangeNode { 1031 public: 1032 CompareAndExchangePNode( Node *c, Node *mem, Node *adr, Node *val, Node *ex, const TypePtr* at, const Type* t, MemNode::MemOrd mem_ord) : CompareAndExchangeNode(c, mem, adr, val, ex, mem_ord, at, t) { } 1033 virtual int Opcode() const; 1034 }; 1035 1036 //------------------------------CompareAndExchangeNNode--------------------------- 1037 class CompareAndExchangeNNode : public CompareAndExchangeNode { 1038 public: 1039 CompareAndExchangeNNode( Node *c, Node *mem, Node *adr, Node *val, Node *ex, const TypePtr* at, const Type* t, MemNode::MemOrd mem_ord) : CompareAndExchangeNode(c, mem, adr, val, ex, mem_ord, at, t) { } 1040 virtual int Opcode() const; 1041 }; 1042 1043 //------------------------------GetAndAddBNode--------------------------- 1044 class GetAndAddBNode : public LoadStoreNode { 1045 public: 1046 GetAndAddBNode( Node *c, Node *mem, Node *adr, Node *val, const TypePtr* at ) : LoadStoreNode(c, mem, adr, val, at, TypeInt::BYTE, 4) { } 1047 virtual int Opcode() const; 1048 }; 1049 1050 //------------------------------GetAndAddSNode--------------------------- 1051 class GetAndAddSNode : public LoadStoreNode { 1052 public: 1053 GetAndAddSNode( Node *c, Node *mem, Node *adr, Node *val, const TypePtr* at ) : LoadStoreNode(c, mem, adr, val, at, TypeInt::SHORT, 4) { } 1054 virtual int Opcode() const; 1055 }; 1056 1057 //------------------------------GetAndAddINode--------------------------- 1058 class GetAndAddINode : public LoadStoreNode { 1059 public: 1060 GetAndAddINode( Node *c, Node *mem, Node *adr, Node *val, const TypePtr* at ) : LoadStoreNode(c, mem, adr, val, at, TypeInt::INT, 4) { } 1061 virtual int Opcode() const; 1062 }; 1063 1064 //------------------------------GetAndAddLNode--------------------------- 1065 class GetAndAddLNode : public LoadStoreNode { 1066 public: 1067 GetAndAddLNode( Node *c, Node *mem, Node *adr, Node *val, const TypePtr* at ) : LoadStoreNode(c, mem, adr, val, at, TypeLong::LONG, 4) { } 1068 virtual int Opcode() const; 1069 }; 1070 1071 //------------------------------GetAndSetBNode--------------------------- 1072 class GetAndSetBNode : public LoadStoreNode { 1073 public: 1074 GetAndSetBNode( Node *c, Node *mem, Node *adr, Node *val, const TypePtr* at ) : LoadStoreNode(c, mem, adr, val, at, TypeInt::BYTE, 4) { } 1075 virtual int Opcode() const; 1076 }; 1077 1078 //------------------------------GetAndSetSNode--------------------------- 1079 class GetAndSetSNode : public LoadStoreNode { 1080 public: 1081 GetAndSetSNode( Node *c, Node *mem, Node *adr, Node *val, const TypePtr* at ) : LoadStoreNode(c, mem, adr, val, at, TypeInt::SHORT, 4) { } 1082 virtual int Opcode() const; 1083 }; 1084 1085 //------------------------------GetAndSetINode--------------------------- 1086 class GetAndSetINode : public LoadStoreNode { 1087 public: 1088 GetAndSetINode( Node *c, Node *mem, Node *adr, Node *val, const TypePtr* at ) : LoadStoreNode(c, mem, adr, val, at, TypeInt::INT, 4) { } 1089 virtual int Opcode() const; 1090 }; 1091 1092 //------------------------------GetAndSetLNode--------------------------- 1093 class GetAndSetLNode : public LoadStoreNode { 1094 public: 1095 GetAndSetLNode( Node *c, Node *mem, Node *adr, Node *val, const TypePtr* at ) : LoadStoreNode(c, mem, adr, val, at, TypeLong::LONG, 4) { } 1096 virtual int Opcode() const; 1097 }; 1098 1099 //------------------------------GetAndSetPNode--------------------------- 1100 class GetAndSetPNode : public LoadStoreNode { 1101 public: 1102 GetAndSetPNode( Node *c, Node *mem, Node *adr, Node *val, const TypePtr* at, const Type* t ) : LoadStoreNode(c, mem, adr, val, at, t, 4) { } 1103 virtual int Opcode() const; 1104 }; 1105 1106 //------------------------------GetAndSetNNode--------------------------- 1107 class GetAndSetNNode : public LoadStoreNode { 1108 public: 1109 GetAndSetNNode( Node *c, Node *mem, Node *adr, Node *val, const TypePtr* at, const Type* t ) : LoadStoreNode(c, mem, adr, val, at, t, 4) { } 1110 virtual int Opcode() const; 1111 }; 1112 1113 //------------------------------ClearArray------------------------------------- 1114 class ClearArrayNode: public Node { 1115 private: 1116 bool _is_large; 1117 bool _word_copy_only; 1118 public: 1119 ClearArrayNode( Node *ctrl, Node *arymem, Node *word_cnt, Node *base, Node* val, bool is_large) 1120 : Node(ctrl, arymem, word_cnt, base, val), _is_large(is_large), 1121 _word_copy_only(val->bottom_type()->isa_long() && (!val->bottom_type()->is_long()->is_con() || val->bottom_type()->is_long()->get_con() != 0)) { 1122 init_class_id(Class_ClearArray); 1123 } 1124 virtual int Opcode() const; 1125 virtual const Type *bottom_type() const { return Type::MEMORY; } 1126 // ClearArray modifies array elements, and so affects only the 1127 // array memory addressed by the bottom_type of its base address. 1128 virtual const class TypePtr *adr_type() const; 1129 virtual Node* Identity(PhaseGVN* phase); 1130 virtual Node *Ideal(PhaseGVN *phase, bool can_reshape); 1131 virtual uint match_edge(uint idx) const; 1132 bool is_large() const { return _is_large; } 1133 bool word_copy_only() const { return _word_copy_only; } 1134 1135 // Clear the given area of an object or array. 1136 // The start offset must always be aligned mod BytesPerInt. 1137 // The end offset must always be aligned mod BytesPerLong. 1138 // Return the new memory. 1139 static Node* clear_memory(Node* control, Node* mem, Node* dest, 1140 Node* val, 1141 Node* raw_val, 1142 intptr_t start_offset, 1143 intptr_t end_offset, 1144 PhaseGVN* phase); 1145 static Node* clear_memory(Node* control, Node* mem, Node* dest, 1146 Node* val, 1147 Node* raw_val, 1148 intptr_t start_offset, 1149 Node* end_offset, 1150 PhaseGVN* phase); 1151 static Node* clear_memory(Node* control, Node* mem, Node* dest, 1152 Node* raw_val, 1153 Node* start_offset, 1154 Node* end_offset, 1155 PhaseGVN* phase); 1156 // Return allocation input memory edge if it is different instance 1157 // or itself if it is the one we are looking for. 1158 static bool step_through(Node** np, uint instance_id, PhaseValues* phase); 1159 }; 1160 1161 //------------------------------MemBar----------------------------------------- 1162 // There are different flavors of Memory Barriers to match the Java Memory 1163 // Model. Monitor-enter and volatile-load act as Acquires: no following ref 1164 // can be moved to before them. We insert a MemBar-Acquire after a FastLock or 1165 // volatile-load. Monitor-exit and volatile-store act as Release: no 1166 // preceding ref can be moved to after them. We insert a MemBar-Release 1167 // before a FastUnlock or volatile-store. All volatiles need to be 1168 // serialized, so we follow all volatile-stores with a MemBar-Volatile to 1169 // separate it from any following volatile-load. 1170 class MemBarNode: public MultiNode { 1171 virtual uint hash() const ; // { return NO_HASH; } 1172 virtual bool cmp( const Node &n ) const ; // Always fail, except on self 1173 1174 virtual uint size_of() const { return sizeof(*this); } 1175 // Memory type this node is serializing. Usually either rawptr or bottom. 1176 const TypePtr* _adr_type; 1177 1178 // How is this membar related to a nearby memory access? 1179 enum { 1180 Standalone, 1181 TrailingLoad, 1182 TrailingStore, 1183 LeadingStore, 1184 TrailingLoadStore, 1185 LeadingLoadStore, 1186 TrailingExpandedArrayCopy 1187 } _kind; 1188 1189 #ifdef ASSERT 1190 uint _pair_idx; 1191 #endif 1192 1193 public: 1194 enum { 1195 Precedent = TypeFunc::Parms // optional edge to force precedence 1196 }; 1197 MemBarNode(Compile* C, int alias_idx, Node* precedent); 1198 virtual int Opcode() const = 0; 1199 virtual const class TypePtr *adr_type() const { return _adr_type; } 1200 virtual const Type* Value(PhaseGVN* phase) const; 1201 virtual Node *Ideal(PhaseGVN *phase, bool can_reshape); 1202 virtual uint match_edge(uint idx) const { return 0; } 1203 virtual const Type *bottom_type() const { return TypeTuple::MEMBAR; } 1204 virtual Node *match(const ProjNode *proj, const Matcher *m, const RegMask* mask); 1205 // Factory method. Builds a wide or narrow membar. 1206 // Optional 'precedent' becomes an extra edge if not null. 1207 static MemBarNode* make(Compile* C, int opcode, 1208 int alias_idx = Compile::AliasIdxBot, 1209 Node* precedent = nullptr); 1210 1211 MemBarNode* trailing_membar() const; 1212 MemBarNode* leading_membar() const; 1213 1214 void set_trailing_load() { _kind = TrailingLoad; } 1215 bool trailing_load() const { return _kind == TrailingLoad; } 1216 bool trailing_store() const { return _kind == TrailingStore; } 1217 bool leading_store() const { return _kind == LeadingStore; } 1218 bool trailing_load_store() const { return _kind == TrailingLoadStore; } 1219 bool leading_load_store() const { return _kind == LeadingLoadStore; } 1220 bool trailing() const { return _kind == TrailingLoad || _kind == TrailingStore || _kind == TrailingLoadStore; } 1221 bool leading() const { return _kind == LeadingStore || _kind == LeadingLoadStore; } 1222 bool standalone() const { return _kind == Standalone; } 1223 void set_trailing_expanded_array_copy() { _kind = TrailingExpandedArrayCopy; } 1224 bool trailing_expanded_array_copy() const { return _kind == TrailingExpandedArrayCopy; } 1225 1226 static void set_store_pair(MemBarNode* leading, MemBarNode* trailing); 1227 static void set_load_store_pair(MemBarNode* leading, MemBarNode* trailing); 1228 1229 void remove(PhaseIterGVN *igvn); 1230 }; 1231 1232 // "Acquire" - no following ref can move before (but earlier refs can 1233 // follow, like an early Load stalled in cache). Requires multi-cpu 1234 // visibility. Inserted after a volatile load. 1235 class MemBarAcquireNode: public MemBarNode { 1236 public: 1237 MemBarAcquireNode(Compile* C, int alias_idx, Node* precedent) 1238 : MemBarNode(C, alias_idx, precedent) {} 1239 virtual int Opcode() const; 1240 }; 1241 1242 // "Acquire" - no following ref can move before (but earlier refs can 1243 // follow, like an early Load stalled in cache). Requires multi-cpu 1244 // visibility. Inserted independent of any load, as required 1245 // for intrinsic Unsafe.loadFence(). 1246 class LoadFenceNode: public MemBarNode { 1247 public: 1248 LoadFenceNode(Compile* C, int alias_idx, Node* precedent) 1249 : MemBarNode(C, alias_idx, precedent) {} 1250 virtual int Opcode() const; 1251 }; 1252 1253 // "Release" - no earlier ref can move after (but later refs can move 1254 // up, like a speculative pipelined cache-hitting Load). Requires 1255 // multi-cpu visibility. Inserted before a volatile store. 1256 class MemBarReleaseNode: public MemBarNode { 1257 public: 1258 MemBarReleaseNode(Compile* C, int alias_idx, Node* precedent) 1259 : MemBarNode(C, alias_idx, precedent) {} 1260 virtual int Opcode() const; 1261 }; 1262 1263 // "Release" - no earlier ref can move after (but later refs can move 1264 // up, like a speculative pipelined cache-hitting Load). Requires 1265 // multi-cpu visibility. Inserted independent of any store, as required 1266 // for intrinsic Unsafe.storeFence(). 1267 class StoreFenceNode: public MemBarNode { 1268 public: 1269 StoreFenceNode(Compile* C, int alias_idx, Node* precedent) 1270 : MemBarNode(C, alias_idx, precedent) {} 1271 virtual int Opcode() const; 1272 }; 1273 1274 // "Acquire" - no following ref can move before (but earlier refs can 1275 // follow, like an early Load stalled in cache). Requires multi-cpu 1276 // visibility. Inserted after a FastLock. 1277 class MemBarAcquireLockNode: public MemBarNode { 1278 public: 1279 MemBarAcquireLockNode(Compile* C, int alias_idx, Node* precedent) 1280 : MemBarNode(C, alias_idx, precedent) {} 1281 virtual int Opcode() const; 1282 }; 1283 1284 // "Release" - no earlier ref can move after (but later refs can move 1285 // up, like a speculative pipelined cache-hitting Load). Requires 1286 // multi-cpu visibility. Inserted before a FastUnLock. 1287 class MemBarReleaseLockNode: public MemBarNode { 1288 public: 1289 MemBarReleaseLockNode(Compile* C, int alias_idx, Node* precedent) 1290 : MemBarNode(C, alias_idx, precedent) {} 1291 virtual int Opcode() const; 1292 }; 1293 1294 class MemBarStoreStoreNode: public MemBarNode { 1295 public: 1296 MemBarStoreStoreNode(Compile* C, int alias_idx, Node* precedent) 1297 : MemBarNode(C, alias_idx, precedent) { 1298 init_class_id(Class_MemBarStoreStore); 1299 } 1300 virtual int Opcode() const; 1301 }; 1302 1303 class StoreStoreFenceNode: public MemBarNode { 1304 public: 1305 StoreStoreFenceNode(Compile* C, int alias_idx, Node* precedent) 1306 : MemBarNode(C, alias_idx, precedent) {} 1307 virtual int Opcode() const; 1308 }; 1309 1310 // Ordering between a volatile store and a following volatile load. 1311 // Requires multi-CPU visibility? 1312 class MemBarVolatileNode: public MemBarNode { 1313 public: 1314 MemBarVolatileNode(Compile* C, int alias_idx, Node* precedent) 1315 : MemBarNode(C, alias_idx, precedent) {} 1316 virtual int Opcode() const; 1317 }; 1318 1319 // Ordering within the same CPU. Used to order unsafe memory references 1320 // inside the compiler when we lack alias info. Not needed "outside" the 1321 // compiler because the CPU does all the ordering for us. 1322 class MemBarCPUOrderNode: public MemBarNode { 1323 public: 1324 MemBarCPUOrderNode(Compile* C, int alias_idx, Node* precedent) 1325 : MemBarNode(C, alias_idx, precedent) {} 1326 virtual int Opcode() const; 1327 virtual uint ideal_reg() const { return 0; } // not matched in the AD file 1328 }; 1329 1330 class OnSpinWaitNode: public MemBarNode { 1331 public: 1332 OnSpinWaitNode(Compile* C, int alias_idx, Node* precedent) 1333 : MemBarNode(C, alias_idx, precedent) {} 1334 virtual int Opcode() const; 1335 }; 1336 1337 // Isolation of object setup after an AllocateNode and before next safepoint. 1338 // (See comment in memnode.cpp near InitializeNode::InitializeNode for semantics.) 1339 class InitializeNode: public MemBarNode { 1340 friend class AllocateNode; 1341 1342 enum { 1343 Incomplete = 0, 1344 Complete = 1, 1345 WithArraycopy = 2 1346 }; 1347 int _is_complete; 1348 1349 bool _does_not_escape; 1350 1351 public: 1352 enum { 1353 Control = TypeFunc::Control, 1354 Memory = TypeFunc::Memory, // MergeMem for states affected by this op 1355 RawAddress = TypeFunc::Parms+0, // the newly-allocated raw address 1356 RawStores = TypeFunc::Parms+1 // zero or more stores (or TOP) 1357 }; 1358 1359 InitializeNode(Compile* C, int adr_type, Node* rawoop); 1360 virtual int Opcode() const; 1361 virtual uint size_of() const { return sizeof(*this); } 1362 virtual uint ideal_reg() const { return 0; } // not matched in the AD file 1363 virtual const RegMask &in_RegMask(uint) const; // mask for RawAddress 1364 1365 // Manage incoming memory edges via a MergeMem on in(Memory): 1366 Node* memory(uint alias_idx); 1367 1368 // The raw memory edge coming directly from the Allocation. 1369 // The contents of this memory are *always* all-zero-bits. 1370 Node* zero_memory() { return memory(Compile::AliasIdxRaw); } 1371 1372 // Return the corresponding allocation for this initialization (or null if none). 1373 // (Note: Both InitializeNode::allocation and AllocateNode::initialization 1374 // are defined in graphKit.cpp, which sets up the bidirectional relation.) 1375 AllocateNode* allocation(); 1376 1377 // Anything other than zeroing in this init? 1378 bool is_non_zero(); 1379 1380 // An InitializeNode must completed before macro expansion is done. 1381 // Completion requires that the AllocateNode must be followed by 1382 // initialization of the new memory to zero, then to any initializers. 1383 bool is_complete() { return _is_complete != Incomplete; } 1384 bool is_complete_with_arraycopy() { return (_is_complete & WithArraycopy) != 0; } 1385 1386 // Mark complete. (Must not yet be complete.) 1387 void set_complete(PhaseGVN* phase); 1388 void set_complete_with_arraycopy() { _is_complete = Complete | WithArraycopy; } 1389 1390 bool does_not_escape() { return _does_not_escape; } 1391 void set_does_not_escape() { _does_not_escape = true; } 1392 1393 #ifdef ASSERT 1394 // ensure all non-degenerate stores are ordered and non-overlapping 1395 bool stores_are_sane(PhaseValues* phase); 1396 #endif //ASSERT 1397 1398 // See if this store can be captured; return offset where it initializes. 1399 // Return 0 if the store cannot be moved (any sort of problem). 1400 intptr_t can_capture_store(StoreNode* st, PhaseGVN* phase, bool can_reshape); 1401 1402 // Capture another store; reformat it to write my internal raw memory. 1403 // Return the captured copy, else null if there is some sort of problem. 1404 Node* capture_store(StoreNode* st, intptr_t start, PhaseGVN* phase, bool can_reshape); 1405 1406 // Find captured store which corresponds to the range [start..start+size). 1407 // Return my own memory projection (meaning the initial zero bits) 1408 // if there is no such store. Return null if there is a problem. 1409 Node* find_captured_store(intptr_t start, int size_in_bytes, PhaseValues* phase); 1410 1411 // Called when the associated AllocateNode is expanded into CFG. 1412 Node* complete_stores(Node* rawctl, Node* rawmem, Node* rawptr, 1413 intptr_t header_size, Node* size_in_bytes, 1414 PhaseIterGVN* phase); 1415 1416 private: 1417 void remove_extra_zeroes(); 1418 1419 // Find out where a captured store should be placed (or already is placed). 1420 int captured_store_insertion_point(intptr_t start, int size_in_bytes, 1421 PhaseValues* phase); 1422 1423 static intptr_t get_store_offset(Node* st, PhaseValues* phase); 1424 1425 Node* make_raw_address(intptr_t offset, PhaseGVN* phase); 1426 1427 bool detect_init_independence(Node* value, PhaseGVN* phase); 1428 1429 void coalesce_subword_stores(intptr_t header_size, Node* size_in_bytes, 1430 PhaseGVN* phase); 1431 1432 intptr_t find_next_fullword_store(uint i, PhaseGVN* phase); 1433 }; 1434 1435 //------------------------------MergeMem--------------------------------------- 1436 // (See comment in memnode.cpp near MergeMemNode::MergeMemNode for semantics.) 1437 class MergeMemNode: public Node { 1438 virtual uint hash() const ; // { return NO_HASH; } 1439 virtual bool cmp( const Node &n ) const ; // Always fail, except on self 1440 friend class MergeMemStream; 1441 MergeMemNode(Node* def); // clients use MergeMemNode::make 1442 1443 public: 1444 // If the input is a whole memory state, clone it with all its slices intact. 1445 // Otherwise, make a new memory state with just that base memory input. 1446 // In either case, the result is a newly created MergeMem. 1447 static MergeMemNode* make(Node* base_memory); 1448 1449 virtual int Opcode() const; 1450 virtual Node* Identity(PhaseGVN* phase); 1451 virtual Node *Ideal(PhaseGVN *phase, bool can_reshape); 1452 virtual uint ideal_reg() const { return NotAMachineReg; } 1453 virtual uint match_edge(uint idx) const { return 0; } 1454 virtual const RegMask &out_RegMask() const; 1455 virtual const Type *bottom_type() const { return Type::MEMORY; } 1456 virtual const TypePtr *adr_type() const { return TypePtr::BOTTOM; } 1457 // sparse accessors 1458 // Fetch the previously stored "set_memory_at", or else the base memory. 1459 // (Caller should clone it if it is a phi-nest.) 1460 Node* memory_at(uint alias_idx) const; 1461 // set the memory, regardless of its previous value 1462 void set_memory_at(uint alias_idx, Node* n); 1463 // the "base" is the memory that provides the non-finite support 1464 Node* base_memory() const { return in(Compile::AliasIdxBot); } 1465 // warning: setting the base can implicitly set any of the other slices too 1466 void set_base_memory(Node* def); 1467 // sentinel value which denotes a copy of the base memory: 1468 Node* empty_memory() const { return in(Compile::AliasIdxTop); } 1469 static Node* make_empty_memory(); // where the sentinel comes from 1470 bool is_empty_memory(Node* n) const { assert((n == empty_memory()) == n->is_top(), "sanity"); return n->is_top(); } 1471 // hook for the iterator, to perform any necessary setup 1472 void iteration_setup(const MergeMemNode* other = nullptr); 1473 // push sentinels until I am at least as long as the other (semantic no-op) 1474 void grow_to_match(const MergeMemNode* other); 1475 bool verify_sparse() const PRODUCT_RETURN0; 1476 #ifndef PRODUCT 1477 virtual void dump_spec(outputStream *st) const; 1478 #endif 1479 }; 1480 1481 class MergeMemStream : public StackObj { 1482 private: 1483 MergeMemNode* _mm; 1484 const MergeMemNode* _mm2; // optional second guy, contributes non-empty iterations 1485 Node* _mm_base; // loop-invariant base memory of _mm 1486 int _idx; 1487 int _cnt; 1488 Node* _mem; 1489 Node* _mem2; 1490 int _cnt2; 1491 1492 void init(MergeMemNode* mm, const MergeMemNode* mm2 = nullptr) { 1493 // subsume_node will break sparseness at times, whenever a memory slice 1494 // folds down to a copy of the base ("fat") memory. In such a case, 1495 // the raw edge will update to base, although it should be top. 1496 // This iterator will recognize either top or base_memory as an 1497 // "empty" slice. See is_empty, is_empty2, and next below. 1498 // 1499 // The sparseness property is repaired in MergeMemNode::Ideal. 1500 // As long as access to a MergeMem goes through this iterator 1501 // or the memory_at accessor, flaws in the sparseness will 1502 // never be observed. 1503 // 1504 // Also, iteration_setup repairs sparseness. 1505 assert(mm->verify_sparse(), "please, no dups of base"); 1506 assert(mm2==nullptr || mm2->verify_sparse(), "please, no dups of base"); 1507 1508 _mm = mm; 1509 _mm_base = mm->base_memory(); 1510 _mm2 = mm2; 1511 _cnt = mm->req(); 1512 _idx = Compile::AliasIdxBot-1; // start at the base memory 1513 _mem = nullptr; 1514 _mem2 = nullptr; 1515 } 1516 1517 #ifdef ASSERT 1518 Node* check_memory() const { 1519 if (at_base_memory()) 1520 return _mm->base_memory(); 1521 else if ((uint)_idx < _mm->req() && !_mm->in(_idx)->is_top()) 1522 return _mm->memory_at(_idx); 1523 else 1524 return _mm_base; 1525 } 1526 Node* check_memory2() const { 1527 return at_base_memory()? _mm2->base_memory(): _mm2->memory_at(_idx); 1528 } 1529 #endif 1530 1531 static bool match_memory(Node* mem, const MergeMemNode* mm, int idx) PRODUCT_RETURN0; 1532 void assert_synch() const { 1533 assert(!_mem || _idx >= _cnt || match_memory(_mem, _mm, _idx), 1534 "no side-effects except through the stream"); 1535 } 1536 1537 public: 1538 1539 // expected usages: 1540 // for (MergeMemStream mms(mem->is_MergeMem()); next_non_empty(); ) { ... } 1541 // for (MergeMemStream mms(mem1, mem2); next_non_empty2(); ) { ... } 1542 1543 // iterate over one merge 1544 MergeMemStream(MergeMemNode* mm) { 1545 mm->iteration_setup(); 1546 init(mm); 1547 DEBUG_ONLY(_cnt2 = 999); 1548 } 1549 // iterate in parallel over two merges 1550 // only iterates through non-empty elements of mm2 1551 MergeMemStream(MergeMemNode* mm, const MergeMemNode* mm2) { 1552 assert(mm2, "second argument must be a MergeMem also"); 1553 ((MergeMemNode*)mm2)->iteration_setup(); // update hidden state 1554 mm->iteration_setup(mm2); 1555 init(mm, mm2); 1556 _cnt2 = mm2->req(); 1557 } 1558 #ifdef ASSERT 1559 ~MergeMemStream() { 1560 assert_synch(); 1561 } 1562 #endif 1563 1564 MergeMemNode* all_memory() const { 1565 return _mm; 1566 } 1567 Node* base_memory() const { 1568 assert(_mm_base == _mm->base_memory(), "no update to base memory, please"); 1569 return _mm_base; 1570 } 1571 const MergeMemNode* all_memory2() const { 1572 assert(_mm2 != nullptr, ""); 1573 return _mm2; 1574 } 1575 bool at_base_memory() const { 1576 return _idx == Compile::AliasIdxBot; 1577 } 1578 int alias_idx() const { 1579 assert(_mem, "must call next 1st"); 1580 return _idx; 1581 } 1582 1583 const TypePtr* adr_type() const { 1584 return Compile::current()->get_adr_type(alias_idx()); 1585 } 1586 1587 const TypePtr* adr_type(Compile* C) const { 1588 return C->get_adr_type(alias_idx()); 1589 } 1590 bool is_empty() const { 1591 assert(_mem, "must call next 1st"); 1592 assert(_mem->is_top() == (_mem==_mm->empty_memory()), "correct sentinel"); 1593 return _mem->is_top(); 1594 } 1595 bool is_empty2() const { 1596 assert(_mem2, "must call next 1st"); 1597 assert(_mem2->is_top() == (_mem2==_mm2->empty_memory()), "correct sentinel"); 1598 return _mem2->is_top(); 1599 } 1600 Node* memory() const { 1601 assert(!is_empty(), "must not be empty"); 1602 assert_synch(); 1603 return _mem; 1604 } 1605 // get the current memory, regardless of empty or non-empty status 1606 Node* force_memory() const { 1607 assert(!is_empty() || !at_base_memory(), ""); 1608 // Use _mm_base to defend against updates to _mem->base_memory(). 1609 Node *mem = _mem->is_top() ? _mm_base : _mem; 1610 assert(mem == check_memory(), ""); 1611 return mem; 1612 } 1613 Node* memory2() const { 1614 assert(_mem2 == check_memory2(), ""); 1615 return _mem2; 1616 } 1617 void set_memory(Node* mem) { 1618 if (at_base_memory()) { 1619 // Note that this does not change the invariant _mm_base. 1620 _mm->set_base_memory(mem); 1621 } else { 1622 _mm->set_memory_at(_idx, mem); 1623 } 1624 _mem = mem; 1625 assert_synch(); 1626 } 1627 1628 // Recover from a side effect to the MergeMemNode. 1629 void set_memory() { 1630 _mem = _mm->in(_idx); 1631 } 1632 1633 bool next() { return next(false); } 1634 bool next2() { return next(true); } 1635 1636 bool next_non_empty() { return next_non_empty(false); } 1637 bool next_non_empty2() { return next_non_empty(true); } 1638 // next_non_empty2 can yield states where is_empty() is true 1639 1640 private: 1641 // find the next item, which might be empty 1642 bool next(bool have_mm2) { 1643 assert((_mm2 != nullptr) == have_mm2, "use other next"); 1644 assert_synch(); 1645 if (++_idx < _cnt) { 1646 // Note: This iterator allows _mm to be non-sparse. 1647 // It behaves the same whether _mem is top or base_memory. 1648 _mem = _mm->in(_idx); 1649 if (have_mm2) 1650 _mem2 = _mm2->in((_idx < _cnt2) ? _idx : Compile::AliasIdxTop); 1651 return true; 1652 } 1653 return false; 1654 } 1655 1656 // find the next non-empty item 1657 bool next_non_empty(bool have_mm2) { 1658 while (next(have_mm2)) { 1659 if (!is_empty()) { 1660 // make sure _mem2 is filled in sensibly 1661 if (have_mm2 && _mem2->is_top()) _mem2 = _mm2->base_memory(); 1662 return true; 1663 } else if (have_mm2 && !is_empty2()) { 1664 return true; // is_empty() == true 1665 } 1666 } 1667 return false; 1668 } 1669 }; 1670 1671 // cachewb node for guaranteeing writeback of the cache line at a 1672 // given address to (non-volatile) RAM 1673 class CacheWBNode : public Node { 1674 public: 1675 CacheWBNode(Node *ctrl, Node *mem, Node *addr) : Node(ctrl, mem, addr) {} 1676 virtual int Opcode() const; 1677 virtual uint ideal_reg() const { return NotAMachineReg; } 1678 virtual uint match_edge(uint idx) const { return (idx == 2); } 1679 virtual const TypePtr *adr_type() const { return TypePtr::BOTTOM; } 1680 virtual const Type *bottom_type() const { return Type::MEMORY; } 1681 }; 1682 1683 // cachewb pre sync node for ensuring that writebacks are serialised 1684 // relative to preceding or following stores 1685 class CacheWBPreSyncNode : public Node { 1686 public: 1687 CacheWBPreSyncNode(Node *ctrl, Node *mem) : Node(ctrl, mem) {} 1688 virtual int Opcode() const; 1689 virtual uint ideal_reg() const { return NotAMachineReg; } 1690 virtual uint match_edge(uint idx) const { return false; } 1691 virtual const TypePtr *adr_type() const { return TypePtr::BOTTOM; } 1692 virtual const Type *bottom_type() const { return Type::MEMORY; } 1693 }; 1694 1695 // cachewb pre sync node for ensuring that writebacks are serialised 1696 // relative to preceding or following stores 1697 class CacheWBPostSyncNode : public Node { 1698 public: 1699 CacheWBPostSyncNode(Node *ctrl, Node *mem) : Node(ctrl, mem) {} 1700 virtual int Opcode() const; 1701 virtual uint ideal_reg() const { return NotAMachineReg; } 1702 virtual uint match_edge(uint idx) const { return false; } 1703 virtual const TypePtr *adr_type() const { return TypePtr::BOTTOM; } 1704 virtual const Type *bottom_type() const { return Type::MEMORY; } 1705 }; 1706 1707 //------------------------------Prefetch--------------------------------------- 1708 1709 // Allocation prefetch which may fault, TLAB size have to be adjusted. 1710 class PrefetchAllocationNode : public Node { 1711 public: 1712 PrefetchAllocationNode(Node *mem, Node *adr) : Node(nullptr,mem,adr) {} 1713 virtual int Opcode() const; 1714 virtual uint ideal_reg() const { return NotAMachineReg; } 1715 virtual uint match_edge(uint idx) const { return idx==2; } 1716 virtual const Type *bottom_type() const { return ( AllocatePrefetchStyle == 3 ) ? Type::MEMORY : Type::ABIO; } 1717 }; 1718 1719 #endif // SHARE_OPTO_MEMNODE_HPP