1 /*
2 * Copyright (c) 1997, 2021, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #ifndef SHARE_OPTO_MEMNODE_HPP
26 #define SHARE_OPTO_MEMNODE_HPP
27
28 #include "opto/multnode.hpp"
29 #include "opto/node.hpp"
30 #include "opto/opcodes.hpp"
31 #include "opto/type.hpp"
32
33 // Portions of code courtesy of Clifford Click
34
35 class MultiNode;
36 class PhaseCCP;
37 class PhaseTransform;
38
39 //------------------------------MemNode----------------------------------------
40 // Load or Store, possibly throwing a NULL pointer exception
41 class MemNode : public Node {
42 private:
43 bool _unaligned_access; // Unaligned access from unsafe
44 bool _mismatched_access; // Mismatched access from unsafe: byte read in integer array for instance
45 bool _unsafe_access; // Access of unsafe origin.
46 uint8_t _barrier_data; // Bit field with barrier information
47
48 protected:
49 #ifdef ASSERT
50 const TypePtr* _adr_type; // What kind of memory is being addressed?
51 #endif
52 virtual uint size_of() const;
53 public:
54 enum { Control, // When is it safe to do this load?
55 Memory, // Chunk of memory is being loaded from
56 Address, // Actually address, derived from base
57 ValueIn, // Value to store
58 OopStore // Preceeding oop store, only in StoreCM
59 };
60 typedef enum { unordered = 0,
61 acquire, // Load has to acquire or be succeeded by MemBarAcquire.
62 release, // Store has to release or be preceded by MemBarRelease.
63 seqcst, // LoadStore has to have both acquire and release semantics.
64 unset // The memory ordering is not set (used for testing)
65 } MemOrd;
66 protected:
67 MemNode( Node *c0, Node *c1, Node *c2, const TypePtr* at ) :
68 Node(c0,c1,c2),
69 _unaligned_access(false),
70 _mismatched_access(false),
71 _unsafe_access(false),
72 _barrier_data(0) {
73 init_class_id(Class_Mem);
74 debug_only(_adr_type=at; adr_type();)
75 }
76 MemNode( Node *c0, Node *c1, Node *c2, const TypePtr* at, Node *c3 ) :
77 Node(c0,c1,c2,c3),
78 _unaligned_access(false),
79 _mismatched_access(false),
80 _unsafe_access(false),
81 _barrier_data(0) {
82 init_class_id(Class_Mem);
83 debug_only(_adr_type=at; adr_type();)
84 }
85 MemNode( Node *c0, Node *c1, Node *c2, const TypePtr* at, Node *c3, Node *c4) :
86 Node(c0,c1,c2,c3,c4),
87 _unaligned_access(false),
88 _mismatched_access(false),
89 _unsafe_access(false),
90 _barrier_data(0) {
91 init_class_id(Class_Mem);
92 debug_only(_adr_type=at; adr_type();)
93 }
94
95 virtual Node* find_previous_arraycopy(PhaseTransform* phase, Node* ld_alloc, Node*& mem, bool can_see_stored_value) const { return NULL; }
96 ArrayCopyNode* find_array_copy_clone(PhaseTransform* phase, Node* ld_alloc, Node* mem) const;
97 static bool check_if_adr_maybe_raw(Node* adr);
98
99 public:
100 // Helpers for the optimizer. Documented in memnode.cpp.
101 static bool detect_ptr_independence(Node* p1, AllocateNode* a1,
102 Node* p2, AllocateNode* a2,
103 PhaseTransform* phase);
104 static bool adr_phi_is_loop_invariant(Node* adr_phi, Node* cast);
105
106 static Node *optimize_simple_memory_chain(Node *mchain, const TypeOopPtr *t_oop, Node *load, PhaseGVN *phase);
107 static Node *optimize_memory_chain(Node *mchain, const TypePtr *t_adr, Node *load, PhaseGVN *phase);
108 // This one should probably be a phase-specific function:
109 static bool all_controls_dominate(Node* dom, Node* sub);
110
111 virtual const class TypePtr *adr_type() const; // returns bottom_type of address
112
113 // Shared code for Ideal methods:
114 Node *Ideal_common(PhaseGVN *phase, bool can_reshape); // Return -1 for short-circuit NULL.
115
116 // Helper function for adr_type() implementations.
117 static const TypePtr* calculate_adr_type(const Type* t, const TypePtr* cross_check = NULL);
118
119 // Raw access function, to allow copying of adr_type efficiently in
120 // product builds and retain the debug info for debug builds.
121 const TypePtr *raw_adr_type() const {
122 #ifdef ASSERT
123 return _adr_type;
124 #else
125 return 0;
126 #endif
127 }
128
129 // Map a load or store opcode to its corresponding store opcode.
130 // (Return -1 if unknown.)
131 virtual int store_Opcode() const { return -1; }
132
133 // What is the type of the value in memory? (T_VOID mean "unspecified".)
134 virtual BasicType memory_type() const = 0;
135 virtual int memory_size() const {
136 #ifdef ASSERT
137 return type2aelembytes(memory_type(), true);
138 #else
139 return type2aelembytes(memory_type());
140 #endif
141 }
142
143 uint8_t barrier_data() { return _barrier_data; }
144 void set_barrier_data(uint8_t barrier_data) { _barrier_data = barrier_data; }
145
146 // Search through memory states which precede this node (load or store).
147 // Look for an exact match for the address, with no intervening
148 // aliased stores.
149 Node* find_previous_store(PhaseTransform* phase);
150
151 // Can this node (load or store) accurately see a stored value in
152 // the given memory state? (The state may or may not be in(Memory).)
153 Node* can_see_stored_value(Node* st, PhaseTransform* phase) const;
154
155 void set_unaligned_access() { _unaligned_access = true; }
156 bool is_unaligned_access() const { return _unaligned_access; }
157 void set_mismatched_access() { _mismatched_access = true; }
158 bool is_mismatched_access() const { return _mismatched_access; }
159 void set_unsafe_access() { _unsafe_access = true; }
160 bool is_unsafe_access() const { return _unsafe_access; }
161
162 #ifndef PRODUCT
163 static void dump_adr_type(const Node* mem, const TypePtr* adr_type, outputStream *st);
164 virtual void dump_spec(outputStream *st) const;
165 #endif
166 };
167
168 //------------------------------LoadNode---------------------------------------
169 // Load value; requires Memory and Address
170 class LoadNode : public MemNode {
171 public:
172 // Some loads (from unsafe) should be pinned: they don't depend only
173 // on the dominating test. The field _control_dependency below records
174 // whether that node depends only on the dominating test.
175 // Pinned and UnknownControl are similar, but differ in that Pinned
176 // loads are not allowed to float across safepoints, whereas UnknownControl
177 // loads are allowed to do that. Therefore, Pinned is stricter.
178 enum ControlDependency {
179 Pinned,
180 UnknownControl,
181 DependsOnlyOnTest
182 };
183
184 private:
185 // LoadNode::hash() doesn't take the _control_dependency field
186 // into account: If the graph already has a non-pinned LoadNode and
187 // we add a pinned LoadNode with the same inputs, it's safe for GVN
188 // to replace the pinned LoadNode with the non-pinned LoadNode,
189 // otherwise it wouldn't be safe to have a non pinned LoadNode with
190 // those inputs in the first place. If the graph already has a
191 // pinned LoadNode and we add a non pinned LoadNode with the same
192 // inputs, it's safe (but suboptimal) for GVN to replace the
193 // non-pinned LoadNode by the pinned LoadNode.
194 ControlDependency _control_dependency;
195
196 // On platforms with weak memory ordering (e.g., PPC, Ia64) we distinguish
197 // loads that can be reordered, and such requiring acquire semantics to
198 // adhere to the Java specification. The required behaviour is stored in
199 // this field.
200 const MemOrd _mo;
201
202 AllocateNode* is_new_object_mark_load(PhaseGVN *phase) const;
203
204 protected:
205 virtual bool cmp(const Node &n) const;
206 virtual uint size_of() const; // Size is bigger
207 // Should LoadNode::Ideal() attempt to remove control edges?
208 virtual bool can_remove_control() const;
209 const Type* const _type; // What kind of value is loaded?
210
211 virtual Node* find_previous_arraycopy(PhaseTransform* phase, Node* ld_alloc, Node*& mem, bool can_see_stored_value) const;
212 public:
213
214 LoadNode(Node *c, Node *mem, Node *adr, const TypePtr* at, const Type *rt, MemOrd mo, ControlDependency control_dependency)
215 : MemNode(c,mem,adr,at), _control_dependency(control_dependency), _mo(mo), _type(rt) {
216 init_class_id(Class_Load);
217 }
218 inline bool is_unordered() const { return !is_acquire(); }
219 inline bool is_acquire() const {
220 assert(_mo == unordered || _mo == acquire, "unexpected");
221 return _mo == acquire;
222 }
223 inline bool is_unsigned() const {
224 int lop = Opcode();
225 return (lop == Op_LoadUB) || (lop == Op_LoadUS);
226 }
227
228 // Polymorphic factory method:
229 static Node* make(PhaseGVN& gvn, Node *c, Node *mem, Node *adr,
230 const TypePtr* at, const Type *rt, BasicType bt,
231 MemOrd mo, ControlDependency control_dependency = DependsOnlyOnTest,
232 bool unaligned = false, bool mismatched = false, bool unsafe = false,
233 uint8_t barrier_data = 0);
234
235 virtual uint hash() const; // Check the type
236
237 // Handle algebraic identities here. If we have an identity, return the Node
238 // we are equivalent to. We look for Load of a Store.
239 virtual Node* Identity(PhaseGVN* phase);
240
241 // If the load is from Field memory and the pointer is non-null, it might be possible to
242 // zero out the control input.
243 // If the offset is constant and the base is an object allocation,
244 // try to hook me up to the exact initializing store.
245 virtual Node *Ideal(PhaseGVN *phase, bool can_reshape);
246
247 // Split instance field load through Phi.
248 Node* split_through_phi(PhaseGVN *phase);
249
250 // Recover original value from boxed values
251 Node *eliminate_autobox(PhaseIterGVN *igvn);
252
253 // Compute a new Type for this node. Basically we just do the pre-check,
254 // then call the virtual add() to set the type.
255 virtual const Type* Value(PhaseGVN* phase) const;
256
257 // Common methods for LoadKlass and LoadNKlass nodes.
258 const Type* klass_value_common(PhaseGVN* phase) const;
259 Node* klass_identity_common(PhaseGVN* phase);
260
261 virtual uint ideal_reg() const;
262 virtual const Type *bottom_type() const;
263 // Following method is copied from TypeNode:
264 void set_type(const Type* t) {
265 assert(t != NULL, "sanity");
266 debug_only(uint check_hash = (VerifyHashTableKeys && _hash_lock) ? hash() : NO_HASH);
267 *(const Type**)&_type = t; // cast away const-ness
268 // If this node is in the hash table, make sure it doesn't need a rehash.
269 assert(check_hash == NO_HASH || check_hash == hash(), "type change must preserve hash code");
270 }
271 const Type* type() const { assert(_type != NULL, "sanity"); return _type; };
272
273 // Do not match memory edge
274 virtual uint match_edge(uint idx) const;
275
276 // Map a load opcode to its corresponding store opcode.
277 virtual int store_Opcode() const = 0;
278
279 // Check if the load's memory input is a Phi node with the same control.
280 bool is_instance_field_load_with_local_phi(Node* ctrl);
281
282 Node* convert_to_unsigned_load(PhaseGVN& gvn);
283 Node* convert_to_signed_load(PhaseGVN& gvn);
284
285 bool has_reinterpret_variant(const Type* rt);
286 Node* convert_to_reinterpret_load(PhaseGVN& gvn, const Type* rt);
287
288 bool has_unknown_control_dependency() const { return _control_dependency == UnknownControl; }
289
290 #ifndef PRODUCT
291 virtual void dump_spec(outputStream *st) const;
292 #endif
293 #ifdef ASSERT
294 // Helper function to allow a raw load without control edge for some cases
295 static bool is_immutable_value(Node* adr);
296 #endif
297 protected:
298 const Type* load_array_final_field(const TypeKlassPtr *tkls,
299 ciKlass* klass) const;
300
301 Node* can_see_arraycopy_value(Node* st, PhaseGVN* phase) const;
302
303 // depends_only_on_test is almost always true, and needs to be almost always
304 // true to enable key hoisting & commoning optimizations. However, for the
305 // special case of RawPtr loads from TLS top & end, and other loads performed by
306 // GC barriers, the control edge carries the dependence preventing hoisting past
307 // a Safepoint instead of the memory edge. (An unfortunate consequence of having
308 // Safepoints not set Raw Memory; itself an unfortunate consequence of having Nodes
309 // which produce results (new raw memory state) inside of loops preventing all
310 // manner of other optimizations). Basically, it's ugly but so is the alternative.
311 // See comment in macro.cpp, around line 125 expand_allocate_common().
312 virtual bool depends_only_on_test() const {
313 return adr_type() != TypeRawPtr::BOTTOM && _control_dependency == DependsOnlyOnTest;
314 }
315 };
316
317 //------------------------------LoadBNode--------------------------------------
318 // Load a byte (8bits signed) from memory
319 class LoadBNode : public LoadNode {
320 public:
321 LoadBNode(Node *c, Node *mem, Node *adr, const TypePtr* at, const TypeInt *ti, MemOrd mo, ControlDependency control_dependency = DependsOnlyOnTest)
322 : LoadNode(c, mem, adr, at, ti, mo, control_dependency) {}
323 virtual int Opcode() const;
324 virtual uint ideal_reg() const { return Op_RegI; }
325 virtual Node *Ideal(PhaseGVN *phase, bool can_reshape);
326 virtual const Type* Value(PhaseGVN* phase) const;
327 virtual int store_Opcode() const { return Op_StoreB; }
328 virtual BasicType memory_type() const { return T_BYTE; }
329 };
330
331 //------------------------------LoadUBNode-------------------------------------
332 // Load a unsigned byte (8bits unsigned) from memory
333 class LoadUBNode : public LoadNode {
334 public:
335 LoadUBNode(Node* c, Node* mem, Node* adr, const TypePtr* at, const TypeInt* ti, MemOrd mo, ControlDependency control_dependency = DependsOnlyOnTest)
336 : LoadNode(c, mem, adr, at, ti, mo, control_dependency) {}
337 virtual int Opcode() const;
338 virtual uint ideal_reg() const { return Op_RegI; }
339 virtual Node* Ideal(PhaseGVN *phase, bool can_reshape);
340 virtual const Type* Value(PhaseGVN* phase) const;
341 virtual int store_Opcode() const { return Op_StoreB; }
342 virtual BasicType memory_type() const { return T_BYTE; }
343 };
344
345 //------------------------------LoadUSNode-------------------------------------
346 // Load an unsigned short/char (16bits unsigned) from memory
347 class LoadUSNode : public LoadNode {
348 public:
349 LoadUSNode(Node *c, Node *mem, Node *adr, const TypePtr* at, const TypeInt *ti, MemOrd mo, ControlDependency control_dependency = DependsOnlyOnTest)
350 : LoadNode(c, mem, adr, at, ti, mo, control_dependency) {}
351 virtual int Opcode() const;
352 virtual uint ideal_reg() const { return Op_RegI; }
353 virtual Node *Ideal(PhaseGVN *phase, bool can_reshape);
354 virtual const Type* Value(PhaseGVN* phase) const;
355 virtual int store_Opcode() const { return Op_StoreC; }
356 virtual BasicType memory_type() const { return T_CHAR; }
357 };
358
359 //------------------------------LoadSNode--------------------------------------
360 // Load a short (16bits signed) from memory
361 class LoadSNode : public LoadNode {
362 public:
363 LoadSNode(Node *c, Node *mem, Node *adr, const TypePtr* at, const TypeInt *ti, MemOrd mo, ControlDependency control_dependency = DependsOnlyOnTest)
364 : LoadNode(c, mem, adr, at, ti, mo, control_dependency) {}
365 virtual int Opcode() const;
366 virtual uint ideal_reg() const { return Op_RegI; }
367 virtual Node *Ideal(PhaseGVN *phase, bool can_reshape);
368 virtual const Type* Value(PhaseGVN* phase) const;
369 virtual int store_Opcode() const { return Op_StoreC; }
370 virtual BasicType memory_type() const { return T_SHORT; }
371 };
372
373 //------------------------------LoadINode--------------------------------------
374 // Load an integer from memory
375 class LoadINode : public LoadNode {
376 public:
377 LoadINode(Node *c, Node *mem, Node *adr, const TypePtr* at, const TypeInt *ti, MemOrd mo, ControlDependency control_dependency = DependsOnlyOnTest)
378 : LoadNode(c, mem, adr, at, ti, mo, control_dependency) {}
379 virtual int Opcode() const;
380 virtual uint ideal_reg() const { return Op_RegI; }
381 virtual int store_Opcode() const { return Op_StoreI; }
382 virtual BasicType memory_type() const { return T_INT; }
383 };
384
385 //------------------------------LoadRangeNode----------------------------------
386 // Load an array length from the array
387 class LoadRangeNode : public LoadINode {
388 public:
389 LoadRangeNode(Node *c, Node *mem, Node *adr, const TypeInt *ti = TypeInt::POS)
390 : LoadINode(c, mem, adr, TypeAryPtr::RANGE, ti, MemNode::unordered) {}
391 virtual int Opcode() const;
392 virtual const Type* Value(PhaseGVN* phase) const;
393 virtual Node* Identity(PhaseGVN* phase);
394 virtual Node *Ideal(PhaseGVN *phase, bool can_reshape);
395 };
396
397 //------------------------------LoadLNode--------------------------------------
398 // Load a long from memory
399 class LoadLNode : public LoadNode {
400 virtual uint hash() const { return LoadNode::hash() + _require_atomic_access; }
401 virtual bool cmp( const Node &n ) const {
402 return _require_atomic_access == ((LoadLNode&)n)._require_atomic_access
403 && LoadNode::cmp(n);
404 }
405 virtual uint size_of() const { return sizeof(*this); }
406 const bool _require_atomic_access; // is piecewise load forbidden?
407
408 public:
409 LoadLNode(Node *c, Node *mem, Node *adr, const TypePtr* at, const TypeLong *tl,
410 MemOrd mo, ControlDependency control_dependency = DependsOnlyOnTest, bool require_atomic_access = false)
411 : LoadNode(c, mem, adr, at, tl, mo, control_dependency), _require_atomic_access(require_atomic_access) {}
412 virtual int Opcode() const;
413 virtual uint ideal_reg() const { return Op_RegL; }
414 virtual int store_Opcode() const { return Op_StoreL; }
415 virtual BasicType memory_type() const { return T_LONG; }
416 bool require_atomic_access() const { return _require_atomic_access; }
417 static LoadLNode* make_atomic(Node* ctl, Node* mem, Node* adr, const TypePtr* adr_type,
418 const Type* rt, MemOrd mo, ControlDependency control_dependency = DependsOnlyOnTest,
419 bool unaligned = false, bool mismatched = false, bool unsafe = false, uint8_t barrier_data = 0);
420 #ifndef PRODUCT
421 virtual void dump_spec(outputStream *st) const {
422 LoadNode::dump_spec(st);
423 if (_require_atomic_access) st->print(" Atomic!");
424 }
425 #endif
426 };
427
428 //------------------------------LoadL_unalignedNode----------------------------
429 // Load a long from unaligned memory
430 class LoadL_unalignedNode : public LoadLNode {
431 public:
432 LoadL_unalignedNode(Node *c, Node *mem, Node *adr, const TypePtr* at, MemOrd mo, ControlDependency control_dependency = DependsOnlyOnTest)
433 : LoadLNode(c, mem, adr, at, TypeLong::LONG, mo, control_dependency) {}
434 virtual int Opcode() const;
435 };
436
437 //------------------------------LoadFNode--------------------------------------
438 // Load a float (64 bits) from memory
439 class LoadFNode : public LoadNode {
440 public:
441 LoadFNode(Node *c, Node *mem, Node *adr, const TypePtr* at, const Type *t, MemOrd mo, ControlDependency control_dependency = DependsOnlyOnTest)
442 : LoadNode(c, mem, adr, at, t, mo, control_dependency) {}
443 virtual int Opcode() const;
444 virtual uint ideal_reg() const { return Op_RegF; }
445 virtual int store_Opcode() const { return Op_StoreF; }
446 virtual BasicType memory_type() const { return T_FLOAT; }
447 };
448
449 //------------------------------LoadDNode--------------------------------------
450 // Load a double (64 bits) from memory
451 class LoadDNode : public LoadNode {
452 virtual uint hash() const { return LoadNode::hash() + _require_atomic_access; }
453 virtual bool cmp( const Node &n ) const {
454 return _require_atomic_access == ((LoadDNode&)n)._require_atomic_access
455 && LoadNode::cmp(n);
456 }
457 virtual uint size_of() const { return sizeof(*this); }
458 const bool _require_atomic_access; // is piecewise load forbidden?
459
460 public:
461 LoadDNode(Node *c, Node *mem, Node *adr, const TypePtr* at, const Type *t,
462 MemOrd mo, ControlDependency control_dependency = DependsOnlyOnTest, bool require_atomic_access = false)
463 : LoadNode(c, mem, adr, at, t, mo, control_dependency), _require_atomic_access(require_atomic_access) {}
464 virtual int Opcode() const;
465 virtual uint ideal_reg() const { return Op_RegD; }
466 virtual int store_Opcode() const { return Op_StoreD; }
467 virtual BasicType memory_type() const { return T_DOUBLE; }
468 bool require_atomic_access() const { return _require_atomic_access; }
469 static LoadDNode* make_atomic(Node* ctl, Node* mem, Node* adr, const TypePtr* adr_type,
470 const Type* rt, MemOrd mo, ControlDependency control_dependency = DependsOnlyOnTest,
471 bool unaligned = false, bool mismatched = false, bool unsafe = false, uint8_t barrier_data = 0);
472 #ifndef PRODUCT
473 virtual void dump_spec(outputStream *st) const {
474 LoadNode::dump_spec(st);
475 if (_require_atomic_access) st->print(" Atomic!");
476 }
477 #endif
478 };
479
480 //------------------------------LoadD_unalignedNode----------------------------
481 // Load a double from unaligned memory
482 class LoadD_unalignedNode : public LoadDNode {
483 public:
484 LoadD_unalignedNode(Node *c, Node *mem, Node *adr, const TypePtr* at, MemOrd mo, ControlDependency control_dependency = DependsOnlyOnTest)
485 : LoadDNode(c, mem, adr, at, Type::DOUBLE, mo, control_dependency) {}
486 virtual int Opcode() const;
487 };
488
489 //------------------------------LoadPNode--------------------------------------
490 // Load a pointer from memory (either object or array)
491 class LoadPNode : public LoadNode {
492 public:
493 LoadPNode(Node *c, Node *mem, Node *adr, const TypePtr *at, const TypePtr* t, MemOrd mo, ControlDependency control_dependency = DependsOnlyOnTest)
494 : LoadNode(c, mem, adr, at, t, mo, control_dependency) {}
495 virtual int Opcode() const;
496 virtual uint ideal_reg() const { return Op_RegP; }
497 virtual int store_Opcode() const { return Op_StoreP; }
498 virtual BasicType memory_type() const { return T_ADDRESS; }
499 };
500
501
502 //------------------------------LoadNNode--------------------------------------
503 // Load a narrow oop from memory (either object or array)
504 class LoadNNode : public LoadNode {
505 public:
506 LoadNNode(Node *c, Node *mem, Node *adr, const TypePtr *at, const Type* t, MemOrd mo, ControlDependency control_dependency = DependsOnlyOnTest)
507 : LoadNode(c, mem, adr, at, t, mo, control_dependency) {}
508 virtual int Opcode() const;
509 virtual uint ideal_reg() const { return Op_RegN; }
510 virtual int store_Opcode() const { return Op_StoreN; }
511 virtual BasicType memory_type() const { return T_NARROWOOP; }
512 };
513
514 //------------------------------LoadKlassNode----------------------------------
515 // Load a Klass from an object
516 class LoadKlassNode : public LoadPNode {
517 protected:
518 // In most cases, LoadKlassNode does not have the control input set. If the control
519 // input is set, it must not be removed (by LoadNode::Ideal()).
520 virtual bool can_remove_control() const;
521 public:
522 LoadKlassNode(Node *c, Node *mem, Node *adr, const TypePtr *at, const TypeKlassPtr *tk, MemOrd mo)
523 : LoadPNode(c, mem, adr, at, tk, mo) {}
524 virtual int Opcode() const;
525 virtual const Type* Value(PhaseGVN* phase) const;
526 virtual Node* Identity(PhaseGVN* phase);
527 virtual bool depends_only_on_test() const { return true; }
528
529 // Polymorphic factory method:
530 static Node* make(PhaseGVN& gvn, Node* ctl, Node* mem, Node* adr, const TypePtr* at,
531 const TypeKlassPtr* tk = TypeInstKlassPtr::OBJECT);
532 };
533
534 //------------------------------LoadNKlassNode---------------------------------
535 // Load a narrow Klass from an object.
536 class LoadNKlassNode : public LoadNNode {
537 public:
538 LoadNKlassNode(Node *c, Node *mem, Node *adr, const TypePtr *at, const TypeNarrowKlass *tk, MemOrd mo)
539 : LoadNNode(c, mem, adr, at, tk, mo) {}
540 virtual int Opcode() const;
541 virtual uint ideal_reg() const { return Op_RegN; }
542 virtual int store_Opcode() const { return Op_StoreNKlass; }
543 virtual BasicType memory_type() const { return T_NARROWKLASS; }
544
545 virtual const Type* Value(PhaseGVN* phase) const;
546 virtual Node* Identity(PhaseGVN* phase);
547 virtual bool depends_only_on_test() const { return true; }
548 };
549
550
551 //------------------------------StoreNode--------------------------------------
552 // Store value; requires Store, Address and Value
553 class StoreNode : public MemNode {
554 private:
555 // On platforms with weak memory ordering (e.g., PPC, Ia64) we distinguish
556 // stores that can be reordered, and such requiring release semantics to
557 // adhere to the Java specification. The required behaviour is stored in
558 // this field.
559 const MemOrd _mo;
560 // Needed for proper cloning.
561 virtual uint size_of() const { return sizeof(*this); }
562 protected:
563 virtual bool cmp( const Node &n ) const;
564 virtual bool depends_only_on_test() const { return false; }
565
566 Node *Ideal_masked_input (PhaseGVN *phase, uint mask);
567 Node *Ideal_sign_extended_input(PhaseGVN *phase, int num_bits);
568
569 public:
570 // We must ensure that stores of object references will be visible
571 // only after the object's initialization. So the callers of this
572 // procedure must indicate that the store requires `release'
573 // semantics, if the stored value is an object reference that might
574 // point to a new object and may become externally visible.
575 StoreNode(Node *c, Node *mem, Node *adr, const TypePtr* at, Node *val, MemOrd mo)
576 : MemNode(c, mem, adr, at, val), _mo(mo) {
577 init_class_id(Class_Store);
578 }
579 StoreNode(Node *c, Node *mem, Node *adr, const TypePtr* at, Node *val, Node *oop_store, MemOrd mo)
580 : MemNode(c, mem, adr, at, val, oop_store), _mo(mo) {
581 init_class_id(Class_Store);
582 }
583
584 inline bool is_unordered() const { return !is_release(); }
585 inline bool is_release() const {
586 assert((_mo == unordered || _mo == release), "unexpected");
587 return _mo == release;
588 }
589
590 // Conservatively release stores of object references in order to
591 // ensure visibility of object initialization.
592 static inline MemOrd release_if_reference(const BasicType t) {
593 #ifdef AARCH64
594 // AArch64 doesn't need a release store here because object
595 // initialization contains the necessary barriers.
596 return unordered;
597 #else
598 const MemOrd mo = (t == T_ARRAY ||
599 t == T_ADDRESS || // Might be the address of an object reference (`boxing').
600 t == T_OBJECT) ? release : unordered;
601 return mo;
602 #endif
603 }
604
605 // Polymorphic factory method
606 //
607 // We must ensure that stores of object references will be visible
608 // only after the object's initialization. So the callers of this
609 // procedure must indicate that the store requires `release'
610 // semantics, if the stored value is an object reference that might
611 // point to a new object and may become externally visible.
612 static StoreNode* make(PhaseGVN& gvn, Node *c, Node *mem, Node *adr,
613 const TypePtr* at, Node *val, BasicType bt, MemOrd mo);
614
615 virtual uint hash() const; // Check the type
616
617 // If the store is to Field memory and the pointer is non-null, we can
618 // zero out the control input.
619 virtual Node *Ideal(PhaseGVN *phase, bool can_reshape);
620
621 // Compute a new Type for this node. Basically we just do the pre-check,
622 // then call the virtual add() to set the type.
623 virtual const Type* Value(PhaseGVN* phase) const;
624
625 // Check for identity function on memory (Load then Store at same address)
626 virtual Node* Identity(PhaseGVN* phase);
627
628 // Do not match memory edge
629 virtual uint match_edge(uint idx) const;
630
631 virtual const Type *bottom_type() const; // returns Type::MEMORY
632
633 // Map a store opcode to its corresponding own opcode, trivially.
634 virtual int store_Opcode() const { return Opcode(); }
635
636 // have all possible loads of the value stored been optimized away?
637 bool value_never_loaded(PhaseTransform *phase) const;
638
639 bool has_reinterpret_variant(const Type* vt);
640 Node* convert_to_reinterpret_store(PhaseGVN& gvn, Node* val, const Type* vt);
641
642 MemBarNode* trailing_membar() const;
643 };
644
645 //------------------------------StoreBNode-------------------------------------
646 // Store byte to memory
647 class StoreBNode : public StoreNode {
648 public:
649 StoreBNode(Node *c, Node *mem, Node *adr, const TypePtr* at, Node *val, MemOrd mo)
650 : StoreNode(c, mem, adr, at, val, mo) {}
651 virtual int Opcode() const;
652 virtual Node *Ideal(PhaseGVN *phase, bool can_reshape);
653 virtual BasicType memory_type() const { return T_BYTE; }
654 };
655
656 //------------------------------StoreCNode-------------------------------------
657 // Store char/short to memory
658 class StoreCNode : public StoreNode {
659 public:
660 StoreCNode(Node *c, Node *mem, Node *adr, const TypePtr* at, Node *val, MemOrd mo)
661 : StoreNode(c, mem, adr, at, val, mo) {}
662 virtual int Opcode() const;
663 virtual Node *Ideal(PhaseGVN *phase, bool can_reshape);
664 virtual BasicType memory_type() const { return T_CHAR; }
665 };
666
667 //------------------------------StoreINode-------------------------------------
668 // Store int to memory
669 class StoreINode : public StoreNode {
670 public:
671 StoreINode(Node *c, Node *mem, Node *adr, const TypePtr* at, Node *val, MemOrd mo)
672 : StoreNode(c, mem, adr, at, val, mo) {}
673 virtual int Opcode() const;
674 virtual BasicType memory_type() const { return T_INT; }
675 };
676
677 //------------------------------StoreLNode-------------------------------------
678 // Store long to memory
679 class StoreLNode : public StoreNode {
680 virtual uint hash() const { return StoreNode::hash() + _require_atomic_access; }
681 virtual bool cmp( const Node &n ) const {
682 return _require_atomic_access == ((StoreLNode&)n)._require_atomic_access
683 && StoreNode::cmp(n);
684 }
685 virtual uint size_of() const { return sizeof(*this); }
686 const bool _require_atomic_access; // is piecewise store forbidden?
687
688 public:
689 StoreLNode(Node *c, Node *mem, Node *adr, const TypePtr* at, Node *val, MemOrd mo, bool require_atomic_access = false)
690 : StoreNode(c, mem, adr, at, val, mo), _require_atomic_access(require_atomic_access) {}
691 virtual int Opcode() const;
692 virtual BasicType memory_type() const { return T_LONG; }
693 bool require_atomic_access() const { return _require_atomic_access; }
694 static StoreLNode* make_atomic(Node* ctl, Node* mem, Node* adr, const TypePtr* adr_type, Node* val, MemOrd mo);
695 #ifndef PRODUCT
696 virtual void dump_spec(outputStream *st) const {
697 StoreNode::dump_spec(st);
698 if (_require_atomic_access) st->print(" Atomic!");
699 }
700 #endif
701 };
702
703 //------------------------------StoreFNode-------------------------------------
704 // Store float to memory
705 class StoreFNode : public StoreNode {
706 public:
707 StoreFNode(Node *c, Node *mem, Node *adr, const TypePtr* at, Node *val, MemOrd mo)
708 : StoreNode(c, mem, adr, at, val, mo) {}
709 virtual int Opcode() const;
710 virtual BasicType memory_type() const { return T_FLOAT; }
711 };
712
713 //------------------------------StoreDNode-------------------------------------
714 // Store double to memory
715 class StoreDNode : public StoreNode {
716 virtual uint hash() const { return StoreNode::hash() + _require_atomic_access; }
717 virtual bool cmp( const Node &n ) const {
718 return _require_atomic_access == ((StoreDNode&)n)._require_atomic_access
719 && StoreNode::cmp(n);
720 }
721 virtual uint size_of() const { return sizeof(*this); }
722 const bool _require_atomic_access; // is piecewise store forbidden?
723 public:
724 StoreDNode(Node *c, Node *mem, Node *adr, const TypePtr* at, Node *val,
725 MemOrd mo, bool require_atomic_access = false)
726 : StoreNode(c, mem, adr, at, val, mo), _require_atomic_access(require_atomic_access) {}
727 virtual int Opcode() const;
728 virtual BasicType memory_type() const { return T_DOUBLE; }
729 bool require_atomic_access() const { return _require_atomic_access; }
730 static StoreDNode* make_atomic(Node* ctl, Node* mem, Node* adr, const TypePtr* adr_type, Node* val, MemOrd mo);
731 #ifndef PRODUCT
732 virtual void dump_spec(outputStream *st) const {
733 StoreNode::dump_spec(st);
734 if (_require_atomic_access) st->print(" Atomic!");
735 }
736 #endif
737
738 };
739
740 //------------------------------StorePNode-------------------------------------
741 // Store pointer to memory
742 class StorePNode : public StoreNode {
743 public:
744 StorePNode(Node *c, Node *mem, Node *adr, const TypePtr* at, Node *val, MemOrd mo)
745 : StoreNode(c, mem, adr, at, val, mo) {}
746 virtual int Opcode() const;
747 virtual BasicType memory_type() const { return T_ADDRESS; }
748 };
749
750 //------------------------------StoreNNode-------------------------------------
751 // Store narrow oop to memory
752 class StoreNNode : public StoreNode {
753 public:
754 StoreNNode(Node *c, Node *mem, Node *adr, const TypePtr* at, Node *val, MemOrd mo)
755 : StoreNode(c, mem, adr, at, val, mo) {}
756 virtual int Opcode() const;
757 virtual BasicType memory_type() const { return T_NARROWOOP; }
758 };
759
760 //------------------------------StoreNKlassNode--------------------------------------
761 // Store narrow klass to memory
762 class StoreNKlassNode : public StoreNNode {
763 public:
764 StoreNKlassNode(Node *c, Node *mem, Node *adr, const TypePtr* at, Node *val, MemOrd mo)
765 : StoreNNode(c, mem, adr, at, val, mo) {}
766 virtual int Opcode() const;
767 virtual BasicType memory_type() const { return T_NARROWKLASS; }
768 };
769
770 //------------------------------StoreCMNode-----------------------------------
771 // Store card-mark byte to memory for CM
772 // The last StoreCM before a SafePoint must be preserved and occur after its "oop" store
773 // Preceeding equivalent StoreCMs may be eliminated.
774 class StoreCMNode : public StoreNode {
775 private:
776 virtual uint hash() const { return StoreNode::hash() + _oop_alias_idx; }
777 virtual bool cmp( const Node &n ) const {
778 return _oop_alias_idx == ((StoreCMNode&)n)._oop_alias_idx
779 && StoreNode::cmp(n);
780 }
781 virtual uint size_of() const { return sizeof(*this); }
782 int _oop_alias_idx; // The alias_idx of OopStore
783
784 public:
785 StoreCMNode( Node *c, Node *mem, Node *adr, const TypePtr* at, Node *val, Node *oop_store, int oop_alias_idx ) :
786 StoreNode(c, mem, adr, at, val, oop_store, MemNode::release),
787 _oop_alias_idx(oop_alias_idx) {
788 assert(_oop_alias_idx >= Compile::AliasIdxRaw ||
789 _oop_alias_idx == Compile::AliasIdxBot && Compile::current()->AliasLevel() == 0,
790 "bad oop alias idx");
791 }
792 virtual int Opcode() const;
793 virtual Node* Identity(PhaseGVN* phase);
794 virtual Node *Ideal(PhaseGVN *phase, bool can_reshape);
795 virtual const Type* Value(PhaseGVN* phase) const;
796 virtual BasicType memory_type() const { return T_VOID; } // unspecific
797 int oop_alias_idx() const { return _oop_alias_idx; }
798 };
799
800 //------------------------------LoadPLockedNode---------------------------------
801 // Load-locked a pointer from memory (either object or array).
802 // On Sparc & Intel this is implemented as a normal pointer load.
803 // On PowerPC and friends it's a real load-locked.
804 class LoadPLockedNode : public LoadPNode {
805 public:
806 LoadPLockedNode(Node *c, Node *mem, Node *adr, MemOrd mo)
807 : LoadPNode(c, mem, adr, TypeRawPtr::BOTTOM, TypeRawPtr::BOTTOM, mo) {}
808 virtual int Opcode() const;
809 virtual int store_Opcode() const { return Op_StorePConditional; }
810 virtual bool depends_only_on_test() const { return true; }
811 };
812
813 //------------------------------SCMemProjNode---------------------------------------
814 // This class defines a projection of the memory state of a store conditional node.
815 // These nodes return a value, but also update memory.
816 class SCMemProjNode : public ProjNode {
817 public:
818 enum {SCMEMPROJCON = (uint)-2};
819 SCMemProjNode( Node *src) : ProjNode( src, SCMEMPROJCON) { }
820 virtual int Opcode() const;
821 virtual bool is_CFG() const { return false; }
822 virtual const Type *bottom_type() const {return Type::MEMORY;}
823 virtual const TypePtr *adr_type() const {
824 Node* ctrl = in(0);
825 if (ctrl == NULL) return NULL; // node is dead
826 return ctrl->in(MemNode::Memory)->adr_type();
827 }
828 virtual uint ideal_reg() const { return 0;} // memory projections don't have a register
829 virtual const Type* Value(PhaseGVN* phase) const;
830 #ifndef PRODUCT
831 virtual void dump_spec(outputStream *st) const {};
832 #endif
833 };
834
835 //------------------------------LoadStoreNode---------------------------
836 // Note: is_Mem() method returns 'true' for this class.
837 class LoadStoreNode : public Node {
838 private:
839 const Type* const _type; // What kind of value is loaded?
840 const TypePtr* _adr_type; // What kind of memory is being addressed?
841 uint8_t _barrier_data; // Bit field with barrier information
842 virtual uint size_of() const; // Size is bigger
843 public:
844 LoadStoreNode( Node *c, Node *mem, Node *adr, Node *val, const TypePtr* at, const Type* rt, uint required );
845 virtual bool depends_only_on_test() const { return false; }
846 virtual uint match_edge(uint idx) const { return idx == MemNode::Address || idx == MemNode::ValueIn; }
847
848 virtual const Type *bottom_type() const { return _type; }
849 virtual uint ideal_reg() const;
850 virtual const class TypePtr *adr_type() const { return _adr_type; } // returns bottom_type of address
851 virtual const Type* Value(PhaseGVN* phase) const;
852
853 bool result_not_used() const;
854 MemBarNode* trailing_membar() const;
855
856 uint8_t barrier_data() { return _barrier_data; }
857 void set_barrier_data(uint8_t barrier_data) { _barrier_data = barrier_data; }
858 };
859
860 class LoadStoreConditionalNode : public LoadStoreNode {
861 public:
862 enum {
863 ExpectedIn = MemNode::ValueIn+1 // One more input than MemNode
864 };
865 LoadStoreConditionalNode(Node *c, Node *mem, Node *adr, Node *val, Node *ex);
866 virtual const Type* Value(PhaseGVN* phase) const;
867 };
868
869 //------------------------------StorePConditionalNode---------------------------
870 // Conditionally store pointer to memory, if no change since prior
871 // load-locked. Sets flags for success or failure of the store.
872 class StorePConditionalNode : public LoadStoreConditionalNode {
873 public:
874 StorePConditionalNode( Node *c, Node *mem, Node *adr, Node *val, Node *ll ) : LoadStoreConditionalNode(c, mem, adr, val, ll) { }
875 virtual int Opcode() const;
876 // Produces flags
877 virtual uint ideal_reg() const { return Op_RegFlags; }
878 };
879
880 //------------------------------StoreIConditionalNode---------------------------
881 // Conditionally store int to memory, if no change since prior
882 // load-locked. Sets flags for success or failure of the store.
883 class StoreIConditionalNode : public LoadStoreConditionalNode {
884 public:
885 StoreIConditionalNode( Node *c, Node *mem, Node *adr, Node *val, Node *ii ) : LoadStoreConditionalNode(c, mem, adr, val, ii) { }
886 virtual int Opcode() const;
887 // Produces flags
888 virtual uint ideal_reg() const { return Op_RegFlags; }
889 };
890
891 //------------------------------StoreLConditionalNode---------------------------
892 // Conditionally store long to memory, if no change since prior
893 // load-locked. Sets flags for success or failure of the store.
894 class StoreLConditionalNode : public LoadStoreConditionalNode {
895 public:
896 StoreLConditionalNode( Node *c, Node *mem, Node *adr, Node *val, Node *ll ) : LoadStoreConditionalNode(c, mem, adr, val, ll) { }
897 virtual int Opcode() const;
898 // Produces flags
899 virtual uint ideal_reg() const { return Op_RegFlags; }
900 };
901
902 class CompareAndSwapNode : public LoadStoreConditionalNode {
903 private:
904 const MemNode::MemOrd _mem_ord;
905 public:
906 CompareAndSwapNode( Node *c, Node *mem, Node *adr, Node *val, Node *ex, MemNode::MemOrd mem_ord) : LoadStoreConditionalNode(c, mem, adr, val, ex), _mem_ord(mem_ord) {}
907 MemNode::MemOrd order() const {
908 return _mem_ord;
909 }
910 virtual uint size_of() const { return sizeof(*this); }
911 };
912
913 class CompareAndExchangeNode : public LoadStoreNode {
914 private:
915 const MemNode::MemOrd _mem_ord;
916 public:
917 enum {
918 ExpectedIn = MemNode::ValueIn+1 // One more input than MemNode
919 };
920 CompareAndExchangeNode( Node *c, Node *mem, Node *adr, Node *val, Node *ex, MemNode::MemOrd mem_ord, const TypePtr* at, const Type* t) :
921 LoadStoreNode(c, mem, adr, val, at, t, 5), _mem_ord(mem_ord) {
922 init_req(ExpectedIn, ex );
923 }
924
925 MemNode::MemOrd order() const {
926 return _mem_ord;
927 }
928 virtual uint size_of() const { return sizeof(*this); }
929 };
930
931 //------------------------------CompareAndSwapBNode---------------------------
932 class CompareAndSwapBNode : public CompareAndSwapNode {
933 public:
934 CompareAndSwapBNode( Node *c, Node *mem, Node *adr, Node *val, Node *ex, MemNode::MemOrd mem_ord) : CompareAndSwapNode(c, mem, adr, val, ex, mem_ord) { }
935 virtual int Opcode() const;
936 };
937
938 //------------------------------CompareAndSwapSNode---------------------------
939 class CompareAndSwapSNode : public CompareAndSwapNode {
940 public:
941 CompareAndSwapSNode( Node *c, Node *mem, Node *adr, Node *val, Node *ex, MemNode::MemOrd mem_ord) : CompareAndSwapNode(c, mem, adr, val, ex, mem_ord) { }
942 virtual int Opcode() const;
943 };
944
945 //------------------------------CompareAndSwapINode---------------------------
946 class CompareAndSwapINode : public CompareAndSwapNode {
947 public:
948 CompareAndSwapINode( Node *c, Node *mem, Node *adr, Node *val, Node *ex, MemNode::MemOrd mem_ord) : CompareAndSwapNode(c, mem, adr, val, ex, mem_ord) { }
949 virtual int Opcode() const;
950 };
951
952 //------------------------------CompareAndSwapLNode---------------------------
953 class CompareAndSwapLNode : public CompareAndSwapNode {
954 public:
955 CompareAndSwapLNode( Node *c, Node *mem, Node *adr, Node *val, Node *ex, MemNode::MemOrd mem_ord) : CompareAndSwapNode(c, mem, adr, val, ex, mem_ord) { }
956 virtual int Opcode() const;
957 };
958
959 //------------------------------CompareAndSwapPNode---------------------------
960 class CompareAndSwapPNode : public CompareAndSwapNode {
961 public:
962 CompareAndSwapPNode( Node *c, Node *mem, Node *adr, Node *val, Node *ex, MemNode::MemOrd mem_ord) : CompareAndSwapNode(c, mem, adr, val, ex, mem_ord) { }
963 virtual int Opcode() const;
964 };
965
966 //------------------------------CompareAndSwapNNode---------------------------
967 class CompareAndSwapNNode : public CompareAndSwapNode {
968 public:
969 CompareAndSwapNNode( Node *c, Node *mem, Node *adr, Node *val, Node *ex, MemNode::MemOrd mem_ord) : CompareAndSwapNode(c, mem, adr, val, ex, mem_ord) { }
970 virtual int Opcode() const;
971 };
972
973 //------------------------------WeakCompareAndSwapBNode---------------------------
974 class WeakCompareAndSwapBNode : public CompareAndSwapNode {
975 public:
976 WeakCompareAndSwapBNode( Node *c, Node *mem, Node *adr, Node *val, Node *ex, MemNode::MemOrd mem_ord) : CompareAndSwapNode(c, mem, adr, val, ex, mem_ord) { }
977 virtual int Opcode() const;
978 };
979
980 //------------------------------WeakCompareAndSwapSNode---------------------------
981 class WeakCompareAndSwapSNode : public CompareAndSwapNode {
982 public:
983 WeakCompareAndSwapSNode( Node *c, Node *mem, Node *adr, Node *val, Node *ex, MemNode::MemOrd mem_ord) : CompareAndSwapNode(c, mem, adr, val, ex, mem_ord) { }
984 virtual int Opcode() const;
985 };
986
987 //------------------------------WeakCompareAndSwapINode---------------------------
988 class WeakCompareAndSwapINode : public CompareAndSwapNode {
989 public:
990 WeakCompareAndSwapINode( Node *c, Node *mem, Node *adr, Node *val, Node *ex, MemNode::MemOrd mem_ord) : CompareAndSwapNode(c, mem, adr, val, ex, mem_ord) { }
991 virtual int Opcode() const;
992 };
993
994 //------------------------------WeakCompareAndSwapLNode---------------------------
995 class WeakCompareAndSwapLNode : public CompareAndSwapNode {
996 public:
997 WeakCompareAndSwapLNode( Node *c, Node *mem, Node *adr, Node *val, Node *ex, MemNode::MemOrd mem_ord) : CompareAndSwapNode(c, mem, adr, val, ex, mem_ord) { }
998 virtual int Opcode() const;
999 };
1000
1001 //------------------------------WeakCompareAndSwapPNode---------------------------
1002 class WeakCompareAndSwapPNode : public CompareAndSwapNode {
1003 public:
1004 WeakCompareAndSwapPNode( Node *c, Node *mem, Node *adr, Node *val, Node *ex, MemNode::MemOrd mem_ord) : CompareAndSwapNode(c, mem, adr, val, ex, mem_ord) { }
1005 virtual int Opcode() const;
1006 };
1007
1008 //------------------------------WeakCompareAndSwapNNode---------------------------
1009 class WeakCompareAndSwapNNode : public CompareAndSwapNode {
1010 public:
1011 WeakCompareAndSwapNNode( Node *c, Node *mem, Node *adr, Node *val, Node *ex, MemNode::MemOrd mem_ord) : CompareAndSwapNode(c, mem, adr, val, ex, mem_ord) { }
1012 virtual int Opcode() const;
1013 };
1014
1015 //------------------------------CompareAndExchangeBNode---------------------------
1016 class CompareAndExchangeBNode : public CompareAndExchangeNode {
1017 public:
1018 CompareAndExchangeBNode( Node *c, Node *mem, Node *adr, Node *val, Node *ex, const TypePtr* at, MemNode::MemOrd mem_ord) : CompareAndExchangeNode(c, mem, adr, val, ex, mem_ord, at, TypeInt::BYTE) { }
1019 virtual int Opcode() const;
1020 };
1021
1022
1023 //------------------------------CompareAndExchangeSNode---------------------------
1024 class CompareAndExchangeSNode : public CompareAndExchangeNode {
1025 public:
1026 CompareAndExchangeSNode( Node *c, Node *mem, Node *adr, Node *val, Node *ex, const TypePtr* at, MemNode::MemOrd mem_ord) : CompareAndExchangeNode(c, mem, adr, val, ex, mem_ord, at, TypeInt::SHORT) { }
1027 virtual int Opcode() const;
1028 };
1029
1030 //------------------------------CompareAndExchangeLNode---------------------------
1031 class CompareAndExchangeLNode : public CompareAndExchangeNode {
1032 public:
1033 CompareAndExchangeLNode( Node *c, Node *mem, Node *adr, Node *val, Node *ex, const TypePtr* at, MemNode::MemOrd mem_ord) : CompareAndExchangeNode(c, mem, adr, val, ex, mem_ord, at, TypeLong::LONG) { }
1034 virtual int Opcode() const;
1035 };
1036
1037
1038 //------------------------------CompareAndExchangeINode---------------------------
1039 class CompareAndExchangeINode : public CompareAndExchangeNode {
1040 public:
1041 CompareAndExchangeINode( Node *c, Node *mem, Node *adr, Node *val, Node *ex, const TypePtr* at, MemNode::MemOrd mem_ord) : CompareAndExchangeNode(c, mem, adr, val, ex, mem_ord, at, TypeInt::INT) { }
1042 virtual int Opcode() const;
1043 };
1044
1045
1046 //------------------------------CompareAndExchangePNode---------------------------
1047 class CompareAndExchangePNode : public CompareAndExchangeNode {
1048 public:
1049 CompareAndExchangePNode( Node *c, Node *mem, Node *adr, Node *val, Node *ex, const TypePtr* at, const Type* t, MemNode::MemOrd mem_ord) : CompareAndExchangeNode(c, mem, adr, val, ex, mem_ord, at, t) { }
1050 virtual int Opcode() const;
1051 };
1052
1053 //------------------------------CompareAndExchangeNNode---------------------------
1054 class CompareAndExchangeNNode : public CompareAndExchangeNode {
1055 public:
1056 CompareAndExchangeNNode( Node *c, Node *mem, Node *adr, Node *val, Node *ex, const TypePtr* at, const Type* t, MemNode::MemOrd mem_ord) : CompareAndExchangeNode(c, mem, adr, val, ex, mem_ord, at, t) { }
1057 virtual int Opcode() const;
1058 };
1059
1060 //------------------------------GetAndAddBNode---------------------------
1061 class GetAndAddBNode : public LoadStoreNode {
1062 public:
1063 GetAndAddBNode( Node *c, Node *mem, Node *adr, Node *val, const TypePtr* at ) : LoadStoreNode(c, mem, adr, val, at, TypeInt::BYTE, 4) { }
1064 virtual int Opcode() const;
1065 };
1066
1067 //------------------------------GetAndAddSNode---------------------------
1068 class GetAndAddSNode : public LoadStoreNode {
1069 public:
1070 GetAndAddSNode( Node *c, Node *mem, Node *adr, Node *val, const TypePtr* at ) : LoadStoreNode(c, mem, adr, val, at, TypeInt::SHORT, 4) { }
1071 virtual int Opcode() const;
1072 };
1073
1074 //------------------------------GetAndAddINode---------------------------
1075 class GetAndAddINode : public LoadStoreNode {
1076 public:
1077 GetAndAddINode( Node *c, Node *mem, Node *adr, Node *val, const TypePtr* at ) : LoadStoreNode(c, mem, adr, val, at, TypeInt::INT, 4) { }
1078 virtual int Opcode() const;
1079 };
1080
1081 //------------------------------GetAndAddLNode---------------------------
1082 class GetAndAddLNode : public LoadStoreNode {
1083 public:
1084 GetAndAddLNode( Node *c, Node *mem, Node *adr, Node *val, const TypePtr* at ) : LoadStoreNode(c, mem, adr, val, at, TypeLong::LONG, 4) { }
1085 virtual int Opcode() const;
1086 };
1087
1088 //------------------------------GetAndSetBNode---------------------------
1089 class GetAndSetBNode : public LoadStoreNode {
1090 public:
1091 GetAndSetBNode( Node *c, Node *mem, Node *adr, Node *val, const TypePtr* at ) : LoadStoreNode(c, mem, adr, val, at, TypeInt::BYTE, 4) { }
1092 virtual int Opcode() const;
1093 };
1094
1095 //------------------------------GetAndSetSNode---------------------------
1096 class GetAndSetSNode : public LoadStoreNode {
1097 public:
1098 GetAndSetSNode( Node *c, Node *mem, Node *adr, Node *val, const TypePtr* at ) : LoadStoreNode(c, mem, adr, val, at, TypeInt::SHORT, 4) { }
1099 virtual int Opcode() const;
1100 };
1101
1102 //------------------------------GetAndSetINode---------------------------
1103 class GetAndSetINode : public LoadStoreNode {
1104 public:
1105 GetAndSetINode( Node *c, Node *mem, Node *adr, Node *val, const TypePtr* at ) : LoadStoreNode(c, mem, adr, val, at, TypeInt::INT, 4) { }
1106 virtual int Opcode() const;
1107 };
1108
1109 //------------------------------GetAndSetLNode---------------------------
1110 class GetAndSetLNode : public LoadStoreNode {
1111 public:
1112 GetAndSetLNode( Node *c, Node *mem, Node *adr, Node *val, const TypePtr* at ) : LoadStoreNode(c, mem, adr, val, at, TypeLong::LONG, 4) { }
1113 virtual int Opcode() const;
1114 };
1115
1116 //------------------------------GetAndSetPNode---------------------------
1117 class GetAndSetPNode : public LoadStoreNode {
1118 public:
1119 GetAndSetPNode( Node *c, Node *mem, Node *adr, Node *val, const TypePtr* at, const Type* t ) : LoadStoreNode(c, mem, adr, val, at, t, 4) { }
1120 virtual int Opcode() const;
1121 };
1122
1123 //------------------------------GetAndSetNNode---------------------------
1124 class GetAndSetNNode : public LoadStoreNode {
1125 public:
1126 GetAndSetNNode( Node *c, Node *mem, Node *adr, Node *val, const TypePtr* at, const Type* t ) : LoadStoreNode(c, mem, adr, val, at, t, 4) { }
1127 virtual int Opcode() const;
1128 };
1129
1130 //------------------------------ClearArray-------------------------------------
1131 class ClearArrayNode: public Node {
1132 private:
1133 bool _is_large;
1134 public:
1135 ClearArrayNode( Node *ctrl, Node *arymem, Node *word_cnt, Node *base, bool is_large)
1136 : Node(ctrl,arymem,word_cnt,base), _is_large(is_large) {
1137 init_class_id(Class_ClearArray);
1138 }
1139 virtual int Opcode() const;
1140 virtual const Type *bottom_type() const { return Type::MEMORY; }
1141 // ClearArray modifies array elements, and so affects only the
1142 // array memory addressed by the bottom_type of its base address.
1143 virtual const class TypePtr *adr_type() const;
1144 virtual Node* Identity(PhaseGVN* phase);
1145 virtual Node *Ideal(PhaseGVN *phase, bool can_reshape);
1146 virtual uint match_edge(uint idx) const;
1147 bool is_large() const { return _is_large; }
1148
1149 // Clear the given area of an object or array.
1150 // The start offset must always be aligned mod BytesPerInt.
1151 // The end offset must always be aligned mod BytesPerLong.
1152 // Return the new memory.
1153 static Node* clear_memory(Node* control, Node* mem, Node* dest,
1154 intptr_t start_offset,
1155 intptr_t end_offset,
1156 PhaseGVN* phase);
1157 static Node* clear_memory(Node* control, Node* mem, Node* dest,
1158 intptr_t start_offset,
1159 Node* end_offset,
1160 PhaseGVN* phase);
1161 static Node* clear_memory(Node* control, Node* mem, Node* dest,
1162 Node* start_offset,
1163 Node* end_offset,
1164 PhaseGVN* phase);
1165 // Return allocation input memory edge if it is different instance
1166 // or itself if it is the one we are looking for.
1167 static bool step_through(Node** np, uint instance_id, PhaseTransform* phase);
1168 };
1169
1170 //------------------------------MemBar-----------------------------------------
1171 // There are different flavors of Memory Barriers to match the Java Memory
1172 // Model. Monitor-enter and volatile-load act as Aquires: no following ref
1173 // can be moved to before them. We insert a MemBar-Acquire after a FastLock or
1174 // volatile-load. Monitor-exit and volatile-store act as Release: no
1175 // preceding ref can be moved to after them. We insert a MemBar-Release
1176 // before a FastUnlock or volatile-store. All volatiles need to be
1177 // serialized, so we follow all volatile-stores with a MemBar-Volatile to
1178 // separate it from any following volatile-load.
1179 class MemBarNode: public MultiNode {
1180 virtual uint hash() const ; // { return NO_HASH; }
1181 virtual bool cmp( const Node &n ) const ; // Always fail, except on self
1182
1183 virtual uint size_of() const { return sizeof(*this); }
1184 // Memory type this node is serializing. Usually either rawptr or bottom.
1185 const TypePtr* _adr_type;
1186
1187 // How is this membar related to a nearby memory access?
1188 enum {
1189 Standalone,
1190 TrailingLoad,
1191 TrailingStore,
1192 LeadingStore,
1193 TrailingLoadStore,
1194 LeadingLoadStore,
1195 TrailingPartialArrayCopy
1196 } _kind;
1197
1198 #ifdef ASSERT
1199 uint _pair_idx;
1200 #endif
1201
1202 public:
1203 enum {
1204 Precedent = TypeFunc::Parms // optional edge to force precedence
1205 };
1206 MemBarNode(Compile* C, int alias_idx, Node* precedent);
1207 virtual int Opcode() const = 0;
1208 virtual const class TypePtr *adr_type() const { return _adr_type; }
1209 virtual const Type* Value(PhaseGVN* phase) const;
1210 virtual Node *Ideal(PhaseGVN *phase, bool can_reshape);
1211 virtual uint match_edge(uint idx) const { return 0; }
1212 virtual const Type *bottom_type() const { return TypeTuple::MEMBAR; }
1213 virtual Node *match( const ProjNode *proj, const Matcher *m );
1214 // Factory method. Builds a wide or narrow membar.
1215 // Optional 'precedent' becomes an extra edge if not null.
1216 static MemBarNode* make(Compile* C, int opcode,
1217 int alias_idx = Compile::AliasIdxBot,
1218 Node* precedent = NULL);
1219
1220 MemBarNode* trailing_membar() const;
1221 MemBarNode* leading_membar() const;
1222
1223 void set_trailing_load() { _kind = TrailingLoad; }
1224 bool trailing_load() const { return _kind == TrailingLoad; }
1225 bool trailing_store() const { return _kind == TrailingStore; }
1226 bool leading_store() const { return _kind == LeadingStore; }
1227 bool trailing_load_store() const { return _kind == TrailingLoadStore; }
1228 bool leading_load_store() const { return _kind == LeadingLoadStore; }
1229 bool trailing() const { return _kind == TrailingLoad || _kind == TrailingStore || _kind == TrailingLoadStore; }
1230 bool leading() const { return _kind == LeadingStore || _kind == LeadingLoadStore; }
1231 bool standalone() const { return _kind == Standalone; }
1232 void set_trailing_partial_array_copy() { _kind = TrailingPartialArrayCopy; }
1233 bool trailing_partial_array_copy() const { return _kind == TrailingPartialArrayCopy; }
1234
1235 static void set_store_pair(MemBarNode* leading, MemBarNode* trailing);
1236 static void set_load_store_pair(MemBarNode* leading, MemBarNode* trailing);
1237
1238 void remove(PhaseIterGVN *igvn);
1239 };
1240
1241 // "Acquire" - no following ref can move before (but earlier refs can
1242 // follow, like an early Load stalled in cache). Requires multi-cpu
1243 // visibility. Inserted after a volatile load.
1244 class MemBarAcquireNode: public MemBarNode {
1245 public:
1246 MemBarAcquireNode(Compile* C, int alias_idx, Node* precedent)
1247 : MemBarNode(C, alias_idx, precedent) {}
1248 virtual int Opcode() const;
1249 };
1250
1251 // "Acquire" - no following ref can move before (but earlier refs can
1252 // follow, like an early Load stalled in cache). Requires multi-cpu
1253 // visibility. Inserted independ of any load, as required
1254 // for intrinsic Unsafe.loadFence().
1255 class LoadFenceNode: public MemBarNode {
1256 public:
1257 LoadFenceNode(Compile* C, int alias_idx, Node* precedent)
1258 : MemBarNode(C, alias_idx, precedent) {}
1259 virtual int Opcode() const;
1260 };
1261
1262 // "Release" - no earlier ref can move after (but later refs can move
1263 // up, like a speculative pipelined cache-hitting Load). Requires
1264 // multi-cpu visibility. Inserted before a volatile store.
1265 class MemBarReleaseNode: public MemBarNode {
1266 public:
1267 MemBarReleaseNode(Compile* C, int alias_idx, Node* precedent)
1268 : MemBarNode(C, alias_idx, precedent) {}
1269 virtual int Opcode() const;
1270 };
1271
1272 // "Release" - no earlier ref can move after (but later refs can move
1273 // up, like a speculative pipelined cache-hitting Load). Requires
1274 // multi-cpu visibility. Inserted independent of any store, as required
1275 // for intrinsic Unsafe.storeFence().
1276 class StoreFenceNode: public MemBarNode {
1277 public:
1278 StoreFenceNode(Compile* C, int alias_idx, Node* precedent)
1279 : MemBarNode(C, alias_idx, precedent) {}
1280 virtual int Opcode() const;
1281 };
1282
1283 // "Acquire" - no following ref can move before (but earlier refs can
1284 // follow, like an early Load stalled in cache). Requires multi-cpu
1285 // visibility. Inserted after a FastLock.
1286 class MemBarAcquireLockNode: public MemBarNode {
1287 public:
1288 MemBarAcquireLockNode(Compile* C, int alias_idx, Node* precedent)
1289 : MemBarNode(C, alias_idx, precedent) {}
1290 virtual int Opcode() const;
1291 };
1292
1293 // "Release" - no earlier ref can move after (but later refs can move
1294 // up, like a speculative pipelined cache-hitting Load). Requires
1295 // multi-cpu visibility. Inserted before a FastUnLock.
1296 class MemBarReleaseLockNode: public MemBarNode {
1297 public:
1298 MemBarReleaseLockNode(Compile* C, int alias_idx, Node* precedent)
1299 : MemBarNode(C, alias_idx, precedent) {}
1300 virtual int Opcode() const;
1301 };
1302
1303 class MemBarStoreStoreNode: public MemBarNode {
1304 public:
1305 MemBarStoreStoreNode(Compile* C, int alias_idx, Node* precedent)
1306 : MemBarNode(C, alias_idx, precedent) {
1307 init_class_id(Class_MemBarStoreStore);
1308 }
1309 virtual int Opcode() const;
1310 };
1311
1312 // Ordering between a volatile store and a following volatile load.
1313 // Requires multi-CPU visibility?
1314 class MemBarVolatileNode: public MemBarNode {
1315 public:
1316 MemBarVolatileNode(Compile* C, int alias_idx, Node* precedent)
1317 : MemBarNode(C, alias_idx, precedent) {}
1318 virtual int Opcode() const;
1319 };
1320
1321 // Ordering within the same CPU. Used to order unsafe memory references
1322 // inside the compiler when we lack alias info. Not needed "outside" the
1323 // compiler because the CPU does all the ordering for us.
1324 class MemBarCPUOrderNode: public MemBarNode {
1325 public:
1326 MemBarCPUOrderNode(Compile* C, int alias_idx, Node* precedent)
1327 : MemBarNode(C, alias_idx, precedent) {}
1328 virtual int Opcode() const;
1329 virtual uint ideal_reg() const { return 0; } // not matched in the AD file
1330 };
1331
1332 class OnSpinWaitNode: public MemBarNode {
1333 public:
1334 OnSpinWaitNode(Compile* C, int alias_idx, Node* precedent)
1335 : MemBarNode(C, alias_idx, precedent) {}
1336 virtual int Opcode() const;
1337 };
1338
1339 //------------------------------BlackholeNode----------------------------
1340 // Blackhole all arguments. This node would survive through the compiler
1341 // the effects on its arguments, and would be finally matched to nothing.
1342 class BlackholeNode : public MemBarNode {
1343 public:
1344 BlackholeNode(Compile* C, int alias_idx, Node* precedent)
1345 : MemBarNode(C, alias_idx, precedent) {}
1346 virtual int Opcode() const;
1347 virtual uint ideal_reg() const { return 0; } // not matched in the AD file
1348 const RegMask &in_RegMask(uint idx) const {
1349 // Fake the incoming arguments mask for blackholes: accept all registers
1350 // and all stack slots. This would avoid any redundant register moves
1351 // for blackhole inputs.
1352 return RegMask::All;
1353 }
1354 #ifndef PRODUCT
1355 virtual void format(PhaseRegAlloc* ra, outputStream* st) const;
1356 #endif
1357 };
1358
1359 // Isolation of object setup after an AllocateNode and before next safepoint.
1360 // (See comment in memnode.cpp near InitializeNode::InitializeNode for semantics.)
1361 class InitializeNode: public MemBarNode {
1362 friend class AllocateNode;
1363
1364 enum {
1365 Incomplete = 0,
1366 Complete = 1,
1367 WithArraycopy = 2
1368 };
1369 int _is_complete;
1370
1371 bool _does_not_escape;
1372
1373 public:
1374 enum {
1375 Control = TypeFunc::Control,
1376 Memory = TypeFunc::Memory, // MergeMem for states affected by this op
1377 RawAddress = TypeFunc::Parms+0, // the newly-allocated raw address
1378 RawStores = TypeFunc::Parms+1 // zero or more stores (or TOP)
1379 };
1380
1381 InitializeNode(Compile* C, int adr_type, Node* rawoop);
1382 virtual int Opcode() const;
1383 virtual uint size_of() const { return sizeof(*this); }
1384 virtual uint ideal_reg() const { return 0; } // not matched in the AD file
1385 virtual const RegMask &in_RegMask(uint) const; // mask for RawAddress
1386
1387 // Manage incoming memory edges via a MergeMem on in(Memory):
1388 Node* memory(uint alias_idx);
1389
1390 // The raw memory edge coming directly from the Allocation.
1391 // The contents of this memory are *always* all-zero-bits.
1392 Node* zero_memory() { return memory(Compile::AliasIdxRaw); }
1393
1394 // Return the corresponding allocation for this initialization (or null if none).
1395 // (Note: Both InitializeNode::allocation and AllocateNode::initialization
1396 // are defined in graphKit.cpp, which sets up the bidirectional relation.)
1397 AllocateNode* allocation();
1398
1399 // Anything other than zeroing in this init?
1400 bool is_non_zero();
1401
1402 // An InitializeNode must completed before macro expansion is done.
1403 // Completion requires that the AllocateNode must be followed by
1404 // initialization of the new memory to zero, then to any initializers.
1405 bool is_complete() { return _is_complete != Incomplete; }
1406 bool is_complete_with_arraycopy() { return (_is_complete & WithArraycopy) != 0; }
1407
1408 // Mark complete. (Must not yet be complete.)
1409 void set_complete(PhaseGVN* phase);
1410 void set_complete_with_arraycopy() { _is_complete = Complete | WithArraycopy; }
1411
1412 bool does_not_escape() { return _does_not_escape; }
1413 void set_does_not_escape() { _does_not_escape = true; }
1414
1415 #ifdef ASSERT
1416 // ensure all non-degenerate stores are ordered and non-overlapping
1417 bool stores_are_sane(PhaseTransform* phase);
1418 #endif //ASSERT
1419
1420 // See if this store can be captured; return offset where it initializes.
1421 // Return 0 if the store cannot be moved (any sort of problem).
1422 intptr_t can_capture_store(StoreNode* st, PhaseGVN* phase, bool can_reshape);
1423
1424 // Capture another store; reformat it to write my internal raw memory.
1425 // Return the captured copy, else NULL if there is some sort of problem.
1426 Node* capture_store(StoreNode* st, intptr_t start, PhaseGVN* phase, bool can_reshape);
1427
1428 // Find captured store which corresponds to the range [start..start+size).
1429 // Return my own memory projection (meaning the initial zero bits)
1430 // if there is no such store. Return NULL if there is a problem.
1431 Node* find_captured_store(intptr_t start, int size_in_bytes, PhaseTransform* phase);
1432
1433 // Called when the associated AllocateNode is expanded into CFG.
1434 Node* complete_stores(Node* rawctl, Node* rawmem, Node* rawptr,
1435 intptr_t header_size, Node* size_in_bytes,
1436 PhaseIterGVN* phase);
1437
1438 private:
1439 void remove_extra_zeroes();
1440
1441 // Find out where a captured store should be placed (or already is placed).
1442 int captured_store_insertion_point(intptr_t start, int size_in_bytes,
1443 PhaseTransform* phase);
1444
1445 static intptr_t get_store_offset(Node* st, PhaseTransform* phase);
1446
1447 Node* make_raw_address(intptr_t offset, PhaseTransform* phase);
1448
1449 bool detect_init_independence(Node* value, PhaseGVN* phase);
1450
1451 void coalesce_subword_stores(intptr_t header_size, Node* size_in_bytes,
1452 PhaseGVN* phase);
1453
1454 intptr_t find_next_fullword_store(uint i, PhaseGVN* phase);
1455 };
1456
1457 //------------------------------MergeMem---------------------------------------
1458 // (See comment in memnode.cpp near MergeMemNode::MergeMemNode for semantics.)
1459 class MergeMemNode: public Node {
1460 virtual uint hash() const ; // { return NO_HASH; }
1461 virtual bool cmp( const Node &n ) const ; // Always fail, except on self
1462 friend class MergeMemStream;
1463 MergeMemNode(Node* def); // clients use MergeMemNode::make
1464
1465 public:
1466 // If the input is a whole memory state, clone it with all its slices intact.
1467 // Otherwise, make a new memory state with just that base memory input.
1468 // In either case, the result is a newly created MergeMem.
1469 static MergeMemNode* make(Node* base_memory);
1470
1471 virtual int Opcode() const;
1472 virtual Node* Identity(PhaseGVN* phase);
1473 virtual Node *Ideal(PhaseGVN *phase, bool can_reshape);
1474 virtual uint ideal_reg() const { return NotAMachineReg; }
1475 virtual uint match_edge(uint idx) const { return 0; }
1476 virtual const RegMask &out_RegMask() const;
1477 virtual const Type *bottom_type() const { return Type::MEMORY; }
1478 virtual const TypePtr *adr_type() const { return TypePtr::BOTTOM; }
1479 // sparse accessors
1480 // Fetch the previously stored "set_memory_at", or else the base memory.
1481 // (Caller should clone it if it is a phi-nest.)
1482 Node* memory_at(uint alias_idx) const;
1483 // set the memory, regardless of its previous value
1484 void set_memory_at(uint alias_idx, Node* n);
1485 // the "base" is the memory that provides the non-finite support
1486 Node* base_memory() const { return in(Compile::AliasIdxBot); }
1487 // warning: setting the base can implicitly set any of the other slices too
1488 void set_base_memory(Node* def);
1489 // sentinel value which denotes a copy of the base memory:
1490 Node* empty_memory() const { return in(Compile::AliasIdxTop); }
1491 static Node* make_empty_memory(); // where the sentinel comes from
1492 bool is_empty_memory(Node* n) const { assert((n == empty_memory()) == n->is_top(), "sanity"); return n->is_top(); }
1493 // hook for the iterator, to perform any necessary setup
1494 void iteration_setup(const MergeMemNode* other = NULL);
1495 // push sentinels until I am at least as long as the other (semantic no-op)
1496 void grow_to_match(const MergeMemNode* other);
1497 bool verify_sparse() const PRODUCT_RETURN0;
1498 #ifndef PRODUCT
1499 virtual void dump_spec(outputStream *st) const;
1500 #endif
1501 };
1502
1503 class MergeMemStream : public StackObj {
1504 private:
1505 MergeMemNode* _mm;
1506 const MergeMemNode* _mm2; // optional second guy, contributes non-empty iterations
1507 Node* _mm_base; // loop-invariant base memory of _mm
1508 int _idx;
1509 int _cnt;
1510 Node* _mem;
1511 Node* _mem2;
1512 int _cnt2;
1513
1514 void init(MergeMemNode* mm, const MergeMemNode* mm2 = NULL) {
1515 // subsume_node will break sparseness at times, whenever a memory slice
1516 // folds down to a copy of the base ("fat") memory. In such a case,
1517 // the raw edge will update to base, although it should be top.
1518 // This iterator will recognize either top or base_memory as an
1519 // "empty" slice. See is_empty, is_empty2, and next below.
1520 //
1521 // The sparseness property is repaired in MergeMemNode::Ideal.
1522 // As long as access to a MergeMem goes through this iterator
1523 // or the memory_at accessor, flaws in the sparseness will
1524 // never be observed.
1525 //
1526 // Also, iteration_setup repairs sparseness.
1527 assert(mm->verify_sparse(), "please, no dups of base");
1528 assert(mm2==NULL || mm2->verify_sparse(), "please, no dups of base");
1529
1530 _mm = mm;
1531 _mm_base = mm->base_memory();
1532 _mm2 = mm2;
1533 _cnt = mm->req();
1534 _idx = Compile::AliasIdxBot-1; // start at the base memory
1535 _mem = NULL;
1536 _mem2 = NULL;
1537 }
1538
1539 #ifdef ASSERT
1540 Node* check_memory() const {
1541 if (at_base_memory())
1542 return _mm->base_memory();
1543 else if ((uint)_idx < _mm->req() && !_mm->in(_idx)->is_top())
1544 return _mm->memory_at(_idx);
1545 else
1546 return _mm_base;
1547 }
1548 Node* check_memory2() const {
1549 return at_base_memory()? _mm2->base_memory(): _mm2->memory_at(_idx);
1550 }
1551 #endif
1552
1553 static bool match_memory(Node* mem, const MergeMemNode* mm, int idx) PRODUCT_RETURN0;
1554 void assert_synch() const {
1555 assert(!_mem || _idx >= _cnt || match_memory(_mem, _mm, _idx),
1556 "no side-effects except through the stream");
1557 }
1558
1559 public:
1560
1561 // expected usages:
1562 // for (MergeMemStream mms(mem->is_MergeMem()); next_non_empty(); ) { ... }
1563 // for (MergeMemStream mms(mem1, mem2); next_non_empty2(); ) { ... }
1564
1565 // iterate over one merge
1566 MergeMemStream(MergeMemNode* mm) {
1567 mm->iteration_setup();
1568 init(mm);
1569 debug_only(_cnt2 = 999);
1570 }
1571 // iterate in parallel over two merges
1572 // only iterates through non-empty elements of mm2
1573 MergeMemStream(MergeMemNode* mm, const MergeMemNode* mm2) {
1574 assert(mm2, "second argument must be a MergeMem also");
1575 ((MergeMemNode*)mm2)->iteration_setup(); // update hidden state
1576 mm->iteration_setup(mm2);
1577 init(mm, mm2);
1578 _cnt2 = mm2->req();
1579 }
1580 #ifdef ASSERT
1581 ~MergeMemStream() {
1582 assert_synch();
1583 }
1584 #endif
1585
1586 MergeMemNode* all_memory() const {
1587 return _mm;
1588 }
1589 Node* base_memory() const {
1590 assert(_mm_base == _mm->base_memory(), "no update to base memory, please");
1591 return _mm_base;
1592 }
1593 const MergeMemNode* all_memory2() const {
1594 assert(_mm2 != NULL, "");
1595 return _mm2;
1596 }
1597 bool at_base_memory() const {
1598 return _idx == Compile::AliasIdxBot;
1599 }
1600 int alias_idx() const {
1601 assert(_mem, "must call next 1st");
1602 return _idx;
1603 }
1604
1605 const TypePtr* adr_type() const {
1606 return Compile::current()->get_adr_type(alias_idx());
1607 }
1608
1609 const TypePtr* adr_type(Compile* C) const {
1610 return C->get_adr_type(alias_idx());
1611 }
1612 bool is_empty() const {
1613 assert(_mem, "must call next 1st");
1614 assert(_mem->is_top() == (_mem==_mm->empty_memory()), "correct sentinel");
1615 return _mem->is_top();
1616 }
1617 bool is_empty2() const {
1618 assert(_mem2, "must call next 1st");
1619 assert(_mem2->is_top() == (_mem2==_mm2->empty_memory()), "correct sentinel");
1620 return _mem2->is_top();
1621 }
1622 Node* memory() const {
1623 assert(!is_empty(), "must not be empty");
1624 assert_synch();
1625 return _mem;
1626 }
1627 // get the current memory, regardless of empty or non-empty status
1628 Node* force_memory() const {
1629 assert(!is_empty() || !at_base_memory(), "");
1630 // Use _mm_base to defend against updates to _mem->base_memory().
1631 Node *mem = _mem->is_top() ? _mm_base : _mem;
1632 assert(mem == check_memory(), "");
1633 return mem;
1634 }
1635 Node* memory2() const {
1636 assert(_mem2 == check_memory2(), "");
1637 return _mem2;
1638 }
1639 void set_memory(Node* mem) {
1640 if (at_base_memory()) {
1641 // Note that this does not change the invariant _mm_base.
1642 _mm->set_base_memory(mem);
1643 } else {
1644 _mm->set_memory_at(_idx, mem);
1645 }
1646 _mem = mem;
1647 assert_synch();
1648 }
1649
1650 // Recover from a side effect to the MergeMemNode.
1651 void set_memory() {
1652 _mem = _mm->in(_idx);
1653 }
1654
1655 bool next() { return next(false); }
1656 bool next2() { return next(true); }
1657
1658 bool next_non_empty() { return next_non_empty(false); }
1659 bool next_non_empty2() { return next_non_empty(true); }
1660 // next_non_empty2 can yield states where is_empty() is true
1661
1662 private:
1663 // find the next item, which might be empty
1664 bool next(bool have_mm2) {
1665 assert((_mm2 != NULL) == have_mm2, "use other next");
1666 assert_synch();
1667 if (++_idx < _cnt) {
1668 // Note: This iterator allows _mm to be non-sparse.
1669 // It behaves the same whether _mem is top or base_memory.
1670 _mem = _mm->in(_idx);
1671 if (have_mm2)
1672 _mem2 = _mm2->in((_idx < _cnt2) ? _idx : Compile::AliasIdxTop);
1673 return true;
1674 }
1675 return false;
1676 }
1677
1678 // find the next non-empty item
1679 bool next_non_empty(bool have_mm2) {
1680 while (next(have_mm2)) {
1681 if (!is_empty()) {
1682 // make sure _mem2 is filled in sensibly
1683 if (have_mm2 && _mem2->is_top()) _mem2 = _mm2->base_memory();
1684 return true;
1685 } else if (have_mm2 && !is_empty2()) {
1686 return true; // is_empty() == true
1687 }
1688 }
1689 return false;
1690 }
1691 };
1692
1693 // cachewb node for guaranteeing writeback of the cache line at a
1694 // given address to (non-volatile) RAM
1695 class CacheWBNode : public Node {
1696 public:
1697 CacheWBNode(Node *ctrl, Node *mem, Node *addr) : Node(ctrl, mem, addr) {}
1698 virtual int Opcode() const;
1699 virtual uint ideal_reg() const { return NotAMachineReg; }
1700 virtual uint match_edge(uint idx) const { return (idx == 2); }
1701 virtual const TypePtr *adr_type() const { return TypePtr::BOTTOM; }
1702 virtual const Type *bottom_type() const { return Type::MEMORY; }
1703 };
1704
1705 // cachewb pre sync node for ensuring that writebacks are serialised
1706 // relative to preceding or following stores
1707 class CacheWBPreSyncNode : public Node {
1708 public:
1709 CacheWBPreSyncNode(Node *ctrl, Node *mem) : Node(ctrl, mem) {}
1710 virtual int Opcode() const;
1711 virtual uint ideal_reg() const { return NotAMachineReg; }
1712 virtual uint match_edge(uint idx) const { return false; }
1713 virtual const TypePtr *adr_type() const { return TypePtr::BOTTOM; }
1714 virtual const Type *bottom_type() const { return Type::MEMORY; }
1715 };
1716
1717 // cachewb pre sync node for ensuring that writebacks are serialised
1718 // relative to preceding or following stores
1719 class CacheWBPostSyncNode : public Node {
1720 public:
1721 CacheWBPostSyncNode(Node *ctrl, Node *mem) : Node(ctrl, mem) {}
1722 virtual int Opcode() const;
1723 virtual uint ideal_reg() const { return NotAMachineReg; }
1724 virtual uint match_edge(uint idx) const { return false; }
1725 virtual const TypePtr *adr_type() const { return TypePtr::BOTTOM; }
1726 virtual const Type *bottom_type() const { return Type::MEMORY; }
1727 };
1728
1729 //------------------------------Prefetch---------------------------------------
1730
1731 // Allocation prefetch which may fault, TLAB size have to be adjusted.
1732 class PrefetchAllocationNode : public Node {
1733 public:
1734 PrefetchAllocationNode(Node *mem, Node *adr) : Node(0,mem,adr) {}
1735 virtual int Opcode() const;
1736 virtual uint ideal_reg() const { return NotAMachineReg; }
1737 virtual uint match_edge(uint idx) const { return idx==2; }
1738 virtual const Type *bottom_type() const { return ( AllocatePrefetchStyle == 3 ) ? Type::MEMORY : Type::ABIO; }
1739 };
1740
1741 #endif // SHARE_OPTO_MEMNODE_HPP