1 /*
2 * Copyright (c) 2016, 2025, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #ifndef SHARE_VM_OPTO_INLINETYPENODE_HPP
26 #define SHARE_VM_OPTO_INLINETYPENODE_HPP
27
28 #include "ci/ciInlineKlass.hpp"
29 #include "gc/shared/c2/barrierSetC2.hpp"
30 #include "oops/accessDecorators.hpp"
31 #include "opto/callnode.hpp"
32 #include "opto/compile.hpp"
33 #include "opto/loopnode.hpp"
34 #include "opto/multnode.hpp"
35 #include "opto/node.hpp"
36
37 class GraphKit;
38
39 //------------------------------InlineTypeNode-------------------------------------
40 // Node representing an inline type in C2 IR
41 class InlineTypeNode : public TypeNode {
42 private:
43 InlineTypeNode(ciInlineKlass* vk, Node* oop, bool null_free)
44 : TypeNode(TypeInstPtr::make(null_free ? TypePtr::NotNull : TypePtr::BotPTR, vk), Values + vk->nof_declared_nonstatic_fields()) {
45 init_class_id(Class_InlineType);
46 init_req(Oop, oop);
47 Compile::current()->add_inline_type(this);
48 }
49
50 enum { Control, // Control input.
51 Oop, // Oop to heap allocated buffer.
52 IsBuffered, // True if inline type is heap allocated (or nullptr), false otherwise.
53 NullMarker, // Needs to be checked before using the field values.
54 // 0 => InlineType is null
55 // 1 => InlineType is non-null
56 // Can be dynamic value, not necessarily statically known
57 Values // Nodes corresponding to values of the inline type's fields.
58 // Nodes are connected in increasing order of the index of the field they correspond to.
59 };
60
61 // Get the klass defining the field layout of the inline type
62 ciInlineKlass* inline_klass() const { return type()->inline_klass(); }
63
64 void make_scalar_in_safepoint(PhaseIterGVN* igvn, Unique_Node_List& worklist, SafePointNode* sfpt);
65 uint add_fields_to_safepoint(Unique_Node_List& worklist, SafePointNode* sfpt);
66
67 // Checks if the inline type is loaded from memory and if so returns the oop
68 Node* is_loaded(PhaseGVN* phase, ciInlineKlass* vk = nullptr, Node* base = nullptr, int holder_offset = 0);
69
70 // Initialize the inline type fields with the inputs or outputs of a MultiNode
71 void initialize_fields(GraphKit* kit, MultiNode* multi, uint& base_input, bool in, bool null_free, Node* null_check_region, GrowableArray<ciType*>& visited);
72 // Initialize the inline type by loading its field values from memory
73 void load(GraphKit* kit, Node* base, Node* ptr, bool immutable_memory, bool trust_null_free_oop, DecoratorSet decorators, GrowableArray<ciType*>& visited);
74 // Store the field values to memory
75 void store(GraphKit* kit, Node* base, Node* ptr, bool immutable_memory, DecoratorSet decorators) const;
76
77 InlineTypeNode* adjust_scalarization_depth_impl(GraphKit* kit, GrowableArray<ciType*>& visited);
78
79 static InlineTypeNode* make_all_zero_impl(PhaseGVN& gvn, ciInlineKlass* vk, GrowableArray<ciType*>& visited);
80 static InlineTypeNode* make_from_oop_impl(GraphKit* kit, Node* oop, ciInlineKlass* vk, GrowableArray<ciType*>& visited);
81 static InlineTypeNode* make_null_impl(PhaseGVN& gvn, ciInlineKlass* vk, GrowableArray<ciType*>& visited, bool transform = true);
82 static InlineTypeNode* make_from_flat_impl(GraphKit* kit, ciInlineKlass* vk, Node* base, Node* ptr, bool atomic, bool immutable_memory,
83 bool null_free, bool trust_null_free_oop, DecoratorSet decorators, GrowableArray<ciType*>& visited);
84
85 public:
86 // Create with all-zero field values
87 static InlineTypeNode* make_all_zero(PhaseGVN& gvn, ciInlineKlass* vk);
88 // Create uninitialized
89 static InlineTypeNode* make_uninitialized(PhaseGVN& gvn, ciInlineKlass* vk, bool null_free = true);
90 // Create and initialize by loading the field values from an oop
91 static InlineTypeNode* make_from_oop(GraphKit* kit, Node* oop, ciInlineKlass* vk);
92 // Create and initialize by loading the field values from a flat field or array
93 static InlineTypeNode* make_from_flat(GraphKit* kit, ciInlineKlass* vk, Node* base, Node* ptr,
94 bool atomic, bool immutable_memory, bool null_free, DecoratorSet decorators);
95 static InlineTypeNode* make_from_flat_array(GraphKit* kit, ciInlineKlass* vk, Node* base, Node* idx);
96 // Create and initialize with the inputs or outputs of a MultiNode (method entry or call)
97 static InlineTypeNode* make_from_multi(GraphKit* kit, MultiNode* multi, ciInlineKlass* vk, uint& base_input, bool in, bool null_free = true);
98 // Create with null field values
99 static InlineTypeNode* make_null(PhaseGVN& gvn, ciInlineKlass* vk, bool transform = true);
100
101 // Support for control flow merges
102 bool has_phi_inputs(Node* region);
103 InlineTypeNode* clone_with_phis(PhaseGVN* gvn, Node* region, SafePointNode* map = nullptr, bool is_non_null = false, bool init_with_top = false);
104 InlineTypeNode* merge_with(PhaseGVN* gvn, const InlineTypeNode* other, int phi_index, bool transform);
105 void add_new_path(Node* region);
106
107 // Get oop for heap allocated inline type (may be TypePtr::NULL_PTR)
108 Node* get_oop() const { return in(Oop); }
109 void set_oop(PhaseGVN& gvn, Node* oop) { set_req_X(Oop, oop, &gvn); }
110 Node* get_null_marker() const { return in(NullMarker); }
111 void set_null_marker(PhaseGVN& gvn, Node* init) { set_req_X(NullMarker, init, &gvn); }
112 void set_null_marker(PhaseGVN& gvn) { set_null_marker(gvn, gvn.intcon(1)); }
113 Node* get_is_buffered() const { return in(IsBuffered); }
114 void set_is_buffered(PhaseGVN& gvn, bool buffered = true) { set_req_X(IsBuffered, gvn.intcon(buffered ? 1 : 0), &gvn); }
115
116 // Checks if the inline type fields are all set to zero
117 bool is_all_zero(PhaseGVN* gvn, bool flat = false) const;
118
119 // Inline type fields
120 uint field_count() const { return req() - Values; }
121 Node* field_value(uint index) const;
122 Node* field_value_by_offset(int offset, bool recursive) const;
123 void set_field_value(uint index, Node* value);
124 void set_field_value_by_offset(int offset, Node* value);
125 int field_offset(uint index) const;
126 uint field_index(int offset) const;
127 ciType* field_type(uint index) const;
128 bool field_is_flat(uint index) const;
129 bool field_is_null_free(uint index) const;
130 bool field_is_volatile(uint index) const;
131 int field_null_marker_offset(uint index) const;
132
133 // Replace InlineTypeNodes in debug info at safepoints with SafePointScalarObjectNodes
134 void make_scalar_in_safepoints(PhaseIterGVN* igvn, bool allow_oop = true);
135
136 // Store the inline type as a flat (headerless) representation
137 void store_flat(GraphKit* kit, Node* base, Node* ptr, bool atomic, bool immutable_memory, bool null_free, DecoratorSet decorators);
138 // Store the inline type as a flat (headerless) representation into an array
139 void store_flat_array(GraphKit* kit, Node* base, Node* idx);
140 // Make sure that inline type is fully scalarized
141 InlineTypeNode* adjust_scalarization_depth(GraphKit* kit);
142
143 // Implementation of the substitutability check for acmp
144 bool can_emit_substitutability_check(Node* other) const;
145 void check_substitutability(PhaseIterGVN* igvn, RegionNode* region, Node* phi, Node** ctrl, Node* mem, Node* base, Node* other, bool flat = false) const;
146
147 // Allocates the inline type (if not yet allocated)
148 InlineTypeNode* buffer(GraphKit* kit, bool safe_for_replace = true);
149 bool is_allocated(PhaseGVN* phase) const;
150
151 void replace_call_results(GraphKit* kit, CallNode* call, Compile* C);
152 void replace_field_projs(Compile* C, CallNode* call, uint& proj_idx);
153
154 // Allocate all non-flat inline type fields
155 InlineTypeNode* allocate_fields(GraphKit* kit);
156
157 Node* tagged_klass(PhaseGVN& gvn) {
158 return tagged_klass(inline_klass(), gvn);
159 }
160 static Node* tagged_klass(ciInlineKlass* vk, PhaseGVN& gvn);
161 // Pass inline type as fields at a call or return
162 void pass_fields(GraphKit* kit, Node* n, uint& base_input, bool in, bool null_free = true);
163
164 // Allocation optimizations
165 void remove_redundant_allocations(PhaseIdealLoop* phase);
166
167 InlineTypeNode* clone_if_required(PhaseGVN* gvn, SafePointNode* map, bool safe_for_replace = true);
168
169 virtual const Type* Value(PhaseGVN* phase) const;
170
171 virtual Node* Ideal(PhaseGVN* phase, bool can_reshape);
172
173 virtual int Opcode() const;
174 };
175
176 // Load from a flat element, the node produces 1 Proj output for each flattened field of the flat
177 // element. The order of the Proj node is the same as that of _vk->_nonstatic_fields, and the null
178 // marker if existing will be the last Proj output. This node acts as if the load happens
179 // atomically and will be expanded to loading the whole payload and extracting the flattened fields
180 // from the loaded payload. In special cases, such as when the object from which this load reads
181 // does not escape, this node can be expanded to multiple loads from each flattened field.
182 // This node allows us to replace its results with the value from a matching store because the
183 // payload value cannot be directly propagated if it contains oops. This effect, in turn, allows
184 // objects with atomic flat fields to be scalar replaced.
185 class LoadFlatNode final : public SafePointNode {
186 private:
187 ciInlineKlass* _vk;
188 const TypeTuple* _type;
189 bool _null_free;
190 DecoratorSet _decorators;
191
192 public:
193 static InlineTypeNode* load(GraphKit* kit, ciInlineKlass* vk, Node* base, Node* ptr, bool null_free, bool trust_null_free_oop, DecoratorSet decorators);
194 Node* base() const { return in(TypeFunc::Parms); }
195 Node* ptr() const { return in(TypeFunc::Parms + 1); }
196 bool expand_non_atomic(PhaseIterGVN& igvn);
197 void expand_atomic(PhaseIterGVN& igvn);
198
199 private:
200 LoadFlatNode(ciInlineKlass* vk, const TypeTuple* type, bool null_free, DecoratorSet decorators)
201 : SafePointNode(TypeFunc::Parms + 2, nullptr, TypePtr::BOTTOM), _vk(vk), _type(type), _null_free(null_free), _decorators(decorators) {
202 init_class_id(Class_LoadFlat);
203 Compile::current()->add_flat_access(this);
204 }
205
206 virtual int Opcode() const override;
207 virtual const Type* bottom_type() const override { return _type; }
208 virtual uint size_of() const override { return sizeof(LoadFlatNode); }
209 virtual Node* Ideal(PhaseGVN* phase, bool can_reshape) override { return nullptr; }
210 virtual Node* Identity(PhaseGVN* phase) override { return this; }
211 virtual const Type* Value(PhaseGVN* phase) const override;
212
213 static void collect_field_types(ciInlineKlass* vk, const Type** field_types, int idx, int limit, bool null_free, bool trust_null_free_oop);
214 InlineTypeNode* collect_projs(GraphKit* kit, ciInlineKlass* vk, int proj_con, bool null_free);
215 void expand_projs_atomic(PhaseIterGVN& gvn, Node* ctrl, Node* payload);
216 static Node* get_payload_value(PhaseIterGVN& igvn, Node* ctrl, BasicType payload_bt, Node* payload, const Type* value_type, BasicType value_bt, int offset);
217 };
218
219 // Store an InlineTypeNode to a flat element, the store acts as if it is atomic. Similar to
220 // LoadFlatNode, this node is expanded to storing a payload created from the field values of the
221 // InlineTypeNode, and under special circumstances, when there is no racing access to the field,
222 // this node can be expanded to multiple stores to each flattened field.
223 // The purposes of this node complement those of LoadFlatNode.
224 class StoreFlatNode final : public SafePointNode {
225 private:
226 bool _null_free;
227 DecoratorSet _decorators;
228
229 public:
230 static void store(GraphKit* kit, Node* base, Node* ptr, InlineTypeNode* value, bool null_free, DecoratorSet decorators);
231 Node* base() const { return in(TypeFunc::Parms); }
232 Node* ptr() const { return in(TypeFunc::Parms + 1); }
233 InlineTypeNode* value() const { return in(TypeFunc::Parms + 2)->as_InlineType(); }
234 bool expand_non_atomic(PhaseIterGVN& igvn);
235 void expand_atomic(PhaseIterGVN& igvn);
236
237 private:
238 StoreFlatNode(bool null_free, DecoratorSet decorators)
239 : SafePointNode(TypeFunc::Parms + 3, nullptr, TypePtr::BOTTOM), _null_free(null_free), _decorators(decorators) {
240 init_class_id(Class_StoreFlat);
241 Compile::current()->add_flat_access(this);
242 }
243
244 virtual int Opcode() const override;
245 virtual const Type* bottom_type() const override { return TypeTuple::MEMBAR; }
246 virtual uint size_of() const override { return sizeof(StoreFlatNode); }
247 virtual Node* Ideal(PhaseGVN* phase, bool can_reshape) override { return nullptr; }
248 virtual Node* Identity(PhaseGVN* phase) override { return this; }
249 virtual const Type* Value(PhaseGVN* phase) const override;
250
251 static Node* convert_to_payload(PhaseIterGVN& igvn, Node* ctrl, InlineTypeNode* value, bool null_free, int& oop_off_1, int& oop_off_2);
252 static Node* set_payload_value(PhaseIterGVN& igvn, BasicType payload_bt, Node* payload, BasicType val_bt, Node* value, int offset);
253 };
254
255 #endif // SHARE_VM_OPTO_INLINETYPENODE_HPP