1 /*
2 * Copyright (c) 2017, 2026, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "ci/ciInlineKlass.hpp"
26 #include "gc/shared/barrierSet.hpp"
27 #include "gc/shared/c2/barrierSetC2.hpp"
28 #include "gc/shared/gc_globals.hpp"
29 #include "memory/resourceArea.hpp"
30 #include "oops/accessDecorators.hpp"
31 #include "opto/addnode.hpp"
32 #include "opto/castnode.hpp"
33 #include "opto/cfgnode.hpp"
34 #include "opto/compile.hpp"
35 #include "opto/convertnode.hpp"
36 #include "opto/graphKit.hpp"
37 #include "opto/inlinetypenode.hpp"
38 #include "opto/memnode.hpp"
39 #include "opto/movenode.hpp"
40 #include "opto/multnode.hpp"
41 #include "opto/narrowptrnode.hpp"
42 #include "opto/node.hpp"
43 #include "opto/opcodes.hpp"
44 #include "opto/phaseX.hpp"
45 #include "opto/rootnode.hpp"
46 #include "opto/subnode.hpp"
47 #include "opto/type.hpp"
48 #include "utilities/globalDefinitions.hpp"
49 #include "utilities/growableArray.hpp"
50 #include "utilities/pair.hpp"
51 #include "utilities/tuple.hpp"
52
53 // Clones the inline type to handle control flow merges involving multiple inline types.
54 // The inputs are replaced by PhiNodes to represent the merged values for the given region.
55 // init_with_top: input of phis above the returned InlineTypeNode are initialized to top.
56 InlineTypeNode* InlineTypeNode::clone_with_phis(PhaseGVN* gvn, Node* region, SafePointNode* map, bool is_non_null, bool init_with_top) {
57 InlineTypeNode* vt = clone_if_required(gvn, map);
58 const Type* t = Type::get_const_type(inline_klass());
59 gvn->set_type(vt, t);
60 vt->as_InlineType()->set_type(t);
61
62 Node* const top = gvn->C->top();
63
64 // Create a PhiNode for merging the oop values
65 PhiNode* oop = PhiNode::make(region, init_with_top ? top : vt->get_oop(), t);
66 gvn->set_type(oop, t);
67 gvn->record_for_igvn(oop);
68 vt->set_oop(*gvn, oop);
69
70 // Create a PhiNode for merging the is_buffered values
71 t = Type::get_const_basic_type(T_BOOLEAN);
72 Node* is_buffered_node = PhiNode::make(region, init_with_top ? top : vt->get_is_buffered(), t);
73 gvn->set_type(is_buffered_node, t);
74 gvn->record_for_igvn(is_buffered_node);
75 vt->set_req(IsBuffered, is_buffered_node);
76
77 // Create a PhiNode for merging the null_marker values
78 Node* null_marker_node;
79 if (is_non_null) {
80 null_marker_node = gvn->intcon(1);
81 } else {
82 t = Type::get_const_basic_type(T_BOOLEAN);
83 null_marker_node = PhiNode::make(region, init_with_top ? top : vt->get_null_marker(), t);
84 gvn->set_type(null_marker_node, t);
85 gvn->record_for_igvn(null_marker_node);
86 }
87 vt->set_req(NullMarker, null_marker_node);
88
89 // Create a PhiNode each for merging the field values
90 for (uint i = 0; i < vt->field_count(); ++i) {
91 ciField* field = vt->field(i);
92 ciType* type = field->type();
93 Node* value = vt->field_value(i);
94 if (field->is_flat()) {
95 // Handle flat fields recursively
96 value = value->as_InlineType()->clone_with_phis(gvn, region, map);
97 } else {
98 t = Type::get_const_type(type);
99 value = PhiNode::make(region, init_with_top ? top : value, t);
100 gvn->set_type(value, t);
101 gvn->record_for_igvn(value);
102 }
103 vt->set_field_value(i, value);
104 }
105 gvn->record_for_igvn(vt);
106 return vt;
107 }
108
109 // Checks if the inputs of the InlineTypeNode were replaced by PhiNodes
110 // for the given region (see InlineTypeNode::clone_with_phis).
111 bool InlineTypeNode::has_phi_inputs(Node* region) const {
112 // Check oop input
113 bool result = get_oop()->is_Phi() && get_oop()->as_Phi()->region() == region;
114 #ifdef ASSERT
115 if (result) {
116 // Check all field value inputs for consistency
117 for (uint i = 0; i < field_count(); ++i) {
118 Node* n = field_value(i);
119 if (n->is_InlineType()) {
120 assert(n->as_InlineType()->has_phi_inputs(region), "inconsistent phi inputs");
121 } else {
122 assert(n->is_Phi() && n->as_Phi()->region() == region, "inconsistent phi inputs");
123 }
124 }
125 }
126 #endif
127 return result;
128 }
129
130 // Merges 'this' with 'other' by updating the input PhiNodes added by 'clone_with_phis'
131 InlineTypeNode* InlineTypeNode::merge_with(PhaseGVN* gvn, const InlineTypeNode* other, int phi_index, bool transform) {
132 assert(inline_klass() == other->inline_klass(), "Merging incompatible types");
133
134 // Merge oop inputs
135 PhiNode* phi = get_oop()->as_Phi();
136 phi->set_req(phi_index, other->get_oop());
137 if (transform) {
138 set_oop(*gvn, gvn->transform(phi));
139 }
140
141 // Merge is_buffered inputs
142 phi = get_is_buffered()->as_Phi();
143 phi->set_req(phi_index, other->get_is_buffered());
144 if (transform) {
145 set_req(IsBuffered, gvn->transform(phi));
146 }
147
148 // Merge null_marker inputs
149 Node* null_marker = get_null_marker();
150 if (null_marker->is_Phi()) {
151 phi = null_marker->as_Phi();
152 phi->set_req(phi_index, other->get_null_marker());
153 if (transform) {
154 set_req(NullMarker, gvn->transform(phi));
155 }
156 } else {
157 assert(null_marker->find_int_con(0) == 1, "only with a non null inline type");
158 }
159
160 // Merge field values
161 for (uint i = 0; i < field_count(); ++i) {
162 Node* val1 = field_value(i);
163 Node* val2 = other->field_value(i);
164 if (field(i)->is_flat()) {
165 if (val2->is_top()) {
166 // The path where 'other' is used is dying. Therefore, we do not need to process the merge with 'other' further.
167 // The phi inputs of 'this' at 'phi_index' will eventually be removed.
168 break;
169 } else if (val2->is_Phi()) {
170 val2 = gvn->transform(val2);
171 }
172
173 assert(val1->is_InlineType() && val2->is_InlineType(), "must be InlineTypeNode: %s - %s", val1->Name(), val2->Name());
174 val1->as_InlineType()->merge_with(gvn, val2->as_InlineType(), phi_index, transform);
175 } else {
176 assert(val1->is_Phi(), "must be a phi node %s", val1->Name());
177 val1->set_req(phi_index, val2);
178 }
179 if (transform) {
180 set_field_value(i, gvn->transform(val1));
181 }
182 }
183 return this;
184 }
185
186 // Adds a new merge path to an inline type node with phi inputs
187 void InlineTypeNode::add_new_path(Node* region) const {
188 assert(has_phi_inputs(region), "must have phi inputs");
189
190 PhiNode* phi = get_oop()->as_Phi();
191 phi->add_req(nullptr);
192 assert(phi->req() == region->req(), "must be same size as region");
193
194 phi = get_is_buffered()->as_Phi();
195 phi->add_req(nullptr);
196 assert(phi->req() == region->req(), "must be same size as region");
197
198 phi = get_null_marker()->as_Phi();
199 phi->add_req(nullptr);
200 assert(phi->req() == region->req(), "must be same size as region");
201
202 for (uint i = 0; i < field_count(); ++i) {
203 Node* val = field_value(i);
204 if (val->is_InlineType()) {
205 val->as_InlineType()->add_new_path(region);
206 } else {
207 val->as_Phi()->add_req(nullptr);
208 assert(val->req() == region->req(), "must be same size as region");
209 }
210 }
211 }
212
213 Node* InlineTypeNode::field_value(uint index) const {
214 assert(index < field_count(), "index out of bounds");
215 return in(Values + index);
216 }
217
218 // Get the value of the field at the given offset.
219 // If 'recursive' is true, flat inline type fields will be resolved recursively.
220 Node* InlineTypeNode::field_value_by_offset(int offset, bool recursive) const {
221 // Find the declared field which contains the field we are looking for
222 int index = inline_klass()->field_index_by_offset(offset);
223 Node* value = field_value(index);
224 assert(value != nullptr, "field value not found");
225 ciField* field = this->field(index);
226 assert(!field->is_flat() || field->type()->is_inlinetype(), "must be an inline type");
227
228 if (value->is_top()) {
229 // The graph is dying but a load may still ask for a nested field
230 // inside a flattened field before the dead load itself is folded away.
231 assert(offset == field->offset_in_bytes() || field->is_flat(), "offset mismatch");
232 return value;
233 }
234 if (!recursive || !field->is_flat()) {
235 assert(offset == field->offset_in_bytes(), "offset mismatch");
236 return value;
237 }
238
239 // Flat inline type field
240 InlineTypeNode* vt = value->as_InlineType();
241 assert(field->is_flat(), "must be flat");
242 if (offset == field->null_marker_offset()) {
243 return vt->get_null_marker();
244 } else {
245 int sub_offset = offset - field->offset_in_bytes(); // Offset of the flattened field inside the declared field
246 sub_offset += vt->inline_klass()->payload_offset(); // Add header size
247 return vt->field_value_by_offset(sub_offset, recursive);
248 }
249 }
250
251 void InlineTypeNode::set_field_value(uint index, Node* value) {
252 assert(index < field_count(), "index out of bounds");
253 set_req(Values + index, value);
254 }
255
256 void InlineTypeNode::set_field_value_by_offset(int offset, Node* value) {
257 set_field_value(field_index(offset), value);
258 }
259
260 uint InlineTypeNode::field_index(int offset) const {
261 uint i = 0;
262 for (; i < field_count() && field(i)->offset_in_bytes() != offset; i++) { }
263 assert(i < field_count(), "field not found");
264 return i;
265 }
266
267 ciField* InlineTypeNode::field(uint index) const {
268 assert(index < field_count(), "index out of bounds");
269 return inline_klass()->declared_nonstatic_field_at(index);
270 }
271
272 uint InlineTypeNode::add_fields_to_safepoint(Unique_Node_List& worklist, SafePointNode* sfpt) const {
273 uint cnt = 0;
274 for (uint i = 0; i < field_count(); ++i) {
275 Node* value = field_value(i);
276 ciField* field = this->field(i);
277 assert(!field->is_flat() || field->type()->is_inlinetype(), "must be an inline type");
278 if (field->is_flat()) {
279 InlineTypeNode* vt = value->as_InlineType();
280 cnt += vt->add_fields_to_safepoint(worklist, sfpt);
281 if (!field->is_null_free()) {
282 // The null marker of a flat field is added right after we scalarize that field
283 sfpt->add_req(vt->get_null_marker());
284 cnt++;
285 }
286 continue;
287 }
288 if (value->is_InlineType()) {
289 // Add inline type to the worklist to process later
290 worklist.push(value);
291 }
292 sfpt->add_req(value);
293 cnt++;
294 }
295 return cnt;
296 }
297
298 void InlineTypeNode::make_scalar_in_safepoint(PhaseIterGVN* igvn, Unique_Node_List& worklist, SafePointNode* sfpt) const {
299 JVMState* jvms = sfpt->jvms();
300 assert(jvms != nullptr, "missing JVMS");
301 uint first_ind = (sfpt->req() - jvms->scloff());
302
303 // Iterate over the inline type fields in order of increasing offset and add the
304 // field values to the safepoint. Nullable inline types have a null marker field that
305 // needs to be checked before using the field values.
306 sfpt->add_req(get_null_marker());
307 uint nfields = add_fields_to_safepoint(worklist, sfpt);
308 jvms->set_endoff(sfpt->req());
309 // Replace safepoint edge by SafePointScalarObjectNode
310 SafePointScalarObjectNode* sobj = new SafePointScalarObjectNode(type()->isa_instptr(),
311 nullptr,
312 first_ind,
313 sfpt->jvms()->depth(),
314 nfields);
315 sobj->init_req(0, igvn->C->root());
316 sobj = igvn->transform(sobj)->as_SafePointScalarObject();
317 igvn->rehash_node_delayed(sfpt);
318 for (uint i = jvms->debug_start(); i < jvms->debug_end(); i++) {
319 Node* debug = sfpt->in(i);
320 if (debug != nullptr && debug->uncast() == this) {
321 sfpt->set_req(i, sobj);
322 }
323 }
324 }
325
326 void InlineTypeNode::make_scalar_in_safepoints(PhaseIterGVN* igvn, bool allow_oop) {
327 make_scalar_in_safepoints(igvn, allow_oop, nullptr);
328 }
329
330 void InlineTypeNode::make_scalar_in_safepoints(PhaseIterGVN* igvn, bool allow_oop, SafePointNode* safepoint) {
331 // If the inline type has a constant or loaded oop, use the oop instead of scalarization
332 // in the safepoint to avoid keeping field loads live just for the debug info.
333 Node* oop = get_oop();
334 bool use_oop = false;
335 if (allow_oop && is_allocated(igvn) && oop->is_Phi()) {
336 Unique_Node_List worklist;
337 VectorSet visited;
338 visited.set(oop->_idx);
339 worklist.push(oop);
340 use_oop = true;
341 while (worklist.size() > 0 && use_oop) {
342 Node* n = worklist.pop();
343 for (uint i = 1; i < n->req(); i++) {
344 Node* in = n->in(i);
345 if (in->is_Phi() && !visited.test_set(in->_idx)) {
346 worklist.push(in);
347 } else if (!(in->is_Con() || in->is_Parm())) {
348 use_oop = false;
349 break;
350 }
351 }
352 }
353 } else {
354 use_oop = allow_oop && is_allocated(igvn) &&
355 (oop->is_Con() || oop->is_Parm() || oop->is_Load() || (oop->isa_DecodeN() && oop->in(1)->is_Load()));
356 }
357
358 ResourceMark rm;
359 Unique_Node_List safepoints;
360 if (safepoint == nullptr) {
361 // Gathering all SafePoint users of `this`...
362 Unique_Node_List worklist;
363 worklist.push(this);
364 while (worklist.size() > 0) {
365 Node* n = worklist.pop();
366 for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) {
367 Node* use = n->fast_out(i);
368 if (use->is_SafePoint() && !use->is_CallLeaf() && (!use->is_Call() || use->as_Call()->has_debug_use(n))) {
369 safepoints.push(use);
370 } else if (use->is_ConstraintCast()) {
371 worklist.push(use);
372 }
373 }
374 }
375 } else {
376 // ...or just the provided one if given.
377 safepoints.push(safepoint);
378 }
379
380 Unique_Node_List vt_worklist;
381 // Process all safepoint uses and scalarize inline type
382 while (safepoints.size() > 0) {
383 SafePointNode* sfpt = safepoints.pop()->as_SafePoint();
384 if (use_oop) {
385 for (uint i = sfpt->jvms()->debug_start(); i < sfpt->jvms()->debug_end(); i++) {
386 Node* debug = sfpt->in(i);
387 if (debug != nullptr && debug->uncast() == this) {
388 sfpt->set_req(i, get_oop());
389 }
390 }
391 igvn->rehash_node_delayed(sfpt);
392 } else {
393 make_scalar_in_safepoint(igvn, vt_worklist, sfpt);
394 }
395 }
396 // Now scalarize non-flat fields
397 for (uint i = 0; i < vt_worklist.size(); ++i) {
398 InlineTypeNode* vt = vt_worklist.at(i)->isa_InlineType();
399 vt->make_scalar_in_safepoints(igvn);
400 }
401 if (outcnt() == 0) {
402 igvn->record_for_igvn(this);
403 }
404 }
405
406 void InlineTypeNode::load(GraphKit* kit, Node* base, Node* ptr, bool immutable_memory, bool trust_null_free_oop, DecoratorSet decorators) {
407 // Initialize the inline type by loading its field values from
408 // memory and adding the values as input edges to the node.
409 ciInlineKlass* vk = inline_klass();
410 for (uint i = 0; i < field_count(); ++i) {
411 ciField* field = this->field(i);
412 assert(!field->is_flat() || field->type()->is_inlinetype(), "must be an inline type");
413 int field_off = field->offset_in_bytes() - vk->payload_offset();
414 Node* field_ptr = kit->basic_plus_adr(base, ptr, field_off);
415 Node* value = nullptr;
416 ciType* ft = field->type();
417 bool field_null_free = field->is_null_free();
418 if (field->is_flat()) {
419 // Recursively load the flat inline type field
420 ciInlineKlass* fvk = ft->as_inline_klass();
421 bool atomic = field->is_atomic();
422 value = make_from_flat_impl(kit, fvk, base, field_ptr, atomic, immutable_memory,
423 field_null_free, trust_null_free_oop && field_null_free, decorators);
424 } else {
425 // Load field value from memory
426 BasicType bt = type2field[ft->basic_type()];
427 assert(is_java_primitive(bt) || field_ptr->bottom_type()->is_ptr_to_narrowoop() == UseCompressedOops, "inconsistent");
428 const Type* val_type = Type::get_const_type(ft);
429 if (trust_null_free_oop && field_null_free) {
430 val_type = val_type->join_speculative(TypePtr::NOTNULL);
431 }
432 const TypePtr* field_ptr_type = (decorators & C2_MISMATCHED) == 0 ? kit->gvn().type(field_ptr)->is_ptr() : TypeRawPtr::BOTTOM;
433 value = kit->access_load_at(base, field_ptr, field_ptr_type, val_type, bt, decorators);
434 }
435 set_field_value(i, value);
436 }
437 }
438
439 void InlineTypeNode::store_flat(GraphKit* kit, Node* base, Node* ptr, bool atomic, bool immutable_memory, bool null_free, DecoratorSet decorators) {
440 ciInlineKlass* vk = inline_klass();
441 bool do_atomic = atomic;
442 // With immutable memory, a non-atomic load and an atomic load are the same
443 if (immutable_memory) {
444 do_atomic = false;
445 }
446 // If there is only one flattened field, a non-atomic load and an atomic load are the same
447 if (vk->is_naturally_atomic(null_free)) {
448 do_atomic = false;
449 }
450
451 if (!do_atomic) {
452 if (!null_free) {
453 int nm_offset = vk->null_marker_offset_in_payload();
454 Node* nm_ptr = kit->basic_plus_adr(base, ptr, nm_offset);
455 const TypePtr* nm_ptr_type = (decorators & C2_MISMATCHED) == 0 ? kit->gvn().type(nm_ptr)->is_ptr() : TypeRawPtr::BOTTOM;
456 kit->access_store_at(base, nm_ptr, nm_ptr_type, get_null_marker(), TypeInt::BOOL, T_BOOLEAN, decorators);
457 }
458 store(kit, base, ptr, immutable_memory, decorators);
459 return;
460 }
461
462 StoreFlatNode::store(kit, base, ptr, this, null_free, decorators);
463 }
464
465 void InlineTypeNode::store_flat_array(GraphKit* kit, Node* base, Node* idx) {
466 PhaseGVN& gvn = kit->gvn();
467 DecoratorSet decorators = IN_HEAP | IS_ARRAY | MO_UNORDERED;
468 kit->C->set_flat_accesses();
469 ciInlineKlass* vk = inline_klass();
470 assert(vk->maybe_flat_in_array(), "element type %s cannot be flat in array", vk->name()->as_utf8());
471
472 RegionNode* region = new RegionNode(4);
473 gvn.set_type(region, Type::CONTROL);
474 kit->record_for_igvn(region);
475
476 Node* input_memory_state = kit->reset_memory();
477 kit->set_all_memory(input_memory_state);
478
479 PhiNode* mem = PhiNode::make(region, input_memory_state, Type::MEMORY, TypePtr::BOTTOM);
480 gvn.set_type(mem, Type::MEMORY);
481 kit->record_for_igvn(mem);
482
483 PhiNode* io = PhiNode::make(region, kit->i_o(), Type::ABIO);
484 gvn.set_type(io, Type::ABIO);
485 kit->record_for_igvn(io);
486
487 Node* bol_null_free = kit->null_free_array_test(base); // Argument evaluation order is undefined in C++ and since this sets control, it needs to come first
488 IfNode* iff_null_free = kit->create_and_map_if(kit->control(), bol_null_free, PROB_FAIR, COUNT_UNKNOWN);
489
490 // Nullable
491 kit->set_control(kit->IfFalse(iff_null_free));
492 if (!kit->stopped()) {
493 assert(vk->has_nullable_atomic_layout(), "element type %s does not have a nullable flat layout", vk->name()->as_utf8());
494 kit->set_all_memory(input_memory_state);
495 Node* cast = kit->cast_to_flat_array_exact(base, vk, false, true);
496 Node* ptr = kit->array_element_address(cast, idx, T_FLAT_ELEMENT);
497 store_flat(kit, cast, ptr, true, false, false, decorators);
498
499 region->init_req(1, kit->control());
500 mem->set_req(1, kit->reset_memory());
501 io->set_req(1, kit->i_o());
502 }
503
504 // Null-free
505 kit->set_control(kit->IfTrue(iff_null_free));
506 if (!kit->stopped()) {
507 kit->set_all_memory(input_memory_state);
508
509 Node* bol_atomic = kit->null_free_atomic_array_test(base, vk);
510 IfNode* iff_atomic = kit->create_and_map_if(kit->control(), bol_atomic, PROB_FAIR, COUNT_UNKNOWN);
511
512 // Atomic
513 kit->set_control(kit->IfTrue(iff_atomic));
514 if (!kit->stopped()) {
515 assert(vk->has_null_free_atomic_layout(), "element type %s does not have a null-free atomic flat layout", vk->name()->as_utf8());
516 kit->set_all_memory(input_memory_state);
517 Node* cast = kit->cast_to_flat_array_exact(base, vk, true, true);
518 Node* ptr = kit->array_element_address(cast, idx, T_FLAT_ELEMENT);
519 store_flat(kit, cast, ptr, true, false, true, decorators);
520
521 region->init_req(2, kit->control());
522 mem->set_req(2, kit->reset_memory());
523 io->set_req(2, kit->i_o());
524 }
525
526 // Non-atomic
527 kit->set_control(kit->IfFalse(iff_atomic));
528 if (!kit->stopped()) {
529 assert(vk->has_null_free_non_atomic_layout(), "element type %s does not have a null-free non-atomic flat layout", vk->name()->as_utf8());
530 kit->set_all_memory(input_memory_state);
531 Node* cast = kit->cast_to_flat_array_exact(base, vk, true, false);
532 Node* ptr = kit->array_element_address(cast, idx, T_FLAT_ELEMENT);
533 store_flat(kit, cast, ptr, false, false, true, decorators);
534
535 region->init_req(3, kit->control());
536 mem->set_req(3, kit->reset_memory());
537 io->set_req(3, kit->i_o());
538 }
539 }
540
541 kit->set_control(gvn.transform(region));
542 kit->set_all_memory(gvn.transform(mem));
543 kit->set_i_o(gvn.transform(io));
544 }
545
546 void InlineTypeNode::store(GraphKit* kit, Node* base, Node* ptr, bool immutable_memory, DecoratorSet decorators) const {
547 // Write field values to memory
548 ciInlineKlass* vk = inline_klass();
549 for (uint i = 0; i < field_count(); ++i) {
550 ciField* field = this->field(i);
551 assert(!field->is_flat() || field->type()->is_inlinetype(), "must be an inline type");
552 int field_off = field->offset_in_bytes() - vk->payload_offset();
553 Node* field_val = field_value(i);
554 bool field_null_free = field->is_null_free();
555 ciType* ft = field->type();
556 Node* field_ptr = kit->basic_plus_adr(base, ptr, field_off);
557 if (field->is_flat()) {
558 // Recursively store the flat inline type field
559 bool atomic = field->is_atomic();
560 field_val->as_InlineType()->store_flat(kit, base, field_ptr, atomic, immutable_memory, field_null_free, decorators);
561 } else {
562 // Store field value to memory
563 BasicType bt = type2field[ft->basic_type()];
564 const TypePtr* field_ptr_type = (decorators & C2_MISMATCHED) == 0 ? kit->gvn().type(field_ptr)->is_ptr() : TypeRawPtr::BOTTOM;
565 const Type* val_type = Type::get_const_type(ft);
566 kit->access_store_at(base, field_ptr, field_ptr_type, field_val, val_type, bt, decorators);
567 }
568 }
569 }
570
571 // If a value class contains cycle, bail out from trying to expand substitutability check involving it
572 static bool check_cycle(ciInlineKlass* vk) {
573 ResourceMark rm;
574 GrowableArray<Pair<ciInlineKlass*, int>> visited;
575 visited.push(Pair(vk, 0));
576 while (visited.is_nonempty()) {
577 ciInlineKlass* current = visited.top().first;
578 bool finish = true;
579 for (int field_idx = visited.top().second; field_idx < current->nof_nonstatic_fields(); field_idx++) {
580 ciField* field = current->nonstatic_field_at(field_idx);
581 ciType* ft = field->type();
582 if (!ft->is_inlinetype()) {
583 continue;
584 } else if (visited.find_if([&](const auto& entry) { return entry.first == ft; }) >= 0) {
585 return true;
586 } else {
587 visited.top().second = field_idx + 1;
588 visited.push(Pair(ft->as_inline_klass(), 0));
589 finish = false;
590 break;
591 }
592 }
593 if (finish) {
594 visited.pop();
595 }
596 }
597 return false;
598 }
599
600 // Check if a substitutability check between 'lhs' and 'rhs' can be implemented in IR
601 bool InlineTypeNode::can_emit_substitutability_check(Node* lhs, Node* rhs) {
602 if (!lhs->bottom_type()->isa_ptr() ||
603 (rhs != nullptr && !rhs->bottom_type()->isa_ptr())) {
604 return false;
605 }
606
607 if (rhs != nullptr && lhs->eqv_uncast(rhs)) {
608 return true;
609 }
610
611 if (!lhs->bottom_type()->is_ptr()->can_be_inline_type() ||
612 (rhs != nullptr && !rhs->bottom_type()->is_ptr()->can_be_inline_type())) {
613 return true;
614 }
615
616 if (!lhs->is_InlineType() && (rhs == nullptr || !rhs->is_InlineType())) {
617 return false;
618 }
619
620 if (!lhs->is_InlineType()) {
621 swap(lhs, rhs);
622 }
623
624 InlineTypeNode* lhs_inline = lhs->as_InlineType();
625 InlineTypeNode* rhs_inline = rhs != nullptr ? rhs->isa_InlineType() : nullptr;
626 if (rhs_inline != nullptr && lhs_inline->type()->inline_klass() != rhs_inline->type()->inline_klass()) {
627 // Dead code, can skip the substitutability check
628 return true;
629 }
630
631 if (check_cycle(lhs_inline->inline_klass())) {
632 return false;
633 }
634
635 for (uint i = 0; i < lhs_inline->field_count(); i++) {
636 ciType* ft = lhs_inline->field(i)->type();
637 if (!ft->can_be_inline_klass()) {
638 continue;
639 }
640
641 Node* lhs_fv = lhs_inline->field_value(i);
642 Node* rhs_fv = rhs_inline != nullptr ? rhs_inline->field_value(i) : nullptr;
643 if (!can_emit_substitutability_check(lhs_fv, rhs_fv)) {
644 return false;
645 }
646 }
647 return true;
648 }
649
650 // Compare lhs and rhs given they are not value objects
651 static void emit_substitutability_check_primitive(GraphKit* kit, PhiNode* result, Node* lhs, Node* rhs, BasicType bt) {
652 PhaseGVN& gvn = kit->gvn();
653 Node* cmp;
654 if (bt == T_INT || is_subword_type(bt)) {
655 cmp = kit->CmpI(lhs, rhs);
656 } else if (bt == T_LONG) {
657 cmp = kit->CmpL(lhs, rhs);
658 } else if (bt == T_FLOAT) {
659 lhs = gvn.transform(new MoveF2INode(lhs));
660 rhs = gvn.transform(new MoveF2INode(rhs));
661 cmp = kit->CmpI(lhs, rhs);
662 } else if (bt == T_DOUBLE) {
663 lhs = gvn.transform(new MoveD2LNode(lhs));
664 rhs = gvn.transform(new MoveD2LNode(rhs));
665 cmp = kit->CmpL(lhs, rhs);
666 } else {
667 assert(is_reference_type(bt), "unexpected bt %s", type2name(bt));
668 cmp = kit->CmpP(lhs, rhs);
669 }
670
671 Node* bol = kit->Bool(cmp, BoolTest::eq);
672 IfNode* iff = kit->create_and_map_if(kit->control(), bol, PROB_FAIR, COUNT_UNKNOWN);
673
674 Node* iff_false = kit->IfFalse(iff);
675 result->add_req(kit->intcon(0));
676 result->region()->add_req(iff_false);
677
678 Node* iff_true = kit->IfTrue(iff);
679 kit->set_control(iff_true);
680 }
681
682 // Try to see what should be done with lhs and rhs. Either we can emit the answer if it is simple,
683 // give up and emit a call to runtime, or start comparing the value objects field-by-field. In the
684 // last case, NodeSentinel is returned.
685 static Node* emit_substitutability_check_pointer(GraphKit* kit, PhiNode* result, Node* lhs, Node* rhs) {
686 PhaseGVN& gvn = kit->gvn();
687 Node* top = kit->C->top();
688 const Type* lhs_type = gvn.type(lhs);
689 const Type* rhs_type = gvn.type(rhs);
690
691 // Dead graph
692 if (lhs_type == Type::TOP || rhs_type == Type::TOP) {
693 kit->set_control(top);
694 return top;
695 }
696
697 Node* cmp = nullptr;
698 if (lhs->eqv_uncast(rhs)) {
699 cmp = kit->intcon(0);
700 } else if (!lhs_type->is_ptr()->can_be_inline_type() || !rhs_type->is_ptr()->can_be_inline_type()) {
701 // If one of the sides is not a value object, can only be substitutable if they are the same
702 cmp = kit->CmpP(lhs, rhs);
703 } else if (lhs_type->is_instptr()->as_klass_type()->join(rhs_type->is_instptr()->as_klass_type())->empty()) {
704 // Both belongs to provably different types, must be different unless both are null. Cannot
705 // rely on the pointer independence alone because different pointers may still be
706 // substitutable.
707 cmp = kit->CmpP(lhs, rhs);
708 }
709 if (cmp != nullptr) {
710 Node* res = kit->Bool(cmp, BoolTest::eq);
711 IfNode* iff = kit->create_and_map_if(kit->control(), res, PROB_FAIR, COUNT_UNKNOWN);
712
713 Node* iff_false = kit->IfFalse(iff);
714 result->region()->add_req(iff_false);
715 result->add_req(gvn.intcon(0));
716
717 Node* iff_true = kit->IfTrue(iff);
718 kit->set_control(iff_true);
719 return iff_true;
720 }
721
722 if (!lhs_type->is_inlinetypeptr() && !rhs_type->is_inlinetypeptr()) {
723 return nullptr;
724 }
725
726 // A cycle, give up
727 ciInlineKlass* vk = lhs_type->is_inlinetypeptr() ? lhs_type->inline_klass() : rhs_type->inline_klass();
728 if (check_cycle(vk)) {
729 return nullptr;
730 }
731
732 return NodeSentinel;
733 }
734
735 Node* InlineTypeNode::emit_substitutability_check(GraphKit* kit, Node* lhs, Node* rhs) {
736 if (!kit->C->allow_macro_nodes()) {
737 // After macro expansion, InlineTypeNodes are also eliminated, creation of new ones then is not
738 // allowed
739 return nullptr;
740 }
741
742 PhaseIterGVN& igvn = *kit->gvn().is_IterGVN();
743 RegionNode* region = new RegionNode(1);
744 PhiNode* result = new PhiNode(region, TypeInt::BOOL);
745 igvn.register_new_node_with_optimizer(region);
746 igvn.register_new_node_with_optimizer(result);
747
748 Node* preprocess = emit_substitutability_check_pointer(kit, result, lhs, rhs);
749 if (preprocess == nullptr) {
750 return nullptr;
751 } else if (preprocess != NodeSentinel) {
752 region->add_req(kit->control());
753 result->add_req(igvn.intcon(1));
754 kit->set_control(region);
755 return result;
756 }
757
758 const Type* lhs_type = igvn.type(lhs);
759 const Type* rhs_type = igvn.type(rhs);
760 assert(!lhs_type->maybe_null() && !rhs_type->maybe_null(), "must check null beforehand");
761 ciInlineKlass* vk = lhs_type->is_inlinetypeptr() ? lhs_type->inline_klass() : rhs_type->inline_klass();
762 const TypeInstPtr* vk_type = TypeOopPtr::make_from_klass(vk)->join(TypePtr::NOTNULL)->is_instptr();
763
764 if (!lhs->is_InlineType()) {
765 if (!lhs_type->higher_equal(vk_type)) {
766 lhs = igvn.transform(new CheckCastPPNode(kit->control(), lhs, vk_type, ConstraintCastNode::DependencyType::NonFloatingNarrowing));
767 }
768 lhs = InlineTypeNode::make_from_oop(kit, lhs, vk);
769 }
770 if (!rhs->is_InlineType()) {
771 if (!rhs_type->higher_equal(vk_type)) {
772 rhs = igvn.transform(new CheckCastPPNode(kit->control(), rhs, vk_type, ConstraintCastNode::DependencyType::NonFloatingNarrowing));
773 }
774 rhs = InlineTypeNode::make_from_oop(kit, rhs, vk);
775 }
776
777 RegionNode* true_region = new RegionNode(3);
778 igvn.register_new_node_with_optimizer(true_region);
779 region->add_req(true_region);
780 result->add_req(igvn.intcon(1));
781
782 // To verify the substitutability of 2 notnull value objects of the same type, we need to check
783 // the substitutability of each field in the objects:
784 //
785 // substitutable(cur_lhs, cur_rhs) {
786 // cur_start_region:
787 //
788 // if (cur_lhs.field1 == null && cur_rhs.field1 == null) {
789 // goto cur_true_field1_region;
790 // } else if ((cur_lhs.field1 == null) != (cur_rhs.field1 == null) {
791 // return false;
792 // } else if (cur_lhs.field1.klass != cur_rhs.field1.klass) {
793 // return false;
794 // }
795 //
796 // cur_start_field1_region:
797 // if (substitutable(cur_lhs.field1, cur_rhs.field1)) {
798 // return false;
799 // }
800 // cur_true_field1_region:
801 //
802 // if (cur_lhs.field2 == null && cur_rhs.field2 == null) {
803 // goto cur_true_field2_region;
804 // } else if ((cur_lhs.field2 == null) != (cur_rhs.field2 == null) {
805 // return false;
806 // } else if (cur_lhs.field2.klass != cur_rhs.field2.klass) {
807 // return false;
808 // }
809 //
810 // cur_start_field2_region:
811 // if (!substitutable(cur_lhs.field1, cur_rhs.field1)) {
812 // return false;
813 // }
814 // cur_true_field2_region:
815 //
816 // ...
817 //
818 // cur_true_fieldn_region:
819 //
820 // cur_true_region:
821 // }
822 //
823 // To avoid recursion, for field values that are value objects, we create the start and end
824 // region for the substitutability check of that field when both sides are notnull, save them on
825 // the worklist to process them later.
826 ResourceMark rm;
827 using WorklistEntry = Tuple<Node*, RegionNode*, InlineTypeNode*, InlineTypeNode*>;
828 GrowableArray<WorklistEntry> worklist;
829 worklist.push(WorklistEntry(kit->control(), true_region, lhs->as_InlineType(), rhs->as_InlineType()));
830 while (worklist.is_nonempty()) {
831 WorklistEntry entry = worklist.pop();
832 Node* cur_start_region = entry.get<0>();
833 RegionNode* cur_true_region = entry.get<1>();
834 InlineTypeNode* cur_lhs = entry.get<2>();
835 InlineTypeNode* cur_rhs = entry.get<3>();
836
837 kit->set_control(cur_start_region);
838 if (kit->stopped()) {
839 break;
840 }
841 ciInlineKlass* vk = cur_lhs->inline_klass();
842 assert(vk == cur_rhs->inline_klass(), "should not reach here otherwise");
843
844 auto is_simple_field = [](ciField* field) {
845 ciType* ft = field->type();
846 return is_java_primitive(ft->basic_type()) || !ft->as_klass()->can_be_inline_klass();
847 };
848
849 // Go through all fields, process the simple ones first
850 for (int field_idx = 0; field_idx < vk->nof_declared_nonstatic_fields(); field_idx++) {
851 ciField* field = vk->declared_nonstatic_field_at(field_idx);
852 if (is_simple_field(field)) {
853 Node* cur_lhs_field = cur_lhs->field_value(field_idx);
854 Node* cur_rhs_field = cur_rhs->field_value(field_idx);
855 emit_substitutability_check_primitive(kit, result, cur_lhs_field, cur_rhs_field, field->type()->basic_type());
856 }
857 }
858
859 for (int field_idx = 0; field_idx < vk->nof_declared_nonstatic_fields(); field_idx++) {
860 ciField* field = vk->declared_nonstatic_field_at(field_idx);
861 if (is_simple_field(field)) {
862 // These fields have already been processed
863 continue;
864 }
865
866 Node* cur_lhs_field = cur_lhs->field_value(field_idx);
867 Node* cur_rhs_field = cur_rhs->field_value(field_idx);
868 InlineTypeNode* cur_lhs_inline = cur_lhs_field->isa_InlineType();
869 InlineTypeNode* cur_rhs_inline = cur_rhs_field->isa_InlineType();
870
871 Node* preprocess = emit_substitutability_check_pointer(kit, result, cur_lhs_field, cur_rhs_field);
872 if (kit->stopped()) {
873 break;
874 } else if (preprocess == nullptr) {
875 // Must emit a substitutability check for this field, give up for now
876 return nullptr;
877 } else if (preprocess != NodeSentinel) {
878 continue;
879 }
880
881 // When field is not null-free, the shape would look like this:
882 //
883 // current_control:
884 // if (cur_lhs_field == null) {
885 // if (cur_rhs_field == null) {
886 // goto field_true_region;
887 // } else {
888 // return false;
889 // }
890 // } else {
891 // if (cur_rhs_field == null) {
892 // return false;
893 // }
894 // }
895 // if (cur_lhs_field.klass != cur_rhs_field.klass) {
896 // return false;
897 // }
898 //
899 // field_start_region:
900 // if (!substitutable(cur_lhs_field, cur_rhs_field)) {
901 // return false;
902 // }
903 // field_true_region:
904 //
905 // The substitutability test between the fields of cur_lhs_field and cur_rhs_field is then
906 // pushed on the worklist to be expanded later.
907 RegionNode* field_true_region = new RegionNode(3);
908 igvn.register_new_node_with_optimizer(field_true_region);
909
910 // Firstly, filter the cases when either is null
911 Node* null_unknown = kit->top();
912 cur_lhs_field = kit->null_check_common(cur_lhs_field, T_OBJECT, false, &null_unknown);
913 if (!null_unknown->is_top()) {
914 PreserveJVMState pjvms(kit);
915 kit->set_control(null_unknown);
916 Node* null_null = kit->top();
917 kit->null_check_common(cur_rhs_field, T_OBJECT, false, &null_null);
918
919 // null - null, skip other checks
920 field_true_region->init_req(1, null_null);
921
922 // null - notnull
923 if (!kit->stopped()) {
924 region->add_req(kit->control());
925 result->add_req(igvn.intcon(0));
926 }
927 }
928 if (kit->stopped()) {
929 // cur_lhs_field is always null
930 kit->set_control(field_true_region);
931 continue;
932 }
933
934 Node* notnull_null = kit->top();
935 cur_rhs_field = kit->null_check_common(cur_rhs_field, T_OBJECT, false, ¬null_null);
936 if (!notnull_null->is_top()) {
937 region->add_req(notnull_null);
938 result->add_req(igvn.intcon(0));
939 }
940
941 if (kit->stopped()) {
942 // cur_rhs_field is always null
943 kit->set_control(field_true_region);
944 continue;
945 }
946 assert(!igvn.type(cur_lhs_field)->maybe_null() && !igvn.type(cur_rhs_field)->maybe_null(), "must be notnull");
947
948 // Can expand this comparison
949 const Type* cur_lhs_field_type = igvn.type(cur_lhs_field);
950 const Type* cur_rhs_field_type = igvn.type(cur_rhs_field);
951 ciInlineKlass* vk = cur_lhs_field_type->is_inlinetypeptr() ? cur_lhs_field_type->inline_klass() : cur_rhs_field_type->inline_klass();
952 const TypeInstKlassPtr* vk_klass_type = TypeInstKlassPtr::make(vk, Type::ignore_interfaces);
953 Node* vk_klass = igvn.makecon(vk_klass_type);
954
955 // Both must be vk
956 // InlineTypeNodes need no (new) field loads, so we can use the uncasted value below.
957 if (cur_lhs_inline != nullptr && cur_lhs_inline->inline_klass() == vk) {
958 cur_lhs_field = cur_lhs_inline;
959 } else {
960 Node* not_vk = kit->top();
961 cur_lhs_field = kit->gen_checkcast(cur_lhs_field, vk_klass, ¬_vk);
962 if (!not_vk->is_top()) {
963 region->add_req(not_vk);
964 result->add_req(igvn.intcon(0));
965 }
966 cur_lhs_field = InlineTypeNode::make_from_oop(kit, cur_lhs_field, vk);
967 }
968
969 if (cur_rhs_inline != nullptr && cur_rhs_inline->inline_klass() == vk) {
970 cur_rhs_field = cur_rhs_inline;
971 } else {
972 Node* not_vk = kit->top();
973 cur_rhs_field = kit->gen_checkcast(cur_rhs_field, vk_klass, ¬_vk);
974 if (!not_vk->is_top()) {
975 region->add_req(not_vk);
976 result->add_req(igvn.intcon(0));
977 }
978 cur_rhs_field = InlineTypeNode::make_from_oop(kit, cur_rhs_field, vk);
979 }
980
981 if (!kit->stopped()) {
982 // Push the expanded InlineTypeNodes for processing later
983 worklist.push(WorklistEntry(kit->control(), field_true_region, cur_lhs_field->as_InlineType(), cur_rhs_field->as_InlineType()));
984 }
985
986 kit->set_control(field_true_region);
987 }
988
989 cur_true_region->init_req(2, kit->control());
990 kit->set_control(cur_true_region);
991 }
992
993 kit->set_control(region);
994 return result;
995 }
996
997 InlineTypeNode* InlineTypeNode::buffer(GraphKit* kit, bool safe_for_replace) {
998 if (is_allocated(&kit->gvn())) {
999 // Already buffered
1000 return this;
1001 }
1002
1003 // Check if inline type is already buffered
1004 Node* not_buffered_ctl = kit->top();
1005 Node* not_null_oop = kit->null_check_oop(get_oop(), ¬_buffered_ctl, /* never_see_null = */ false, safe_for_replace);
1006 if (not_buffered_ctl->is_top()) {
1007 // Already buffered
1008 InlineTypeNode* vt = clone_if_required(&kit->gvn(), kit->map(), safe_for_replace);
1009 vt->set_is_buffered(kit->gvn());
1010 vt = kit->gvn().transform(vt)->as_InlineType();
1011 if (safe_for_replace) {
1012 kit->replace_in_map(this, vt);
1013 }
1014 return vt;
1015 }
1016 Node* buffered_ctl = kit->control();
1017 kit->set_control(not_buffered_ctl);
1018
1019 // Inline type is not buffered, check if it is null.
1020 Node* null_ctl = kit->top();
1021 kit->null_check_common(get_null_marker(), T_INT, false, &null_ctl);
1022 bool null_free = null_ctl->is_top();
1023
1024 RegionNode* region = new RegionNode(4);
1025 PhiNode* oop = PhiNode::make(region, not_null_oop, type()->join_speculative(null_free ? TypePtr::NOTNULL : TypePtr::BOTTOM));
1026
1027 // InlineType is already buffered
1028 region->init_req(1, buffered_ctl);
1029 oop->init_req(1, not_null_oop);
1030
1031 // InlineType is null
1032 region->init_req(2, null_ctl);
1033 oop->init_req(2, kit->gvn().zerocon(T_OBJECT));
1034
1035 PhiNode* io = PhiNode::make(region, kit->i_o(), Type::ABIO);
1036 PhiNode* mem = PhiNode::make(region, kit->merged_memory(), Type::MEMORY, TypePtr::BOTTOM);
1037
1038 if (!kit->stopped()) {
1039 assert(!is_allocated(&kit->gvn()), "already buffered");
1040 PreserveJVMState pjvms(kit);
1041 ciInlineKlass* vk = inline_klass();
1042 // Allocate and initialize buffer, re-execute on deoptimization.
1043 kit->jvms()->set_bci(kit->bci());
1044 kit->jvms()->set_should_reexecute(true);
1045 kit->kill_dead_locals();
1046 Node* klass_node = kit->makecon(TypeKlassPtr::make(vk));
1047 Node* alloc_oop = kit->new_instance(klass_node, nullptr, nullptr, /* deoptimize_on_exception */ true, this);
1048 Node* payload_alloc_oop = kit->basic_plus_adr(alloc_oop, vk->payload_offset());
1049 store(kit, alloc_oop, payload_alloc_oop, true, IN_HEAP | MO_UNORDERED | C2_TIGHTLY_COUPLED_ALLOC);
1050
1051 // Do not let stores that initialize this buffer be reordered with a subsequent
1052 // store that would make this buffer accessible by other threads.
1053 AllocateNode* alloc = AllocateNode::Ideal_allocation(alloc_oop);
1054 assert(alloc != nullptr, "must have an allocation node");
1055 kit->insert_mem_bar(Op_MemBarStoreStore, alloc->proj_out_or_null(AllocateNode::RawAddress));
1056 oop->init_req(3, alloc_oop);
1057 region->init_req(3, kit->control());
1058 io ->init_req(3, kit->i_o());
1059 mem ->init_req(3, kit->merged_memory());
1060 }
1061
1062 // Update GraphKit
1063 kit->set_control(kit->gvn().transform(region));
1064 kit->set_i_o(kit->gvn().transform(io));
1065 kit->set_all_memory(kit->gvn().transform(mem));
1066 kit->record_for_igvn(region);
1067 kit->record_for_igvn(oop);
1068 kit->record_for_igvn(io);
1069 kit->record_for_igvn(mem);
1070
1071 // Use cloned InlineTypeNode to propagate oop from now on
1072 Node* res_oop = kit->gvn().transform(oop);
1073 InlineTypeNode* vt = clone_if_required(&kit->gvn(), kit->map(), safe_for_replace);
1074 vt->set_oop(kit->gvn(), res_oop);
1075 vt->set_is_buffered(kit->gvn());
1076 vt = kit->gvn().transform(vt)->as_InlineType();
1077 kit->record_for_igvn(vt);
1078 if (safe_for_replace) {
1079 kit->replace_in_map(this, vt);
1080 }
1081 // InlineTypeNode::remove_redundant_allocations piggybacks on split if.
1082 // Make sure it gets a chance to remove this allocation.
1083 kit->C->set_has_split_ifs(true);
1084 return vt;
1085 }
1086
1087 bool InlineTypeNode::is_allocated(PhaseGVN* phase) const {
1088 if (phase->type(get_is_buffered()) == TypeInt::ONE) {
1089 return true;
1090 }
1091 Node* oop = get_oop();
1092 const Type* oop_type = (phase != nullptr) ? phase->type(oop) : oop->bottom_type();
1093 return !oop_type->maybe_null();
1094 }
1095
1096 static void replace_proj(Compile* C, CallNode* call, uint& proj_idx, Node* value, BasicType bt) {
1097 ProjNode* pn = call->proj_out_or_null(proj_idx);
1098 if (pn != nullptr) {
1099 C->gvn_replace_by(pn, value);
1100 C->initial_gvn()->hash_delete(pn);
1101 pn->set_req(0, C->top());
1102 }
1103 proj_idx += type2size[bt];
1104 }
1105
1106 // When a call returns multiple values, it has several result
1107 // projections, one per field. Replacing the result of the call by an
1108 // inline type node (after late inlining) requires that for each result
1109 // projection, we find the corresponding inline type field.
1110 void InlineTypeNode::replace_call_results(GraphKit* kit, CallNode* call, Compile* C) const {
1111 uint proj_idx = TypeFunc::Parms;
1112 // Replace oop projection
1113 replace_proj(C, call, proj_idx, get_oop(), T_OBJECT);
1114 // Replace field projections
1115 replace_field_projs(C, call, proj_idx);
1116 // Replace null_marker projection
1117 replace_proj(C, call, proj_idx, get_null_marker(), T_BOOLEAN);
1118 assert(proj_idx == call->tf()->range_cc()->cnt(), "missed a projection");
1119 }
1120
1121 void InlineTypeNode::replace_field_projs(Compile* C, CallNode* call, uint& proj_idx) const {
1122 for (uint i = 0; i < field_count(); ++i) {
1123 Node* value = field_value(i);
1124 ciField* field = this->field(i);
1125 assert(!field->is_flat() || field->type()->is_inlinetype(), "must be an inline type");
1126 if (field->is_flat()) {
1127 InlineTypeNode* vt = value->as_InlineType();
1128 // Replace field projections for flat field
1129 vt->replace_field_projs(C, call, proj_idx);
1130 if (!field->is_null_free()) {
1131 // Replace null_marker projection for nullable field
1132 replace_proj(C, call, proj_idx, vt->get_null_marker(), T_BOOLEAN);
1133 }
1134 continue;
1135 }
1136 // Replace projection for field value
1137 replace_proj(C, call, proj_idx, value, field->type()->basic_type());
1138 }
1139 }
1140
1141 InlineTypeNode* InlineTypeNode::allocate_fields(GraphKit* kit) {
1142 InlineTypeNode* vt = clone_if_required(&kit->gvn(), kit->map());
1143 for (uint i = 0; i < field_count(); i++) {
1144 Node* value = field_value(i);
1145 ciField* field = this->field(i);
1146 assert(!field->is_flat() || field->type()->is_inlinetype(), "must be an inline type");
1147 if (field->is_flat()) {
1148 // Flat inline type field
1149 vt->set_field_value(i, value->as_InlineType()->allocate_fields(kit));
1150 } else if (value->is_InlineType()) {
1151 // Non-flat inline type field
1152 vt->set_field_value(i, value->as_InlineType()->buffer(kit));
1153 }
1154 }
1155 vt = kit->gvn().transform(vt)->as_InlineType();
1156 kit->replace_in_map(this, vt);
1157 return vt;
1158 }
1159
1160 // Replace a buffer allocation by a dominating allocation
1161 static void replace_allocation(PhaseIterGVN* igvn, Node* res, Node* dom) {
1162 // Remove initializing stores and GC barriers
1163 for (DUIterator_Fast imax, i = res->fast_outs(imax); i < imax; i++) {
1164 Node* use = res->fast_out(i);
1165 if (use->is_AddP()) {
1166 for (DUIterator_Fast jmax, j = use->fast_outs(jmax); j < jmax; j++) {
1167 Node* store = use->fast_out(j)->isa_Store();
1168 if (store != nullptr) {
1169 igvn->rehash_node_delayed(store);
1170 igvn->replace_in_uses(store, store->in(MemNode::Memory));
1171 }
1172 }
1173 } else if (use->Opcode() == Op_CastP2X) {
1174 if (UseG1GC && use->find_out_with(Op_XorX)->in(1) != use) {
1175 // The G1 pre-barrier uses a CastP2X both for the pointer of the object
1176 // we store into, as well as the value we are storing. Skip if this is a
1177 // barrier for storing 'res' into another object.
1178 continue;
1179 }
1180 BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
1181 bs->eliminate_gc_barrier(igvn, use);
1182 --i; --imax;
1183 }
1184 }
1185 igvn->replace_node(res, dom);
1186 }
1187
1188 Node* InlineTypeNode::Ideal(PhaseGVN* phase, bool can_reshape) {
1189 Node* oop = get_oop();
1190 if (oop->isa_InlineType() && !phase->type(oop)->maybe_null()) {
1191 InlineTypeNode* vtptr = oop->as_InlineType();
1192 assert(inline_klass() == vtptr->inline_klass(), "inconsistent types");
1193 set_oop(*phase, vtptr->get_oop());
1194 set_is_buffered(*phase);
1195 set_null_marker(*phase);
1196 for (uint i = Values; i < vtptr->req(); ++i) {
1197 set_req(i, vtptr->in(i));
1198 }
1199 return this;
1200 }
1201
1202 // Use base oop if fields are loaded from memory, don't do so if base is the CheckCastPP of an
1203 // allocation because the only case we load from a naked CheckCastPP is when we exit a
1204 // constructor of an inline type and we want to relinquish the larval oop there. This has a
1205 // couple of benefits:
1206 // - The allocation is likely to be elided earlier if it is not an input of an InlineTypeNode.
1207 // - The InlineTypeNode without an allocation input is more likely to be GVN-ed. This may emerge
1208 // when we try to clone a value object.
1209 // - The buffering, if needed, is delayed until it is required. This new allocation, since it is
1210 // created from an InlineTypeNode, is recognized as not having a unique identity and in the
1211 // future, we can move them around more freely such as hoisting out of loops. This is not true
1212 // for the old allocation since larval value objects do have unique identities.
1213 Node* base = is_loaded(phase);
1214 if (base != nullptr && !base->is_InlineType() && !phase->type(base)->maybe_null() && phase->C->allow_macro_nodes() && AllocateNode::Ideal_allocation(base) == nullptr) {
1215 if (oop != base || !is_allocated(phase)) {
1216 set_oop(*phase, base);
1217 set_is_buffered(*phase);
1218 return this;
1219 }
1220 }
1221
1222 if (can_reshape) {
1223 PhaseIterGVN* igvn = phase->is_IterGVN();
1224 if (is_allocated(phase)) {
1225 // Search for and remove re-allocations of this inline type. Ignore scalar replaceable ones,
1226 // they will be removed anyway and changing the memory chain will confuse other optimizations.
1227 // This can happen with late inlining when we first allocate an inline type argument
1228 // but later decide to inline the call after the callee code also triggered allocation.
1229 for (DUIterator_Fast imax, i = fast_outs(imax); i < imax; i++) {
1230 AllocateNode* alloc = fast_out(i)->isa_Allocate();
1231 if (alloc != nullptr && alloc->in(AllocateNode::InlineType) == this && !alloc->_is_scalar_replaceable) {
1232 // Found a re-allocation
1233 Node* res = alloc->result_cast();
1234 if (res != nullptr && res->is_CheckCastPP()) {
1235 // Replace allocation by oop and unlink AllocateNode
1236 replace_allocation(igvn, res, oop);
1237 igvn->replace_input_of(alloc, AllocateNode::InlineType, igvn->C->top());
1238 --i; --imax;
1239 }
1240 }
1241 }
1242 }
1243 }
1244
1245 return nullptr;
1246 }
1247
1248 InlineTypeNode* InlineTypeNode::make_uninitialized(PhaseGVN& gvn, ciInlineKlass* vk, bool null_free) {
1249 // Create a new InlineTypeNode with uninitialized values and nullptr oop
1250 InlineTypeNode* vt = new InlineTypeNode(vk, gvn.zerocon(T_OBJECT), null_free);
1251 vt->set_is_buffered(gvn, false);
1252 vt->set_null_marker(gvn);
1253 return vt;
1254 }
1255
1256 InlineTypeNode* InlineTypeNode::make_all_zero(PhaseGVN& gvn, ciInlineKlass* vk) {
1257 return make_all_zero_impl(gvn, vk);
1258 }
1259
1260 InlineTypeNode* InlineTypeNode::make_all_zero_impl(PhaseGVN& gvn, ciInlineKlass* vk) {
1261 // Create a new InlineTypeNode initialized with all zero
1262 InlineTypeNode* vt = new InlineTypeNode(vk, gvn.zerocon(T_OBJECT), /* null_free= */ true);
1263 vt->set_is_buffered(gvn, false);
1264 vt->set_null_marker(gvn);
1265 for (uint i = 0; i < vt->field_count(); ++i) {
1266 ciField* field = vt->field(i);
1267 assert(!field->is_flat() || field->type()->is_inlinetype(), "must be an inline type");
1268 ciType* ft = field->type();
1269 Node* value;
1270 if (field->is_null_free()) {
1271 value = make_all_zero_impl(gvn, ft->as_inline_klass());
1272 } else if (field->is_flat()) {
1273 value = make_null_impl(gvn, ft->as_inline_klass());
1274 } else {
1275 value = gvn.zerocon(ft->basic_type());
1276 }
1277 vt->set_field_value(i, value);
1278 }
1279 vt = gvn.transform(vt)->as_InlineType();
1280 assert(vt->is_all_zero(&gvn), "must be the all-zero inline type");
1281 return vt;
1282 }
1283
1284 bool InlineTypeNode::is_all_zero(PhaseGVN* gvn, bool flat) const {
1285 const TypeInt* tinit = gvn->type(get_null_marker())->isa_int();
1286 if (tinit == nullptr || !tinit->is_con(1)) {
1287 return false; // May be null
1288 }
1289 for (uint i = 0; i < field_count(); ++i) {
1290 Node* value = field_value(i);
1291 ciField* field = this->field(i);
1292 assert(!field->is_flat() || field->type()->is_inlinetype(), "must be an inline type");
1293 if (field->is_null_free()) {
1294 // Null-free value class field must have the all-zero value. If 'flat' is set,
1295 // reject non-flat fields because they need to be initialized with an oop to a buffer.
1296 if (!value->is_InlineType() || !value->as_InlineType()->is_all_zero(gvn) || (flat && !field->is_flat())) {
1297 return false;
1298 }
1299 continue;
1300 } else if (value->is_InlineType()) {
1301 // Nullable value class field must be null
1302 tinit = gvn->type(value->as_InlineType()->get_null_marker())->isa_int();
1303 if (tinit != nullptr && tinit->is_con(0)) {
1304 continue;
1305 }
1306 return false;
1307 } else if (!gvn->type(value)->is_zero_type()) {
1308 return false;
1309 }
1310 }
1311 return true;
1312 }
1313
1314 InlineTypeNode* InlineTypeNode::make_from_oop(GraphKit* kit, Node* oop, ciInlineKlass* vk) {
1315 return make_from_oop_impl(kit, oop, vk);
1316 }
1317
1318 InlineTypeNode* InlineTypeNode::make_from_oop_impl(GraphKit* kit, Node* oop, ciInlineKlass* vk) {
1319 PhaseGVN& gvn = kit->gvn();
1320
1321 // Create and initialize an InlineTypeNode by loading all field
1322 // values from a heap-allocated version and also save the oop.
1323 InlineTypeNode* vt = oop->isa_InlineType();
1324 if (vt != nullptr) {
1325 return vt;
1326 }
1327
1328 if (gvn.type(oop) == TypePtr::NULL_PTR) {
1329 return make_null_impl(gvn, vk);
1330 }
1331
1332 if (gvn.type(oop)->maybe_null()) {
1333 // Add a null check because the oop may be null
1334 Node* null_ctl = kit->top();
1335 Node* not_null_oop = kit->null_check_oop(oop, &null_ctl);
1336 if (kit->stopped()) {
1337 // Constant null
1338 kit->set_control(null_ctl);
1339 vt = make_null_impl(gvn, vk);
1340 kit->record_for_igvn(vt);
1341 return vt;
1342 }
1343 vt = new InlineTypeNode(vk, not_null_oop, /* null_free= */ false);
1344 vt->set_is_buffered(gvn);
1345 vt->set_null_marker(gvn);
1346 Node* payload_ptr = kit->basic_plus_adr(not_null_oop, vk->payload_offset());
1347 vt->load(kit, not_null_oop, payload_ptr, true, true, IN_HEAP | MO_UNORDERED);
1348
1349 if (null_ctl != kit->top()) {
1350 InlineTypeNode* null_vt = make_null_impl(gvn, vk);
1351 Node* region = new RegionNode(3);
1352 region->init_req(1, kit->control());
1353 region->init_req(2, null_ctl);
1354 vt = vt->clone_with_phis(&gvn, region, kit->map());
1355 vt->merge_with(&gvn, null_vt, 2, true);
1356 vt->set_oop(gvn, oop);
1357 vt->set_is_buffered(gvn);
1358 kit->set_control(gvn.transform(region));
1359 }
1360 } else {
1361 // Oop can never be null
1362 vt = new InlineTypeNode(vk, oop, /* null_free= */ true);
1363 vt->set_is_buffered(gvn);
1364 vt->set_null_marker(gvn);
1365 Node* payload_ptr = kit->basic_plus_adr(oop, vk->payload_offset());
1366 vt->load(kit, oop, payload_ptr, true, true, IN_HEAP | MO_UNORDERED);
1367 }
1368 assert(vt->is_allocated(&gvn), "inline type should be allocated");
1369 kit->record_for_igvn(vt);
1370 return gvn.transform(vt)->as_InlineType();
1371 }
1372
1373 InlineTypeNode* InlineTypeNode::make_from_flat(GraphKit* kit, ciInlineKlass* vk, Node* base, Node* ptr,
1374 bool atomic, bool immutable_memory, bool null_free, DecoratorSet decorators) {
1375 return make_from_flat_impl(kit, vk, base, ptr, atomic, immutable_memory, null_free, null_free, decorators);
1376 }
1377
1378 // GraphKit wrapper for the 'make_from_flat' method
1379 InlineTypeNode* InlineTypeNode::make_from_flat_impl(GraphKit* kit, ciInlineKlass* vk, Node* base, Node* ptr, bool atomic, bool immutable_memory,
1380 bool null_free, bool trust_null_free_oop, DecoratorSet decorators) {
1381 assert(null_free || !trust_null_free_oop, "cannot trust null-free oop when the holder object is not null-free");
1382 PhaseGVN& gvn = kit->gvn();
1383 bool do_atomic = atomic;
1384 // With immutable memory, a non-atomic load and an atomic load are the same
1385 if (immutable_memory) {
1386 do_atomic = false;
1387 }
1388 // If there is only one flattened field, a non-atomic load and an atomic load are the same
1389 if (vk->is_naturally_atomic(null_free)) {
1390 do_atomic = false;
1391 }
1392
1393 if (!do_atomic) {
1394 InlineTypeNode* vt = make_uninitialized(kit->gvn(), vk, null_free);
1395 if (!null_free) {
1396 int nm_offset = vk->null_marker_offset_in_payload();
1397 Node* nm_ptr = kit->basic_plus_adr(base, ptr, nm_offset);
1398 const TypePtr* nm_ptr_type = (decorators & C2_MISMATCHED) == 0 ? gvn.type(nm_ptr)->is_ptr() : TypeRawPtr::BOTTOM;
1399 Node* nm_value = kit->access_load_at(base, nm_ptr, nm_ptr_type, TypeInt::BOOL, T_BOOLEAN, decorators);
1400 vt->set_req(NullMarker, nm_value);
1401 }
1402
1403 vt->load(kit, base, ptr, immutable_memory, trust_null_free_oop, decorators);
1404 return gvn.transform(vt)->as_InlineType();
1405 }
1406
1407 assert(!immutable_memory, "immutable memory does not need explicit atomic access");
1408 return LoadFlatNode::load(kit, vk, base, ptr, null_free, trust_null_free_oop, decorators);
1409 }
1410
1411 InlineTypeNode* InlineTypeNode::make_from_flat_array(GraphKit* kit, ciInlineKlass* vk, Node* base, Node* idx) {
1412 assert(vk->maybe_flat_in_array(), "element type %s cannot be flat in array", vk->name()->as_utf8());
1413 PhaseGVN& gvn = kit->gvn();
1414 // The flat field loads are dependent on both the array layout checks as well as the range check.
1415 DecoratorSet decorators = IN_HEAP | IS_ARRAY | MO_UNORDERED | C2_CONTROL_DEPENDENT_LOAD | C2_UNKNOWN_CONTROL_LOAD;
1416 kit->C->set_flat_accesses();
1417 InlineTypeNode* vt_nullable = nullptr;
1418 InlineTypeNode* vt_null_free = nullptr;
1419 InlineTypeNode* vt_non_atomic = nullptr;
1420
1421 RegionNode* region = new RegionNode(4);
1422 gvn.set_type(region, Type::CONTROL);
1423 kit->record_for_igvn(region);
1424
1425 Node* input_memory_state = kit->reset_memory();
1426 kit->set_all_memory(input_memory_state);
1427
1428 PhiNode* mem = PhiNode::make(region, input_memory_state, Type::MEMORY, TypePtr::BOTTOM);
1429 gvn.set_type(mem, Type::MEMORY);
1430 kit->record_for_igvn(mem);
1431
1432 PhiNode* io = PhiNode::make(region, kit->i_o(), Type::ABIO);
1433 gvn.set_type(io, Type::ABIO);
1434 kit->record_for_igvn(io);
1435
1436 Node* bol_null_free = kit->null_free_array_test(base); // Argument evaluation order is undefined in C++ and since this sets control, it needs to come first
1437 IfNode* iff_null_free = kit->create_and_map_if(kit->control(), bol_null_free, PROB_FAIR, COUNT_UNKNOWN);
1438
1439 // Nullable
1440 kit->set_control(kit->IfFalse(iff_null_free));
1441 if (!kit->stopped()) {
1442 assert(vk->has_nullable_atomic_layout(), "element type %s does not have a nullable flat layout", vk->name()->as_utf8());
1443 kit->set_all_memory(input_memory_state);
1444 Node* cast = kit->cast_to_flat_array_exact(base, vk, false, true);
1445 Node* ptr = kit->array_element_address(cast, idx, T_FLAT_ELEMENT);
1446 vt_nullable = InlineTypeNode::make_from_flat(kit, vk, cast, ptr, true, false, false, decorators);
1447
1448 region->init_req(1, kit->control());
1449 mem->set_req(1, kit->reset_memory());
1450 io->set_req(1, kit->i_o());
1451 }
1452
1453 // Null-free
1454 kit->set_control(kit->IfTrue(iff_null_free));
1455 if (!kit->stopped()) {
1456 kit->set_all_memory(input_memory_state);
1457
1458 Node* bol_atomic = kit->null_free_atomic_array_test(base, vk);
1459 IfNode* iff_atomic = kit->create_and_map_if(kit->control(), bol_atomic, PROB_FAIR, COUNT_UNKNOWN);
1460
1461 // Atomic
1462 kit->set_control(kit->IfTrue(iff_atomic));
1463 if (!kit->stopped()) {
1464 assert(vk->has_null_free_atomic_layout(), "element type %s does not have a null-free atomic flat layout", vk->name()->as_utf8());
1465 kit->set_all_memory(input_memory_state);
1466 Node* cast = kit->cast_to_flat_array_exact(base, vk, true, true);
1467 Node* ptr = kit->array_element_address(cast, idx, T_FLAT_ELEMENT);
1468 vt_null_free = InlineTypeNode::make_from_flat(kit, vk, cast, ptr, true, false, true, decorators);
1469
1470 region->init_req(2, kit->control());
1471 mem->set_req(2, kit->reset_memory());
1472 io->set_req(2, kit->i_o());
1473 }
1474
1475 // Non-Atomic
1476 kit->set_control(kit->IfFalse(iff_atomic));
1477 if (!kit->stopped()) {
1478 assert(vk->has_null_free_non_atomic_layout(), "element type %s does not have a null-free non-atomic flat layout", vk->name()->as_utf8());
1479 kit->set_all_memory(input_memory_state);
1480 Node* cast = kit->cast_to_flat_array_exact(base, vk, true, false);
1481 Node* ptr = kit->array_element_address(cast, idx, T_FLAT_ELEMENT);
1482 vt_non_atomic = InlineTypeNode::make_from_flat(kit, vk, cast, ptr, false, false, true, decorators);
1483
1484 region->init_req(3, kit->control());
1485 mem->set_req(3, kit->reset_memory());
1486 io->set_req(3, kit->i_o());
1487 }
1488 }
1489
1490 InlineTypeNode* vt = nullptr;
1491 if (vt_nullable == nullptr && vt_null_free == nullptr && vt_non_atomic == nullptr) {
1492 // All paths are dead
1493 vt = make_null(gvn, vk);
1494 } else if (vt_nullable == nullptr && vt_null_free == nullptr) {
1495 vt = vt_non_atomic;
1496 } else if (vt_nullable == nullptr && vt_non_atomic == nullptr) {
1497 vt = vt_null_free;
1498 } else if (vt_null_free == nullptr && vt_non_atomic == nullptr) {
1499 vt = vt_nullable;
1500 }
1501 if (vt != nullptr) {
1502 kit->set_control(kit->gvn().transform(region));
1503 kit->set_all_memory(kit->gvn().transform(mem));
1504 kit->set_i_o(kit->gvn().transform(io));
1505 return vt;
1506 }
1507
1508 InlineTypeNode* zero = InlineTypeNode::make_null(gvn, vk);
1509 vt = zero->clone_with_phis(&gvn, region);
1510 if (vt_nullable != nullptr) {
1511 vt = vt->merge_with(&gvn, vt_nullable, 1, false);
1512 }
1513 if (vt_null_free != nullptr) {
1514 vt = vt->merge_with(&gvn, vt_null_free, 2, false);
1515 }
1516 if (vt_non_atomic != nullptr) {
1517 vt = vt->merge_with(&gvn, vt_non_atomic, 3, false);
1518 }
1519
1520 kit->set_control(kit->gvn().transform(region));
1521 kit->set_all_memory(kit->gvn().transform(mem));
1522 kit->set_i_o(kit->gvn().transform(io));
1523 return gvn.transform(vt)->as_InlineType();
1524 }
1525
1526 InlineTypeNode* InlineTypeNode::make_from_multi(GraphKit* kit, MultiNode* multi, ciInlineKlass* vk, uint& base_input, bool in, bool null_free) {
1527 InlineTypeNode* vt = make_uninitialized(kit->gvn(), vk, null_free);
1528 if (!in || multi->is_Start()) {
1529 // Keep track of the oop. The inline type might already be buffered.
1530 Node* oop = nullptr;
1531 if (multi->is_Start()) {
1532 oop = kit->gvn().transform(new ParmNode(multi->as_Start(), base_input++));
1533 } else {
1534 oop = kit->gvn().transform(new ProjNode(multi, base_input++));
1535 }
1536 vt->set_oop(kit->gvn(), oop);
1537 } else {
1538 Node* oop = multi->as_Call()->in(base_input++);
1539 vt->set_oop(kit->gvn(), oop);
1540 }
1541 vt->initialize_fields(kit, multi, base_input, in, null_free, nullptr);
1542 return kit->gvn().transform(vt)->as_InlineType();
1543 }
1544
1545 Node* InlineTypeNode::is_loaded(PhaseGVN* phase, ciInlineKlass* vk, Node* base, int holder_offset) const {
1546 if (vk == nullptr) {
1547 vk = inline_klass();
1548 }
1549 for (uint i = 0; i < field_count(); ++i) {
1550 ciField* field = this->field(i);
1551 int offset = holder_offset + field->offset_in_bytes();
1552 Node* value = field_value(i);
1553 if (value->is_InlineType()) {
1554 assert(!field->is_flat() || field->type()->is_inlinetype(), "must be an inline type");
1555 InlineTypeNode* vt = value->as_InlineType();
1556 if (vt->type()->inline_klass()->is_empty()) {
1557 continue;
1558 } else if (field->is_flat() && vt->is_InlineType()) {
1559 // Check inline type field load recursively
1560 base = vt->as_InlineType()->is_loaded(phase, vk, base, offset - vt->type()->inline_klass()->payload_offset());
1561 if (base == nullptr) {
1562 return nullptr;
1563 }
1564 continue;
1565 } else {
1566 value = vt->get_oop();
1567 if (value->Opcode() == Op_CastPP) {
1568 // Skip CastPP
1569 value = value->in(1);
1570 }
1571 }
1572 }
1573 if (value->isa_DecodeN()) {
1574 // Skip DecodeN
1575 value = value->in(1);
1576 }
1577 if (value->isa_Load()) {
1578 // Check if base and offset of field load matches inline type layout
1579 intptr_t loffset = 0;
1580 Node* lbase = AddPNode::Ideal_base_and_offset(value->in(MemNode::Address), phase, loffset);
1581 if (lbase == nullptr || (lbase != base && base != nullptr) || loffset != offset) {
1582 return nullptr;
1583 } else if (base == nullptr) {
1584 // Set base and check if pointer type matches
1585 base = lbase;
1586 const TypeInstPtr* vtptr = phase->type(base)->isa_instptr();
1587 if (vtptr == nullptr || !vtptr->instance_klass()->equals(vk)) {
1588 return nullptr;
1589 }
1590 }
1591 } else {
1592 return nullptr;
1593 }
1594 }
1595 return base;
1596 }
1597
1598 Node* InlineTypeNode::tagged_klass(ciInlineKlass* vk, PhaseGVN& gvn) {
1599 const TypeKlassPtr* tk = TypeKlassPtr::make(vk);
1600 intptr_t bits = tk->get_con();
1601 set_nth_bit(bits, 0);
1602 return gvn.longcon((jlong)bits);
1603 }
1604
1605 void InlineTypeNode::pass_fields(GraphKit* kit, Node* n, uint& base_input, bool in, bool null_free, bool root) {
1606 if (root) {
1607 if (is_allocated(&kit->gvn())) {
1608 // Keep the information that 'this' is buffered
1609 n->init_req(base_input++, this);
1610 } else {
1611 n->init_req(base_input++, get_oop());
1612 }
1613 }
1614 if (!null_free && in) {
1615 n->init_req(base_input++, get_null_marker());
1616 }
1617 for (uint i = 0; i < field_count(); i++) {
1618 Node* arg = field_value(i);
1619 ciField* field = this->field(i);
1620 assert(!field->is_flat() || field->type()->is_inlinetype(), "must be an inline type");
1621 if (field->is_flat()) {
1622 // Flat inline type field
1623 arg->as_InlineType()->pass_fields(kit, n, base_input, in);
1624 if (!field->is_null_free()) {
1625 assert(field->null_marker_offset() != -1, "inconsistency");
1626 n->init_req(base_input++, arg->as_InlineType()->get_null_marker());
1627 }
1628 } else {
1629 if (arg->is_InlineType()) {
1630 // Non-flat inline type field
1631 InlineTypeNode* vt = arg->as_InlineType();
1632 assert(n->Opcode() != Op_Return || vt->is_allocated(&kit->gvn()), "inline type field should be allocated on return");
1633 arg = vt->buffer(kit);
1634 }
1635 // Initialize call/return arguments
1636 n->init_req(base_input++, arg);
1637 if (field->type()->size() == 2) {
1638 n->init_req(base_input++, kit->top());
1639 }
1640 }
1641 }
1642 }
1643
1644 void InlineTypeNode::initialize_fields(GraphKit* kit, MultiNode* multi, uint& base_input, bool in, bool no_null_marker, Node* null_check_region) {
1645 PhaseGVN& gvn = kit->gvn();
1646 Node* null_marker = nullptr;
1647 if (!no_null_marker) {
1648 // Nullable inline type
1649 if (in) {
1650 // Set null marker
1651 if (multi->is_Start()) {
1652 null_marker = gvn.transform(new ParmNode(multi->as_Start(), base_input));
1653 } else {
1654 null_marker = multi->as_Call()->in(base_input);
1655 }
1656 set_req(NullMarker, null_marker);
1657 base_input++;
1658 }
1659 // Add a null check to make subsequent loads dependent on
1660 assert(null_check_region == nullptr, "already set");
1661 if (null_marker == nullptr) {
1662 // Will only be initialized below, use dummy node for now
1663 null_marker = new Node(1);
1664 null_marker->init_req(0, kit->control()); // Add an input to prevent dummy from being dead
1665 gvn.set_type_bottom(null_marker);
1666 }
1667 Node* null_ctrl = kit->top();
1668 kit->null_check_common(null_marker, T_INT, false, &null_ctrl);
1669 Node* non_null_ctrl = kit->control();
1670 null_check_region = new RegionNode(3);
1671 null_check_region->init_req(1, non_null_ctrl);
1672 null_check_region->init_req(2, null_ctrl);
1673 null_check_region = gvn.transform(null_check_region);
1674 kit->set_control(null_check_region);
1675 }
1676
1677 for (uint i = 0; i < field_count(); ++i) {
1678 ciField* field = this->field(i);
1679 ciType* type = field->type();
1680 Node* parm = nullptr;
1681 assert(!field->is_flat() || field->type()->is_inlinetype(), "must be an inline type");
1682 if (field->is_flat()) {
1683 // Flat inline type field
1684 InlineTypeNode* vt = make_uninitialized(gvn, type->as_inline_klass(), field->is_null_free());
1685 vt->initialize_fields(kit, multi, base_input, in, true, null_check_region);
1686 if (!field->is_null_free()) {
1687 assert(field->null_marker_offset() != -1, "inconsistency");
1688 Node* null_marker_field_vt = nullptr;
1689 if (multi->is_Start()) {
1690 null_marker_field_vt = gvn.transform(new ParmNode(multi->as_Start(), base_input));
1691 } else if (in) {
1692 null_marker_field_vt = multi->as_Call()->in(base_input);
1693 } else {
1694 null_marker_field_vt = gvn.transform(new ProjNode(multi->as_Call(), base_input));
1695 }
1696 vt->set_req(NullMarker, null_marker_field_vt);
1697 base_input++;
1698 }
1699 parm = gvn.transform(vt);
1700 } else {
1701 if (multi->is_Start()) {
1702 assert(in, "return from start?");
1703 parm = gvn.transform(new ParmNode(multi->as_Start(), base_input));
1704 } else if (in) {
1705 parm = multi->as_Call()->in(base_input);
1706 } else {
1707 parm = gvn.transform(new ProjNode(multi->as_Call(), base_input));
1708 }
1709 base_input += type->size();
1710 }
1711 assert(parm != nullptr, "should never be null");
1712 assert(field_value(i) == nullptr, "already set");
1713 set_field_value(i, parm);
1714 gvn.record_for_igvn(parm);
1715 }
1716 // The last argument is used to pass the null marker to compiled code
1717 if (!no_null_marker && !in) {
1718 Node* cmp = null_marker->raw_out(0);
1719 null_marker = gvn.transform(new ProjNode(multi->as_Call(), base_input));
1720 set_req(NullMarker, null_marker);
1721 gvn.hash_delete(cmp);
1722 cmp->set_req(1, null_marker);
1723 gvn.hash_find_insert(cmp);
1724 gvn.record_for_igvn(cmp);
1725 base_input++;
1726 }
1727 }
1728
1729 // Search for multiple allocations of this inline type and try to replace them by dominating allocations.
1730 // Equivalent InlineTypeNodes are merged by GVN, so we just need to search for AllocateNode users to find redundant allocations.
1731 void InlineTypeNode::remove_redundant_allocations(PhaseIdealLoop* phase) const {
1732 PhaseIterGVN* igvn = &phase->igvn();
1733 // Search for allocations of this inline type. Ignore scalar replaceable ones, they
1734 // will be removed anyway and changing the memory chain will confuse other optimizations.
1735 for (DUIterator_Fast imax, i = fast_outs(imax); i < imax; i++) {
1736 AllocateNode* alloc = fast_out(i)->isa_Allocate();
1737 if (alloc != nullptr && alloc->in(AllocateNode::InlineType) == this && !alloc->_is_scalar_replaceable) {
1738 Node* res = alloc->result_cast();
1739 if (res == nullptr || !res->is_CheckCastPP()) {
1740 break; // No unique CheckCastPP
1741 }
1742 // Search for a dominating allocation of the same inline type
1743 Node* res_dom = res;
1744 for (DUIterator_Fast jmax, j = fast_outs(jmax); j < jmax; j++) {
1745 AllocateNode* alloc_other = fast_out(j)->isa_Allocate();
1746 if (alloc_other != nullptr && alloc_other->in(AllocateNode::InlineType) == this && !alloc_other->_is_scalar_replaceable) {
1747 Node* res_other = alloc_other->result_cast();
1748 if (res_other != nullptr && res_other->is_CheckCastPP() && res_other != res_dom &&
1749 phase->is_dominator(res_other->in(0), res_dom->in(0))) {
1750 res_dom = res_other;
1751 }
1752 }
1753 }
1754 if (res_dom != res) {
1755 // Replace allocation by dominating one.
1756 replace_allocation(igvn, res, res_dom);
1757 // The result of the dominated allocation is now unused and will be removed
1758 // later in PhaseMacroExpand::eliminate_allocate_node to not confuse loop opts.
1759 igvn->_worklist.push(alloc);
1760 }
1761 }
1762 }
1763 }
1764
1765 InlineTypeNode* InlineTypeNode::make_null(PhaseGVN& gvn, ciInlineKlass* vk, bool transform) {
1766 return make_null_impl(gvn, vk, transform);
1767 }
1768
1769 InlineTypeNode* InlineTypeNode::make_null_impl(PhaseGVN& gvn, ciInlineKlass* vk, bool transform) {
1770 InlineTypeNode* vt = new InlineTypeNode(vk, gvn.zerocon(T_OBJECT), /* null_free= */ false);
1771 vt->set_is_buffered(gvn);
1772 vt->set_null_marker(gvn, gvn.intcon(0));
1773 for (uint i = 0; i < vt->field_count(); i++) {
1774 ciField* field = vt->field(i);
1775 ciType* ft = field->type();
1776 Node* value;
1777 if (field->is_flat()) {
1778 value = make_null_impl(gvn, ft->as_inline_klass());
1779 } else {
1780 value = gvn.zerocon(ft->basic_type());
1781 }
1782 vt->set_field_value(i, value);
1783 }
1784 return transform ? gvn.transform(vt)->as_InlineType() : vt;
1785 }
1786
1787 InlineTypeNode* InlineTypeNode::clone_if_required(PhaseGVN* gvn, SafePointNode* map, bool safe_for_replace) {
1788 if (!safe_for_replace || (map == nullptr && outcnt() != 0)) {
1789 return clone()->as_InlineType();
1790 }
1791 for (DUIterator_Fast imax, i = fast_outs(imax); i < imax; i++) {
1792 if (fast_out(i) != map) {
1793 return clone()->as_InlineType();
1794 }
1795 }
1796 gvn->hash_delete(this);
1797 return this;
1798 }
1799
1800 const Type* InlineTypeNode::Value(PhaseGVN* phase) const {
1801 Node* oop = get_oop();
1802 const Type* toop = phase->type(oop);
1803 #ifdef ASSERT
1804 if (oop->is_Con() && toop->is_zero_type() && _type->isa_oopptr()->is_known_instance()) {
1805 // We are not allocated (anymore) and should therefore not have an instance id
1806 dump(1);
1807 assert(false, "Unbuffered inline type should not have known instance id");
1808 }
1809 #endif
1810 if (toop == Type::TOP) {
1811 return Type::TOP;
1812 }
1813 const Type* t = toop->filter_speculative(_type);
1814 // Because of contradicting type profiling, we can end up with top as speculative type,
1815 // which would then get removed by cleanup_speculative. In this case we have to run filter_speculative
1816 // again, otherwise we would break the idempotence of Value
1817 if (t->speculative() == nullptr && toop->speculative() != nullptr) {
1818 t = toop->filter_speculative(t);
1819 }
1820 if (t->singleton()) {
1821 // Don't replace InlineType by a constant
1822 t = _type;
1823 }
1824 const Type* tinit = phase->type(in(NullMarker));
1825 if (tinit == Type::TOP) {
1826 return Type::TOP;
1827 }
1828 if (tinit->isa_int() && tinit->is_int()->is_con(1)) {
1829 t = t->join_speculative(TypePtr::NOTNULL);
1830 }
1831 return t;
1832 }
1833
1834 InlineTypeNode* LoadFlatNode::load(GraphKit* kit, ciInlineKlass* vk, Node* base, Node* ptr, bool null_free, bool trust_null_free_oop, DecoratorSet decorators) {
1835 int output_type_size = vk->nof_nonstatic_fields() + (null_free ? 0 : 1);
1836 const Type** output_types = TypeTuple::fields(output_type_size);
1837 collect_field_types(vk, output_types + TypeFunc::Parms, 0, output_type_size, null_free, trust_null_free_oop);
1838 const TypeTuple* type = TypeTuple::make(output_type_size + TypeFunc::Parms, output_types);
1839
1840 LoadFlatNode* load = new LoadFlatNode(vk, type, null_free, decorators);
1841 load->init_req(TypeFunc::Control, kit->control());
1842 load->init_req(TypeFunc::I_O, kit->top());
1843 load->init_req(TypeFunc::Memory, kit->reset_memory());
1844 load->init_req(TypeFunc::FramePtr, kit->frameptr());
1845 load->init_req(TypeFunc::ReturnAdr, kit->top());
1846
1847 load->init_req(TypeFunc::Parms, base);
1848 load->init_req(TypeFunc::Parms + 1, ptr);
1849 kit->kill_dead_locals();
1850 kit->add_safepoint_edges(load);
1851 load = kit->gvn().transform(load)->as_LoadFlat();
1852 kit->record_for_igvn(load);
1853
1854 kit->set_control(kit->gvn().transform(new ProjNode(load, TypeFunc::Control)));
1855 kit->set_all_memory(kit->gvn().transform(new ProjNode(load, TypeFunc::Memory)));
1856 return load->collect_projs(kit, vk, TypeFunc::Parms, null_free);
1857 }
1858
1859 bool LoadFlatNode::expand_constant(PhaseIterGVN& igvn, ciInstance* inst) const {
1860 precond(inst != nullptr);
1861 assert(igvn.delay_transform(), "transformation must be delayed");
1862 if ((_decorators & C2_MISMATCHED) != 0) {
1863 return false;
1864 }
1865
1866 GraphKit kit(this, igvn);
1867 for (int i = 0; i < _vk->nof_nonstatic_fields(); i++) {
1868 ProjNode* proj_out = proj_out_or_null(TypeFunc::Parms + i);
1869 if (proj_out == nullptr) {
1870 continue;
1871 }
1872
1873 ciField* field = _vk->nonstatic_field_at(i);
1874 BasicType bt = field->type()->basic_type();
1875 if (inst == nullptr) {
1876 Node* cst_node = igvn.zerocon(bt);
1877 igvn.replace_node(proj_out, cst_node);
1878 } else {
1879 bool is_unsigned_load = bt == T_BOOLEAN || bt == T_CHAR;
1880 const Type* cst_type = Type::make_constant_from_field(field, inst, bt, is_unsigned_load);
1881 Node* cst_node = igvn.makecon(cst_type);
1882 igvn.replace_node(proj_out, cst_node);
1883 }
1884 }
1885
1886 if (!_null_free) {
1887 ProjNode* proj_out = proj_out_or_null(TypeFunc::Parms + _vk->nof_nonstatic_fields());
1888 if (proj_out != nullptr) {
1889 igvn.replace_node(proj_out, igvn.intcon(1));
1890 }
1891 }
1892
1893 Node* old_ctrl = proj_out_or_null(TypeFunc::Control);
1894 if (old_ctrl != nullptr) {
1895 igvn.replace_node(old_ctrl, kit.control());
1896 }
1897 Node* old_mem = proj_out_or_null(TypeFunc::Memory);
1898 Node* new_mem = kit.reset_memory();
1899 if (old_mem != nullptr) {
1900 igvn.replace_node(old_mem, new_mem);
1901 }
1902 return true;
1903 }
1904
1905 bool LoadFlatNode::expand_non_atomic(PhaseIterGVN& igvn) const {
1906 assert(igvn.delay_transform(), "transformation must be delayed");
1907 if ((_decorators & C2_MISMATCHED) != 0) {
1908 return false;
1909 }
1910
1911 GraphKit kit(this, igvn);
1912 Node* base = this->base();
1913 Node* ptr = this->ptr();
1914
1915 for (int i = 0; i < _vk->nof_nonstatic_fields(); i++) {
1916 ProjNode* proj_out = proj_out_or_null(TypeFunc::Parms + i);
1917 if (proj_out == nullptr) {
1918 continue;
1919 }
1920
1921 ciField* field = _vk->nonstatic_field_at(i);
1922 Node* field_ptr = kit.basic_plus_adr(base, ptr, field->offset_in_bytes() - _vk->payload_offset());
1923 const TypePtr* field_ptr_type = field_ptr->Value(&igvn)->is_ptr();
1924 igvn.set_type(field_ptr, field_ptr_type);
1925
1926 Node* field_value = kit.access_load_at(base, field_ptr, field_ptr_type, igvn.type(proj_out), field->type()->basic_type(), _decorators);
1927 igvn.replace_node(proj_out, field_value);
1928 }
1929
1930 if (!_null_free) {
1931 ProjNode* proj_out = proj_out_or_null(TypeFunc::Parms + _vk->nof_nonstatic_fields());
1932 if (proj_out != nullptr) {
1933 Node* null_marker_ptr = kit.basic_plus_adr(base, ptr, _vk->null_marker_offset_in_payload());
1934 const TypePtr* null_marker_ptr_type = null_marker_ptr->Value(&igvn)->is_ptr();
1935 igvn.set_type(null_marker_ptr, null_marker_ptr_type);
1936 Node* null_marker_value = kit.access_load_at(base, null_marker_ptr, null_marker_ptr_type, TypeInt::BOOL, T_BOOLEAN, _decorators);
1937 igvn.replace_node(proj_out, null_marker_value);
1938 }
1939 }
1940
1941 Node* old_ctrl = proj_out_or_null(TypeFunc::Control);
1942 if (old_ctrl != nullptr) {
1943 igvn.replace_node(old_ctrl, kit.control());
1944 }
1945 Node* old_mem = proj_out_or_null(TypeFunc::Memory);
1946 Node* new_mem = kit.reset_memory();
1947 if (old_mem != nullptr) {
1948 igvn.replace_node(old_mem, new_mem);
1949 }
1950 return true;
1951 }
1952
1953 void LoadFlatNode::expand_atomic(PhaseIterGVN& igvn) const {
1954 assert(igvn.delay_transform(), "transformation must be delayed");
1955 GraphKit kit(this, igvn);
1956 Node* base = this->base();
1957 Node* ptr = this->ptr();
1958
1959 BasicType payload_bt = _vk->atomic_size_to_basic_type(_null_free);
1960 kit.insert_mem_bar(Op_MemBarCPUOrder);
1961 Node* payload = kit.access_load_at(base, ptr, TypeRawPtr::BOTTOM, Type::get_const_basic_type(payload_bt), payload_bt,
1962 _decorators | C2_MISMATCHED | C2_CONTROL_DEPENDENT_LOAD | C2_UNKNOWN_CONTROL_LOAD, kit.control());
1963 kit.insert_mem_bar(Op_MemBarCPUOrder);
1964
1965 Node* old_ctrl = proj_out_or_null(TypeFunc::Control);
1966 if (old_ctrl != nullptr) {
1967 igvn.replace_node(old_ctrl, kit.control());
1968 }
1969 Node* old_mem = proj_out_or_null(TypeFunc::Memory);
1970 Node* new_mem = kit.reset_memory();
1971 if (old_mem != nullptr) {
1972 igvn.replace_node(old_mem, new_mem);
1973 }
1974
1975 expand_projs_atomic(igvn, kit.control(), payload);
1976 }
1977
1978 void LoadFlatNode::collect_field_types(ciInlineKlass* vk, const Type** field_types, int idx, int limit, bool null_free, bool trust_null_free_oop) {
1979 assert(null_free || !trust_null_free_oop, "cannot trust null-free oop when the holder object is not null-free");
1980 for (int i = 0; i < vk->nof_declared_nonstatic_fields(); i++) {
1981 ciField* field = vk->declared_nonstatic_field_at(i);
1982 if (field->is_flat()) {
1983 ciInlineKlass* field_klass = field->type()->as_inline_klass();
1984 collect_field_types(field_klass, field_types, idx, limit, field->is_null_free(), trust_null_free_oop && field->is_null_free());
1985 idx += field_klass->nof_nonstatic_fields() + (field->is_null_free() ? 0 : 1);
1986 continue;
1987 }
1988
1989 const Type* field_type = Type::get_const_type(field->type());
1990 if (trust_null_free_oop && field->is_null_free()) {
1991 field_type = field_type->filter(TypePtr::NOTNULL);
1992 }
1993
1994 assert(idx >= 0 && idx < limit, "field type out of bounds, %d - %d", idx, limit);
1995 field_types[idx] = field_type;
1996 idx++;
1997 }
1998
1999 if (!null_free) {
2000 assert(idx >= 0 && idx < limit, "field type out of bounds, %d - %d", idx, limit);
2001 field_types[idx] = TypeInt::BOOL;
2002 }
2003 }
2004
2005 // Create an InlineTypeNode from a LoadFlatNode with its fields being extracted from the
2006 // LoadFlatNode
2007 InlineTypeNode* LoadFlatNode::collect_projs(GraphKit* kit, ciInlineKlass* vk, int proj_con, bool null_free) {
2008 PhaseGVN& gvn = kit->gvn();
2009 InlineTypeNode* res = InlineTypeNode::make_uninitialized(gvn, vk, null_free);
2010 for (int i = 0; i < vk->nof_declared_nonstatic_fields(); i++) {
2011 ciField* field = vk->declared_nonstatic_field_at(i);
2012 Node* field_value;
2013 if (field->is_flat()) {
2014 ciInlineKlass* field_klass = field->type()->as_inline_klass();
2015 field_value = collect_projs(kit, field_klass, proj_con, field->is_null_free());
2016 proj_con += field_klass->nof_nonstatic_fields() + (field->is_null_free() ? 0 : 1);
2017 } else {
2018 field_value = gvn.transform(new ProjNode(this, proj_con));
2019 proj_con++;
2020 }
2021 res->set_field_value(i, field_value);
2022 }
2023
2024 if (null_free) {
2025 res->set_null_marker(gvn);
2026 } else {
2027 res->set_null_marker(gvn, gvn.transform(new ProjNode(this, proj_con)));
2028 }
2029 return gvn.transform(res)->as_InlineType();
2030 }
2031
2032 // Extract the values of the flattened fields from the loaded payload
2033 void LoadFlatNode::expand_projs_atomic(PhaseIterGVN& igvn, Node* ctrl, Node* payload) const {
2034 BasicType payload_bt = _vk->atomic_size_to_basic_type(_null_free);
2035 for (int i = 0; i < _vk->nof_nonstatic_fields(); i++) {
2036 ProjNode* proj_out = proj_out_or_null(TypeFunc::Parms + i);
2037 if (proj_out == nullptr) {
2038 continue;
2039 }
2040
2041 ciField* field = _vk->nonstatic_field_at(i);
2042 int field_offset = field->offset_in_bytes() - _vk->payload_offset();
2043 const Type* field_type = igvn.type(proj_out);
2044 Node* field_value = get_payload_value(igvn, ctrl, payload_bt, payload, field_type, field->type()->basic_type(), field_offset);
2045 igvn.replace_node(proj_out, field_value);
2046 }
2047
2048 if (!_null_free) {
2049 ProjNode* proj_out = proj_out_or_null(TypeFunc::Parms + _vk->nof_nonstatic_fields());
2050 if (proj_out == nullptr) {
2051 return;
2052 }
2053
2054 int null_marker_offset = _vk->null_marker_offset_in_payload();
2055 Node* null_marker_value = get_payload_value(igvn, ctrl, payload_bt, payload, TypeInt::BOOL, T_BOOLEAN, null_marker_offset);
2056 igvn.replace_node(proj_out, null_marker_value);
2057 }
2058 }
2059
2060 Node* LoadFlatNode::get_payload_value(PhaseIterGVN& igvn, Node* ctrl, BasicType payload_bt, Node* payload, const Type* value_type, BasicType value_bt, int offset) {
2061 assert((offset + type2aelembytes(value_bt)) <= type2aelembytes(payload_bt), "Value does not fit into payload");
2062 Node* value = nullptr;
2063 // Shift to the right position in the long value
2064 Node* shift_val = igvn.intcon(offset << LogBitsPerByte);
2065 if (payload_bt == T_LONG) {
2066 value = igvn.transform(new URShiftLNode(payload, shift_val));
2067 value = igvn.transform(new ConvL2INode(value));
2068 } else {
2069 value = igvn.transform(new URShiftINode(payload, shift_val));
2070 }
2071
2072 if (value_bt == T_INT) {
2073 return value;
2074 } else if (!is_java_primitive(value_bt)) {
2075 assert(UseCompressedOops && payload_bt == T_LONG, "Naturally atomic");
2076 value = igvn.transform(new CastI2NNode(ctrl, value, value_type->make_narrowoop()));
2077 value = igvn.transform(new DecodeNNode(value, value_type));
2078
2079 // Similar to CheckCastPP nodes with raw input, CastI2N nodes require special handling in 'PhaseCFG::schedule_late' to ensure the
2080 // register allocator does not move the CastI2N below a safepoint. This is necessary to avoid having the raw pointer span a safepoint,
2081 // making it opaque to the GC. Unlike CheckCastPPs, which need extra handling in 'Scheduling::ComputeRegisterAntidependencies' due to
2082 // scalarization, CastI2N nodes are always used by a load if scalarization happens which inherently keeps them pinned above the safepoint.
2083 return value;
2084 } else {
2085 // Make sure to zero unused bits in the 32-bit value
2086 return Compile::narrow_value(value_bt, value, value_type, &igvn, true);
2087 }
2088 }
2089
2090 void StoreFlatNode::store(GraphKit* kit, Node* base, Node* ptr, InlineTypeNode* value, bool null_free, DecoratorSet decorators) {
2091 value = value->allocate_fields(kit);
2092 StoreFlatNode* store = new StoreFlatNode(null_free, decorators);
2093 store->init_req(TypeFunc::Control, kit->control());
2094 store->init_req(TypeFunc::I_O, kit->top());
2095 store->init_req(TypeFunc::Memory, kit->reset_memory());
2096 store->init_req(TypeFunc::FramePtr, kit->frameptr());
2097 store->init_req(TypeFunc::ReturnAdr, kit->top());
2098
2099 store->init_req(TypeFunc::Parms, base);
2100 store->init_req(TypeFunc::Parms + 1, ptr);
2101 store->init_req(TypeFunc::Parms + 2, value);
2102 kit->kill_dead_locals();
2103 kit->add_safepoint_edges(store);
2104 store = kit->gvn().transform(store)->as_StoreFlat();
2105 kit->record_for_igvn(store);
2106
2107 kit->set_control(kit->gvn().transform(new ProjNode(store, TypeFunc::Control)));
2108 kit->set_all_memory(kit->gvn().transform(new ProjNode(store, TypeFunc::Memory)));
2109 }
2110
2111 bool StoreFlatNode::expand_non_atomic(PhaseIterGVN& igvn) const {
2112 assert(igvn.delay_transform(), "transformation must be delayed");
2113 if ((_decorators & C2_MISMATCHED) != 0) {
2114 return false;
2115 }
2116
2117 GraphKit kit(this, igvn);
2118 Node* base = this->base();
2119 Node* ptr = this->ptr();
2120 InlineTypeNode* value = this->value();
2121
2122 ciInlineKlass* vk = igvn.type(value)->inline_klass();
2123 for (int i = 0; i < vk->nof_nonstatic_fields(); i++) {
2124 ciField* field = vk->nonstatic_field_at(i);
2125 Node* field_ptr = kit.basic_plus_adr(base, ptr, field->offset_in_bytes() - vk->payload_offset());
2126 const TypePtr* field_ptr_type = field_ptr->Value(&igvn)->is_ptr();
2127 igvn.set_type(field_ptr, field_ptr_type);
2128 Node* field_value = value->field_value_by_offset(field->offset_in_bytes(), true);
2129 kit.access_store_at(base, field_ptr, field_ptr_type, field_value, igvn.type(field_value), field->type()->basic_type(), _decorators);
2130 }
2131
2132 if (!_null_free) {
2133 Node* null_marker_ptr = kit.basic_plus_adr(base, ptr, vk->null_marker_offset_in_payload());
2134 const TypePtr* null_marker_ptr_type = null_marker_ptr->Value(&igvn)->is_ptr();
2135 igvn.set_type(null_marker_ptr, null_marker_ptr_type);
2136 Node* null_marker_value = value->get_null_marker();
2137 kit.access_store_at(base, null_marker_ptr, null_marker_ptr_type, null_marker_value, TypeInt::BOOL, T_BOOLEAN, _decorators);
2138 }
2139
2140 Node* old_ctrl = proj_out_or_null(TypeFunc::Control);
2141 if (old_ctrl != nullptr) {
2142 igvn.replace_node(old_ctrl, kit.control());
2143 }
2144 Node* old_mem = proj_out_or_null(TypeFunc::Memory);
2145 Node* new_mem = kit.reset_memory();
2146 if (old_mem != nullptr) {
2147 igvn.replace_node(old_mem, new_mem);
2148 }
2149 return true;
2150 }
2151
2152 void StoreFlatNode::expand_atomic(PhaseIterGVN& igvn) const {
2153 // Convert to a payload value <= 64-bit and write atomically.
2154 // The payload might contain at most two oop fields that must be narrow because otherwise they would be 64-bit
2155 // in size and would then be written by a "normal" oop store. If the payload contains oops, its size is always
2156 // 64-bit because the next smaller (power-of-two) size would be 32-bit which could only hold one narrow oop that
2157 // would then be written by a normal narrow oop store. These properties are asserted in 'convert_to_payload'.
2158 assert(igvn.delay_transform(), "transformation must be delayed");
2159 GraphKit kit(this, igvn);
2160 Node* base = this->base();
2161 Node* ptr = this->ptr();
2162 InlineTypeNode* value = this->value();
2163
2164 int oop_off_1 = -1;
2165 int oop_off_2 = -1;
2166 Node* payload = convert_to_payload(igvn, kit.control(), value, _null_free, oop_off_1, oop_off_2);
2167
2168 ciInlineKlass* vk = igvn.type(value)->inline_klass();
2169 assert(oop_off_1 == -1 || oop_off_1 == 0 || oop_off_1 == 4, "invalid layout for %s, first oop at offset %d", vk->name()->as_utf8(), oop_off_1);
2170 assert(oop_off_2 == -1 || oop_off_2 == 4, "invalid layout for %s, second oop at offset %d", vk->name()->as_utf8(), oop_off_2);
2171 BasicType payload_bt = vk->atomic_size_to_basic_type(_null_free);
2172 kit.insert_mem_bar(Op_MemBarCPUOrder);
2173 if (!UseG1GC || oop_off_1 == -1) {
2174 // No oop fields or no late barrier expansion. Emit an atomic store of the payload and add GC barriers if needed.
2175 assert(oop_off_2 == -1 || !UseG1GC, "sanity");
2176 // ZGC does not support compressed oops, so only one oop can be in the payload which is written by a "normal" oop store.
2177 assert((oop_off_1 == -1 && oop_off_2 == -1) || !UseZGC, "ZGC does not support embedded oops in flat fields");
2178 kit.access_store_at(base, ptr, TypeRawPtr::BOTTOM, payload, Type::get_const_basic_type(payload_bt), payload_bt, _decorators | C2_MISMATCHED, true, value);
2179 } else {
2180 // Contains oops and requires late barrier expansion. Emit a special store node that allows to emit GC barriers in the backend.
2181 assert(UseG1GC, "Unexpected GC");
2182 assert(payload_bt == T_LONG, "Unexpected payload type");
2183 // If one oop, set the offset (if no offset is set, two oops are assumed by the backend)
2184 Node* oop_offset = (oop_off_2 == -1) ? igvn.intcon(oop_off_1) : nullptr;
2185 Node* mem = kit.reset_memory();
2186 kit.set_all_memory(mem);
2187 Node* store = igvn.transform(new StoreLSpecialNode(kit.control(), mem, ptr, TypeRawPtr::BOTTOM, payload, oop_offset, MemNode::unordered));
2188 kit.set_memory(store, TypeRawPtr::BOTTOM);
2189 }
2190 kit.insert_mem_bar(Op_MemBarCPUOrder);
2191
2192 Node* old_ctrl = proj_out_or_null(TypeFunc::Control);
2193 if (old_ctrl != nullptr) {
2194 igvn.replace_node(old_ctrl, kit.control());
2195 }
2196 Node* old_mem = proj_out_or_null(TypeFunc::Memory);
2197 Node* new_mem = kit.reset_memory();
2198 if (old_mem != nullptr) {
2199 igvn.replace_node(old_mem, new_mem);
2200 }
2201 }
2202
2203 // Convert the field values to a payload value of type 'bt'
2204 Node* StoreFlatNode::convert_to_payload(PhaseIterGVN& igvn, Node* ctrl, InlineTypeNode* value, bool null_free, int& oop_off_1, int& oop_off_2) {
2205 ciInlineKlass* vk = igvn.type(value)->inline_klass();
2206 BasicType payload_bt = vk->atomic_size_to_basic_type(null_free);
2207 Node* payload = igvn.zerocon(payload_bt);
2208 if (!null_free) {
2209 // Set the null marker
2210 payload = set_payload_value(igvn, payload_bt, payload, T_BOOLEAN, value->get_null_marker(), vk->null_marker_offset_in_payload());
2211 }
2212
2213 // Iterate over the fields and add their values to the payload
2214 for (int i = 0; i < vk->nof_nonstatic_fields(); i++) {
2215 ciField* field = vk->nonstatic_field_at(i);
2216 Node* field_value = value->field_value_by_offset(field->offset_in_bytes(), true);
2217 ciType* field_klass = field->type();
2218 BasicType field_bt = field_klass->basic_type();
2219 int field_offset_in_payload = field->offset_in_bytes() - vk->payload_offset();
2220 if (!field_klass->is_primitive_type()) {
2221 // Narrow oop field
2222 assert(UseCompressedOops && payload_bt == T_LONG, "Naturally atomic");
2223 if (oop_off_1 == -1) {
2224 oop_off_1 = field_offset_in_payload;
2225 } else {
2226 assert(oop_off_2 == -1, "already set");
2227 oop_off_2 = field_offset_in_payload;
2228 }
2229
2230 const Type* val_type = Type::get_const_type(field_klass)->make_narrowoop();
2231 if (field_value->is_InlineType()) {
2232 assert(field_value->as_InlineType()->is_allocated(&igvn), "must be allocated");
2233 }
2234
2235 field_value = igvn.transform(new EncodePNode(field_value, val_type));
2236 field_value = igvn.transform(new CastP2XNode(ctrl, field_value));
2237 field_value = igvn.transform(new ConvL2INode(field_value));
2238 field_bt = T_INT;
2239 }
2240 payload = set_payload_value(igvn, payload_bt, payload, field_bt, field_value, field_offset_in_payload);
2241 }
2242
2243 return payload;
2244 }
2245
2246 Node* StoreFlatNode::set_payload_value(PhaseIterGVN& igvn, BasicType payload_bt, Node* payload, BasicType val_bt, Node* value, int offset) {
2247 assert((offset + type2aelembytes(val_bt)) <= type2aelembytes(payload_bt), "Value does not fit into payload");
2248
2249 // Make sure to zero unused bits in the 32-bit value
2250 if (val_bt == T_BYTE || val_bt == T_BOOLEAN) {
2251 value = igvn.transform(new AndINode(value, igvn.intcon(0xFF)));
2252 } else if (val_bt == T_CHAR || val_bt == T_SHORT) {
2253 value = igvn.transform(new AndINode(value, igvn.intcon(0xFFFF)));
2254 } else if (val_bt == T_FLOAT) {
2255 value = igvn.transform(new MoveF2INode(value));
2256 } else {
2257 assert(val_bt == T_INT, "Unsupported type: %s", type2name(val_bt));
2258 }
2259
2260 Node* shift_val = igvn.intcon(offset << LogBitsPerByte);
2261 if (payload_bt == T_LONG) {
2262 // Convert to long and remove the sign bit (the backend will fold this and emit a zero extend i2l)
2263 value = igvn.transform(new ConvI2LNode(value));
2264 value = igvn.transform(new AndLNode(value, igvn.longcon(0xFFFFFFFF)));
2265
2266 Node* shift_value = igvn.transform(new LShiftLNode(value, shift_val));
2267 payload = new OrLNode(shift_value, payload);
2268 } else {
2269 Node* shift_value = igvn.transform(new LShiftINode(value, shift_val));
2270 payload = new OrINode(shift_value, payload);
2271 }
2272 return igvn.transform(payload);
2273 }
2274
2275 const Type* LoadFlatNode::Value(PhaseGVN* phase) const {
2276 if (phase->type(in(TypeFunc::Control)) == Type::TOP || phase->type(in(TypeFunc::Memory)) == Type::TOP ||
2277 phase->type(base()) == Type::TOP || phase->type(ptr()) == Type::TOP) {
2278 return Type::TOP;
2279 }
2280 return bottom_type();
2281 }
2282
2283 const Type* StoreFlatNode::Value(PhaseGVN* phase) const {
2284 if (phase->type(in(TypeFunc::Control)) == Type::TOP || phase->type(in(TypeFunc::Memory)) == Type::TOP ||
2285 phase->type(base()) == Type::TOP || phase->type(ptr()) == Type::TOP || phase->type(value()) == Type::TOP) {
2286 return Type::TOP;
2287 }
2288 return bottom_type();
2289 }
2290
2291 bool LoadFlatNode::is_mismatched() const {
2292 return (_decorators & C2_MISMATCHED) != 0;
2293 }
2294
2295 bool StoreFlatNode::is_mismatched() const {
2296 return (_decorators & C2_MISMATCHED) != 0;
2297 }