1 /*
2 * Copyright (c) 2017, 2026, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "ci/ciInlineKlass.hpp"
26 #include "gc/shared/barrierSet.hpp"
27 #include "gc/shared/c2/barrierSetC2.hpp"
28 #include "gc/shared/gc_globals.hpp"
29 #include "memory/resourceArea.hpp"
30 #include "oops/accessDecorators.hpp"
31 #include "opto/addnode.hpp"
32 #include "opto/castnode.hpp"
33 #include "opto/cfgnode.hpp"
34 #include "opto/compile.hpp"
35 #include "opto/convertnode.hpp"
36 #include "opto/graphKit.hpp"
37 #include "opto/inlinetypenode.hpp"
38 #include "opto/memnode.hpp"
39 #include "opto/movenode.hpp"
40 #include "opto/multnode.hpp"
41 #include "opto/narrowptrnode.hpp"
42 #include "opto/node.hpp"
43 #include "opto/opaquenode.hpp"
44 #include "opto/opcodes.hpp"
45 #include "opto/phaseX.hpp"
46 #include "opto/rootnode.hpp"
47 #include "opto/subnode.hpp"
48 #include "opto/type.hpp"
49 #include "utilities/globalDefinitions.hpp"
50 #include "utilities/growableArray.hpp"
51 #include "utilities/pair.hpp"
52 #include "utilities/tuple.hpp"
53
54 // Clones the inline type to handle control flow merges involving multiple inline types.
55 // The inputs are replaced by PhiNodes to represent the merged values for the given region.
56 // init_with_top: input of phis above the returned InlineTypeNode are initialized to top.
57 InlineTypeNode* InlineTypeNode::clone_with_phis(PhaseGVN* gvn, Node* region, SafePointNode* map, bool is_non_null, bool init_with_top) {
58 InlineTypeNode* vt = clone_if_required(gvn, map);
59 const Type* t = Type::get_const_type(inline_klass());
60 gvn->set_type(vt, t);
61 vt->as_InlineType()->set_type(t);
62
63 Node* const top = gvn->C->top();
64
65 // Create a PhiNode for merging the oop values
66 PhiNode* oop = PhiNode::make(region, init_with_top ? top : vt->get_oop(), t);
67 gvn->set_type(oop, t);
68 gvn->record_for_igvn(oop);
69 vt->set_oop(*gvn, oop);
70
71 // Create a PhiNode for merging the is_buffered values
72 t = Type::get_const_basic_type(T_BOOLEAN);
73 Node* is_buffered_node = PhiNode::make(region, init_with_top ? top : vt->get_is_buffered(), t);
74 gvn->set_type(is_buffered_node, t);
75 gvn->record_for_igvn(is_buffered_node);
76 vt->set_req(IsBuffered, is_buffered_node);
77
78 // Create a PhiNode for merging the null_marker values
79 Node* null_marker_node;
80 if (is_non_null) {
81 null_marker_node = gvn->intcon(1);
82 } else {
83 t = Type::get_const_basic_type(T_BOOLEAN);
84 null_marker_node = PhiNode::make(region, init_with_top ? top : vt->get_null_marker(), t);
85 gvn->set_type(null_marker_node, t);
86 gvn->record_for_igvn(null_marker_node);
87 }
88 vt->set_req(NullMarker, null_marker_node);
89
90 // Create a PhiNode each for merging the field values
91 for (uint i = 0; i < vt->field_count(); ++i) {
92 ciField* field = vt->field(i);
93 ciType* type = field->type();
94 Node* value = vt->field_value(i);
95 if (field->is_flat()) {
96 // Handle flat fields recursively
97 value = value->as_InlineType()->clone_with_phis(gvn, region, map);
98 } else {
99 t = Type::get_const_type(type);
100 value = PhiNode::make(region, init_with_top ? top : value, t);
101 gvn->set_type(value, t);
102 gvn->record_for_igvn(value);
103 }
104 vt->set_field_value(i, value);
105 }
106 gvn->record_for_igvn(vt);
107 return vt;
108 }
109
110 // Checks if the inputs of the InlineTypeNode were replaced by PhiNodes
111 // for the given region (see InlineTypeNode::clone_with_phis).
112 bool InlineTypeNode::has_phi_inputs(Node* region) const {
113 // Check oop input
114 bool result = get_oop()->is_Phi() && get_oop()->as_Phi()->region() == region;
115 #ifdef ASSERT
116 if (result) {
117 // Check all field value inputs for consistency
118 for (uint i = 0; i < field_count(); ++i) {
119 Node* n = field_value(i);
120 if (n->is_InlineType()) {
121 assert(n->as_InlineType()->has_phi_inputs(region), "inconsistent phi inputs");
122 } else {
123 assert(n->is_Phi() && n->as_Phi()->region() == region, "inconsistent phi inputs");
124 }
125 }
126 }
127 #endif
128 return result;
129 }
130
131 // Merges 'this' with 'other' by updating the input PhiNodes added by 'clone_with_phis'
132 InlineTypeNode* InlineTypeNode::merge_with(PhaseGVN* gvn, const InlineTypeNode* other, int phi_index, bool transform) {
133 assert(inline_klass() == other->inline_klass(), "Merging incompatible types");
134
135 // Merge oop inputs
136 PhiNode* phi = get_oop()->as_Phi();
137 phi->set_req(phi_index, other->get_oop());
138 if (transform) {
139 set_oop(*gvn, gvn->transform(phi));
140 }
141
142 // Merge is_buffered inputs
143 phi = get_is_buffered()->as_Phi();
144 phi->set_req(phi_index, other->get_is_buffered());
145 if (transform) {
146 set_req(IsBuffered, gvn->transform(phi));
147 }
148
149 // Merge null_marker inputs
150 Node* null_marker = get_null_marker();
151 if (null_marker->is_Phi()) {
152 phi = null_marker->as_Phi();
153 phi->set_req(phi_index, other->get_null_marker());
154 if (transform) {
155 set_req(NullMarker, gvn->transform(phi));
156 }
157 } else {
158 assert(null_marker->find_int_con(0) == 1, "only with a non null inline type");
159 }
160
161 // Merge field values
162 for (uint i = 0; i < field_count(); ++i) {
163 Node* val1 = field_value(i);
164 Node* val2 = other->field_value(i);
165 if (field(i)->is_flat()) {
166 if (val2->is_top()) {
167 // The path where 'other' is used is dying. Therefore, we do not need to process the merge with 'other' further.
168 // The phi inputs of 'this' at 'phi_index' will eventually be removed.
169 break;
170 } else if (val2->is_Phi()) {
171 val2 = gvn->transform(val2);
172 }
173
174 assert(val1->is_InlineType() && val2->is_InlineType(), "must be InlineTypeNode: %s - %s", val1->Name(), val2->Name());
175 val1->as_InlineType()->merge_with(gvn, val2->as_InlineType(), phi_index, transform);
176 } else {
177 assert(val1->is_Phi(), "must be a phi node %s", val1->Name());
178 val1->set_req(phi_index, val2);
179 }
180 if (transform) {
181 set_field_value(i, gvn->transform(val1));
182 }
183 }
184 return this;
185 }
186
187 // Adds a new merge path to an inline type node with phi inputs
188 void InlineTypeNode::add_new_path(Node* region) const {
189 assert(has_phi_inputs(region), "must have phi inputs");
190
191 PhiNode* phi = get_oop()->as_Phi();
192 phi->add_req(nullptr);
193 assert(phi->req() == region->req(), "must be same size as region");
194
195 phi = get_is_buffered()->as_Phi();
196 phi->add_req(nullptr);
197 assert(phi->req() == region->req(), "must be same size as region");
198
199 phi = get_null_marker()->as_Phi();
200 phi->add_req(nullptr);
201 assert(phi->req() == region->req(), "must be same size as region");
202
203 for (uint i = 0; i < field_count(); ++i) {
204 Node* val = field_value(i);
205 if (val->is_InlineType()) {
206 val->as_InlineType()->add_new_path(region);
207 } else {
208 val->as_Phi()->add_req(nullptr);
209 assert(val->req() == region->req(), "must be same size as region");
210 }
211 }
212 }
213
214 Node* InlineTypeNode::field_value(uint index) const {
215 assert(index < field_count(), "index out of bounds");
216 return in(Values + index);
217 }
218
219 // Get the value of the field at the given offset.
220 // If 'recursive' is true, flat inline type fields will be resolved recursively.
221 Node* InlineTypeNode::field_value_by_offset(int offset, bool recursive) const {
222 // Find the declared field which contains the field we are looking for
223 int index = inline_klass()->field_index_by_offset(offset);
224 Node* value = field_value(index);
225 assert(value != nullptr, "field value not found");
226 ciField* field = this->field(index);
227 assert(!field->is_flat() || field->type()->is_inlinetype(), "must be an inline type");
228
229 if (value->is_top()) {
230 // The graph is dying but a load may still ask for a nested field
231 // inside a flattened field before the dead load itself is folded away.
232 assert(offset == field->offset_in_bytes() || field->is_flat(), "offset mismatch");
233 return value;
234 }
235 if (!recursive || !field->is_flat()) {
236 assert(offset == field->offset_in_bytes(), "offset mismatch");
237 return value;
238 }
239
240 // Flat inline type field
241 InlineTypeNode* vt = value->as_InlineType();
242 assert(field->is_flat(), "must be flat");
243 if (offset == field->null_marker_offset()) {
244 return vt->get_null_marker();
245 } else {
246 int sub_offset = offset - field->offset_in_bytes(); // Offset of the flattened field inside the declared field
247 sub_offset += vt->inline_klass()->payload_offset(); // Add header size
248 return vt->field_value_by_offset(sub_offset, recursive);
249 }
250 }
251
252 void InlineTypeNode::set_field_value(uint index, Node* value) {
253 assert(index < field_count(), "index out of bounds");
254 set_req(Values + index, value);
255 }
256
257 void InlineTypeNode::set_field_value_by_offset(int offset, Node* value) {
258 set_field_value(field_index(offset), value);
259 }
260
261 uint InlineTypeNode::field_index(int offset) const {
262 uint i = 0;
263 for (; i < field_count() && field(i)->offset_in_bytes() != offset; i++) { }
264 assert(i < field_count(), "field not found");
265 return i;
266 }
267
268 ciField* InlineTypeNode::field(uint index) const {
269 assert(index < field_count(), "index out of bounds");
270 return inline_klass()->declared_nonstatic_field_at(index);
271 }
272
273 uint InlineTypeNode::add_fields_to_safepoint(Unique_Node_List& worklist, SafePointNode* sfpt) const {
274 uint cnt = 0;
275 for (uint i = 0; i < field_count(); ++i) {
276 Node* value = field_value(i);
277 ciField* field = this->field(i);
278 assert(!field->is_flat() || field->type()->is_inlinetype(), "must be an inline type");
279 if (field->is_flat()) {
280 InlineTypeNode* vt = value->as_InlineType();
281 cnt += vt->add_fields_to_safepoint(worklist, sfpt);
282 if (!field->is_null_free()) {
283 // The null marker of a flat field is added right after we scalarize that field
284 sfpt->add_req(vt->get_null_marker());
285 cnt++;
286 }
287 continue;
288 }
289 if (value->is_InlineType()) {
290 // Add inline type to the worklist to process later
291 worklist.push(value);
292 }
293 sfpt->add_req(value);
294 cnt++;
295 }
296 return cnt;
297 }
298
299 void InlineTypeNode::make_scalar_in_safepoint(PhaseIterGVN* igvn, Unique_Node_List& worklist, SafePointNode* sfpt) const {
300 JVMState* jvms = sfpt->jvms();
301 assert(jvms != nullptr, "missing JVMS");
302 uint first_ind = (sfpt->req() - jvms->scloff());
303
304 // Iterate over the inline type fields in order of increasing offset and add the
305 // field values to the safepoint. Nullable inline types have a null marker field that
306 // needs to be checked before using the field values.
307 sfpt->add_req(get_null_marker());
308 uint nfields = add_fields_to_safepoint(worklist, sfpt);
309 jvms->set_endoff(sfpt->req());
310 // Replace safepoint edge by SafePointScalarObjectNode
311 SafePointScalarObjectNode* sobj = new SafePointScalarObjectNode(type()->isa_instptr(),
312 nullptr,
313 first_ind,
314 sfpt->jvms()->depth(),
315 nfields);
316 sobj->init_req(0, igvn->C->root());
317 sobj = igvn->transform(sobj)->as_SafePointScalarObject();
318 igvn->rehash_node_delayed(sfpt);
319 for (uint i = jvms->debug_start(); i < jvms->debug_end(); i++) {
320 Node* debug = sfpt->in(i);
321 if (debug != nullptr && debug->uncast() == this) {
322 sfpt->set_req(i, sobj);
323 }
324 }
325 }
326
327 void InlineTypeNode::make_scalar_in_safepoints(PhaseIterGVN* igvn, bool allow_oop) {
328 make_scalar_in_safepoints(igvn, allow_oop, nullptr);
329 }
330
331 void InlineTypeNode::make_scalar_in_safepoints(PhaseIterGVN* igvn, bool allow_oop, SafePointNode* safepoint) {
332 // If the inline type has a constant or loaded oop, use the oop instead of scalarization
333 // in the safepoint to avoid keeping field loads live just for the debug info.
334 Node* oop = get_oop();
335 bool use_oop = false;
336 if (allow_oop && is_allocated(igvn) && oop->is_Phi()) {
337 Unique_Node_List worklist;
338 VectorSet visited;
339 visited.set(oop->_idx);
340 worklist.push(oop);
341 use_oop = true;
342 while (worklist.size() > 0 && use_oop) {
343 Node* n = worklist.pop();
344 for (uint i = 1; i < n->req(); i++) {
345 Node* in = n->in(i);
346 if (in->is_Phi() && !visited.test_set(in->_idx)) {
347 worklist.push(in);
348 } else if (!(in->is_Con() || in->is_Parm())) {
349 use_oop = false;
350 break;
351 }
352 }
353 }
354 } else {
355 use_oop = allow_oop && is_allocated(igvn) &&
356 (oop->is_Con() || oop->is_Parm() || oop->is_Load() || (oop->isa_DecodeN() && oop->in(1)->is_Load()));
357 }
358
359 ResourceMark rm;
360 Unique_Node_List safepoints;
361 if (safepoint == nullptr) {
362 // Gathering all SafePoint users of `this`...
363 Unique_Node_List worklist;
364 worklist.push(this);
365 while (worklist.size() > 0) {
366 Node* n = worklist.pop();
367 for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) {
368 Node* use = n->fast_out(i);
369 if (use->is_SafePoint() && !use->is_CallLeaf() && (!use->is_Call() || use->as_Call()->has_debug_use(n))) {
370 safepoints.push(use);
371 } else if (use->is_ConstraintCast()) {
372 worklist.push(use);
373 }
374 }
375 }
376 } else {
377 // ...or just the provided one if given.
378 safepoints.push(safepoint);
379 }
380
381 Unique_Node_List vt_worklist;
382 // Process all safepoint uses and scalarize inline type
383 while (safepoints.size() > 0) {
384 SafePointNode* sfpt = safepoints.pop()->as_SafePoint();
385 if (use_oop) {
386 for (uint i = sfpt->jvms()->debug_start(); i < sfpt->jvms()->debug_end(); i++) {
387 Node* debug = sfpt->in(i);
388 if (debug != nullptr && debug->uncast() == this) {
389 sfpt->set_req(i, get_oop());
390 }
391 }
392 igvn->rehash_node_delayed(sfpt);
393 } else {
394 make_scalar_in_safepoint(igvn, vt_worklist, sfpt);
395 }
396 }
397 // Now scalarize non-flat fields
398 for (uint i = 0; i < vt_worklist.size(); ++i) {
399 InlineTypeNode* vt = vt_worklist.at(i)->isa_InlineType();
400 vt->make_scalar_in_safepoints(igvn);
401 }
402 if (outcnt() == 0) {
403 igvn->record_for_igvn(this);
404 }
405 }
406
407 void InlineTypeNode::load(GraphKit* kit, Node* base, Node* ptr, bool immutable_memory, bool trust_null_free_oop, DecoratorSet decorators) {
408 // Initialize the inline type by loading its field values from
409 // memory and adding the values as input edges to the node.
410 ciInlineKlass* vk = inline_klass();
411 for (uint i = 0; i < field_count(); ++i) {
412 ciField* field = this->field(i);
413 assert(!field->is_flat() || field->type()->is_inlinetype(), "must be an inline type");
414 int field_off = field->offset_in_bytes() - vk->payload_offset();
415 Node* field_ptr = kit->basic_plus_adr(base, ptr, field_off);
416 Node* value = nullptr;
417 ciType* ft = field->type();
418 bool field_null_free = field->is_null_free();
419 if (field->is_flat()) {
420 // Recursively load the flat inline type field
421 ciInlineKlass* fvk = ft->as_inline_klass();
422 bool atomic = field->is_atomic();
423 value = make_from_flat_impl(kit, fvk, base, field_ptr, atomic, immutable_memory,
424 field_null_free, trust_null_free_oop && field_null_free, decorators);
425 } else {
426 // Load field value from memory
427 BasicType bt = type2field[ft->basic_type()];
428 assert(is_java_primitive(bt) || field_ptr->bottom_type()->is_ptr_to_narrowoop() == UseCompressedOops, "inconsistent");
429 const Type* val_type = Type::get_const_type(ft);
430 if (trust_null_free_oop && field_null_free) {
431 val_type = val_type->join_speculative(TypePtr::NOTNULL);
432 }
433 const TypePtr* field_ptr_type = (decorators & C2_MISMATCHED) == 0 ? kit->gvn().type(field_ptr)->is_ptr() : TypeRawPtr::BOTTOM;
434 value = kit->access_load_at(base, field_ptr, field_ptr_type, val_type, bt, decorators);
435 }
436 set_field_value(i, value);
437 }
438 }
439
440 void InlineTypeNode::store_flat(GraphKit* kit, Node* base, Node* ptr, bool atomic, bool immutable_memory, bool null_free, DecoratorSet decorators) {
441 ciInlineKlass* vk = inline_klass();
442 bool do_atomic = atomic;
443 // With immutable memory, a non-atomic load and an atomic load are the same
444 if (immutable_memory) {
445 do_atomic = false;
446 }
447 // If there is only one flattened field, a non-atomic load and an atomic load are the same
448 if (vk->is_naturally_atomic(null_free)) {
449 do_atomic = false;
450 }
451
452 if (!do_atomic) {
453 if (!null_free) {
454 int nm_offset = vk->null_marker_offset_in_payload();
455 Node* nm_ptr = kit->basic_plus_adr(base, ptr, nm_offset);
456 const TypePtr* nm_ptr_type = (decorators & C2_MISMATCHED) == 0 ? kit->gvn().type(nm_ptr)->is_ptr() : TypeRawPtr::BOTTOM;
457 kit->access_store_at(base, nm_ptr, nm_ptr_type, get_null_marker(), TypeInt::BOOL, T_BOOLEAN, decorators);
458 }
459 store(kit, base, ptr, immutable_memory, decorators);
460 return;
461 }
462
463 StoreFlatNode::store(kit, base, ptr, this, null_free, decorators);
464 }
465
466 void InlineTypeNode::store_flat_array(GraphKit* kit, Node* base, Node* idx) {
467 PhaseGVN& gvn = kit->gvn();
468 DecoratorSet decorators = IN_HEAP | IS_ARRAY | MO_UNORDERED;
469 kit->C->set_flat_accesses();
470 ciInlineKlass* vk = inline_klass();
471 assert(vk->maybe_flat_in_array(), "element type %s cannot be flat in array", vk->name()->as_utf8());
472
473 RegionNode* region = new RegionNode(4);
474 gvn.set_type(region, Type::CONTROL);
475 kit->record_for_igvn(region);
476
477 Node* input_memory_state = kit->reset_memory();
478 kit->set_all_memory(input_memory_state);
479
480 PhiNode* mem = PhiNode::make(region, input_memory_state, Type::MEMORY, TypePtr::BOTTOM);
481 gvn.set_type(mem, Type::MEMORY);
482 kit->record_for_igvn(mem);
483
484 PhiNode* io = PhiNode::make(region, kit->i_o(), Type::ABIO);
485 gvn.set_type(io, Type::ABIO);
486 kit->record_for_igvn(io);
487
488 Node* bol_null_free = kit->null_free_array_test(base); // Argument evaluation order is undefined in C++ and since this sets control, it needs to come first
489 IfNode* iff_null_free = kit->create_and_map_if(kit->control(), bol_null_free, PROB_FAIR, COUNT_UNKNOWN);
490
491 // Nullable
492 kit->set_control(kit->IfFalse(iff_null_free));
493 if (!kit->stopped()) {
494 assert(vk->has_nullable_atomic_layout(), "element type %s does not have a nullable flat layout", vk->name()->as_utf8());
495 kit->set_all_memory(input_memory_state);
496 Node* cast = kit->cast_to_flat_array_exact(base, vk, false, true);
497 Node* ptr = kit->array_element_address(cast, idx, T_FLAT_ELEMENT);
498 store_flat(kit, cast, ptr, true, false, false, decorators);
499
500 region->init_req(1, kit->control());
501 mem->set_req(1, kit->reset_memory());
502 io->set_req(1, kit->i_o());
503 }
504
505 // Null-free
506 kit->set_control(kit->IfTrue(iff_null_free));
507 if (!kit->stopped()) {
508 kit->set_all_memory(input_memory_state);
509
510 Node* bol_atomic = kit->null_free_atomic_array_test(base, vk);
511 IfNode* iff_atomic = kit->create_and_map_if(kit->control(), bol_atomic, PROB_FAIR, COUNT_UNKNOWN);
512
513 // Atomic
514 kit->set_control(kit->IfTrue(iff_atomic));
515 if (!kit->stopped()) {
516 assert(vk->has_null_free_atomic_layout(), "element type %s does not have a null-free atomic flat layout", vk->name()->as_utf8());
517 kit->set_all_memory(input_memory_state);
518 Node* cast = kit->cast_to_flat_array_exact(base, vk, true, true);
519 Node* ptr = kit->array_element_address(cast, idx, T_FLAT_ELEMENT);
520 store_flat(kit, cast, ptr, true, false, true, decorators);
521
522 region->init_req(2, kit->control());
523 mem->set_req(2, kit->reset_memory());
524 io->set_req(2, kit->i_o());
525 }
526
527 // Non-atomic
528 kit->set_control(kit->IfFalse(iff_atomic));
529 if (!kit->stopped()) {
530 assert(vk->has_null_free_non_atomic_layout(), "element type %s does not have a null-free non-atomic flat layout", vk->name()->as_utf8());
531 kit->set_all_memory(input_memory_state);
532 Node* cast = kit->cast_to_flat_array_exact(base, vk, true, false);
533 Node* ptr = kit->array_element_address(cast, idx, T_FLAT_ELEMENT);
534 store_flat(kit, cast, ptr, false, false, true, decorators);
535
536 region->init_req(3, kit->control());
537 mem->set_req(3, kit->reset_memory());
538 io->set_req(3, kit->i_o());
539 }
540 }
541
542 kit->set_control(gvn.transform(region));
543 kit->set_all_memory(gvn.transform(mem));
544 kit->set_i_o(gvn.transform(io));
545 }
546
547 void InlineTypeNode::store(GraphKit* kit, Node* base, Node* ptr, bool immutable_memory, DecoratorSet decorators) const {
548 // Write field values to memory
549 ciInlineKlass* vk = inline_klass();
550 for (uint i = 0; i < field_count(); ++i) {
551 ciField* field = this->field(i);
552 assert(!field->is_flat() || field->type()->is_inlinetype(), "must be an inline type");
553 int field_off = field->offset_in_bytes() - vk->payload_offset();
554 Node* field_val = field_value(i);
555 bool field_null_free = field->is_null_free();
556 ciType* ft = field->type();
557 Node* field_ptr = kit->basic_plus_adr(base, ptr, field_off);
558 if (field->is_flat()) {
559 // Recursively store the flat inline type field
560 bool atomic = field->is_atomic();
561 field_val->as_InlineType()->store_flat(kit, base, field_ptr, atomic, immutable_memory, field_null_free, decorators);
562 } else {
563 // Store field value to memory
564 BasicType bt = type2field[ft->basic_type()];
565 const TypePtr* field_ptr_type = (decorators & C2_MISMATCHED) == 0 ? kit->gvn().type(field_ptr)->is_ptr() : TypeRawPtr::BOTTOM;
566 const Type* val_type = Type::get_const_type(ft);
567 kit->access_store_at(base, field_ptr, field_ptr_type, field_val, val_type, bt, decorators);
568 }
569 }
570 }
571
572 // If a value class contains cycle, bail out from trying to expand substitutability check involving it
573 static bool check_cycle(ciInlineKlass* vk) {
574 ResourceMark rm;
575 GrowableArray<Pair<ciInlineKlass*, int>> visited;
576 visited.push(Pair(vk, 0));
577 while (visited.is_nonempty()) {
578 ciInlineKlass* current = visited.top().first;
579 bool finish = true;
580 for (int field_idx = visited.top().second; field_idx < current->nof_nonstatic_fields(); field_idx++) {
581 ciField* field = current->nonstatic_field_at(field_idx);
582 ciType* ft = field->type();
583 if (!ft->is_inlinetype()) {
584 continue;
585 } else if (visited.find_if([&](const auto& entry) { return entry.first == ft; }) >= 0) {
586 return true;
587 } else {
588 visited.top().second = field_idx + 1;
589 visited.push(Pair(ft->as_inline_klass(), 0));
590 finish = false;
591 break;
592 }
593 }
594 if (finish) {
595 visited.pop();
596 }
597 }
598 return false;
599 }
600
601 // Check if a substitutability check between 'lhs' and 'rhs' can be implemented in IR
602 bool InlineTypeNode::can_emit_substitutability_check(Node* lhs, Node* rhs) {
603 if (!lhs->bottom_type()->isa_ptr() ||
604 (rhs != nullptr && !rhs->bottom_type()->isa_ptr())) {
605 return false;
606 }
607
608 if (rhs != nullptr && lhs->eqv_uncast(rhs)) {
609 return true;
610 }
611
612 if (!lhs->bottom_type()->is_ptr()->can_be_inline_type() ||
613 (rhs != nullptr && !rhs->bottom_type()->is_ptr()->can_be_inline_type())) {
614 return true;
615 }
616
617 if (!lhs->is_InlineType() && (rhs == nullptr || !rhs->is_InlineType())) {
618 return false;
619 }
620
621 if (!lhs->is_InlineType()) {
622 swap(lhs, rhs);
623 }
624
625 InlineTypeNode* lhs_inline = lhs->as_InlineType();
626 InlineTypeNode* rhs_inline = rhs != nullptr ? rhs->isa_InlineType() : nullptr;
627 if (rhs_inline != nullptr && lhs_inline->type()->inline_klass() != rhs_inline->type()->inline_klass()) {
628 // Dead code, can skip the substitutability check
629 return true;
630 }
631
632 if (check_cycle(lhs_inline->inline_klass())) {
633 return false;
634 }
635
636 for (uint i = 0; i < lhs_inline->field_count(); i++) {
637 ciType* ft = lhs_inline->field(i)->type();
638 if (!ft->can_be_inline_klass()) {
639 continue;
640 }
641
642 Node* lhs_fv = lhs_inline->field_value(i);
643 Node* rhs_fv = rhs_inline != nullptr ? rhs_inline->field_value(i) : nullptr;
644 if (!can_emit_substitutability_check(lhs_fv, rhs_fv)) {
645 return false;
646 }
647 }
648 return true;
649 }
650
651 // Compare lhs and rhs given they are not value objects
652 static void emit_substitutability_check_primitive(GraphKit* kit, PhiNode* result, Node* lhs, Node* rhs, BasicType bt) {
653 PhaseGVN& gvn = kit->gvn();
654 Node* cmp;
655 if (bt == T_INT || is_subword_type(bt)) {
656 cmp = kit->CmpI(lhs, rhs);
657 } else if (bt == T_LONG) {
658 cmp = kit->CmpL(lhs, rhs);
659 } else if (bt == T_FLOAT) {
660 lhs = gvn.transform(new MoveF2INode(lhs));
661 rhs = gvn.transform(new MoveF2INode(rhs));
662 cmp = kit->CmpI(lhs, rhs);
663 } else if (bt == T_DOUBLE) {
664 lhs = gvn.transform(new MoveD2LNode(lhs));
665 rhs = gvn.transform(new MoveD2LNode(rhs));
666 cmp = kit->CmpL(lhs, rhs);
667 } else {
668 assert(is_reference_type(bt), "unexpected bt %s", type2name(bt));
669 cmp = kit->CmpP(lhs, rhs);
670 }
671
672 Node* bol = kit->Bool(cmp, BoolTest::eq);
673 IfNode* iff = kit->create_and_map_if(kit->control(), bol, PROB_FAIR, COUNT_UNKNOWN);
674
675 Node* iff_false = kit->IfFalse(iff);
676 result->add_req(kit->intcon(0));
677 result->region()->add_req(iff_false);
678
679 Node* iff_true = kit->IfTrue(iff);
680 kit->set_control(iff_true);
681 }
682
683 // Try to see what should be done with lhs and rhs. Either we can emit the answer if it is simple,
684 // give up and emit a call to runtime, or start comparing the value objects field-by-field. In the
685 // last case, NodeSentinel is returned.
686 static Node* emit_substitutability_check_pointer(GraphKit* kit, PhiNode* result, Node* lhs, Node* rhs) {
687 PhaseGVN& gvn = kit->gvn();
688 Node* top = kit->C->top();
689 const Type* lhs_type = gvn.type(lhs);
690 const Type* rhs_type = gvn.type(rhs);
691
692 // Dead graph
693 if (lhs_type == Type::TOP || rhs_type == Type::TOP) {
694 kit->set_control(top);
695 return top;
696 }
697
698 Node* cmp = nullptr;
699 if (lhs->eqv_uncast(rhs)) {
700 cmp = kit->intcon(0);
701 } else if (!lhs_type->is_ptr()->can_be_inline_type() || !rhs_type->is_ptr()->can_be_inline_type()) {
702 // If one of the sides is not a value object, can only be substitutable if they are the same
703 cmp = kit->CmpP(lhs, rhs);
704 } else if (lhs_type->is_instptr()->as_klass_type()->join(rhs_type->is_instptr()->as_klass_type())->empty()) {
705 // Both belongs to provably different types, must be different unless both are null. Cannot
706 // rely on the pointer independence alone because different pointers may still be
707 // substitutable.
708 cmp = kit->CmpP(lhs, rhs);
709 }
710 if (cmp != nullptr) {
711 Node* res = kit->Bool(cmp, BoolTest::eq);
712 IfNode* iff = kit->create_and_map_if(kit->control(), res, PROB_FAIR, COUNT_UNKNOWN);
713
714 Node* iff_false = kit->IfFalse(iff);
715 result->region()->add_req(iff_false);
716 result->add_req(gvn.intcon(0));
717
718 Node* iff_true = kit->IfTrue(iff);
719 kit->set_control(iff_true);
720 return iff_true;
721 }
722
723 if (!lhs_type->is_inlinetypeptr() && !rhs_type->is_inlinetypeptr()) {
724 return nullptr;
725 }
726
727 // A cycle, give up
728 ciInlineKlass* vk = lhs_type->is_inlinetypeptr() ? lhs_type->inline_klass() : rhs_type->inline_klass();
729 if (check_cycle(vk)) {
730 return nullptr;
731 }
732
733 return NodeSentinel;
734 }
735
736 Node* InlineTypeNode::emit_substitutability_check(GraphKit* kit, Node* lhs, Node* rhs) {
737 if (!kit->C->allow_macro_nodes()) {
738 // After macro expansion, InlineTypeNodes are also eliminated, creation of new ones then is not
739 // allowed
740 return nullptr;
741 }
742
743 PhaseIterGVN& igvn = *kit->gvn().is_IterGVN();
744 RegionNode* region = new RegionNode(1);
745 PhiNode* result = new PhiNode(region, TypeInt::BOOL);
746 igvn.register_new_node_with_optimizer(region);
747 igvn.register_new_node_with_optimizer(result);
748
749 Node* preprocess = emit_substitutability_check_pointer(kit, result, lhs, rhs);
750 if (preprocess == nullptr) {
751 return nullptr;
752 } else if (preprocess != NodeSentinel) {
753 region->add_req(kit->control());
754 result->add_req(igvn.intcon(1));
755 kit->set_control(region);
756 return result;
757 }
758
759 const Type* lhs_type = igvn.type(lhs);
760 const Type* rhs_type = igvn.type(rhs);
761 assert(!lhs_type->maybe_null() && !rhs_type->maybe_null(), "must check null beforehand");
762 ciInlineKlass* vk = lhs_type->is_inlinetypeptr() ? lhs_type->inline_klass() : rhs_type->inline_klass();
763 const TypeInstPtr* vk_type = TypeOopPtr::make_from_klass(vk)->join(TypePtr::NOTNULL)->is_instptr();
764
765 if (!lhs->is_InlineType()) {
766 if (!lhs_type->higher_equal(vk_type)) {
767 lhs = igvn.transform(new CheckCastPPNode(kit->control(), lhs, vk_type, ConstraintCastNode::DependencyType::NonFloatingNarrowing));
768 }
769 lhs = InlineTypeNode::make_from_oop(kit, lhs, vk);
770 }
771 if (!rhs->is_InlineType()) {
772 if (!rhs_type->higher_equal(vk_type)) {
773 rhs = igvn.transform(new CheckCastPPNode(kit->control(), rhs, vk_type, ConstraintCastNode::DependencyType::NonFloatingNarrowing));
774 }
775 rhs = InlineTypeNode::make_from_oop(kit, rhs, vk);
776 }
777
778 RegionNode* true_region = new RegionNode(3);
779 igvn.register_new_node_with_optimizer(true_region);
780 region->add_req(true_region);
781 result->add_req(igvn.intcon(1));
782
783 // To verify the substitutability of 2 notnull value objects of the same type, we need to check
784 // the substitutability of each field in the objects:
785 //
786 // substitutable(cur_lhs, cur_rhs) {
787 // cur_start_region:
788 //
789 // if (cur_lhs.field1 == null && cur_rhs.field1 == null) {
790 // goto cur_true_field1_region;
791 // } else if ((cur_lhs.field1 == null) != (cur_rhs.field1 == null) {
792 // return false;
793 // } else if (cur_lhs.field1.klass != cur_rhs.field1.klass) {
794 // return false;
795 // }
796 //
797 // cur_start_field1_region:
798 // if (substitutable(cur_lhs.field1, cur_rhs.field1)) {
799 // return false;
800 // }
801 // cur_true_field1_region:
802 //
803 // if (cur_lhs.field2 == null && cur_rhs.field2 == null) {
804 // goto cur_true_field2_region;
805 // } else if ((cur_lhs.field2 == null) != (cur_rhs.field2 == null) {
806 // return false;
807 // } else if (cur_lhs.field2.klass != cur_rhs.field2.klass) {
808 // return false;
809 // }
810 //
811 // cur_start_field2_region:
812 // if (!substitutable(cur_lhs.field1, cur_rhs.field1)) {
813 // return false;
814 // }
815 // cur_true_field2_region:
816 //
817 // ...
818 //
819 // cur_true_fieldn_region:
820 //
821 // cur_true_region:
822 // }
823 //
824 // To avoid recursion, for field values that are value objects, we create the start and end
825 // region for the substitutability check of that field when both sides are notnull, save them on
826 // the worklist to process them later.
827 ResourceMark rm;
828 using WorklistEntry = Tuple<Node*, RegionNode*, InlineTypeNode*, InlineTypeNode*>;
829 GrowableArray<WorklistEntry> worklist;
830 worklist.push(WorklistEntry(kit->control(), true_region, lhs->as_InlineType(), rhs->as_InlineType()));
831 while (worklist.is_nonempty()) {
832 WorklistEntry entry = worklist.pop();
833 Node* cur_start_region = entry.get<0>();
834 RegionNode* cur_true_region = entry.get<1>();
835 InlineTypeNode* cur_lhs = entry.get<2>();
836 InlineTypeNode* cur_rhs = entry.get<3>();
837
838 kit->set_control(cur_start_region);
839 if (kit->stopped()) {
840 break;
841 }
842 ciInlineKlass* vk = cur_lhs->inline_klass();
843 assert(vk == cur_rhs->inline_klass(), "should not reach here otherwise");
844
845 auto is_simple_field = [](ciField* field) {
846 ciType* ft = field->type();
847 return is_java_primitive(ft->basic_type()) || !ft->as_klass()->can_be_inline_klass();
848 };
849
850 // Go through all fields, process the simple ones first
851 for (int field_idx = 0; field_idx < vk->nof_declared_nonstatic_fields(); field_idx++) {
852 ciField* field = vk->declared_nonstatic_field_at(field_idx);
853 if (is_simple_field(field)) {
854 Node* cur_lhs_field = cur_lhs->field_value(field_idx);
855 Node* cur_rhs_field = cur_rhs->field_value(field_idx);
856 emit_substitutability_check_primitive(kit, result, cur_lhs_field, cur_rhs_field, field->type()->basic_type());
857 }
858 }
859
860 for (int field_idx = 0; field_idx < vk->nof_declared_nonstatic_fields(); field_idx++) {
861 ciField* field = vk->declared_nonstatic_field_at(field_idx);
862 if (is_simple_field(field)) {
863 // These fields have already been processed
864 continue;
865 }
866
867 Node* cur_lhs_field = cur_lhs->field_value(field_idx);
868 Node* cur_rhs_field = cur_rhs->field_value(field_idx);
869 InlineTypeNode* cur_lhs_inline = cur_lhs_field->isa_InlineType();
870 InlineTypeNode* cur_rhs_inline = cur_rhs_field->isa_InlineType();
871
872 Node* preprocess = emit_substitutability_check_pointer(kit, result, cur_lhs_field, cur_rhs_field);
873 if (kit->stopped()) {
874 break;
875 } else if (preprocess == nullptr) {
876 // Must emit a substitutability check for this field, give up for now
877 return nullptr;
878 } else if (preprocess != NodeSentinel) {
879 continue;
880 }
881
882 // When field is not null-free, the shape would look like this:
883 //
884 // current_control:
885 // if (cur_lhs_field == null) {
886 // if (cur_rhs_field == null) {
887 // goto field_true_region;
888 // } else {
889 // return false;
890 // }
891 // } else {
892 // if (cur_rhs_field == null) {
893 // return false;
894 // }
895 // }
896 // if (cur_lhs_field.klass != cur_rhs_field.klass) {
897 // return false;
898 // }
899 //
900 // field_start_region:
901 // if (!substitutable(cur_lhs_field, cur_rhs_field)) {
902 // return false;
903 // }
904 // field_true_region:
905 //
906 // The substitutability test between the fields of cur_lhs_field and cur_rhs_field is then
907 // pushed on the worklist to be expanded later.
908 RegionNode* field_true_region = new RegionNode(3);
909 igvn.register_new_node_with_optimizer(field_true_region);
910
911 // Firstly, filter the cases when either is null
912 Node* null_unknown = kit->top();
913 cur_lhs_field = kit->null_check_common(cur_lhs_field, T_OBJECT, false, &null_unknown);
914 if (!null_unknown->is_top()) {
915 PreserveJVMState pjvms(kit);
916 kit->set_control(null_unknown);
917 Node* null_null = kit->top();
918 kit->null_check_common(cur_rhs_field, T_OBJECT, false, &null_null);
919
920 // null - null, skip other checks
921 field_true_region->init_req(1, null_null);
922
923 // null - notnull
924 if (!kit->stopped()) {
925 region->add_req(kit->control());
926 result->add_req(igvn.intcon(0));
927 }
928 }
929 if (kit->stopped()) {
930 // cur_lhs_field is always null
931 kit->set_control(field_true_region);
932 continue;
933 }
934
935 Node* notnull_null = kit->top();
936 cur_rhs_field = kit->null_check_common(cur_rhs_field, T_OBJECT, false, ¬null_null);
937 if (!notnull_null->is_top()) {
938 region->add_req(notnull_null);
939 result->add_req(igvn.intcon(0));
940 }
941
942 if (kit->stopped()) {
943 // cur_rhs_field is always null
944 kit->set_control(field_true_region);
945 continue;
946 }
947 assert(!igvn.type(cur_lhs_field)->maybe_null() && !igvn.type(cur_rhs_field)->maybe_null(), "must be notnull");
948
949 // Can expand this comparison
950 const Type* cur_lhs_field_type = igvn.type(cur_lhs_field);
951 const Type* cur_rhs_field_type = igvn.type(cur_rhs_field);
952 ciInlineKlass* vk = cur_lhs_field_type->is_inlinetypeptr() ? cur_lhs_field_type->inline_klass() : cur_rhs_field_type->inline_klass();
953 const TypeInstKlassPtr* vk_klass_type = TypeInstKlassPtr::make(vk, Type::ignore_interfaces);
954 Node* vk_klass = igvn.makecon(vk_klass_type);
955
956 // Both must be vk
957 // InlineTypeNodes need no (new) field loads, so we can use the uncasted value below.
958 if (cur_lhs_inline != nullptr && cur_lhs_inline->inline_klass() == vk) {
959 cur_lhs_field = cur_lhs_inline;
960 } else {
961 Node* not_vk = kit->top();
962 cur_lhs_field = kit->gen_checkcast(cur_lhs_field, vk_klass, ¬_vk);
963 if (!not_vk->is_top()) {
964 region->add_req(not_vk);
965 result->add_req(igvn.intcon(0));
966 }
967 cur_lhs_field = InlineTypeNode::make_from_oop(kit, cur_lhs_field, vk);
968 }
969
970 if (cur_rhs_inline != nullptr && cur_rhs_inline->inline_klass() == vk) {
971 cur_rhs_field = cur_rhs_inline;
972 } else {
973 Node* not_vk = kit->top();
974 cur_rhs_field = kit->gen_checkcast(cur_rhs_field, vk_klass, ¬_vk);
975 if (!not_vk->is_top()) {
976 region->add_req(not_vk);
977 result->add_req(igvn.intcon(0));
978 }
979 cur_rhs_field = InlineTypeNode::make_from_oop(kit, cur_rhs_field, vk);
980 }
981
982 if (!kit->stopped()) {
983 // Push the expanded InlineTypeNodes for processing later
984 worklist.push(WorklistEntry(kit->control(), field_true_region, cur_lhs_field->as_InlineType(), cur_rhs_field->as_InlineType()));
985 }
986
987 kit->set_control(field_true_region);
988 }
989
990 cur_true_region->init_req(2, kit->control());
991 kit->set_control(cur_true_region);
992 }
993
994 kit->set_control(region);
995 return result;
996 }
997
998 InlineTypeNode* InlineTypeNode::buffer(GraphKit* kit, bool safe_for_replace) {
999 if (is_allocated(&kit->gvn())) {
1000 // Already buffered
1001 return this;
1002 }
1003
1004 // Check if inline type is already buffered
1005 Node* not_buffered_ctl = kit->top();
1006 Node* not_null_oop = kit->null_check_oop(get_oop(), ¬_buffered_ctl, /* never_see_null = */ false, safe_for_replace);
1007 if (not_buffered_ctl->is_top()) {
1008 // Already buffered
1009 InlineTypeNode* vt = clone_if_required(&kit->gvn(), kit->map(), safe_for_replace);
1010 vt->set_is_buffered(kit->gvn());
1011 vt = kit->gvn().transform(vt)->as_InlineType();
1012 if (safe_for_replace) {
1013 kit->replace_in_map(this, vt);
1014 }
1015 return vt;
1016 }
1017 Node* buffered_ctl = kit->control();
1018 kit->set_control(not_buffered_ctl);
1019
1020 // Inline type is not buffered, check if it is null.
1021 Node* null_ctl = kit->top();
1022 kit->null_check_common(get_null_marker(), T_INT, false, &null_ctl);
1023 bool null_free = null_ctl->is_top();
1024
1025 RegionNode* region = new RegionNode(4);
1026 PhiNode* oop = PhiNode::make(region, not_null_oop, type()->join_speculative(null_free ? TypePtr::NOTNULL : TypePtr::BOTTOM));
1027
1028 // InlineType is already buffered
1029 region->init_req(1, buffered_ctl);
1030 oop->init_req(1, not_null_oop);
1031
1032 // InlineType is null
1033 region->init_req(2, null_ctl);
1034 oop->init_req(2, kit->gvn().zerocon(T_OBJECT));
1035
1036 PhiNode* io = PhiNode::make(region, kit->i_o(), Type::ABIO);
1037 PhiNode* mem = PhiNode::make(region, kit->merged_memory(), Type::MEMORY, TypePtr::BOTTOM);
1038
1039 if (!kit->stopped()) {
1040 assert(!is_allocated(&kit->gvn()), "already buffered");
1041 PreserveJVMState pjvms(kit);
1042 ciInlineKlass* vk = inline_klass();
1043 // Allocate and initialize buffer, re-execute on deoptimization.
1044 kit->jvms()->set_bci(kit->bci());
1045 kit->jvms()->set_should_reexecute(true);
1046 kit->kill_dead_locals();
1047 Node* klass_node = kit->makecon(TypeKlassPtr::make(vk));
1048 Node* alloc_oop = kit->new_instance(klass_node, nullptr, nullptr, /* deoptimize_on_exception */ true, this);
1049 Node* payload_alloc_oop = kit->basic_plus_adr(alloc_oop, vk->payload_offset());
1050 store(kit, alloc_oop, payload_alloc_oop, true, IN_HEAP | MO_UNORDERED | C2_TIGHTLY_COUPLED_ALLOC);
1051
1052 // Do not let stores that initialize this buffer be reordered with a subsequent
1053 // store that would make this buffer accessible by other threads.
1054 AllocateNode* alloc = AllocateNode::Ideal_allocation(alloc_oop);
1055 assert(alloc != nullptr, "must have an allocation node");
1056 kit->insert_mem_bar(Op_MemBarStoreStore, alloc->proj_out_or_null(AllocateNode::RawAddress));
1057 oop->init_req(3, alloc_oop);
1058 region->init_req(3, kit->control());
1059 io ->init_req(3, kit->i_o());
1060 mem ->init_req(3, kit->merged_memory());
1061 }
1062
1063 // Update GraphKit
1064 kit->set_control(kit->gvn().transform(region));
1065 kit->set_i_o(kit->gvn().transform(io));
1066 kit->set_all_memory(kit->gvn().transform(mem));
1067 kit->record_for_igvn(region);
1068 kit->record_for_igvn(oop);
1069 kit->record_for_igvn(io);
1070 kit->record_for_igvn(mem);
1071
1072 // Use cloned InlineTypeNode to propagate oop from now on
1073 Node* res_oop = kit->gvn().transform(oop);
1074 InlineTypeNode* vt = clone_if_required(&kit->gvn(), kit->map(), safe_for_replace);
1075 vt->set_oop(kit->gvn(), res_oop);
1076 vt->set_is_buffered(kit->gvn());
1077 vt = kit->gvn().transform(vt)->as_InlineType();
1078 kit->record_for_igvn(vt);
1079 if (safe_for_replace) {
1080 kit->replace_in_map(this, vt);
1081 }
1082 // InlineTypeNode::remove_redundant_allocations piggybacks on split if.
1083 // Make sure it gets a chance to remove this allocation.
1084 kit->C->set_has_split_ifs(true);
1085 return vt;
1086 }
1087
1088 bool InlineTypeNode::is_allocated(PhaseGVN* phase) const {
1089 if (phase->type(get_is_buffered()) == TypeInt::ONE) {
1090 return true;
1091 }
1092 Node* oop = get_oop();
1093 const Type* oop_type = (phase != nullptr) ? phase->type(oop) : oop->bottom_type();
1094 return !oop_type->maybe_null();
1095 }
1096
1097 static void replace_proj(Compile* C, CallNode* call, uint& proj_idx, Node* value, BasicType bt) {
1098 ProjNode* pn = call->proj_out_or_null(proj_idx);
1099 if (pn != nullptr) {
1100 if (pn->outcnt() > 0) {
1101 PhaseGVN* gvn = C->initial_gvn();
1102 const Type* result_type = gvn->type(pn);
1103 value = gvn->transform(new OpaqueParseNode(C, value, result_type));
1104 }
1105 C->gvn_replace_by(pn, value);
1106 C->initial_gvn()->hash_delete(pn);
1107 pn->set_req(0, C->top());
1108 }
1109 proj_idx += type2size[bt];
1110 }
1111
1112 // When a call returns multiple values, it has several result
1113 // projections, one per field. Replacing the result of the call by an
1114 // inline type node (after late inlining) requires that for each result
1115 // projection, we find the corresponding inline type field.
1116 void InlineTypeNode::replace_call_results(GraphKit* kit, CallNode* call, Compile* C) const {
1117 uint proj_idx = TypeFunc::Parms;
1118 // Replace oop projection
1119 replace_proj(C, call, proj_idx, get_oop(), T_OBJECT);
1120 // Replace field projections
1121 replace_field_projs(C, call, proj_idx);
1122 // Replace null_marker projection
1123 replace_proj(C, call, proj_idx, get_null_marker(), T_BOOLEAN);
1124 assert(proj_idx == call->tf()->range_cc()->cnt(), "missed a projection");
1125 }
1126
1127 void InlineTypeNode::replace_field_projs(Compile* C, CallNode* call, uint& proj_idx) const {
1128 for (uint i = 0; i < field_count(); ++i) {
1129 Node* value = field_value(i);
1130 ciField* field = this->field(i);
1131 assert(!field->is_flat() || field->type()->is_inlinetype(), "must be an inline type");
1132 if (field->is_flat()) {
1133 InlineTypeNode* vt = value->as_InlineType();
1134 // Replace field projections for flat field
1135 vt->replace_field_projs(C, call, proj_idx);
1136 if (!field->is_null_free()) {
1137 // Replace null_marker projection for nullable field
1138 replace_proj(C, call, proj_idx, vt->get_null_marker(), T_BOOLEAN);
1139 }
1140 continue;
1141 }
1142 // Replace projection for field value
1143 replace_proj(C, call, proj_idx, value, field->type()->basic_type());
1144 }
1145 }
1146
1147 InlineTypeNode* InlineTypeNode::allocate_fields(GraphKit* kit) {
1148 InlineTypeNode* vt = clone_if_required(&kit->gvn(), kit->map());
1149 for (uint i = 0; i < field_count(); i++) {
1150 Node* value = field_value(i);
1151 ciField* field = this->field(i);
1152 assert(!field->is_flat() || field->type()->is_inlinetype(), "must be an inline type");
1153 if (field->is_flat()) {
1154 // Flat inline type field
1155 vt->set_field_value(i, value->as_InlineType()->allocate_fields(kit));
1156 } else if (value->is_InlineType()) {
1157 // Non-flat inline type field
1158 vt->set_field_value(i, value->as_InlineType()->buffer(kit));
1159 }
1160 }
1161 vt = kit->gvn().transform(vt)->as_InlineType();
1162 kit->replace_in_map(this, vt);
1163 return vt;
1164 }
1165
1166 // Replace a buffer allocation by a dominating allocation
1167 static void replace_allocation(PhaseIterGVN* igvn, Node* res, Node* dom) {
1168 // Remove initializing stores and GC barriers
1169 for (DUIterator_Fast imax, i = res->fast_outs(imax); i < imax; i++) {
1170 Node* use = res->fast_out(i);
1171 if (use->is_AddP()) {
1172 for (DUIterator_Fast jmax, j = use->fast_outs(jmax); j < jmax; j++) {
1173 Node* store = use->fast_out(j)->isa_Store();
1174 if (store != nullptr) {
1175 igvn->rehash_node_delayed(store);
1176 igvn->replace_in_uses(store, store->in(MemNode::Memory));
1177 }
1178 }
1179 } else if (use->Opcode() == Op_CastP2X) {
1180 if (UseG1GC && use->find_out_with(Op_XorX)->in(1) != use) {
1181 // The G1 pre-barrier uses a CastP2X both for the pointer of the object
1182 // we store into, as well as the value we are storing. Skip if this is a
1183 // barrier for storing 'res' into another object.
1184 continue;
1185 }
1186 BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
1187 bs->eliminate_gc_barrier(igvn, use);
1188 --i; --imax;
1189 }
1190 }
1191 igvn->replace_node(res, dom);
1192 }
1193
1194 Node* InlineTypeNode::Ideal(PhaseGVN* phase, bool can_reshape) {
1195 Node* oop = get_oop();
1196 if (oop->isa_InlineType() && !phase->type(oop)->maybe_null()) {
1197 InlineTypeNode* vtptr = oop->as_InlineType();
1198 assert(inline_klass() == vtptr->inline_klass(), "inconsistent types");
1199 set_oop(*phase, vtptr->get_oop());
1200 set_is_buffered(*phase);
1201 set_null_marker(*phase);
1202 for (uint i = Values; i < vtptr->req(); ++i) {
1203 set_req(i, vtptr->in(i));
1204 }
1205 return this;
1206 }
1207
1208 // Use base oop if fields are loaded from memory, don't do so if base is the CheckCastPP of an
1209 // allocation because the only case we load from a naked CheckCastPP is when we exit a
1210 // constructor of an inline type and we want to relinquish the larval oop there. This has a
1211 // couple of benefits:
1212 // - The allocation is likely to be elided earlier if it is not an input of an InlineTypeNode.
1213 // - The InlineTypeNode without an allocation input is more likely to be GVN-ed. This may emerge
1214 // when we try to clone a value object.
1215 // - The buffering, if needed, is delayed until it is required. This new allocation, since it is
1216 // created from an InlineTypeNode, is recognized as not having a unique identity and in the
1217 // future, we can move them around more freely such as hoisting out of loops. This is not true
1218 // for the old allocation since larval value objects do have unique identities.
1219 Node* base = is_loaded(phase);
1220 if (base != nullptr && !base->is_InlineType() && !phase->type(base)->maybe_null() && phase->C->allow_macro_nodes() && AllocateNode::Ideal_allocation(base) == nullptr) {
1221 if (oop != base || !is_allocated(phase)) {
1222 set_oop(*phase, base);
1223 set_is_buffered(*phase);
1224 return this;
1225 }
1226 }
1227
1228 if (can_reshape) {
1229 PhaseIterGVN* igvn = phase->is_IterGVN();
1230 if (is_allocated(phase)) {
1231 // Search for and remove re-allocations of this inline type. Ignore scalar replaceable ones,
1232 // they will be removed anyway and changing the memory chain will confuse other optimizations.
1233 // This can happen with late inlining when we first allocate an inline type argument
1234 // but later decide to inline the call after the callee code also triggered allocation.
1235 for (DUIterator_Fast imax, i = fast_outs(imax); i < imax; i++) {
1236 AllocateNode* alloc = fast_out(i)->isa_Allocate();
1237 if (alloc != nullptr && alloc->in(AllocateNode::InlineType) == this && !alloc->_is_scalar_replaceable) {
1238 // Found a re-allocation
1239 Node* res = alloc->result_cast();
1240 if (res != nullptr && res->is_CheckCastPP()) {
1241 // Replace allocation by oop and unlink AllocateNode
1242 replace_allocation(igvn, res, oop);
1243 igvn->replace_input_of(alloc, AllocateNode::InlineType, igvn->C->top());
1244 --i; --imax;
1245 }
1246 }
1247 }
1248 }
1249 }
1250
1251 return nullptr;
1252 }
1253
1254 InlineTypeNode* InlineTypeNode::make_uninitialized(PhaseGVN& gvn, ciInlineKlass* vk, bool null_free) {
1255 // Create a new InlineTypeNode with uninitialized values and nullptr oop
1256 InlineTypeNode* vt = new InlineTypeNode(vk, gvn.zerocon(T_OBJECT), null_free);
1257 vt->set_is_buffered(gvn, false);
1258 vt->set_null_marker(gvn);
1259 return vt;
1260 }
1261
1262 InlineTypeNode* InlineTypeNode::make_all_zero(PhaseGVN& gvn, ciInlineKlass* vk) {
1263 return make_all_zero_impl(gvn, vk);
1264 }
1265
1266 InlineTypeNode* InlineTypeNode::make_all_zero_impl(PhaseGVN& gvn, ciInlineKlass* vk) {
1267 // Create a new InlineTypeNode initialized with all zero
1268 InlineTypeNode* vt = new InlineTypeNode(vk, gvn.zerocon(T_OBJECT), /* null_free= */ true);
1269 vt->set_is_buffered(gvn, false);
1270 vt->set_null_marker(gvn);
1271 for (uint i = 0; i < vt->field_count(); ++i) {
1272 ciField* field = vt->field(i);
1273 assert(!field->is_flat() || field->type()->is_inlinetype(), "must be an inline type");
1274 ciType* ft = field->type();
1275 Node* value;
1276 if (field->is_null_free()) {
1277 value = make_all_zero_impl(gvn, ft->as_inline_klass());
1278 } else if (field->is_flat()) {
1279 value = make_null_impl(gvn, ft->as_inline_klass());
1280 } else {
1281 value = gvn.zerocon(ft->basic_type());
1282 }
1283 vt->set_field_value(i, value);
1284 }
1285 vt = gvn.transform(vt)->as_InlineType();
1286 assert(vt->is_all_zero(&gvn), "must be the all-zero inline type");
1287 return vt;
1288 }
1289
1290 bool InlineTypeNode::is_all_zero(PhaseGVN* gvn, bool flat) const {
1291 const TypeInt* tinit = gvn->type(get_null_marker())->isa_int();
1292 if (tinit == nullptr || !tinit->is_con(1)) {
1293 return false; // May be null
1294 }
1295 for (uint i = 0; i < field_count(); ++i) {
1296 Node* value = field_value(i);
1297 ciField* field = this->field(i);
1298 assert(!field->is_flat() || field->type()->is_inlinetype(), "must be an inline type");
1299 if (field->is_null_free()) {
1300 // Null-free value class field must have the all-zero value. If 'flat' is set,
1301 // reject non-flat fields because they need to be initialized with an oop to a buffer.
1302 if (!value->is_InlineType() || !value->as_InlineType()->is_all_zero(gvn) || (flat && !field->is_flat())) {
1303 return false;
1304 }
1305 continue;
1306 } else if (value->is_InlineType()) {
1307 // Nullable value class field must be null
1308 tinit = gvn->type(value->as_InlineType()->get_null_marker())->isa_int();
1309 if (tinit != nullptr && tinit->is_con(0)) {
1310 continue;
1311 }
1312 return false;
1313 } else if (!gvn->type(value)->is_zero_type()) {
1314 return false;
1315 }
1316 }
1317 return true;
1318 }
1319
1320 InlineTypeNode* InlineTypeNode::make_from_oop(GraphKit* kit, Node* oop, ciInlineKlass* vk) {
1321 return make_from_oop_impl(kit, oop, vk);
1322 }
1323
1324 InlineTypeNode* InlineTypeNode::make_from_oop_impl(GraphKit* kit, Node* oop, ciInlineKlass* vk) {
1325 PhaseGVN& gvn = kit->gvn();
1326
1327 // Create and initialize an InlineTypeNode by loading all field
1328 // values from a heap-allocated version and also save the oop.
1329 InlineTypeNode* vt = oop->isa_InlineType();
1330 if (vt != nullptr) {
1331 return vt;
1332 }
1333
1334 if (gvn.type(oop) == TypePtr::NULL_PTR) {
1335 return make_null_impl(gvn, vk);
1336 }
1337
1338 if (gvn.type(oop)->maybe_null()) {
1339 // Add a null check because the oop may be null
1340 Node* null_ctl = kit->top();
1341 Node* not_null_oop = kit->null_check_oop(oop, &null_ctl);
1342 if (kit->stopped()) {
1343 // Constant null
1344 kit->set_control(null_ctl);
1345 vt = make_null_impl(gvn, vk);
1346 kit->record_for_igvn(vt);
1347 return vt;
1348 }
1349 vt = new InlineTypeNode(vk, not_null_oop, /* null_free= */ false);
1350 vt->set_is_buffered(gvn);
1351 vt->set_null_marker(gvn);
1352 Node* payload_ptr = kit->basic_plus_adr(not_null_oop, vk->payload_offset());
1353 vt->load(kit, not_null_oop, payload_ptr, true, true, IN_HEAP | MO_UNORDERED);
1354
1355 if (null_ctl != kit->top()) {
1356 InlineTypeNode* null_vt = make_null_impl(gvn, vk);
1357 Node* region = new RegionNode(3);
1358 region->init_req(1, kit->control());
1359 region->init_req(2, null_ctl);
1360 vt = vt->clone_with_phis(&gvn, region, kit->map());
1361 vt->merge_with(&gvn, null_vt, 2, true);
1362 vt->set_oop(gvn, oop);
1363 vt->set_is_buffered(gvn);
1364 kit->set_control(gvn.transform(region));
1365 }
1366 } else {
1367 // Oop can never be null
1368 vt = new InlineTypeNode(vk, oop, /* null_free= */ true);
1369 vt->set_is_buffered(gvn);
1370 vt->set_null_marker(gvn);
1371 Node* payload_ptr = kit->basic_plus_adr(oop, vk->payload_offset());
1372 vt->load(kit, oop, payload_ptr, true, true, IN_HEAP | MO_UNORDERED);
1373 }
1374 assert(vt->is_allocated(&gvn), "inline type should be allocated");
1375 kit->record_for_igvn(vt);
1376 return gvn.transform(vt)->as_InlineType();
1377 }
1378
1379 InlineTypeNode* InlineTypeNode::make_from_flat(GraphKit* kit, ciInlineKlass* vk, Node* base, Node* ptr,
1380 bool atomic, bool immutable_memory, bool null_free, DecoratorSet decorators) {
1381 return make_from_flat_impl(kit, vk, base, ptr, atomic, immutable_memory, null_free, null_free, decorators);
1382 }
1383
1384 // GraphKit wrapper for the 'make_from_flat' method
1385 InlineTypeNode* InlineTypeNode::make_from_flat_impl(GraphKit* kit, ciInlineKlass* vk, Node* base, Node* ptr, bool atomic, bool immutable_memory,
1386 bool null_free, bool trust_null_free_oop, DecoratorSet decorators) {
1387 assert(null_free || !trust_null_free_oop, "cannot trust null-free oop when the holder object is not null-free");
1388 PhaseGVN& gvn = kit->gvn();
1389 bool do_atomic = atomic;
1390 // With immutable memory, a non-atomic load and an atomic load are the same
1391 if (immutable_memory) {
1392 do_atomic = false;
1393 }
1394 // If there is only one flattened field, a non-atomic load and an atomic load are the same
1395 if (vk->is_naturally_atomic(null_free)) {
1396 do_atomic = false;
1397 }
1398
1399 if (!do_atomic) {
1400 InlineTypeNode* vt = make_uninitialized(kit->gvn(), vk, null_free);
1401 if (!null_free) {
1402 int nm_offset = vk->null_marker_offset_in_payload();
1403 Node* nm_ptr = kit->basic_plus_adr(base, ptr, nm_offset);
1404 const TypePtr* nm_ptr_type = (decorators & C2_MISMATCHED) == 0 ? gvn.type(nm_ptr)->is_ptr() : TypeRawPtr::BOTTOM;
1405 Node* nm_value = kit->access_load_at(base, nm_ptr, nm_ptr_type, TypeInt::BOOL, T_BOOLEAN, decorators);
1406 vt->set_req(NullMarker, nm_value);
1407 }
1408
1409 vt->load(kit, base, ptr, immutable_memory, trust_null_free_oop, decorators);
1410 return gvn.transform(vt)->as_InlineType();
1411 }
1412
1413 assert(!immutable_memory, "immutable memory does not need explicit atomic access");
1414 return LoadFlatNode::load(kit, vk, base, ptr, null_free, trust_null_free_oop, decorators);
1415 }
1416
1417 InlineTypeNode* InlineTypeNode::make_from_flat_array(GraphKit* kit, ciInlineKlass* vk, Node* base, Node* idx) {
1418 assert(vk->maybe_flat_in_array(), "element type %s cannot be flat in array", vk->name()->as_utf8());
1419 PhaseGVN& gvn = kit->gvn();
1420 // The flat field loads are dependent on both the array layout checks as well as the range check.
1421 DecoratorSet decorators = IN_HEAP | IS_ARRAY | MO_UNORDERED | C2_CONTROL_DEPENDENT_LOAD | C2_UNKNOWN_CONTROL_LOAD;
1422 kit->C->set_flat_accesses();
1423 InlineTypeNode* vt_nullable = nullptr;
1424 InlineTypeNode* vt_null_free = nullptr;
1425 InlineTypeNode* vt_non_atomic = nullptr;
1426
1427 RegionNode* region = new RegionNode(4);
1428 gvn.set_type(region, Type::CONTROL);
1429 kit->record_for_igvn(region);
1430
1431 Node* input_memory_state = kit->reset_memory();
1432 kit->set_all_memory(input_memory_state);
1433
1434 PhiNode* mem = PhiNode::make(region, input_memory_state, Type::MEMORY, TypePtr::BOTTOM);
1435 gvn.set_type(mem, Type::MEMORY);
1436 kit->record_for_igvn(mem);
1437
1438 PhiNode* io = PhiNode::make(region, kit->i_o(), Type::ABIO);
1439 gvn.set_type(io, Type::ABIO);
1440 kit->record_for_igvn(io);
1441
1442 Node* bol_null_free = kit->null_free_array_test(base); // Argument evaluation order is undefined in C++ and since this sets control, it needs to come first
1443 IfNode* iff_null_free = kit->create_and_map_if(kit->control(), bol_null_free, PROB_FAIR, COUNT_UNKNOWN);
1444
1445 // Nullable
1446 kit->set_control(kit->IfFalse(iff_null_free));
1447 if (!kit->stopped()) {
1448 assert(vk->has_nullable_atomic_layout(), "element type %s does not have a nullable flat layout", vk->name()->as_utf8());
1449 kit->set_all_memory(input_memory_state);
1450 Node* cast = kit->cast_to_flat_array_exact(base, vk, false, true);
1451 Node* ptr = kit->array_element_address(cast, idx, T_FLAT_ELEMENT);
1452 vt_nullable = InlineTypeNode::make_from_flat(kit, vk, cast, ptr, true, false, false, decorators);
1453
1454 region->init_req(1, kit->control());
1455 mem->set_req(1, kit->reset_memory());
1456 io->set_req(1, kit->i_o());
1457 }
1458
1459 // Null-free
1460 kit->set_control(kit->IfTrue(iff_null_free));
1461 if (!kit->stopped()) {
1462 kit->set_all_memory(input_memory_state);
1463
1464 Node* bol_atomic = kit->null_free_atomic_array_test(base, vk);
1465 IfNode* iff_atomic = kit->create_and_map_if(kit->control(), bol_atomic, PROB_FAIR, COUNT_UNKNOWN);
1466
1467 // Atomic
1468 kit->set_control(kit->IfTrue(iff_atomic));
1469 if (!kit->stopped()) {
1470 assert(vk->has_null_free_atomic_layout(), "element type %s does not have a null-free atomic flat layout", vk->name()->as_utf8());
1471 kit->set_all_memory(input_memory_state);
1472 Node* cast = kit->cast_to_flat_array_exact(base, vk, true, true);
1473 Node* ptr = kit->array_element_address(cast, idx, T_FLAT_ELEMENT);
1474 vt_null_free = InlineTypeNode::make_from_flat(kit, vk, cast, ptr, true, false, true, decorators);
1475
1476 region->init_req(2, kit->control());
1477 mem->set_req(2, kit->reset_memory());
1478 io->set_req(2, kit->i_o());
1479 }
1480
1481 // Non-Atomic
1482 kit->set_control(kit->IfFalse(iff_atomic));
1483 if (!kit->stopped()) {
1484 assert(vk->has_null_free_non_atomic_layout(), "element type %s does not have a null-free non-atomic flat layout", vk->name()->as_utf8());
1485 kit->set_all_memory(input_memory_state);
1486 Node* cast = kit->cast_to_flat_array_exact(base, vk, true, false);
1487 Node* ptr = kit->array_element_address(cast, idx, T_FLAT_ELEMENT);
1488 vt_non_atomic = InlineTypeNode::make_from_flat(kit, vk, cast, ptr, false, false, true, decorators);
1489
1490 region->init_req(3, kit->control());
1491 mem->set_req(3, kit->reset_memory());
1492 io->set_req(3, kit->i_o());
1493 }
1494 }
1495
1496 InlineTypeNode* vt = nullptr;
1497 if (vt_nullable == nullptr && vt_null_free == nullptr && vt_non_atomic == nullptr) {
1498 // All paths are dead
1499 vt = make_null(gvn, vk);
1500 } else if (vt_nullable == nullptr && vt_null_free == nullptr) {
1501 vt = vt_non_atomic;
1502 } else if (vt_nullable == nullptr && vt_non_atomic == nullptr) {
1503 vt = vt_null_free;
1504 } else if (vt_null_free == nullptr && vt_non_atomic == nullptr) {
1505 vt = vt_nullable;
1506 }
1507 if (vt != nullptr) {
1508 kit->set_control(kit->gvn().transform(region));
1509 kit->set_all_memory(kit->gvn().transform(mem));
1510 kit->set_i_o(kit->gvn().transform(io));
1511 return vt;
1512 }
1513
1514 InlineTypeNode* zero = InlineTypeNode::make_null(gvn, vk);
1515 vt = zero->clone_with_phis(&gvn, region);
1516 if (vt_nullable != nullptr) {
1517 vt = vt->merge_with(&gvn, vt_nullable, 1, false);
1518 }
1519 if (vt_null_free != nullptr) {
1520 vt = vt->merge_with(&gvn, vt_null_free, 2, false);
1521 }
1522 if (vt_non_atomic != nullptr) {
1523 vt = vt->merge_with(&gvn, vt_non_atomic, 3, false);
1524 }
1525
1526 kit->set_control(kit->gvn().transform(region));
1527 kit->set_all_memory(kit->gvn().transform(mem));
1528 kit->set_i_o(kit->gvn().transform(io));
1529 return gvn.transform(vt)->as_InlineType();
1530 }
1531
1532 InlineTypeNode* InlineTypeNode::make_from_multi(GraphKit* kit, MultiNode* multi, ciInlineKlass* vk, uint& base_input, bool in, bool null_free) {
1533 InlineTypeNode* vt = make_uninitialized(kit->gvn(), vk, null_free);
1534 if (!in || multi->is_Start()) {
1535 // Keep track of the oop. The inline type might already be buffered.
1536 Node* oop = nullptr;
1537 if (multi->is_Start()) {
1538 oop = kit->gvn().transform(new ParmNode(multi->as_Start(), base_input++));
1539 } else {
1540 oop = kit->gvn().transform(new ProjNode(multi, base_input++));
1541 }
1542 vt->set_oop(kit->gvn(), oop);
1543 } else {
1544 Node* oop = multi->as_Call()->in(base_input++);
1545 vt->set_oop(kit->gvn(), oop);
1546 }
1547 vt->initialize_fields(kit, multi, base_input, in, null_free, nullptr);
1548 return kit->gvn().transform(vt)->as_InlineType();
1549 }
1550
1551 Node* InlineTypeNode::is_loaded(PhaseGVN* phase, ciInlineKlass* vk, Node* base, int holder_offset) const {
1552 if (vk == nullptr) {
1553 vk = inline_klass();
1554 }
1555 for (uint i = 0; i < field_count(); ++i) {
1556 ciField* field = this->field(i);
1557 int offset = holder_offset + field->offset_in_bytes();
1558 Node* value = field_value(i);
1559 if (value->is_InlineType()) {
1560 assert(!field->is_flat() || field->type()->is_inlinetype(), "must be an inline type");
1561 InlineTypeNode* vt = value->as_InlineType();
1562 if (vt->type()->inline_klass()->is_empty()) {
1563 continue;
1564 } else if (field->is_flat() && vt->is_InlineType()) {
1565 // Check inline type field load recursively
1566 base = vt->as_InlineType()->is_loaded(phase, vk, base, offset - vt->type()->inline_klass()->payload_offset());
1567 if (base == nullptr) {
1568 return nullptr;
1569 }
1570 continue;
1571 } else {
1572 value = vt->get_oop();
1573 if (value->Opcode() == Op_CastPP) {
1574 // Skip CastPP
1575 value = value->in(1);
1576 }
1577 }
1578 }
1579 if (value->isa_DecodeN()) {
1580 // Skip DecodeN
1581 value = value->in(1);
1582 }
1583 if (value->isa_Load()) {
1584 // Check if base and offset of field load matches inline type layout
1585 intptr_t loffset = 0;
1586 Node* lbase = AddPNode::Ideal_base_and_offset(value->in(MemNode::Address), phase, loffset);
1587 if (lbase == nullptr || (lbase != base && base != nullptr) || loffset != offset) {
1588 return nullptr;
1589 } else if (base == nullptr) {
1590 // Set base and check if pointer type matches
1591 base = lbase;
1592 const TypeInstPtr* vtptr = phase->type(base)->isa_instptr();
1593 if (vtptr == nullptr || !vtptr->instance_klass()->equals(vk)) {
1594 return nullptr;
1595 }
1596 }
1597 } else {
1598 return nullptr;
1599 }
1600 }
1601 return base;
1602 }
1603
1604 Node* InlineTypeNode::tagged_klass(ciInlineKlass* vk, PhaseGVN& gvn) {
1605 const TypeKlassPtr* tk = TypeKlassPtr::make(vk);
1606 intptr_t bits = tk->get_con();
1607 set_nth_bit(bits, 0);
1608 return gvn.longcon((jlong)bits);
1609 }
1610
1611 void InlineTypeNode::pass_fields(GraphKit* kit, Node* n, uint& base_input, bool in, bool null_free, bool root) {
1612 if (root) {
1613 if (is_allocated(&kit->gvn())) {
1614 // Keep the information that 'this' is buffered
1615 n->init_req(base_input++, this);
1616 } else {
1617 n->init_req(base_input++, get_oop());
1618 }
1619 }
1620 if (!null_free && in) {
1621 n->init_req(base_input++, get_null_marker());
1622 }
1623 for (uint i = 0; i < field_count(); i++) {
1624 Node* arg = field_value(i);
1625 ciField* field = this->field(i);
1626 assert(!field->is_flat() || field->type()->is_inlinetype(), "must be an inline type");
1627 if (field->is_flat()) {
1628 // Flat inline type field
1629 arg->as_InlineType()->pass_fields(kit, n, base_input, in);
1630 if (!field->is_null_free()) {
1631 assert(field->null_marker_offset() != -1, "inconsistency");
1632 n->init_req(base_input++, arg->as_InlineType()->get_null_marker());
1633 }
1634 } else {
1635 if (arg->is_InlineType()) {
1636 // Non-flat inline type field
1637 InlineTypeNode* vt = arg->as_InlineType();
1638 assert(n->Opcode() != Op_Return || vt->is_allocated(&kit->gvn()), "inline type field should be allocated on return");
1639 arg = vt->buffer(kit);
1640 }
1641 // Initialize call/return arguments
1642 n->init_req(base_input++, arg);
1643 if (field->type()->size() == 2) {
1644 n->init_req(base_input++, kit->top());
1645 }
1646 }
1647 }
1648 }
1649
1650 void InlineTypeNode::initialize_fields(GraphKit* kit, MultiNode* multi, uint& base_input, bool in, bool no_null_marker, Node* null_check_region) {
1651 PhaseGVN& gvn = kit->gvn();
1652 Node* null_marker = nullptr;
1653 if (!no_null_marker) {
1654 // Nullable inline type
1655 if (in) {
1656 // Set null marker
1657 if (multi->is_Start()) {
1658 null_marker = gvn.transform(new ParmNode(multi->as_Start(), base_input));
1659 } else {
1660 null_marker = multi->as_Call()->in(base_input);
1661 }
1662 set_req(NullMarker, null_marker);
1663 base_input++;
1664 }
1665 // Add a null check to make subsequent loads dependent on
1666 assert(null_check_region == nullptr, "already set");
1667 if (null_marker == nullptr) {
1668 // Will only be initialized below, use dummy node for now
1669 null_marker = new Node(1);
1670 null_marker->init_req(0, kit->control()); // Add an input to prevent dummy from being dead
1671 gvn.set_type_bottom(null_marker);
1672 }
1673 Node* null_ctrl = kit->top();
1674 kit->null_check_common(null_marker, T_INT, false, &null_ctrl);
1675 Node* non_null_ctrl = kit->control();
1676 null_check_region = new RegionNode(3);
1677 null_check_region->init_req(1, non_null_ctrl);
1678 null_check_region->init_req(2, null_ctrl);
1679 null_check_region = gvn.transform(null_check_region);
1680 kit->set_control(null_check_region);
1681 }
1682
1683 for (uint i = 0; i < field_count(); ++i) {
1684 ciField* field = this->field(i);
1685 ciType* type = field->type();
1686 Node* parm = nullptr;
1687 assert(!field->is_flat() || field->type()->is_inlinetype(), "must be an inline type");
1688 if (field->is_flat()) {
1689 // Flat inline type field
1690 InlineTypeNode* vt = make_uninitialized(gvn, type->as_inline_klass(), field->is_null_free());
1691 vt->initialize_fields(kit, multi, base_input, in, true, null_check_region);
1692 if (!field->is_null_free()) {
1693 assert(field->null_marker_offset() != -1, "inconsistency");
1694 Node* null_marker_field_vt = nullptr;
1695 if (multi->is_Start()) {
1696 null_marker_field_vt = gvn.transform(new ParmNode(multi->as_Start(), base_input));
1697 } else if (in) {
1698 null_marker_field_vt = multi->as_Call()->in(base_input);
1699 } else {
1700 null_marker_field_vt = gvn.transform(new ProjNode(multi->as_Call(), base_input));
1701 }
1702 vt->set_req(NullMarker, null_marker_field_vt);
1703 base_input++;
1704 }
1705 parm = gvn.transform(vt);
1706 } else {
1707 if (multi->is_Start()) {
1708 assert(in, "return from start?");
1709 parm = gvn.transform(new ParmNode(multi->as_Start(), base_input));
1710 } else if (in) {
1711 parm = multi->as_Call()->in(base_input);
1712 } else {
1713 parm = gvn.transform(new ProjNode(multi->as_Call(), base_input));
1714 }
1715 base_input += type->size();
1716 }
1717 assert(parm != nullptr, "should never be null");
1718 assert(field_value(i) == nullptr, "already set");
1719 set_field_value(i, parm);
1720 gvn.record_for_igvn(parm);
1721 }
1722 // The last argument is used to pass the null marker to compiled code
1723 if (!no_null_marker && !in) {
1724 Node* cmp = null_marker->raw_out(0);
1725 null_marker = gvn.transform(new ProjNode(multi->as_Call(), base_input));
1726 set_req(NullMarker, null_marker);
1727 gvn.hash_delete(cmp);
1728 cmp->set_req(1, null_marker);
1729 gvn.hash_find_insert(cmp);
1730 gvn.record_for_igvn(cmp);
1731 base_input++;
1732 }
1733 }
1734
1735 // Search for multiple allocations of this inline type and try to replace them by dominating allocations.
1736 // Equivalent InlineTypeNodes are merged by GVN, so we just need to search for AllocateNode users to find redundant allocations.
1737 void InlineTypeNode::remove_redundant_allocations(PhaseIdealLoop* phase) const {
1738 PhaseIterGVN* igvn = &phase->igvn();
1739 // Search for allocations of this inline type. Ignore scalar replaceable ones, they
1740 // will be removed anyway and changing the memory chain will confuse other optimizations.
1741 for (DUIterator_Fast imax, i = fast_outs(imax); i < imax; i++) {
1742 AllocateNode* alloc = fast_out(i)->isa_Allocate();
1743 if (alloc != nullptr && alloc->in(AllocateNode::InlineType) == this && !alloc->_is_scalar_replaceable) {
1744 Node* res = alloc->result_cast();
1745 if (res == nullptr || !res->is_CheckCastPP()) {
1746 break; // No unique CheckCastPP
1747 }
1748 // Search for a dominating allocation of the same inline type
1749 Node* res_dom = res;
1750 for (DUIterator_Fast jmax, j = fast_outs(jmax); j < jmax; j++) {
1751 AllocateNode* alloc_other = fast_out(j)->isa_Allocate();
1752 if (alloc_other != nullptr && alloc_other->in(AllocateNode::InlineType) == this && !alloc_other->_is_scalar_replaceable) {
1753 Node* res_other = alloc_other->result_cast();
1754 if (res_other != nullptr && res_other->is_CheckCastPP() && res_other != res_dom &&
1755 phase->is_dominator(res_other->in(0), res_dom->in(0))) {
1756 res_dom = res_other;
1757 }
1758 }
1759 }
1760 if (res_dom != res) {
1761 // Replace allocation by dominating one.
1762 replace_allocation(igvn, res, res_dom);
1763 // The result of the dominated allocation is now unused and will be removed
1764 // later in PhaseMacroExpand::eliminate_allocate_node to not confuse loop opts.
1765 igvn->_worklist.push(alloc);
1766 }
1767 }
1768 }
1769 }
1770
1771 InlineTypeNode* InlineTypeNode::make_null(PhaseGVN& gvn, ciInlineKlass* vk, bool transform) {
1772 return make_null_impl(gvn, vk, transform);
1773 }
1774
1775 InlineTypeNode* InlineTypeNode::make_null_impl(PhaseGVN& gvn, ciInlineKlass* vk, bool transform) {
1776 InlineTypeNode* vt = new InlineTypeNode(vk, gvn.zerocon(T_OBJECT), /* null_free= */ false);
1777 vt->set_is_buffered(gvn);
1778 vt->set_null_marker(gvn, gvn.intcon(0));
1779 for (uint i = 0; i < vt->field_count(); i++) {
1780 ciField* field = vt->field(i);
1781 ciType* ft = field->type();
1782 Node* value;
1783 if (field->is_flat()) {
1784 value = make_null_impl(gvn, ft->as_inline_klass());
1785 } else {
1786 value = gvn.zerocon(ft->basic_type());
1787 }
1788 vt->set_field_value(i, value);
1789 }
1790 return transform ? gvn.transform(vt)->as_InlineType() : vt;
1791 }
1792
1793 InlineTypeNode* InlineTypeNode::clone_if_required(PhaseGVN* gvn, SafePointNode* map, bool safe_for_replace) {
1794 if (!safe_for_replace || (map == nullptr && outcnt() != 0)) {
1795 return clone()->as_InlineType();
1796 }
1797 for (DUIterator_Fast imax, i = fast_outs(imax); i < imax; i++) {
1798 if (fast_out(i) != map) {
1799 return clone()->as_InlineType();
1800 }
1801 }
1802 gvn->hash_delete(this);
1803 return this;
1804 }
1805
1806 const Type* InlineTypeNode::Value(PhaseGVN* phase) const {
1807 Node* oop = get_oop();
1808 const Type* toop = phase->type(oop);
1809 #ifdef ASSERT
1810 if (oop->is_Con() && toop->is_zero_type() && _type->isa_oopptr()->is_known_instance()) {
1811 // We are not allocated (anymore) and should therefore not have an instance id
1812 dump(1);
1813 assert(false, "Unbuffered inline type should not have known instance id");
1814 }
1815 #endif
1816 if (toop == Type::TOP) {
1817 return Type::TOP;
1818 }
1819 const Type* t = toop->filter_speculative(_type);
1820 // Because of contradicting type profiling, we can end up with top as speculative type,
1821 // which would then get removed by cleanup_speculative. In this case we have to run filter_speculative
1822 // again, otherwise we would break the idempotence of Value
1823 if (t->speculative() == nullptr && toop->speculative() != nullptr) {
1824 t = toop->filter_speculative(t);
1825 }
1826 if (t->singleton()) {
1827 // Don't replace InlineType by a constant
1828 t = _type;
1829 }
1830 const Type* tinit = phase->type(in(NullMarker));
1831 if (tinit == Type::TOP) {
1832 return Type::TOP;
1833 }
1834 if (tinit->isa_int() && tinit->is_int()->is_con(1)) {
1835 t = t->join_speculative(TypePtr::NOTNULL);
1836 }
1837 return t;
1838 }
1839
1840 InlineTypeNode* LoadFlatNode::load(GraphKit* kit, ciInlineKlass* vk, Node* base, Node* ptr, bool null_free, bool trust_null_free_oop, DecoratorSet decorators) {
1841 int output_type_size = vk->nof_nonstatic_fields() + (null_free ? 0 : 1);
1842 const Type** output_types = TypeTuple::fields(output_type_size);
1843 collect_field_types(vk, output_types + TypeFunc::Parms, 0, output_type_size, null_free, trust_null_free_oop);
1844 const TypeTuple* type = TypeTuple::make(output_type_size + TypeFunc::Parms, output_types);
1845
1846 LoadFlatNode* load = new LoadFlatNode(vk, type, null_free, decorators);
1847 load->init_req(TypeFunc::Control, kit->control());
1848 load->init_req(TypeFunc::I_O, kit->top());
1849 load->init_req(TypeFunc::Memory, kit->reset_memory());
1850 load->init_req(TypeFunc::FramePtr, kit->frameptr());
1851 load->init_req(TypeFunc::ReturnAdr, kit->top());
1852
1853 load->init_req(TypeFunc::Parms, base);
1854 load->init_req(TypeFunc::Parms + 1, ptr);
1855 kit->kill_dead_locals();
1856 kit->add_safepoint_edges(load);
1857 load = kit->gvn().transform(load)->as_LoadFlat();
1858 kit->record_for_igvn(load);
1859
1860 kit->set_control(kit->gvn().transform(new ProjNode(load, TypeFunc::Control)));
1861 kit->set_all_memory(kit->gvn().transform(new ProjNode(load, TypeFunc::Memory)));
1862 return load->collect_projs(kit, vk, TypeFunc::Parms, null_free);
1863 }
1864
1865 bool LoadFlatNode::expand_constant(PhaseIterGVN& igvn, ciInstance* inst) const {
1866 precond(inst != nullptr);
1867 assert(igvn.delay_transform(), "transformation must be delayed");
1868 if ((_decorators & C2_MISMATCHED) != 0) {
1869 return false;
1870 }
1871
1872 GraphKit kit(this, igvn);
1873 for (int i = 0; i < _vk->nof_nonstatic_fields(); i++) {
1874 ProjNode* proj_out = proj_out_or_null(TypeFunc::Parms + i);
1875 if (proj_out == nullptr) {
1876 continue;
1877 }
1878
1879 ciField* field = _vk->nonstatic_field_at(i);
1880 BasicType bt = field->type()->basic_type();
1881 if (inst == nullptr) {
1882 Node* cst_node = igvn.zerocon(bt);
1883 igvn.replace_node(proj_out, cst_node);
1884 } else {
1885 bool is_unsigned_load = bt == T_BOOLEAN || bt == T_CHAR;
1886 const Type* cst_type = Type::make_constant_from_field(field, inst, bt, is_unsigned_load);
1887 Node* cst_node = igvn.makecon(cst_type);
1888 igvn.replace_node(proj_out, cst_node);
1889 }
1890 }
1891
1892 if (!_null_free) {
1893 ProjNode* proj_out = proj_out_or_null(TypeFunc::Parms + _vk->nof_nonstatic_fields());
1894 if (proj_out != nullptr) {
1895 igvn.replace_node(proj_out, igvn.intcon(1));
1896 }
1897 }
1898
1899 Node* old_ctrl = proj_out_or_null(TypeFunc::Control);
1900 if (old_ctrl != nullptr) {
1901 igvn.replace_node(old_ctrl, kit.control());
1902 }
1903 Node* old_mem = proj_out_or_null(TypeFunc::Memory);
1904 Node* new_mem = kit.reset_memory();
1905 if (old_mem != nullptr) {
1906 igvn.replace_node(old_mem, new_mem);
1907 }
1908 return true;
1909 }
1910
1911 bool LoadFlatNode::expand_non_atomic(PhaseIterGVN& igvn) const {
1912 assert(igvn.delay_transform(), "transformation must be delayed");
1913 if ((_decorators & C2_MISMATCHED) != 0) {
1914 return false;
1915 }
1916
1917 GraphKit kit(this, igvn);
1918 Node* base = this->base();
1919 Node* ptr = this->ptr();
1920
1921 for (int i = 0; i < _vk->nof_nonstatic_fields(); i++) {
1922 ProjNode* proj_out = proj_out_or_null(TypeFunc::Parms + i);
1923 if (proj_out == nullptr) {
1924 continue;
1925 }
1926
1927 ciField* field = _vk->nonstatic_field_at(i);
1928 Node* field_ptr = kit.basic_plus_adr(base, ptr, field->offset_in_bytes() - _vk->payload_offset());
1929 const TypePtr* field_ptr_type = field_ptr->Value(&igvn)->is_ptr();
1930 igvn.set_type(field_ptr, field_ptr_type);
1931
1932 Node* field_value = kit.access_load_at(base, field_ptr, field_ptr_type, igvn.type(proj_out), field->type()->basic_type(), _decorators);
1933 igvn.replace_node(proj_out, field_value);
1934 }
1935
1936 if (!_null_free) {
1937 ProjNode* proj_out = proj_out_or_null(TypeFunc::Parms + _vk->nof_nonstatic_fields());
1938 if (proj_out != nullptr) {
1939 Node* null_marker_ptr = kit.basic_plus_adr(base, ptr, _vk->null_marker_offset_in_payload());
1940 const TypePtr* null_marker_ptr_type = null_marker_ptr->Value(&igvn)->is_ptr();
1941 igvn.set_type(null_marker_ptr, null_marker_ptr_type);
1942 Node* null_marker_value = kit.access_load_at(base, null_marker_ptr, null_marker_ptr_type, TypeInt::BOOL, T_BOOLEAN, _decorators);
1943 igvn.replace_node(proj_out, null_marker_value);
1944 }
1945 }
1946
1947 Node* old_ctrl = proj_out_or_null(TypeFunc::Control);
1948 if (old_ctrl != nullptr) {
1949 igvn.replace_node(old_ctrl, kit.control());
1950 }
1951 Node* old_mem = proj_out_or_null(TypeFunc::Memory);
1952 Node* new_mem = kit.reset_memory();
1953 if (old_mem != nullptr) {
1954 igvn.replace_node(old_mem, new_mem);
1955 }
1956 return true;
1957 }
1958
1959 void LoadFlatNode::expand_atomic(PhaseIterGVN& igvn) const {
1960 assert(igvn.delay_transform(), "transformation must be delayed");
1961 GraphKit kit(this, igvn);
1962 Node* base = this->base();
1963 Node* ptr = this->ptr();
1964
1965 BasicType payload_bt = _vk->atomic_size_to_basic_type(_null_free);
1966 kit.insert_mem_bar(Op_MemBarCPUOrder);
1967 Node* payload = kit.access_load_at(base, ptr, TypeRawPtr::BOTTOM, Type::get_const_basic_type(payload_bt), payload_bt,
1968 _decorators | C2_MISMATCHED | C2_CONTROL_DEPENDENT_LOAD | C2_UNKNOWN_CONTROL_LOAD, kit.control());
1969 kit.insert_mem_bar(Op_MemBarCPUOrder);
1970
1971 Node* old_ctrl = proj_out_or_null(TypeFunc::Control);
1972 if (old_ctrl != nullptr) {
1973 igvn.replace_node(old_ctrl, kit.control());
1974 }
1975 Node* old_mem = proj_out_or_null(TypeFunc::Memory);
1976 Node* new_mem = kit.reset_memory();
1977 if (old_mem != nullptr) {
1978 igvn.replace_node(old_mem, new_mem);
1979 }
1980
1981 expand_projs_atomic(igvn, kit.control(), payload);
1982 }
1983
1984 void LoadFlatNode::collect_field_types(ciInlineKlass* vk, const Type** field_types, int idx, int limit, bool null_free, bool trust_null_free_oop) {
1985 assert(null_free || !trust_null_free_oop, "cannot trust null-free oop when the holder object is not null-free");
1986 for (int i = 0; i < vk->nof_declared_nonstatic_fields(); i++) {
1987 ciField* field = vk->declared_nonstatic_field_at(i);
1988 if (field->is_flat()) {
1989 ciInlineKlass* field_klass = field->type()->as_inline_klass();
1990 collect_field_types(field_klass, field_types, idx, limit, field->is_null_free(), trust_null_free_oop && field->is_null_free());
1991 idx += field_klass->nof_nonstatic_fields() + (field->is_null_free() ? 0 : 1);
1992 continue;
1993 }
1994
1995 const Type* field_type = Type::get_const_type(field->type());
1996 if (trust_null_free_oop && field->is_null_free()) {
1997 field_type = field_type->filter(TypePtr::NOTNULL);
1998 }
1999
2000 assert(idx >= 0 && idx < limit, "field type out of bounds, %d - %d", idx, limit);
2001 field_types[idx] = field_type;
2002 idx++;
2003 }
2004
2005 if (!null_free) {
2006 assert(idx >= 0 && idx < limit, "field type out of bounds, %d - %d", idx, limit);
2007 field_types[idx] = TypeInt::BOOL;
2008 }
2009 }
2010
2011 // Create an InlineTypeNode from a LoadFlatNode with its fields being extracted from the
2012 // LoadFlatNode
2013 InlineTypeNode* LoadFlatNode::collect_projs(GraphKit* kit, ciInlineKlass* vk, int proj_con, bool null_free) {
2014 PhaseGVN& gvn = kit->gvn();
2015 InlineTypeNode* res = InlineTypeNode::make_uninitialized(gvn, vk, null_free);
2016 for (int i = 0; i < vk->nof_declared_nonstatic_fields(); i++) {
2017 ciField* field = vk->declared_nonstatic_field_at(i);
2018 Node* field_value;
2019 if (field->is_flat()) {
2020 ciInlineKlass* field_klass = field->type()->as_inline_klass();
2021 field_value = collect_projs(kit, field_klass, proj_con, field->is_null_free());
2022 proj_con += field_klass->nof_nonstatic_fields() + (field->is_null_free() ? 0 : 1);
2023 } else {
2024 field_value = gvn.transform(new ProjNode(this, proj_con));
2025 proj_con++;
2026 }
2027 res->set_field_value(i, field_value);
2028 }
2029
2030 if (null_free) {
2031 res->set_null_marker(gvn);
2032 } else {
2033 res->set_null_marker(gvn, gvn.transform(new ProjNode(this, proj_con)));
2034 }
2035 return gvn.transform(res)->as_InlineType();
2036 }
2037
2038 // Extract the values of the flattened fields from the loaded payload
2039 void LoadFlatNode::expand_projs_atomic(PhaseIterGVN& igvn, Node* ctrl, Node* payload) const {
2040 BasicType payload_bt = _vk->atomic_size_to_basic_type(_null_free);
2041 for (int i = 0; i < _vk->nof_nonstatic_fields(); i++) {
2042 ProjNode* proj_out = proj_out_or_null(TypeFunc::Parms + i);
2043 if (proj_out == nullptr) {
2044 continue;
2045 }
2046
2047 ciField* field = _vk->nonstatic_field_at(i);
2048 int field_offset = field->offset_in_bytes() - _vk->payload_offset();
2049 const Type* field_type = igvn.type(proj_out);
2050 Node* field_value = get_payload_value(igvn, ctrl, payload_bt, payload, field_type, field->type()->basic_type(), field_offset);
2051 igvn.replace_node(proj_out, field_value);
2052 }
2053
2054 if (!_null_free) {
2055 ProjNode* proj_out = proj_out_or_null(TypeFunc::Parms + _vk->nof_nonstatic_fields());
2056 if (proj_out == nullptr) {
2057 return;
2058 }
2059
2060 int null_marker_offset = _vk->null_marker_offset_in_payload();
2061 Node* null_marker_value = get_payload_value(igvn, ctrl, payload_bt, payload, TypeInt::BOOL, T_BOOLEAN, null_marker_offset);
2062 igvn.replace_node(proj_out, null_marker_value);
2063 }
2064 }
2065
2066 Node* LoadFlatNode::get_payload_value(PhaseIterGVN& igvn, Node* ctrl, BasicType payload_bt, Node* payload, const Type* value_type, BasicType value_bt, int offset) {
2067 assert((offset + type2aelembytes(value_bt)) <= type2aelembytes(payload_bt), "Value does not fit into payload");
2068 Node* value = nullptr;
2069 // Shift to the right position in the long value
2070 Node* shift_val = igvn.intcon(offset << LogBitsPerByte);
2071 if (payload_bt == T_LONG) {
2072 value = igvn.transform(new URShiftLNode(payload, shift_val));
2073 value = igvn.transform(new ConvL2INode(value));
2074 } else {
2075 value = igvn.transform(new URShiftINode(payload, shift_val));
2076 }
2077
2078 if (value_bt == T_INT) {
2079 return value;
2080 } else if (!is_java_primitive(value_bt)) {
2081 assert(UseCompressedOops && payload_bt == T_LONG, "Naturally atomic");
2082 value = igvn.transform(new CastI2NNode(ctrl, value, value_type->make_narrowoop()));
2083 value = igvn.transform(new DecodeNNode(value, value_type));
2084
2085 // Similar to CheckCastPP nodes with raw input, CastI2N nodes require special handling in 'PhaseCFG::schedule_late' to ensure the
2086 // register allocator does not move the CastI2N below a safepoint. This is necessary to avoid having the raw pointer span a safepoint,
2087 // making it opaque to the GC. Unlike CheckCastPPs, which need extra handling in 'Scheduling::ComputeRegisterAntidependencies' due to
2088 // scalarization, CastI2N nodes are always used by a load if scalarization happens which inherently keeps them pinned above the safepoint.
2089 return value;
2090 } else {
2091 // Make sure to zero unused bits in the 32-bit value
2092 return Compile::narrow_value(value_bt, value, value_type, &igvn, true);
2093 }
2094 }
2095
2096 void StoreFlatNode::store(GraphKit* kit, Node* base, Node* ptr, InlineTypeNode* value, bool null_free, DecoratorSet decorators) {
2097 value = value->allocate_fields(kit);
2098 StoreFlatNode* store = new StoreFlatNode(null_free, decorators);
2099 store->init_req(TypeFunc::Control, kit->control());
2100 store->init_req(TypeFunc::I_O, kit->top());
2101 store->init_req(TypeFunc::Memory, kit->reset_memory());
2102 store->init_req(TypeFunc::FramePtr, kit->frameptr());
2103 store->init_req(TypeFunc::ReturnAdr, kit->top());
2104
2105 store->init_req(TypeFunc::Parms, base);
2106 store->init_req(TypeFunc::Parms + 1, ptr);
2107 store->init_req(TypeFunc::Parms + 2, value);
2108 kit->kill_dead_locals();
2109 kit->add_safepoint_edges(store);
2110 store = kit->gvn().transform(store)->as_StoreFlat();
2111 kit->record_for_igvn(store);
2112
2113 kit->set_control(kit->gvn().transform(new ProjNode(store, TypeFunc::Control)));
2114 kit->set_all_memory(kit->gvn().transform(new ProjNode(store, TypeFunc::Memory)));
2115 }
2116
2117 bool StoreFlatNode::expand_non_atomic(PhaseIterGVN& igvn) const {
2118 assert(igvn.delay_transform(), "transformation must be delayed");
2119 if ((_decorators & C2_MISMATCHED) != 0) {
2120 return false;
2121 }
2122
2123 GraphKit kit(this, igvn);
2124 Node* base = this->base();
2125 Node* ptr = this->ptr();
2126 InlineTypeNode* value = this->value();
2127
2128 ciInlineKlass* vk = igvn.type(value)->inline_klass();
2129 for (int i = 0; i < vk->nof_nonstatic_fields(); i++) {
2130 ciField* field = vk->nonstatic_field_at(i);
2131 Node* field_ptr = kit.basic_plus_adr(base, ptr, field->offset_in_bytes() - vk->payload_offset());
2132 const TypePtr* field_ptr_type = field_ptr->Value(&igvn)->is_ptr();
2133 igvn.set_type(field_ptr, field_ptr_type);
2134 Node* field_value = value->field_value_by_offset(field->offset_in_bytes(), true);
2135 kit.access_store_at(base, field_ptr, field_ptr_type, field_value, igvn.type(field_value), field->type()->basic_type(), _decorators);
2136 }
2137
2138 if (!_null_free) {
2139 Node* null_marker_ptr = kit.basic_plus_adr(base, ptr, vk->null_marker_offset_in_payload());
2140 const TypePtr* null_marker_ptr_type = null_marker_ptr->Value(&igvn)->is_ptr();
2141 igvn.set_type(null_marker_ptr, null_marker_ptr_type);
2142 Node* null_marker_value = value->get_null_marker();
2143 kit.access_store_at(base, null_marker_ptr, null_marker_ptr_type, null_marker_value, TypeInt::BOOL, T_BOOLEAN, _decorators);
2144 }
2145
2146 Node* old_ctrl = proj_out_or_null(TypeFunc::Control);
2147 if (old_ctrl != nullptr) {
2148 igvn.replace_node(old_ctrl, kit.control());
2149 }
2150 Node* old_mem = proj_out_or_null(TypeFunc::Memory);
2151 Node* new_mem = kit.reset_memory();
2152 if (old_mem != nullptr) {
2153 igvn.replace_node(old_mem, new_mem);
2154 }
2155 return true;
2156 }
2157
2158 void StoreFlatNode::expand_atomic(PhaseIterGVN& igvn) const {
2159 // Convert to a payload value <= 64-bit and write atomically.
2160 // The payload might contain at most two oop fields that must be narrow because otherwise they would be 64-bit
2161 // in size and would then be written by a "normal" oop store. If the payload contains oops, its size is always
2162 // 64-bit because the next smaller (power-of-two) size would be 32-bit which could only hold one narrow oop that
2163 // would then be written by a normal narrow oop store. These properties are asserted in 'convert_to_payload'.
2164 assert(igvn.delay_transform(), "transformation must be delayed");
2165 GraphKit kit(this, igvn);
2166 Node* base = this->base();
2167 Node* ptr = this->ptr();
2168 InlineTypeNode* value = this->value();
2169
2170 int oop_off_1 = -1;
2171 int oop_off_2 = -1;
2172 Node* payload = convert_to_payload(igvn, kit.control(), value, _null_free, oop_off_1, oop_off_2);
2173
2174 ciInlineKlass* vk = igvn.type(value)->inline_klass();
2175 assert(oop_off_1 == -1 || oop_off_1 == 0 || oop_off_1 == 4, "invalid layout for %s, first oop at offset %d", vk->name()->as_utf8(), oop_off_1);
2176 assert(oop_off_2 == -1 || oop_off_2 == 4, "invalid layout for %s, second oop at offset %d", vk->name()->as_utf8(), oop_off_2);
2177 BasicType payload_bt = vk->atomic_size_to_basic_type(_null_free);
2178 kit.insert_mem_bar(Op_MemBarCPUOrder);
2179 if (!UseG1GC || oop_off_1 == -1) {
2180 // No oop fields or no late barrier expansion. Emit an atomic store of the payload and add GC barriers if needed.
2181 assert(oop_off_2 == -1 || !UseG1GC, "sanity");
2182 // ZGC does not support compressed oops, so only one oop can be in the payload which is written by a "normal" oop store.
2183 assert((oop_off_1 == -1 && oop_off_2 == -1) || !UseZGC, "ZGC does not support embedded oops in flat fields");
2184 kit.access_store_at(base, ptr, TypeRawPtr::BOTTOM, payload, Type::get_const_basic_type(payload_bt), payload_bt, _decorators | C2_MISMATCHED, true, value);
2185 } else {
2186 // Contains oops and requires late barrier expansion. Emit a special store node that allows to emit GC barriers in the backend.
2187 assert(UseG1GC, "Unexpected GC");
2188 assert(payload_bt == T_LONG, "Unexpected payload type");
2189 // If one oop, set the offset (if no offset is set, two oops are assumed by the backend)
2190 Node* oop_offset = (oop_off_2 == -1) ? igvn.intcon(oop_off_1) : nullptr;
2191 Node* mem = kit.reset_memory();
2192 kit.set_all_memory(mem);
2193 Node* store = igvn.transform(new StoreLSpecialNode(kit.control(), mem, ptr, TypeRawPtr::BOTTOM, payload, oop_offset, MemNode::unordered));
2194 kit.set_memory(store, TypeRawPtr::BOTTOM);
2195 }
2196 kit.insert_mem_bar(Op_MemBarCPUOrder);
2197
2198 Node* old_ctrl = proj_out_or_null(TypeFunc::Control);
2199 if (old_ctrl != nullptr) {
2200 igvn.replace_node(old_ctrl, kit.control());
2201 }
2202 Node* old_mem = proj_out_or_null(TypeFunc::Memory);
2203 Node* new_mem = kit.reset_memory();
2204 if (old_mem != nullptr) {
2205 igvn.replace_node(old_mem, new_mem);
2206 }
2207 }
2208
2209 // Convert the field values to a payload value of type 'bt'
2210 Node* StoreFlatNode::convert_to_payload(PhaseIterGVN& igvn, Node* ctrl, InlineTypeNode* value, bool null_free, int& oop_off_1, int& oop_off_2) {
2211 ciInlineKlass* vk = igvn.type(value)->inline_klass();
2212 BasicType payload_bt = vk->atomic_size_to_basic_type(null_free);
2213 Node* payload = igvn.zerocon(payload_bt);
2214 if (!null_free) {
2215 // Set the null marker
2216 payload = set_payload_value(igvn, payload_bt, payload, T_BOOLEAN, value->get_null_marker(), vk->null_marker_offset_in_payload());
2217 }
2218
2219 // Iterate over the fields and add their values to the payload
2220 for (int i = 0; i < vk->nof_nonstatic_fields(); i++) {
2221 ciField* field = vk->nonstatic_field_at(i);
2222 Node* field_value = value->field_value_by_offset(field->offset_in_bytes(), true);
2223 ciType* field_klass = field->type();
2224 BasicType field_bt = field_klass->basic_type();
2225 int field_offset_in_payload = field->offset_in_bytes() - vk->payload_offset();
2226 if (!field_klass->is_primitive_type()) {
2227 // Narrow oop field
2228 assert(UseCompressedOops && payload_bt == T_LONG, "Naturally atomic");
2229 if (oop_off_1 == -1) {
2230 oop_off_1 = field_offset_in_payload;
2231 } else {
2232 assert(oop_off_2 == -1, "already set");
2233 oop_off_2 = field_offset_in_payload;
2234 }
2235
2236 const Type* val_type = Type::get_const_type(field_klass)->make_narrowoop();
2237 if (field_value->is_InlineType()) {
2238 assert(field_value->as_InlineType()->is_allocated(&igvn), "must be allocated");
2239 }
2240
2241 field_value = igvn.transform(new EncodePNode(field_value, val_type));
2242 field_value = igvn.transform(new CastP2XNode(ctrl, field_value));
2243 field_value = igvn.transform(new ConvL2INode(field_value));
2244 field_bt = T_INT;
2245 }
2246 payload = set_payload_value(igvn, payload_bt, payload, field_bt, field_value, field_offset_in_payload);
2247 }
2248
2249 return payload;
2250 }
2251
2252 Node* StoreFlatNode::set_payload_value(PhaseIterGVN& igvn, BasicType payload_bt, Node* payload, BasicType val_bt, Node* value, int offset) {
2253 assert((offset + type2aelembytes(val_bt)) <= type2aelembytes(payload_bt), "Value does not fit into payload");
2254
2255 // Make sure to zero unused bits in the 32-bit value
2256 if (val_bt == T_BYTE || val_bt == T_BOOLEAN) {
2257 value = igvn.transform(new AndINode(value, igvn.intcon(0xFF)));
2258 } else if (val_bt == T_CHAR || val_bt == T_SHORT) {
2259 value = igvn.transform(new AndINode(value, igvn.intcon(0xFFFF)));
2260 } else if (val_bt == T_FLOAT) {
2261 value = igvn.transform(new MoveF2INode(value));
2262 } else {
2263 assert(val_bt == T_INT, "Unsupported type: %s", type2name(val_bt));
2264 }
2265
2266 Node* shift_val = igvn.intcon(offset << LogBitsPerByte);
2267 if (payload_bt == T_LONG) {
2268 // Convert to long and remove the sign bit (the backend will fold this and emit a zero extend i2l)
2269 value = igvn.transform(new ConvI2LNode(value));
2270 value = igvn.transform(new AndLNode(value, igvn.longcon(0xFFFFFFFF)));
2271
2272 Node* shift_value = igvn.transform(new LShiftLNode(value, shift_val));
2273 payload = new OrLNode(shift_value, payload);
2274 } else {
2275 Node* shift_value = igvn.transform(new LShiftINode(value, shift_val));
2276 payload = new OrINode(shift_value, payload);
2277 }
2278 return igvn.transform(payload);
2279 }
2280
2281 const Type* LoadFlatNode::Value(PhaseGVN* phase) const {
2282 if (phase->type(in(TypeFunc::Control)) == Type::TOP || phase->type(in(TypeFunc::Memory)) == Type::TOP ||
2283 phase->type(base()) == Type::TOP || phase->type(ptr()) == Type::TOP) {
2284 return Type::TOP;
2285 }
2286 return bottom_type();
2287 }
2288
2289 const Type* StoreFlatNode::Value(PhaseGVN* phase) const {
2290 if (phase->type(in(TypeFunc::Control)) == Type::TOP || phase->type(in(TypeFunc::Memory)) == Type::TOP ||
2291 phase->type(base()) == Type::TOP || phase->type(ptr()) == Type::TOP || phase->type(value()) == Type::TOP) {
2292 return Type::TOP;
2293 }
2294 return bottom_type();
2295 }
2296
2297 bool LoadFlatNode::is_mismatched() const {
2298 return (_decorators & C2_MISMATCHED) != 0;
2299 }
2300
2301 bool StoreFlatNode::is_mismatched() const {
2302 return (_decorators & C2_MISMATCHED) != 0;
2303 }