1 /* 2 * Copyright (c) 2017, 2020, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 * 23 */ 24 25 #include "precompiled.hpp" 26 #include "ci/ciInlineKlass.hpp" 27 #include "gc/shared/barrierSet.hpp" 28 #include "gc/shared/gc_globals.hpp" 29 #include "opto/addnode.hpp" 30 #include "opto/castnode.hpp" 31 #include "opto/graphKit.hpp" 32 #include "opto/inlinetypenode.hpp" 33 #include "opto/rootnode.hpp" 34 #include "opto/phaseX.hpp" 35 36 // Clones the inline type to handle control flow merges involving multiple inline types. 37 // The inputs are replaced by PhiNodes to represent the merged values for the given region. 38 InlineTypeBaseNode* InlineTypeBaseNode::clone_with_phis(PhaseGVN* gvn, Node* region, bool is_init) { 39 InlineTypeBaseNode* vt = clone()->as_InlineTypeBase(); 40 if (vt->is_InlineTypePtr()) { 41 // Use nullable type 42 const Type* t = Type::get_const_type(inline_klass()); 43 gvn->set_type(vt, t); 44 vt->as_InlineTypePtr()->set_type(t); 45 } 46 47 // Create a PhiNode for merging the oop values 48 const Type* phi_type = Type::get_const_type(inline_klass()); 49 PhiNode* oop = PhiNode::make(region, vt->get_oop(), phi_type); 50 gvn->set_type(oop, phi_type); 51 gvn->record_for_igvn(oop); 52 vt->set_oop(oop); 53 54 // Create a PhiNode for merging the is_init values 55 Node* is_init_node; 56 if (is_init) { 57 is_init_node = gvn->intcon(1); 58 } else { 59 phi_type = Type::get_const_basic_type(T_BOOLEAN); 60 is_init_node = PhiNode::make(region, vt->get_is_init(), phi_type); 61 gvn->set_type(is_init_node, phi_type); 62 gvn->record_for_igvn(is_init_node); 63 } 64 vt->set_req(IsInit, is_init_node); 65 66 // Create a PhiNode each for merging the field values 67 for (uint i = 0; i < vt->field_count(); ++i) { 68 ciType* type = vt->field_type(i); 69 Node* value = vt->field_value(i); 70 if (value->is_InlineTypeBase()) { 71 // Handle inline type fields recursively 72 value = value->as_InlineTypeBase()->clone_with_phis(gvn, region); 73 } else { 74 phi_type = Type::get_const_type(type); 75 value = PhiNode::make(region, value, phi_type); 76 gvn->set_type(value, phi_type); 77 gvn->record_for_igvn(value); 78 } 79 vt->set_field_value(i, value); 80 } 81 gvn->set_type(vt, vt->bottom_type()); 82 gvn->record_for_igvn(vt); 83 return vt; 84 } 85 86 // Checks if the inputs of the InlineTypeBaseTypeNode were replaced by PhiNodes 87 // for the given region (see InlineTypeBaseTypeNode::clone_with_phis). 88 bool InlineTypeBaseNode::has_phi_inputs(Node* region) { 89 // Check oop input 90 bool result = get_oop()->is_Phi() && get_oop()->as_Phi()->region() == region; 91 #ifdef ASSERT 92 if (result) { 93 // Check all field value inputs for consistency 94 for (uint i = Values; i < field_count(); ++i) { 95 Node* n = in(i); 96 if (n->is_InlineTypeBase()) { 97 assert(n->as_InlineTypeBase()->has_phi_inputs(region), "inconsistent phi inputs"); 98 } else { 99 assert(n->is_Phi() && n->as_Phi()->region() == region, "inconsistent phi inputs"); 100 } 101 } 102 } 103 #endif 104 return result; 105 } 106 107 // Check if all inline type fields have inline type node values 108 bool InlineTypeBaseNode::can_merge() { 109 for (uint i = 0; i < field_count(); ++i) { 110 ciType* type = field_type(i); 111 Node* val = field_value(i); 112 if (type->is_inlinetype() && 113 (!val->is_InlineTypeBase() || !val->as_InlineTypeBase()->can_merge())) { 114 return false; 115 } 116 } 117 return true; 118 } 119 120 // Merges 'this' with 'other' by updating the input PhiNodes added by 'clone_with_phis' 121 InlineTypeBaseNode* InlineTypeBaseNode::merge_with(PhaseGVN* gvn, const InlineTypeBaseNode* other, int pnum, bool transform) { 122 // Merge oop inputs 123 PhiNode* phi = get_oop()->as_Phi(); 124 phi->set_req(pnum, other->get_oop()); 125 if (transform) { 126 set_oop(gvn->transform(phi)); 127 } 128 129 Node* is_init = get_is_init(); 130 if (is_init->is_Phi()) { 131 phi = is_init->as_Phi(); 132 phi->set_req(pnum, other->get_is_init()); 133 if (transform) { 134 set_req(IsInit, gvn->transform(phi)); 135 } 136 } else { 137 assert(is_init->find_int_con(0) == 1, "only with a non null inline type"); 138 } 139 140 // Merge field values 141 for (uint i = 0; i < field_count(); ++i) { 142 Node* val1 = field_value(i); 143 Node* val2 = other->field_value(i); 144 if (val1->is_InlineTypeBase()) { 145 val1->as_InlineTypeBase()->merge_with(gvn, val2->as_InlineTypeBase(), pnum, transform); 146 } else { 147 assert(val1->is_Phi(), "must be a phi node"); 148 val1->set_req(pnum, val2); 149 } 150 if (transform) { 151 set_field_value(i, gvn->transform(val1)); 152 } 153 } 154 return this; 155 } 156 157 // Adds a new merge path to an inline type node with phi inputs 158 void InlineTypeBaseNode::add_new_path(Node* region) { 159 assert(has_phi_inputs(region), "must have phi inputs"); 160 161 PhiNode* phi = get_oop()->as_Phi(); 162 phi->add_req(NULL); 163 assert(phi->req() == region->req(), "must be same size as region"); 164 165 phi = get_is_init()->as_Phi(); 166 phi->add_req(NULL); 167 assert(phi->req() == region->req(), "must be same size as region"); 168 169 for (uint i = 0; i < field_count(); ++i) { 170 Node* val = field_value(i); 171 if (val->is_InlineTypeBase()) { 172 val->as_InlineTypeBase()->add_new_path(region); 173 } else { 174 val->as_Phi()->add_req(NULL); 175 assert(val->req() == region->req(), "must be same size as region"); 176 } 177 } 178 } 179 180 Node* InlineTypeBaseNode::field_value(uint index) const { 181 assert(index < field_count(), "index out of bounds"); 182 return in(Values + index); 183 } 184 185 // Get the value of the field at the given offset. 186 // If 'recursive' is true, flattened inline type fields will be resolved recursively. 187 Node* InlineTypeBaseNode::field_value_by_offset(int offset, bool recursive) const { 188 // If the field at 'offset' belongs to a flattened inline type field, 'index' refers to the 189 // corresponding InlineTypeNode input and 'sub_offset' is the offset in flattened inline type. 190 int index = inline_klass()->field_index_by_offset(offset); 191 int sub_offset = offset - field_offset(index); 192 Node* value = field_value(index); 193 assert(value != NULL, "field value not found"); 194 if (recursive && value->is_InlineTypeBase()) { 195 if (field_is_flattened(index)) { 196 // Flattened inline type field 197 InlineTypeBaseNode* vt = value->as_InlineTypeBase(); 198 sub_offset += vt->inline_klass()->first_field_offset(); // Add header size 199 return vt->field_value_by_offset(sub_offset, recursive); 200 } else { 201 assert(sub_offset == 0, "should not have a sub offset"); 202 return value; 203 } 204 } 205 assert(!(recursive && value->is_InlineTypeBase()), "should not be an inline type"); 206 assert(sub_offset == 0, "offset mismatch"); 207 return value; 208 } 209 210 void InlineTypeBaseNode::set_field_value(uint index, Node* value) { 211 assert(index < field_count(), "index out of bounds"); 212 set_req(Values + index, value); 213 } 214 215 void InlineTypeBaseNode::set_field_value_by_offset(int offset, Node* value) { 216 set_field_value(field_index(offset), value); 217 } 218 219 int InlineTypeBaseNode::field_offset(uint index) const { 220 assert(index < field_count(), "index out of bounds"); 221 return inline_klass()->declared_nonstatic_field_at(index)->offset(); 222 } 223 224 uint InlineTypeBaseNode::field_index(int offset) const { 225 uint i = 0; 226 for (; i < field_count() && field_offset(i) != offset; i++) { } 227 assert(i < field_count(), "field not found"); 228 return i; 229 } 230 231 ciType* InlineTypeBaseNode::field_type(uint index) const { 232 assert(index < field_count(), "index out of bounds"); 233 return inline_klass()->declared_nonstatic_field_at(index)->type(); 234 } 235 236 bool InlineTypeBaseNode::field_is_flattened(uint index) const { 237 assert(index < field_count(), "index out of bounds"); 238 ciField* field = inline_klass()->declared_nonstatic_field_at(index); 239 assert(!field->is_flattened() || field->type()->is_inlinetype(), "must be an inline type"); 240 return field->is_flattened(); 241 } 242 243 bool InlineTypeBaseNode::field_is_null_free(uint index) const { 244 assert(index < field_count(), "index out of bounds"); 245 ciField* field = inline_klass()->declared_nonstatic_field_at(index); 246 assert(!field->is_flattened() || field->type()->is_inlinetype(), "must be an inline type"); 247 return field->is_null_free(); 248 } 249 250 void InlineTypeBaseNode::make_scalar_in_safepoint(PhaseIterGVN* igvn, Unique_Node_List& worklist, SafePointNode* sfpt) { 251 ciInlineKlass* vk = inline_klass(); 252 uint nfields = vk->nof_nonstatic_fields(); 253 JVMState* jvms = sfpt->jvms(); 254 // Replace safepoint edge by SafePointScalarObjectNode and add field values 255 assert(jvms != NULL, "missing JVMS"); 256 uint first_ind = (sfpt->req() - jvms->scloff()); 257 SafePointScalarObjectNode* sobj = new SafePointScalarObjectNode(inline_ptr(), 258 #ifdef ASSERT 259 NULL, 260 #endif 261 first_ind, nfields); 262 sobj->init_req(0, igvn->C->root()); 263 // Nullable inline types have an IsInit field that needs 264 // to be checked before using the field values. 265 if (!igvn->type(get_is_init())->is_int()->is_con(1)) { 266 sfpt->add_req(get_is_init()); 267 } else { 268 sfpt->add_req(igvn->C->top()); 269 } 270 // Iterate over the inline type fields in order of increasing 271 // offset and add the field values to the safepoint. 272 for (uint j = 0; j < nfields; ++j) { 273 int offset = vk->nonstatic_field_at(j)->offset(); 274 Node* value = field_value_by_offset(offset, true /* include flattened inline type fields */); 275 if (value->is_InlineTypeBase()) { 276 // Add inline type field to the worklist to process later 277 worklist.push(value); 278 } 279 sfpt->add_req(value); 280 } 281 jvms->set_endoff(sfpt->req()); 282 sobj = igvn->transform(sobj)->as_SafePointScalarObject(); 283 igvn->rehash_node_delayed(sfpt); 284 for (uint i = jvms->debug_start(); i < jvms->debug_end(); i++) { 285 Node* debug = sfpt->in(i); 286 if (debug != NULL && debug->uncast() == this) { 287 sfpt->set_req(i, sobj); 288 } 289 } 290 } 291 292 void InlineTypeBaseNode::make_scalar_in_safepoints(PhaseIterGVN* igvn, bool allow_oop) { 293 // If the inline type has a constant or loaded oop, use the oop instead of scalarization 294 // in the safepoint to avoid keeping field loads live just for the debug info. 295 Node* oop = get_oop(); 296 bool use_oop = allow_oop && (is_InlineTypePtr() || is_allocated(igvn)) && 297 (oop->is_Con() || oop->is_Parm() || oop->is_Load() || (oop->isa_DecodeN() && oop->in(1)->is_Load())); 298 299 ResourceMark rm; 300 Unique_Node_List safepoints; 301 Unique_Node_List vt_worklist; 302 Unique_Node_List worklist; 303 worklist.push(this); 304 while (worklist.size() > 0) { 305 Node* n = worklist.pop(); 306 for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) { 307 Node* use = n->fast_out(i); 308 if (use->is_SafePoint() && !use->is_CallLeaf() && (!use->is_Call() || use->as_Call()->has_debug_use(n))) { 309 safepoints.push(use); 310 } else if (use->is_ConstraintCast()) { 311 worklist.push(use); 312 } 313 } 314 } 315 316 // Process all safepoint uses and scalarize inline type 317 while (safepoints.size() > 0) { 318 SafePointNode* sfpt = safepoints.pop()->as_SafePoint(); 319 if (use_oop) { 320 for (uint i = sfpt->jvms()->debug_start(); i < sfpt->jvms()->debug_end(); i++) { 321 Node* debug = sfpt->in(i); 322 if (debug != NULL && debug->uncast() == this) { 323 sfpt->set_req(i, get_oop()); 324 } 325 } 326 igvn->rehash_node_delayed(sfpt); 327 } else { 328 make_scalar_in_safepoint(igvn, vt_worklist, sfpt); 329 } 330 } 331 // Now scalarize non-flattened fields 332 for (uint i = 0; i < vt_worklist.size(); ++i) { 333 InlineTypeBaseNode* vt = vt_worklist.at(i)->isa_InlineTypeBase(); 334 vt->make_scalar_in_safepoints(igvn); 335 } 336 if (outcnt() == 0) { 337 igvn->_worklist.push(this); 338 } 339 } 340 341 const TypePtr* InlineTypeBaseNode::field_adr_type(Node* base, int offset, ciInstanceKlass* holder, DecoratorSet decorators, PhaseGVN& gvn) const { 342 const TypeAryPtr* ary_type = gvn.type(base)->isa_aryptr(); 343 const TypePtr* adr_type = NULL; 344 bool is_array = ary_type != NULL; 345 if ((decorators & C2_MISMATCHED) != 0) { 346 adr_type = TypeRawPtr::BOTTOM; 347 } else if (is_array) { 348 // In the case of a flattened inline type array, each field has its own slice 349 adr_type = ary_type->with_field_offset(offset)->add_offset(Type::OffsetBot); 350 } else { 351 ciField* field = holder->get_field_by_offset(offset, false); 352 assert(field != NULL, "field not found"); 353 adr_type = gvn.C->alias_type(field)->adr_type(); 354 } 355 return adr_type; 356 } 357 358 void InlineTypeBaseNode::load(GraphKit* kit, Node* base, Node* ptr, ciInstanceKlass* holder, int holder_offset, DecoratorSet decorators) { 359 // Initialize the inline type by loading its field values from 360 // memory and adding the values as input edges to the node. 361 for (uint i = 0; i < field_count(); ++i) { 362 int offset = holder_offset + field_offset(i); 363 Node* value = NULL; 364 ciType* ft = field_type(i); 365 bool null_free = field_is_null_free(i); 366 if (null_free && ft->as_inline_klass()->is_empty()) { 367 // Loading from a field of an empty inline type. Just return the default instance. 368 value = InlineTypeNode::make_default(kit->gvn(), ft->as_inline_klass()); 369 } else if (field_is_flattened(i)) { 370 // Recursively load the flattened inline type field 371 value = InlineTypeNode::make_from_flattened(kit, ft->as_inline_klass(), base, ptr, holder, offset, decorators); 372 } else { 373 const TypeOopPtr* oop_ptr = kit->gvn().type(base)->isa_oopptr(); 374 bool is_array = (oop_ptr->isa_aryptr() != NULL); 375 bool mismatched = (decorators & C2_MISMATCHED) != 0; 376 if (base->is_Con() && !is_array && !mismatched) { 377 // If the oop to the inline type is constant (static final field), we can 378 // also treat the fields as constants because the inline type is immutable. 379 ciObject* constant_oop = oop_ptr->const_oop(); 380 ciField* field = holder->get_field_by_offset(offset, false); 381 assert(field != NULL, "field not found"); 382 ciConstant constant = constant_oop->as_instance()->field_value(field); 383 const Type* con_type = Type::make_from_constant(constant, /*require_const=*/ true); 384 assert(con_type != NULL, "type not found"); 385 value = kit->gvn().transform(kit->makecon(con_type)); 386 // Check type of constant which might be more precise than the static field type 387 if (con_type->is_inlinetypeptr() && !con_type->is_zero_type()) { 388 ft = con_type->inline_klass(); 389 null_free = true; 390 } 391 } else { 392 // Load field value from memory 393 const TypePtr* adr_type = field_adr_type(base, offset, holder, decorators, kit->gvn()); 394 Node* adr = kit->basic_plus_adr(base, ptr, offset); 395 BasicType bt = type2field[ft->basic_type()]; 396 assert(is_java_primitive(bt) || adr->bottom_type()->is_ptr_to_narrowoop() == UseCompressedOops, "inconsistent"); 397 const Type* val_type = Type::get_const_type(ft); 398 if (is_array) { 399 decorators |= IS_ARRAY; 400 } 401 value = kit->access_load_at(base, adr, adr_type, val_type, bt, decorators); 402 } 403 // Loading a non-flattened inline type from memory 404 if (ft->is_inlinetype()) { 405 value = InlineTypeNode::make_from_oop(kit, value, ft->as_inline_klass(), null_free); 406 } 407 } 408 set_field_value(i, value); 409 } 410 } 411 412 void InlineTypeBaseNode::store_flattened(GraphKit* kit, Node* base, Node* ptr, ciInstanceKlass* holder, int holder_offset, DecoratorSet decorators) const { 413 if (kit->gvn().type(base)->isa_aryptr()) { 414 kit->C->set_flattened_accesses(); 415 } 416 // The inline type is embedded into the object without an oop header. Subtract the 417 // offset of the first field to account for the missing header when storing the values. 418 if (holder == NULL) { 419 holder = inline_klass(); 420 } 421 holder_offset -= inline_klass()->first_field_offset(); 422 store(kit, base, ptr, holder, holder_offset, decorators); 423 } 424 425 void InlineTypeBaseNode::store(GraphKit* kit, Node* base, Node* ptr, ciInstanceKlass* holder, int holder_offset, DecoratorSet decorators) const { 426 // Write field values to memory 427 for (uint i = 0; i < field_count(); ++i) { 428 int offset = holder_offset + field_offset(i); 429 Node* value = field_value(i); 430 ciType* ft = field_type(i); 431 if (field_is_flattened(i)) { 432 // Recursively store the flattened inline type field 433 if (!value->is_InlineTypeBase()) { 434 value = InlineTypeNode::make_from_oop(kit, value, ft->as_inline_klass()); 435 } 436 value->as_InlineTypeBase()->store_flattened(kit, base, ptr, holder, offset, decorators); 437 } else { 438 // Store field value to memory 439 const TypePtr* adr_type = field_adr_type(base, offset, holder, decorators, kit->gvn()); 440 Node* adr = kit->basic_plus_adr(base, ptr, offset); 441 BasicType bt = type2field[ft->basic_type()]; 442 assert(is_java_primitive(bt) || adr->bottom_type()->is_ptr_to_narrowoop() == UseCompressedOops, "inconsistent"); 443 const Type* val_type = Type::get_const_type(ft); 444 const TypeAryPtr* ary_type = kit->gvn().type(base)->isa_aryptr(); 445 if (ary_type != NULL) { 446 decorators |= IS_ARRAY; 447 } 448 kit->access_store_at(base, adr, adr_type, value, val_type, bt, decorators); 449 } 450 } 451 } 452 453 InlineTypePtrNode* InlineTypeBaseNode::buffer(GraphKit* kit, bool safe_for_replace) { 454 assert(is_InlineType(), "sanity"); 455 456 // Check if inline type is already buffered 457 Node* not_buffered_ctl = kit->top(); 458 Node* not_null_oop = kit->null_check_oop(get_oop(), ¬_buffered_ctl, /* never_see_null = */ false, safe_for_replace); 459 if (not_buffered_ctl->is_top()) { 460 // Already buffered 461 InlineTypePtrNode* ptr = as_ptr(&kit->gvn(), false); 462 if (safe_for_replace) { 463 kit->replace_in_map(this, ptr); 464 } 465 return ptr; 466 } 467 Node* buffered_ctl = kit->control(); 468 kit->set_control(not_buffered_ctl); 469 470 // Inline type is not buffered, check if it is null. 471 Node* null_ctl = kit->top(); 472 kit->null_check_common(get_is_init(), T_INT, false, &null_ctl); 473 bool null_free = null_ctl->is_top(); 474 475 RegionNode* region = new RegionNode(4); 476 PhiNode* oop = PhiNode::make(region, not_null_oop, inline_ptr()->join_speculative(null_free ? TypePtr::NOTNULL : TypePtr::BOTTOM)); 477 478 // InlineType is already buffered 479 region->init_req(1, buffered_ctl); 480 oop->init_req(1, not_null_oop); 481 482 // InlineType is null 483 region->init_req(2, null_ctl); 484 oop->init_req(2, kit->gvn().zerocon(T_OBJECT)); 485 486 PhiNode* io = PhiNode::make(region, kit->i_o(), Type::ABIO); 487 PhiNode* mem = PhiNode::make(region, kit->merged_memory(), Type::MEMORY, TypePtr::BOTTOM); 488 489 int bci = kit->bci(); 490 bool reexecute = kit->jvms()->should_reexecute(); 491 if (!kit->stopped()) { 492 assert(!is_allocated(&kit->gvn()), "already buffered"); 493 494 // Allocate and initialize buffer 495 PreserveJVMState pjvms(kit); 496 // Propagate re-execution state and bci 497 kit->set_bci(bci); 498 kit->jvms()->set_bci(bci); 499 kit->jvms()->set_should_reexecute(reexecute); 500 501 kit->kill_dead_locals(); 502 ciInlineKlass* vk = inline_klass(); 503 Node* klass_node = kit->makecon(TypeKlassPtr::make(vk)); 504 Node* alloc_oop = kit->new_instance(klass_node, NULL, NULL, /* deoptimize_on_exception */ true, this); 505 store(kit, alloc_oop, alloc_oop, vk); 506 507 // Do not let stores that initialize this buffer be reordered with a subsequent 508 // store that would make this buffer accessible by other threads. 509 AllocateNode* alloc = AllocateNode::Ideal_allocation(alloc_oop, &kit->gvn()); 510 assert(alloc != NULL, "must have an allocation node"); 511 kit->insert_mem_bar(Op_MemBarStoreStore, alloc->proj_out_or_null(AllocateNode::RawAddress)); 512 513 region->init_req(3, kit->control()); 514 oop ->init_req(3, alloc_oop); 515 io ->init_req(3, kit->i_o()); 516 mem ->init_req(3, kit->merged_memory()); 517 } 518 519 // Update GraphKit 520 kit->set_control(kit->gvn().transform(region)); 521 kit->set_i_o(kit->gvn().transform(io)); 522 kit->set_all_memory(kit->gvn().transform(mem)); 523 kit->record_for_igvn(region); 524 kit->record_for_igvn(oop); 525 kit->record_for_igvn(io); 526 kit->record_for_igvn(mem); 527 528 // Use cloned InlineTypeNode to propagate oop from now on 529 Node* res_oop = kit->gvn().transform(oop); 530 InlineTypeBaseNode* vt = clone()->as_InlineTypeBase(); 531 vt->set_oop(res_oop); 532 vt = kit->gvn().transform(vt)->as_InlineTypeBase(); 533 if (safe_for_replace) { 534 kit->replace_in_map(this, vt); 535 } 536 // InlineTypeNode::remove_redundant_allocations piggybacks on split if. 537 // Make sure it gets a chance to remove this allocation. 538 kit->C->set_has_split_ifs(true); 539 return vt->as_ptr(&kit->gvn(), null_free); 540 } 541 542 bool InlineTypeBaseNode::is_allocated(PhaseGVN* phase) const { 543 Node* oop = get_oop(); 544 const Type* oop_type = (phase != NULL) ? phase->type(oop) : oop->bottom_type(); 545 return !oop_type->maybe_null(); 546 } 547 548 InlineTypePtrNode* InlineTypeBaseNode::as_ptr(PhaseGVN* phase, bool null_free) const { 549 assert(!null_free || is_allocated(phase), "must be allocated"); 550 if (is_InlineTypePtr()) { 551 return as_InlineTypePtr(); 552 } 553 return phase->transform(new InlineTypePtrNode(this, null_free))->as_InlineTypePtr(); 554 } 555 556 // When a call returns multiple values, it has several result 557 // projections, one per field. Replacing the result of the call by an 558 // inline type node (after late inlining) requires that for each result 559 // projection, we find the corresponding inline type field. 560 void InlineTypeBaseNode::replace_call_results(GraphKit* kit, CallNode* call, Compile* C, bool null_free) { 561 ciInlineKlass* vk = inline_klass(); 562 for (DUIterator_Fast imax, i = call->fast_outs(imax); i < imax; i++) { 563 ProjNode* pn = call->fast_out(i)->as_Proj(); 564 uint con = pn->_con; 565 Node* field = NULL; 566 if (con == TypeFunc::Parms) { 567 field = get_oop(); 568 } else if (!null_free && con == (call->tf()->range_cc()->cnt() - 1)) { 569 field = get_is_init(); 570 } else if (con > TypeFunc::Parms) { 571 uint field_nb = con - (TypeFunc::Parms+1); 572 int extra = 0; 573 for (uint j = 0; j < field_nb - extra; j++) { 574 ciField* f = vk->nonstatic_field_at(j); 575 BasicType bt = f->type()->basic_type(); 576 if (bt == T_LONG || bt == T_DOUBLE) { 577 extra++; 578 } 579 } 580 ciField* f = vk->nonstatic_field_at(field_nb - extra); 581 field = field_value_by_offset(f->offset(), true); 582 if (field->is_InlineType()) { 583 assert(field->as_InlineType()->is_allocated(&kit->gvn()), "must be allocated"); 584 field = field->as_InlineType()->get_oop(); 585 } 586 } 587 if (field != NULL) { 588 C->gvn_replace_by(pn, field); 589 C->initial_gvn()->hash_delete(pn); 590 pn->set_req(0, C->top()); 591 --i; --imax; 592 } 593 } 594 } 595 596 Node* InlineTypeBaseNode::allocate_fields(GraphKit* kit) { 597 InlineTypeBaseNode* vt = clone()->as_InlineTypeBase(); 598 for (uint i = 0; i < field_count(); i++) { 599 Node* value = field_value(i); 600 if (field_is_flattened(i)) { 601 // Flattened inline type field 602 vt->set_field_value(i, value->as_InlineTypeBase()->allocate_fields(kit)); 603 } else if (value->is_InlineType()) { 604 // Non-flattened inline type field 605 vt->set_field_value(i, value->as_InlineType()->buffer(kit)); 606 } 607 } 608 vt = kit->gvn().transform(vt)->as_InlineTypeBase(); 609 kit->replace_in_map(this, vt); 610 return vt; 611 } 612 613 Node* InlineTypeBaseNode::Ideal(PhaseGVN* phase, bool can_reshape) { 614 Node* is_init = get_is_init(); 615 if (is_init->isa_InlineTypePtr()) { 616 set_req(IsInit, is_init->as_InlineTypePtr()->get_is_init()); 617 return this; 618 } 619 Node* oop = get_oop(); 620 if (oop->isa_InlineTypePtr() && !phase->type(oop)->maybe_null()) { 621 InlineTypePtrNode* vtptr = oop->as_InlineTypePtr(); 622 set_oop(vtptr->get_oop()); 623 set_is_init(*phase); 624 for (uint i = Values; i < vtptr->req(); ++i) { 625 set_req(i, vtptr->in(i)); 626 } 627 return this; 628 } 629 return NULL; 630 } 631 632 InlineTypeNode* InlineTypeNode::make_uninitialized(PhaseGVN& gvn, ciInlineKlass* vk) { 633 // Create a new InlineTypeNode with uninitialized values and NULL oop 634 Node* oop = (vk->is_empty() && vk->is_initialized()) ? default_oop(gvn, vk) : gvn.zerocon(T_PRIMITIVE_OBJECT); 635 InlineTypeNode* vt = new InlineTypeNode(vk, oop); 636 vt->set_is_init(gvn); 637 return vt; 638 } 639 640 Node* InlineTypeBaseNode::default_oop(PhaseGVN& gvn, ciInlineKlass* vk) { 641 // Returns the constant oop of the default inline type allocation 642 return gvn.makecon(TypeInstPtr::make(vk->default_instance())); 643 } 644 645 InlineTypeNode* InlineTypeNode::make_default(PhaseGVN& gvn, ciInlineKlass* vk) { 646 // Create a new InlineTypeNode with default values 647 Node* oop = vk->is_initialized() ? default_oop(gvn, vk) : gvn.zerocon(T_PRIMITIVE_OBJECT); 648 InlineTypeNode* vt = new InlineTypeNode(vk, oop); 649 vt->set_is_init(gvn); 650 for (uint i = 0; i < vt->field_count(); ++i) { 651 ciType* field_type = vt->field_type(i); 652 Node* value = gvn.zerocon(field_type->basic_type()); 653 if (field_type->is_inlinetype()) { 654 ciInlineKlass* vk = field_type->as_inline_klass(); 655 if (vt->field_is_null_free(i)) { 656 value = make_default(gvn, vk); 657 } else { 658 value = InlineTypePtrNode::make_null(gvn, vk); 659 } 660 } 661 vt->set_field_value(i, value); 662 } 663 vt = gvn.transform(vt)->as_InlineType(); 664 assert(vt->is_default(&gvn), "must be the default inline type"); 665 return vt; 666 } 667 668 InlineTypeNode* InlineTypeNode::make_null(PhaseGVN& gvn, ciInlineKlass* vk) { 669 InlineTypeNode* vt = new InlineTypeNode(vk, gvn.zerocon(T_OBJECT)); 670 vt->set_req(IsInit, gvn.intcon(0)); 671 for (uint i = 0; i < vt->field_count(); i++) { 672 ciType* field_type = vt->field_type(i); 673 Node* value = gvn.zerocon(field_type->basic_type()); 674 if (field_type->is_inlinetype()) { 675 if (vt->field_is_null_free(i)) { 676 value = InlineTypeNode::make_null(gvn, field_type->as_inline_klass()); 677 } else { 678 value = InlineTypePtrNode::make_null(gvn, field_type->as_inline_klass()); 679 } 680 } 681 vt->set_field_value(i, value); 682 } 683 return gvn.transform(vt)->as_InlineType(); 684 } 685 686 bool InlineTypeBaseNode::is_default(PhaseGVN* gvn) const { 687 const Type* tinit = gvn->type(in(IsInit)); 688 if (!tinit->isa_int() || !tinit->is_int()->is_con(1)) { 689 return false; // May be null 690 } 691 for (uint i = 0; i < field_count(); ++i) { 692 Node* value = field_value(i); 693 if (value->is_InlineTypePtr()) { 694 value = value->as_InlineTypePtr()->get_oop(); 695 } 696 if (!gvn->type(value)->is_zero_type() && 697 !(field_is_null_free(i) && value->is_InlineType() && value->as_InlineType()->is_default(gvn))) { 698 return false; 699 } 700 } 701 return true; 702 } 703 704 InlineTypeBaseNode* InlineTypeNode::make_from_oop(GraphKit* kit, Node* oop, ciInlineKlass* vk, bool null_free) { 705 PhaseGVN& gvn = kit->gvn(); 706 707 if (vk->is_empty() && null_free) { 708 InlineTypeNode* def = make_default(gvn, vk); 709 kit->record_for_igvn(def); 710 return def; 711 } 712 // Create and initialize an InlineTypeNode by loading all field 713 // values from a heap-allocated version and also save the oop. 714 InlineTypeBaseNode* vt = NULL; 715 716 if (oop->isa_InlineTypePtr()) { 717 return oop->as_InlineTypePtr(); 718 } else if (gvn.type(oop)->maybe_null()) { 719 // Add a null check because the oop may be null 720 Node* null_ctl = kit->top(); 721 Node* not_null_oop = kit->null_check_oop(oop, &null_ctl); 722 if (kit->stopped()) { 723 // Constant null 724 kit->set_control(null_ctl); 725 if (null_free) { 726 vt = make_default(gvn, vk); 727 } else { 728 vt = InlineTypePtrNode::make_null(gvn, vk); 729 } 730 kit->record_for_igvn(vt); 731 return vt; 732 } 733 vt = new InlineTypePtrNode(vk, not_null_oop, null_free); 734 vt->set_is_init(gvn); 735 vt->load(kit, not_null_oop, not_null_oop, vk, /* holder_offset */ 0); 736 737 if (null_ctl != kit->top()) { 738 InlineTypeBaseNode* null_vt = NULL; 739 if (null_free) { 740 null_vt = make_default(gvn, vk)->as_ptr(&gvn); 741 } else { 742 null_vt = InlineTypePtrNode::make_null(gvn, vk); 743 } 744 Node* region = new RegionNode(3); 745 region->init_req(1, kit->control()); 746 region->init_req(2, null_ctl); 747 748 vt = vt->clone_with_phis(&gvn, region); 749 vt->merge_with(&gvn, null_vt, 2, true); 750 if (!null_free) { 751 vt->set_oop(oop); 752 } 753 kit->set_control(gvn.transform(region)); 754 } 755 } else { 756 // Oop can never be null 757 vt = new InlineTypePtrNode(vk, oop, /* null_free= */ true); 758 Node* init_ctl = kit->control(); 759 vt->set_is_init(gvn); 760 vt->load(kit, oop, oop, vk, /* holder_offset */ 0); 761 // TODO fix with JDK-8278390 762 // assert(!null_free || vt->as_InlineType()->is_default(&gvn) || init_ctl != kit->control() || !gvn.type(oop)->is_inlinetypeptr() || oop->is_Con() || oop->Opcode() == Op_InlineTypePtr || 763 // AllocateNode::Ideal_allocation(oop, &gvn) != NULL || vt->as_InlineType()->is_loaded(&gvn) == oop, "inline type should be loaded"); 764 } 765 assert(!null_free || vt->is_allocated(&gvn), "inline type should be allocated"); 766 kit->record_for_igvn(vt); 767 return gvn.transform(vt)->as_InlineTypeBase(); 768 } 769 770 // GraphKit wrapper for the 'make_from_flattened' method 771 InlineTypeNode* InlineTypeNode::make_from_flattened(GraphKit* kit, ciInlineKlass* vk, Node* obj, Node* ptr, ciInstanceKlass* holder, int holder_offset, DecoratorSet decorators) { 772 if (kit->gvn().type(obj)->isa_aryptr()) { 773 kit->C->set_flattened_accesses(); 774 } 775 // Create and initialize an InlineTypeNode by loading all field values from 776 // a flattened inline type field at 'holder_offset' or from an inline type array. 777 InlineTypeNode* vt = make_uninitialized(kit->gvn(), vk); 778 // The inline type is flattened into the object without an oop header. Subtract the 779 // offset of the first field to account for the missing header when loading the values. 780 holder_offset -= vk->first_field_offset(); 781 vt->load(kit, obj, ptr, holder, holder_offset, decorators); 782 assert(vt->is_loaded(&kit->gvn()) != obj, "holder oop should not be used as flattened inline type oop"); 783 return kit->gvn().transform(vt)->as_InlineType(); 784 } 785 786 InlineTypeBaseNode* InlineTypeNode::make_from_multi(GraphKit* kit, MultiNode* multi, ciInlineKlass* vk, uint& base_input, bool in, bool null_free) { 787 InlineTypeNode* vt = make_uninitialized(kit->gvn(), vk); 788 if (!in) { 789 // Keep track of the oop. The returned inline type might already be buffered. 790 Node* oop = kit->gvn().transform(new ProjNode(multi, base_input++)); 791 vt->set_oop(oop); 792 } 793 vt->initialize_fields(kit, multi, base_input, in, null_free); 794 return kit->gvn().transform(vt)->as_InlineTypeBase(); 795 } 796 797 InlineTypeNode* InlineTypeBaseNode::make_larval(GraphKit* kit, bool allocate) const { 798 ciInlineKlass* vk = inline_klass(); 799 InlineTypeNode* res = InlineTypeNode::make_uninitialized(kit->gvn(), vk); 800 for (uint i = 1; i < req(); ++i) { 801 res->set_req(i, in(i)); 802 } 803 804 if (allocate) { 805 // Re-execute if buffering triggers deoptimization 806 PreserveReexecuteState preexecs(kit); 807 kit->jvms()->set_should_reexecute(true); 808 Node* klass_node = kit->makecon(TypeKlassPtr::make(vk)); 809 Node* alloc_oop = kit->new_instance(klass_node, NULL, NULL, true); 810 AllocateNode* alloc = AllocateNode::Ideal_allocation(alloc_oop, &kit->gvn()); 811 alloc->_larval = true; 812 813 store(kit, alloc_oop, alloc_oop, vk); 814 res->set_oop(alloc_oop); 815 } 816 res->set_type(TypeInlineType::make(vk, true)); 817 res = kit->gvn().transform(res)->as_InlineType(); 818 assert(!allocate || res->is_allocated(&kit->gvn()), "must be allocated"); 819 return res; 820 } 821 822 InlineTypeNode* InlineTypeBaseNode::finish_larval(GraphKit* kit) const { 823 Node* obj = get_oop(); 824 Node* mark_addr = kit->basic_plus_adr(obj, oopDesc::mark_offset_in_bytes()); 825 Node* mark = kit->make_load(NULL, mark_addr, TypeX_X, TypeX_X->basic_type(), MemNode::unordered); 826 mark = kit->gvn().transform(new AndXNode(mark, kit->MakeConX(~markWord::larval_bit_in_place))); 827 kit->store_to_memory(kit->control(), mark_addr, mark, TypeX_X->basic_type(), kit->gvn().type(mark_addr)->is_ptr(), MemNode::unordered); 828 829 // Do not let stores that initialize this buffer be reordered with a subsequent 830 // store that would make this buffer accessible by other threads. 831 AllocateNode* alloc = AllocateNode::Ideal_allocation(obj, &kit->gvn()); 832 assert(alloc != NULL, "must have an allocation node"); 833 kit->insert_mem_bar(Op_MemBarStoreStore, alloc->proj_out_or_null(AllocateNode::RawAddress)); 834 835 ciInlineKlass* vk = inline_klass(); 836 InlineTypeNode* res = InlineTypeNode::make_uninitialized(kit->gvn(), vk); 837 for (uint i = 1; i < req(); ++i) { 838 res->set_req(i, in(i)); 839 } 840 res->set_type(TypeInlineType::make(vk, false)); 841 res = kit->gvn().transform(res)->as_InlineType(); 842 return res; 843 } 844 845 Node* InlineTypeNode::is_loaded(PhaseGVN* phase, ciInlineKlass* vk, Node* base, int holder_offset) { 846 if (vk == NULL) { 847 vk = inline_klass(); 848 } 849 if (field_count() == 0 && vk->is_initialized()) { 850 const Type* tinit = phase->type(in(IsInit)); 851 if (tinit->isa_int() && tinit->is_int()->is_con(1)) { 852 assert(is_allocated(phase), "must be allocated"); 853 return get_oop(); 854 } else { 855 // TODO 8284443 856 return NULL; 857 } 858 } 859 for (uint i = 0; i < field_count(); ++i) { 860 int offset = holder_offset + field_offset(i); 861 Node* value = field_value(i); 862 if (value->is_InlineTypeBase()) { 863 InlineTypeBaseNode* vt = value->as_InlineTypeBase(); 864 if (vt->type()->inline_klass()->is_empty()) { 865 continue; 866 } else if (field_is_flattened(i) && vt->is_InlineType()) { 867 // Check inline type field load recursively 868 base = vt->as_InlineType()->is_loaded(phase, vk, base, offset - vt->type()->inline_klass()->first_field_offset()); 869 if (base == NULL) { 870 return NULL; 871 } 872 continue; 873 } else { 874 value = vt->get_oop(); 875 if (value->Opcode() == Op_CastPP) { 876 // Skip CastPP 877 value = value->in(1); 878 } 879 } 880 } 881 if (value->isa_DecodeN()) { 882 // Skip DecodeN 883 value = value->in(1); 884 } 885 if (value->isa_Load()) { 886 // Check if base and offset of field load matches inline type layout 887 intptr_t loffset = 0; 888 Node* lbase = AddPNode::Ideal_base_and_offset(value->in(MemNode::Address), phase, loffset); 889 if (lbase == NULL || (lbase != base && base != NULL) || loffset != offset) { 890 return NULL; 891 } else if (base == NULL) { 892 // Set base and check if pointer type matches 893 base = lbase; 894 const TypeInstPtr* vtptr = phase->type(base)->isa_instptr(); 895 if (vtptr == NULL || !vtptr->klass()->equals(vk)) { 896 return NULL; 897 } 898 } 899 } else { 900 return NULL; 901 } 902 } 903 return base; 904 } 905 906 Node* InlineTypeBaseNode::tagged_klass(ciInlineKlass* vk, PhaseGVN& gvn) { 907 const TypeKlassPtr* tk = TypeKlassPtr::make(vk); 908 intptr_t bits = tk->get_con(); 909 set_nth_bit(bits, 0); 910 return gvn.longcon((jlong)bits); 911 } 912 913 void InlineTypeBaseNode::pass_fields(GraphKit* kit, Node* n, uint& base_input, bool in, bool null_free) { 914 if (!null_free && in) { 915 n->init_req(base_input++, get_is_init()); 916 } 917 for (uint i = 0; i < field_count(); i++) { 918 Node* arg = field_value(i); 919 if (field_is_flattened(i)) { 920 // Flattened inline type field 921 arg->as_InlineTypeBase()->pass_fields(kit, n, base_input, in); 922 } else { 923 if (arg->is_InlineType()) { 924 // Non-flattened inline type field 925 InlineTypeNode* vt = arg->as_InlineType(); 926 assert(n->Opcode() != Op_Return || vt->is_allocated(&kit->gvn()), "inline type field should be allocated on return"); 927 arg = vt->buffer(kit); 928 } 929 // Initialize call/return arguments 930 n->init_req(base_input++, arg); 931 if (field_type(i)->size() == 2) { 932 n->init_req(base_input++, kit->top()); 933 } 934 } 935 } 936 // The last argument is used to pass IsInit information to compiled code and not required here. 937 if (!null_free && !in) { 938 n->init_req(base_input++, kit->top()); 939 } 940 } 941 942 void InlineTypeNode::initialize_fields(GraphKit* kit, MultiNode* multi, uint& base_input, bool in, bool null_free, Node* null_check_region) { 943 PhaseGVN& gvn = kit->gvn(); 944 Node* is_init = NULL; 945 if (!null_free) { 946 // Nullable inline type 947 if (in) { 948 // Set IsInit field 949 if (multi->is_Start()) { 950 is_init = gvn.transform(new ParmNode(multi->as_Start(), base_input)); 951 } else { 952 is_init = multi->as_Call()->in(base_input); 953 } 954 set_req(IsInit, is_init); 955 base_input++; 956 } 957 // Add a null check to make subsequent loads dependent on 958 assert(null_check_region == NULL, "already set"); 959 if (is_init == NULL) { 960 // Will only be initialized below, use dummy node for now 961 is_init = new Node(1); 962 gvn.set_type_bottom(is_init); 963 } 964 Node* null_ctrl = kit->top(); 965 kit->null_check_common(is_init, T_INT, false, &null_ctrl); 966 Node* non_null_ctrl = kit->control(); 967 null_check_region = new RegionNode(3); 968 null_check_region->init_req(1, non_null_ctrl); 969 null_check_region->init_req(2, null_ctrl); 970 null_check_region = gvn.transform(null_check_region); 971 kit->set_control(null_check_region); 972 } 973 974 for (uint i = 0; i < field_count(); ++i) { 975 ciType* type = field_type(i); 976 Node* parm = NULL; 977 if (field_is_flattened(i)) { 978 // Flattened inline type field 979 InlineTypeNode* vt = make_uninitialized(gvn, type->as_inline_klass()); 980 vt->initialize_fields(kit, multi, base_input, in, true, null_check_region); 981 parm = gvn.transform(vt); 982 } else { 983 if (multi->is_Start()) { 984 assert(in, "return from start?"); 985 parm = gvn.transform(new ParmNode(multi->as_Start(), base_input)); 986 } else if (in) { 987 parm = multi->as_Call()->in(base_input); 988 } else { 989 parm = gvn.transform(new ProjNode(multi->as_Call(), base_input)); 990 } 991 // Non-flattened inline type field 992 if (type->is_inlinetype()) { 993 if (null_check_region != NULL) { 994 // Holder is nullable, set field to NULL if holder is NULL to avoid loading from uninitialized memory 995 parm = PhiNode::make(null_check_region, parm, TypeInstPtr::make(TypePtr::BotPTR, type->as_inline_klass())); 996 parm->set_req(2, kit->zerocon(T_OBJECT)); 997 parm = gvn.transform(parm); 998 } 999 parm = make_from_oop(kit, parm, type->as_inline_klass(), field_is_null_free(i)); 1000 } 1001 base_input += type->size(); 1002 } 1003 assert(parm != NULL, "should never be null"); 1004 assert(field_value(i) == NULL, "already set"); 1005 set_field_value(i, parm); 1006 gvn.record_for_igvn(parm); 1007 } 1008 // The last argument is used to pass IsInit information to compiled code 1009 if (!null_free && !in) { 1010 Node* cmp = is_init->raw_out(0); 1011 is_init= gvn.transform(new ProjNode(multi->as_Call(), base_input)); 1012 set_req(IsInit, is_init); 1013 gvn.hash_delete(cmp); 1014 cmp->set_req(1, is_init); 1015 gvn.hash_find_insert(cmp); 1016 base_input++; 1017 } 1018 } 1019 1020 // Replace a buffer allocation by a dominating allocation 1021 static void replace_allocation(PhaseIterGVN* igvn, Node* res, Node* dom) { 1022 // Remove initializing stores and GC barriers 1023 for (DUIterator_Fast imax, i = res->fast_outs(imax); i < imax; i++) { 1024 Node* use = res->fast_out(i); 1025 if (use->is_AddP()) { 1026 for (DUIterator_Fast jmax, j = use->fast_outs(jmax); j < jmax; j++) { 1027 Node* store = use->fast_out(j)->isa_Store(); 1028 if (store != NULL) { 1029 igvn->rehash_node_delayed(store); 1030 igvn->replace_in_uses(store, store->in(MemNode::Memory)); 1031 } 1032 } 1033 } else if (use->Opcode() == Op_CastP2X) { 1034 if (UseG1GC && use->find_out_with(Op_XorX)->in(1) != use) { 1035 // The G1 pre-barrier uses a CastP2X both for the pointer of the object 1036 // we store into, as well as the value we are storing. Skip if this is a 1037 // barrier for storing 'res' into another object. 1038 continue; 1039 } 1040 BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2(); 1041 bs->eliminate_gc_barrier(igvn, use); 1042 --i; --imax; 1043 } 1044 } 1045 igvn->replace_node(res, dom); 1046 } 1047 1048 Node* InlineTypeNode::Ideal(PhaseGVN* phase, bool can_reshape) { 1049 Node* oop = get_oop(); 1050 if (is_default(phase) && inline_klass()->is_initialized() && 1051 (!oop->is_Con() || phase->type(oop)->is_zero_type())) { 1052 // Use the pre-allocated oop for default inline types 1053 set_oop(default_oop(*phase, inline_klass())); 1054 assert(is_allocated(phase), "should now be allocated"); 1055 return this; 1056 } 1057 1058 if (!is_allocated(phase)) { 1059 // Save base oop if fields are loaded from memory and the inline 1060 // type is not buffered (in this case we should not use the oop). 1061 Node* base = is_loaded(phase); 1062 if (base != NULL && !phase->type(base)->maybe_null()) { 1063 set_oop(base); 1064 assert(is_allocated(phase), "should now be allocated"); 1065 return this; 1066 } 1067 } 1068 1069 if (can_reshape) { 1070 PhaseIterGVN* igvn = phase->is_IterGVN(); 1071 1072 if (is_allocated(phase)) { 1073 // Search for and remove re-allocations of this inline type. Ignore scalar replaceable ones, 1074 // they will be removed anyway and changing the memory chain will confuse other optimizations. 1075 // This can happen with late inlining when we first allocate an inline type argument 1076 // but later decide to inline the call after the callee code also triggered allocation. 1077 for (DUIterator_Fast imax, i = fast_outs(imax); i < imax; i++) { 1078 AllocateNode* alloc = fast_out(i)->isa_Allocate(); 1079 if (alloc != NULL && alloc->in(AllocateNode::InlineTypeNode) == this && !alloc->_is_scalar_replaceable) { 1080 // Found a re-allocation 1081 Node* res = alloc->result_cast(); 1082 if (res != NULL && res->is_CheckCastPP()) { 1083 // Replace allocation by oop and unlink AllocateNode 1084 replace_allocation(igvn, res, oop); 1085 igvn->replace_input_of(alloc, AllocateNode::InlineTypeNode, igvn->C->top()); 1086 --i; --imax; 1087 } 1088 } 1089 } 1090 } 1091 } 1092 return InlineTypeBaseNode::Ideal(phase, can_reshape); 1093 } 1094 1095 // Search for multiple allocations of this inline type and try to replace them by dominating allocations. 1096 void InlineTypeNode::remove_redundant_allocations(PhaseIterGVN* igvn, PhaseIdealLoop* phase) { 1097 // Search for allocations of this inline type. Ignore scalar replaceable ones, they 1098 // will be removed anyway and changing the memory chain will confuse other optimizations. 1099 for (DUIterator_Fast imax, i = fast_outs(imax); i < imax; i++) { 1100 AllocateNode* alloc = fast_out(i)->isa_Allocate(); 1101 if (alloc != NULL && alloc->in(AllocateNode::InlineTypeNode) == this && !alloc->_is_scalar_replaceable) { 1102 Node* res = alloc->result_cast(); 1103 if (res == NULL || !res->is_CheckCastPP()) { 1104 break; // No unique CheckCastPP 1105 } 1106 assert((!is_default(igvn) || !inline_klass()->is_initialized()) && !is_allocated(igvn), "re-allocation should be removed by Ideal transformation"); 1107 // Search for a dominating allocation of the same inline type 1108 Node* res_dom = res; 1109 for (DUIterator_Fast jmax, j = fast_outs(jmax); j < jmax; j++) { 1110 AllocateNode* alloc_other = fast_out(j)->isa_Allocate(); 1111 if (alloc_other != NULL && alloc_other->in(AllocateNode::InlineTypeNode) == this && !alloc_other->_is_scalar_replaceable) { 1112 Node* res_other = alloc_other->result_cast(); 1113 if (res_other != NULL && res_other->is_CheckCastPP() && res_other != res_dom && 1114 phase->is_dominator(res_other->in(0), res_dom->in(0))) { 1115 res_dom = res_other; 1116 } 1117 } 1118 } 1119 if (res_dom != res) { 1120 // Replace allocation by dominating one. 1121 replace_allocation(igvn, res, res_dom); 1122 // The result of the dominated allocation is now unused and will be removed 1123 // later in PhaseMacroExpand::eliminate_allocate_node to not confuse loop opts. 1124 igvn->_worklist.push(alloc); 1125 } 1126 } 1127 } 1128 1129 // Process users 1130 for (DUIterator_Fast imax, i = fast_outs(imax); i < imax; i++) { 1131 Node* out = fast_out(i); 1132 if (out->is_InlineType()) { 1133 // Recursively process inline type users 1134 igvn->rehash_node_delayed(out); 1135 out->as_InlineType()->remove_redundant_allocations(igvn, phase); 1136 } else if (out->isa_Allocate() != NULL) { 1137 // Unlink AllocateNode 1138 assert(out->in(AllocateNode::InlineTypeNode) == this, "should be linked"); 1139 igvn->replace_input_of(out, AllocateNode::InlineTypeNode, igvn->C->top()); 1140 --i; --imax; 1141 } 1142 } 1143 } 1144 1145 InlineTypePtrNode* InlineTypePtrNode::make_null(PhaseGVN& gvn, ciInlineKlass* vk) { 1146 InlineTypePtrNode* ptr = new InlineTypePtrNode(vk, gvn.zerocon(T_OBJECT), /* null_free= */ false); 1147 ptr->set_req(IsInit, gvn.intcon(0)); 1148 for (uint i = 0; i < ptr->field_count(); i++) { 1149 ciType* field_type = ptr->field_type(i); 1150 Node* value = gvn.zerocon(field_type->basic_type()); 1151 if (field_type->is_inlinetype()) { 1152 if (ptr->field_is_null_free(i)) { 1153 value = InlineTypeNode::make_null(gvn, field_type->as_inline_klass()); 1154 } else { 1155 value = InlineTypePtrNode::make_null(gvn, field_type->as_inline_klass()); 1156 } 1157 } 1158 ptr->set_field_value(i, value); 1159 } 1160 return gvn.transform(ptr)->as_InlineTypePtr(); 1161 } 1162 1163 Node* InlineTypePtrNode::Identity(PhaseGVN* phase) { 1164 if (get_oop()->is_InlineTypePtr()) { 1165 return get_oop(); 1166 } 1167 return this; 1168 } 1169 1170 const Type* InlineTypePtrNode::Value(PhaseGVN* phase) const { 1171 const Type* t = phase->type(get_oop())->filter_speculative(_type); 1172 if (t->singleton()) { 1173 // Don't replace InlineTypePtr by a constant 1174 t = _type; 1175 } 1176 const Type* tinit = phase->type(in(IsInit)); 1177 if (tinit->isa_int() && tinit->is_int()->is_con(1)) { 1178 t = t->join_speculative(TypePtr::NOTNULL); 1179 } 1180 return t; 1181 }