1 /* 2 * Copyright (c) 2020, 2023, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 * 23 */ 24 25 #include "precompiled.hpp" 26 #include "ci/ciSymbols.hpp" 27 #include "gc/shared/barrierSet.hpp" 28 #include "opto/castnode.hpp" 29 #include "opto/graphKit.hpp" 30 #include "opto/phaseX.hpp" 31 #include "opto/rootnode.hpp" 32 #include "opto/vector.hpp" 33 #include "utilities/macros.hpp" 34 35 static bool is_vector_mask(ciKlass* klass) { 36 return klass->is_subclass_of(ciEnv::current()->vector_VectorMask_klass()); 37 } 38 39 void PhaseVector::optimize_vector_boxes() { 40 Compile::TracePhase tp("vector_elimination", &timers[_t_vector_elimination]); 41 42 // Signal GraphKit it's post-parse phase. 43 assert(C->inlining_incrementally() == false, "sanity"); 44 C->set_inlining_incrementally(true); 45 46 C->for_igvn()->clear(); 47 C->initial_gvn()->replace_with(&_igvn); 48 49 expand_vunbox_nodes(); 50 scalarize_vbox_nodes(); 51 52 C->inline_vector_reboxing_calls(); 53 54 expand_vbox_nodes(); 55 eliminate_vbox_alloc_nodes(); 56 57 C->set_inlining_incrementally(false); 58 59 do_cleanup(); 60 } 61 62 void PhaseVector::do_cleanup() { 63 if (C->failing()) return; 64 { 65 Compile::TracePhase tp("vector_pru", &timers[_t_vector_pru]); 66 ResourceMark rm; 67 PhaseRemoveUseless pru(C->initial_gvn(), C->for_igvn()); 68 if (C->failing()) return; 69 } 70 { 71 Compile::TracePhase tp("incrementalInline_igvn", &timers[_t_vector_igvn]); 72 _igvn = PhaseIterGVN(C->initial_gvn()); 73 _igvn.optimize(); 74 if (C->failing()) return; 75 } 76 C->print_method(PHASE_ITER_GVN_BEFORE_EA, 3); 77 } 78 79 void PhaseVector::scalarize_vbox_nodes() { 80 if (C->failing()) return; 81 82 if (!EnableVectorReboxing) { 83 return; // don't scalarize vector boxes 84 } 85 86 int macro_idx = C->macro_count() - 1; 87 while (macro_idx >= 0) { 88 Node * n = C->macro_node(macro_idx); 89 assert(n->is_macro(), "only macro nodes expected here"); 90 if (n->Opcode() == Op_VectorBox) { 91 VectorBoxNode* vbox = static_cast<VectorBoxNode*>(n); 92 scalarize_vbox_node(vbox); 93 if (C->failing()) return; 94 C->print_method(PHASE_SCALARIZE_VBOX, 3, vbox); 95 } 96 if (C->failing()) return; 97 macro_idx = MIN2(macro_idx - 1, C->macro_count() - 1); 98 } 99 } 100 101 void PhaseVector::expand_vbox_nodes() { 102 if (C->failing()) return; 103 104 int macro_idx = C->macro_count() - 1; 105 while (macro_idx >= 0) { 106 Node * n = C->macro_node(macro_idx); 107 assert(n->is_macro(), "only macro nodes expected here"); 108 if (n->Opcode() == Op_VectorBox) { 109 VectorBoxNode* vbox = static_cast<VectorBoxNode*>(n); 110 expand_vbox_node(vbox); 111 if (C->failing()) return; 112 } 113 if (C->failing()) return; 114 macro_idx = MIN2(macro_idx - 1, C->macro_count() - 1); 115 } 116 } 117 118 void PhaseVector::expand_vunbox_nodes() { 119 if (C->failing()) return; 120 121 int macro_idx = C->macro_count() - 1; 122 while (macro_idx >= 0) { 123 Node * n = C->macro_node(macro_idx); 124 assert(n->is_macro(), "only macro nodes expected here"); 125 if (n->Opcode() == Op_VectorUnbox) { 126 VectorUnboxNode* vec_unbox = static_cast<VectorUnboxNode*>(n); 127 expand_vunbox_node(vec_unbox); 128 if (C->failing()) return; 129 C->print_method(PHASE_EXPAND_VUNBOX, 3, vec_unbox); 130 } 131 if (C->failing()) return; 132 macro_idx = MIN2(macro_idx - 1, C->macro_count() - 1); 133 } 134 } 135 136 void PhaseVector::eliminate_vbox_alloc_nodes() { 137 if (C->failing()) return; 138 139 int macro_idx = C->macro_count() - 1; 140 while (macro_idx >= 0) { 141 Node * n = C->macro_node(macro_idx); 142 assert(n->is_macro(), "only macro nodes expected here"); 143 if (n->Opcode() == Op_VectorBoxAllocate) { 144 VectorBoxAllocateNode* vbox_alloc = static_cast<VectorBoxAllocateNode*>(n); 145 eliminate_vbox_alloc_node(vbox_alloc); 146 if (C->failing()) return; 147 C->print_method(PHASE_ELIMINATE_VBOX_ALLOC, 3, vbox_alloc); 148 } 149 if (C->failing()) return; 150 macro_idx = MIN2(macro_idx - 1, C->macro_count() - 1); 151 } 152 } 153 154 static JVMState* clone_jvms(Compile* C, SafePointNode* sfpt) { 155 JVMState* new_jvms = sfpt->jvms()->clone_shallow(C); 156 uint size = sfpt->req(); 157 SafePointNode* map = new SafePointNode(size, new_jvms); 158 for (uint i = 0; i < size; i++) { 159 map->init_req(i, sfpt->in(i)); 160 } 161 Node* mem = map->memory(); 162 if (!mem->is_MergeMem()) { 163 // Since we are not in parsing, the SafePointNode does not guarantee that the memory 164 // input is necessarily a MergeMemNode. But we need to ensure that there is that 165 // MergeMemNode, since the GraphKit assumes the memory input of the map to be a 166 // MergeMemNode, so that it can directly access the memory slices. 167 PhaseGVN& gvn = *C->initial_gvn(); 168 Node* mergemem = MergeMemNode::make(mem); 169 gvn.set_type_bottom(mergemem); 170 map->set_memory(mergemem); 171 } 172 new_jvms->set_map(map); 173 return new_jvms; 174 } 175 176 void PhaseVector::scalarize_vbox_node(VectorBoxNode* vec_box) { 177 Node* vec_value = vec_box->in(VectorBoxNode::Value); 178 PhaseGVN& gvn = *C->initial_gvn(); 179 180 // Process merged VBAs 181 182 if (EnableVectorAggressiveReboxing) { 183 Unique_Node_List calls(C->comp_arena()); 184 for (DUIterator_Fast imax, i = vec_box->fast_outs(imax); i < imax; i++) { 185 Node* use = vec_box->fast_out(i); 186 if (use->is_CallJava()) { 187 CallJavaNode* call = use->as_CallJava(); 188 if (call->has_non_debug_use(vec_box) && vec_box->in(VectorBoxNode::Box)->is_Phi()) { 189 calls.push(call); 190 } 191 } 192 } 193 194 while (calls.size() > 0) { 195 CallJavaNode* call = calls.pop()->as_CallJava(); 196 // Attach new VBA to the call and use it instead of Phi (VBA ... VBA). 197 198 JVMState* jvms = clone_jvms(C, call); 199 GraphKit kit(jvms); 200 PhaseGVN& gvn = kit.gvn(); 201 202 // Adjust JVMS from post-call to pre-call state: put args on stack 203 uint nargs = call->method()->arg_size(); 204 kit.ensure_stack(kit.sp() + nargs); 205 for (uint i = TypeFunc::Parms; i < call->tf()->domain_sig()->cnt(); i++) { 206 kit.push(call->in(i)); 207 } 208 jvms = kit.sync_jvms(); 209 210 Node* new_vbox = nullptr; 211 { 212 Node* vect = vec_box->in(VectorBoxNode::Value); 213 const TypeInstPtr* vbox_type = vec_box->box_type(); 214 const TypeVect* vt = vec_box->vec_type(); 215 BasicType elem_bt = vt->element_basic_type(); 216 int num_elem = vt->length(); 217 218 new_vbox = kit.box_vector(vect, vbox_type, elem_bt, num_elem, /*deoptimize=*/true); 219 220 kit.replace_in_map(vec_box, new_vbox); 221 } 222 223 kit.dec_sp(nargs); 224 jvms = kit.sync_jvms(); 225 226 call->set_req(TypeFunc::Control , kit.control()); 227 call->set_req(TypeFunc::I_O , kit.i_o()); 228 call->set_req(TypeFunc::Memory , kit.reset_memory()); 229 call->set_req(TypeFunc::FramePtr, kit.frameptr()); 230 call->replace_edge(vec_box, new_vbox); 231 232 C->record_for_igvn(call); 233 } 234 } 235 236 // Process debug uses at safepoints 237 Unique_Node_List safepoints(C->comp_arena()); 238 239 Unique_Node_List worklist(C->comp_arena()); 240 worklist.push(vec_box); 241 while (worklist.size() > 0) { 242 Node* n = worklist.pop(); 243 for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) { 244 Node* use = n->fast_out(i); 245 if (use->is_SafePoint()) { 246 SafePointNode* sfpt = use->as_SafePoint(); 247 if (!sfpt->is_Call() || !sfpt->as_Call()->has_non_debug_use(n)) { 248 safepoints.push(sfpt); 249 } 250 } else if (use->is_ConstraintCast()) { 251 worklist.push(use); // reversed version of Node::uncast() 252 } 253 } 254 } 255 256 ciInstanceKlass* iklass = vec_box->box_type()->instance_klass(); 257 int n_fields = iklass->nof_nonstatic_fields(); 258 assert(n_fields == 1, "sanity"); 259 260 // If a mask is feeding into safepoint[s], then its value should be 261 // packed into a boolean/byte vector first, this will simplify the 262 // re-materialization logic for both predicated and non-predicated 263 // targets. 264 bool is_mask = is_vector_mask(iklass); 265 if (is_mask && vec_value->Opcode() != Op_VectorStoreMask) { 266 const TypeVect* vt = vec_value->bottom_type()->is_vect(); 267 BasicType bt = vt->element_basic_type(); 268 vec_value = gvn.transform(VectorStoreMaskNode::make(gvn, vec_value, bt, vt->length())); 269 } 270 271 while (safepoints.size() > 0) { 272 SafePointNode* sfpt = safepoints.pop()->as_SafePoint(); 273 274 uint first_ind = (sfpt->req() - sfpt->jvms()->scloff()); 275 Node* sobj = new SafePointScalarObjectNode(vec_box->box_type(), 276 #ifdef ASSERT 277 vec_box, 278 #endif // ASSERT 279 first_ind, n_fields); 280 sobj->init_req(0, C->root()); 281 sfpt->add_req(vec_value); 282 283 sobj = gvn.transform(sobj); 284 285 JVMState *jvms = sfpt->jvms(); 286 287 jvms->set_endoff(sfpt->req()); 288 // Now make a pass over the debug information replacing any references 289 // to the allocated object with vector value. 290 for (uint i = jvms->debug_start(); i < jvms->debug_end(); i++) { 291 Node* debug = sfpt->in(i); 292 if (debug != nullptr && debug->uncast(/*keep_deps*/false) == vec_box) { 293 sfpt->set_req(i, sobj); 294 } 295 } 296 C->record_for_igvn(sfpt); 297 } 298 } 299 300 void PhaseVector::expand_vbox_node(VectorBoxNode* vec_box) { 301 if (vec_box->outcnt() > 0) { 302 VectorSet visited; 303 Node* vbox = vec_box->in(VectorBoxNode::Box); 304 Node* vect = vec_box->in(VectorBoxNode::Value); 305 Node* result = expand_vbox_node_helper(vbox, vect, vec_box->box_type(), 306 vec_box->vec_type(), visited); 307 C->gvn_replace_by(vec_box, result); 308 C->print_method(PHASE_EXPAND_VBOX, 3, vec_box); 309 } 310 C->remove_macro_node(vec_box); 311 } 312 313 Node* PhaseVector::expand_vbox_node_helper(Node* vbox, 314 Node* vect, 315 const TypeInstPtr* box_type, 316 const TypeVect* vect_type, 317 VectorSet &visited) { 318 // JDK-8304948 shows an example that there may be a cycle in the graph. 319 if (visited.test_set(vbox->_idx)) { 320 assert(vbox->is_Phi(), "should be phi"); 321 return vbox; // already visited 322 } 323 324 // Handle the case when the allocation input to VectorBoxNode is a Proj. 325 // This is the normal case before expanding. 326 if (vbox->is_Proj() && vbox->in(0)->Opcode() == Op_VectorBoxAllocate) { 327 VectorBoxAllocateNode* vbox_alloc = static_cast<VectorBoxAllocateNode*>(vbox->in(0)); 328 return expand_vbox_alloc_node(vbox_alloc, vect, box_type, vect_type); 329 } 330 331 // Handle the case when both the allocation input and vector input to 332 // VectorBoxNode are Phi. This case is generated after the transformation of 333 // Phi: Phi (VectorBox1 VectorBox2) => VectorBox (Phi1 Phi2). 334 // With this optimization, the relative two allocation inputs of VectorBox1 and 335 // VectorBox2 are gathered into Phi1 now. Similarly, the original vector 336 // inputs of two VectorBox nodes are in Phi2. 337 // 338 // See PhiNode::merge_through_phi in cfg.cpp for more details. 339 if (vbox->is_Phi() && vect->is_Phi()) { 340 assert(vbox->as_Phi()->region() == vect->as_Phi()->region(), ""); 341 for (uint i = 1; i < vbox->req(); i++) { 342 Node* new_box = expand_vbox_node_helper(vbox->in(i), vect->in(i), 343 box_type, vect_type, visited); 344 if (!new_box->is_Phi()) { 345 C->initial_gvn()->hash_delete(vbox); 346 vbox->set_req(i, new_box); 347 } 348 } 349 return C->initial_gvn()->transform(vbox); 350 } 351 352 // Handle the case when the allocation input to VectorBoxNode is a phi 353 // but the vector input is not, which can definitely be the case if the 354 // vector input has been value-numbered. It seems to be safe to do by 355 // construction because VectorBoxNode and VectorBoxAllocate come in a 356 // specific order as a result of expanding an intrinsic call. After that, if 357 // any of the inputs to VectorBoxNode are value-numbered they can only 358 // move up and are guaranteed to dominate. 359 if (vbox->is_Phi() && (vect->is_Vector() || vect->is_LoadVector())) { 360 for (uint i = 1; i < vbox->req(); i++) { 361 Node* new_box = expand_vbox_node_helper(vbox->in(i), vect, 362 box_type, vect_type, visited); 363 if (!new_box->is_Phi()) { 364 C->initial_gvn()->hash_delete(vbox); 365 vbox->set_req(i, new_box); 366 } 367 } 368 return C->initial_gvn()->transform(vbox); 369 } 370 371 assert(!vbox->is_Phi(), "should be expanded"); 372 // TODO: assert that expanded vbox is initialized with the same value (vect). 373 return vbox; // already expanded 374 } 375 376 Node* PhaseVector::expand_vbox_alloc_node(VectorBoxAllocateNode* vbox_alloc, 377 Node* value, 378 const TypeInstPtr* box_type, 379 const TypeVect* vect_type) { 380 JVMState* jvms = clone_jvms(C, vbox_alloc); 381 GraphKit kit(jvms); 382 PhaseGVN& gvn = kit.gvn(); 383 384 ciInstanceKlass* box_klass = box_type->instance_klass(); 385 BasicType bt = vect_type->element_basic_type(); 386 int num_elem = vect_type->length(); 387 388 bool is_mask = is_vector_mask(box_klass); 389 // If boxed mask value is present in a predicate register, it must be 390 // spilled to a vector though a VectorStoreMaskOperation before actual StoreVector 391 // operation to vector payload field. 392 if (is_mask && (value->bottom_type()->isa_vectmask() || bt != T_BOOLEAN)) { 393 value = gvn.transform(VectorStoreMaskNode::make(gvn, value, bt, num_elem)); 394 // Although type of mask depends on its definition, in terms of storage everything is stored in boolean array. 395 bt = T_BOOLEAN; 396 assert(value->bottom_type()->is_vect()->element_basic_type() == bt, 397 "must be consistent with mask representation"); 398 } 399 400 // Generate array allocation for the field which holds the values. 401 const TypeKlassPtr* array_klass = TypeKlassPtr::make(ciTypeArrayKlass::make(bt)); 402 Node* arr = kit.new_array(kit.makecon(array_klass), kit.intcon(num_elem), 1); 403 404 // Store the vector value into the array. 405 // (The store should be captured by InitializeNode and turned into initialized store later.) 406 Node* arr_adr = kit.array_element_address(arr, kit.intcon(0), bt); 407 const TypePtr* arr_adr_type = arr_adr->bottom_type()->is_ptr(); 408 Node* arr_mem = kit.memory(arr_adr); 409 Node* vstore = gvn.transform(StoreVectorNode::make(0, 410 kit.control(), 411 arr_mem, 412 arr_adr, 413 arr_adr_type, 414 value, 415 num_elem)); 416 kit.set_memory(vstore, arr_adr_type); 417 418 C->set_max_vector_size(MAX2(C->max_vector_size(), vect_type->length_in_bytes())); 419 420 // Generate the allocate for the Vector object. 421 const TypeKlassPtr* klass_type = box_type->as_klass_type(); 422 Node* klass_node = kit.makecon(klass_type); 423 Node* vec_obj = kit.new_instance(klass_node); 424 425 // Store the allocated array into object. 426 ciField* field = ciEnv::current()->vector_VectorPayload_klass()->get_field_by_name(ciSymbols::payload_name(), 427 ciSymbols::object_signature(), 428 false); 429 assert(field != nullptr, ""); 430 Node* vec_field = kit.basic_plus_adr(vec_obj, field->offset_in_bytes()); 431 const TypePtr* vec_adr_type = vec_field->bottom_type()->is_ptr(); 432 433 // The store should be captured by InitializeNode and turned into initialized store later. 434 Node* field_store = gvn.transform(kit.access_store_at(vec_obj, 435 vec_field, 436 vec_adr_type, 437 arr, 438 TypeOopPtr::make_from_klass(field->type()->as_klass()), 439 T_OBJECT, 440 IN_HEAP)); 441 kit.set_memory(field_store, vec_adr_type); 442 443 kit.replace_call(vbox_alloc, vec_obj, true); 444 C->remove_macro_node(vbox_alloc); 445 446 return vec_obj; 447 } 448 449 void PhaseVector::expand_vunbox_node(VectorUnboxNode* vec_unbox) { 450 if (vec_unbox->outcnt() > 0) { 451 GraphKit kit; 452 PhaseGVN& gvn = kit.gvn(); 453 454 Node* obj = vec_unbox->obj(); 455 const TypeInstPtr* tinst = gvn.type(obj)->isa_instptr(); 456 ciInstanceKlass* from_kls = tinst->instance_klass(); 457 const TypeVect* vt = vec_unbox->bottom_type()->is_vect(); 458 BasicType bt = vt->element_basic_type(); 459 BasicType masktype = bt; 460 461 if (is_vector_mask(from_kls)) { 462 bt = T_BOOLEAN; 463 } 464 465 ciField* field = ciEnv::current()->vector_VectorPayload_klass()->get_field_by_name(ciSymbols::payload_name(), 466 ciSymbols::object_signature(), 467 false); 468 assert(field != nullptr, ""); 469 int offset = field->offset_in_bytes(); 470 Node* vec_adr = kit.basic_plus_adr(obj, offset); 471 472 Node* mem = vec_unbox->mem(); 473 Node* ctrl = vec_unbox->in(0); 474 Node* vec_field_ld; 475 { 476 DecoratorSet decorators = MO_UNORDERED | IN_HEAP; 477 C2AccessValuePtr addr(vec_adr, vec_adr->bottom_type()->is_ptr()); 478 MergeMemNode* local_mem = MergeMemNode::make(mem); 479 gvn.record_for_igvn(local_mem); 480 BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2(); 481 C2OptAccess access(gvn, ctrl, local_mem, decorators, T_OBJECT, obj, addr); 482 const Type* type = TypeOopPtr::make_from_klass(field->type()->as_klass()); 483 vec_field_ld = bs->load_at(access, type); 484 } 485 486 // For proper aliasing, attach concrete payload type. 487 ciKlass* payload_klass = ciTypeArrayKlass::make(bt); 488 const Type* payload_type = TypeAryPtr::make_from_klass(payload_klass)->cast_to_ptr_type(TypePtr::NotNull); 489 vec_field_ld = gvn.transform(new CastPPNode(vec_field_ld, payload_type)); 490 491 Node* adr = kit.array_element_address(vec_field_ld, gvn.intcon(0), bt); 492 const TypePtr* adr_type = adr->bottom_type()->is_ptr(); 493 int num_elem = vt->length(); 494 Node* vec_val_load = LoadVectorNode::make(0, 495 ctrl, 496 mem, 497 adr, 498 adr_type, 499 num_elem, 500 bt); 501 vec_val_load = gvn.transform(vec_val_load); 502 503 C->set_max_vector_size(MAX2(C->max_vector_size(), vt->length_in_bytes())); 504 505 if (is_vector_mask(from_kls)) { 506 vec_val_load = gvn.transform(new VectorLoadMaskNode(vec_val_load, TypeVect::makemask(masktype, num_elem))); 507 } 508 509 gvn.hash_delete(vec_unbox); 510 vec_unbox->disconnect_inputs(C); 511 C->gvn_replace_by(vec_unbox, vec_val_load); 512 } 513 C->remove_macro_node(vec_unbox); 514 } 515 516 void PhaseVector::eliminate_vbox_alloc_node(VectorBoxAllocateNode* vbox_alloc) { 517 JVMState* jvms = clone_jvms(C, vbox_alloc); 518 GraphKit kit(jvms); 519 // Remove VBA, but leave a safepoint behind. 520 // Otherwise, it may end up with a loop without any safepoint polls. 521 kit.replace_call(vbox_alloc, kit.map(), true); 522 C->remove_macro_node(vbox_alloc); 523 }