1 /* 2 * Copyright (c) 2020, 2024, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 * 23 */ 24 25 #include "precompiled.hpp" 26 #include "ci/ciSymbols.hpp" 27 #include "gc/shared/barrierSet.hpp" 28 #include "opto/castnode.hpp" 29 #include "opto/graphKit.hpp" 30 #include "opto/phaseX.hpp" 31 #include "opto/rootnode.hpp" 32 #include "opto/vector.hpp" 33 #include "utilities/macros.hpp" 34 35 static bool is_vector_mask(ciKlass* klass) { 36 return klass->is_subclass_of(ciEnv::current()->vector_VectorMask_klass()); 37 } 38 39 static bool is_vector_shuffle(ciKlass* klass) { 40 return klass->is_subclass_of(ciEnv::current()->vector_VectorShuffle_klass()); 41 } 42 43 44 void PhaseVector::optimize_vector_boxes() { 45 Compile::TracePhase tp("vector_elimination", &timers[_t_vector_elimination]); 46 47 // Signal GraphKit it's post-parse phase. 48 assert(C->inlining_incrementally() == false, "sanity"); 49 C->set_inlining_incrementally(true); 50 51 C->igvn_worklist()->ensure_empty(); // should be done with igvn 52 53 expand_vunbox_nodes(); 54 scalarize_vbox_nodes(); 55 56 C->inline_vector_reboxing_calls(); 57 58 expand_vbox_nodes(); 59 eliminate_vbox_alloc_nodes(); 60 61 C->set_inlining_incrementally(false); 62 63 do_cleanup(); 64 } 65 66 void PhaseVector::do_cleanup() { 67 if (C->failing()) return; 68 { 69 Compile::TracePhase tp("vector_pru", &timers[_t_vector_pru]); 70 ResourceMark rm; 71 PhaseRemoveUseless pru(C->initial_gvn(), *C->igvn_worklist()); 72 if (C->failing()) return; 73 } 74 { 75 Compile::TracePhase tp("incrementalInline_igvn", &timers[_t_vector_igvn]); 76 _igvn.reset_from_gvn(C->initial_gvn()); 77 _igvn.optimize(); 78 if (C->failing()) return; 79 } 80 C->print_method(PHASE_ITER_GVN_BEFORE_EA, 3); 81 } 82 83 void PhaseVector::scalarize_vbox_nodes() { 84 if (C->failing()) return; 85 86 if (!EnableVectorReboxing) { 87 return; // don't scalarize vector boxes 88 } 89 90 int macro_idx = C->macro_count() - 1; 91 while (macro_idx >= 0) { 92 Node * n = C->macro_node(macro_idx); 93 assert(n->is_macro(), "only macro nodes expected here"); 94 if (n->Opcode() == Op_VectorBox) { 95 VectorBoxNode* vbox = static_cast<VectorBoxNode*>(n); 96 scalarize_vbox_node(vbox); 97 if (C->failing()) return; 98 C->print_method(PHASE_SCALARIZE_VBOX, 3, vbox); 99 } 100 if (C->failing()) return; 101 macro_idx = MIN2(macro_idx - 1, C->macro_count() - 1); 102 } 103 } 104 105 void PhaseVector::expand_vbox_nodes() { 106 if (C->failing()) return; 107 108 int macro_idx = C->macro_count() - 1; 109 while (macro_idx >= 0) { 110 Node * n = C->macro_node(macro_idx); 111 assert(n->is_macro(), "only macro nodes expected here"); 112 if (n->Opcode() == Op_VectorBox) { 113 VectorBoxNode* vbox = static_cast<VectorBoxNode*>(n); 114 expand_vbox_node(vbox); 115 if (C->failing()) return; 116 } 117 if (C->failing()) return; 118 macro_idx = MIN2(macro_idx - 1, C->macro_count() - 1); 119 } 120 } 121 122 void PhaseVector::expand_vunbox_nodes() { 123 if (C->failing()) return; 124 125 int macro_idx = C->macro_count() - 1; 126 while (macro_idx >= 0) { 127 Node * n = C->macro_node(macro_idx); 128 assert(n->is_macro(), "only macro nodes expected here"); 129 if (n->Opcode() == Op_VectorUnbox) { 130 VectorUnboxNode* vec_unbox = static_cast<VectorUnboxNode*>(n); 131 expand_vunbox_node(vec_unbox); 132 if (C->failing()) return; 133 C->print_method(PHASE_EXPAND_VUNBOX, 3, vec_unbox); 134 } 135 if (C->failing()) return; 136 macro_idx = MIN2(macro_idx - 1, C->macro_count() - 1); 137 } 138 } 139 140 void PhaseVector::eliminate_vbox_alloc_nodes() { 141 if (C->failing()) return; 142 143 int macro_idx = C->macro_count() - 1; 144 while (macro_idx >= 0) { 145 Node * n = C->macro_node(macro_idx); 146 assert(n->is_macro(), "only macro nodes expected here"); 147 if (n->Opcode() == Op_VectorBoxAllocate) { 148 VectorBoxAllocateNode* vbox_alloc = static_cast<VectorBoxAllocateNode*>(n); 149 eliminate_vbox_alloc_node(vbox_alloc); 150 if (C->failing()) return; 151 C->print_method(PHASE_ELIMINATE_VBOX_ALLOC, 3, vbox_alloc); 152 } 153 if (C->failing()) return; 154 macro_idx = MIN2(macro_idx - 1, C->macro_count() - 1); 155 } 156 } 157 158 static JVMState* clone_jvms(Compile* C, SafePointNode* sfpt) { 159 JVMState* new_jvms = sfpt->jvms()->clone_shallow(C); 160 uint size = sfpt->req(); 161 SafePointNode* map = new SafePointNode(size, new_jvms); 162 for (uint i = 0; i < size; i++) { 163 map->init_req(i, sfpt->in(i)); 164 } 165 Node* mem = map->memory(); 166 if (!mem->is_MergeMem()) { 167 // Since we are not in parsing, the SafePointNode does not guarantee that the memory 168 // input is necessarily a MergeMemNode. But we need to ensure that there is that 169 // MergeMemNode, since the GraphKit assumes the memory input of the map to be a 170 // MergeMemNode, so that it can directly access the memory slices. 171 PhaseGVN& gvn = *C->initial_gvn(); 172 Node* mergemem = MergeMemNode::make(mem); 173 gvn.set_type_bottom(mergemem); 174 map->set_memory(mergemem); 175 } 176 new_jvms->set_map(map); 177 return new_jvms; 178 } 179 180 void PhaseVector::scalarize_vbox_node(VectorBoxNode* vec_box) { 181 Node* vec_value = vec_box->in(VectorBoxNode::Value); 182 PhaseGVN& gvn = *C->initial_gvn(); 183 184 // Process merged VBAs 185 186 if (EnableVectorAggressiveReboxing) { 187 Unique_Node_List calls; 188 for (DUIterator_Fast imax, i = vec_box->fast_outs(imax); i < imax; i++) { 189 Node* use = vec_box->fast_out(i); 190 if (use->is_CallJava()) { 191 CallJavaNode* call = use->as_CallJava(); 192 if (call->has_non_debug_use(vec_box) && vec_box->in(VectorBoxNode::Box)->is_Phi()) { 193 calls.push(call); 194 } 195 } 196 } 197 198 while (calls.size() > 0) { 199 CallJavaNode* call = calls.pop()->as_CallJava(); 200 // Attach new VBA to the call and use it instead of Phi (VBA ... VBA). 201 202 JVMState* jvms = clone_jvms(C, call); 203 GraphKit kit(jvms); 204 PhaseGVN& gvn = kit.gvn(); 205 206 // Adjust JVMS from post-call to pre-call state: put args on stack 207 uint nargs = call->method()->arg_size(); 208 kit.ensure_stack(kit.sp() + nargs); 209 for (uint i = TypeFunc::Parms; i < call->tf()->domain()->cnt(); i++) { 210 kit.push(call->in(i)); 211 } 212 jvms = kit.sync_jvms(); 213 214 Node* new_vbox = nullptr; 215 { 216 Node* vect = vec_box->in(VectorBoxNode::Value); 217 const TypeInstPtr* vbox_type = vec_box->box_type(); 218 const TypeVect* vt = vec_box->vec_type(); 219 BasicType elem_bt = vt->element_basic_type(); 220 int num_elem = vt->length(); 221 222 new_vbox = kit.box_vector(vect, vbox_type, elem_bt, num_elem, /*deoptimize=*/true); 223 224 kit.replace_in_map(vec_box, new_vbox); 225 } 226 227 kit.dec_sp(nargs); 228 jvms = kit.sync_jvms(); 229 230 call->set_req(TypeFunc::Control , kit.control()); 231 call->set_req(TypeFunc::I_O , kit.i_o()); 232 call->set_req(TypeFunc::Memory , kit.reset_memory()); 233 call->set_req(TypeFunc::FramePtr, kit.frameptr()); 234 call->replace_edge(vec_box, new_vbox); 235 236 C->record_for_igvn(call); 237 } 238 } 239 240 // Process debug uses at safepoints 241 Unique_Node_List safepoints; 242 243 Unique_Node_List worklist; 244 worklist.push(vec_box); 245 while (worklist.size() > 0) { 246 Node* n = worklist.pop(); 247 for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) { 248 Node* use = n->fast_out(i); 249 if (use->is_SafePoint()) { 250 SafePointNode* sfpt = use->as_SafePoint(); 251 if (!sfpt->is_Call() || !sfpt->as_Call()->has_non_debug_use(n)) { 252 safepoints.push(sfpt); 253 } 254 } else if (use->is_ConstraintCast()) { 255 worklist.push(use); // reversed version of Node::uncast() 256 } 257 } 258 } 259 260 ciInstanceKlass* iklass = vec_box->box_type()->instance_klass(); 261 int n_fields = iklass->nof_nonstatic_fields(); 262 assert(n_fields == 1, "sanity"); 263 264 // If a mask is feeding into safepoint[s], then its value should be 265 // packed into a boolean/byte vector first, this will simplify the 266 // re-materialization logic for both predicated and non-predicated 267 // targets. 268 bool is_mask = is_vector_mask(iklass); 269 if (is_mask && vec_value->Opcode() != Op_VectorStoreMask) { 270 const TypeVect* vt = vec_value->bottom_type()->is_vect(); 271 BasicType bt = vt->element_basic_type(); 272 vec_value = gvn.transform(VectorStoreMaskNode::make(gvn, vec_value, bt, vt->length())); 273 } 274 275 while (safepoints.size() > 0) { 276 SafePointNode* sfpt = safepoints.pop()->as_SafePoint(); 277 278 uint first_ind = (sfpt->req() - sfpt->jvms()->scloff()); 279 Node* sobj = new SafePointScalarObjectNode(vec_box->box_type(), vec_box, first_ind, sfpt->jvms()->depth(), n_fields); 280 sobj->init_req(0, C->root()); 281 sfpt->add_req(vec_value); 282 283 sobj = gvn.transform(sobj); 284 285 JVMState *jvms = sfpt->jvms(); 286 287 jvms->set_endoff(sfpt->req()); 288 // Now make a pass over the debug information replacing any references 289 // to the allocated object with vector value. 290 for (uint i = jvms->debug_start(); i < jvms->debug_end(); i++) { 291 Node* debug = sfpt->in(i); 292 if (debug != nullptr && debug->uncast(/*keep_deps*/false) == vec_box) { 293 sfpt->set_req(i, sobj); 294 } 295 } 296 C->record_for_igvn(sfpt); 297 } 298 } 299 300 void PhaseVector::expand_vbox_node(VectorBoxNode* vec_box) { 301 if (vec_box->outcnt() > 0) { 302 VectorSet visited; 303 Node* vbox = vec_box->in(VectorBoxNode::Box); 304 Node* vect = vec_box->in(VectorBoxNode::Value); 305 Node* result = expand_vbox_node_helper(vbox, vect, vec_box->box_type(), 306 vec_box->vec_type(), visited); 307 C->gvn_replace_by(vec_box, result); 308 C->print_method(PHASE_EXPAND_VBOX, 3, vec_box); 309 } 310 C->remove_macro_node(vec_box); 311 } 312 313 Node* PhaseVector::expand_vbox_node_helper(Node* vbox, 314 Node* vect, 315 const TypeInstPtr* box_type, 316 const TypeVect* vect_type, 317 VectorSet &visited) { 318 // JDK-8304948 shows an example that there may be a cycle in the graph. 319 if (visited.test_set(vbox->_idx)) { 320 assert(vbox->is_Phi(), "should be phi"); 321 return vbox; // already visited 322 } 323 324 // Handle the case when the allocation input to VectorBoxNode is a Proj. 325 // This is the normal case before expanding. 326 if (vbox->is_Proj() && vbox->in(0)->Opcode() == Op_VectorBoxAllocate) { 327 VectorBoxAllocateNode* vbox_alloc = static_cast<VectorBoxAllocateNode*>(vbox->in(0)); 328 return expand_vbox_alloc_node(vbox_alloc, vect, box_type, vect_type); 329 } 330 331 // Handle the case when both the allocation input and vector input to 332 // VectorBoxNode are Phi. This case is generated after the transformation of 333 // Phi: Phi (VectorBox1 VectorBox2) => VectorBox (Phi1 Phi2). 334 // With this optimization, the relative two allocation inputs of VectorBox1 and 335 // VectorBox2 are gathered into Phi1 now. Similarly, the original vector 336 // inputs of two VectorBox nodes are in Phi2. 337 // 338 // See PhiNode::merge_through_phi in cfg.cpp for more details. 339 if (vbox->is_Phi() && vect->is_Phi()) { 340 assert(vbox->as_Phi()->region() == vect->as_Phi()->region(), ""); 341 for (uint i = 1; i < vbox->req(); i++) { 342 Node* new_box = expand_vbox_node_helper(vbox->in(i), vect->in(i), 343 box_type, vect_type, visited); 344 if (!new_box->is_Phi()) { 345 C->initial_gvn()->hash_delete(vbox); 346 vbox->set_req(i, new_box); 347 } 348 } 349 return C->initial_gvn()->transform(vbox); 350 } 351 352 // Handle the case when the allocation input to VectorBoxNode is a phi 353 // but the vector input is not, which can definitely be the case if the 354 // vector input has been value-numbered. It seems to be safe to do by 355 // construction because VectorBoxNode and VectorBoxAllocate come in a 356 // specific order as a result of expanding an intrinsic call. After that, if 357 // any of the inputs to VectorBoxNode are value-numbered they can only 358 // move up and are guaranteed to dominate. 359 if (vbox->is_Phi() && (vect->is_Vector() || vect->is_LoadVector())) { 360 for (uint i = 1; i < vbox->req(); i++) { 361 Node* new_box = expand_vbox_node_helper(vbox->in(i), vect, 362 box_type, vect_type, visited); 363 if (!new_box->is_Phi()) { 364 C->initial_gvn()->hash_delete(vbox); 365 vbox->set_req(i, new_box); 366 } 367 } 368 return C->initial_gvn()->transform(vbox); 369 } 370 371 assert(!vbox->is_Phi(), "should be expanded"); 372 // TODO: assert that expanded vbox is initialized with the same value (vect). 373 return vbox; // already expanded 374 } 375 376 Node* PhaseVector::expand_vbox_alloc_node(VectorBoxAllocateNode* vbox_alloc, 377 Node* value, 378 const TypeInstPtr* box_type, 379 const TypeVect* vect_type) { 380 JVMState* jvms = clone_jvms(C, vbox_alloc); 381 GraphKit kit(jvms); 382 PhaseGVN& gvn = kit.gvn(); 383 384 ciInstanceKlass* box_klass = box_type->instance_klass(); 385 BasicType bt = vect_type->element_basic_type(); 386 int num_elem = vect_type->length(); 387 388 bool is_mask = is_vector_mask(box_klass); 389 // If boxed mask value is present in a predicate register, it must be 390 // spilled to a vector though a VectorStoreMaskOperation before actual StoreVector 391 // operation to vector payload field. 392 if (is_mask && (value->bottom_type()->isa_vectmask() || bt != T_BOOLEAN)) { 393 value = gvn.transform(VectorStoreMaskNode::make(gvn, value, bt, num_elem)); 394 // Although type of mask depends on its definition, in terms of storage everything is stored in boolean array. 395 bt = T_BOOLEAN; 396 assert(value->bottom_type()->is_vect()->element_basic_type() == bt, 397 "must be consistent with mask representation"); 398 } 399 400 // Generate array allocation for the field which holds the values. 401 const TypeKlassPtr* array_klass = TypeKlassPtr::make(ciTypeArrayKlass::make(bt)); 402 Node* arr = kit.new_array(kit.makecon(array_klass), kit.intcon(num_elem), 1); 403 404 // Store the vector value into the array. 405 // (The store should be captured by InitializeNode and turned into initialized store later.) 406 Node* arr_adr = kit.array_element_address(arr, kit.intcon(0), bt); 407 const TypePtr* arr_adr_type = arr_adr->bottom_type()->is_ptr(); 408 Node* arr_mem = kit.memory(arr_adr); 409 Node* vstore = gvn.transform(StoreVectorNode::make(0, 410 kit.control(), 411 arr_mem, 412 arr_adr, 413 arr_adr_type, 414 value, 415 num_elem)); 416 kit.set_memory(vstore, arr_adr_type); 417 418 C->set_max_vector_size(MAX2(C->max_vector_size(), vect_type->length_in_bytes())); 419 420 // Generate the allocate for the Vector object. 421 const TypeKlassPtr* klass_type = box_type->as_klass_type(); 422 Node* klass_node = kit.makecon(klass_type); 423 Node* vec_obj = kit.new_instance(klass_node); 424 425 // Store the allocated array into object. 426 ciField* field = ciEnv::current()->vector_VectorPayload_klass()->get_field_by_name(ciSymbols::payload_name(), 427 ciSymbols::object_signature(), 428 false); 429 assert(field != nullptr, ""); 430 Node* vec_field = kit.basic_plus_adr(vec_obj, field->offset_in_bytes()); 431 const TypePtr* vec_adr_type = vec_field->bottom_type()->is_ptr(); 432 433 // The store should be captured by InitializeNode and turned into initialized store later. 434 Node* field_store = gvn.transform(kit.access_store_at(vec_obj, 435 vec_field, 436 vec_adr_type, 437 arr, 438 TypeOopPtr::make_from_klass(field->type()->as_klass()), 439 T_OBJECT, 440 IN_HEAP)); 441 kit.set_memory(field_store, vec_adr_type); 442 443 kit.replace_call(vbox_alloc, vec_obj, true); 444 C->remove_macro_node(vbox_alloc); 445 446 return vec_obj; 447 } 448 449 void PhaseVector::expand_vunbox_node(VectorUnboxNode* vec_unbox) { 450 if (vec_unbox->outcnt() > 0) { 451 GraphKit kit; 452 PhaseGVN& gvn = kit.gvn(); 453 454 Node* obj = vec_unbox->obj(); 455 const TypeInstPtr* tinst = gvn.type(obj)->isa_instptr(); 456 ciInstanceKlass* from_kls = tinst->instance_klass(); 457 const TypeVect* vt = vec_unbox->bottom_type()->is_vect(); 458 BasicType bt = vt->element_basic_type(); 459 BasicType masktype = bt; 460 461 if (is_vector_mask(from_kls)) { 462 bt = T_BOOLEAN; 463 } else if (is_vector_shuffle(from_kls)) { 464 bt = T_BYTE; 465 } 466 467 ciField* field = ciEnv::current()->vector_VectorPayload_klass()->get_field_by_name(ciSymbols::payload_name(), 468 ciSymbols::object_signature(), 469 false); 470 assert(field != nullptr, ""); 471 int offset = field->offset_in_bytes(); 472 Node* vec_adr = kit.basic_plus_adr(obj, offset); 473 474 Node* mem = vec_unbox->mem(); 475 Node* ctrl = vec_unbox->in(0); 476 Node* vec_field_ld; 477 { 478 DecoratorSet decorators = MO_UNORDERED | IN_HEAP; 479 C2AccessValuePtr addr(vec_adr, vec_adr->bottom_type()->is_ptr()); 480 MergeMemNode* local_mem = MergeMemNode::make(mem); 481 gvn.record_for_igvn(local_mem); 482 BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2(); 483 C2OptAccess access(gvn, ctrl, local_mem, decorators, T_OBJECT, obj, addr); 484 const Type* type = TypeOopPtr::make_from_klass(field->type()->as_klass()); 485 vec_field_ld = bs->load_at(access, type); 486 } 487 488 // For proper aliasing, attach concrete payload type. 489 ciKlass* payload_klass = ciTypeArrayKlass::make(bt); 490 const Type* payload_type = TypeAryPtr::make_from_klass(payload_klass)->cast_to_ptr_type(TypePtr::NotNull); 491 vec_field_ld = gvn.transform(new CastPPNode(nullptr, vec_field_ld, payload_type)); 492 493 Node* adr = kit.array_element_address(vec_field_ld, gvn.intcon(0), bt); 494 const TypePtr* adr_type = adr->bottom_type()->is_ptr(); 495 int num_elem = vt->length(); 496 Node* vec_val_load = LoadVectorNode::make(0, 497 ctrl, 498 mem, 499 adr, 500 adr_type, 501 num_elem, 502 bt); 503 vec_val_load = gvn.transform(vec_val_load); 504 505 C->set_max_vector_size(MAX2(C->max_vector_size(), vt->length_in_bytes())); 506 507 if (is_vector_mask(from_kls)) { 508 vec_val_load = gvn.transform(new VectorLoadMaskNode(vec_val_load, TypeVect::makemask(masktype, num_elem))); 509 } else if (is_vector_shuffle(from_kls) && !vec_unbox->is_shuffle_to_vector()) { 510 assert(vec_unbox->bottom_type()->is_vect()->element_basic_type() == masktype, "expect shuffle type consistency"); 511 vec_val_load = gvn.transform(new VectorLoadShuffleNode(vec_val_load, TypeVect::make(masktype, num_elem))); 512 } 513 514 gvn.hash_delete(vec_unbox); 515 vec_unbox->disconnect_inputs(C); 516 C->gvn_replace_by(vec_unbox, vec_val_load); 517 } 518 C->remove_macro_node(vec_unbox); 519 } 520 521 void PhaseVector::eliminate_vbox_alloc_node(VectorBoxAllocateNode* vbox_alloc) { 522 JVMState* jvms = clone_jvms(C, vbox_alloc); 523 GraphKit kit(jvms); 524 // Remove VBA, but leave a safepoint behind. 525 // Otherwise, it may end up with a loop without any safepoint polls. 526 kit.replace_call(vbox_alloc, kit.map(), true); 527 C->remove_macro_node(vbox_alloc); 528 }