1 /*
2 * Copyright (c) 2020, 2025, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "ci/ciSymbols.hpp"
26 #include "gc/shared/barrierSet.hpp"
27 #include "opto/castnode.hpp"
28 #include "opto/graphKit.hpp"
29 #include "opto/phaseX.hpp"
30 #include "opto/rootnode.hpp"
31 #include "opto/vector.hpp"
32 #include "utilities/macros.hpp"
33
34 static bool is_vector_mask(ciKlass* klass) {
35 return klass->is_subclass_of(ciEnv::current()->vector_VectorMask_klass());
36 }
37
38 void PhaseVector::optimize_vector_boxes() {
39 Compile::TracePhase tp(_t_vector_elimination);
40
41 // Signal GraphKit it's post-parse phase.
42 assert(C->inlining_incrementally() == false, "sanity");
43 C->set_inlining_incrementally(true);
44
45 C->igvn_worklist()->ensure_empty(); // should be done with igvn
46
47 if (StressMacroExpansion) {
48 C->shuffle_macro_nodes();
49 }
50
51 expand_vunbox_nodes();
52 scalarize_vbox_nodes();
53
54 C->inline_vector_reboxing_calls();
55
56 expand_vbox_nodes();
57 eliminate_vbox_alloc_nodes();
58
59 C->set_inlining_incrementally(false);
60
61 do_cleanup();
62 }
63
64 void PhaseVector::do_cleanup() {
65 if (C->failing()) return;
66 {
67 Compile::TracePhase tp(_t_vector_pru);
68 ResourceMark rm;
69 PhaseRemoveUseless pru(C->initial_gvn(), *C->igvn_worklist());
70 if (C->failing()) return;
71 }
72 {
73 Compile::TracePhase tp(_t_vector_igvn);
74 _igvn.reset_from_gvn(C->initial_gvn());
75 _igvn.optimize();
76 if (C->failing()) return;
77 }
78 C->print_method(PHASE_ITER_GVN_BEFORE_EA, 3);
79 }
80
81 void PhaseVector::scalarize_vbox_nodes() {
82 if (C->failing()) return;
83
84 if (!EnableVectorReboxing) {
85 return; // don't scalarize vector boxes
86 }
87
88 int macro_idx = C->macro_count() - 1;
89 while (macro_idx >= 0) {
90 Node * n = C->macro_node(macro_idx);
91 assert(n->is_macro(), "only macro nodes expected here");
92 if (n->Opcode() == Op_VectorBox) {
93 VectorBoxNode* vbox = static_cast<VectorBoxNode*>(n);
94 scalarize_vbox_node(vbox);
95 if (C->failing()) return;
96 C->print_method(PHASE_SCALARIZE_VBOX, 3, vbox);
97 }
98 if (C->failing()) return;
99 macro_idx = MIN2(macro_idx - 1, C->macro_count() - 1);
100 }
101 }
102
103 void PhaseVector::expand_vbox_nodes() {
104 if (C->failing()) return;
105
106 int macro_idx = C->macro_count() - 1;
107 while (macro_idx >= 0) {
108 Node * n = C->macro_node(macro_idx);
109 assert(n->is_macro(), "only macro nodes expected here");
110 if (n->Opcode() == Op_VectorBox) {
111 VectorBoxNode* vbox = static_cast<VectorBoxNode*>(n);
112 expand_vbox_node(vbox);
113 if (C->failing()) return;
114 }
115 if (C->failing()) return;
116 macro_idx = MIN2(macro_idx - 1, C->macro_count() - 1);
117 }
118 }
119
120 void PhaseVector::expand_vunbox_nodes() {
121 if (C->failing()) return;
122
123 int macro_idx = C->macro_count() - 1;
124 while (macro_idx >= 0) {
125 Node * n = C->macro_node(macro_idx);
126 assert(n->is_macro(), "only macro nodes expected here");
127 if (n->Opcode() == Op_VectorUnbox) {
128 VectorUnboxNode* vec_unbox = static_cast<VectorUnboxNode*>(n);
129 expand_vunbox_node(vec_unbox);
130 if (C->failing()) return;
131 C->print_method(PHASE_EXPAND_VUNBOX, 3, vec_unbox);
132 }
133 if (C->failing()) return;
134 macro_idx = MIN2(macro_idx - 1, C->macro_count() - 1);
135 }
136 }
137
138 void PhaseVector::eliminate_vbox_alloc_nodes() {
139 if (C->failing()) return;
140
141 int macro_idx = C->macro_count() - 1;
142 while (macro_idx >= 0) {
143 Node * n = C->macro_node(macro_idx);
144 assert(n->is_macro(), "only macro nodes expected here");
145 if (n->Opcode() == Op_VectorBoxAllocate) {
146 VectorBoxAllocateNode* vbox_alloc = static_cast<VectorBoxAllocateNode*>(n);
147 eliminate_vbox_alloc_node(vbox_alloc);
148 if (C->failing()) return;
149 C->print_method(PHASE_ELIMINATE_VBOX_ALLOC, 3, vbox_alloc);
150 }
151 if (C->failing()) return;
152 macro_idx = MIN2(macro_idx - 1, C->macro_count() - 1);
153 }
154 }
155
156 static JVMState* clone_jvms(Compile* C, SafePointNode* sfpt) {
157 JVMState* new_jvms = sfpt->jvms()->clone_shallow(C);
158 uint size = sfpt->req();
159 SafePointNode* map = new SafePointNode(size, new_jvms);
160 for (uint i = 0; i < size; i++) {
161 map->init_req(i, sfpt->in(i));
162 }
163 Node* mem = map->memory();
164 if (!mem->is_MergeMem()) {
165 // Since we are not in parsing, the SafePointNode does not guarantee that the memory
166 // input is necessarily a MergeMemNode. But we need to ensure that there is that
167 // MergeMemNode, since the GraphKit assumes the memory input of the map to be a
168 // MergeMemNode, so that it can directly access the memory slices.
169 PhaseGVN& gvn = *C->initial_gvn();
170 Node* mergemem = MergeMemNode::make(mem);
171 gvn.set_type_bottom(mergemem);
172 map->set_memory(mergemem);
173 }
174 new_jvms->set_map(map);
175 return new_jvms;
176 }
177
178 void PhaseVector::scalarize_vbox_node(VectorBoxNode* vec_box) {
179 Node* vec_value = vec_box->in(VectorBoxNode::Value);
180 PhaseGVN& gvn = *C->initial_gvn();
181
182 // Process merged VBAs
183
184 if (EnableVectorAggressiveReboxing) {
185 Unique_Node_List calls;
186 for (DUIterator_Fast imax, i = vec_box->fast_outs(imax); i < imax; i++) {
187 Node* use = vec_box->fast_out(i);
188 if (use->is_CallJava()) {
189 CallJavaNode* call = use->as_CallJava();
190 if (call->has_non_debug_use(vec_box) && vec_box->in(VectorBoxNode::Box)->is_Phi()) {
191 calls.push(call);
192 }
193 }
194 }
195
196 while (calls.size() > 0) {
197 CallJavaNode* call = calls.pop()->as_CallJava();
198 // Attach new VBA to the call and use it instead of Phi (VBA ... VBA).
199
200 JVMState* jvms = clone_jvms(C, call);
201 GraphKit kit(jvms);
202 PhaseGVN& gvn = kit.gvn();
203
204 // Adjust JVMS from post-call to pre-call state: put args on stack
205 uint nargs = call->method()->arg_size();
206 kit.ensure_stack(kit.sp() + nargs);
207 for (uint i = TypeFunc::Parms; i < call->tf()->domain()->cnt(); i++) {
208 kit.push(call->in(i));
209 }
210 jvms = kit.sync_jvms();
211
212 Node* new_vbox = nullptr;
213 {
214 Node* vect = vec_box->in(VectorBoxNode::Value);
215 const TypeInstPtr* vbox_type = vec_box->box_type();
216 const TypeVect* vt = vec_box->vec_type();
217 BasicType elem_bt = vt->element_basic_type();
218 int num_elem = vt->length();
219
220 new_vbox = kit.box_vector(vect, vbox_type, elem_bt, num_elem, /*deoptimize=*/true);
221
222 kit.replace_in_map(vec_box, new_vbox);
223 }
224
225 kit.dec_sp(nargs);
226 jvms = kit.sync_jvms();
227
228 call->set_req(TypeFunc::Control , kit.control());
229 call->set_req(TypeFunc::I_O , kit.i_o());
230 call->set_req(TypeFunc::Memory , kit.reset_memory());
231 call->set_req(TypeFunc::FramePtr, kit.frameptr());
232 call->replace_edge(vec_box, new_vbox);
233
234 C->record_for_igvn(call);
235 }
236 }
237
238 // Process debug uses at safepoints
239 Unique_Node_List safepoints;
240
241 Unique_Node_List worklist;
242 worklist.push(vec_box);
243 while (worklist.size() > 0) {
244 Node* n = worklist.pop();
245 for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) {
246 Node* use = n->fast_out(i);
247 if (use->is_SafePoint()) {
248 SafePointNode* sfpt = use->as_SafePoint();
249 if (!sfpt->is_Call() || !sfpt->as_Call()->has_non_debug_use(n)) {
250 safepoints.push(sfpt);
251 }
252 } else if (use->is_ConstraintCast()) {
253 worklist.push(use); // reversed version of Node::uncast()
254 }
255 }
256 }
257
258 ciInstanceKlass* iklass = vec_box->box_type()->instance_klass();
259 int n_fields = iklass->nof_nonstatic_fields();
260 assert(n_fields == 1, "sanity");
261
262 // If a mask is feeding into safepoint[s], then its value should be
263 // packed into a boolean/byte vector first, this will simplify the
264 // re-materialization logic for both predicated and non-predicated
265 // targets.
266 bool is_mask = is_vector_mask(iklass);
267 if (is_mask && vec_value->Opcode() != Op_VectorStoreMask) {
268 const TypeVect* vt = vec_value->bottom_type()->is_vect();
269 BasicType bt = vt->element_basic_type();
270 vec_value = gvn.transform(VectorStoreMaskNode::make(gvn, vec_value, bt, vt->length()));
271 }
272
273 while (safepoints.size() > 0) {
274 SafePointNode* sfpt = safepoints.pop()->as_SafePoint();
275
276 uint first_ind = (sfpt->req() - sfpt->jvms()->scloff());
277 Node* sobj = new SafePointScalarObjectNode(vec_box->box_type(), vec_box, first_ind, sfpt->jvms()->depth(), n_fields);
278 sobj->init_req(0, C->root());
279 sfpt->add_req(vec_value);
280
281 sobj = gvn.transform(sobj);
282
283 JVMState *jvms = sfpt->jvms();
284
285 jvms->set_endoff(sfpt->req());
286 // Now make a pass over the debug information replacing any references
287 // to the allocated object with vector value.
288 for (uint i = jvms->debug_start(); i < jvms->debug_end(); i++) {
289 Node* debug = sfpt->in(i);
290 if (debug != nullptr && debug->uncast(/*keep_deps*/false) == vec_box) {
291 sfpt->set_req(i, sobj);
292 }
293 }
294 C->record_for_igvn(sfpt);
295 }
296 }
297
298 void PhaseVector::expand_vbox_node(VectorBoxNode* vec_box) {
299 if (vec_box->outcnt() > 0) {
300 VectorSet visited;
301 Node* vbox = vec_box->in(VectorBoxNode::Box);
302 Node* vect = vec_box->in(VectorBoxNode::Value);
303 Node* result = expand_vbox_node_helper(vbox, vect, vec_box->box_type(),
304 vec_box->vec_type(), visited);
305 C->gvn_replace_by(vec_box, result);
306 C->print_method(PHASE_EXPAND_VBOX, 3, vec_box);
307 }
308 C->remove_macro_node(vec_box);
309 }
310
311 Node* PhaseVector::expand_vbox_node_helper(Node* vbox,
312 Node* vect,
313 const TypeInstPtr* box_type,
314 const TypeVect* vect_type,
315 VectorSet &visited) {
316 // JDK-8304948 shows an example that there may be a cycle in the graph.
317 if (visited.test_set(vbox->_idx)) {
318 assert(vbox->is_Phi() || vbox->is_CheckCastPP(), "either phi or expanded");
319 return vbox; // already visited
320 }
321
322 // Handle the case when the allocation input to VectorBoxNode is a Proj.
323 // This is the normal case before expanding.
324 if (vbox->is_Proj() && vbox->in(0)->Opcode() == Op_VectorBoxAllocate) {
325 VectorBoxAllocateNode* vbox_alloc = static_cast<VectorBoxAllocateNode*>(vbox->in(0));
326 return expand_vbox_alloc_node(vbox_alloc, vect, box_type, vect_type);
327 }
328
329 // Handle the case when both the allocation input and vector input to
330 // VectorBoxNode are Phi. This case is generated after the transformation of
331 // Phi: Phi (VectorBox1 VectorBox2) => VectorBox (Phi1 Phi2).
332 // With this optimization, the relative two allocation inputs of VectorBox1 and
333 // VectorBox2 are gathered into Phi1 now. Similarly, the original vector
334 // inputs of two VectorBox nodes are in Phi2.
335 //
336 // See PhiNode::merge_through_phi in cfg.cpp for more details.
337 if (vbox->is_Phi() && vect->is_Phi()) {
338 assert(vbox->as_Phi()->region() == vect->as_Phi()->region(), "");
339 for (uint i = 1; i < vbox->req(); i++) {
340 Node* new_box = expand_vbox_node_helper(vbox->in(i), vect->in(i),
341 box_type, vect_type, visited);
342 if (!new_box->is_Phi()) {
343 C->initial_gvn()->hash_delete(vbox);
344 vbox->set_req(i, new_box);
345 }
346 }
347 return C->initial_gvn()->transform(vbox);
348 }
349
350 // Handle the case when the allocation input to VectorBoxNode is a phi
351 // but the vector input is not, which can definitely be the case if the
352 // vector input has been value-numbered. It seems to be safe to do by
353 // construction because VectorBoxNode and VectorBoxAllocate come in a
354 // specific order as a result of expanding an intrinsic call. After that, if
355 // any of the inputs to VectorBoxNode are value-numbered they can only
356 // move up and are guaranteed to dominate.
357 if (vbox->is_Phi() && (vect->is_Vector() || vect->is_LoadVector())) {
358 for (uint i = 1; i < vbox->req(); i++) {
359 Node* new_box = expand_vbox_node_helper(vbox->in(i), vect,
360 box_type, vect_type, visited);
361 if (!new_box->is_Phi()) {
362 C->initial_gvn()->hash_delete(vbox);
363 vbox->set_req(i, new_box);
364 }
365 }
366 return C->initial_gvn()->transform(vbox);
367 }
368
369 assert(vbox->is_CheckCastPP(), "should be expanded");
370 // TODO: assert that expanded vbox is initialized with the same value (vect).
371 return vbox; // already expanded
372 }
373
374 Node* PhaseVector::expand_vbox_alloc_node(VectorBoxAllocateNode* vbox_alloc,
375 Node* value,
376 const TypeInstPtr* box_type,
377 const TypeVect* vect_type) {
378 JVMState* jvms = clone_jvms(C, vbox_alloc);
379 GraphKit kit(jvms);
380 PhaseGVN& gvn = kit.gvn();
381
382 ciInstanceKlass* box_klass = box_type->instance_klass();
383 BasicType bt = vect_type->element_basic_type();
384 int num_elem = vect_type->length();
385
386 bool is_mask = is_vector_mask(box_klass);
387 // If boxed mask value is present in a predicate register, it must be
388 // spilled to a vector though a VectorStoreMaskOperation before actual StoreVector
389 // operation to vector payload field.
390 if (is_mask && (value->bottom_type()->isa_vectmask() || bt != T_BOOLEAN)) {
391 value = gvn.transform(VectorStoreMaskNode::make(gvn, value, bt, num_elem));
392 // Although type of mask depends on its definition, in terms of storage everything is stored in boolean array.
393 bt = T_BOOLEAN;
394 assert(value->bottom_type()->is_vect()->element_basic_type() == bt,
395 "must be consistent with mask representation");
396 }
397
398 // Generate array allocation for the field which holds the values.
399 const TypeKlassPtr* array_klass = TypeKlassPtr::make(ciTypeArrayKlass::make(bt));
400 Node* arr = kit.new_array(kit.makecon(array_klass), kit.intcon(num_elem), 1);
401
402 // Store the vector value into the array.
403 // (The store should be captured by InitializeNode and turned into initialized store later.)
404 Node* arr_adr = kit.array_element_address(arr, kit.intcon(0), bt);
405 const TypePtr* arr_adr_type = arr_adr->bottom_type()->is_ptr();
406 Node* arr_mem = kit.memory(arr_adr);
407 Node* vstore = gvn.transform(StoreVectorNode::make(0,
408 kit.control(),
409 arr_mem,
410 arr_adr,
411 arr_adr_type,
412 value,
413 num_elem));
414 kit.set_memory(vstore, arr_adr_type);
415
416 C->set_max_vector_size(MAX2(C->max_vector_size(), vect_type->length_in_bytes()));
417
418 // Generate the allocate for the Vector object.
419 const TypeKlassPtr* klass_type = box_type->as_klass_type();
420 Node* klass_node = kit.makecon(klass_type);
421 Node* vec_obj = kit.new_instance(klass_node);
422
423 // Store the allocated array into object.
424 ciField* field = ciEnv::current()->vector_VectorPayload_klass()->get_field_by_name(ciSymbols::payload_name(),
425 ciSymbols::object_signature(),
426 false);
427 assert(field != nullptr, "");
428 Node* vec_field = kit.basic_plus_adr(vec_obj, field->offset_in_bytes());
429 const TypePtr* vec_adr_type = vec_field->bottom_type()->is_ptr();
430
431 // The store should be captured by InitializeNode and turned into initialized store later.
432 Node* field_store = gvn.transform(kit.access_store_at(vec_obj,
433 vec_field,
434 vec_adr_type,
435 arr,
436 TypeOopPtr::make_from_klass(field->type()->as_klass()),
437 T_OBJECT,
438 IN_HEAP));
439 kit.set_memory(field_store, vec_adr_type);
440
441 kit.replace_call(vbox_alloc, vec_obj, true);
442 C->remove_macro_node(vbox_alloc);
443
444 return vec_obj;
445 }
446
447 void PhaseVector::expand_vunbox_node(VectorUnboxNode* vec_unbox) {
448 if (vec_unbox->outcnt() > 0) {
449 GraphKit kit;
450 PhaseGVN& gvn = kit.gvn();
451
452 Node* obj = vec_unbox->obj();
453 const TypeInstPtr* tinst = gvn.type(obj)->isa_instptr();
454 ciInstanceKlass* from_kls = tinst->instance_klass();
455 const TypeVect* vt = vec_unbox->bottom_type()->is_vect();
456 BasicType bt = vt->element_basic_type();
457 BasicType masktype = bt;
458
459 if (is_vector_mask(from_kls)) {
460 bt = T_BOOLEAN;
461 }
462
463 ciField* field = ciEnv::current()->vector_VectorPayload_klass()->get_field_by_name(ciSymbols::payload_name(),
464 ciSymbols::object_signature(),
465 false);
466 assert(field != nullptr, "");
467 int offset = field->offset_in_bytes();
468 Node* vec_adr = kit.basic_plus_adr(obj, offset);
469
470 Node* mem = vec_unbox->mem();
471 Node* ctrl = vec_unbox->in(0);
472 Node* vec_field_ld;
473 {
474 DecoratorSet decorators = MO_UNORDERED | IN_HEAP;
475 C2AccessValuePtr addr(vec_adr, vec_adr->bottom_type()->is_ptr());
476 MergeMemNode* local_mem = MergeMemNode::make(mem);
477 gvn.record_for_igvn(local_mem);
478 BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
479 C2OptAccess access(gvn, ctrl, local_mem, decorators, T_OBJECT, obj, addr);
480 const Type* type = TypeOopPtr::make_from_klass(field->type()->as_klass());
481 vec_field_ld = bs->load_at(access, type);
482 }
483
484 // For proper aliasing, attach concrete payload type.
485 ciKlass* payload_klass = ciTypeArrayKlass::make(bt);
486 const Type* payload_type = TypeAryPtr::make_from_klass(payload_klass)->cast_to_ptr_type(TypePtr::NotNull);
487 vec_field_ld = gvn.transform(new CastPPNode(nullptr, vec_field_ld, payload_type));
488
489 Node* adr = kit.array_element_address(vec_field_ld, gvn.intcon(0), bt);
490 const TypePtr* adr_type = adr->bottom_type()->is_ptr();
491 int num_elem = vt->length();
492 Node* vec_val_load = LoadVectorNode::make(0,
493 ctrl,
494 mem,
495 adr,
496 adr_type,
497 num_elem,
498 bt);
499 vec_val_load = gvn.transform(vec_val_load);
500
501 C->set_max_vector_size(MAX2(C->max_vector_size(), vt->length_in_bytes()));
502
503 if (is_vector_mask(from_kls)) {
504 vec_val_load = gvn.transform(new VectorLoadMaskNode(vec_val_load, TypeVect::makemask(masktype, num_elem)));
505 }
506
507 gvn.hash_delete(vec_unbox);
508 vec_unbox->disconnect_inputs(C);
509 C->gvn_replace_by(vec_unbox, vec_val_load);
510 }
511 C->remove_macro_node(vec_unbox);
512 }
513
514 void PhaseVector::eliminate_vbox_alloc_node(VectorBoxAllocateNode* vbox_alloc) {
515 JVMState* jvms = clone_jvms(C, vbox_alloc);
516 GraphKit kit(jvms);
517 // Remove VBA, but leave a safepoint behind.
518 // Otherwise, it may end up with a loop without any safepoint polls.
519 kit.replace_call(vbox_alloc, kit.map(), true);
520 C->remove_macro_node(vbox_alloc);
521 }