1 /*
2 * Copyright (c) 2020, 2026, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "ci/ciSymbols.hpp"
26 #include "gc/shared/barrierSet.hpp"
27 #include "opto/castnode.hpp"
28 #include "opto/graphKit.hpp"
29 #include "opto/phaseX.hpp"
30 #include "opto/rootnode.hpp"
31 #include "opto/vector.hpp"
32 #include "utilities/globalDefinitions.hpp"
33 #include "utilities/macros.hpp"
34
35 static bool is_vector_mask(ciKlass* klass) {
36 return klass->is_subclass_of(ciEnv::current()->vector_VectorMask_klass());
37 }
38
39 void PhaseVector::optimize_vector_boxes() {
40 Compile::TracePhase tp(_t_vector_elimination);
41
42 // Signal GraphKit it's post-parse phase.
43 assert(C->inlining_incrementally() == false, "sanity");
44 C->set_inlining_incrementally(true);
45
46 C->igvn_worklist()->ensure_empty(); // should be done with igvn
47
48 if (StressMacroExpansion) {
49 C->shuffle_macro_nodes();
50 }
51
52 expand_vunbox_nodes();
53 scalarize_vbox_nodes();
54
55 C->inline_vector_reboxing_calls();
56
57 expand_vbox_nodes();
58 eliminate_vbox_alloc_nodes();
59
60 C->set_inlining_incrementally(false);
61
62 do_cleanup();
63 }
64
65 void PhaseVector::do_cleanup() {
66 if (C->failing()) return;
67 {
68 Compile::TracePhase tp(_t_vector_pru);
69 ResourceMark rm;
70 PhaseRemoveUseless pru(C->initial_gvn(), *C->igvn_worklist());
71 if (C->failing()) return;
72 }
73 {
74 Compile::TracePhase tp(_t_vector_igvn);
75 _igvn.reset();
76 _igvn.optimize();
77 if (C->failing()) return;
78 }
79 C->print_method(PHASE_ITER_GVN_BEFORE_EA, 3);
80 }
81
82 void PhaseVector::scalarize_vbox_nodes() {
83 if (C->failing()) return;
84
85 if (!EnableVectorReboxing) {
86 return; // don't scalarize vector boxes
87 }
88
89 int macro_idx = C->macro_count() - 1;
90 while (macro_idx >= 0) {
91 Node * n = C->macro_node(macro_idx);
92 assert(n->is_macro(), "only macro nodes expected here");
93 if (n->Opcode() == Op_VectorBox) {
94 VectorBoxNode* vbox = static_cast<VectorBoxNode*>(n);
95 scalarize_vbox_node(vbox);
96 if (C->failing()) return;
97 C->print_method(PHASE_SCALARIZE_VBOX, 3, vbox);
98 }
99 if (C->failing()) return;
100 macro_idx = MIN2(macro_idx - 1, C->macro_count() - 1);
101 }
102 }
103
104 void PhaseVector::expand_vbox_nodes() {
105 if (C->failing()) return;
106
107 int macro_idx = C->macro_count() - 1;
108 while (macro_idx >= 0) {
109 Node * n = C->macro_node(macro_idx);
110 assert(n->is_macro(), "only macro nodes expected here");
111 if (n->Opcode() == Op_VectorBox) {
112 VectorBoxNode* vbox = static_cast<VectorBoxNode*>(n);
113 expand_vbox_node(vbox);
114 if (C->failing()) return;
115 }
116 if (C->failing()) return;
117 macro_idx = MIN2(macro_idx - 1, C->macro_count() - 1);
118 }
119 }
120
121 void PhaseVector::expand_vunbox_nodes() {
122 if (C->failing()) return;
123
124 int macro_idx = C->macro_count() - 1;
125 while (macro_idx >= 0) {
126 Node * n = C->macro_node(macro_idx);
127 assert(n->is_macro(), "only macro nodes expected here");
128 if (n->Opcode() == Op_VectorUnbox) {
129 VectorUnboxNode* vec_unbox = static_cast<VectorUnboxNode*>(n);
130 expand_vunbox_node(vec_unbox);
131 if (C->failing()) return;
132 C->print_method(PHASE_EXPAND_VUNBOX, 3, vec_unbox);
133 }
134 if (C->failing()) return;
135 macro_idx = MIN2(macro_idx - 1, C->macro_count() - 1);
136 }
137 }
138
139 void PhaseVector::eliminate_vbox_alloc_nodes() {
140 if (C->failing()) return;
141
142 int macro_idx = C->macro_count() - 1;
143 while (macro_idx >= 0) {
144 Node * n = C->macro_node(macro_idx);
145 assert(n->is_macro(), "only macro nodes expected here");
146 if (n->Opcode() == Op_VectorBoxAllocate) {
147 VectorBoxAllocateNode* vbox_alloc = static_cast<VectorBoxAllocateNode*>(n);
148 eliminate_vbox_alloc_node(vbox_alloc);
149 if (C->failing()) return;
150 C->print_method(PHASE_ELIMINATE_VBOX_ALLOC, 3, vbox_alloc);
151 }
152 if (C->failing()) return;
153 macro_idx = MIN2(macro_idx - 1, C->macro_count() - 1);
154 }
155 }
156
157 static JVMState* clone_jvms(Compile* C, SafePointNode* sfpt) {
158 JVMState* new_jvms = sfpt->jvms()->clone_shallow(C);
159 uint size = sfpt->req();
160 SafePointNode* map = new SafePointNode(size, new_jvms);
161 for (uint i = 0; i < size; i++) {
162 map->init_req(i, sfpt->in(i));
163 }
164 Node* mem = map->memory();
165 if (!mem->is_MergeMem()) {
166 // Since we are not in parsing, the SafePointNode does not guarantee that the memory
167 // input is necessarily a MergeMemNode. But we need to ensure that there is that
168 // MergeMemNode, since the GraphKit assumes the memory input of the map to be a
169 // MergeMemNode, so that it can directly access the memory slices.
170 PhaseGVN& gvn = *C->initial_gvn();
171 Node* mergemem = MergeMemNode::make(mem);
172 gvn.set_type_bottom(mergemem);
173 map->set_memory(mergemem);
174 }
175 new_jvms->set_map(map);
176 return new_jvms;
177 }
178
179 void PhaseVector::scalarize_vbox_node(VectorBoxNode* vec_box) {
180 Node* vec_value = vec_box->in(VectorBoxNode::Value);
181 PhaseGVN& gvn = *C->initial_gvn();
182
183 // Process merged VBAs
184
185 if (EnableVectorAggressiveReboxing) {
186 Unique_Node_List calls;
187 for (DUIterator_Fast imax, i = vec_box->fast_outs(imax); i < imax; i++) {
188 Node* use = vec_box->fast_out(i);
189 if (use->is_CallJava()) {
190 CallJavaNode* call = use->as_CallJava();
191 if (call->has_non_debug_use(vec_box) && vec_box->in(VectorBoxNode::Box)->is_Phi()) {
192 calls.push(call);
193 }
194 }
195 }
196
197 while (calls.size() > 0) {
198 CallJavaNode* call = calls.pop()->as_CallJava();
199 // Attach new VBA to the call and use it instead of Phi (VBA ... VBA).
200
201 JVMState* jvms = clone_jvms(C, call);
202 GraphKit kit(jvms);
203 PhaseGVN& gvn = kit.gvn();
204
205 // Adjust JVMS from post-call to pre-call state: put args on stack
206 uint nargs = call->method()->arg_size();
207 kit.ensure_stack(kit.sp() + nargs);
208 uint in_idx = TypeFunc::Parms; // The index of the call input, using scalarized calling convention.
209 int parm_idx = 0; // The index of the Java parameter: double/long take one slot
210 for (uint i = 0; i < nargs; i++) { // The index of the argument on the JVM stack: double/long take two slots
211 const Type* arg_type = call->tf()->domain_sig()->field_at(TypeFunc::Parms + i);
212 if (arg_type->is_inlinetypeptr() && !call->method()->mismatch() && call->method()->is_scalarized_arg(parm_idx)) {
213 bool nullable = arg_type->maybe_null();
214 ciInlineKlass* vk = arg_type->inline_klass();
215 InlineTypeNode* it = InlineTypeNode::make_from_multi(&kit, call, vk, in_idx, true, !nullable);
216 kit.push(gvn.transform(it));
217 } else {
218 kit.push(call->in(in_idx));
219 in_idx++;
220 }
221 if (arg_type != Type::HALF) {
222 parm_idx++;
223 }
224 }
225 assert(in_idx == call->tf()->domain_cc()->cnt(), "should have processed exactly as many input as the scalarized calling convention; %d vs %d", in_idx, call->tf()->domain_cc()->cnt());
226 assert(parm_idx == (call->method()->is_static() ? 0 : 1) + call->method()->signature()->count(), "should have processed all the parameters; %d vs %d", parm_idx, (call->method()->is_static() ? 0 : 1) + call->method()->signature()->count());
227 jvms = kit.sync_jvms();
228
229 Node* new_vbox = nullptr;
230 {
231 Node* vect = vec_box->in(VectorBoxNode::Value);
232 const TypeInstPtr* vbox_type = vec_box->box_type();
233 const TypeVect* vt = vec_box->vec_type();
234 BasicType elem_bt = vt->element_basic_type();
235 int num_elem = vt->length();
236
237 new_vbox = kit.box_vector(vect, vbox_type, elem_bt, num_elem, /*deoptimize=*/true);
238
239 kit.replace_in_map(vec_box, new_vbox);
240 }
241
242 kit.dec_sp(nargs);
243 jvms = kit.sync_jvms();
244
245 call->set_req(TypeFunc::Control , kit.control());
246 call->set_req(TypeFunc::I_O , kit.i_o());
247 call->set_req(TypeFunc::Memory , kit.reset_memory());
248 call->set_req(TypeFunc::FramePtr, kit.frameptr());
249 call->replace_edge(vec_box, new_vbox);
250
251 C->record_for_igvn(call);
252 }
253 }
254
255 // Process debug uses at safepoints
256 Unique_Node_List safepoints;
257
258 Unique_Node_List worklist;
259 worklist.push(vec_box);
260 while (worklist.size() > 0) {
261 Node* n = worklist.pop();
262 for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) {
263 Node* use = n->fast_out(i);
264 if (use->is_SafePoint()) {
265 SafePointNode* sfpt = use->as_SafePoint();
266 if (!sfpt->is_Call() || !sfpt->as_Call()->has_non_debug_use(n)) {
267 safepoints.push(sfpt);
268 }
269 } else if (use->is_ConstraintCast()) {
270 worklist.push(use); // reversed version of Node::uncast()
271 }
272 }
273 }
274
275 ciInstanceKlass* iklass = vec_box->box_type()->instance_klass();
276 int n_fields = iklass->nof_nonstatic_fields();
277 assert(n_fields == 1, "sanity");
278
279 // If a mask is feeding into safepoint[s], then its value should be
280 // packed into a boolean/byte vector first, this will simplify the
281 // re-materialization logic for both predicated and non-predicated
282 // targets.
283 bool is_mask = is_vector_mask(iklass);
284 if (is_mask && vec_value->Opcode() != Op_VectorStoreMask) {
285 const TypeVect* vt = vec_value->bottom_type()->is_vect();
286 BasicType bt = vt->element_basic_type();
287 vec_value = gvn.transform(VectorStoreMaskNode::make(gvn, vec_value, bt, vt->length()));
288 }
289
290 while (safepoints.size() > 0) {
291 SafePointNode* sfpt = safepoints.pop()->as_SafePoint();
292
293 uint first_ind = (sfpt->req() - sfpt->jvms()->scloff());
294 Node* sobj = new SafePointScalarObjectNode(vec_box->box_type(), vec_box, first_ind, sfpt->jvms()->depth(), n_fields);
295 sobj->init_req(0, C->root());
296 sfpt->add_req(vec_value);
297
298 sobj = gvn.transform(sobj);
299
300 JVMState *jvms = sfpt->jvms();
301
302 jvms->set_endoff(sfpt->req());
303 // Now make a pass over the debug information replacing any references
304 // to the allocated object with vector value.
305 for (uint i = jvms->debug_start(); i < jvms->debug_end(); i++) {
306 Node* debug = sfpt->in(i);
307 if (debug != nullptr && debug->uncast(/*keep_deps*/false) == vec_box) {
308 sfpt->set_req(i, sobj);
309 }
310 }
311 C->record_for_igvn(sfpt);
312 }
313 }
314
315 void PhaseVector::expand_vbox_node(VectorBoxNode* vec_box) {
316 if (vec_box->outcnt() > 0) {
317 VectorSet visited;
318 Node* vbox = vec_box->in(VectorBoxNode::Box);
319 Node* vect = vec_box->in(VectorBoxNode::Value);
320 Node* result = expand_vbox_node_helper(vbox, vect, vec_box->box_type(),
321 vec_box->vec_type(), visited);
322 C->gvn_replace_by(vec_box, result);
323 C->print_method(PHASE_EXPAND_VBOX, 3, vec_box);
324 }
325 C->remove_macro_node(vec_box);
326 }
327
328 Node* PhaseVector::expand_vbox_node_helper(Node* vbox,
329 Node* vect,
330 const TypeInstPtr* box_type,
331 const TypeVect* vect_type,
332 VectorSet &visited) {
333 // JDK-8304948 shows an example that there may be a cycle in the graph.
334 if (visited.test_set(vbox->_idx)) {
335 assert(vbox->is_Phi() || vbox->is_CheckCastPP(), "either phi or expanded");
336 return vbox; // already visited
337 }
338
339 // Handle the case when the allocation input to VectorBoxNode is a Proj.
340 // This is the normal case before expanding.
341 if (vbox->is_Proj() && vbox->in(0)->Opcode() == Op_VectorBoxAllocate) {
342 VectorBoxAllocateNode* vbox_alloc = static_cast<VectorBoxAllocateNode*>(vbox->in(0));
343 return expand_vbox_alloc_node(vbox_alloc, vect, box_type, vect_type);
344 }
345
346 // Handle the case when the allocation input to VectorBoxNode is a Phi.
347 // This is generated after the transformation in PhiNode::merge_through_phi:
348 // Phi (VectorBox1 VectorBox2) => VectorBox (Phi1 Phi2)
349 // The vector input may also be a Phi (Phi2 above), or it may have been
350 // value-numbered to a single node if all inputs were identical.
351 if (vbox->is_Phi()) {
352 bool same_region = vect->is_Phi() && vbox->as_Phi()->region() == vect->as_Phi()->region();
353 for (uint i = 1; i < vbox->req(); i++) {
354 Node* new_box = expand_vbox_node_helper(vbox->in(i), same_region ? vect->in(i) : vect,
355 box_type, vect_type, visited);
356 if (!new_box->is_Phi()) {
357 C->initial_gvn()->hash_delete(vbox);
358 vbox->set_req(i, new_box);
359 }
360 }
361 return C->initial_gvn()->transform(vbox);
362 }
363
364 assert(vbox->is_CheckCastPP(), "should be expanded");
365 // TODO: assert that expanded vbox is initialized with the same value (vect).
366 return vbox; // already expanded
367 }
368
369 Node* PhaseVector::expand_vbox_alloc_node(VectorBoxAllocateNode* vbox_alloc,
370 Node* value,
371 const TypeInstPtr* box_type,
372 const TypeVect* vect_type) {
373 JVMState* jvms = clone_jvms(C, vbox_alloc);
374 GraphKit kit(jvms);
375 PhaseGVN& gvn = kit.gvn();
376
377 ciInstanceKlass* box_klass = box_type->instance_klass();
378 BasicType bt = vect_type->element_basic_type();
379 int num_elem = vect_type->length();
380
381 bool is_mask = is_vector_mask(box_klass);
382 // If boxed mask value is present in a predicate register, it must be
383 // spilled to a vector though a VectorStoreMaskOperation before actual StoreVector
384 // operation to vector payload field.
385 if (is_mask && (value->bottom_type()->isa_pvectmask() || bt != T_BOOLEAN)) {
386 value = gvn.transform(VectorStoreMaskNode::make(gvn, value, bt, num_elem));
387 // Although type of mask depends on its definition, in terms of storage everything is stored in boolean array.
388 bt = T_BOOLEAN;
389 assert(value->bottom_type()->is_vect()->element_basic_type() == bt,
390 "must be consistent with mask representation");
391 }
392
393 // Generate array allocation for the field which holds the values.
394 const TypeKlassPtr* array_klass = TypeKlassPtr::make(ciTypeArrayKlass::make(bt));
395 Node* arr = kit.new_array(kit.makecon(array_klass), kit.intcon(num_elem), 1);
396
397 // Store the vector value into the array.
398 // (The store should be captured by InitializeNode and turned into initialized store later.)
399 Node* arr_adr = kit.array_element_address(arr, kit.intcon(0), bt);
400 const TypePtr* arr_adr_type = arr_adr->bottom_type()->is_ptr();
401 Node* arr_mem = kit.memory(arr_adr);
402 Node* vstore = gvn.transform(StoreVectorNode::make(0,
403 kit.control(),
404 arr_mem,
405 arr_adr,
406 arr_adr_type,
407 value,
408 num_elem));
409 kit.set_memory(vstore, arr_adr_type);
410
411 C->set_max_vector_size(MAX2(C->max_vector_size(), vect_type->length_in_bytes()));
412
413 // Generate the allocate for the Vector object.
414 const TypeKlassPtr* klass_type = box_type->as_klass_type();
415 Node* klass_node = kit.makecon(klass_type);
416 Node* vec_obj = kit.new_instance(klass_node);
417
418 // Store the allocated array into object.
419 ciField* field = ciEnv::current()->vector_VectorPayload_klass()->get_field_by_name(ciSymbols::payload_name(),
420 ciSymbols::object_signature(),
421 false);
422 assert(field != nullptr, "");
423 Node* vec_field = kit.basic_plus_adr(vec_obj, field->offset_in_bytes());
424 const TypePtr* vec_adr_type = vec_field->bottom_type()->is_ptr();
425
426 // The store should be captured by InitializeNode and turned into initialized store later.
427 Node* field_store = gvn.transform(kit.access_store_at(vec_obj,
428 vec_field,
429 vec_adr_type,
430 arr,
431 TypeOopPtr::make_from_klass(field->type()->as_klass()),
432 T_OBJECT,
433 IN_HEAP));
434 kit.set_memory(field_store, vec_adr_type);
435
436 kit.replace_call(vbox_alloc, vec_obj, true);
437 C->remove_macro_node(vbox_alloc);
438
439 return vec_obj;
440 }
441
442 void PhaseVector::expand_vunbox_node(VectorUnboxNode* vec_unbox) {
443 if (vec_unbox->outcnt() > 0) {
444 GraphKit kit;
445 PhaseGVN& gvn = kit.gvn();
446
447 Node* obj = vec_unbox->obj();
448 const TypeInstPtr* tinst = gvn.type(obj)->isa_instptr();
449 ciInstanceKlass* from_kls = tinst->instance_klass();
450 const TypeVect* vt = vec_unbox->bottom_type()->is_vect();
451 BasicType bt = vt->element_basic_type();
452 BasicType masktype = bt;
453
454 if (is_vector_mask(from_kls)) {
455 bt = T_BOOLEAN;
456 }
457
458 ciField* field = ciEnv::current()->vector_VectorPayload_klass()->get_field_by_name(ciSymbols::payload_name(),
459 ciSymbols::object_signature(),
460 false);
461 assert(field != nullptr, "");
462 int offset = field->offset_in_bytes();
463 Node* vec_adr = kit.basic_plus_adr(obj, offset);
464
465 Node* mem = vec_unbox->mem();
466 Node* ctrl = vec_unbox->in(0);
467 Node* vec_field_ld;
468 {
469 DecoratorSet decorators = MO_UNORDERED | IN_HEAP;
470 C2AccessValuePtr addr(vec_adr, vec_adr->bottom_type()->is_ptr());
471 MergeMemNode* local_mem = MergeMemNode::make(mem);
472 gvn.record_for_igvn(local_mem);
473 BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
474 C2OptAccess access(gvn, ctrl, local_mem, decorators, T_OBJECT, obj, addr);
475 vec_field_ld = bs->load_at(access, Type::get_const_basic_type(T_OBJECT));
476
477 // For proper aliasing, attach concrete payload type.
478 ciKlass* payload_klass = ciTypeArrayKlass::make(bt);
479 const Type* payload_type = TypeAryPtr::make_from_klass(payload_klass)->cast_to_ptr_type(TypePtr::NotNull);
480 vec_field_ld = gvn.transform(new CheckCastPPNode(ctrl, vec_field_ld, payload_type, ConstraintCastNode::DependencyType::NonFloatingNarrowing));
481 }
482
483 Node* adr = kit.array_element_address(vec_field_ld, gvn.intcon(0), bt);
484 const TypePtr* adr_type = adr->bottom_type()->is_ptr();
485 int num_elem = vt->length();
486 Node* vec_val_load = LoadVectorNode::make(0,
487 ctrl,
488 mem,
489 adr,
490 adr_type,
491 num_elem,
492 bt);
493 vec_val_load = gvn.transform(vec_val_load);
494
495 C->set_max_vector_size(MAX2(C->max_vector_size(), vt->length_in_bytes()));
496
497 if (is_vector_mask(from_kls)) {
498 vec_val_load = gvn.transform(new VectorLoadMaskNode(vec_val_load, TypeVect::makemask(masktype, num_elem)));
499 }
500
501 gvn.hash_delete(vec_unbox);
502 vec_unbox->disconnect_inputs(C);
503 C->gvn_replace_by(vec_unbox, vec_val_load);
504 }
505 C->remove_macro_node(vec_unbox);
506 }
507
508 void PhaseVector::eliminate_vbox_alloc_node(VectorBoxAllocateNode* vbox_alloc) {
509 JVMState* jvms = clone_jvms(C, vbox_alloc);
510 GraphKit kit(jvms);
511 // Remove VBA, but leave a safepoint behind.
512 // Otherwise, it may end up with a loop without any safepoint polls.
513 kit.replace_call(vbox_alloc, kit.map(), true);
514 C->remove_macro_node(vbox_alloc);
515 }