1 /*
2 * Copyright (c) 2020, 2023, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "ci/ciSymbols.hpp"
27 #include "gc/shared/barrierSet.hpp"
28 #include "opto/castnode.hpp"
29 #include "opto/graphKit.hpp"
30 #include "opto/phaseX.hpp"
31 #include "opto/rootnode.hpp"
32 #include "opto/vector.hpp"
33 #include "utilities/macros.hpp"
34
35 static bool is_vector_mask(ciKlass* klass) {
36 return klass->is_subclass_of(ciEnv::current()->vector_VectorMask_klass());
37 }
38
39 static bool is_vector_shuffle(ciKlass* klass) {
40 return klass->is_subclass_of(ciEnv::current()->vector_VectorShuffle_klass());
41 }
42
43
44 void PhaseVector::optimize_vector_boxes() {
45 Compile::TracePhase tp("vector_elimination", &timers[_t_vector_elimination]);
46
47 // Signal GraphKit it's post-parse phase.
48 assert(C->inlining_incrementally() == false, "sanity");
49 C->set_inlining_incrementally(true);
50
51 C->igvn_worklist()->ensure_empty(); // should be done with igvn
52
53 expand_vunbox_nodes();
54 scalarize_vbox_nodes();
55
56 C->inline_vector_reboxing_calls();
57
58 expand_vbox_nodes();
59 eliminate_vbox_alloc_nodes();
60
61 C->set_inlining_incrementally(false);
62
63 do_cleanup();
64 }
65
66 void PhaseVector::do_cleanup() {
67 if (C->failing()) return;
68 {
69 Compile::TracePhase tp("vector_pru", &timers[_t_vector_pru]);
70 ResourceMark rm;
71 PhaseRemoveUseless pru(C->initial_gvn(), *C->igvn_worklist());
72 if (C->failing()) return;
73 }
74 {
75 Compile::TracePhase tp("incrementalInline_igvn", &timers[_t_vector_igvn]);
76 _igvn.reset_from_gvn(C->initial_gvn());
77 _igvn.optimize();
78 if (C->failing()) return;
79 }
80 C->print_method(PHASE_ITER_GVN_BEFORE_EA, 3);
81 }
82
83 void PhaseVector::scalarize_vbox_nodes() {
84 if (C->failing()) return;
85
86 if (!EnableVectorReboxing) {
87 return; // don't scalarize vector boxes
88 }
89
90 int macro_idx = C->macro_count() - 1;
91 while (macro_idx >= 0) {
92 Node * n = C->macro_node(macro_idx);
93 assert(n->is_macro(), "only macro nodes expected here");
94 if (n->Opcode() == Op_VectorBox) {
95 VectorBoxNode* vbox = static_cast<VectorBoxNode*>(n);
96 scalarize_vbox_node(vbox);
97 if (C->failing()) return;
98 C->print_method(PHASE_SCALARIZE_VBOX, 3, vbox);
99 }
100 if (C->failing()) return;
101 macro_idx = MIN2(macro_idx - 1, C->macro_count() - 1);
102 }
103 }
104
105 void PhaseVector::expand_vbox_nodes() {
106 if (C->failing()) return;
107
108 int macro_idx = C->macro_count() - 1;
109 while (macro_idx >= 0) {
110 Node * n = C->macro_node(macro_idx);
111 assert(n->is_macro(), "only macro nodes expected here");
112 if (n->Opcode() == Op_VectorBox) {
113 VectorBoxNode* vbox = static_cast<VectorBoxNode*>(n);
114 expand_vbox_node(vbox);
115 if (C->failing()) return;
116 }
117 if (C->failing()) return;
118 macro_idx = MIN2(macro_idx - 1, C->macro_count() - 1);
119 }
120 }
121
122 void PhaseVector::expand_vunbox_nodes() {
123 if (C->failing()) return;
124
125 int macro_idx = C->macro_count() - 1;
126 while (macro_idx >= 0) {
127 Node * n = C->macro_node(macro_idx);
128 assert(n->is_macro(), "only macro nodes expected here");
129 if (n->Opcode() == Op_VectorUnbox) {
130 VectorUnboxNode* vec_unbox = static_cast<VectorUnboxNode*>(n);
131 expand_vunbox_node(vec_unbox);
132 if (C->failing()) return;
133 C->print_method(PHASE_EXPAND_VUNBOX, 3, vec_unbox);
134 }
135 if (C->failing()) return;
136 macro_idx = MIN2(macro_idx - 1, C->macro_count() - 1);
137 }
138 }
139
140 void PhaseVector::eliminate_vbox_alloc_nodes() {
141 if (C->failing()) return;
142
143 int macro_idx = C->macro_count() - 1;
144 while (macro_idx >= 0) {
145 Node * n = C->macro_node(macro_idx);
146 assert(n->is_macro(), "only macro nodes expected here");
147 if (n->Opcode() == Op_VectorBoxAllocate) {
148 VectorBoxAllocateNode* vbox_alloc = static_cast<VectorBoxAllocateNode*>(n);
149 eliminate_vbox_alloc_node(vbox_alloc);
150 if (C->failing()) return;
151 C->print_method(PHASE_ELIMINATE_VBOX_ALLOC, 3, vbox_alloc);
152 }
153 if (C->failing()) return;
154 macro_idx = MIN2(macro_idx - 1, C->macro_count() - 1);
155 }
156 }
157
158 static JVMState* clone_jvms(Compile* C, SafePointNode* sfpt) {
159 JVMState* new_jvms = sfpt->jvms()->clone_shallow(C);
160 uint size = sfpt->req();
161 SafePointNode* map = new SafePointNode(size, new_jvms);
162 for (uint i = 0; i < size; i++) {
163 map->init_req(i, sfpt->in(i));
164 }
165 Node* mem = map->memory();
166 if (!mem->is_MergeMem()) {
167 // Since we are not in parsing, the SafePointNode does not guarantee that the memory
168 // input is necessarily a MergeMemNode. But we need to ensure that there is that
169 // MergeMemNode, since the GraphKit assumes the memory input of the map to be a
170 // MergeMemNode, so that it can directly access the memory slices.
171 PhaseGVN& gvn = *C->initial_gvn();
172 Node* mergemem = MergeMemNode::make(mem);
173 gvn.set_type_bottom(mergemem);
174 map->set_memory(mergemem);
175 }
176 new_jvms->set_map(map);
177 return new_jvms;
178 }
179
180 void PhaseVector::scalarize_vbox_node(VectorBoxNode* vec_box) {
181 Node* vec_value = vec_box->in(VectorBoxNode::Value);
182 PhaseGVN& gvn = *C->initial_gvn();
183
184 // Process merged VBAs
185
186 if (EnableVectorAggressiveReboxing) {
187 Unique_Node_List calls(C->comp_arena());
188 for (DUIterator_Fast imax, i = vec_box->fast_outs(imax); i < imax; i++) {
189 Node* use = vec_box->fast_out(i);
190 if (use->is_CallJava()) {
191 CallJavaNode* call = use->as_CallJava();
192 if (call->has_non_debug_use(vec_box) && vec_box->in(VectorBoxNode::Box)->is_Phi()) {
193 calls.push(call);
194 }
195 }
196 }
197
198 while (calls.size() > 0) {
199 CallJavaNode* call = calls.pop()->as_CallJava();
200 // Attach new VBA to the call and use it instead of Phi (VBA ... VBA).
201
202 JVMState* jvms = clone_jvms(C, call);
203 GraphKit kit(jvms);
204 PhaseGVN& gvn = kit.gvn();
205
206 // Adjust JVMS from post-call to pre-call state: put args on stack
207 uint nargs = call->method()->arg_size();
208 kit.ensure_stack(kit.sp() + nargs);
209 for (uint i = TypeFunc::Parms; i < call->tf()->domain_sig()->cnt(); i++) {
210 kit.push(call->in(i));
211 }
212 jvms = kit.sync_jvms();
213
214 Node* new_vbox = nullptr;
215 {
216 Node* vect = vec_box->in(VectorBoxNode::Value);
217 const TypeInstPtr* vbox_type = vec_box->box_type();
218 const TypeVect* vt = vec_box->vec_type();
219 BasicType elem_bt = vt->element_basic_type();
220 int num_elem = vt->length();
221
222 new_vbox = kit.box_vector(vect, vbox_type, elem_bt, num_elem, /*deoptimize=*/true);
223
224 kit.replace_in_map(vec_box, new_vbox);
225 }
226
227 kit.dec_sp(nargs);
228 jvms = kit.sync_jvms();
229
230 call->set_req(TypeFunc::Control , kit.control());
231 call->set_req(TypeFunc::I_O , kit.i_o());
232 call->set_req(TypeFunc::Memory , kit.reset_memory());
233 call->set_req(TypeFunc::FramePtr, kit.frameptr());
234 call->replace_edge(vec_box, new_vbox);
235
236 C->record_for_igvn(call);
237 }
238 }
239
240 // Process debug uses at safepoints
241 Unique_Node_List safepoints(C->comp_arena());
242
243 Unique_Node_List worklist(C->comp_arena());
244 worklist.push(vec_box);
245 while (worklist.size() > 0) {
246 Node* n = worklist.pop();
247 for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) {
248 Node* use = n->fast_out(i);
249 if (use->is_SafePoint()) {
250 SafePointNode* sfpt = use->as_SafePoint();
251 if (!sfpt->is_Call() || !sfpt->as_Call()->has_non_debug_use(n)) {
252 safepoints.push(sfpt);
253 }
254 } else if (use->is_ConstraintCast()) {
255 worklist.push(use); // reversed version of Node::uncast()
256 }
257 }
258 }
259
260 ciInstanceKlass* iklass = vec_box->box_type()->instance_klass();
261 int n_fields = iklass->nof_nonstatic_fields();
262 assert(n_fields == 1, "sanity");
263
264 // If a mask is feeding into safepoint[s], then its value should be
265 // packed into a boolean/byte vector first, this will simplify the
266 // re-materialization logic for both predicated and non-predicated
267 // targets.
268 bool is_mask = is_vector_mask(iklass);
269 if (is_mask && vec_value->Opcode() != Op_VectorStoreMask) {
270 const TypeVect* vt = vec_value->bottom_type()->is_vect();
271 BasicType bt = vt->element_basic_type();
272 vec_value = gvn.transform(VectorStoreMaskNode::make(gvn, vec_value, bt, vt->length()));
273 }
274
275 while (safepoints.size() > 0) {
276 SafePointNode* sfpt = safepoints.pop()->as_SafePoint();
277
278 uint first_ind = (sfpt->req() - sfpt->jvms()->scloff());
279 Node* sobj = new SafePointScalarObjectNode(vec_box->box_type(), vec_box, first_ind, n_fields);
280 sobj->init_req(0, C->root());
281 sfpt->add_req(vec_value);
282
283 sobj = gvn.transform(sobj);
284
285 JVMState *jvms = sfpt->jvms();
286
287 jvms->set_endoff(sfpt->req());
288 // Now make a pass over the debug information replacing any references
289 // to the allocated object with vector value.
290 for (uint i = jvms->debug_start(); i < jvms->debug_end(); i++) {
291 Node* debug = sfpt->in(i);
292 if (debug != nullptr && debug->uncast(/*keep_deps*/false) == vec_box) {
293 sfpt->set_req(i, sobj);
294 }
295 }
296 C->record_for_igvn(sfpt);
297 }
298 }
299
300 void PhaseVector::expand_vbox_node(VectorBoxNode* vec_box) {
301 if (vec_box->outcnt() > 0) {
302 VectorSet visited;
303 Node* vbox = vec_box->in(VectorBoxNode::Box);
304 Node* vect = vec_box->in(VectorBoxNode::Value);
305 Node* result = expand_vbox_node_helper(vbox, vect, vec_box->box_type(),
306 vec_box->vec_type(), visited);
307 C->gvn_replace_by(vec_box, result);
308 C->print_method(PHASE_EXPAND_VBOX, 3, vec_box);
309 }
310 C->remove_macro_node(vec_box);
311 }
312
313 Node* PhaseVector::expand_vbox_node_helper(Node* vbox,
314 Node* vect,
315 const TypeInstPtr* box_type,
316 const TypeVect* vect_type,
317 VectorSet &visited) {
318 // JDK-8304948 shows an example that there may be a cycle in the graph.
319 if (visited.test_set(vbox->_idx)) {
320 assert(vbox->is_Phi(), "should be phi");
321 return vbox; // already visited
322 }
323
324 // Handle the case when the allocation input to VectorBoxNode is a Proj.
325 // This is the normal case before expanding.
326 if (vbox->is_Proj() && vbox->in(0)->Opcode() == Op_VectorBoxAllocate) {
327 VectorBoxAllocateNode* vbox_alloc = static_cast<VectorBoxAllocateNode*>(vbox->in(0));
328 return expand_vbox_alloc_node(vbox_alloc, vect, box_type, vect_type);
329 }
330
331 // Handle the case when both the allocation input and vector input to
332 // VectorBoxNode are Phi. This case is generated after the transformation of
333 // Phi: Phi (VectorBox1 VectorBox2) => VectorBox (Phi1 Phi2).
334 // With this optimization, the relative two allocation inputs of VectorBox1 and
335 // VectorBox2 are gathered into Phi1 now. Similarly, the original vector
336 // inputs of two VectorBox nodes are in Phi2.
337 //
338 // See PhiNode::merge_through_phi in cfg.cpp for more details.
339 if (vbox->is_Phi() && vect->is_Phi()) {
340 assert(vbox->as_Phi()->region() == vect->as_Phi()->region(), "");
341 for (uint i = 1; i < vbox->req(); i++) {
342 Node* new_box = expand_vbox_node_helper(vbox->in(i), vect->in(i),
343 box_type, vect_type, visited);
344 if (!new_box->is_Phi()) {
345 C->initial_gvn()->hash_delete(vbox);
346 vbox->set_req(i, new_box);
347 }
348 }
349 return C->initial_gvn()->transform(vbox);
350 }
351
352 // Handle the case when the allocation input to VectorBoxNode is a phi
353 // but the vector input is not, which can definitely be the case if the
354 // vector input has been value-numbered. It seems to be safe to do by
355 // construction because VectorBoxNode and VectorBoxAllocate come in a
356 // specific order as a result of expanding an intrinsic call. After that, if
357 // any of the inputs to VectorBoxNode are value-numbered they can only
358 // move up and are guaranteed to dominate.
359 if (vbox->is_Phi() && (vect->is_Vector() || vect->is_LoadVector())) {
360 for (uint i = 1; i < vbox->req(); i++) {
361 Node* new_box = expand_vbox_node_helper(vbox->in(i), vect,
362 box_type, vect_type, visited);
363 if (!new_box->is_Phi()) {
364 C->initial_gvn()->hash_delete(vbox);
365 vbox->set_req(i, new_box);
366 }
367 }
368 return C->initial_gvn()->transform(vbox);
369 }
370
371 assert(!vbox->is_Phi(), "should be expanded");
372 // TODO: assert that expanded vbox is initialized with the same value (vect).
373 return vbox; // already expanded
374 }
375
376 Node* PhaseVector::expand_vbox_alloc_node(VectorBoxAllocateNode* vbox_alloc,
377 Node* value,
378 const TypeInstPtr* box_type,
379 const TypeVect* vect_type) {
380 JVMState* jvms = clone_jvms(C, vbox_alloc);
381 GraphKit kit(jvms);
382 PhaseGVN& gvn = kit.gvn();
383
384 ciInstanceKlass* box_klass = box_type->instance_klass();
385 BasicType bt = vect_type->element_basic_type();
386 int num_elem = vect_type->length();
387
388 bool is_mask = is_vector_mask(box_klass);
389 // If boxed mask value is present in a predicate register, it must be
390 // spilled to a vector though a VectorStoreMaskOperation before actual StoreVector
391 // operation to vector payload field.
392 if (is_mask && (value->bottom_type()->isa_vectmask() || bt != T_BOOLEAN)) {
393 value = gvn.transform(VectorStoreMaskNode::make(gvn, value, bt, num_elem));
394 // Although type of mask depends on its definition, in terms of storage everything is stored in boolean array.
395 bt = T_BOOLEAN;
396 assert(value->bottom_type()->is_vect()->element_basic_type() == bt,
397 "must be consistent with mask representation");
398 }
399
400 // Generate array allocation for the field which holds the values.
401 const TypeKlassPtr* array_klass = TypeKlassPtr::make(ciTypeArrayKlass::make(bt));
402 Node* arr = kit.new_array(kit.makecon(array_klass), kit.intcon(num_elem), 1);
403
404 // Store the vector value into the array.
405 // (The store should be captured by InitializeNode and turned into initialized store later.)
406 Node* arr_adr = kit.array_element_address(arr, kit.intcon(0), bt);
407 const TypePtr* arr_adr_type = arr_adr->bottom_type()->is_ptr();
408 Node* arr_mem = kit.memory(arr_adr);
409 Node* vstore = gvn.transform(StoreVectorNode::make(0,
410 kit.control(),
411 arr_mem,
412 arr_adr,
413 arr_adr_type,
414 value,
415 num_elem));
416 kit.set_memory(vstore, arr_adr_type);
417
418 C->set_max_vector_size(MAX2(C->max_vector_size(), vect_type->length_in_bytes()));
419
420 // Generate the allocate for the Vector object.
421 const TypeKlassPtr* klass_type = box_type->as_klass_type();
422 Node* klass_node = kit.makecon(klass_type);
423 Node* vec_obj = kit.new_instance(klass_node);
424
425 // Store the allocated array into object.
426 ciField* field = ciEnv::current()->vector_VectorPayload_klass()->get_field_by_name(ciSymbols::payload_name(),
427 ciSymbols::object_signature(),
428 false);
429 assert(field != nullptr, "");
430 Node* vec_field = kit.basic_plus_adr(vec_obj, field->offset_in_bytes());
431 const TypePtr* vec_adr_type = vec_field->bottom_type()->is_ptr();
432
433 // The store should be captured by InitializeNode and turned into initialized store later.
434 Node* field_store = gvn.transform(kit.access_store_at(vec_obj,
435 vec_field,
436 vec_adr_type,
437 arr,
438 TypeOopPtr::make_from_klass(field->type()->as_klass()),
439 T_OBJECT,
440 IN_HEAP));
441 kit.set_memory(field_store, vec_adr_type);
442
443 kit.replace_call(vbox_alloc, vec_obj, true);
444 C->remove_macro_node(vbox_alloc);
445
446 return vec_obj;
447 }
448
449 void PhaseVector::expand_vunbox_node(VectorUnboxNode* vec_unbox) {
450 if (vec_unbox->outcnt() > 0) {
451 GraphKit kit;
452 PhaseGVN& gvn = kit.gvn();
453
454 Node* obj = vec_unbox->obj();
455 const TypeInstPtr* tinst = gvn.type(obj)->isa_instptr();
456 ciInstanceKlass* from_kls = tinst->instance_klass();
457 const TypeVect* vt = vec_unbox->bottom_type()->is_vect();
458 BasicType bt = vt->element_basic_type();
459 BasicType masktype = bt;
460
461 if (is_vector_mask(from_kls)) {
462 bt = T_BOOLEAN;
463 } else if (is_vector_shuffle(from_kls)) {
464 bt = T_BYTE;
465 }
466
467 ciField* field = ciEnv::current()->vector_VectorPayload_klass()->get_field_by_name(ciSymbols::payload_name(),
468 ciSymbols::object_signature(),
469 false);
470 assert(field != nullptr, "");
471 int offset = field->offset_in_bytes();
472 Node* vec_adr = kit.basic_plus_adr(obj, offset);
473
474 Node* mem = vec_unbox->mem();
475 Node* ctrl = vec_unbox->in(0);
476 Node* vec_field_ld;
477 {
478 DecoratorSet decorators = MO_UNORDERED | IN_HEAP;
479 C2AccessValuePtr addr(vec_adr, vec_adr->bottom_type()->is_ptr());
480 MergeMemNode* local_mem = MergeMemNode::make(mem);
481 gvn.record_for_igvn(local_mem);
482 BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
483 C2OptAccess access(gvn, ctrl, local_mem, decorators, T_OBJECT, obj, addr);
484 const Type* type = TypeOopPtr::make_from_klass(field->type()->as_klass());
485 vec_field_ld = bs->load_at(access, type);
486 }
487
488 // For proper aliasing, attach concrete payload type.
489 ciKlass* payload_klass = ciTypeArrayKlass::make(bt);
490 const Type* payload_type = TypeAryPtr::make_from_klass(payload_klass)->cast_to_ptr_type(TypePtr::NotNull);
491 vec_field_ld = gvn.transform(new CastPPNode(vec_field_ld, payload_type));
492
493 Node* adr = kit.array_element_address(vec_field_ld, gvn.intcon(0), bt);
494 const TypePtr* adr_type = adr->bottom_type()->is_ptr();
495 int num_elem = vt->length();
496 Node* vec_val_load = LoadVectorNode::make(0,
497 ctrl,
498 mem,
499 adr,
500 adr_type,
501 num_elem,
502 bt);
503 vec_val_load = gvn.transform(vec_val_load);
504
505 C->set_max_vector_size(MAX2(C->max_vector_size(), vt->length_in_bytes()));
506
507 if (is_vector_mask(from_kls)) {
508 vec_val_load = gvn.transform(new VectorLoadMaskNode(vec_val_load, TypeVect::makemask(masktype, num_elem)));
509 } else if (is_vector_shuffle(from_kls) && !vec_unbox->is_shuffle_to_vector()) {
510 assert(vec_unbox->bottom_type()->is_vect()->element_basic_type() == masktype, "expect shuffle type consistency");
511 vec_val_load = gvn.transform(new VectorLoadShuffleNode(vec_val_load, TypeVect::make(masktype, num_elem)));
512 }
513
514 gvn.hash_delete(vec_unbox);
515 vec_unbox->disconnect_inputs(C);
516 C->gvn_replace_by(vec_unbox, vec_val_load);
517 }
518 C->remove_macro_node(vec_unbox);
519 }
520
521 void PhaseVector::eliminate_vbox_alloc_node(VectorBoxAllocateNode* vbox_alloc) {
522 JVMState* jvms = clone_jvms(C, vbox_alloc);
523 GraphKit kit(jvms);
524 // Remove VBA, but leave a safepoint behind.
525 // Otherwise, it may end up with a loop without any safepoint polls.
526 kit.replace_call(vbox_alloc, kit.map(), true);
527 C->remove_macro_node(vbox_alloc);
528 }