1 /*
2 * Copyright (c) 2020, 2023, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "ci/ciSymbols.hpp"
27 #include "gc/shared/barrierSet.hpp"
28 #include "opto/castnode.hpp"
29 #include "opto/graphKit.hpp"
30 #include "opto/phaseX.hpp"
31 #include "opto/rootnode.hpp"
32 #include "opto/vector.hpp"
33 #include "utilities/macros.hpp"
34
35 static bool is_vector_mask(ciKlass* klass) {
36 return klass->is_subclass_of(ciEnv::current()->vector_VectorMask_klass());
37 }
38
39 void PhaseVector::optimize_vector_boxes() {
40 Compile::TracePhase tp("vector_elimination", &timers[_t_vector_elimination]);
41
42 // Signal GraphKit it's post-parse phase.
43 assert(C->inlining_incrementally() == false, "sanity");
44 C->set_inlining_incrementally(true);
45
46 C->for_igvn()->clear();
47 C->initial_gvn()->replace_with(&_igvn);
48
49 expand_vunbox_nodes();
50 scalarize_vbox_nodes();
51
52 C->inline_vector_reboxing_calls();
53
54 expand_vbox_nodes();
55 eliminate_vbox_alloc_nodes();
56
57 C->set_inlining_incrementally(false);
58
59 do_cleanup();
60 }
61
62 void PhaseVector::do_cleanup() {
63 if (C->failing()) return;
64 {
65 Compile::TracePhase tp("vector_pru", &timers[_t_vector_pru]);
66 ResourceMark rm;
67 PhaseRemoveUseless pru(C->initial_gvn(), C->for_igvn());
68 if (C->failing()) return;
69 }
70 {
71 Compile::TracePhase tp("incrementalInline_igvn", &timers[_t_vector_igvn]);
72 _igvn = PhaseIterGVN(C->initial_gvn());
73 _igvn.optimize();
74 if (C->failing()) return;
75 }
76 C->print_method(PHASE_ITER_GVN_BEFORE_EA, 3);
77 }
78
79 void PhaseVector::scalarize_vbox_nodes() {
80 if (C->failing()) return;
81
82 if (!EnableVectorReboxing) {
83 return; // don't scalarize vector boxes
84 }
85
86 int macro_idx = C->macro_count() - 1;
87 while (macro_idx >= 0) {
88 Node * n = C->macro_node(macro_idx);
89 assert(n->is_macro(), "only macro nodes expected here");
90 if (n->Opcode() == Op_VectorBox) {
91 VectorBoxNode* vbox = static_cast<VectorBoxNode*>(n);
92 scalarize_vbox_node(vbox);
93 if (C->failing()) return;
94 C->print_method(PHASE_SCALARIZE_VBOX, 3, vbox);
95 }
96 if (C->failing()) return;
97 macro_idx = MIN2(macro_idx - 1, C->macro_count() - 1);
98 }
99 }
100
101 void PhaseVector::expand_vbox_nodes() {
102 if (C->failing()) return;
103
104 int macro_idx = C->macro_count() - 1;
105 while (macro_idx >= 0) {
106 Node * n = C->macro_node(macro_idx);
107 assert(n->is_macro(), "only macro nodes expected here");
108 if (n->Opcode() == Op_VectorBox) {
109 VectorBoxNode* vbox = static_cast<VectorBoxNode*>(n);
110 expand_vbox_node(vbox);
111 if (C->failing()) return;
112 }
113 if (C->failing()) return;
114 macro_idx = MIN2(macro_idx - 1, C->macro_count() - 1);
115 }
116 }
117
118 void PhaseVector::expand_vunbox_nodes() {
119 if (C->failing()) return;
120
121 int macro_idx = C->macro_count() - 1;
122 while (macro_idx >= 0) {
123 Node * n = C->macro_node(macro_idx);
124 assert(n->is_macro(), "only macro nodes expected here");
125 if (n->Opcode() == Op_VectorUnbox) {
126 VectorUnboxNode* vec_unbox = static_cast<VectorUnboxNode*>(n);
127 expand_vunbox_node(vec_unbox);
128 if (C->failing()) return;
129 C->print_method(PHASE_EXPAND_VUNBOX, 3, vec_unbox);
130 }
131 if (C->failing()) return;
132 macro_idx = MIN2(macro_idx - 1, C->macro_count() - 1);
133 }
134 }
135
136 void PhaseVector::eliminate_vbox_alloc_nodes() {
137 if (C->failing()) return;
138
139 int macro_idx = C->macro_count() - 1;
140 while (macro_idx >= 0) {
141 Node * n = C->macro_node(macro_idx);
142 assert(n->is_macro(), "only macro nodes expected here");
143 if (n->Opcode() == Op_VectorBoxAllocate) {
144 VectorBoxAllocateNode* vbox_alloc = static_cast<VectorBoxAllocateNode*>(n);
145 eliminate_vbox_alloc_node(vbox_alloc);
146 if (C->failing()) return;
147 C->print_method(PHASE_ELIMINATE_VBOX_ALLOC, 3, vbox_alloc);
148 }
149 if (C->failing()) return;
150 macro_idx = MIN2(macro_idx - 1, C->macro_count() - 1);
151 }
152 }
153
154 static JVMState* clone_jvms(Compile* C, SafePointNode* sfpt) {
155 JVMState* new_jvms = sfpt->jvms()->clone_shallow(C);
156 uint size = sfpt->req();
157 SafePointNode* map = new SafePointNode(size, new_jvms);
158 for (uint i = 0; i < size; i++) {
159 map->init_req(i, sfpt->in(i));
160 }
161 Node* mem = map->memory();
162 if (!mem->is_MergeMem()) {
163 // Since we are not in parsing, the SafePointNode does not guarantee that the memory
164 // input is necessarily a MergeMemNode. But we need to ensure that there is that
165 // MergeMemNode, since the GraphKit assumes the memory input of the map to be a
166 // MergeMemNode, so that it can directly access the memory slices.
167 PhaseGVN& gvn = *C->initial_gvn();
168 Node* mergemem = MergeMemNode::make(mem);
169 gvn.set_type_bottom(mergemem);
170 map->set_memory(mergemem);
171 }
172 new_jvms->set_map(map);
173 return new_jvms;
174 }
175
176 void PhaseVector::scalarize_vbox_node(VectorBoxNode* vec_box) {
177 Node* vec_value = vec_box->in(VectorBoxNode::Value);
178 PhaseGVN& gvn = *C->initial_gvn();
179
180 // Process merged VBAs
181
182 if (EnableVectorAggressiveReboxing) {
183 Unique_Node_List calls(C->comp_arena());
184 for (DUIterator_Fast imax, i = vec_box->fast_outs(imax); i < imax; i++) {
185 Node* use = vec_box->fast_out(i);
186 if (use->is_CallJava()) {
187 CallJavaNode* call = use->as_CallJava();
188 if (call->has_non_debug_use(vec_box) && vec_box->in(VectorBoxNode::Box)->is_Phi()) {
189 calls.push(call);
190 }
191 }
192 }
193
194 while (calls.size() > 0) {
195 CallJavaNode* call = calls.pop()->as_CallJava();
196 // Attach new VBA to the call and use it instead of Phi (VBA ... VBA).
197
198 JVMState* jvms = clone_jvms(C, call);
199 GraphKit kit(jvms);
200 PhaseGVN& gvn = kit.gvn();
201
202 // Adjust JVMS from post-call to pre-call state: put args on stack
203 uint nargs = call->method()->arg_size();
204 kit.ensure_stack(kit.sp() + nargs);
205 for (uint i = TypeFunc::Parms; i < call->tf()->domain_sig()->cnt(); i++) {
206 kit.push(call->in(i));
207 }
208 jvms = kit.sync_jvms();
209
210 Node* new_vbox = nullptr;
211 {
212 Node* vect = vec_box->in(VectorBoxNode::Value);
213 const TypeInstPtr* vbox_type = vec_box->box_type();
214 const TypeVect* vt = vec_box->vec_type();
215 BasicType elem_bt = vt->element_basic_type();
216 int num_elem = vt->length();
217
218 new_vbox = kit.box_vector(vect, vbox_type, elem_bt, num_elem, /*deoptimize=*/true);
219
220 kit.replace_in_map(vec_box, new_vbox);
221 }
222
223 kit.dec_sp(nargs);
224 jvms = kit.sync_jvms();
225
226 call->set_req(TypeFunc::Control , kit.control());
227 call->set_req(TypeFunc::I_O , kit.i_o());
228 call->set_req(TypeFunc::Memory , kit.reset_memory());
229 call->set_req(TypeFunc::FramePtr, kit.frameptr());
230 call->replace_edge(vec_box, new_vbox);
231
232 C->record_for_igvn(call);
233 }
234 }
235
236 // Process debug uses at safepoints
237 Unique_Node_List safepoints(C->comp_arena());
238
239 Unique_Node_List worklist(C->comp_arena());
240 worklist.push(vec_box);
241 while (worklist.size() > 0) {
242 Node* n = worklist.pop();
243 for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) {
244 Node* use = n->fast_out(i);
245 if (use->is_SafePoint()) {
246 SafePointNode* sfpt = use->as_SafePoint();
247 if (!sfpt->is_Call() || !sfpt->as_Call()->has_non_debug_use(n)) {
248 safepoints.push(sfpt);
249 }
250 } else if (use->is_ConstraintCast()) {
251 worklist.push(use); // reversed version of Node::uncast()
252 }
253 }
254 }
255
256 ciInstanceKlass* iklass = vec_box->box_type()->instance_klass();
257 int n_fields = iklass->nof_nonstatic_fields();
258 assert(n_fields == 1, "sanity");
259
260 // If a mask is feeding into safepoint[s], then its value should be
261 // packed into a boolean/byte vector first, this will simplify the
262 // re-materialization logic for both predicated and non-predicated
263 // targets.
264 bool is_mask = is_vector_mask(iklass);
265 if (is_mask && vec_value->Opcode() != Op_VectorStoreMask) {
266 const TypeVect* vt = vec_value->bottom_type()->is_vect();
267 BasicType bt = vt->element_basic_type();
268 vec_value = gvn.transform(VectorStoreMaskNode::make(gvn, vec_value, bt, vt->length()));
269 }
270
271 while (safepoints.size() > 0) {
272 SafePointNode* sfpt = safepoints.pop()->as_SafePoint();
273
274 uint first_ind = (sfpt->req() - sfpt->jvms()->scloff());
275 Node* sobj = new SafePointScalarObjectNode(vec_box->box_type(),
276 #ifdef ASSERT
277 vec_box,
278 #endif // ASSERT
279 first_ind, n_fields);
280 sobj->init_req(0, C->root());
281 sfpt->add_req(vec_value);
282
283 sobj = gvn.transform(sobj);
284
285 JVMState *jvms = sfpt->jvms();
286
287 jvms->set_endoff(sfpt->req());
288 // Now make a pass over the debug information replacing any references
289 // to the allocated object with vector value.
290 for (uint i = jvms->debug_start(); i < jvms->debug_end(); i++) {
291 Node* debug = sfpt->in(i);
292 if (debug != nullptr && debug->uncast(/*keep_deps*/false) == vec_box) {
293 sfpt->set_req(i, sobj);
294 }
295 }
296 C->record_for_igvn(sfpt);
297 }
298 }
299
300 void PhaseVector::expand_vbox_node(VectorBoxNode* vec_box) {
301 if (vec_box->outcnt() > 0) {
302 VectorSet visited;
303 Node* vbox = vec_box->in(VectorBoxNode::Box);
304 Node* vect = vec_box->in(VectorBoxNode::Value);
305 Node* result = expand_vbox_node_helper(vbox, vect, vec_box->box_type(),
306 vec_box->vec_type(), visited);
307 C->gvn_replace_by(vec_box, result);
308 C->print_method(PHASE_EXPAND_VBOX, 3, vec_box);
309 }
310 C->remove_macro_node(vec_box);
311 }
312
313 Node* PhaseVector::expand_vbox_node_helper(Node* vbox,
314 Node* vect,
315 const TypeInstPtr* box_type,
316 const TypeVect* vect_type,
317 VectorSet &visited) {
318 // JDK-8304948 shows an example that there may be a cycle in the graph.
319 if (visited.test_set(vbox->_idx)) {
320 assert(vbox->is_Phi(), "should be phi");
321 return vbox; // already visited
322 }
323
324 // Handle the case when the allocation input to VectorBoxNode is a Proj.
325 // This is the normal case before expanding.
326 if (vbox->is_Proj() && vbox->in(0)->Opcode() == Op_VectorBoxAllocate) {
327 VectorBoxAllocateNode* vbox_alloc = static_cast<VectorBoxAllocateNode*>(vbox->in(0));
328 return expand_vbox_alloc_node(vbox_alloc, vect, box_type, vect_type);
329 }
330
331 // Handle the case when both the allocation input and vector input to
332 // VectorBoxNode are Phi. This case is generated after the transformation of
333 // Phi: Phi (VectorBox1 VectorBox2) => VectorBox (Phi1 Phi2).
334 // With this optimization, the relative two allocation inputs of VectorBox1 and
335 // VectorBox2 are gathered into Phi1 now. Similarly, the original vector
336 // inputs of two VectorBox nodes are in Phi2.
337 //
338 // See PhiNode::merge_through_phi in cfg.cpp for more details.
339 if (vbox->is_Phi() && vect->is_Phi()) {
340 assert(vbox->as_Phi()->region() == vect->as_Phi()->region(), "");
341 for (uint i = 1; i < vbox->req(); i++) {
342 Node* new_box = expand_vbox_node_helper(vbox->in(i), vect->in(i),
343 box_type, vect_type, visited);
344 if (!new_box->is_Phi()) {
345 C->initial_gvn()->hash_delete(vbox);
346 vbox->set_req(i, new_box);
347 }
348 }
349 return C->initial_gvn()->transform(vbox);
350 }
351
352 // Handle the case when the allocation input to VectorBoxNode is a phi
353 // but the vector input is not, which can definitely be the case if the
354 // vector input has been value-numbered. It seems to be safe to do by
355 // construction because VectorBoxNode and VectorBoxAllocate come in a
356 // specific order as a result of expanding an intrinsic call. After that, if
357 // any of the inputs to VectorBoxNode are value-numbered they can only
358 // move up and are guaranteed to dominate.
359 if (vbox->is_Phi() && (vect->is_Vector() || vect->is_LoadVector())) {
360 for (uint i = 1; i < vbox->req(); i++) {
361 Node* new_box = expand_vbox_node_helper(vbox->in(i), vect,
362 box_type, vect_type, visited);
363 if (!new_box->is_Phi()) {
364 C->initial_gvn()->hash_delete(vbox);
365 vbox->set_req(i, new_box);
366 }
367 }
368 return C->initial_gvn()->transform(vbox);
369 }
370
371 assert(!vbox->is_Phi(), "should be expanded");
372 // TODO: assert that expanded vbox is initialized with the same value (vect).
373 return vbox; // already expanded
374 }
375
376 Node* PhaseVector::expand_vbox_alloc_node(VectorBoxAllocateNode* vbox_alloc,
377 Node* value,
378 const TypeInstPtr* box_type,
379 const TypeVect* vect_type) {
380 JVMState* jvms = clone_jvms(C, vbox_alloc);
381 GraphKit kit(jvms);
382 PhaseGVN& gvn = kit.gvn();
383
384 ciInstanceKlass* box_klass = box_type->instance_klass();
385 BasicType bt = vect_type->element_basic_type();
386 int num_elem = vect_type->length();
387
388 bool is_mask = is_vector_mask(box_klass);
389 // If boxed mask value is present in a predicate register, it must be
390 // spilled to a vector though a VectorStoreMaskOperation before actual StoreVector
391 // operation to vector payload field.
392 if (is_mask && (value->bottom_type()->isa_vectmask() || bt != T_BOOLEAN)) {
393 value = gvn.transform(VectorStoreMaskNode::make(gvn, value, bt, num_elem));
394 // Although type of mask depends on its definition, in terms of storage everything is stored in boolean array.
395 bt = T_BOOLEAN;
396 assert(value->bottom_type()->is_vect()->element_basic_type() == bt,
397 "must be consistent with mask representation");
398 }
399
400 // Generate array allocation for the field which holds the values.
401 const TypeKlassPtr* array_klass = TypeKlassPtr::make(ciTypeArrayKlass::make(bt));
402 Node* arr = kit.new_array(kit.makecon(array_klass), kit.intcon(num_elem), 1);
403
404 // Store the vector value into the array.
405 // (The store should be captured by InitializeNode and turned into initialized store later.)
406 Node* arr_adr = kit.array_element_address(arr, kit.intcon(0), bt);
407 const TypePtr* arr_adr_type = arr_adr->bottom_type()->is_ptr();
408 Node* arr_mem = kit.memory(arr_adr);
409 Node* vstore = gvn.transform(StoreVectorNode::make(0,
410 kit.control(),
411 arr_mem,
412 arr_adr,
413 arr_adr_type,
414 value,
415 num_elem));
416 kit.set_memory(vstore, arr_adr_type);
417
418 C->set_max_vector_size(MAX2(C->max_vector_size(), vect_type->length_in_bytes()));
419
420 // Generate the allocate for the Vector object.
421 const TypeKlassPtr* klass_type = box_type->as_klass_type();
422 Node* klass_node = kit.makecon(klass_type);
423 Node* vec_obj = kit.new_instance(klass_node);
424
425 // Store the allocated array into object.
426 ciField* field = ciEnv::current()->vector_VectorPayload_klass()->get_field_by_name(ciSymbols::payload_name(),
427 ciSymbols::object_signature(),
428 false);
429 assert(field != nullptr, "");
430 Node* vec_field = kit.basic_plus_adr(vec_obj, field->offset_in_bytes());
431 const TypePtr* vec_adr_type = vec_field->bottom_type()->is_ptr();
432
433 // The store should be captured by InitializeNode and turned into initialized store later.
434 Node* field_store = gvn.transform(kit.access_store_at(vec_obj,
435 vec_field,
436 vec_adr_type,
437 arr,
438 TypeOopPtr::make_from_klass(field->type()->as_klass()),
439 T_OBJECT,
440 IN_HEAP));
441 kit.set_memory(field_store, vec_adr_type);
442
443 kit.replace_call(vbox_alloc, vec_obj, true);
444 C->remove_macro_node(vbox_alloc);
445
446 return vec_obj;
447 }
448
449 void PhaseVector::expand_vunbox_node(VectorUnboxNode* vec_unbox) {
450 if (vec_unbox->outcnt() > 0) {
451 GraphKit kit;
452 PhaseGVN& gvn = kit.gvn();
453
454 Node* obj = vec_unbox->obj();
455 const TypeInstPtr* tinst = gvn.type(obj)->isa_instptr();
456 ciInstanceKlass* from_kls = tinst->instance_klass();
457 const TypeVect* vt = vec_unbox->bottom_type()->is_vect();
458 BasicType bt = vt->element_basic_type();
459 BasicType masktype = bt;
460
461 if (is_vector_mask(from_kls)) {
462 bt = T_BOOLEAN;
463 }
464
465 ciField* field = ciEnv::current()->vector_VectorPayload_klass()->get_field_by_name(ciSymbols::payload_name(),
466 ciSymbols::object_signature(),
467 false);
468 assert(field != nullptr, "");
469 int offset = field->offset_in_bytes();
470 Node* vec_adr = kit.basic_plus_adr(obj, offset);
471
472 Node* mem = vec_unbox->mem();
473 Node* ctrl = vec_unbox->in(0);
474 Node* vec_field_ld;
475 {
476 DecoratorSet decorators = MO_UNORDERED | IN_HEAP;
477 C2AccessValuePtr addr(vec_adr, vec_adr->bottom_type()->is_ptr());
478 MergeMemNode* local_mem = MergeMemNode::make(mem);
479 gvn.record_for_igvn(local_mem);
480 BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
481 C2OptAccess access(gvn, ctrl, local_mem, decorators, T_OBJECT, obj, addr);
482 const Type* type = TypeOopPtr::make_from_klass(field->type()->as_klass());
483 vec_field_ld = bs->load_at(access, type);
484 }
485
486 // For proper aliasing, attach concrete payload type.
487 ciKlass* payload_klass = ciTypeArrayKlass::make(bt);
488 const Type* payload_type = TypeAryPtr::make_from_klass(payload_klass)->cast_to_ptr_type(TypePtr::NotNull);
489 vec_field_ld = gvn.transform(new CastPPNode(vec_field_ld, payload_type));
490
491 Node* adr = kit.array_element_address(vec_field_ld, gvn.intcon(0), bt);
492 const TypePtr* adr_type = adr->bottom_type()->is_ptr();
493 int num_elem = vt->length();
494 Node* vec_val_load = LoadVectorNode::make(0,
495 ctrl,
496 mem,
497 adr,
498 adr_type,
499 num_elem,
500 bt);
501 vec_val_load = gvn.transform(vec_val_load);
502
503 C->set_max_vector_size(MAX2(C->max_vector_size(), vt->length_in_bytes()));
504
505 if (is_vector_mask(from_kls)) {
506 vec_val_load = gvn.transform(new VectorLoadMaskNode(vec_val_load, TypeVect::makemask(masktype, num_elem)));
507 }
508
509 gvn.hash_delete(vec_unbox);
510 vec_unbox->disconnect_inputs(C);
511 C->gvn_replace_by(vec_unbox, vec_val_load);
512 }
513 C->remove_macro_node(vec_unbox);
514 }
515
516 void PhaseVector::eliminate_vbox_alloc_node(VectorBoxAllocateNode* vbox_alloc) {
517 JVMState* jvms = clone_jvms(C, vbox_alloc);
518 GraphKit kit(jvms);
519 // Remove VBA, but leave a safepoint behind.
520 // Otherwise, it may end up with a loop without any safepoint polls.
521 kit.replace_call(vbox_alloc, kit.map(), true);
522 C->remove_macro_node(vbox_alloc);
523 }