1 /*
  2  * Copyright (c) 2020, 2023, Oracle and/or its affiliates. All rights reserved.
  3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  4  *
  5  * This code is free software; you can redistribute it and/or modify it
  6  * under the terms of the GNU General Public License version 2 only, as
  7  * published by the Free Software Foundation.
  8  *
  9  * This code is distributed in the hope that it will be useful, but WITHOUT
 10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 12  * version 2 for more details (a copy is included in the LICENSE file that
 13  * accompanied this code).
 14  *
 15  * You should have received a copy of the GNU General Public License version
 16  * 2 along with this work; if not, write to the Free Software Foundation,
 17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 18  *
 19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 20  * or visit www.oracle.com if you need additional information or have any
 21  * questions.
 22  *
 23  */
 24 
 25 #include "precompiled.hpp"
 26 #include "ci/ciSymbols.hpp"
 27 #include "gc/shared/barrierSet.hpp"
 28 #include "opto/castnode.hpp"
 29 #include "opto/graphKit.hpp"
 30 #include "opto/phaseX.hpp"
 31 #include "opto/rootnode.hpp"
 32 #include "opto/vector.hpp"
 33 #include "utilities/macros.hpp"
 34 
 35 static bool is_vector_mask(ciKlass* klass) {
 36   return klass->is_subclass_of(ciEnv::current()->vector_VectorMask_klass());
 37 }
 38 
 39 void PhaseVector::optimize_vector_boxes() {
 40   Compile::TracePhase tp("vector_elimination", &timers[_t_vector_elimination]);
 41 
 42   // Signal GraphKit it's post-parse phase.
 43   assert(C->inlining_incrementally() == false, "sanity");
 44   C->set_inlining_incrementally(true);
 45 
 46   C->for_igvn()->clear();
 47   C->initial_gvn()->replace_with(&_igvn);
 48 
 49   expand_vunbox_nodes();
 50   scalarize_vbox_nodes();
 51 
 52   C->inline_vector_reboxing_calls();
 53 
 54   expand_vbox_nodes();
 55   eliminate_vbox_alloc_nodes();
 56 
 57   C->set_inlining_incrementally(false);
 58 
 59   do_cleanup();
 60 }
 61 
 62 void PhaseVector::do_cleanup() {
 63   if (C->failing())  return;
 64   {
 65     Compile::TracePhase tp("vector_pru", &timers[_t_vector_pru]);
 66     ResourceMark rm;
 67     PhaseRemoveUseless pru(C->initial_gvn(), C->for_igvn());
 68     if (C->failing())  return;
 69   }
 70   {
 71     Compile::TracePhase tp("incrementalInline_igvn", &timers[_t_vector_igvn]);
 72     _igvn = PhaseIterGVN(C->initial_gvn());
 73     _igvn.optimize();
 74     if (C->failing())  return;
 75   }
 76   C->print_method(PHASE_ITER_GVN_BEFORE_EA, 3);
 77 }
 78 
 79 void PhaseVector::scalarize_vbox_nodes() {
 80   if (C->failing())  return;
 81 
 82   if (!EnableVectorReboxing) {
 83     return; // don't scalarize vector boxes
 84   }
 85 
 86   int macro_idx = C->macro_count() - 1;
 87   while (macro_idx >= 0) {
 88     Node * n = C->macro_node(macro_idx);
 89     assert(n->is_macro(), "only macro nodes expected here");
 90     if (n->Opcode() == Op_VectorBox) {
 91       VectorBoxNode* vbox = static_cast<VectorBoxNode*>(n);
 92       scalarize_vbox_node(vbox);
 93       if (C->failing())  return;
 94       C->print_method(PHASE_SCALARIZE_VBOX, 3, vbox);
 95     }
 96     if (C->failing())  return;
 97     macro_idx = MIN2(macro_idx - 1, C->macro_count() - 1);
 98   }
 99 }
100 
101 void PhaseVector::expand_vbox_nodes() {
102   if (C->failing())  return;
103 
104   int macro_idx = C->macro_count() - 1;
105   while (macro_idx >= 0) {
106     Node * n = C->macro_node(macro_idx);
107     assert(n->is_macro(), "only macro nodes expected here");
108     if (n->Opcode() == Op_VectorBox) {
109       VectorBoxNode* vbox = static_cast<VectorBoxNode*>(n);
110       expand_vbox_node(vbox);
111       if (C->failing())  return;
112     }
113     if (C->failing())  return;
114     macro_idx = MIN2(macro_idx - 1, C->macro_count() - 1);
115   }
116 }
117 
118 void PhaseVector::expand_vunbox_nodes() {
119   if (C->failing())  return;
120 
121   int macro_idx = C->macro_count() - 1;
122   while (macro_idx >= 0) {
123     Node * n = C->macro_node(macro_idx);
124     assert(n->is_macro(), "only macro nodes expected here");
125     if (n->Opcode() == Op_VectorUnbox) {
126       VectorUnboxNode* vec_unbox = static_cast<VectorUnboxNode*>(n);
127       expand_vunbox_node(vec_unbox);
128       if (C->failing())  return;
129       C->print_method(PHASE_EXPAND_VUNBOX, 3, vec_unbox);
130     }
131     if (C->failing())  return;
132     macro_idx = MIN2(macro_idx - 1, C->macro_count() - 1);
133   }
134 }
135 
136 void PhaseVector::eliminate_vbox_alloc_nodes() {
137   if (C->failing())  return;
138 
139   int macro_idx = C->macro_count() - 1;
140   while (macro_idx >= 0) {
141     Node * n = C->macro_node(macro_idx);
142     assert(n->is_macro(), "only macro nodes expected here");
143     if (n->Opcode() == Op_VectorBoxAllocate) {
144       VectorBoxAllocateNode* vbox_alloc = static_cast<VectorBoxAllocateNode*>(n);
145       eliminate_vbox_alloc_node(vbox_alloc);
146       if (C->failing())  return;
147       C->print_method(PHASE_ELIMINATE_VBOX_ALLOC, 3, vbox_alloc);
148     }
149     if (C->failing())  return;
150     macro_idx = MIN2(macro_idx - 1, C->macro_count() - 1);
151   }
152 }
153 
154 static JVMState* clone_jvms(Compile* C, SafePointNode* sfpt) {
155   JVMState* new_jvms = sfpt->jvms()->clone_shallow(C);
156   uint size = sfpt->req();
157   SafePointNode* map = new SafePointNode(size, new_jvms);
158   for (uint i = 0; i < size; i++) {
159     map->init_req(i, sfpt->in(i));
160   }
161   Node* mem = map->memory();
162   if (!mem->is_MergeMem()) {
163     // Since we are not in parsing, the SafePointNode does not guarantee that the memory
164     // input is necessarily a MergeMemNode. But we need to ensure that there is that
165     // MergeMemNode, since the GraphKit assumes the memory input of the map to be a
166     // MergeMemNode, so that it can directly access the memory slices.
167     PhaseGVN& gvn = *C->initial_gvn();
168     Node* mergemem = MergeMemNode::make(mem);
169     gvn.set_type_bottom(mergemem);
170     map->set_memory(mergemem);
171   }
172   new_jvms->set_map(map);
173   return new_jvms;
174 }
175 
176 void PhaseVector::scalarize_vbox_node(VectorBoxNode* vec_box) {
177   Node* vec_value = vec_box->in(VectorBoxNode::Value);
178   PhaseGVN& gvn = *C->initial_gvn();
179 
180   // Process merged VBAs
181 
182   if (EnableVectorAggressiveReboxing) {
183     Unique_Node_List calls(C->comp_arena());
184     for (DUIterator_Fast imax, i = vec_box->fast_outs(imax); i < imax; i++) {
185       Node* use = vec_box->fast_out(i);
186       if (use->is_CallJava()) {
187         CallJavaNode* call = use->as_CallJava();
188         if (call->has_non_debug_use(vec_box) && vec_box->in(VectorBoxNode::Box)->is_Phi()) {
189           calls.push(call);
190         }
191       }
192     }
193 
194     while (calls.size() > 0) {
195       CallJavaNode* call = calls.pop()->as_CallJava();
196       // Attach new VBA to the call and use it instead of Phi (VBA ... VBA).
197 
198       JVMState* jvms = clone_jvms(C, call);
199       GraphKit kit(jvms);
200       PhaseGVN& gvn = kit.gvn();
201 
202       // Adjust JVMS from post-call to pre-call state: put args on stack
203       uint nargs = call->method()->arg_size();
204       kit.ensure_stack(kit.sp() + nargs);
205       for (uint i = TypeFunc::Parms; i < call->tf()->domain_sig()->cnt(); i++) {
206         kit.push(call->in(i));
207       }
208       jvms = kit.sync_jvms();
209 
210       Node* new_vbox = nullptr;
211       {
212         Node* vect = vec_box->in(VectorBoxNode::Value);
213         const TypeInstPtr* vbox_type = vec_box->box_type();
214         const TypeVect* vt = vec_box->vec_type();
215         BasicType elem_bt = vt->element_basic_type();
216         int num_elem = vt->length();
217 
218         new_vbox = kit.box_vector(vect, vbox_type, elem_bt, num_elem, /*deoptimize=*/true);
219 
220         kit.replace_in_map(vec_box, new_vbox);
221       }
222 
223       kit.dec_sp(nargs);
224       jvms = kit.sync_jvms();
225 
226       call->set_req(TypeFunc::Control , kit.control());
227       call->set_req(TypeFunc::I_O     , kit.i_o());
228       call->set_req(TypeFunc::Memory  , kit.reset_memory());
229       call->set_req(TypeFunc::FramePtr, kit.frameptr());
230       call->replace_edge(vec_box, new_vbox);
231 
232       C->record_for_igvn(call);
233     }
234   }
235 
236   // Process debug uses at safepoints
237   Unique_Node_List safepoints(C->comp_arena());
238 
239   Unique_Node_List worklist(C->comp_arena());
240   worklist.push(vec_box);
241   while (worklist.size() > 0) {
242     Node* n = worklist.pop();
243     for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) {
244       Node* use = n->fast_out(i);
245       if (use->is_SafePoint()) {
246         SafePointNode* sfpt = use->as_SafePoint();
247         if (!sfpt->is_Call() || !sfpt->as_Call()->has_non_debug_use(n)) {
248           safepoints.push(sfpt);
249         }
250       } else if (use->is_ConstraintCast()) {
251         worklist.push(use); // reversed version of Node::uncast()
252       }
253     }
254   }
255 
256   ciInstanceKlass* iklass = vec_box->box_type()->instance_klass();
257   int n_fields = iklass->nof_nonstatic_fields();
258   assert(n_fields == 1, "sanity");
259 
260   // If a mask is feeding into safepoint[s], then its value should be
261   // packed into a boolean/byte vector first, this will simplify the
262   // re-materialization logic for both predicated and non-predicated
263   // targets.
264   bool is_mask = is_vector_mask(iklass);
265   if (is_mask && vec_value->Opcode() != Op_VectorStoreMask) {
266     const TypeVect* vt = vec_value->bottom_type()->is_vect();
267     BasicType bt = vt->element_basic_type();
268     vec_value = gvn.transform(VectorStoreMaskNode::make(gvn, vec_value, bt, vt->length()));
269   }
270 
271   while (safepoints.size() > 0) {
272     SafePointNode* sfpt = safepoints.pop()->as_SafePoint();
273 
274     uint first_ind = (sfpt->req() - sfpt->jvms()->scloff());
275     Node* sobj = new SafePointScalarObjectNode(vec_box->box_type(),
276 #ifdef ASSERT
277                                                vec_box,
278 #endif // ASSERT
279                                                first_ind, n_fields);
280     sobj->init_req(0, C->root());
281     sfpt->add_req(vec_value);
282 
283     sobj = gvn.transform(sobj);
284 
285     JVMState *jvms = sfpt->jvms();
286 
287     jvms->set_endoff(sfpt->req());
288     // Now make a pass over the debug information replacing any references
289     // to the allocated object with vector value.
290     for (uint i = jvms->debug_start(); i < jvms->debug_end(); i++) {
291       Node* debug = sfpt->in(i);
292       if (debug != nullptr && debug->uncast(/*keep_deps*/false) == vec_box) {
293         sfpt->set_req(i, sobj);
294       }
295     }
296     C->record_for_igvn(sfpt);
297   }
298 }
299 
300 void PhaseVector::expand_vbox_node(VectorBoxNode* vec_box) {
301   if (vec_box->outcnt() > 0) {
302     VectorSet visited;
303     Node* vbox = vec_box->in(VectorBoxNode::Box);
304     Node* vect = vec_box->in(VectorBoxNode::Value);
305     Node* result = expand_vbox_node_helper(vbox, vect, vec_box->box_type(),
306                                            vec_box->vec_type(), visited);
307     C->gvn_replace_by(vec_box, result);
308     C->print_method(PHASE_EXPAND_VBOX, 3, vec_box);
309   }
310   C->remove_macro_node(vec_box);
311 }
312 
313 Node* PhaseVector::expand_vbox_node_helper(Node* vbox,
314                                            Node* vect,
315                                            const TypeInstPtr* box_type,
316                                            const TypeVect* vect_type,
317                                            VectorSet &visited) {
318   // JDK-8304948 shows an example that there may be a cycle in the graph.
319   if (visited.test_set(vbox->_idx)) {
320     assert(vbox->is_Phi(), "should be phi");
321     return vbox; // already visited
322   }
323 
324   // Handle the case when the allocation input to VectorBoxNode is a Proj.
325   // This is the normal case before expanding.
326   if (vbox->is_Proj() && vbox->in(0)->Opcode() == Op_VectorBoxAllocate) {
327     VectorBoxAllocateNode* vbox_alloc = static_cast<VectorBoxAllocateNode*>(vbox->in(0));
328     return expand_vbox_alloc_node(vbox_alloc, vect, box_type, vect_type);
329   }
330 
331   // Handle the case when both the allocation input and vector input to
332   // VectorBoxNode are Phi. This case is generated after the transformation of
333   // Phi: Phi (VectorBox1 VectorBox2) => VectorBox (Phi1 Phi2).
334   // With this optimization, the relative two allocation inputs of VectorBox1 and
335   // VectorBox2 are gathered into Phi1 now. Similarly, the original vector
336   // inputs of two VectorBox nodes are in Phi2.
337   //
338   // See PhiNode::merge_through_phi in cfg.cpp for more details.
339   if (vbox->is_Phi() && vect->is_Phi()) {
340     assert(vbox->as_Phi()->region() == vect->as_Phi()->region(), "");
341     for (uint i = 1; i < vbox->req(); i++) {
342       Node* new_box = expand_vbox_node_helper(vbox->in(i), vect->in(i),
343                                               box_type, vect_type, visited);
344       if (!new_box->is_Phi()) {
345         C->initial_gvn()->hash_delete(vbox);
346         vbox->set_req(i, new_box);
347       }
348     }
349     return C->initial_gvn()->transform(vbox);
350   }
351 
352   // Handle the case when the allocation input to VectorBoxNode is a phi
353   // but the vector input is not, which can definitely be the case if the
354   // vector input has been value-numbered. It seems to be safe to do by
355   // construction because VectorBoxNode and VectorBoxAllocate come in a
356   // specific order as a result of expanding an intrinsic call. After that, if
357   // any of the inputs to VectorBoxNode are value-numbered they can only
358   // move up and are guaranteed to dominate.
359   if (vbox->is_Phi() && (vect->is_Vector() || vect->is_LoadVector())) {
360     for (uint i = 1; i < vbox->req(); i++) {
361       Node* new_box = expand_vbox_node_helper(vbox->in(i), vect,
362                                               box_type, vect_type, visited);
363       if (!new_box->is_Phi()) {
364         C->initial_gvn()->hash_delete(vbox);
365         vbox->set_req(i, new_box);
366       }
367     }
368     return C->initial_gvn()->transform(vbox);
369   }
370 
371   assert(!vbox->is_Phi(), "should be expanded");
372   // TODO: assert that expanded vbox is initialized with the same value (vect).
373   return vbox; // already expanded
374 }
375 
376 Node* PhaseVector::expand_vbox_alloc_node(VectorBoxAllocateNode* vbox_alloc,
377                                           Node* value,
378                                           const TypeInstPtr* box_type,
379                                           const TypeVect* vect_type) {
380   JVMState* jvms = clone_jvms(C, vbox_alloc);
381   GraphKit kit(jvms);
382   PhaseGVN& gvn = kit.gvn();
383 
384   ciInstanceKlass* box_klass = box_type->instance_klass();
385   BasicType bt = vect_type->element_basic_type();
386   int num_elem = vect_type->length();
387 
388   bool is_mask = is_vector_mask(box_klass);
389   // If boxed mask value is present in a predicate register, it must be
390   // spilled to a vector though a VectorStoreMaskOperation before actual StoreVector
391   // operation to vector payload field.
392   if (is_mask && (value->bottom_type()->isa_vectmask() || bt != T_BOOLEAN)) {
393     value = gvn.transform(VectorStoreMaskNode::make(gvn, value, bt, num_elem));
394     // Although type of mask depends on its definition, in terms of storage everything is stored in boolean array.
395     bt = T_BOOLEAN;
396     assert(value->bottom_type()->is_vect()->element_basic_type() == bt,
397            "must be consistent with mask representation");
398   }
399 
400   // Generate array allocation for the field which holds the values.
401   const TypeKlassPtr* array_klass = TypeKlassPtr::make(ciTypeArrayKlass::make(bt));
402   Node* arr = kit.new_array(kit.makecon(array_klass), kit.intcon(num_elem), 1);
403 
404   // Store the vector value into the array.
405   // (The store should be captured by InitializeNode and turned into initialized store later.)
406   Node* arr_adr = kit.array_element_address(arr, kit.intcon(0), bt);
407   const TypePtr* arr_adr_type = arr_adr->bottom_type()->is_ptr();
408   Node* arr_mem = kit.memory(arr_adr);
409   Node* vstore = gvn.transform(StoreVectorNode::make(0,
410                                                      kit.control(),
411                                                      arr_mem,
412                                                      arr_adr,
413                                                      arr_adr_type,
414                                                      value,
415                                                      num_elem));
416   kit.set_memory(vstore, arr_adr_type);
417 
418   C->set_max_vector_size(MAX2(C->max_vector_size(), vect_type->length_in_bytes()));
419 
420   // Generate the allocate for the Vector object.
421   const TypeKlassPtr* klass_type = box_type->as_klass_type();
422   Node* klass_node = kit.makecon(klass_type);
423   Node* vec_obj = kit.new_instance(klass_node);
424 
425   // Store the allocated array into object.
426   ciField* field = ciEnv::current()->vector_VectorPayload_klass()->get_field_by_name(ciSymbols::payload_name(),
427                                                                                      ciSymbols::object_signature(),
428                                                                                      false);
429   assert(field != nullptr, "");
430   Node* vec_field = kit.basic_plus_adr(vec_obj, field->offset_in_bytes());
431   const TypePtr* vec_adr_type = vec_field->bottom_type()->is_ptr();
432 
433   // The store should be captured by InitializeNode and turned into initialized store later.
434   Node* field_store = gvn.transform(kit.access_store_at(vec_obj,
435                                                         vec_field,
436                                                         vec_adr_type,
437                                                         arr,
438                                                         TypeOopPtr::make_from_klass(field->type()->as_klass()),
439                                                         T_OBJECT,
440                                                         IN_HEAP));
441   kit.set_memory(field_store, vec_adr_type);
442 
443   kit.replace_call(vbox_alloc, vec_obj, true);
444   C->remove_macro_node(vbox_alloc);
445 
446   return vec_obj;
447 }
448 
449 void PhaseVector::expand_vunbox_node(VectorUnboxNode* vec_unbox) {
450   if (vec_unbox->outcnt() > 0) {
451     GraphKit kit;
452     PhaseGVN& gvn = kit.gvn();
453 
454     Node* obj = vec_unbox->obj();
455     const TypeInstPtr* tinst = gvn.type(obj)->isa_instptr();
456     ciInstanceKlass* from_kls = tinst->instance_klass();
457     const TypeVect* vt = vec_unbox->bottom_type()->is_vect();
458     BasicType bt = vt->element_basic_type();
459     BasicType masktype = bt;
460 
461     if (is_vector_mask(from_kls)) {
462       bt = T_BOOLEAN;
463     }
464 
465     ciField* field = ciEnv::current()->vector_VectorPayload_klass()->get_field_by_name(ciSymbols::payload_name(),
466                                                                                        ciSymbols::object_signature(),
467                                                                                        false);
468     assert(field != nullptr, "");
469     int offset = field->offset_in_bytes();
470     Node* vec_adr = kit.basic_plus_adr(obj, offset);
471 
472     Node* mem = vec_unbox->mem();
473     Node* ctrl = vec_unbox->in(0);
474     Node* vec_field_ld;
475     {
476       DecoratorSet decorators = MO_UNORDERED | IN_HEAP;
477       C2AccessValuePtr addr(vec_adr, vec_adr->bottom_type()->is_ptr());
478       MergeMemNode* local_mem = MergeMemNode::make(mem);
479       gvn.record_for_igvn(local_mem);
480       BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
481       C2OptAccess access(gvn, ctrl, local_mem, decorators, T_OBJECT, obj, addr);
482       const Type* type = TypeOopPtr::make_from_klass(field->type()->as_klass());
483       vec_field_ld = bs->load_at(access, type);
484     }
485 
486     // For proper aliasing, attach concrete payload type.
487     ciKlass* payload_klass = ciTypeArrayKlass::make(bt);
488     const Type* payload_type = TypeAryPtr::make_from_klass(payload_klass)->cast_to_ptr_type(TypePtr::NotNull);
489     vec_field_ld = gvn.transform(new CastPPNode(vec_field_ld, payload_type));
490 
491     Node* adr = kit.array_element_address(vec_field_ld, gvn.intcon(0), bt);
492     const TypePtr* adr_type = adr->bottom_type()->is_ptr();
493     int num_elem = vt->length();
494     Node* vec_val_load = LoadVectorNode::make(0,
495                                               ctrl,
496                                               mem,
497                                               adr,
498                                               adr_type,
499                                               num_elem,
500                                               bt);
501     vec_val_load = gvn.transform(vec_val_load);
502 
503     C->set_max_vector_size(MAX2(C->max_vector_size(), vt->length_in_bytes()));
504 
505     if (is_vector_mask(from_kls)) {
506       vec_val_load = gvn.transform(new VectorLoadMaskNode(vec_val_load, TypeVect::makemask(masktype, num_elem)));
507     }
508 
509     gvn.hash_delete(vec_unbox);
510     vec_unbox->disconnect_inputs(C);
511     C->gvn_replace_by(vec_unbox, vec_val_load);
512   }
513   C->remove_macro_node(vec_unbox);
514 }
515 
516 void PhaseVector::eliminate_vbox_alloc_node(VectorBoxAllocateNode* vbox_alloc) {
517   JVMState* jvms = clone_jvms(C, vbox_alloc);
518   GraphKit kit(jvms);
519   // Remove VBA, but leave a safepoint behind.
520   // Otherwise, it may end up with a loop without any safepoint polls.
521   kit.replace_call(vbox_alloc, kit.map(), true);
522   C->remove_macro_node(vbox_alloc);
523 }