1 /*
  2  * Copyright (c) 2020, 2021, Oracle and/or its affiliates. All rights reserved.
  3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  4  *
  5  * This code is free software; you can redistribute it and/or modify it
  6  * under the terms of the GNU General Public License version 2 only, as
  7  * published by the Free Software Foundation.
  8  *
  9  * This code is distributed in the hope that it will be useful, but WITHOUT
 10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 12  * version 2 for more details (a copy is included in the LICENSE file that
 13  * accompanied this code).
 14  *
 15  * You should have received a copy of the GNU General Public License version
 16  * 2 along with this work; if not, write to the Free Software Foundation,
 17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 18  *
 19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 20  * or visit www.oracle.com if you need additional information or have any
 21  * questions.
 22  *
 23  */
 24 
 25 #include "precompiled.hpp"
 26 #include "ci/ciSymbols.hpp"
 27 #include "gc/shared/barrierSet.hpp"
 28 #include "opto/castnode.hpp"
 29 #include "opto/graphKit.hpp"
 30 #include "opto/phaseX.hpp"
 31 #include "opto/rootnode.hpp"
 32 #include "opto/vector.hpp"
 33 #include "utilities/macros.hpp"
 34 
 35 static bool is_vector_mask(ciKlass* klass) {
 36   return klass->is_subclass_of(ciEnv::current()->vector_VectorMask_klass());
 37 }
 38 
 39 static bool is_vector_shuffle(ciKlass* klass) {
 40   return klass->is_subclass_of(ciEnv::current()->vector_VectorShuffle_klass());
 41 }
 42 
 43 
 44 void PhaseVector::optimize_vector_boxes() {
 45   Compile::TracePhase tp("vector_elimination", &timers[_t_vector_elimination]);
 46 
 47   // Signal GraphKit it's post-parse phase.
 48   assert(C->inlining_incrementally() == false, "sanity");
 49   C->set_inlining_incrementally(true);
 50 
 51   C->for_igvn()->clear();
 52   C->initial_gvn()->replace_with(&_igvn);
 53 
 54   expand_vunbox_nodes();
 55   scalarize_vbox_nodes();
 56 
 57   C->inline_vector_reboxing_calls();
 58 
 59   expand_vbox_nodes();
 60   eliminate_vbox_alloc_nodes();
 61 
 62   C->set_inlining_incrementally(false);
 63 
 64   do_cleanup();
 65 }
 66 
 67 void PhaseVector::do_cleanup() {
 68   if (C->failing())  return;
 69   {
 70     Compile::TracePhase tp("vector_pru", &timers[_t_vector_pru]);
 71     ResourceMark rm;
 72     PhaseRemoveUseless pru(C->initial_gvn(), C->for_igvn());
 73     if (C->failing())  return;
 74   }
 75   {
 76     Compile::TracePhase tp("incrementalInline_igvn", &timers[_t_vector_igvn]);
 77     _igvn = PhaseIterGVN(C->initial_gvn());
 78     _igvn.optimize();
 79     if (C->failing())  return;
 80   }
 81   C->print_method(PHASE_ITER_GVN_BEFORE_EA, 3);
 82 }
 83 
 84 void PhaseVector::scalarize_vbox_nodes() {
 85   if (C->failing())  return;
 86 
 87   if (!EnableVectorReboxing) {
 88     return; // don't scalarize vector boxes
 89   }
 90 
 91   int macro_idx = C->macro_count() - 1;
 92   while (macro_idx >= 0) {
 93     Node * n = C->macro_node(macro_idx);
 94     assert(n->is_macro(), "only macro nodes expected here");
 95     if (n->Opcode() == Op_VectorBox) {
 96       VectorBoxNode* vbox = static_cast<VectorBoxNode*>(n);
 97       scalarize_vbox_node(vbox);
 98       if (C->failing())  return;
 99       C->print_method(PHASE_SCALARIZE_VBOX, vbox, 3);
100     }
101     if (C->failing())  return;
102     macro_idx = MIN2(macro_idx - 1, C->macro_count() - 1);
103   }
104 }
105 
106 void PhaseVector::expand_vbox_nodes() {
107   if (C->failing())  return;
108 
109   int macro_idx = C->macro_count() - 1;
110   while (macro_idx >= 0) {
111     Node * n = C->macro_node(macro_idx);
112     assert(n->is_macro(), "only macro nodes expected here");
113     if (n->Opcode() == Op_VectorBox) {
114       VectorBoxNode* vbox = static_cast<VectorBoxNode*>(n);
115       expand_vbox_node(vbox);
116       if (C->failing())  return;
117     }
118     if (C->failing())  return;
119     macro_idx = MIN2(macro_idx - 1, C->macro_count() - 1);
120   }
121 }
122 
123 void PhaseVector::expand_vunbox_nodes() {
124   if (C->failing())  return;
125 
126   int macro_idx = C->macro_count() - 1;
127   while (macro_idx >= 0) {
128     Node * n = C->macro_node(macro_idx);
129     assert(n->is_macro(), "only macro nodes expected here");
130     if (n->Opcode() == Op_VectorUnbox) {
131       VectorUnboxNode* vec_unbox = static_cast<VectorUnboxNode*>(n);
132       expand_vunbox_node(vec_unbox);
133       if (C->failing())  return;
134       C->print_method(PHASE_EXPAND_VUNBOX, vec_unbox, 3);
135     }
136     if (C->failing())  return;
137     macro_idx = MIN2(macro_idx - 1, C->macro_count() - 1);
138   }
139 }
140 
141 void PhaseVector::eliminate_vbox_alloc_nodes() {
142   if (C->failing())  return;
143 
144   int macro_idx = C->macro_count() - 1;
145   while (macro_idx >= 0) {
146     Node * n = C->macro_node(macro_idx);
147     assert(n->is_macro(), "only macro nodes expected here");
148     if (n->Opcode() == Op_VectorBoxAllocate) {
149       VectorBoxAllocateNode* vbox_alloc = static_cast<VectorBoxAllocateNode*>(n);
150       eliminate_vbox_alloc_node(vbox_alloc);
151       if (C->failing())  return;
152       C->print_method(PHASE_ELIMINATE_VBOX_ALLOC, vbox_alloc, 3);
153     }
154     if (C->failing())  return;
155     macro_idx = MIN2(macro_idx - 1, C->macro_count() - 1);
156   }
157 }
158 
159 static JVMState* clone_jvms(Compile* C, SafePointNode* sfpt) {
160   JVMState* new_jvms = sfpt->jvms()->clone_shallow(C);
161   uint size = sfpt->req();
162   SafePointNode* map = new SafePointNode(size, new_jvms);
163   for (uint i = 0; i < size; i++) {
164     map->init_req(i, sfpt->in(i));
165   }
166   new_jvms->set_map(map);
167   return new_jvms;
168 }
169 
170 void PhaseVector::scalarize_vbox_node(VectorBoxNode* vec_box) {
171   Node* vec_value = vec_box->in(VectorBoxNode::Value);
172   PhaseGVN& gvn = *C->initial_gvn();
173 
174   // Process merged VBAs
175 
176   if (EnableVectorAggressiveReboxing) {
177     Unique_Node_List calls(C->comp_arena());
178     for (DUIterator_Fast imax, i = vec_box->fast_outs(imax); i < imax; i++) {
179       Node* use = vec_box->fast_out(i);
180       if (use->is_CallJava()) {
181         CallJavaNode* call = use->as_CallJava();
182         if (call->has_non_debug_use(vec_box) && vec_box->in(VectorBoxNode::Box)->is_Phi()) {
183           calls.push(call);
184         }
185       }
186     }
187 
188     while (calls.size() > 0) {
189       CallJavaNode* call = calls.pop()->as_CallJava();
190       // Attach new VBA to the call and use it instead of Phi (VBA ... VBA).
191 
192       JVMState* jvms = clone_jvms(C, call);
193       GraphKit kit(jvms);
194       PhaseGVN& gvn = kit.gvn();
195 
196       // Adjust JVMS from post-call to pre-call state: put args on stack
197       uint nargs = call->method()->arg_size();
198       kit.ensure_stack(kit.sp() + nargs);
199       for (uint i = TypeFunc::Parms; i < call->tf()->domain()->cnt(); i++) {
200         kit.push(call->in(i));
201       }
202       jvms = kit.sync_jvms();
203 
204       Node* new_vbox = NULL;
205       {
206         Node* vect = vec_box->in(VectorBoxNode::Value);
207         const TypeInstPtr* vbox_type = vec_box->box_type();
208         const TypeVect* vt = vec_box->vec_type();
209         BasicType elem_bt = vt->element_basic_type();
210         int num_elem = vt->length();
211 
212         new_vbox = kit.box_vector(vect, vbox_type, elem_bt, num_elem, /*deoptimize=*/true);
213 
214         kit.replace_in_map(vec_box, new_vbox);
215       }
216 
217       kit.dec_sp(nargs);
218       jvms = kit.sync_jvms();
219 
220       call->set_req(TypeFunc::Control , kit.control());
221       call->set_req(TypeFunc::I_O     , kit.i_o());
222       call->set_req(TypeFunc::Memory  , kit.reset_memory());
223       call->set_req(TypeFunc::FramePtr, kit.frameptr());
224       call->replace_edge(vec_box, new_vbox);
225 
226       C->record_for_igvn(call);
227     }
228   }
229 
230   // Process debug uses at safepoints
231   Unique_Node_List safepoints(C->comp_arena());
232 
233   Unique_Node_List worklist(C->comp_arena());
234   worklist.push(vec_box);
235   while (worklist.size() > 0) {
236     Node* n = worklist.pop();
237     for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) {
238       Node* use = n->fast_out(i);
239       if (use->is_SafePoint()) {
240         SafePointNode* sfpt = use->as_SafePoint();
241         if (!sfpt->is_Call() || !sfpt->as_Call()->has_non_debug_use(n)) {
242           safepoints.push(sfpt);
243         }
244       } else if (use->is_ConstraintCast()) {
245         worklist.push(use); // reversed version of Node::uncast()
246       }
247     }
248   }
249 
250   ciInstanceKlass* iklass = vec_box->box_type()->klass()->as_instance_klass();
251   int n_fields = iklass->nof_nonstatic_fields();
252   assert(n_fields == 1, "sanity");
253 
254   // If a mask is feeding into safepoint[s], then its value should be
255   // packed into a boolean/byte vector first, this will simplify the
256   // re-materialization logic for both predicated and non-predicated
257   // targets.
258   bool is_mask = is_vector_mask(iklass);
259   if (is_mask && vec_value->Opcode() != Op_VectorStoreMask) {
260     const TypeVect* vt = vec_value->bottom_type()->is_vect();
261     BasicType bt = vt->element_basic_type();
262     vec_value = gvn.transform(VectorStoreMaskNode::make(gvn, vec_value, bt, vt->length()));
263   }
264 
265   while (safepoints.size() > 0) {
266     SafePointNode* sfpt = safepoints.pop()->as_SafePoint();
267 
268     uint first_ind = (sfpt->req() - sfpt->jvms()->scloff());
269     Node* sobj = new SafePointScalarObjectNode(vec_box->box_type(),
270 #ifdef ASSERT
271                                                vec_box,
272 #endif // ASSERT
273                                                first_ind, n_fields);
274     sobj->init_req(0, C->root());
275     sfpt->add_req(vec_value);
276 
277     sobj = gvn.transform(sobj);
278 
279     JVMState *jvms = sfpt->jvms();
280 
281     jvms->set_endoff(sfpt->req());
282     // Now make a pass over the debug information replacing any references
283     // to the allocated object with vector value.
284     for (uint i = jvms->debug_start(); i < jvms->debug_end(); i++) {
285       Node* debug = sfpt->in(i);
286       if (debug != NULL && debug->uncast(/*keep_deps*/false) == vec_box) {
287         sfpt->set_req(i, sobj);
288       }
289     }
290     C->record_for_igvn(sfpt);
291   }
292 }
293 
294 void PhaseVector::expand_vbox_node(VectorBoxNode* vec_box) {
295   if (vec_box->outcnt() > 0) {
296     Node* vbox = vec_box->in(VectorBoxNode::Box);
297     Node* vect = vec_box->in(VectorBoxNode::Value);
298     Node* result = expand_vbox_node_helper(vbox, vect, vec_box->box_type(), vec_box->vec_type());
299     C->gvn_replace_by(vec_box, result);
300     C->print_method(PHASE_EXPAND_VBOX, vec_box, 3);
301   }
302   C->remove_macro_node(vec_box);
303 }
304 
305 Node* PhaseVector::expand_vbox_node_helper(Node* vbox,
306                                            Node* vect,
307                                            const TypeInstPtr* box_type,
308                                            const TypeVect* vect_type) {
309   if (vbox->is_Phi() && vect->is_Phi()) {
310     assert(vbox->as_Phi()->region() == vect->as_Phi()->region(), "");
311     Node* new_phi = new PhiNode(vbox->as_Phi()->region(), box_type);
312     for (uint i = 1; i < vbox->req(); i++) {
313       Node* new_box = expand_vbox_node_helper(vbox->in(i), vect->in(i), box_type, vect_type);
314       new_phi->set_req(i, new_box);
315     }
316     new_phi = C->initial_gvn()->transform(new_phi);
317     return new_phi;
318   } else if (vbox->is_Proj() && vbox->in(0)->Opcode() == Op_VectorBoxAllocate) {
319     VectorBoxAllocateNode* vbox_alloc = static_cast<VectorBoxAllocateNode*>(vbox->in(0));
320     return expand_vbox_alloc_node(vbox_alloc, vect, box_type, vect_type);
321   } else {
322     assert(!vbox->is_Phi(), "");
323     // TODO: assert that expanded vbox is initialized with the same value (vect).
324     return vbox; // already expanded
325   }
326 }
327 
328 Node* PhaseVector::expand_vbox_alloc_node(VectorBoxAllocateNode* vbox_alloc,
329                                           Node* value,
330                                           const TypeInstPtr* box_type,
331                                           const TypeVect* vect_type) {
332   JVMState* jvms = clone_jvms(C, vbox_alloc);
333   GraphKit kit(jvms);
334   PhaseGVN& gvn = kit.gvn();
335 
336   ciInstanceKlass* box_klass = box_type->klass()->as_instance_klass();
337   BasicType bt = vect_type->element_basic_type();
338   int num_elem = vect_type->length();
339 
340   bool is_mask = is_vector_mask(box_klass);
341   // If boxed mask value is present in a predicate register, it must be
342   // spilled to a vector though a VectorStoreMaskOperation before actual StoreVector
343   // operation to vector payload field.
344   if (is_mask && (value->bottom_type()->isa_vectmask() || bt != T_BOOLEAN)) {
345     value = gvn.transform(VectorStoreMaskNode::make(gvn, value, bt, num_elem));
346     // Although type of mask depends on its definition, in terms of storage everything is stored in boolean array.
347     bt = T_BOOLEAN;
348     assert(value->bottom_type()->is_vect()->element_basic_type() == bt,
349            "must be consistent with mask representation");
350   }
351 
352   // Generate array allocation for the field which holds the values.
353   const TypeKlassPtr* array_klass = TypeKlassPtr::make(ciTypeArrayKlass::make(bt));
354   Node* arr = kit.new_array(kit.makecon(array_klass), kit.intcon(num_elem), 1);
355 
356   // Store the vector value into the array.
357   // (The store should be captured by InitializeNode and turned into initialized store later.)
358   Node* arr_adr = kit.array_element_address(arr, kit.intcon(0), bt);
359   const TypePtr* arr_adr_type = arr_adr->bottom_type()->is_ptr();
360   Node* arr_mem = kit.memory(arr_adr);
361   Node* vstore = gvn.transform(StoreVectorNode::make(0,
362                                                      kit.control(),
363                                                      arr_mem,
364                                                      arr_adr,
365                                                      arr_adr_type,
366                                                      value,
367                                                      num_elem));
368   kit.set_memory(vstore, arr_adr_type);
369 
370   C->set_max_vector_size(MAX2(C->max_vector_size(), vect_type->length_in_bytes()));
371 
372   // Generate the allocate for the Vector object.
373   const TypeKlassPtr* klass_type = box_type->as_klass_type();
374   Node* klass_node = kit.makecon(klass_type);
375   Node* vec_obj = kit.new_instance(klass_node);
376 
377   // Store the allocated array into object.
378   ciField* field = ciEnv::current()->vector_VectorPayload_klass()->get_field_by_name(ciSymbols::payload_name(),
379                                                                                      ciSymbols::object_signature(),
380                                                                                      false);
381   assert(field != NULL, "");
382   Node* vec_field = kit.basic_plus_adr(vec_obj, field->offset_in_bytes());
383   const TypePtr* vec_adr_type = vec_field->bottom_type()->is_ptr();
384 
385   // The store should be captured by InitializeNode and turned into initialized store later.
386   Node* field_store = gvn.transform(kit.access_store_at(vec_obj,
387                                                         vec_field,
388                                                         vec_adr_type,
389                                                         arr,
390                                                         TypeOopPtr::make_from_klass(field->type()->as_klass()),
391                                                         T_OBJECT,
392                                                         IN_HEAP));
393   kit.set_memory(field_store, vec_adr_type);
394 
395   kit.replace_call(vbox_alloc, vec_obj, true);
396   C->remove_macro_node(vbox_alloc);
397 
398   return vec_obj;
399 }
400 
401 void PhaseVector::expand_vunbox_node(VectorUnboxNode* vec_unbox) {
402   if (vec_unbox->outcnt() > 0) {
403     GraphKit kit;
404     PhaseGVN& gvn = kit.gvn();
405 
406     Node* obj = vec_unbox->obj();
407     const TypeInstPtr* tinst = gvn.type(obj)->isa_instptr();
408     ciInstanceKlass* from_kls = tinst->klass()->as_instance_klass();
409     const TypeVect* vt = vec_unbox->bottom_type()->is_vect();
410     BasicType bt = vt->element_basic_type();
411     BasicType masktype = bt;
412 
413     if (is_vector_mask(from_kls)) {
414       bt = T_BOOLEAN;
415     } else if (is_vector_shuffle(from_kls)) {
416       bt = T_BYTE;
417     }
418 
419     ciField* field = ciEnv::current()->vector_VectorPayload_klass()->get_field_by_name(ciSymbols::payload_name(),
420                                                                                        ciSymbols::object_signature(),
421                                                                                        false);
422     assert(field != NULL, "");
423     int offset = field->offset_in_bytes();
424     Node* vec_adr = kit.basic_plus_adr(obj, offset);
425 
426     Node* mem = vec_unbox->mem();
427     Node* ctrl = vec_unbox->in(0);
428     Node* vec_field_ld;
429     {
430       DecoratorSet decorators = MO_UNORDERED | IN_HEAP;
431       C2AccessValuePtr addr(vec_adr, vec_adr->bottom_type()->is_ptr());
432       MergeMemNode* local_mem = MergeMemNode::make(mem);
433       gvn.record_for_igvn(local_mem);
434       BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
435       C2OptAccess access(gvn, ctrl, local_mem, decorators, T_OBJECT, obj, addr);
436       const Type* type = TypeOopPtr::make_from_klass(field->type()->as_klass());
437       vec_field_ld = bs->load_at(access, type);
438     }
439 
440     // For proper aliasing, attach concrete payload type.
441     ciKlass* payload_klass = ciTypeArrayKlass::make(bt);
442     const Type* payload_type = TypeAryPtr::make_from_klass(payload_klass)->cast_to_ptr_type(TypePtr::NotNull);
443     vec_field_ld = gvn.transform(new CastPPNode(vec_field_ld, payload_type));
444 
445     Node* adr = kit.array_element_address(vec_field_ld, gvn.intcon(0), bt);
446     const TypePtr* adr_type = adr->bottom_type()->is_ptr();
447     int num_elem = vt->length();
448     Node* vec_val_load = LoadVectorNode::make(0,
449                                               ctrl,
450                                               mem,
451                                               adr,
452                                               adr_type,
453                                               num_elem,
454                                               bt);
455     vec_val_load = gvn.transform(vec_val_load);
456 
457     C->set_max_vector_size(MAX2(C->max_vector_size(), vt->length_in_bytes()));
458 
459     if (is_vector_mask(from_kls)) {
460       vec_val_load = gvn.transform(new VectorLoadMaskNode(vec_val_load, TypeVect::makemask(masktype, num_elem)));
461     } else if (is_vector_shuffle(from_kls) && !vec_unbox->is_shuffle_to_vector()) {
462       assert(vec_unbox->bottom_type()->is_vect()->element_basic_type() == masktype, "expect shuffle type consistency");
463       vec_val_load = gvn.transform(new VectorLoadShuffleNode(vec_val_load, TypeVect::make(masktype, num_elem)));
464     }
465 
466     gvn.hash_delete(vec_unbox);
467     vec_unbox->disconnect_inputs(C);
468     C->gvn_replace_by(vec_unbox, vec_val_load);
469   }
470   C->remove_macro_node(vec_unbox);
471 }
472 
473 void PhaseVector::eliminate_vbox_alloc_node(VectorBoxAllocateNode* vbox_alloc) {
474   JVMState* jvms = clone_jvms(C, vbox_alloc);
475   GraphKit kit(jvms);
476   // Remove VBA, but leave a safepoint behind.
477   // Otherwise, it may end up with a loop without any safepoint polls.
478   kit.replace_call(vbox_alloc, kit.map(), true);
479   C->remove_macro_node(vbox_alloc);
480 }