1 /*
2 * Copyright (c) 2020, 2024, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "ci/ciSymbols.hpp"
27 #include "classfile/vmSymbols.hpp"
28 #include "opto/library_call.hpp"
29 #include "opto/runtime.hpp"
30 #include "opto/vectornode.hpp"
31 #include "prims/vectorSupport.hpp"
32 #include "runtime/stubRoutines.hpp"
33
34 #ifdef ASSERT
35 static bool is_vector(ciKlass* klass) {
36 return klass->is_subclass_of(ciEnv::current()->vector_VectorPayload_klass());
37 }
38
39 static bool check_vbox(const TypeInstPtr* vbox_type) {
40 assert(vbox_type->klass_is_exact(), "");
41
42 ciInstanceKlass* ik = vbox_type->instance_klass();
43 assert(is_vector(ik), "not a vector");
44
45 ciField* fd1 = ik->get_field_by_name(ciSymbols::ETYPE_name(), ciSymbols::class_signature(), /* is_static */ true);
46 assert(fd1 != nullptr, "element type info is missing");
47
48 ciConstant val1 = fd1->constant_value();
49 BasicType elem_bt = val1.as_object()->as_instance()->java_mirror_type()->basic_type();
50 assert(is_java_primitive(elem_bt), "element type info is missing");
51
52 ciField* fd2 = ik->get_field_by_name(ciSymbols::VLENGTH_name(), ciSymbols::int_signature(), /* is_static */ true);
53 assert(fd2 != nullptr, "vector length info is missing");
54
55 ciConstant val2 = fd2->constant_value();
56 assert(val2.as_int() > 0, "vector length info is missing");
57
58 return true;
59 }
60 #endif
61
62 #define log_if_needed(...) \
63 if (C->print_intrinsics()) { \
64 tty->print_cr(__VA_ARGS__); \
65 }
66
67 #ifndef PRODUCT
68 #define non_product_log_if_needed(...) log_if_needed(__VA_ARGS__)
69 #else
70 #define non_product_log_if_needed(...)
71 #endif
72
73 static bool is_vector_mask(ciKlass* klass) {
74 return klass->is_subclass_of(ciEnv::current()->vector_VectorMask_klass());
75 }
76
77 bool LibraryCallKit::arch_supports_vector_rotate(int opc, int num_elem, BasicType elem_bt,
78 VectorMaskUseType mask_use_type, bool has_scalar_args) {
79 bool is_supported = true;
80
81 // has_scalar_args flag is true only for non-constant scalar shift count,
82 // since in this case shift needs to be broadcasted.
83 if (!Matcher::match_rule_supported_vector(opc, num_elem, elem_bt) ||
84 (has_scalar_args && !arch_supports_vector(Op_Replicate, num_elem, elem_bt, VecMaskNotUsed))) {
85 is_supported = false;
86 }
87
88 if (is_supported) {
89 // Check if mask unboxing is supported, this is a two step process which first loads the contents
90 // of boolean array into vector followed by either lane expansion to match the lane size of masked
91 // vector operation or populate the predicate register.
92 if ((mask_use_type & VecMaskUseLoad) != 0) {
93 if (!Matcher::match_rule_supported_vector(Op_VectorLoadMask, num_elem, elem_bt) ||
94 !Matcher::match_rule_supported_vector(Op_LoadVector, num_elem, T_BOOLEAN)) {
95 non_product_log_if_needed(" ** Rejected vector mask loading (%s,%s,%d) because architecture does not support it",
96 NodeClassNames[Op_VectorLoadMask], type2name(elem_bt), num_elem);
97 return false;
98 }
99 }
100
101 if ((mask_use_type & VecMaskUsePred) != 0) {
102 if (!Matcher::has_predicated_vectors() ||
103 !Matcher::match_rule_supported_vector_masked(opc, num_elem, elem_bt)) {
104 non_product_log_if_needed("Rejected vector mask predicate using (%s,%s,%d) because architecture does not support it",
105 NodeClassNames[opc], type2name(elem_bt), num_elem);
106 return false;
107 }
108 }
109 }
110
111 int lshiftopc, rshiftopc;
112 switch(elem_bt) {
113 case T_BYTE:
114 lshiftopc = Op_LShiftI;
115 rshiftopc = Op_URShiftB;
116 break;
117 case T_SHORT:
118 lshiftopc = Op_LShiftI;
119 rshiftopc = Op_URShiftS;
120 break;
121 case T_INT:
122 lshiftopc = Op_LShiftI;
123 rshiftopc = Op_URShiftI;
124 break;
125 case T_LONG:
126 lshiftopc = Op_LShiftL;
127 rshiftopc = Op_URShiftL;
128 break;
129 default: fatal("Unexpected type: %s", type2name(elem_bt));
130 }
131 int lshiftvopc = VectorNode::opcode(lshiftopc, elem_bt);
132 int rshiftvopc = VectorNode::opcode(rshiftopc, elem_bt);
133 if (!is_supported &&
134 arch_supports_vector(lshiftvopc, num_elem, elem_bt, VecMaskNotUsed, has_scalar_args) &&
135 arch_supports_vector(rshiftvopc, num_elem, elem_bt, VecMaskNotUsed, has_scalar_args) &&
136 arch_supports_vector(Op_OrV, num_elem, elem_bt, VecMaskNotUsed)) {
137 is_supported = true;
138 }
139 return is_supported;
140 }
141
142 Node* GraphKit::box_vector(Node* vector, const TypeInstPtr* vbox_type, BasicType elem_bt, int num_elem, bool deoptimize_on_exception) {
143 assert(EnableVectorSupport, "");
144
145 PreserveReexecuteState preexecs(this);
146 jvms()->set_should_reexecute(true);
147
148 VectorBoxAllocateNode* alloc = new VectorBoxAllocateNode(C, vbox_type);
149 set_edges_for_java_call(alloc, /*must_throw=*/false, /*separate_io_proj=*/true);
150 make_slow_call_ex(alloc, env()->Throwable_klass(), /*separate_io_proj=*/true, deoptimize_on_exception);
151 set_i_o(gvn().transform( new ProjNode(alloc, TypeFunc::I_O) ));
152 set_all_memory(gvn().transform( new ProjNode(alloc, TypeFunc::Memory) ));
153 Node* ret = gvn().transform(new ProjNode(alloc, TypeFunc::Parms));
154
155 assert(check_vbox(vbox_type), "");
156 const TypeVect* vt = TypeVect::make(elem_bt, num_elem, is_vector_mask(vbox_type->instance_klass()));
157 VectorBoxNode* vbox = new VectorBoxNode(C, ret, vector, vbox_type, vt);
158 return gvn().transform(vbox);
159 }
160
161 Node* GraphKit::unbox_vector(Node* v, const TypeInstPtr* vbox_type, BasicType elem_bt, int num_elem) {
162 assert(EnableVectorSupport, "");
163 const TypeInstPtr* vbox_type_v = gvn().type(v)->isa_instptr();
164 if (vbox_type_v == nullptr || vbox_type->instance_klass() != vbox_type_v->instance_klass()) {
165 return nullptr; // arguments don't agree on vector shapes
166 }
167 if (vbox_type_v->maybe_null()) {
168 return nullptr; // no nulls are allowed
169 }
170 assert(check_vbox(vbox_type), "");
171 const TypeVect* vt = TypeVect::make(elem_bt, num_elem, is_vector_mask(vbox_type->instance_klass()));
172 Node* unbox = gvn().transform(new VectorUnboxNode(C, vt, v, merged_memory()));
173 return unbox;
174 }
175
176 Node* GraphKit::vector_shift_count(Node* cnt, int shift_op, BasicType bt, int num_elem) {
177 assert(bt == T_INT || bt == T_LONG || bt == T_SHORT || bt == T_BYTE, "byte, short, long and int are supported");
178 juint mask = (type2aelembytes(bt) * BitsPerByte - 1);
179 Node* nmask = gvn().transform(ConNode::make(TypeInt::make(mask)));
180 Node* mcnt = gvn().transform(new AndINode(cnt, nmask));
181 return gvn().transform(VectorNode::shift_count(shift_op, mcnt, num_elem, bt));
182 }
183
184 bool LibraryCallKit::arch_supports_vector(int sopc, int num_elem, BasicType type, VectorMaskUseType mask_use_type, bool has_scalar_args) {
185 // Check that the operation is valid.
186 if (sopc <= 0) {
187 non_product_log_if_needed(" ** Rejected intrinsification because no valid vector op could be extracted");
188 return false;
189 }
190
191 if (VectorNode::is_vector_rotate(sopc)) {
192 if(!arch_supports_vector_rotate(sopc, num_elem, type, mask_use_type, has_scalar_args)) {
193 non_product_log_if_needed(" ** Rejected vector op (%s,%s,%d) because architecture does not support variable vector shifts",
194 NodeClassNames[sopc], type2name(type), num_elem);
195 return false;
196 }
197 } else if (VectorNode::is_vector_integral_negate(sopc)) {
198 if (!VectorNode::is_vector_integral_negate_supported(sopc, num_elem, type, false)) {
199 non_product_log_if_needed(" ** Rejected vector op (%s,%s,%d) because architecture does not support integral vector negate",
200 NodeClassNames[sopc], type2name(type), num_elem);
201 return false;
202 }
203 } else {
204 // Check that architecture supports this op-size-type combination.
205 if (!Matcher::match_rule_supported_vector(sopc, num_elem, type)) {
206 non_product_log_if_needed(" ** Rejected vector op (%s,%s,%d) because architecture does not support it",
207 NodeClassNames[sopc], type2name(type), num_elem);
208 return false;
209 } else {
210 assert(Matcher::match_rule_supported(sopc), "must be supported");
211 }
212 }
213
214 if (num_elem == 1) {
215 if (mask_use_type != VecMaskNotUsed) {
216 non_product_log_if_needed(" ** Rejected vector mask op (%s,%s,%d) because architecture does not support it",
217 NodeClassNames[sopc], type2name(type), num_elem);
218 return false;
219 }
220
221 if (sopc != 0) {
222 if (sopc != Op_LoadVector && sopc != Op_StoreVector) {
223 non_product_log_if_needed(" ** Not a svml call or load/store vector op (%s,%s,%d)",
224 NodeClassNames[sopc], type2name(type), num_elem);
225 return false;
226 }
227 }
228 }
229
230 if (!has_scalar_args && VectorNode::is_vector_shift(sopc) &&
231 Matcher::supports_vector_variable_shifts() == false) {
232 log_if_needed(" ** Rejected vector op (%s,%s,%d) because architecture does not support variable vector shifts",
233 NodeClassNames[sopc], type2name(type), num_elem);
234 return false;
235 }
236
237 // Check if mask unboxing is supported, this is a two step process which first loads the contents
238 // of boolean array into vector followed by either lane expansion to match the lane size of masked
239 // vector operation or populate the predicate register.
240 if ((mask_use_type & VecMaskUseLoad) != 0) {
241 if (!Matcher::match_rule_supported_vector(Op_VectorLoadMask, num_elem, type) ||
242 !Matcher::match_rule_supported_vector(Op_LoadVector, num_elem, T_BOOLEAN)) {
243 non_product_log_if_needed(" ** Rejected vector mask loading (%s,%s,%d) because architecture does not support it",
244 NodeClassNames[Op_VectorLoadMask], type2name(type), num_elem);
245 return false;
246 }
247 }
248
249 // Check if mask boxing is supported, this is a two step process which first stores the contents
250 // of mask vector / predicate register into a boolean vector followed by vector store operation to
251 // transfer the contents to underlined storage of mask boxes which is a boolean array.
252 if ((mask_use_type & VecMaskUseStore) != 0) {
253 if (!Matcher::match_rule_supported_vector(Op_VectorStoreMask, num_elem, type) ||
254 !Matcher::match_rule_supported_vector(Op_StoreVector, num_elem, T_BOOLEAN)) {
255 non_product_log_if_needed("Rejected vector mask storing (%s,%s,%d) because architecture does not support it",
256 NodeClassNames[Op_VectorStoreMask], type2name(type), num_elem);
257 return false;
258 }
259 }
260
261 if ((mask_use_type & VecMaskUsePred) != 0) {
262 bool is_supported = false;
263 if (Matcher::has_predicated_vectors()) {
264 if (VectorNode::is_vector_integral_negate(sopc)) {
265 is_supported = VectorNode::is_vector_integral_negate_supported(sopc, num_elem, type, true);
266 } else {
267 is_supported = Matcher::match_rule_supported_vector_masked(sopc, num_elem, type);
268 }
269 }
270 is_supported |= Matcher::supports_vector_predicate_op_emulation(sopc, num_elem, type);
271
272 if (!is_supported) {
273 non_product_log_if_needed("Rejected vector mask predicate using (%s,%s,%d) because architecture does not support it",
274 NodeClassNames[sopc], type2name(type), num_elem);
275 return false;
276 }
277 }
278
279 return true;
280 }
281
282 static bool is_klass_initialized(const TypeInstPtr* vec_klass) {
283 if (vec_klass->const_oop() == nullptr) {
284 return false; // uninitialized or some kind of unsafe access
285 }
286 assert(vec_klass->const_oop()->as_instance()->java_lang_Class_klass() != nullptr, "klass instance expected");
287 ciInstanceKlass* klass = vec_klass->const_oop()->as_instance()->java_lang_Class_klass()->as_instance_klass();
288 return klass->is_initialized();
289 }
290
291 // public static
292 // <V extends Vector<E>,
293 // M extends VectorMask<E>,
294 // E>
295 // V unaryOp(int oprId, Class<? extends V> vmClass, Class<? extends M> maskClass, Class<E> elementType,
296 // int length, V v, M m,
297 // UnaryOperation<V, M> defaultImpl)
298 //
299 // public static
300 // <V,
301 // M extends VectorMask<E>,
302 // E>
303 // V binaryOp(int oprId, Class<? extends V> vmClass, Class<? extends M> maskClass, Class<E> elementType,
304 // int length, V v1, V v2, M m,
305 // BinaryOperation<V, M> defaultImpl)
306 //
307 // public static
308 // <V extends Vector<E>,
309 // M extends VectorMask<E>,
310 // E>
311 // V ternaryOp(int oprId, Class<? extends V> vmClass, Class<? extends M> maskClass, Class<E> elementType,
312 // int length, V v1, V v2, V v3, M m,
313 // TernaryOperation<V, M> defaultImpl)
314 //
315 bool LibraryCallKit::inline_vector_nary_operation(int n) {
316 const TypeInt* opr = gvn().type(argument(0))->isa_int();
317 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
318 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
319 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
320 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
321
322 if (opr == nullptr || vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr ||
323 !opr->is_con() || vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
324 log_if_needed(" ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
325 NodeClassNames[argument(0)->Opcode()],
326 NodeClassNames[argument(1)->Opcode()],
327 NodeClassNames[argument(3)->Opcode()],
328 NodeClassNames[argument(4)->Opcode()]);
329 return false; // not enough info for intrinsification
330 }
331
332 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
333 if (!elem_type->is_primitive_type()) {
334 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
335 return false; // should be primitive type
336 }
337 if (!is_klass_initialized(vector_klass)) {
338 log_if_needed(" ** klass argument not initialized");
339 return false;
340 }
341
342 // "argument(n + 5)" should be the mask object. We assume it is "null" when no mask
343 // is used to control this operation.
344 const Type* vmask_type = gvn().type(argument(n + 5));
345 bool is_masked_op = vmask_type != TypePtr::NULL_PTR;
346 if (is_masked_op) {
347 if (mask_klass == nullptr || mask_klass->const_oop() == nullptr) {
348 log_if_needed(" ** missing constant: maskclass=%s", NodeClassNames[argument(2)->Opcode()]);
349 return false; // not enough info for intrinsification
350 }
351
352 if (!is_klass_initialized(mask_klass)) {
353 log_if_needed(" ** mask klass argument not initialized");
354 return false;
355 }
356
357 if (vmask_type->maybe_null()) {
358 log_if_needed(" ** null mask values are not allowed for masked op");
359 return false;
360 }
361 }
362
363 BasicType elem_bt = elem_type->basic_type();
364 bool has_scalar_op = VectorSupport::has_scalar_op(opr->get_con());
365 bool is_unsigned = VectorSupport::is_unsigned_op(opr->get_con());
366
367 int num_elem = vlen->get_con();
368 int opc = VectorSupport::vop2ideal(opr->get_con(), elem_bt);
369 int sopc = has_scalar_op ? VectorNode::opcode(opc, elem_bt) : opc;
370 if ((opc != Op_CallLeafVector) && (sopc == 0)) {
371 log_if_needed(" ** operation not supported: opc=%s bt=%s", NodeClassNames[opc], type2name(elem_bt));
372 return false; // operation not supported
373 }
374 if (num_elem == 1) {
375 if (opc != Op_CallLeafVector || elem_bt != T_DOUBLE) {
376 log_if_needed(" ** not a svml call: arity=%d opc=%d vlen=%d etype=%s",
377 n, opc, num_elem, type2name(elem_bt));
378 return false;
379 }
380 }
381 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
382 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
383
384 if (is_vector_mask(vbox_klass)) {
385 assert(!is_masked_op, "mask operations do not need mask to control");
386 }
387
388 if (opc == Op_CallLeafVector) {
389 if (!UseVectorStubs) {
390 log_if_needed(" ** vector stubs support is disabled");
391 return false;
392 }
393 if (!Matcher::supports_vector_calling_convention()) {
394 log_if_needed(" ** no vector calling conventions supported");
395 return false;
396 }
397 if (!Matcher::vector_size_supported(elem_bt, num_elem)) {
398 log_if_needed(" ** vector size (vlen=%d, etype=%s) is not supported",
399 num_elem, type2name(elem_bt));
400 return false;
401 }
402 }
403
404 // When using mask, mask use type needs to be VecMaskUseLoad.
405 VectorMaskUseType mask_use_type = is_vector_mask(vbox_klass) ? VecMaskUseAll
406 : is_masked_op ? VecMaskUseLoad : VecMaskNotUsed;
407 if ((sopc != 0) && !arch_supports_vector(sopc, num_elem, elem_bt, mask_use_type)) {
408 log_if_needed(" ** not supported: arity=%d opc=%d vlen=%d etype=%s ismask=%d is_masked_op=%d",
409 n, sopc, num_elem, type2name(elem_bt),
410 is_vector_mask(vbox_klass) ? 1 : 0, is_masked_op ? 1 : 0);
411 return false; // not supported
412 }
413
414 // Return true if current platform has implemented the masked operation with predicate feature.
415 bool use_predicate = is_masked_op && sopc != 0 && arch_supports_vector(sopc, num_elem, elem_bt, VecMaskUsePred);
416 if (is_masked_op && !use_predicate && !arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad)) {
417 log_if_needed(" ** not supported: arity=%d opc=%d vlen=%d etype=%s ismask=0 is_masked_op=1",
418 n, sopc, num_elem, type2name(elem_bt));
419 return false;
420 }
421
422 Node* opd1 = nullptr; Node* opd2 = nullptr; Node* opd3 = nullptr;
423 switch (n) {
424 case 3: {
425 opd3 = unbox_vector(argument(7), vbox_type, elem_bt, num_elem);
426 if (opd3 == nullptr) {
427 log_if_needed(" ** unbox failed v3=%s",
428 NodeClassNames[argument(7)->Opcode()]);
429 return false;
430 }
431 // fall-through
432 }
433 case 2: {
434 opd2 = unbox_vector(argument(6), vbox_type, elem_bt, num_elem);
435 if (opd2 == nullptr) {
436 log_if_needed(" ** unbox failed v2=%s",
437 NodeClassNames[argument(6)->Opcode()]);
438 return false;
439 }
440 // fall-through
441 }
442 case 1: {
443 opd1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
444 if (opd1 == nullptr) {
445 log_if_needed(" ** unbox failed v1=%s",
446 NodeClassNames[argument(5)->Opcode()]);
447 return false;
448 }
449 break;
450 }
451 default: fatal("unsupported arity: %d", n);
452 }
453
454 Node* mask = nullptr;
455 if (is_masked_op) {
456 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
457 assert(is_vector_mask(mbox_klass), "argument(2) should be a mask class");
458 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
459 mask = unbox_vector(argument(n + 5), mbox_type, elem_bt, num_elem);
460 if (mask == nullptr) {
461 log_if_needed(" ** unbox failed mask=%s",
462 NodeClassNames[argument(n + 5)->Opcode()]);
463 return false;
464 }
465 }
466
467 Node* operation = nullptr;
468 if (opc == Op_CallLeafVector) {
469 assert(UseVectorStubs, "sanity");
470 operation = gen_call_to_vector_math(opr->get_con(), elem_bt, num_elem, opd1, opd2);
471 if (operation == nullptr) {
472 log_if_needed(" ** Vector math call failed for %s_%s_%d",
473 (elem_bt == T_FLOAT) ? "float" : "double",
474 VectorSupport::mathname[opr->get_con() - VectorSupport::VECTOR_OP_MATH_START],
475 num_elem * type2aelembytes(elem_bt));
476 return false;
477 }
478 } else {
479 const TypeVect* vt = TypeVect::make(elem_bt, num_elem, is_vector_mask(vbox_klass));
480 switch (n) {
481 case 1:
482 case 2: {
483 operation = VectorNode::make(sopc, opd1, opd2, vt, is_vector_mask(vbox_klass), VectorNode::is_shift_opcode(opc), is_unsigned);
484 break;
485 }
486 case 3: {
487 operation = VectorNode::make(sopc, opd1, opd2, opd3, vt);
488 break;
489 }
490 default: fatal("unsupported arity: %d", n);
491 }
492 }
493
494 if (is_masked_op && mask != nullptr) {
495 if (use_predicate) {
496 operation->add_req(mask);
497 operation->add_flag(Node::Flag_is_predicated_vector);
498 } else {
499 operation->add_flag(Node::Flag_is_predicated_using_blend);
500 operation = gvn().transform(operation);
501 operation = new VectorBlendNode(opd1, operation, mask);
502 }
503 }
504 operation = gvn().transform(operation);
505
506 // Wrap it up in VectorBox to keep object type information.
507 Node* vbox = box_vector(operation, vbox_type, elem_bt, num_elem);
508 set_result(vbox);
509 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
510 return true;
511 }
512
513 // public static
514 // <E, M>
515 // long maskReductionCoerced(int oper, Class<? extends M> maskClass, Class<?> elemClass,
516 // int length, M m, VectorMaskOp<M> defaultImpl)
517 bool LibraryCallKit::inline_vector_mask_operation() {
518 const TypeInt* oper = gvn().type(argument(0))->isa_int();
519 const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
520 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
521 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
522 Node* mask = argument(4);
523
524 if (mask_klass == nullptr || elem_klass == nullptr || mask->is_top() || vlen == nullptr) {
525 return false; // dead code
526 }
527
528 if (!is_klass_initialized(mask_klass)) {
529 log_if_needed(" ** klass argument not initialized");
530 return false;
531 }
532
533 int num_elem = vlen->get_con();
534 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
535 BasicType elem_bt = elem_type->basic_type();
536
537 int mopc = VectorSupport::vop2ideal(oper->get_con(), elem_bt);
538 if (!arch_supports_vector(mopc, num_elem, elem_bt, VecMaskUseLoad)) {
539 log_if_needed(" ** not supported: arity=1 op=cast#%d/3 vlen2=%d etype2=%s",
540 mopc, num_elem, type2name(elem_bt));
541 return false; // not supported
542 }
543
544 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
545 const TypeInstPtr* mask_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
546 Node* mask_vec = unbox_vector(mask, mask_box_type, elem_bt, num_elem);
547 if (mask_vec == nullptr) {
548 log_if_needed(" ** unbox failed mask=%s",
549 NodeClassNames[argument(4)->Opcode()]);
550 return false;
551 }
552
553 if (mask_vec->bottom_type()->isa_vectmask() == nullptr) {
554 mask_vec = gvn().transform(VectorStoreMaskNode::make(gvn(), mask_vec, elem_bt, num_elem));
555 }
556 const Type* maskoper_ty = mopc == Op_VectorMaskToLong ? (const Type*)TypeLong::LONG : (const Type*)TypeInt::INT;
557 Node* maskoper = gvn().transform(VectorMaskOpNode::make(mask_vec, maskoper_ty, mopc));
558 if (mopc != Op_VectorMaskToLong) {
559 maskoper = ConvI2L(maskoper);
560 }
561 set_result(maskoper);
562
563 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
564 return true;
565 }
566
567 // public static
568 // <M,
569 // S extends VectorSpecies<E>,
570 // E>
571 // M fromBitsCoerced(Class<? extends M> vmClass, Class<E> elementType, int length,
572 // long bits, int mode, S s,
573 // BroadcastOperation<M, E, S> defaultImpl)
574 bool LibraryCallKit::inline_vector_frombits_coerced() {
575 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
576 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
577 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
578 const TypeLong* bits_type = gvn().type(argument(3))->isa_long();
579 // Mode argument determines the mode of operation it can take following values:-
580 // MODE_BROADCAST for vector Vector.broadcast and VectorMask.maskAll operations.
581 // MODE_BITS_COERCED_LONG_TO_MASK for VectorMask.fromLong operation.
582 const TypeInt* mode = gvn().type(argument(5))->isa_int();
583
584 if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr || mode == nullptr ||
585 bits_type == nullptr || vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr ||
586 !vlen->is_con() || !mode->is_con()) {
587 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s bitwise=%s",
588 NodeClassNames[argument(0)->Opcode()],
589 NodeClassNames[argument(1)->Opcode()],
590 NodeClassNames[argument(2)->Opcode()],
591 NodeClassNames[argument(5)->Opcode()]);
592 return false; // not enough info for intrinsification
593 }
594
595 if (!is_klass_initialized(vector_klass)) {
596 log_if_needed(" ** klass argument not initialized");
597 return false;
598 }
599 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
600 if (!elem_type->is_primitive_type()) {
601 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
602 return false; // should be primitive type
603 }
604 BasicType elem_bt = elem_type->basic_type();
605 int num_elem = vlen->get_con();
606 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
607 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
608
609 bool is_mask = is_vector_mask(vbox_klass);
610 int bcast_mode = mode->get_con();
611 VectorMaskUseType checkFlags = (VectorMaskUseType)(is_mask ? VecMaskUseAll : VecMaskNotUsed);
612 int opc = bcast_mode == VectorSupport::MODE_BITS_COERCED_LONG_TO_MASK ? Op_VectorLongToMask : Op_Replicate;
613
614 if (!arch_supports_vector(opc, num_elem, elem_bt, checkFlags, true /*has_scalar_args*/)) {
615 log_if_needed(" ** not supported: arity=0 op=broadcast vlen=%d etype=%s ismask=%d bcast_mode=%d",
616 num_elem, type2name(elem_bt),
617 is_mask ? 1 : 0,
618 bcast_mode);
619 return false; // not supported
620 }
621
622 Node* broadcast = nullptr;
623 Node* bits = argument(3);
624 Node* elem = bits;
625
626 if (opc == Op_VectorLongToMask) {
627 const TypeVect* vt = TypeVect::makemask(elem_bt, num_elem);
628 if (vt->isa_vectmask()) {
629 broadcast = gvn().transform(new VectorLongToMaskNode(elem, vt));
630 } else {
631 const TypeVect* mvt = TypeVect::make(T_BOOLEAN, num_elem);
632 broadcast = gvn().transform(new VectorLongToMaskNode(elem, mvt));
633 broadcast = gvn().transform(new VectorLoadMaskNode(broadcast, vt));
634 }
635 } else {
636 switch (elem_bt) {
637 case T_BOOLEAN: // fall-through
638 case T_BYTE: // fall-through
639 case T_SHORT: // fall-through
640 case T_CHAR: // fall-through
641 case T_INT: {
642 elem = gvn().transform(new ConvL2INode(bits));
643 break;
644 }
645 case T_DOUBLE: {
646 elem = gvn().transform(new MoveL2DNode(bits));
647 break;
648 }
649 case T_FLOAT: {
650 bits = gvn().transform(new ConvL2INode(bits));
651 elem = gvn().transform(new MoveI2FNode(bits));
652 break;
653 }
654 case T_LONG: {
655 // no conversion needed
656 break;
657 }
658 default: fatal("%s", type2name(elem_bt));
659 }
660 broadcast = VectorNode::scalar2vector(elem, num_elem, elem_bt, is_mask);
661 broadcast = gvn().transform(broadcast);
662 }
663
664 Node* box = box_vector(broadcast, vbox_type, elem_bt, num_elem);
665 set_result(box);
666 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
667 return true;
668 }
669
670 static bool elem_consistent_with_arr(BasicType elem_bt, const TypeAryPtr* arr_type, bool mismatched_ms) {
671 assert(arr_type != nullptr, "unexpected");
672 BasicType arr_elem_bt = arr_type->elem()->array_element_basic_type();
673 if (elem_bt == arr_elem_bt) {
674 return true;
675 } else if (elem_bt == T_SHORT && arr_elem_bt == T_CHAR) {
676 // Load/store of short vector from/to char[] is supported
677 return true;
678 } else if (elem_bt == T_BYTE && arr_elem_bt == T_BOOLEAN) {
679 // Load/store of byte vector from/to boolean[] is supported
680 return true;
681 } else {
682 return mismatched_ms;
683 }
684 }
685
686 // public static
687 // <C,
688 // VM extends VectorPayload,
689 // E,
690 // S extends VectorSpecies<E>>
691 // VM load(Class<? extends VM> vmClass, Class<E> eClass,
692 // int length,
693 // Object base, long offset, // Unsafe addressing
694 // boolean fromSegment,
695 // C container, long index, S s, // Arguments for default implementation
696 // LoadOperation<C, VM, S> defaultImpl) {
697 // public static
698 // <C,
699 // V extends VectorPayload>
700 // void store(Class<?> vClass, Class<?> eClass,
701 // int length,
702 // Object base, long offset, // Unsafe addressing
703 // boolean fromSegment,
704 // V v, C container, long index, // Arguments for default implementation
705 // StoreVectorOperation<C, V> defaultImpl) {
706 bool LibraryCallKit::inline_vector_mem_operation(bool is_store) {
707 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
708 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
709 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
710 const TypeInt* from_ms = gvn().type(argument(6))->isa_int();
711
712 if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr || !from_ms->is_con() ||
713 vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
714 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s from_ms=%s",
715 NodeClassNames[argument(0)->Opcode()],
716 NodeClassNames[argument(1)->Opcode()],
717 NodeClassNames[argument(2)->Opcode()],
718 NodeClassNames[argument(6)->Opcode()]);
719 return false; // not enough info for intrinsification
720 }
721 if (!is_klass_initialized(vector_klass)) {
722 log_if_needed(" ** klass argument not initialized");
723 return false;
724 }
725
726 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
727 if (!elem_type->is_primitive_type()) {
728 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
729 return false; // should be primitive type
730 }
731 BasicType elem_bt = elem_type->basic_type();
732 int num_elem = vlen->get_con();
733
734 // TODO When mask usage is supported, VecMaskNotUsed needs to be VecMaskUseLoad.
735 if (!arch_supports_vector(is_store ? Op_StoreVector : Op_LoadVector, num_elem, elem_bt, VecMaskNotUsed)) {
736 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d etype=%s ismask=no",
737 is_store, is_store ? "store" : "load",
738 num_elem, type2name(elem_bt));
739 return false; // not supported
740 }
741
742 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
743 bool is_mask = is_vector_mask(vbox_klass);
744
745 Node* base = argument(3);
746 Node* offset = ConvL2X(argument(4));
747
748 // Save state and restore on bailout
749 uint old_sp = sp();
750 SafePointNode* old_map = clone_map();
751
752 Node* addr = make_unsafe_address(base, offset, (is_mask ? T_BOOLEAN : elem_bt), true);
753
754 // The memory barrier checks are based on ones for unsafe access.
755 // This is not 1-1 implementation.
756 const Type *const base_type = gvn().type(base);
757
758 const TypePtr *addr_type = gvn().type(addr)->isa_ptr();
759 const TypeAryPtr* arr_type = addr_type->isa_aryptr();
760
761 const bool in_native = TypePtr::NULL_PTR == base_type; // base always null
762 const bool in_heap = !TypePtr::NULL_PTR->higher_equal(base_type); // base never null
763
764 const bool is_mixed_access = !in_heap && !in_native;
765
766 const bool is_mismatched_access = in_heap && (addr_type->isa_aryptr() == nullptr);
767
768 const bool needs_cpu_membar = is_mixed_access || is_mismatched_access;
769
770 // For non-masked mismatched memory segment vector read/write accesses, intrinsification can continue
771 // with unknown backing storage type and compiler can skip inserting explicit reinterpretation IR after
772 // loading from or before storing to backing storage which is mandatory for semantic correctness of
773 // big-endian memory layout.
774 bool mismatched_ms = LITTLE_ENDIAN_ONLY(false)
775 BIG_ENDIAN_ONLY(from_ms->get_con() && !is_mask && arr_type != nullptr &&
776 arr_type->elem()->array_element_basic_type() != elem_bt);
777 BasicType mem_elem_bt = mismatched_ms ? arr_type->elem()->array_element_basic_type() : elem_bt;
778 if (!is_java_primitive(mem_elem_bt)) {
779 log_if_needed(" ** non-primitive array element type");
780 return false;
781 }
782 int mem_num_elem = mismatched_ms ? (num_elem * type2aelembytes(elem_bt)) / type2aelembytes(mem_elem_bt) : num_elem;
783 if (arr_type != nullptr && !is_mask && !elem_consistent_with_arr(elem_bt, arr_type, mismatched_ms)) {
784 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d etype=%s atype=%s ismask=no",
785 is_store, is_store ? "store" : "load",
786 num_elem, type2name(elem_bt), type2name(arr_type->elem()->array_element_basic_type()));
787 set_map(old_map);
788 set_sp(old_sp);
789 return false;
790 }
791
792 // In case of mismatched memory segment accesses, we need to double check that the source type memory operations are supported by backend.
793 if (mismatched_ms) {
794 if (is_store) {
795 if (!arch_supports_vector(Op_StoreVector, num_elem, elem_bt, VecMaskNotUsed)
796 || !arch_supports_vector(Op_VectorReinterpret, mem_num_elem, mem_elem_bt, VecMaskNotUsed)) {
797 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d*8 etype=%s/8 ismask=no",
798 is_store, "store",
799 num_elem, type2name(elem_bt));
800 set_map(old_map);
801 set_sp(old_sp);
802 return false; // not supported
803 }
804 } else {
805 if (!arch_supports_vector(Op_LoadVector, mem_num_elem, mem_elem_bt, VecMaskNotUsed)
806 || !arch_supports_vector(Op_VectorReinterpret, num_elem, elem_bt, VecMaskNotUsed)) {
807 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d*8 etype=%s/8 ismask=no",
808 is_store, "load",
809 mem_num_elem, type2name(mem_elem_bt));
810 set_map(old_map);
811 set_sp(old_sp);
812 return false; // not supported
813 }
814 }
815 }
816 if (is_mask) {
817 if (!is_store) {
818 if (!arch_supports_vector(Op_LoadVector, num_elem, elem_bt, VecMaskUseLoad)) {
819 set_map(old_map);
820 set_sp(old_sp);
821 return false; // not supported
822 }
823 } else {
824 if (!arch_supports_vector(Op_StoreVector, num_elem, elem_bt, VecMaskUseStore)) {
825 set_map(old_map);
826 set_sp(old_sp);
827 return false; // not supported
828 }
829 }
830 }
831
832 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
833
834 if (needs_cpu_membar) {
835 insert_mem_bar(Op_MemBarCPUOrder);
836 }
837
838 if (is_store) {
839 Node* val = unbox_vector(argument(7), vbox_type, elem_bt, num_elem);
840 if (val == nullptr) {
841 set_map(old_map);
842 set_sp(old_sp);
843 return false; // operand unboxing failed
844 }
845 set_all_memory(reset_memory());
846
847 // In case the store needs to happen to byte array, reinterpret the incoming vector to byte vector.
848 int store_num_elem = num_elem;
849 if (mismatched_ms) {
850 store_num_elem = mem_num_elem;
851 const TypeVect* to_vect_type = TypeVect::make(mem_elem_bt, store_num_elem);
852 val = gvn().transform(new VectorReinterpretNode(val, val->bottom_type()->is_vect(), to_vect_type));
853 }
854 if (is_mask) {
855 val = gvn().transform(VectorStoreMaskNode::make(gvn(), val, elem_bt, num_elem));
856 }
857 Node* vstore = gvn().transform(StoreVectorNode::make(0, control(), memory(addr), addr, addr_type, val, store_num_elem));
858 set_memory(vstore, addr_type);
859 } else {
860 // When using byte array, we need to load as byte then reinterpret the value. Otherwise, do a simple vector load.
861 Node* vload = nullptr;
862 if (mismatched_ms) {
863 vload = gvn().transform(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, mem_num_elem, mem_elem_bt));
864 const TypeVect* to_vect_type = TypeVect::make(elem_bt, num_elem);
865 vload = gvn().transform(new VectorReinterpretNode(vload, vload->bottom_type()->is_vect(), to_vect_type));
866 } else {
867 // Special handle for masks
868 if (is_mask) {
869 vload = gvn().transform(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, num_elem, T_BOOLEAN));
870 vload = gvn().transform(new VectorLoadMaskNode(vload, TypeVect::makemask(elem_bt, num_elem)));
871 } else {
872 vload = gvn().transform(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, num_elem, elem_bt));
873 }
874 }
875 Node* box = box_vector(vload, vbox_type, elem_bt, num_elem);
876 set_result(box);
877 }
878
879 destruct_map_clone(old_map);
880
881 if (needs_cpu_membar) {
882 insert_mem_bar(Op_MemBarCPUOrder);
883 }
884
885 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
886 return true;
887 }
888
889 // public static
890 // <C,
891 // V extends Vector<?>,
892 // E,
893 // S extends VectorSpecies<E>,
894 // M extends VectorMask<E>>
895 // V loadMasked(Class<? extends V> vClass, Class<M> mClass, Class<E> eClass,
896 // int length, Object base, long offset, // Unsafe addressing
897 // boolean fromSegment,
898 // M m, int offsetInRange,
899 // C container, long index, S s, // Arguments for default implementation
900 // LoadVectorMaskedOperation<C, V, S, M> defaultImpl) {
901 // public static
902 // <C,
903 // V extends Vector<E>,
904 // M extends VectorMask<E>,
905 // E>
906 // void storeMasked(Class<? extends V> vClass, Class<M> mClass, Class<E> eClass,
907 // int length,
908 // Object base, long offset, // Unsafe addressing
909 // boolean fromSegment,
910 // V v, M m, C container, long index, // Arguments for default implementation
911 // StoreVectorMaskedOperation<C, V, M> defaultImpl) {
912
913 bool LibraryCallKit::inline_vector_mem_masked_operation(bool is_store) {
914 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
915 const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
916 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
917 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
918 const TypeInt* from_ms = gvn().type(argument(7))->isa_int();
919
920 if (vector_klass == nullptr || mask_klass == nullptr || elem_klass == nullptr || vlen == nullptr ||
921 vector_klass->const_oop() == nullptr || mask_klass->const_oop() == nullptr || from_ms == nullptr ||
922 elem_klass->const_oop() == nullptr || !vlen->is_con() || !from_ms->is_con()) {
923 log_if_needed(" ** missing constant: vclass=%s mclass=%s etype=%s vlen=%s from_ms=%s",
924 NodeClassNames[argument(0)->Opcode()],
925 NodeClassNames[argument(1)->Opcode()],
926 NodeClassNames[argument(2)->Opcode()],
927 NodeClassNames[argument(3)->Opcode()],
928 NodeClassNames[argument(7)->Opcode()]);
929 return false; // not enough info for intrinsification
930 }
931 if (!is_klass_initialized(vector_klass)) {
932 log_if_needed(" ** klass argument not initialized");
933 return false;
934 }
935
936 if (!is_klass_initialized(mask_klass)) {
937 log_if_needed(" ** mask klass argument not initialized");
938 return false;
939 }
940
941 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
942 if (!elem_type->is_primitive_type()) {
943 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
944 return false; // should be primitive type
945 }
946
947 BasicType elem_bt = elem_type->basic_type();
948 int num_elem = vlen->get_con();
949
950 Node* base = argument(4);
951 Node* offset = ConvL2X(argument(5));
952
953 // Save state and restore on bailout
954 uint old_sp = sp();
955 SafePointNode* old_map = clone_map();
956
957 Node* addr = make_unsafe_address(base, offset, elem_bt, true);
958 const TypePtr *addr_type = gvn().type(addr)->isa_ptr();
959 const TypeAryPtr* arr_type = addr_type->isa_aryptr();
960
961 bool mismatched_ms = from_ms->get_con() && arr_type != nullptr && arr_type->elem()->array_element_basic_type() != elem_bt;
962 BIG_ENDIAN_ONLY(if (mismatched_ms) return false;)
963 // If there is no consistency between array and vector element types, it must be special byte array case
964 if (arr_type != nullptr && !elem_consistent_with_arr(elem_bt, arr_type, mismatched_ms)) {
965 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d etype=%s atype=%s",
966 is_store, is_store ? "storeMasked" : "loadMasked",
967 num_elem, type2name(elem_bt), type2name(arr_type->elem()->array_element_basic_type()));
968 set_map(old_map);
969 set_sp(old_sp);
970 return false;
971 }
972
973 int mem_num_elem = mismatched_ms ? num_elem * type2aelembytes(elem_bt) : num_elem;
974 BasicType mem_elem_bt = mismatched_ms ? T_BYTE : elem_bt;
975 bool supports_predicate = arch_supports_vector(is_store ? Op_StoreVectorMasked : Op_LoadVectorMasked,
976 mem_num_elem, mem_elem_bt, VecMaskUseLoad);
977
978 // If current arch does not support the predicated operations, we have to bail
979 // out when current case uses the predicate feature.
980 if (!supports_predicate) {
981 bool needs_predicate = false;
982 if (is_store) {
983 // Masked vector store always uses the predicated store.
984 needs_predicate = true;
985 } else {
986 // Masked vector load with IOOBE always uses the predicated load.
987 const TypeInt* offset_in_range = gvn().type(argument(9))->isa_int();
988 if (!offset_in_range->is_con()) {
989 log_if_needed(" ** missing constant: offsetInRange=%s",
990 NodeClassNames[argument(8)->Opcode()]);
991 set_map(old_map);
992 set_sp(old_sp);
993 return false;
994 }
995 needs_predicate = (offset_in_range->get_con() == 0);
996 }
997
998 if (needs_predicate) {
999 log_if_needed(" ** not supported: op=%s vlen=%d etype=%s mismatched_ms=%d",
1000 is_store ? "storeMasked" : "loadMasked",
1001 num_elem, type2name(elem_bt), mismatched_ms ? 1 : 0);
1002 set_map(old_map);
1003 set_sp(old_sp);
1004 return false;
1005 }
1006 }
1007
1008 // This only happens for masked vector load. If predicate is not supported, then check whether
1009 // the normal vector load and blend operations are supported by backend.
1010 if (!supports_predicate && (!arch_supports_vector(Op_LoadVector, mem_num_elem, mem_elem_bt, VecMaskNotUsed) ||
1011 !arch_supports_vector(Op_VectorBlend, mem_num_elem, mem_elem_bt, VecMaskUseLoad))) {
1012 log_if_needed(" ** not supported: op=loadMasked vlen=%d etype=%s mismatched_ms=%d",
1013 num_elem, type2name(elem_bt), mismatched_ms ? 1 : 0);
1014 set_map(old_map);
1015 set_sp(old_sp);
1016 return false;
1017 }
1018
1019 // Since we are using byte array, we need to double check that the vector reinterpret operation
1020 // with byte type is supported by backend.
1021 if (mismatched_ms) {
1022 if (!arch_supports_vector(Op_VectorReinterpret, mem_num_elem, T_BYTE, VecMaskNotUsed)) {
1023 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d etype=%s mismatched_ms=1",
1024 is_store, is_store ? "storeMasked" : "loadMasked",
1025 num_elem, type2name(elem_bt));
1026 set_map(old_map);
1027 set_sp(old_sp);
1028 return false;
1029 }
1030 }
1031
1032 // Since it needs to unbox the mask, we need to double check that the related load operations
1033 // for mask are supported by backend.
1034 if (!arch_supports_vector(Op_LoadVector, num_elem, elem_bt, VecMaskUseLoad)) {
1035 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d etype=%s",
1036 is_store, is_store ? "storeMasked" : "loadMasked",
1037 num_elem, type2name(elem_bt));
1038 set_map(old_map);
1039 set_sp(old_sp);
1040 return false;
1041 }
1042
1043 // Can base be null? Otherwise, always on-heap access.
1044 bool can_access_non_heap = TypePtr::NULL_PTR->higher_equal(gvn().type(base));
1045 if (can_access_non_heap) {
1046 insert_mem_bar(Op_MemBarCPUOrder);
1047 }
1048
1049 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1050 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1051 assert(!is_vector_mask(vbox_klass) && is_vector_mask(mbox_klass), "Invalid class type");
1052 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1053 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1054
1055 Node* mask = unbox_vector(is_store ? argument(9) : argument(8), mbox_type, elem_bt, num_elem);
1056 if (mask == nullptr) {
1057 log_if_needed(" ** unbox failed mask=%s",
1058 is_store ? NodeClassNames[argument(9)->Opcode()]
1059 : NodeClassNames[argument(8)->Opcode()]);
1060 set_map(old_map);
1061 set_sp(old_sp);
1062 return false;
1063 }
1064
1065 if (is_store) {
1066 Node* val = unbox_vector(argument(8), vbox_type, elem_bt, num_elem);
1067 if (val == nullptr) {
1068 log_if_needed(" ** unbox failed vector=%s",
1069 NodeClassNames[argument(8)->Opcode()]);
1070 set_map(old_map);
1071 set_sp(old_sp);
1072 return false; // operand unboxing failed
1073 }
1074 set_all_memory(reset_memory());
1075
1076 if (mismatched_ms) {
1077 // Reinterpret the incoming vector to byte vector.
1078 const TypeVect* to_vect_type = TypeVect::make(mem_elem_bt, mem_num_elem);
1079 val = gvn().transform(new VectorReinterpretNode(val, val->bottom_type()->is_vect(), to_vect_type));
1080 // Reinterpret the vector mask to byte type.
1081 const TypeVect* from_mask_type = TypeVect::makemask(elem_bt, num_elem);
1082 const TypeVect* to_mask_type = TypeVect::makemask(mem_elem_bt, mem_num_elem);
1083 mask = gvn().transform(new VectorReinterpretNode(mask, from_mask_type, to_mask_type));
1084 }
1085 Node* vstore = gvn().transform(new StoreVectorMaskedNode(control(), memory(addr), addr, val, addr_type, mask));
1086 set_memory(vstore, addr_type);
1087 } else {
1088 Node* vload = nullptr;
1089
1090 if (mismatched_ms) {
1091 // Reinterpret the vector mask to byte type.
1092 const TypeVect* from_mask_type = TypeVect::makemask(elem_bt, num_elem);
1093 const TypeVect* to_mask_type = TypeVect::makemask(mem_elem_bt, mem_num_elem);
1094 mask = gvn().transform(new VectorReinterpretNode(mask, from_mask_type, to_mask_type));
1095 }
1096
1097 if (supports_predicate) {
1098 // Generate masked load vector node if predicate feature is supported.
1099 const TypeVect* vt = TypeVect::make(mem_elem_bt, mem_num_elem);
1100 vload = gvn().transform(new LoadVectorMaskedNode(control(), memory(addr), addr, addr_type, vt, mask));
1101 } else {
1102 // Use the vector blend to implement the masked load vector. The biased elements are zeros.
1103 Node* zero = gvn().transform(gvn().zerocon(mem_elem_bt));
1104 zero = gvn().transform(VectorNode::scalar2vector(zero, mem_num_elem, mem_elem_bt));
1105 vload = gvn().transform(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, mem_num_elem, mem_elem_bt));
1106 vload = gvn().transform(new VectorBlendNode(zero, vload, mask));
1107 }
1108
1109 if (mismatched_ms) {
1110 const TypeVect* to_vect_type = TypeVect::make(elem_bt, num_elem);
1111 vload = gvn().transform(new VectorReinterpretNode(vload, vload->bottom_type()->is_vect(), to_vect_type));
1112 }
1113
1114 Node* box = box_vector(vload, vbox_type, elem_bt, num_elem);
1115 set_result(box);
1116 }
1117
1118 destruct_map_clone(old_map);
1119
1120 if (can_access_non_heap) {
1121 insert_mem_bar(Op_MemBarCPUOrder);
1122 }
1123
1124 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1125 return true;
1126 }
1127
1128 // <C,
1129 // V extends Vector<?>,
1130 // W extends Vector<Integer>,
1131 // S extends VectorSpecies<E>,
1132 // M extends VectorMask<E>,
1133 // E>
1134 // V loadWithMap(Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType, int length,
1135 // Class<? extends Vector<Integer>> vectorIndexClass,
1136 // Object base, long offset, // Unsafe addressing
1137 // W index_vector, M m,
1138 // C container, int index, int[] indexMap, int indexM, S s, // Arguments for default implementation
1139 // LoadVectorOperationWithMap<C, V, E, S, M> defaultImpl)
1140 //
1141 // <C,
1142 // V extends Vector<E>,
1143 // W extends Vector<Integer>,
1144 // M extends VectorMask<E>,
1145 // E>
1146 // void storeWithMap(Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType,
1147 // int length, Class<? extends Vector<Integer>> vectorIndexClass, Object base, long offset, // Unsafe addressing
1148 // W index_vector, V v, M m,
1149 // C container, int index, int[] indexMap, int indexM, // Arguments for default implementation
1150 // StoreVectorOperationWithMap<C, V, M, E> defaultImpl)
1151 //
1152 bool LibraryCallKit::inline_vector_gather_scatter(bool is_scatter) {
1153 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
1154 const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
1155 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
1156 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
1157 const TypeInstPtr* vector_idx_klass = gvn().type(argument(4))->isa_instptr();
1158
1159 if (vector_klass == nullptr || elem_klass == nullptr || vector_idx_klass == nullptr || vlen == nullptr ||
1160 vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || vector_idx_klass->const_oop() == nullptr || !vlen->is_con()) {
1161 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s viclass=%s",
1162 NodeClassNames[argument(0)->Opcode()],
1163 NodeClassNames[argument(2)->Opcode()],
1164 NodeClassNames[argument(3)->Opcode()],
1165 NodeClassNames[argument(4)->Opcode()]);
1166 return false; // not enough info for intrinsification
1167 }
1168
1169 if (!is_klass_initialized(vector_klass) || !is_klass_initialized(vector_idx_klass)) {
1170 log_if_needed(" ** klass argument not initialized");
1171 return false;
1172 }
1173
1174 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1175 if (!elem_type->is_primitive_type()) {
1176 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1177 return false; // should be primitive type
1178 }
1179
1180 BasicType elem_bt = elem_type->basic_type();
1181 int num_elem = vlen->get_con();
1182
1183 const Type* vmask_type = gvn().type(is_scatter ? argument(10) : argument(9));
1184 bool is_masked_op = vmask_type != TypePtr::NULL_PTR;
1185 if (is_masked_op) {
1186 if (mask_klass == nullptr || mask_klass->const_oop() == nullptr) {
1187 log_if_needed(" ** missing constant: maskclass=%s", NodeClassNames[argument(1)->Opcode()]);
1188 return false; // not enough info for intrinsification
1189 }
1190
1191 if (!is_klass_initialized(mask_klass)) {
1192 log_if_needed(" ** mask klass argument not initialized");
1193 return false;
1194 }
1195
1196 if (vmask_type->maybe_null()) {
1197 log_if_needed(" ** null mask values are not allowed for masked op");
1198 return false;
1199 }
1200
1201 // Check whether the predicated gather/scatter node is supported by architecture.
1202 VectorMaskUseType mask = (VectorMaskUseType) (VecMaskUseLoad | VecMaskUsePred);
1203 if (!arch_supports_vector(is_scatter ? Op_StoreVectorScatterMasked : Op_LoadVectorGatherMasked, num_elem, elem_bt, mask)) {
1204 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d etype=%s is_masked_op=1",
1205 is_scatter, is_scatter ? "scatterMasked" : "gatherMasked",
1206 num_elem, type2name(elem_bt));
1207 return false; // not supported
1208 }
1209 } else {
1210 // Check whether the normal gather/scatter node is supported for non-masked operation.
1211 if (!arch_supports_vector(is_scatter ? Op_StoreVectorScatter : Op_LoadVectorGather, num_elem, elem_bt, VecMaskNotUsed)) {
1212 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d etype=%s is_masked_op=0",
1213 is_scatter, is_scatter ? "scatter" : "gather",
1214 num_elem, type2name(elem_bt));
1215 return false; // not supported
1216 }
1217 }
1218
1219 // Check that the vector holding indices is supported by architecture
1220 // For sub-word gathers expander receive index array.
1221 if (!is_subword_type(elem_bt) && !arch_supports_vector(Op_LoadVector, num_elem, T_INT, VecMaskNotUsed)) {
1222 log_if_needed(" ** not supported: arity=%d op=%s/loadindex vlen=%d etype=int is_masked_op=%d",
1223 is_scatter, is_scatter ? "scatter" : "gather",
1224 num_elem, is_masked_op ? 1 : 0);
1225 return false; // not supported
1226 }
1227
1228 Node* base = argument(5);
1229 Node* offset = ConvL2X(argument(6));
1230
1231 // Save state and restore on bailout
1232 uint old_sp = sp();
1233 SafePointNode* old_map = clone_map();
1234
1235 Node* addr = make_unsafe_address(base, offset, elem_bt, true);
1236
1237 const TypePtr *addr_type = gvn().type(addr)->isa_ptr();
1238 const TypeAryPtr* arr_type = addr_type->isa_aryptr();
1239
1240 // The array must be consistent with vector type
1241 if (arr_type == nullptr || (arr_type != nullptr && !elem_consistent_with_arr(elem_bt, arr_type, false))) {
1242 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d etype=%s atype=%s ismask=no",
1243 is_scatter, is_scatter ? "scatter" : "gather",
1244 num_elem, type2name(elem_bt), type2name(arr_type->elem()->array_element_basic_type()));
1245 set_map(old_map);
1246 set_sp(old_sp);
1247 return false;
1248 }
1249
1250 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1251 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1252 ciKlass* vbox_idx_klass = vector_idx_klass->const_oop()->as_instance()->java_lang_Class_klass();
1253 if (vbox_idx_klass == nullptr) {
1254 set_map(old_map);
1255 set_sp(old_sp);
1256 return false;
1257 }
1258
1259 Node* index_vect = nullptr;
1260 const TypeInstPtr* vbox_idx_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_idx_klass);
1261 if (!is_subword_type(elem_bt)) {
1262 index_vect = unbox_vector(argument(8), vbox_idx_type, T_INT, num_elem);
1263 if (index_vect == nullptr) {
1264 set_map(old_map);
1265 set_sp(old_sp);
1266 return false;
1267 }
1268 }
1269
1270 Node* mask = nullptr;
1271 if (is_masked_op) {
1272 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1273 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1274 mask = unbox_vector(is_scatter ? argument(10) : argument(9), mbox_type, elem_bt, num_elem);
1275 if (mask == nullptr) {
1276 log_if_needed(" ** unbox failed mask=%s",
1277 is_scatter ? NodeClassNames[argument(10)->Opcode()]
1278 : NodeClassNames[argument(9)->Opcode()]);
1279 set_map(old_map);
1280 set_sp(old_sp);
1281 return false;
1282 }
1283 }
1284
1285 const TypeVect* vector_type = TypeVect::make(elem_bt, num_elem);
1286 if (is_scatter) {
1287 Node* val = unbox_vector(argument(9), vbox_type, elem_bt, num_elem);
1288 if (val == nullptr) {
1289 set_map(old_map);
1290 set_sp(old_sp);
1291 return false; // operand unboxing failed
1292 }
1293 set_all_memory(reset_memory());
1294
1295 Node* vstore = nullptr;
1296 if (mask != nullptr) {
1297 vstore = gvn().transform(new StoreVectorScatterMaskedNode(control(), memory(addr), addr, addr_type, val, index_vect, mask));
1298 } else {
1299 vstore = gvn().transform(new StoreVectorScatterNode(control(), memory(addr), addr, addr_type, val, index_vect));
1300 }
1301 set_memory(vstore, addr_type);
1302 } else {
1303 Node* vload = nullptr;
1304 Node* index = argument(11);
1305 Node* indexMap = argument(12);
1306 Node* indexM = argument(13);
1307 if (mask != nullptr) {
1308 if (is_subword_type(elem_bt)) {
1309 Node* index_arr_base = array_element_address(indexMap, indexM, T_INT);
1310 vload = gvn().transform(new LoadVectorGatherMaskedNode(control(), memory(addr), addr, addr_type, vector_type, index_arr_base, mask, index));
1311 } else {
1312 vload = gvn().transform(new LoadVectorGatherMaskedNode(control(), memory(addr), addr, addr_type, vector_type, index_vect, mask));
1313 }
1314 } else {
1315 if (is_subword_type(elem_bt)) {
1316 Node* index_arr_base = array_element_address(indexMap, indexM, T_INT);
1317 vload = gvn().transform(new LoadVectorGatherNode(control(), memory(addr), addr, addr_type, vector_type, index_arr_base, index));
1318 } else {
1319 vload = gvn().transform(new LoadVectorGatherNode(control(), memory(addr), addr, addr_type, vector_type, index_vect));
1320 }
1321 }
1322 Node* box = box_vector(vload, vbox_type, elem_bt, num_elem);
1323 set_result(box);
1324 }
1325
1326 destruct_map_clone(old_map);
1327
1328 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1329 return true;
1330 }
1331
1332 // public static
1333 // <V extends Vector<E>,
1334 // M extends VectorMask<E>,
1335 // E>
1336 // long reductionCoerced(int oprId, Class<? extends V> vectorClass, Class<? extends M> maskClass,
1337 // Class<E> elementType, int length, V v, M m,
1338 // ReductionOperation<V, M> defaultImpl)
1339 bool LibraryCallKit::inline_vector_reduction() {
1340 const TypeInt* opr = gvn().type(argument(0))->isa_int();
1341 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
1342 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
1343 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
1344 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
1345
1346 if (opr == nullptr || vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr ||
1347 !opr->is_con() || vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
1348 log_if_needed(" ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
1349 NodeClassNames[argument(0)->Opcode()],
1350 NodeClassNames[argument(1)->Opcode()],
1351 NodeClassNames[argument(3)->Opcode()],
1352 NodeClassNames[argument(4)->Opcode()]);
1353 return false; // not enough info for intrinsification
1354 }
1355 if (!is_klass_initialized(vector_klass)) {
1356 log_if_needed(" ** klass argument not initialized");
1357 return false;
1358 }
1359 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1360 if (!elem_type->is_primitive_type()) {
1361 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1362 return false; // should be primitive type
1363 }
1364
1365 const Type* vmask_type = gvn().type(argument(6));
1366 bool is_masked_op = vmask_type != TypePtr::NULL_PTR;
1367 if (is_masked_op) {
1368 if (mask_klass == nullptr || mask_klass->const_oop() == nullptr) {
1369 log_if_needed(" ** missing constant: maskclass=%s", NodeClassNames[argument(2)->Opcode()]);
1370 return false; // not enough info for intrinsification
1371 }
1372
1373 if (!is_klass_initialized(mask_klass)) {
1374 log_if_needed(" ** mask klass argument not initialized");
1375 return false;
1376 }
1377
1378 if (vmask_type->maybe_null()) {
1379 log_if_needed(" ** null mask values are not allowed for masked op");
1380 return false;
1381 }
1382 }
1383
1384 BasicType elem_bt = elem_type->basic_type();
1385 int num_elem = vlen->get_con();
1386 int opc = VectorSupport::vop2ideal(opr->get_con(), elem_bt);
1387 int sopc = ReductionNode::opcode(opc, elem_bt);
1388
1389 // Ensure reduction operation for lanewise operation
1390 // When using mask, mask use type needs to be VecMaskUseLoad.
1391 if (sopc == opc || !arch_supports_vector(sopc, num_elem, elem_bt, is_masked_op ? VecMaskUseLoad : VecMaskNotUsed)) {
1392 log_if_needed(" ** not supported: arity=1 op=%d/reduce vlen=%d etype=%s is_masked_op=%d",
1393 sopc, num_elem, type2name(elem_bt), is_masked_op ? 1 : 0);
1394 return false;
1395 }
1396
1397 // Return true if current platform has implemented the masked operation with predicate feature.
1398 bool use_predicate = is_masked_op && arch_supports_vector(sopc, num_elem, elem_bt, VecMaskUsePred);
1399 if (is_masked_op && !use_predicate && !arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad)) {
1400 log_if_needed(" ** not supported: arity=1 op=%d/reduce vlen=%d etype=%s is_masked_op=1",
1401 sopc, num_elem, type2name(elem_bt));
1402 return false;
1403 }
1404
1405 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1406 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1407
1408 Node* opd = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
1409 if (opd == nullptr) {
1410 return false; // operand unboxing failed
1411 }
1412
1413 Node* mask = nullptr;
1414 if (is_masked_op) {
1415 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1416 assert(is_vector_mask(mbox_klass), "argument(2) should be a mask class");
1417 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1418 mask = unbox_vector(argument(6), mbox_type, elem_bt, num_elem);
1419 if (mask == nullptr) {
1420 log_if_needed(" ** unbox failed mask=%s",
1421 NodeClassNames[argument(6)->Opcode()]);
1422 return false;
1423 }
1424 }
1425
1426 Node* init = ReductionNode::make_identity_con_scalar(gvn(), opc, elem_bt);
1427 Node* value = opd;
1428
1429 assert(mask != nullptr || !is_masked_op, "Masked op needs the mask value never null");
1430 if (mask != nullptr && !use_predicate) {
1431 Node* reduce_identity = gvn().transform(VectorNode::scalar2vector(init, num_elem, elem_bt));
1432 value = gvn().transform(new VectorBlendNode(reduce_identity, value, mask));
1433 }
1434
1435 // Make an unordered Reduction node. This affects only AddReductionVF/VD and MulReductionVF/VD,
1436 // as these operations are allowed to be associative (not requiring strict order) in VectorAPI.
1437 value = ReductionNode::make(opc, nullptr, init, value, elem_bt, /* requires_strict_order */ false);
1438
1439 if (mask != nullptr && use_predicate) {
1440 value->add_req(mask);
1441 value->add_flag(Node::Flag_is_predicated_vector);
1442 }
1443
1444 value = gvn().transform(value);
1445
1446 Node* bits = nullptr;
1447 switch (elem_bt) {
1448 case T_BYTE:
1449 case T_SHORT:
1450 case T_INT: {
1451 bits = gvn().transform(new ConvI2LNode(value));
1452 break;
1453 }
1454 case T_FLOAT: {
1455 value = gvn().transform(new MoveF2INode(value));
1456 bits = gvn().transform(new ConvI2LNode(value));
1457 break;
1458 }
1459 case T_DOUBLE: {
1460 bits = gvn().transform(new MoveD2LNode(value));
1461 break;
1462 }
1463 case T_LONG: {
1464 bits = value; // no conversion needed
1465 break;
1466 }
1467 default: fatal("%s", type2name(elem_bt));
1468 }
1469 set_result(bits);
1470 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1471 return true;
1472 }
1473
1474 // public static <V> boolean test(int cond, Class<?> vectorClass, Class<?> elementType, int vlen,
1475 // V v1, V v2,
1476 // BiFunction<V, V, Boolean> defaultImpl)
1477 //
1478 bool LibraryCallKit::inline_vector_test() {
1479 const TypeInt* cond = gvn().type(argument(0))->isa_int();
1480 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
1481 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
1482 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
1483
1484 if (cond == nullptr || vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr ||
1485 !cond->is_con() || vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
1486 log_if_needed(" ** missing constant: cond=%s vclass=%s etype=%s vlen=%s",
1487 NodeClassNames[argument(0)->Opcode()],
1488 NodeClassNames[argument(1)->Opcode()],
1489 NodeClassNames[argument(2)->Opcode()],
1490 NodeClassNames[argument(3)->Opcode()]);
1491 return false; // not enough info for intrinsification
1492 }
1493 if (!is_klass_initialized(vector_klass)) {
1494 log_if_needed(" ** klass argument not initialized");
1495 return false;
1496 }
1497 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1498 if (!elem_type->is_primitive_type()) {
1499 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1500 return false; // should be primitive type
1501 }
1502 BasicType elem_bt = elem_type->basic_type();
1503 int num_elem = vlen->get_con();
1504 BoolTest::mask booltest = (BoolTest::mask)cond->get_con();
1505 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1506 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1507
1508 if (!arch_supports_vector(Op_VectorTest, num_elem, elem_bt, is_vector_mask(vbox_klass) ? VecMaskUseLoad : VecMaskNotUsed)) {
1509 log_if_needed(" ** not supported: arity=2 op=test/%d vlen=%d etype=%s ismask=%d",
1510 cond->get_con(), num_elem, type2name(elem_bt),
1511 is_vector_mask(vbox_klass));
1512 return false;
1513 }
1514
1515 Node* opd1 = unbox_vector(argument(4), vbox_type, elem_bt, num_elem);
1516 Node* opd2;
1517 if (Matcher::vectortest_needs_second_argument(booltest == BoolTest::overflow,
1518 opd1->bottom_type()->isa_vectmask())) {
1519 opd2 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
1520 } else {
1521 opd2 = opd1;
1522 }
1523 if (opd1 == nullptr || opd2 == nullptr) {
1524 return false; // operand unboxing failed
1525 }
1526
1527 Node* cmp = gvn().transform(new VectorTestNode(opd1, opd2, booltest));
1528 BoolTest::mask test = Matcher::vectortest_mask(booltest == BoolTest::overflow,
1529 opd1->bottom_type()->isa_vectmask(), num_elem);
1530 Node* bol = gvn().transform(new BoolNode(cmp, test));
1531 Node* res = gvn().transform(new CMoveINode(bol, gvn().intcon(0), gvn().intcon(1), TypeInt::BOOL));
1532
1533 set_result(res);
1534 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1535 return true;
1536 }
1537
1538 // public static
1539 // <V extends Vector<E>,
1540 // M extends VectorMask<E>,
1541 // E>
1542 // V blend(Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType, int vlen,
1543 // V v1, V v2, M m,
1544 // VectorBlendOp<V, M, E> defaultImpl)
1545 bool LibraryCallKit::inline_vector_blend() {
1546 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
1547 const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
1548 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
1549 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
1550
1551 if (mask_klass == nullptr || vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr) {
1552 return false; // dead code
1553 }
1554 if (mask_klass->const_oop() == nullptr || vector_klass->const_oop() == nullptr ||
1555 elem_klass->const_oop() == nullptr || !vlen->is_con()) {
1556 log_if_needed(" ** missing constant: vclass=%s mclass=%s etype=%s vlen=%s",
1557 NodeClassNames[argument(0)->Opcode()],
1558 NodeClassNames[argument(1)->Opcode()],
1559 NodeClassNames[argument(2)->Opcode()],
1560 NodeClassNames[argument(3)->Opcode()]);
1561 return false; // not enough info for intrinsification
1562 }
1563 if (!is_klass_initialized(vector_klass) || !is_klass_initialized(mask_klass)) {
1564 log_if_needed(" ** klass argument not initialized");
1565 return false;
1566 }
1567 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1568 if (!elem_type->is_primitive_type()) {
1569 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1570 return false; // should be primitive type
1571 }
1572 BasicType elem_bt = elem_type->basic_type();
1573 BasicType mask_bt = elem_bt;
1574 int num_elem = vlen->get_con();
1575
1576 if (!arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad)) {
1577 log_if_needed(" ** not supported: arity=2 op=blend vlen=%d etype=%s ismask=useload",
1578 num_elem, type2name(elem_bt));
1579 return false; // not supported
1580 }
1581 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1582 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1583
1584 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1585 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1586
1587 Node* v1 = unbox_vector(argument(4), vbox_type, elem_bt, num_elem);
1588 Node* v2 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
1589 Node* mask = unbox_vector(argument(6), mbox_type, mask_bt, num_elem);
1590
1591 if (v1 == nullptr || v2 == nullptr || mask == nullptr) {
1592 return false; // operand unboxing failed
1593 }
1594
1595 Node* blend = gvn().transform(new VectorBlendNode(v1, v2, mask));
1596
1597 Node* box = box_vector(blend, vbox_type, elem_bt, num_elem);
1598 set_result(box);
1599 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1600 return true;
1601 }
1602
1603 // public static
1604 // <V extends Vector<E>,
1605 // M extends VectorMask<E>,
1606 // E>
1607 // M compare(int cond, Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType, int vlen,
1608 // V v1, V v2, M m,
1609 // VectorCompareOp<V,M> defaultImpl)
1610 bool LibraryCallKit::inline_vector_compare() {
1611 const TypeInt* cond = gvn().type(argument(0))->isa_int();
1612 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
1613 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
1614 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
1615 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
1616
1617 if (cond == nullptr || vector_klass == nullptr || mask_klass == nullptr || elem_klass == nullptr || vlen == nullptr) {
1618 return false; // dead code
1619 }
1620 if (!cond->is_con() || vector_klass->const_oop() == nullptr || mask_klass->const_oop() == nullptr ||
1621 elem_klass->const_oop() == nullptr || !vlen->is_con()) {
1622 log_if_needed(" ** missing constant: cond=%s vclass=%s mclass=%s etype=%s vlen=%s",
1623 NodeClassNames[argument(0)->Opcode()],
1624 NodeClassNames[argument(1)->Opcode()],
1625 NodeClassNames[argument(2)->Opcode()],
1626 NodeClassNames[argument(3)->Opcode()],
1627 NodeClassNames[argument(4)->Opcode()]);
1628 return false; // not enough info for intrinsification
1629 }
1630 if (!is_klass_initialized(vector_klass) || !is_klass_initialized(mask_klass)) {
1631 log_if_needed(" ** klass argument not initialized");
1632 return false;
1633 }
1634 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1635 if (!elem_type->is_primitive_type()) {
1636 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1637 return false; // should be primitive type
1638 }
1639
1640 int num_elem = vlen->get_con();
1641 BasicType elem_bt = elem_type->basic_type();
1642 BasicType mask_bt = elem_bt;
1643
1644 if ((cond->get_con() & BoolTest::unsigned_compare) != 0) {
1645 if (!Matcher::supports_vector_comparison_unsigned(num_elem, elem_bt)) {
1646 log_if_needed(" ** not supported: unsigned comparison op=comp/%d vlen=%d etype=%s ismask=usestore",
1647 cond->get_con() & (BoolTest::unsigned_compare - 1), num_elem, type2name(elem_bt));
1648 return false;
1649 }
1650 }
1651
1652 if (!arch_supports_vector(Op_VectorMaskCmp, num_elem, elem_bt, VecMaskUseStore)) {
1653 log_if_needed(" ** not supported: arity=2 op=comp/%d vlen=%d etype=%s ismask=usestore",
1654 cond->get_con(), num_elem, type2name(elem_bt));
1655 return false;
1656 }
1657
1658 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1659 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1660
1661 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1662 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1663
1664 Node* v1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
1665 Node* v2 = unbox_vector(argument(6), vbox_type, elem_bt, num_elem);
1666
1667 bool is_masked_op = argument(7)->bottom_type() != TypePtr::NULL_PTR;
1668 Node* mask = is_masked_op ? unbox_vector(argument(7), mbox_type, elem_bt, num_elem) : nullptr;
1669 if (is_masked_op && mask == nullptr) {
1670 log_if_needed(" ** not supported: mask = null arity=2 op=comp/%d vlen=%d etype=%s ismask=usestore is_masked_op=1",
1671 cond->get_con(), num_elem, type2name(elem_bt));
1672 return false;
1673 }
1674
1675 bool use_predicate = is_masked_op && arch_supports_vector(Op_VectorMaskCmp, num_elem, elem_bt, VecMaskUsePred);
1676 if (is_masked_op && !use_predicate && !arch_supports_vector(Op_AndV, num_elem, elem_bt, VecMaskUseLoad)) {
1677 log_if_needed(" ** not supported: arity=2 op=comp/%d vlen=%d etype=%s ismask=usestore is_masked_op=1",
1678 cond->get_con(), num_elem, type2name(elem_bt));
1679 return false;
1680 }
1681
1682 if (v1 == nullptr || v2 == nullptr) {
1683 return false; // operand unboxing failed
1684 }
1685 BoolTest::mask pred = (BoolTest::mask)cond->get_con();
1686 ConINode* pred_node = (ConINode*)gvn().makecon(cond);
1687
1688 const TypeVect* vmask_type = TypeVect::makemask(mask_bt, num_elem);
1689 Node* operation = new VectorMaskCmpNode(pred, v1, v2, pred_node, vmask_type);
1690
1691 if (is_masked_op) {
1692 if (use_predicate) {
1693 operation->add_req(mask);
1694 operation->add_flag(Node::Flag_is_predicated_vector);
1695 } else {
1696 operation = gvn().transform(operation);
1697 operation = VectorNode::make(Op_AndV, operation, mask, vmask_type);
1698 }
1699 }
1700
1701 operation = gvn().transform(operation);
1702
1703 Node* box = box_vector(operation, mbox_type, mask_bt, num_elem);
1704 set_result(box);
1705 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1706 return true;
1707 }
1708
1709 // public static
1710 // <V extends Vector<E>,
1711 // Sh extends VectorShuffle<E>,
1712 // M extends VectorMask<E>,
1713 // E>
1714 // V rearrangeOp(Class<? extends V> vectorClass, Class<Sh> shuffleClass, Class<M> maskClass, Class<E> elementType, int vlen,
1715 // V v1, Sh sh, M m,
1716 // VectorRearrangeOp<V, Sh, M, E> defaultImpl)
1717 bool LibraryCallKit::inline_vector_rearrange() {
1718 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
1719 const TypeInstPtr* shuffle_klass = gvn().type(argument(1))->isa_instptr();
1720 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
1721 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
1722 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
1723
1724 if (vector_klass == nullptr || shuffle_klass == nullptr || elem_klass == nullptr || vlen == nullptr) {
1725 return false; // dead code
1726 }
1727 if (shuffle_klass->const_oop() == nullptr ||
1728 vector_klass->const_oop() == nullptr ||
1729 elem_klass->const_oop() == nullptr ||
1730 !vlen->is_con()) {
1731 log_if_needed(" ** missing constant: vclass=%s sclass=%s etype=%s vlen=%s",
1732 NodeClassNames[argument(0)->Opcode()],
1733 NodeClassNames[argument(1)->Opcode()],
1734 NodeClassNames[argument(3)->Opcode()],
1735 NodeClassNames[argument(4)->Opcode()]);
1736 return false; // not enough info for intrinsification
1737 }
1738 if (!is_klass_initialized(vector_klass) ||
1739 !is_klass_initialized(shuffle_klass)) {
1740 log_if_needed(" ** klass argument not initialized");
1741 return false;
1742 }
1743 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1744 if (!elem_type->is_primitive_type()) {
1745 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1746 return false; // should be primitive type
1747 }
1748
1749 BasicType elem_bt = elem_type->basic_type();
1750 BasicType shuffle_bt = elem_bt;
1751 if (shuffle_bt == T_FLOAT) {
1752 shuffle_bt = T_INT;
1753 } else if (shuffle_bt == T_DOUBLE) {
1754 shuffle_bt = T_LONG;
1755 }
1756
1757 int num_elem = vlen->get_con();
1758 bool need_load_shuffle = Matcher::vector_rearrange_requires_load_shuffle(shuffle_bt, num_elem);
1759
1760 if (need_load_shuffle && !arch_supports_vector(Op_VectorLoadShuffle, num_elem, shuffle_bt, VecMaskNotUsed)) {
1761 if (C->print_intrinsics()) {
1762 tty->print_cr(" ** not supported: arity=0 op=load/shuffle vlen=%d etype=%s ismask=no",
1763 num_elem, type2name(shuffle_bt));
1764 }
1765 return false; // not supported
1766 }
1767
1768 bool is_masked_op = argument(7)->bottom_type() != TypePtr::NULL_PTR;
1769 bool use_predicate = is_masked_op;
1770 if (is_masked_op &&
1771 (mask_klass == nullptr ||
1772 mask_klass->const_oop() == nullptr ||
1773 !is_klass_initialized(mask_klass))) {
1774 log_if_needed(" ** mask_klass argument not initialized");
1775 }
1776 if (!arch_supports_vector(Op_AndV, num_elem, elem_bt, VecMaskNotUsed)) {
1777 log_if_needed(" ** not supported: arity=2 op=and vlen=%d etype=%s ismask=no",
1778 num_elem, type2name(elem_bt));
1779 return false;
1780 }
1781 VectorMaskUseType checkFlags = (VectorMaskUseType)(is_masked_op ? (VecMaskUseLoad | VecMaskUsePred) : VecMaskNotUsed);
1782 if (!arch_supports_vector(Op_VectorRearrange, num_elem, elem_bt, checkFlags)) {
1783 use_predicate = false;
1784 if(!is_masked_op ||
1785 (!arch_supports_vector(Op_VectorRearrange, num_elem, elem_bt, VecMaskNotUsed) ||
1786 !arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad) ||
1787 !arch_supports_vector(Op_Replicate, num_elem, elem_bt, VecMaskNotUsed))) {
1788 log_if_needed(" ** not supported: arity=2 op=shuffle/rearrange vlen=%d etype=%s ismask=no",
1789 num_elem, type2name(elem_bt));
1790 return false; // not supported
1791 }
1792 }
1793 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1794 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1795
1796 ciKlass* shbox_klass = shuffle_klass->const_oop()->as_instance()->java_lang_Class_klass();
1797 const TypeInstPtr* shbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, shbox_klass);
1798
1799 Node* v1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
1800 Node* shuffle = unbox_vector(argument(6), shbox_type, shuffle_bt, num_elem);
1801 const TypeVect* st = TypeVect::make(shuffle_bt, num_elem);
1802
1803 if (v1 == nullptr || shuffle == nullptr) {
1804 return false; // operand unboxing failed
1805 }
1806
1807 assert(is_power_of_2(num_elem), "wrapping invalid");
1808 Node* wrapping_mask_elem = gvn().makecon(TypeInteger::make(num_elem - 1, num_elem - 1, Type::WidenMin, shuffle_bt == T_LONG ? T_LONG : T_INT));
1809 Node* wrapping_mask = gvn().transform(VectorNode::scalar2vector(wrapping_mask_elem, num_elem, shuffle_bt));
1810 shuffle = gvn().transform(new AndVNode(shuffle, wrapping_mask, st));
1811
1812 Node* mask = nullptr;
1813 if (is_masked_op) {
1814 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1815 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1816 mask = unbox_vector(argument(7), mbox_type, elem_bt, num_elem);
1817 if (mask == nullptr) {
1818 log_if_needed(" ** not supported: arity=3 op=shuffle/rearrange vlen=%d etype=%s ismask=useload is_masked_op=1",
1819 num_elem, type2name(elem_bt));
1820 return false;
1821 }
1822 }
1823
1824 if (need_load_shuffle) {
1825 shuffle = gvn().transform(new VectorLoadShuffleNode(shuffle, st));
1826 }
1827
1828 Node* rearrange = new VectorRearrangeNode(v1, shuffle);
1829 if (is_masked_op) {
1830 if (use_predicate) {
1831 rearrange->add_req(mask);
1832 rearrange->add_flag(Node::Flag_is_predicated_vector);
1833 } else {
1834 rearrange = gvn().transform(rearrange);
1835 Node* zero = gvn().makecon(Type::get_zero_type(elem_bt));
1836 Node* zerovec = gvn().transform(VectorNode::scalar2vector(zero, num_elem, elem_bt));
1837 rearrange = new VectorBlendNode(zerovec, rearrange, mask);
1838 }
1839 }
1840 rearrange = gvn().transform(rearrange);
1841
1842 Node* box = box_vector(rearrange, vbox_type, elem_bt, num_elem);
1843 set_result(box);
1844 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1845 return true;
1846 }
1847
1848 static address get_vector_math_address(int vop, int bits, BasicType bt, char* name_ptr, int name_len) {
1849 address addr = nullptr;
1850 assert(UseVectorStubs, "sanity");
1851 assert(name_ptr != nullptr, "unexpected");
1852 assert((vop >= VectorSupport::VECTOR_OP_MATH_START) && (vop <= VectorSupport::VECTOR_OP_MATH_END), "unexpected");
1853 int op = vop - VectorSupport::VECTOR_OP_MATH_START;
1854
1855 switch(bits) {
1856 case 64: //fallthough
1857 case 128: //fallthough
1858 case 256: //fallthough
1859 case 512:
1860 if (bt == T_FLOAT) {
1861 snprintf(name_ptr, name_len, "vector_%s_float_%dbits_fixed", VectorSupport::mathname[op], bits);
1862 addr = StubRoutines::_vector_f_math[exact_log2(bits/64)][op];
1863 } else {
1864 assert(bt == T_DOUBLE, "must be FP type only");
1865 snprintf(name_ptr, name_len, "vector_%s_double_%dbits_fixed", VectorSupport::mathname[op], bits);
1866 addr = StubRoutines::_vector_d_math[exact_log2(bits/64)][op];
1867 }
1868 break;
1869 default:
1870 if (!Matcher::supports_scalable_vector() || !Matcher::vector_size_supported(bt, bits/type2aelembytes(bt)) ) {
1871 snprintf(name_ptr, name_len, "invalid");
1872 addr = nullptr;
1873 Unimplemented();
1874 }
1875 break;
1876 }
1877
1878 if (addr == nullptr && Matcher::supports_scalable_vector()) {
1879 if (bt == T_FLOAT) {
1880 snprintf(name_ptr, name_len, "vector_%s_float_%dbits_scalable", VectorSupport::mathname[op], bits);
1881 addr = StubRoutines::_vector_f_math[VectorSupport::VEC_SIZE_SCALABLE][op];
1882 } else {
1883 assert(bt == T_DOUBLE, "must be FP type only");
1884 snprintf(name_ptr, name_len, "vector_%s_double_%dbits_scalable", VectorSupport::mathname[op], bits);
1885 addr = StubRoutines::_vector_d_math[VectorSupport::VEC_SIZE_SCALABLE][op];
1886 }
1887 }
1888
1889 return addr;
1890 }
1891
1892 // public static
1893 // <V extends Vector<E>,
1894 // M extends VectorMask<E>,
1895 // E>
1896 // V selectFromOp(Class<? extends V> vClass, Class<M> mClass, Class<E> eClass,
1897 // int length, V v1, V v2, M m,
1898 // VectorSelectFromOp<V, M> defaultImpl)
1899 bool LibraryCallKit::inline_vector_select_from() {
1900 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
1901 const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
1902 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
1903 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
1904
1905 if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr ||
1906 vector_klass->const_oop() == nullptr ||
1907 elem_klass->const_oop() == nullptr ||
1908 !vlen->is_con()) {
1909 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s",
1910 NodeClassNames[argument(0)->Opcode()],
1911 NodeClassNames[argument(2)->Opcode()],
1912 NodeClassNames[argument(3)->Opcode()]);
1913 return false; // not enough info for intrinsification
1914 }
1915 if (!is_klass_initialized(vector_klass)) {
1916 log_if_needed(" ** klass argument not initialized");
1917 return false;
1918 }
1919 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1920 if (!elem_type->is_primitive_type()) {
1921 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1922 return false; // should be primitive type
1923 }
1924 BasicType elem_bt = elem_type->basic_type();
1925 int num_elem = vlen->get_con();
1926 if (!is_power_of_2(num_elem)) {
1927 log_if_needed(" ** vlen not power of two=%d", num_elem);
1928 return false;
1929 }
1930
1931 BasicType shuffle_bt = elem_bt;
1932 if (shuffle_bt == T_FLOAT) {
1933 shuffle_bt = T_INT;
1934 } else if (shuffle_bt == T_DOUBLE) {
1935 shuffle_bt = T_LONG;
1936 }
1937 bool need_load_shuffle = Matcher::vector_rearrange_requires_load_shuffle(shuffle_bt, num_elem);
1938
1939 int cast_vopc = VectorCastNode::opcode(-1, elem_bt); // from vector of type elem_bt
1940 if ((need_load_shuffle && !arch_supports_vector(Op_VectorLoadShuffle, num_elem, elem_bt, VecMaskNotUsed)) ||
1941 (elem_bt != shuffle_bt && !arch_supports_vector(cast_vopc, num_elem, shuffle_bt, VecMaskNotUsed)) ||
1942 !arch_supports_vector(Op_AndV, num_elem, shuffle_bt, VecMaskNotUsed) ||
1943 !arch_supports_vector(Op_Replicate, num_elem, shuffle_bt, VecMaskNotUsed)) {
1944 log_if_needed(" ** not supported: arity=0 op=selectFrom vlen=%d etype=%s ismask=no",
1945 num_elem, type2name(elem_bt));
1946 return false; // not supported
1947 }
1948
1949 bool is_masked_op = argument(6)->bottom_type() != TypePtr::NULL_PTR;
1950 bool use_predicate = is_masked_op;
1951 if (is_masked_op &&
1952 (mask_klass == nullptr ||
1953 mask_klass->const_oop() == nullptr ||
1954 !is_klass_initialized(mask_klass))) {
1955 log_if_needed(" ** mask_klass argument not initialized");
1956 return false; // not supported
1957 }
1958 VectorMaskUseType checkFlags = (VectorMaskUseType)(is_masked_op ? (VecMaskUseLoad | VecMaskUsePred) : VecMaskNotUsed);
1959 if (!arch_supports_vector(Op_VectorRearrange, num_elem, elem_bt, checkFlags)) {
1960 use_predicate = false;
1961 if(!is_masked_op ||
1962 (!arch_supports_vector(Op_VectorRearrange, num_elem, elem_bt, VecMaskNotUsed) ||
1963 !arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad) ||
1964 !arch_supports_vector(Op_Replicate, num_elem, elem_bt, VecMaskNotUsed))) {
1965 log_if_needed(" ** not supported: op=selectFrom vlen=%d etype=%s is_masked_op=%d",
1966 num_elem, type2name(elem_bt), is_masked_op);
1967 return false; // not supported
1968 }
1969 }
1970 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1971 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1972
1973 // v1 is the index vector
1974 Node* v1 = unbox_vector(argument(4), vbox_type, elem_bt, num_elem);
1975 // v2 is the vector being rearranged
1976 Node* v2 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
1977
1978 if (v1 == nullptr) {
1979 log_if_needed(" ** unbox failed v1=%s", NodeClassNames[argument(4)->Opcode()]);
1980 return false; // operand unboxing failed
1981 }
1982
1983 if (v2 == nullptr) {
1984 log_if_needed(" ** unbox failed v2=%s", NodeClassNames[argument(5)->Opcode()]);
1985 return false; // operand unboxing failed
1986 }
1987
1988 Node* mask = nullptr;
1989 if (is_masked_op) {
1990 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1991 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1992 mask = unbox_vector(argument(6), mbox_type, elem_bt, num_elem);
1993 if (mask == nullptr) {
1994 log_if_needed(" ** unbox failed mask=%s", NodeClassNames[argument(6)->Opcode()]);
1995 return false;
1996 }
1997 }
1998
1999 // cast index vector from elem_bt vector to byte vector
2000 const TypeVect* shuffle_vt = TypeVect::make(shuffle_bt, num_elem);
2001 Node* shuffle = v1;
2002
2003 if (shuffle_bt != elem_bt) {
2004 shuffle = gvn().transform(VectorCastNode::make(cast_vopc, v1, shuffle_bt, num_elem));
2005 }
2006
2007 // wrap the byte vector lanes to (num_elem - 1) to form the shuffle vector where num_elem is vector length
2008 // this is a simple AND operation as we come here only for power of two vector length
2009 Node* mod_val = gvn().makecon(TypeInteger::make(num_elem - 1, num_elem - 1, Type::WidenMin, shuffle_bt == T_LONG ? T_LONG : T_INT));
2010 Node* bcast_mod = gvn().transform(VectorNode::scalar2vector(mod_val, num_elem, shuffle_bt));
2011 shuffle = gvn().transform(VectorNode::make(Op_AndV, shuffle, bcast_mod, shuffle_vt));
2012
2013 // load the shuffle to use in rearrange
2014 if (need_load_shuffle) {
2015 shuffle = gvn().transform(new VectorLoadShuffleNode(shuffle, shuffle_vt));
2016 }
2017
2018 // and finally rearrange
2019 Node* rearrange = new VectorRearrangeNode(v2, shuffle);
2020 if (is_masked_op) {
2021 if (use_predicate) {
2022 // masked rearrange is supported so use that directly
2023 rearrange->add_req(mask);
2024 rearrange->add_flag(Node::Flag_is_predicated_vector);
2025 } else {
2026 // masked rearrange is not supported so emulate usig blend
2027 const TypeVect* vt = v1->bottom_type()->is_vect();
2028 rearrange = gvn().transform(rearrange);
2029
2030 // create a zero vector with each lane element set as zero
2031 Node* zero = gvn().makecon(Type::get_zero_type(elem_bt));
2032 Node* zerovec = gvn().transform(VectorNode::scalar2vector(zero, num_elem, elem_bt));
2033
2034 // For each lane for which mask is set, blend in the rearranged lane into zero vector
2035 rearrange = new VectorBlendNode(zerovec, rearrange, mask);
2036 }
2037 }
2038 rearrange = gvn().transform(rearrange);
2039
2040 // box the result
2041 Node* box = box_vector(rearrange, vbox_type, elem_bt, num_elem);
2042 set_result(box);
2043
2044 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2045 return true;
2046 }
2047
2048 Node* LibraryCallKit::gen_call_to_vector_math(int vector_api_op_id, BasicType bt, int num_elem, Node* opd1, Node* opd2) {
2049 assert(UseVectorStubs, "sanity");
2050 assert(vector_api_op_id >= VectorSupport::VECTOR_OP_MATH_START && vector_api_op_id <= VectorSupport::VECTOR_OP_MATH_END, "need valid op id");
2051 assert(opd1 != nullptr, "must not be null");
2052 const TypeVect* vt = TypeVect::make(bt, num_elem);
2053 const TypeFunc* call_type = OptoRuntime::Math_Vector_Vector_Type(opd2 != nullptr ? 2 : 1, vt, vt);
2054 char name[100] = "";
2055
2056 // Get address for vector math method.
2057 address addr = get_vector_math_address(vector_api_op_id, vt->length_in_bytes() * BitsPerByte, bt, name, 100);
2058
2059 if (addr == nullptr) {
2060 return nullptr;
2061 }
2062
2063 assert(name[0] != '\0', "name must not be null");
2064 Node* operation = make_runtime_call(RC_VECTOR,
2065 call_type,
2066 addr,
2067 name,
2068 TypePtr::BOTTOM,
2069 opd1,
2070 opd2);
2071 return gvn().transform(new ProjNode(gvn().transform(operation), TypeFunc::Parms));
2072 }
2073
2074 // public static
2075 // <V extends Vector<E>,
2076 // M extends VectorMask<E>,
2077 // E>
2078 // V broadcastInt(int opr, Class<? extends V> vectorClass, Class<? extends M> maskClass,
2079 // Class<E> elementType, int length,
2080 // V v, int n, M m,
2081 // VectorBroadcastIntOp<V, M> defaultImpl)
2082 bool LibraryCallKit::inline_vector_broadcast_int() {
2083 const TypeInt* opr = gvn().type(argument(0))->isa_int();
2084 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
2085 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
2086 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
2087 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
2088
2089 if (opr == nullptr || vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr) {
2090 return false; // dead code
2091 }
2092 if (!opr->is_con() || vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
2093 log_if_needed(" ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
2094 NodeClassNames[argument(0)->Opcode()],
2095 NodeClassNames[argument(1)->Opcode()],
2096 NodeClassNames[argument(3)->Opcode()],
2097 NodeClassNames[argument(4)->Opcode()]);
2098 return false; // not enough info for intrinsification
2099 }
2100 if (!is_klass_initialized(vector_klass)) {
2101 log_if_needed(" ** klass argument not initialized");
2102 return false;
2103 }
2104
2105 const Type* vmask_type = gvn().type(argument(7));
2106 bool is_masked_op = vmask_type != TypePtr::NULL_PTR;
2107 if (is_masked_op) {
2108 if (mask_klass == nullptr || mask_klass->const_oop() == nullptr) {
2109 log_if_needed(" ** missing constant: maskclass=%s", NodeClassNames[argument(2)->Opcode()]);
2110 return false; // not enough info for intrinsification
2111 }
2112
2113 if (!is_klass_initialized(mask_klass)) {
2114 log_if_needed(" ** mask klass argument not initialized");
2115 return false;
2116 }
2117
2118 if (vmask_type->maybe_null()) {
2119 log_if_needed(" ** null mask values are not allowed for masked op");
2120 return false;
2121 }
2122 }
2123
2124 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2125 if (!elem_type->is_primitive_type()) {
2126 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2127 return false; // should be primitive type
2128 }
2129
2130 int num_elem = vlen->get_con();
2131 BasicType elem_bt = elem_type->basic_type();
2132 int opc = VectorSupport::vop2ideal(opr->get_con(), elem_bt);
2133
2134 bool is_shift = VectorNode::is_shift_opcode(opc);
2135 bool is_rotate = VectorNode::is_rotate_opcode(opc);
2136
2137 if (opc == 0 || (!is_shift && !is_rotate)) {
2138 log_if_needed(" ** operation not supported: op=%d bt=%s", opr->get_con(), type2name(elem_bt));
2139 return false; // operation not supported
2140 }
2141
2142 int sopc = VectorNode::opcode(opc, elem_bt);
2143 if (sopc == 0) {
2144 log_if_needed(" ** operation not supported: opc=%s bt=%s", NodeClassNames[opc], type2name(elem_bt));
2145 return false; // operation not supported
2146 }
2147
2148 Node* cnt = argument(6);
2149 const TypeInt* cnt_type = cnt->bottom_type()->isa_int();
2150
2151 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2152 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2153
2154 // If CPU supports vector constant rotate instructions pass it directly
2155 bool is_const_rotate = is_rotate && cnt_type && cnt_type->is_con() &&
2156 Matcher::supports_vector_constant_rotates(cnt_type->get_con());
2157 bool has_scalar_args = is_rotate ? !is_const_rotate : true;
2158
2159 VectorMaskUseType checkFlags = (VectorMaskUseType)(is_masked_op ? (VecMaskUseLoad | VecMaskUsePred) : VecMaskNotUsed);
2160 bool use_predicate = is_masked_op;
2161
2162 if (!arch_supports_vector(sopc, num_elem, elem_bt, checkFlags, has_scalar_args)) {
2163 use_predicate = false;
2164 if (!is_masked_op ||
2165 (!arch_supports_vector(sopc, num_elem, elem_bt, VecMaskNotUsed, has_scalar_args) ||
2166 !arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad))) {
2167
2168 log_if_needed(" ** not supported: arity=0 op=int/%d vlen=%d etype=%s is_masked_op=%d",
2169 sopc, num_elem, type2name(elem_bt), is_masked_op ? 1 : 0);
2170 return false; // not supported
2171 }
2172 }
2173
2174 Node* opd1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
2175 Node* opd2 = nullptr;
2176 if (is_shift) {
2177 opd2 = vector_shift_count(cnt, opc, elem_bt, num_elem);
2178 } else {
2179 assert(is_rotate, "unexpected operation");
2180 if (!is_const_rotate) {
2181 cnt = elem_bt == T_LONG ? gvn().transform(new ConvI2LNode(cnt)) : cnt;
2182 opd2 = gvn().transform(VectorNode::scalar2vector(cnt, num_elem, elem_bt));
2183 } else {
2184 // Constant shift value.
2185 opd2 = cnt;
2186 }
2187 }
2188
2189 if (opd1 == nullptr || opd2 == nullptr) {
2190 return false;
2191 }
2192
2193 Node* mask = nullptr;
2194 if (is_masked_op) {
2195 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
2196 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
2197 mask = unbox_vector(argument(7), mbox_type, elem_bt, num_elem);
2198 if (mask == nullptr) {
2199 log_if_needed(" ** unbox failed mask=%s", NodeClassNames[argument(7)->Opcode()]);
2200 return false;
2201 }
2202 }
2203
2204 Node* operation = VectorNode::make(opc, opd1, opd2, num_elem, elem_bt);
2205 if (is_masked_op && mask != nullptr) {
2206 if (use_predicate) {
2207 operation->add_req(mask);
2208 operation->add_flag(Node::Flag_is_predicated_vector);
2209 } else {
2210 operation = gvn().transform(operation);
2211 operation = new VectorBlendNode(opd1, operation, mask);
2212 }
2213 }
2214 operation = gvn().transform(operation);
2215 Node* vbox = box_vector(operation, vbox_type, elem_bt, num_elem);
2216 set_result(vbox);
2217 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2218 return true;
2219 }
2220
2221 // public static <VOUT extends VectorPayload,
2222 // VIN extends VectorPayload,
2223 // S extends VectorSpecies>
2224 // VOUT convert(int oprId,
2225 // Class<?> fromVectorClass, Class<?> fromElementType, int fromVLen,
2226 // Class<?> toVectorClass, Class<?> toElementType, int toVLen,
2227 // VIN v, S s,
2228 // VectorConvertOp<VOUT, VIN, S> defaultImpl)
2229 //
2230 bool LibraryCallKit::inline_vector_convert() {
2231 const TypeInt* opr = gvn().type(argument(0))->isa_int();
2232
2233 const TypeInstPtr* vector_klass_from = gvn().type(argument(1))->isa_instptr();
2234 const TypeInstPtr* elem_klass_from = gvn().type(argument(2))->isa_instptr();
2235 const TypeInt* vlen_from = gvn().type(argument(3))->isa_int();
2236
2237 const TypeInstPtr* vector_klass_to = gvn().type(argument(4))->isa_instptr();
2238 const TypeInstPtr* elem_klass_to = gvn().type(argument(5))->isa_instptr();
2239 const TypeInt* vlen_to = gvn().type(argument(6))->isa_int();
2240
2241 if (opr == nullptr ||
2242 vector_klass_from == nullptr || elem_klass_from == nullptr || vlen_from == nullptr ||
2243 vector_klass_to == nullptr || elem_klass_to == nullptr || vlen_to == nullptr) {
2244 return false; // dead code
2245 }
2246 if (!opr->is_con() ||
2247 vector_klass_from->const_oop() == nullptr || elem_klass_from->const_oop() == nullptr || !vlen_from->is_con() ||
2248 vector_klass_to->const_oop() == nullptr || elem_klass_to->const_oop() == nullptr || !vlen_to->is_con()) {
2249 log_if_needed(" ** missing constant: opr=%s vclass_from=%s etype_from=%s vlen_from=%s vclass_to=%s etype_to=%s vlen_to=%s",
2250 NodeClassNames[argument(0)->Opcode()],
2251 NodeClassNames[argument(1)->Opcode()],
2252 NodeClassNames[argument(2)->Opcode()],
2253 NodeClassNames[argument(3)->Opcode()],
2254 NodeClassNames[argument(4)->Opcode()],
2255 NodeClassNames[argument(5)->Opcode()],
2256 NodeClassNames[argument(6)->Opcode()]);
2257 return false; // not enough info for intrinsification
2258 }
2259 if (!is_klass_initialized(vector_klass_from) || !is_klass_initialized(vector_klass_to)) {
2260 log_if_needed(" ** klass argument not initialized");
2261 return false;
2262 }
2263
2264 assert(opr->get_con() == VectorSupport::VECTOR_OP_CAST ||
2265 opr->get_con() == VectorSupport::VECTOR_OP_UCAST ||
2266 opr->get_con() == VectorSupport::VECTOR_OP_REINTERPRET, "wrong opcode");
2267 bool is_cast = (opr->get_con() == VectorSupport::VECTOR_OP_CAST || opr->get_con() == VectorSupport::VECTOR_OP_UCAST);
2268 bool is_ucast = (opr->get_con() == VectorSupport::VECTOR_OP_UCAST);
2269
2270 ciKlass* vbox_klass_from = vector_klass_from->const_oop()->as_instance()->java_lang_Class_klass();
2271 ciKlass* vbox_klass_to = vector_klass_to->const_oop()->as_instance()->java_lang_Class_klass();
2272
2273 bool is_mask = is_vector_mask(vbox_klass_from);
2274
2275 ciType* elem_type_from = elem_klass_from->const_oop()->as_instance()->java_mirror_type();
2276 if (!elem_type_from->is_primitive_type()) {
2277 return false; // should be primitive type
2278 }
2279 BasicType elem_bt_from = elem_type_from->basic_type();
2280 ciType* elem_type_to = elem_klass_to->const_oop()->as_instance()->java_mirror_type();
2281 if (!elem_type_to->is_primitive_type()) {
2282 return false; // should be primitive type
2283 }
2284 BasicType elem_bt_to = elem_type_to->basic_type();
2285
2286 int num_elem_from = vlen_from->get_con();
2287 int num_elem_to = vlen_to->get_con();
2288
2289 // Check whether we can unbox to appropriate size. Even with casting, checking for reinterpret is needed
2290 // since we may need to change size.
2291 if (!arch_supports_vector(Op_VectorReinterpret,
2292 num_elem_from,
2293 elem_bt_from,
2294 is_mask ? VecMaskUseAll : VecMaskNotUsed)) {
2295 log_if_needed(" ** not supported: arity=1 op=%s/1 vlen1=%d etype1=%s ismask=%d",
2296 is_cast ? "cast" : "reinterpret",
2297 num_elem_from, type2name(elem_bt_from), is_mask);
2298 return false;
2299 }
2300
2301 // Check whether we can support resizing/reinterpreting to the new size.
2302 if (!arch_supports_vector(Op_VectorReinterpret,
2303 num_elem_to,
2304 elem_bt_to,
2305 is_mask ? VecMaskUseAll : VecMaskNotUsed)) {
2306 log_if_needed(" ** not supported: arity=1 op=%s/2 vlen2=%d etype2=%s ismask=%d",
2307 is_cast ? "cast" : "reinterpret",
2308 num_elem_to, type2name(elem_bt_to), is_mask);
2309 return false;
2310 }
2311
2312 // At this point, we know that both input and output vector registers are supported
2313 // by the architecture. Next check if the casted type is simply to same type - which means
2314 // that it is actually a resize and not a cast.
2315 if (is_cast && elem_bt_from == elem_bt_to) {
2316 is_cast = false;
2317 }
2318
2319 const TypeInstPtr* vbox_type_from = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass_from);
2320
2321 Node* opd1 = unbox_vector(argument(7), vbox_type_from, elem_bt_from, num_elem_from);
2322 if (opd1 == nullptr) {
2323 return false;
2324 }
2325
2326 const TypeVect* src_type = TypeVect::make(elem_bt_from, num_elem_from, is_mask);
2327 const TypeVect* dst_type = TypeVect::make(elem_bt_to, num_elem_to, is_mask);
2328
2329 // Safety check to prevent casting if source mask is of type vector
2330 // and destination mask of type predicate vector and vice-versa.
2331 // From X86 standpoint, this case will only arise over KNL target,
2332 // where certain masks (depending on the species) are either propagated
2333 // through a vector or predicate register.
2334 if (is_mask &&
2335 ((src_type->isa_vectmask() == nullptr && dst_type->isa_vectmask()) ||
2336 (dst_type->isa_vectmask() == nullptr && src_type->isa_vectmask()))) {
2337 return false;
2338 }
2339
2340 Node* op = opd1;
2341 if (is_cast) {
2342 assert(!is_mask || num_elem_from == num_elem_to, "vector mask cast needs the same elem num");
2343 int cast_vopc = VectorCastNode::opcode(-1, elem_bt_from, !is_ucast);
2344
2345 // Make sure that vector cast is implemented to particular type/size combination if it is
2346 // not a mask casting.
2347 if (!is_mask && !arch_supports_vector(cast_vopc, num_elem_to, elem_bt_to, VecMaskNotUsed)) {
2348 log_if_needed(" ** not supported: arity=1 op=cast#%d/3 vlen2=%d etype2=%s ismask=%d",
2349 cast_vopc, num_elem_to, type2name(elem_bt_to), is_mask);
2350 return false;
2351 }
2352
2353 if (num_elem_from < num_elem_to) {
2354 // Since input and output number of elements are not consistent, we need to make sure we
2355 // properly size. Thus, first make a cast that retains the number of elements from source.
2356 int num_elem_for_cast = num_elem_from;
2357
2358 // It is possible that arch does not support this intermediate vector size
2359 // TODO More complex logic required here to handle this corner case for the sizes.
2360 if (!arch_supports_vector(cast_vopc, num_elem_for_cast, elem_bt_to, VecMaskNotUsed)) {
2361 log_if_needed(" ** not supported: arity=1 op=cast#%d/4 vlen1=%d etype2=%s ismask=%d",
2362 cast_vopc,
2363 num_elem_for_cast, type2name(elem_bt_to), is_mask);
2364 return false;
2365 }
2366
2367 op = gvn().transform(VectorCastNode::make(cast_vopc, op, elem_bt_to, num_elem_for_cast));
2368 // Now ensure that the destination gets properly resized to needed size.
2369 op = gvn().transform(new VectorReinterpretNode(op, op->bottom_type()->is_vect(), dst_type));
2370 } else if (num_elem_from > num_elem_to) {
2371 // Since number of elements from input is larger than output, simply reduce size of input
2372 // (we are supposed to drop top elements anyway).
2373 int num_elem_for_resize = num_elem_to;
2374
2375 // It is possible that arch does not support this intermediate vector size
2376 // TODO More complex logic required here to handle this corner case for the sizes.
2377 if (!arch_supports_vector(Op_VectorReinterpret,
2378 num_elem_for_resize,
2379 elem_bt_from,
2380 VecMaskNotUsed)) {
2381 log_if_needed(" ** not supported: arity=1 op=cast/5 vlen2=%d etype1=%s ismask=%d",
2382 num_elem_for_resize, type2name(elem_bt_from), is_mask);
2383 return false;
2384 }
2385
2386 const TypeVect* resize_type = TypeVect::make(elem_bt_from, num_elem_for_resize);
2387 op = gvn().transform(new VectorReinterpretNode(op, src_type, resize_type));
2388 op = gvn().transform(VectorCastNode::make(cast_vopc, op, elem_bt_to, num_elem_to));
2389 } else { // num_elem_from == num_elem_to
2390 if (is_mask) {
2391 // Make sure that cast for vector mask is implemented to particular type/size combination.
2392 if (!arch_supports_vector(Op_VectorMaskCast, num_elem_to, elem_bt_to, VecMaskNotUsed)) {
2393 log_if_needed(" ** not supported: arity=1 op=maskcast vlen2=%d etype2=%s ismask=%d",
2394 num_elem_to, type2name(elem_bt_to), is_mask);
2395 return false;
2396 }
2397 op = gvn().transform(new VectorMaskCastNode(op, dst_type));
2398 } else {
2399 // Since input and output number of elements match, and since we know this vector size is
2400 // supported, simply do a cast with no resize needed.
2401 op = gvn().transform(VectorCastNode::make(cast_vopc, op, elem_bt_to, num_elem_to));
2402 }
2403 }
2404 } else if (!Type::equals(src_type, dst_type)) {
2405 assert(!is_cast, "must be reinterpret");
2406 op = gvn().transform(new VectorReinterpretNode(op, src_type, dst_type));
2407 }
2408
2409 const TypeInstPtr* vbox_type_to = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass_to);
2410 Node* vbox = box_vector(op, vbox_type_to, elem_bt_to, num_elem_to);
2411 set_result(vbox);
2412 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem_to * type2aelembytes(elem_bt_to))));
2413 return true;
2414 }
2415
2416 // public static
2417 // <V extends Vector<E>,
2418 // E>
2419 // V insert(Class<? extends V> vectorClass, Class<E> elementType, int vlen,
2420 // V vec, int ix, long val,
2421 // VecInsertOp<V> defaultImpl)
2422 bool LibraryCallKit::inline_vector_insert() {
2423 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
2424 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
2425 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
2426 const TypeInt* idx = gvn().type(argument(4))->isa_int();
2427
2428 if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr || idx == nullptr) {
2429 return false; // dead code
2430 }
2431 if (vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con() || !idx->is_con()) {
2432 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s idx=%s",
2433 NodeClassNames[argument(0)->Opcode()],
2434 NodeClassNames[argument(1)->Opcode()],
2435 NodeClassNames[argument(2)->Opcode()],
2436 NodeClassNames[argument(4)->Opcode()]);
2437 return false; // not enough info for intrinsification
2438 }
2439 if (!is_klass_initialized(vector_klass)) {
2440 log_if_needed(" ** klass argument not initialized");
2441 return false;
2442 }
2443 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2444 if (!elem_type->is_primitive_type()) {
2445 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2446 return false; // should be primitive type
2447 }
2448 BasicType elem_bt = elem_type->basic_type();
2449 int num_elem = vlen->get_con();
2450 if (!arch_supports_vector(Op_VectorInsert, num_elem, elem_bt, VecMaskNotUsed)) {
2451 log_if_needed(" ** not supported: arity=1 op=insert vlen=%d etype=%s ismask=no",
2452 num_elem, type2name(elem_bt));
2453 return false; // not supported
2454 }
2455
2456 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2457 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2458
2459 Node* opd = unbox_vector(argument(3), vbox_type, elem_bt, num_elem);
2460 if (opd == nullptr) {
2461 return false;
2462 }
2463
2464 Node* insert_val = argument(5);
2465 assert(gvn().type(insert_val)->isa_long() != nullptr, "expected to be long");
2466
2467 // Convert insert value back to its appropriate type.
2468 switch (elem_bt) {
2469 case T_BYTE:
2470 insert_val = gvn().transform(new ConvL2INode(insert_val, TypeInt::BYTE));
2471 break;
2472 case T_SHORT:
2473 insert_val = gvn().transform(new ConvL2INode(insert_val, TypeInt::SHORT));
2474 break;
2475 case T_INT:
2476 insert_val = gvn().transform(new ConvL2INode(insert_val));
2477 break;
2478 case T_FLOAT:
2479 insert_val = gvn().transform(new ConvL2INode(insert_val));
2480 insert_val = gvn().transform(new MoveI2FNode(insert_val));
2481 break;
2482 case T_DOUBLE:
2483 insert_val = gvn().transform(new MoveL2DNode(insert_val));
2484 break;
2485 case T_LONG:
2486 // no conversion needed
2487 break;
2488 default: fatal("%s", type2name(elem_bt)); break;
2489 }
2490
2491 Node* operation = gvn().transform(VectorInsertNode::make(opd, insert_val, idx->get_con(), gvn()));
2492
2493 Node* vbox = box_vector(operation, vbox_type, elem_bt, num_elem);
2494 set_result(vbox);
2495 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2496 return true;
2497 }
2498
2499 // public static
2500 // <VM extends VectorPayload,
2501 // E>
2502 // long extract(Class<? extends VM> vClass, Class<E> eClass,
2503 // int length,
2504 // VM vm, int i,
2505 // VecExtractOp<VM> defaultImpl)
2506 bool LibraryCallKit::inline_vector_extract() {
2507 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
2508 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
2509 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
2510 const TypeInt* idx = gvn().type(argument(4))->isa_int();
2511
2512 if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr || idx == nullptr) {
2513 return false; // dead code
2514 }
2515 if (vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
2516 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s",
2517 NodeClassNames[argument(0)->Opcode()],
2518 NodeClassNames[argument(1)->Opcode()],
2519 NodeClassNames[argument(2)->Opcode()]);
2520 return false; // not enough info for intrinsification
2521 }
2522 if (!is_klass_initialized(vector_klass)) {
2523 log_if_needed(" ** klass argument not initialized");
2524 return false;
2525 }
2526 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2527 if (!elem_type->is_primitive_type()) {
2528 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2529 return false; // should be primitive type
2530 }
2531 BasicType elem_bt = elem_type->basic_type();
2532 int num_elem = vlen->get_con();
2533
2534 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2535 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2536
2537 Node* opd = nullptr;
2538
2539 if (is_vector_mask(vbox_klass)) {
2540 // vbox_klass is mask. This is used for VectorMask.laneIsSet(int).
2541
2542 Node* pos = argument(4); // can be variable
2543 if (arch_supports_vector(Op_ExtractUB, num_elem, elem_bt, VecMaskUseAll)) {
2544 // Transform mask to vector with type of boolean and utilize ExtractUB node.
2545 opd = unbox_vector(argument(3), vbox_type, elem_bt, num_elem);
2546 if (opd == nullptr) {
2547 return false;
2548 }
2549 opd = gvn().transform(VectorStoreMaskNode::make(gvn(), opd, elem_bt, num_elem));
2550 opd = gvn().transform(new ExtractUBNode(opd, pos));
2551 opd = gvn().transform(new ConvI2LNode(opd));
2552 } else if (arch_supports_vector(Op_VectorMaskToLong, num_elem, elem_bt, VecMaskUseLoad)) {
2553 opd = unbox_vector(argument(3), vbox_type, elem_bt, num_elem);
2554 if (opd == nullptr) {
2555 return false;
2556 }
2557 // VectorMaskToLongNode requires the input is either a mask or a vector with BOOLEAN type.
2558 if (opd->bottom_type()->isa_vectmask() == nullptr) {
2559 opd = gvn().transform(VectorStoreMaskNode::make(gvn(), opd, elem_bt, num_elem));
2560 }
2561 // ((toLong() >>> pos) & 1L
2562 opd = gvn().transform(new VectorMaskToLongNode(opd, TypeLong::LONG));
2563 opd = gvn().transform(new URShiftLNode(opd, pos));
2564 opd = gvn().transform(new AndLNode(opd, gvn().makecon(TypeLong::ONE)));
2565 } else {
2566 log_if_needed(" ** Rejected mask extraction because architecture does not support it");
2567 return false; // not supported
2568 }
2569 } else {
2570 // vbox_klass is vector. This is used for Vector.lane(int).
2571 if (!idx->is_con()) {
2572 log_if_needed(" ** missing constant: idx=%s", NodeClassNames[argument(4)->Opcode()]);
2573 return false; // not enough info for intrinsification
2574 }
2575
2576 int vopc = ExtractNode::opcode(elem_bt);
2577 if (!arch_supports_vector(vopc, num_elem, elem_bt, VecMaskNotUsed)) {
2578 log_if_needed(" ** not supported: arity=1 op=extract vlen=%d etype=%s ismask=no",
2579 num_elem, type2name(elem_bt));
2580 return false; // not supported
2581 }
2582
2583 opd = unbox_vector(argument(3), vbox_type, elem_bt, num_elem);
2584 if (opd == nullptr) {
2585 return false;
2586 }
2587 ConINode* idx_con = gvn().intcon(idx->get_con())->as_ConI();
2588
2589 opd = gvn().transform(ExtractNode::make(opd, idx_con, elem_bt));
2590 switch (elem_bt) {
2591 case T_BYTE:
2592 case T_SHORT:
2593 case T_INT: {
2594 opd = gvn().transform(new ConvI2LNode(opd));
2595 break;
2596 }
2597 case T_FLOAT: {
2598 opd = gvn().transform(new MoveF2INode(opd));
2599 opd = gvn().transform(new ConvI2LNode(opd));
2600 break;
2601 }
2602 case T_DOUBLE: {
2603 opd = gvn().transform(new MoveD2LNode(opd));
2604 break;
2605 }
2606 case T_LONG: {
2607 // no conversion needed
2608 break;
2609 }
2610 default: fatal("%s", type2name(elem_bt));
2611 }
2612 }
2613 set_result(opd);
2614 return true;
2615 }
2616
2617 static Node* LowerSelectFromTwoVectorOperation(PhaseGVN& phase, Node* index_vec, Node* src1, Node* src2, const TypeVect* vt) {
2618 int num_elem = vt->length();
2619 BasicType elem_bt = vt->element_basic_type();
2620
2621 // Lower selectFrom operation into its constituent operations.
2622 // SelectFromTwoVectorNode =
2623 // (VectorBlend
2624 // (VectorRearrange SRC1 (WRAPED_INDEX AND (VLEN-1))
2625 // (VectorRearrange SRC2 (WRAPED_INDEX AND (VLEN-1))
2626 // MASK)
2627 // Where
2628 // WRAPED_INDEX are computed by wrapping incoming indexes
2629 // to two vector index range [0, VLEN*2) and
2630 // MASK = WRAPED_INDEX < VLEN
2631 //
2632 // IR lowering prevents intrinsification failure and associated argument
2633 // boxing penalties.
2634 //
2635
2636 BasicType shuffle_bt = elem_bt;
2637 if (shuffle_bt == T_FLOAT) {
2638 shuffle_bt = T_INT;
2639 } else if (shuffle_bt == T_DOUBLE) {
2640 shuffle_bt = T_LONG;
2641 }
2642 const TypeVect* st = TypeVect::make(shuffle_bt, num_elem);
2643
2644 // Cast index vector to the corresponding bit type
2645 if (elem_bt != shuffle_bt) {
2646 int cast_vopc = VectorCastNode::opcode(0, elem_bt, true);
2647 index_vec = phase.transform(VectorCastNode::make(cast_vopc, index_vec, shuffle_bt, num_elem));
2648 }
2649
2650 // Wrap indexes into two vector index range [0, VLEN * 2)
2651 Node* two_vect_lane_cnt_m1 = phase.makecon(TypeInteger::make(2 * num_elem - 1, 2 * num_elem - 1, Type::WidenMin, shuffle_bt == T_LONG ? T_LONG : T_INT));
2652 Node* bcast_two_vect_lane_cnt_m1_vec = phase.transform(VectorNode::scalar2vector(two_vect_lane_cnt_m1, num_elem,
2653 shuffle_bt, false));
2654 index_vec = phase.transform(VectorNode::make(Op_AndV, index_vec, bcast_two_vect_lane_cnt_m1_vec, st));
2655
2656 // Compute the blend mask for merging two independently permitted vectors
2657 // using shuffle index in two vector index range [0, VLEN * 2).
2658 BoolTest::mask pred = BoolTest::le;
2659 ConINode* pred_node = phase.makecon(TypeInt::make(pred))->as_ConI();
2660 const TypeVect* vmask_type = TypeVect::makemask(shuffle_bt, num_elem);
2661 Node* lane_cnt_m1 = phase.makecon(TypeInteger::make(num_elem - 1, num_elem - 1, Type::WidenMin, shuffle_bt == T_LONG ? T_LONG : T_INT));
2662 Node* bcast_lane_cnt_m1_vec = phase.transform(VectorNode::scalar2vector(lane_cnt_m1, num_elem, shuffle_bt, false));
2663 Node* mask = phase.transform(new VectorMaskCmpNode(pred, index_vec, bcast_lane_cnt_m1_vec, pred_node, vmask_type));
2664
2665 // Rearrange expects the indexes to lie within single vector index range [0, VLEN).
2666 Node* wrapped_index_vec = phase.transform(VectorNode::make(Op_AndV, index_vec, bcast_lane_cnt_m1_vec, st));
2667
2668 // Load indexes from byte vector and appropriately transform them to target
2669 // specific permutation index format.
2670 if (Matcher::vector_rearrange_requires_load_shuffle(shuffle_bt, num_elem)) {
2671 wrapped_index_vec = phase.transform(new VectorLoadShuffleNode(wrapped_index_vec, st));
2672 }
2673
2674 vmask_type = TypeVect::makemask(elem_bt, num_elem);
2675 mask = phase.transform(new VectorMaskCastNode(mask, vmask_type));
2676
2677 Node* p1 = phase.transform(new VectorRearrangeNode(src1, wrapped_index_vec));
2678 Node* p2 = phase.transform(new VectorRearrangeNode(src2, wrapped_index_vec));
2679
2680 return new VectorBlendNode(p2, p1, mask);
2681 }
2682
2683 // public static
2684 // <V extends Vector<E>,
2685 // E>
2686 // V selectFromTwoVectorOp(Class<? extends V> vClass, Class<E> eClass, int length,
2687 // V v1, V v2, V v3,
2688 // SelectFromTwoVector<V> defaultImpl)
2689 bool LibraryCallKit::inline_vector_select_from_two_vectors() {
2690 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
2691 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
2692 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
2693
2694 if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr || vector_klass->const_oop() == nullptr ||
2695 elem_klass->const_oop() == nullptr ||!vlen->is_con()) {
2696 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s",
2697 NodeClassNames[argument(0)->Opcode()],
2698 NodeClassNames[argument(1)->Opcode()],
2699 NodeClassNames[argument(2)->Opcode()]);
2700 return false; // not enough info for intrinsification
2701 }
2702
2703 if (!is_klass_initialized(vector_klass)) {
2704 log_if_needed(" ** klass argument not initialized");
2705 return false;
2706 }
2707
2708 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2709 if (!elem_type->is_primitive_type()) {
2710 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2711 return false; // should be primitive type
2712 }
2713
2714 int num_elem = vlen->get_con();
2715 if (!is_power_of_2(num_elem)) {
2716 log_if_needed(" ** vlen is not power of two=%d", num_elem);
2717 return false;
2718 }
2719
2720 BasicType elem_bt = elem_type->basic_type();
2721 BasicType index_elem_bt = elem_bt;
2722 if (elem_bt == T_FLOAT) {
2723 index_elem_bt = T_INT;
2724 } else if (elem_bt == T_DOUBLE) {
2725 index_elem_bt = T_LONG;
2726 }
2727
2728 bool lowerSelectFromOp = false;
2729 if (!arch_supports_vector(Op_SelectFromTwoVector, num_elem, elem_bt, VecMaskNotUsed)) {
2730 int cast_vopc = VectorCastNode::opcode(-1, elem_bt, true);
2731 if ((elem_bt != index_elem_bt && !arch_supports_vector(cast_vopc, num_elem, index_elem_bt, VecMaskNotUsed)) ||
2732 !arch_supports_vector(Op_VectorMaskCmp, num_elem, index_elem_bt, VecMaskNotUsed) ||
2733 !arch_supports_vector(Op_AndV, num_elem, index_elem_bt, VecMaskNotUsed) ||
2734 !arch_supports_vector(Op_VectorMaskCast, num_elem, elem_bt, VecMaskNotUsed) ||
2735 !arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad) ||
2736 !arch_supports_vector(Op_VectorRearrange, num_elem, elem_bt, VecMaskNotUsed) ||
2737 !arch_supports_vector(Op_VectorLoadShuffle, num_elem, index_elem_bt, VecMaskNotUsed) ||
2738 !arch_supports_vector(Op_Replicate, num_elem, index_elem_bt, VecMaskNotUsed)) {
2739 log_if_needed(" ** not supported: opc=%d vlen=%d etype=%s ismask=useload",
2740 Op_SelectFromTwoVector, num_elem, type2name(elem_bt));
2741 return false; // not supported
2742 }
2743 lowerSelectFromOp = true;
2744 }
2745
2746 int cast_vopc = VectorCastNode::opcode(-1, elem_bt, true);
2747 if (!lowerSelectFromOp) {
2748 if (!arch_supports_vector(Op_AndV, num_elem, index_elem_bt, VecMaskNotUsed) ||
2749 !arch_supports_vector(Op_Replicate, num_elem, index_elem_bt, VecMaskNotUsed) ||
2750 (is_floating_point_type(elem_bt) &&
2751 !arch_supports_vector(cast_vopc, num_elem, index_elem_bt, VecMaskNotUsed))) {
2752 log_if_needed(" ** index wrapping not supported: vlen=%d etype=%s" ,
2753 num_elem, type2name(elem_bt));
2754 return false; // not supported
2755 }
2756 }
2757
2758 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2759 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2760
2761 Node* opd1 = unbox_vector(argument(3), vbox_type, elem_bt, num_elem);
2762 if (opd1 == nullptr) {
2763 log_if_needed(" ** unbox failed v1=%s",
2764 NodeClassNames[argument(3)->Opcode()]);
2765 return false;
2766 }
2767 Node* opd2 = unbox_vector(argument(4), vbox_type, elem_bt, num_elem);
2768 if (opd2 == nullptr) {
2769 log_if_needed(" ** unbox failed v2=%s",
2770 NodeClassNames[argument(4)->Opcode()]);
2771 return false;
2772 }
2773 Node* opd3 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
2774 if (opd3 == nullptr) {
2775 log_if_needed(" ** unbox failed v3=%s",
2776 NodeClassNames[argument(5)->Opcode()]);
2777 return false;
2778 }
2779
2780 const TypeVect* vt = TypeVect::make(elem_bt, num_elem);
2781
2782 Node* operation = nullptr;
2783 if (lowerSelectFromOp) {
2784 operation = gvn().transform(LowerSelectFromTwoVectorOperation(gvn(), opd1, opd2, opd3, vt));
2785 } else {
2786 if (index_elem_bt != elem_bt) {
2787 opd1 = gvn().transform(VectorCastNode::make(cast_vopc, opd1, index_elem_bt, num_elem));
2788 }
2789 int indexRangeMask = 2 * num_elem - 1;
2790 Node* wrap_mask = gvn().makecon(TypeInteger::make(indexRangeMask, indexRangeMask, Type::WidenMin, index_elem_bt != T_LONG ? T_INT : index_elem_bt));
2791 Node* wrap_mask_vec = gvn().transform(VectorNode::scalar2vector(wrap_mask, num_elem, index_elem_bt, false));
2792 opd1 = gvn().transform(VectorNode::make(Op_AndV, opd1, wrap_mask_vec, opd1->bottom_type()->is_vect()));
2793 operation = gvn().transform(VectorNode::make(Op_SelectFromTwoVector, opd1, opd2, opd3, vt));
2794 }
2795
2796 // Wrap it up in VectorBox to keep object type information.
2797 Node* vbox = box_vector(operation, vbox_type, elem_bt, num_elem);
2798 set_result(vbox);
2799 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2800 return true;
2801 }
2802
2803 // public static
2804 // <V extends Vector<E>,
2805 // M extends VectorMask<E>,
2806 // E>
2807 // V compressExpandOp(int opr,
2808 // Class<? extends V> vClass, Class<? extends M> mClass, Class<E> eClass,
2809 // int length, V v, M m,
2810 // CompressExpandOperation<V, M> defaultImpl)
2811 bool LibraryCallKit::inline_vector_compress_expand() {
2812 const TypeInt* opr = gvn().type(argument(0))->isa_int();
2813 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
2814 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
2815 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
2816 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
2817
2818 if (vector_klass == nullptr || elem_klass == nullptr || mask_klass == nullptr || vlen == nullptr ||
2819 vector_klass->const_oop() == nullptr || mask_klass->const_oop() == nullptr ||
2820 elem_klass->const_oop() == nullptr || !vlen->is_con() || !opr->is_con()) {
2821 log_if_needed(" ** missing constant: opr=%s vclass=%s mclass=%s etype=%s vlen=%s",
2822 NodeClassNames[argument(0)->Opcode()],
2823 NodeClassNames[argument(1)->Opcode()],
2824 NodeClassNames[argument(2)->Opcode()],
2825 NodeClassNames[argument(3)->Opcode()],
2826 NodeClassNames[argument(4)->Opcode()]);
2827 return false; // not enough info for intrinsification
2828 }
2829
2830 if (!is_klass_initialized(vector_klass) || !is_klass_initialized(mask_klass)) {
2831 log_if_needed(" ** klass argument not initialized");
2832 return false;
2833 }
2834
2835 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2836 if (!elem_type->is_primitive_type()) {
2837 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2838 return false; // should be primitive type
2839 }
2840
2841 int num_elem = vlen->get_con();
2842 BasicType elem_bt = elem_type->basic_type();
2843 int opc = VectorSupport::vop2ideal(opr->get_con(), elem_bt);
2844
2845 if (!arch_supports_vector(opc, num_elem, elem_bt, VecMaskUseLoad)) {
2846 log_if_needed(" ** not supported: opc=%d vlen=%d etype=%s ismask=useload",
2847 opc, num_elem, type2name(elem_bt));
2848 return false; // not supported
2849 }
2850
2851 Node* opd1 = nullptr;
2852 const TypeInstPtr* vbox_type = nullptr;
2853 if (opc != Op_CompressM) {
2854 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2855 vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2856 opd1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
2857 if (opd1 == nullptr) {
2858 log_if_needed(" ** unbox failed vector=%s",
2859 NodeClassNames[argument(5)->Opcode()]);
2860 return false;
2861 }
2862 }
2863
2864 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
2865 assert(is_vector_mask(mbox_klass), "argument(6) should be a mask class");
2866 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
2867
2868 Node* mask = unbox_vector(argument(6), mbox_type, elem_bt, num_elem);
2869 if (mask == nullptr) {
2870 log_if_needed(" ** unbox failed mask=%s",
2871 NodeClassNames[argument(6)->Opcode()]);
2872 return false;
2873 }
2874
2875 const TypeVect* vt = TypeVect::make(elem_bt, num_elem, opc == Op_CompressM);
2876 Node* operation = gvn().transform(VectorNode::make(opc, opd1, mask, vt));
2877
2878 // Wrap it up in VectorBox to keep object type information.
2879 const TypeInstPtr* box_type = opc == Op_CompressM ? mbox_type : vbox_type;
2880 Node* vbox = box_vector(operation, box_type, elem_bt, num_elem);
2881 set_result(vbox);
2882 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2883 return true;
2884 }
2885
2886 // public static
2887 // <V extends Vector<E>,
2888 // E,
2889 // S extends VectorSpecies<E>>
2890 // V indexVector(Class<? extends V> vClass, Class<E> eClass,
2891 // int length,
2892 // V v, int step, S s,
2893 // IndexOperation<V, S> defaultImpl)
2894 bool LibraryCallKit::inline_index_vector() {
2895 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
2896 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
2897 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
2898
2899 if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr ||
2900 vector_klass->const_oop() == nullptr || !vlen->is_con() ||
2901 elem_klass->const_oop() == nullptr) {
2902 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s",
2903 NodeClassNames[argument(0)->Opcode()],
2904 NodeClassNames[argument(1)->Opcode()],
2905 NodeClassNames[argument(2)->Opcode()]);
2906 return false; // not enough info for intrinsification
2907 }
2908
2909 if (!is_klass_initialized(vector_klass)) {
2910 log_if_needed(" ** klass argument not initialized");
2911 return false;
2912 }
2913
2914 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2915 if (!elem_type->is_primitive_type()) {
2916 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2917 return false; // should be primitive type
2918 }
2919
2920 int num_elem = vlen->get_con();
2921 BasicType elem_bt = elem_type->basic_type();
2922
2923 // Check whether the iota index generation op is supported by the current hardware
2924 if (!arch_supports_vector(Op_VectorLoadConst, num_elem, elem_bt, VecMaskNotUsed)) {
2925 log_if_needed(" ** not supported: vlen=%d etype=%s", num_elem, type2name(elem_bt));
2926 return false; // not supported
2927 }
2928
2929 int mul_op = VectorSupport::vop2ideal(VectorSupport::VECTOR_OP_MUL, elem_bt);
2930 int vmul_op = VectorNode::opcode(mul_op, elem_bt);
2931 bool needs_mul = true;
2932 Node* scale = argument(4);
2933 const TypeInt* scale_type = gvn().type(scale)->isa_int();
2934 // Multiply is not needed if the scale is a constant "1".
2935 if (scale_type && scale_type->is_con() && scale_type->get_con() == 1) {
2936 needs_mul = false;
2937 } else {
2938 // Check whether the vector multiply op is supported by the current hardware
2939 if (!arch_supports_vector(vmul_op, num_elem, elem_bt, VecMaskNotUsed)) {
2940 log_if_needed(" ** not supported: vlen=%d etype=%s", num_elem, type2name(elem_bt));
2941 return false; // not supported
2942 }
2943
2944 // Check whether the scalar cast op is supported by the current hardware
2945 if (is_floating_point_type(elem_bt) || elem_bt == T_LONG) {
2946 int cast_op = elem_bt == T_LONG ? Op_ConvI2L :
2947 elem_bt == T_FLOAT? Op_ConvI2F : Op_ConvI2D;
2948 if (!Matcher::match_rule_supported(cast_op)) {
2949 log_if_needed(" ** Rejected op (%s) because architecture does not support it",
2950 NodeClassNames[cast_op]);
2951 return false; // not supported
2952 }
2953 }
2954 }
2955
2956 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2957 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2958 Node* opd = unbox_vector(argument(3), vbox_type, elem_bt, num_elem);
2959 if (opd == nullptr) {
2960 log_if_needed(" ** unbox failed vector=%s",
2961 NodeClassNames[argument(3)->Opcode()]);
2962 return false;
2963 }
2964
2965 int add_op = VectorSupport::vop2ideal(VectorSupport::VECTOR_OP_ADD, elem_bt);
2966 int vadd_op = VectorNode::opcode(add_op, elem_bt);
2967 bool needs_add = true;
2968 // The addition is not needed if all the element values of "opd" are zero
2969 if (VectorNode::is_all_zeros_vector(opd)) {
2970 needs_add = false;
2971 } else {
2972 // Check whether the vector addition op is supported by the current hardware
2973 if (!arch_supports_vector(vadd_op, num_elem, elem_bt, VecMaskNotUsed)) {
2974 log_if_needed(" ** not supported: vlen=%d etype=%s", num_elem, type2name(elem_bt));
2975 return false; // not supported
2976 }
2977 }
2978
2979 // Compute the iota indice vector
2980 const TypeVect* vt = TypeVect::make(elem_bt, num_elem);
2981 Node* index = gvn().transform(new VectorLoadConstNode(gvn().makecon(TypeInt::ZERO), vt));
2982
2983 // Broadcast the "scale" to a vector, and multiply the "scale" with iota indice vector.
2984 if (needs_mul) {
2985 switch (elem_bt) {
2986 case T_BOOLEAN: // fall-through
2987 case T_BYTE: // fall-through
2988 case T_SHORT: // fall-through
2989 case T_CHAR: // fall-through
2990 case T_INT: {
2991 // no conversion needed
2992 break;
2993 }
2994 case T_LONG: {
2995 scale = gvn().transform(new ConvI2LNode(scale));
2996 break;
2997 }
2998 case T_FLOAT: {
2999 scale = gvn().transform(new ConvI2FNode(scale));
3000 break;
3001 }
3002 case T_DOUBLE: {
3003 scale = gvn().transform(new ConvI2DNode(scale));
3004 break;
3005 }
3006 default: fatal("%s", type2name(elem_bt));
3007 }
3008 scale = gvn().transform(VectorNode::scalar2vector(scale, num_elem, elem_bt));
3009 index = gvn().transform(VectorNode::make(vmul_op, index, scale, vt));
3010 }
3011
3012 // Add "opd" if addition is needed.
3013 if (needs_add) {
3014 index = gvn().transform(VectorNode::make(vadd_op, opd, index, vt));
3015 }
3016 Node* vbox = box_vector(index, vbox_type, elem_bt, num_elem);
3017 set_result(vbox);
3018 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
3019 return true;
3020 }
3021
3022 // public static
3023 // <E,
3024 // M extends VectorMask<E>>
3025 // M indexPartiallyInUpperRange(Class<? extends M> mClass, Class<E> eClass, int length,
3026 // long offset, long limit,
3027 // IndexPartiallyInUpperRangeOperation<E, M> defaultImpl)
3028 bool LibraryCallKit::inline_index_partially_in_upper_range() {
3029 const TypeInstPtr* mask_klass = gvn().type(argument(0))->isa_instptr();
3030 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
3031 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
3032
3033 if (mask_klass == nullptr || elem_klass == nullptr || vlen == nullptr ||
3034 mask_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
3035 log_if_needed(" ** missing constant: mclass=%s etype=%s vlen=%s",
3036 NodeClassNames[argument(0)->Opcode()],
3037 NodeClassNames[argument(1)->Opcode()],
3038 NodeClassNames[argument(2)->Opcode()]);
3039 return false; // not enough info for intrinsification
3040 }
3041
3042 if (!is_klass_initialized(mask_klass)) {
3043 log_if_needed(" ** klass argument not initialized");
3044 return false;
3045 }
3046
3047 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
3048 if (!elem_type->is_primitive_type()) {
3049 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
3050 return false; // should be primitive type
3051 }
3052
3053 int num_elem = vlen->get_con();
3054 BasicType elem_bt = elem_type->basic_type();
3055
3056 // Check whether the necessary ops are supported by current hardware.
3057 bool supports_mask_gen = arch_supports_vector(Op_VectorMaskGen, num_elem, elem_bt, VecMaskUseStore);
3058 if (!supports_mask_gen) {
3059 if (!arch_supports_vector(Op_VectorLoadConst, num_elem, elem_bt, VecMaskNotUsed) ||
3060 !arch_supports_vector(Op_Replicate, num_elem, elem_bt, VecMaskNotUsed) ||
3061 !arch_supports_vector(Op_VectorMaskCmp, num_elem, elem_bt, VecMaskUseStore)) {
3062 log_if_needed(" ** not supported: vlen=%d etype=%s", num_elem, type2name(elem_bt));
3063 return false; // not supported
3064 }
3065
3066 // Check whether the scalar cast operation is supported by current hardware.
3067 if (elem_bt != T_LONG) {
3068 int cast_op = is_integral_type(elem_bt) ? Op_ConvL2I
3069 : (elem_bt == T_FLOAT ? Op_ConvL2F : Op_ConvL2D);
3070 if (!Matcher::match_rule_supported(cast_op)) {
3071 log_if_needed(" ** Rejected op (%s) because architecture does not support it",
3072 NodeClassNames[cast_op]);
3073 return false; // not supported
3074 }
3075 }
3076 }
3077
3078 Node* offset = argument(3);
3079 Node* limit = argument(5);
3080 if (offset == nullptr || limit == nullptr) {
3081 log_if_needed(" ** offset or limit argument is null");
3082 return false; // not supported
3083 }
3084
3085 ciKlass* box_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
3086 assert(is_vector_mask(box_klass), "argument(0) should be a mask class");
3087 const TypeInstPtr* box_type = TypeInstPtr::make_exact(TypePtr::NotNull, box_klass);
3088
3089 // We assume "offset > 0 && limit >= offset && limit - offset < num_elem".
3090 // So directly get indexLimit with "indexLimit = limit - offset".
3091 Node* indexLimit = gvn().transform(new SubLNode(limit, offset));
3092 Node* mask = nullptr;
3093 if (supports_mask_gen) {
3094 mask = gvn().transform(VectorMaskGenNode::make(indexLimit, elem_bt, num_elem));
3095 } else {
3096 // Generate the vector mask based on "mask = iota < indexLimit".
3097 // Broadcast "indexLimit" to a vector.
3098 switch (elem_bt) {
3099 case T_BOOLEAN: // fall-through
3100 case T_BYTE: // fall-through
3101 case T_SHORT: // fall-through
3102 case T_CHAR: // fall-through
3103 case T_INT: {
3104 indexLimit = gvn().transform(new ConvL2INode(indexLimit));
3105 break;
3106 }
3107 case T_DOUBLE: {
3108 indexLimit = gvn().transform(new ConvL2DNode(indexLimit));
3109 break;
3110 }
3111 case T_FLOAT: {
3112 indexLimit = gvn().transform(new ConvL2FNode(indexLimit));
3113 break;
3114 }
3115 case T_LONG: {
3116 // no conversion needed
3117 break;
3118 }
3119 default: fatal("%s", type2name(elem_bt));
3120 }
3121 indexLimit = gvn().transform(VectorNode::scalar2vector(indexLimit, num_elem, elem_bt));
3122
3123 // Load the "iota" vector.
3124 const TypeVect* vt = TypeVect::make(elem_bt, num_elem);
3125 Node* iota = gvn().transform(new VectorLoadConstNode(gvn().makecon(TypeInt::ZERO), vt));
3126
3127 // Compute the vector mask with "mask = iota < indexLimit".
3128 ConINode* pred_node = (ConINode*)gvn().makecon(TypeInt::make(BoolTest::lt));
3129 const TypeVect* vmask_type = TypeVect::makemask(elem_bt, num_elem);
3130 mask = gvn().transform(new VectorMaskCmpNode(BoolTest::lt, iota, indexLimit, pred_node, vmask_type));
3131 }
3132 Node* vbox = box_vector(mask, box_type, elem_bt, num_elem);
3133 set_result(vbox);
3134 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
3135 return true;
3136 }
3137
3138 #undef non_product_log_if_needed
3139 #undef log_if_needed