1 /*
2 * Copyright (c) 2020, 2025, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "ci/ciSymbols.hpp"
26 #include "classfile/vmSymbols.hpp"
27 #include "opto/library_call.hpp"
28 #include "opto/runtime.hpp"
29 #include "opto/vectornode.hpp"
30 #include "prims/vectorSupport.hpp"
31 #include "runtime/stubRoutines.hpp"
32
33 #ifdef ASSERT
34 static bool is_vector(ciKlass* klass) {
35 return klass->is_subclass_of(ciEnv::current()->vector_VectorPayload_klass());
36 }
37
38 static bool check_vbox(const TypeInstPtr* vbox_type) {
39 assert(vbox_type->klass_is_exact(), "");
40
41 ciInstanceKlass* ik = vbox_type->instance_klass();
42 assert(is_vector(ik), "not a vector");
43
44 ciField* fd1 = ik->get_field_by_name(ciSymbols::ETYPE_name(), ciSymbols::class_signature(), /* is_static */ true);
45 assert(fd1 != nullptr, "element type info is missing");
46
47 ciConstant val1 = fd1->constant_value();
48 BasicType elem_bt = val1.as_object()->as_instance()->java_mirror_type()->basic_type();
49 assert(is_java_primitive(elem_bt), "element type info is missing");
50
51 ciField* fd2 = ik->get_field_by_name(ciSymbols::VLENGTH_name(), ciSymbols::int_signature(), /* is_static */ true);
52 assert(fd2 != nullptr, "vector length info is missing");
53
54 ciConstant val2 = fd2->constant_value();
55 assert(val2.as_int() > 0, "vector length info is missing");
56
57 return true;
58 }
59 #endif
60
61 #define log_if_needed(...) \
62 if (C->print_intrinsics()) { \
63 tty->print_cr(__VA_ARGS__); \
64 }
65
66 #ifndef PRODUCT
67 #define non_product_log_if_needed(...) log_if_needed(__VA_ARGS__)
68 #else
69 #define non_product_log_if_needed(...)
70 #endif
71
72 static bool is_vector_mask(ciKlass* klass) {
73 return klass->is_subclass_of(ciEnv::current()->vector_VectorMask_klass());
74 }
75
76 bool LibraryCallKit::arch_supports_vector_rotate(int opc, int num_elem, BasicType elem_bt,
77 VectorMaskUseType mask_use_type, bool has_scalar_args) {
78 bool is_supported = true;
79
80 // has_scalar_args flag is true only for non-constant scalar shift count,
81 // since in this case shift needs to be broadcasted.
82 if (!Matcher::match_rule_supported_vector(opc, num_elem, elem_bt) ||
83 (has_scalar_args && !arch_supports_vector(Op_Replicate, num_elem, elem_bt, VecMaskNotUsed))) {
84 is_supported = false;
85 }
86
87 if (is_supported) {
88 // Check if mask unboxing is supported, this is a two step process which first loads the contents
89 // of boolean array into vector followed by either lane expansion to match the lane size of masked
90 // vector operation or populate the predicate register.
91 if ((mask_use_type & VecMaskUseLoad) != 0) {
92 if (!Matcher::match_rule_supported_vector(Op_VectorLoadMask, num_elem, elem_bt) ||
93 !Matcher::match_rule_supported_vector(Op_LoadVector, num_elem, T_BOOLEAN)) {
94 non_product_log_if_needed(" ** Rejected vector mask loading (%s,%s,%d) because architecture does not support it",
95 NodeClassNames[Op_VectorLoadMask], type2name(elem_bt), num_elem);
96 return false;
97 }
98 }
99
100 if ((mask_use_type & VecMaskUsePred) != 0) {
101 if (!Matcher::has_predicated_vectors() ||
102 !Matcher::match_rule_supported_vector_masked(opc, num_elem, elem_bt)) {
103 non_product_log_if_needed("Rejected vector mask predicate using (%s,%s,%d) because architecture does not support it",
104 NodeClassNames[opc], type2name(elem_bt), num_elem);
105 return false;
106 }
107 }
108 }
109
110 int lshiftopc, rshiftopc;
111 switch(elem_bt) {
112 case T_BYTE:
113 lshiftopc = Op_LShiftI;
114 rshiftopc = Op_URShiftB;
115 break;
116 case T_SHORT:
117 lshiftopc = Op_LShiftI;
118 rshiftopc = Op_URShiftS;
119 break;
120 case T_INT:
121 lshiftopc = Op_LShiftI;
122 rshiftopc = Op_URShiftI;
123 break;
124 case T_LONG:
125 lshiftopc = Op_LShiftL;
126 rshiftopc = Op_URShiftL;
127 break;
128 default: fatal("Unexpected type: %s", type2name(elem_bt));
129 }
130 int lshiftvopc = VectorNode::opcode(lshiftopc, elem_bt);
131 int rshiftvopc = VectorNode::opcode(rshiftopc, elem_bt);
132 if (!is_supported &&
133 arch_supports_vector(lshiftvopc, num_elem, elem_bt, VecMaskNotUsed, has_scalar_args) &&
134 arch_supports_vector(rshiftvopc, num_elem, elem_bt, VecMaskNotUsed, has_scalar_args) &&
135 arch_supports_vector(Op_OrV, num_elem, elem_bt, VecMaskNotUsed)) {
136 is_supported = true;
137 }
138 return is_supported;
139 }
140
141 Node* GraphKit::box_vector(Node* vector, const TypeInstPtr* vbox_type, BasicType elem_bt, int num_elem, bool deoptimize_on_exception) {
142 assert(EnableVectorSupport, "");
143
144 PreserveReexecuteState preexecs(this);
145 jvms()->set_should_reexecute(true);
146
147 VectorBoxAllocateNode* alloc = new VectorBoxAllocateNode(C, vbox_type);
148 set_edges_for_java_call(alloc, /*must_throw=*/false, /*separate_io_proj=*/true);
149 make_slow_call_ex(alloc, env()->Throwable_klass(), /*separate_io_proj=*/true, deoptimize_on_exception);
150 set_i_o(gvn().transform( new ProjNode(alloc, TypeFunc::I_O) ));
151 set_all_memory(gvn().transform( new ProjNode(alloc, TypeFunc::Memory) ));
152 Node* ret = gvn().transform(new ProjNode(alloc, TypeFunc::Parms));
153
154 assert(check_vbox(vbox_type), "");
155 const TypeVect* vt = TypeVect::make(elem_bt, num_elem, is_vector_mask(vbox_type->instance_klass()));
156 VectorBoxNode* vbox = new VectorBoxNode(C, ret, vector, vbox_type, vt);
157 return gvn().transform(vbox);
158 }
159
160 Node* GraphKit::unbox_vector(Node* v, const TypeInstPtr* vbox_type, BasicType elem_bt, int num_elem) {
161 assert(EnableVectorSupport, "");
162 const TypeInstPtr* vbox_type_v = gvn().type(v)->isa_instptr();
163 if (vbox_type_v == nullptr || vbox_type->instance_klass() != vbox_type_v->instance_klass()) {
164 return nullptr; // arguments don't agree on vector shapes
165 }
166 if (vbox_type_v->maybe_null()) {
167 return nullptr; // no nulls are allowed
168 }
169 assert(check_vbox(vbox_type), "");
170 const TypeVect* vt = TypeVect::make(elem_bt, num_elem, is_vector_mask(vbox_type->instance_klass()));
171 Node* unbox = gvn().transform(new VectorUnboxNode(C, vt, v, merged_memory()));
172 return unbox;
173 }
174
175 Node* GraphKit::vector_shift_count(Node* cnt, int shift_op, BasicType bt, int num_elem) {
176 assert(bt == T_INT || bt == T_LONG || bt == T_SHORT || bt == T_BYTE, "byte, short, long and int are supported");
177 juint mask = (type2aelembytes(bt) * BitsPerByte - 1);
178 Node* nmask = gvn().transform(ConNode::make(TypeInt::make(mask)));
179 Node* mcnt = gvn().transform(new AndINode(cnt, nmask));
180 return gvn().transform(VectorNode::shift_count(shift_op, mcnt, num_elem, bt));
181 }
182
183 bool LibraryCallKit::arch_supports_vector(int sopc, int num_elem, BasicType type, VectorMaskUseType mask_use_type, bool has_scalar_args) {
184 // Check that the operation is valid.
185 if (sopc <= 0) {
186 non_product_log_if_needed(" ** Rejected intrinsification because no valid vector op could be extracted");
187 return false;
188 }
189
190 if (VectorNode::is_vector_rotate(sopc)) {
191 if(!arch_supports_vector_rotate(sopc, num_elem, type, mask_use_type, has_scalar_args)) {
192 non_product_log_if_needed(" ** Rejected vector op (%s,%s,%d) because architecture does not support variable vector shifts",
193 NodeClassNames[sopc], type2name(type), num_elem);
194 return false;
195 }
196 } else if (VectorNode::is_vector_integral_negate(sopc)) {
197 if (!VectorNode::is_vector_integral_negate_supported(sopc, num_elem, type, false)) {
198 non_product_log_if_needed(" ** Rejected vector op (%s,%s,%d) because architecture does not support integral vector negate",
199 NodeClassNames[sopc], type2name(type), num_elem);
200 return false;
201 }
202 } else {
203 // Check that architecture supports this op-size-type combination.
204 if (!Matcher::match_rule_supported_vector(sopc, num_elem, type)) {
205 non_product_log_if_needed(" ** Rejected vector op (%s,%s,%d) because architecture does not support it",
206 NodeClassNames[sopc], type2name(type), num_elem);
207 return false;
208 } else {
209 assert(Matcher::match_rule_supported(sopc), "must be supported");
210 }
211 }
212
213 if (num_elem == 1) {
214 if (mask_use_type != VecMaskNotUsed) {
215 non_product_log_if_needed(" ** Rejected vector mask op (%s,%s,%d) because architecture does not support it",
216 NodeClassNames[sopc], type2name(type), num_elem);
217 return false;
218 }
219
220 if (sopc != 0) {
221 if (sopc != Op_LoadVector && sopc != Op_StoreVector) {
222 non_product_log_if_needed(" ** Not a svml call or load/store vector op (%s,%s,%d)",
223 NodeClassNames[sopc], type2name(type), num_elem);
224 return false;
225 }
226 }
227 }
228
229 if (!has_scalar_args && VectorNode::is_vector_shift(sopc) &&
230 Matcher::supports_vector_variable_shifts() == false) {
231 log_if_needed(" ** Rejected vector op (%s,%s,%d) because architecture does not support variable vector shifts",
232 NodeClassNames[sopc], type2name(type), num_elem);
233 return false;
234 }
235
236 // Check if mask unboxing is supported, this is a two step process which first loads the contents
237 // of boolean array into vector followed by either lane expansion to match the lane size of masked
238 // vector operation or populate the predicate register.
239 if ((mask_use_type & VecMaskUseLoad) != 0) {
240 if (!Matcher::match_rule_supported_vector(Op_VectorLoadMask, num_elem, type) ||
241 !Matcher::match_rule_supported_vector(Op_LoadVector, num_elem, T_BOOLEAN)) {
242 non_product_log_if_needed(" ** Rejected vector mask loading (%s,%s,%d) because architecture does not support it",
243 NodeClassNames[Op_VectorLoadMask], type2name(type), num_elem);
244 return false;
245 }
246 }
247
248 // Check if mask boxing is supported, this is a two step process which first stores the contents
249 // of mask vector / predicate register into a boolean vector followed by vector store operation to
250 // transfer the contents to underlined storage of mask boxes which is a boolean array.
251 if ((mask_use_type & VecMaskUseStore) != 0) {
252 if (!Matcher::match_rule_supported_vector(Op_VectorStoreMask, num_elem, type) ||
253 !Matcher::match_rule_supported_vector(Op_StoreVector, num_elem, T_BOOLEAN)) {
254 non_product_log_if_needed("Rejected vector mask storing (%s,%s,%d) because architecture does not support it",
255 NodeClassNames[Op_VectorStoreMask], type2name(type), num_elem);
256 return false;
257 }
258 }
259
260 if ((mask_use_type & VecMaskUsePred) != 0) {
261 bool is_supported = false;
262 if (Matcher::has_predicated_vectors()) {
263 if (VectorNode::is_vector_integral_negate(sopc)) {
264 is_supported = VectorNode::is_vector_integral_negate_supported(sopc, num_elem, type, true);
265 } else {
266 is_supported = Matcher::match_rule_supported_vector_masked(sopc, num_elem, type);
267 }
268 }
269 is_supported |= Matcher::supports_vector_predicate_op_emulation(sopc, num_elem, type);
270
271 if (!is_supported) {
272 non_product_log_if_needed("Rejected vector mask predicate using (%s,%s,%d) because architecture does not support it",
273 NodeClassNames[sopc], type2name(type), num_elem);
274 return false;
275 }
276 }
277
278 return true;
279 }
280
281 static bool is_klass_initialized(const TypeInstPtr* vec_klass) {
282 if (vec_klass->const_oop() == nullptr) {
283 return false; // uninitialized or some kind of unsafe access
284 }
285 assert(vec_klass->const_oop()->as_instance()->java_lang_Class_klass() != nullptr, "klass instance expected");
286 ciInstanceKlass* klass = vec_klass->const_oop()->as_instance()->java_lang_Class_klass()->as_instance_klass();
287 return klass->is_initialized();
288 }
289
290 // public static
291 // <V extends Vector<E>,
292 // M extends VectorMask<E>,
293 // E>
294 // V unaryOp(int oprId, Class<? extends V> vmClass, Class<? extends M> maskClass, Class<E> elementType,
295 // int length, V v, M m,
296 // UnaryOperation<V, M> defaultImpl)
297 //
298 // public static
299 // <V,
300 // M extends VectorMask<E>,
301 // E>
302 // V binaryOp(int oprId, Class<? extends V> vmClass, Class<? extends M> maskClass, Class<E> elementType,
303 // int length, V v1, V v2, M m,
304 // BinaryOperation<V, M> defaultImpl)
305 //
306 // public static
307 // <V extends Vector<E>,
308 // M extends VectorMask<E>,
309 // E>
310 // V ternaryOp(int oprId, Class<? extends V> vmClass, Class<? extends M> maskClass, Class<E> elementType,
311 // int length, V v1, V v2, V v3, M m,
312 // TernaryOperation<V, M> defaultImpl)
313 //
314 bool LibraryCallKit::inline_vector_nary_operation(int n) {
315 const TypeInt* opr = gvn().type(argument(0))->isa_int();
316 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
317 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
318 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
319 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
320
321 if (opr == nullptr || vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr ||
322 !opr->is_con() || vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
323 log_if_needed(" ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
324 NodeClassNames[argument(0)->Opcode()],
325 NodeClassNames[argument(1)->Opcode()],
326 NodeClassNames[argument(3)->Opcode()],
327 NodeClassNames[argument(4)->Opcode()]);
328 return false; // not enough info for intrinsification
329 }
330
331 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
332 if (!elem_type->is_primitive_type()) {
333 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
334 return false; // should be primitive type
335 }
336 if (!is_klass_initialized(vector_klass)) {
337 log_if_needed(" ** klass argument not initialized");
338 return false;
339 }
340
341 // "argument(n + 5)" should be the mask object. We assume it is "null" when no mask
342 // is used to control this operation.
343 const Type* vmask_type = gvn().type(argument(n + 5));
344 bool is_masked_op = vmask_type != TypePtr::NULL_PTR;
345 if (is_masked_op) {
346 if (mask_klass == nullptr || mask_klass->const_oop() == nullptr) {
347 log_if_needed(" ** missing constant: maskclass=%s", NodeClassNames[argument(2)->Opcode()]);
348 return false; // not enough info for intrinsification
349 }
350
351 if (!is_klass_initialized(mask_klass)) {
352 log_if_needed(" ** mask klass argument not initialized");
353 return false;
354 }
355
356 if (vmask_type->maybe_null()) {
357 log_if_needed(" ** null mask values are not allowed for masked op");
358 return false;
359 }
360 }
361
362 BasicType elem_bt = elem_type->basic_type();
363 bool has_scalar_op = VectorSupport::has_scalar_op(opr->get_con());
364 bool is_unsigned = VectorSupport::is_unsigned_op(opr->get_con());
365
366 int num_elem = vlen->get_con();
367 int opc = VectorSupport::vop2ideal(opr->get_con(), elem_bt);
368 int sopc = has_scalar_op ? VectorNode::opcode(opc, elem_bt) : opc;
369 if ((opc != Op_CallLeafVector) && (sopc == 0)) {
370 log_if_needed(" ** operation not supported: opc=%s bt=%s", NodeClassNames[opc], type2name(elem_bt));
371 return false; // operation not supported
372 }
373 if (num_elem == 1) {
374 if (opc != Op_CallLeafVector || elem_bt != T_DOUBLE) {
375 log_if_needed(" ** not a svml call: arity=%d opc=%d vlen=%d etype=%s",
376 n, opc, num_elem, type2name(elem_bt));
377 return false;
378 }
379 }
380 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
381 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
382
383 if (is_vector_mask(vbox_klass)) {
384 assert(!is_masked_op, "mask operations do not need mask to control");
385 }
386
387 if (opc == Op_CallLeafVector) {
388 if (!UseVectorStubs) {
389 log_if_needed(" ** vector stubs support is disabled");
390 return false;
391 }
392 if (!Matcher::supports_vector_calling_convention()) {
393 log_if_needed(" ** no vector calling conventions supported");
394 return false;
395 }
396 if (!Matcher::vector_size_supported(elem_bt, num_elem)) {
397 log_if_needed(" ** vector size (vlen=%d, etype=%s) is not supported",
398 num_elem, type2name(elem_bt));
399 return false;
400 }
401 }
402
403 // When using mask, mask use type needs to be VecMaskUseLoad.
404 VectorMaskUseType mask_use_type = is_vector_mask(vbox_klass) ? VecMaskUseAll
405 : is_masked_op ? VecMaskUseLoad : VecMaskNotUsed;
406 if ((sopc != 0) && !arch_supports_vector(sopc, num_elem, elem_bt, mask_use_type)) {
407 log_if_needed(" ** not supported: arity=%d opc=%d vlen=%d etype=%s ismask=%d is_masked_op=%d",
408 n, sopc, num_elem, type2name(elem_bt),
409 is_vector_mask(vbox_klass) ? 1 : 0, is_masked_op ? 1 : 0);
410 return false; // not supported
411 }
412
413 // Return true if current platform has implemented the masked operation with predicate feature.
414 bool use_predicate = is_masked_op && sopc != 0 && arch_supports_vector(sopc, num_elem, elem_bt, VecMaskUsePred);
415 if (is_masked_op && !use_predicate && !arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad)) {
416 log_if_needed(" ** not supported: arity=%d opc=%d vlen=%d etype=%s ismask=0 is_masked_op=1",
417 n, sopc, num_elem, type2name(elem_bt));
418 return false;
419 }
420
421 Node* opd1 = nullptr; Node* opd2 = nullptr; Node* opd3 = nullptr;
422 switch (n) {
423 case 3: {
424 opd3 = unbox_vector(argument(7), vbox_type, elem_bt, num_elem);
425 if (opd3 == nullptr) {
426 log_if_needed(" ** unbox failed v3=%s",
427 NodeClassNames[argument(7)->Opcode()]);
428 return false;
429 }
430 // fall-through
431 }
432 case 2: {
433 opd2 = unbox_vector(argument(6), vbox_type, elem_bt, num_elem);
434 if (opd2 == nullptr) {
435 log_if_needed(" ** unbox failed v2=%s",
436 NodeClassNames[argument(6)->Opcode()]);
437 return false;
438 }
439 // fall-through
440 }
441 case 1: {
442 opd1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
443 if (opd1 == nullptr) {
444 log_if_needed(" ** unbox failed v1=%s",
445 NodeClassNames[argument(5)->Opcode()]);
446 return false;
447 }
448 break;
449 }
450 default: fatal("unsupported arity: %d", n);
451 }
452
453 Node* mask = nullptr;
454 if (is_masked_op) {
455 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
456 assert(is_vector_mask(mbox_klass), "argument(2) should be a mask class");
457 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
458 mask = unbox_vector(argument(n + 5), mbox_type, elem_bt, num_elem);
459 if (mask == nullptr) {
460 log_if_needed(" ** unbox failed mask=%s",
461 NodeClassNames[argument(n + 5)->Opcode()]);
462 return false;
463 }
464 }
465
466 Node* operation = nullptr;
467 if (opc == Op_CallLeafVector) {
468 assert(UseVectorStubs, "sanity");
469 operation = gen_call_to_vector_math(opr->get_con(), elem_bt, num_elem, opd1, opd2);
470 if (operation == nullptr) {
471 log_if_needed(" ** Vector math call failed for %s_%s_%d",
472 (elem_bt == T_FLOAT) ? "float" : "double",
473 VectorSupport::mathname[opr->get_con() - VectorSupport::VECTOR_OP_MATH_START],
474 num_elem * type2aelembytes(elem_bt));
475 return false;
476 }
477 } else {
478 const TypeVect* vt = TypeVect::make(elem_bt, num_elem, is_vector_mask(vbox_klass));
479 switch (n) {
480 case 1:
481 case 2: {
482 operation = VectorNode::make(sopc, opd1, opd2, vt, is_vector_mask(vbox_klass), VectorNode::is_shift_opcode(opc), is_unsigned);
483 break;
484 }
485 case 3: {
486 operation = VectorNode::make(sopc, opd1, opd2, opd3, vt);
487 break;
488 }
489 default: fatal("unsupported arity: %d", n);
490 }
491 }
492
493 if (is_masked_op && mask != nullptr) {
494 if (use_predicate) {
495 operation->add_req(mask);
496 operation->add_flag(Node::Flag_is_predicated_vector);
497 } else {
498 operation->add_flag(Node::Flag_is_predicated_using_blend);
499 operation = gvn().transform(operation);
500 operation = new VectorBlendNode(opd1, operation, mask);
501 }
502 }
503 operation = gvn().transform(operation);
504
505 // Wrap it up in VectorBox to keep object type information.
506 Node* vbox = box_vector(operation, vbox_type, elem_bt, num_elem);
507 set_result(vbox);
508 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
509 return true;
510 }
511
512 // public static
513 // <E, M>
514 // long maskReductionCoerced(int oper, Class<? extends M> maskClass, Class<?> elemClass,
515 // int length, M m, VectorMaskOp<M> defaultImpl)
516 bool LibraryCallKit::inline_vector_mask_operation() {
517 const TypeInt* oper = gvn().type(argument(0))->isa_int();
518 const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
519 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
520 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
521 Node* mask = argument(4);
522
523 if (mask_klass == nullptr || elem_klass == nullptr || mask->is_top() || vlen == nullptr) {
524 return false; // dead code
525 }
526
527 if (!is_klass_initialized(mask_klass)) {
528 log_if_needed(" ** klass argument not initialized");
529 return false;
530 }
531
532 int num_elem = vlen->get_con();
533 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
534 BasicType elem_bt = elem_type->basic_type();
535
536 int mopc = VectorSupport::vop2ideal(oper->get_con(), elem_bt);
537 if (!arch_supports_vector(mopc, num_elem, elem_bt, VecMaskUseLoad)) {
538 log_if_needed(" ** not supported: arity=1 op=cast#%d/3 vlen2=%d etype2=%s",
539 mopc, num_elem, type2name(elem_bt));
540 return false; // not supported
541 }
542
543 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
544 const TypeInstPtr* mask_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
545 Node* mask_vec = unbox_vector(mask, mask_box_type, elem_bt, num_elem);
546 if (mask_vec == nullptr) {
547 log_if_needed(" ** unbox failed mask=%s",
548 NodeClassNames[argument(4)->Opcode()]);
549 return false;
550 }
551
552 if (mask_vec->bottom_type()->isa_vectmask() == nullptr) {
553 mask_vec = gvn().transform(VectorStoreMaskNode::make(gvn(), mask_vec, elem_bt, num_elem));
554 }
555 const Type* maskoper_ty = mopc == Op_VectorMaskToLong ? (const Type*)TypeLong::LONG : (const Type*)TypeInt::INT;
556 Node* maskoper = gvn().transform(VectorMaskOpNode::make(mask_vec, maskoper_ty, mopc));
557 if (mopc != Op_VectorMaskToLong) {
558 maskoper = ConvI2L(maskoper);
559 }
560 set_result(maskoper);
561
562 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
563 return true;
564 }
565
566 // public static
567 // <M,
568 // S extends VectorSpecies<E>,
569 // E>
570 // M fromBitsCoerced(Class<? extends M> vmClass, Class<E> elementType, int length,
571 // long bits, int mode, S s,
572 // BroadcastOperation<M, E, S> defaultImpl)
573 bool LibraryCallKit::inline_vector_frombits_coerced() {
574 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
575 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
576 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
577 const TypeLong* bits_type = gvn().type(argument(3))->isa_long();
578 // Mode argument determines the mode of operation it can take following values:-
579 // MODE_BROADCAST for vector Vector.broadcast and VectorMask.maskAll operations.
580 // MODE_BITS_COERCED_LONG_TO_MASK for VectorMask.fromLong operation.
581 const TypeInt* mode = gvn().type(argument(5))->isa_int();
582
583 if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr || mode == nullptr ||
584 bits_type == nullptr || vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr ||
585 !vlen->is_con() || !mode->is_con()) {
586 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s bitwise=%s",
587 NodeClassNames[argument(0)->Opcode()],
588 NodeClassNames[argument(1)->Opcode()],
589 NodeClassNames[argument(2)->Opcode()],
590 NodeClassNames[argument(5)->Opcode()]);
591 return false; // not enough info for intrinsification
592 }
593
594 if (!is_klass_initialized(vector_klass)) {
595 log_if_needed(" ** klass argument not initialized");
596 return false;
597 }
598 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
599 if (!elem_type->is_primitive_type()) {
600 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
601 return false; // should be primitive type
602 }
603 BasicType elem_bt = elem_type->basic_type();
604 int num_elem = vlen->get_con();
605 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
606 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
607
608 bool is_mask = is_vector_mask(vbox_klass);
609 int bcast_mode = mode->get_con();
610 VectorMaskUseType checkFlags = (VectorMaskUseType)(is_mask ? VecMaskUseAll : VecMaskNotUsed);
611 int opc = bcast_mode == VectorSupport::MODE_BITS_COERCED_LONG_TO_MASK ? Op_VectorLongToMask : Op_Replicate;
612
613 if (!arch_supports_vector(opc, num_elem, elem_bt, checkFlags, true /*has_scalar_args*/)) {
614 log_if_needed(" ** not supported: arity=0 op=broadcast vlen=%d etype=%s ismask=%d bcast_mode=%d",
615 num_elem, type2name(elem_bt),
616 is_mask ? 1 : 0,
617 bcast_mode);
618 return false; // not supported
619 }
620
621 Node* broadcast = nullptr;
622 Node* bits = argument(3);
623 Node* elem = bits;
624
625 if (opc == Op_VectorLongToMask) {
626 const TypeVect* vt = TypeVect::makemask(elem_bt, num_elem);
627 if (vt->isa_vectmask()) {
628 broadcast = gvn().transform(new VectorLongToMaskNode(elem, vt));
629 } else {
630 const TypeVect* mvt = TypeVect::make(T_BOOLEAN, num_elem);
631 broadcast = gvn().transform(new VectorLongToMaskNode(elem, mvt));
632 broadcast = gvn().transform(new VectorLoadMaskNode(broadcast, vt));
633 }
634 } else {
635 switch (elem_bt) {
636 case T_BOOLEAN: // fall-through
637 case T_BYTE: // fall-through
638 case T_SHORT: // fall-through
639 case T_CHAR: // fall-through
640 case T_INT: {
641 elem = gvn().transform(new ConvL2INode(bits));
642 break;
643 }
644 case T_DOUBLE: {
645 elem = gvn().transform(new MoveL2DNode(bits));
646 break;
647 }
648 case T_FLOAT: {
649 bits = gvn().transform(new ConvL2INode(bits));
650 elem = gvn().transform(new MoveI2FNode(bits));
651 break;
652 }
653 case T_LONG: {
654 // no conversion needed
655 break;
656 }
657 default: fatal("%s", type2name(elem_bt));
658 }
659 broadcast = VectorNode::scalar2vector(elem, num_elem, elem_bt, is_mask);
660 broadcast = gvn().transform(broadcast);
661 }
662
663 Node* box = box_vector(broadcast, vbox_type, elem_bt, num_elem);
664 set_result(box);
665 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
666 return true;
667 }
668
669 static bool elem_consistent_with_arr(BasicType elem_bt, const TypeAryPtr* arr_type, bool mismatched_ms) {
670 assert(arr_type != nullptr, "unexpected");
671 BasicType arr_elem_bt = arr_type->elem()->array_element_basic_type();
672 if (elem_bt == arr_elem_bt) {
673 return true;
674 } else if (elem_bt == T_SHORT && arr_elem_bt == T_CHAR) {
675 // Load/store of short vector from/to char[] is supported
676 return true;
677 } else if (elem_bt == T_BYTE && arr_elem_bt == T_BOOLEAN) {
678 // Load/store of byte vector from/to boolean[] is supported
679 return true;
680 } else {
681 return mismatched_ms;
682 }
683 }
684
685 // public static
686 // <C,
687 // VM extends VectorPayload,
688 // E,
689 // S extends VectorSpecies<E>>
690 // VM load(Class<? extends VM> vmClass, Class<E> eClass,
691 // int length,
692 // Object base, long offset, // Unsafe addressing
693 // boolean fromSegment,
694 // C container, long index, S s, // Arguments for default implementation
695 // LoadOperation<C, VM, S> defaultImpl) {
696 // public static
697 // <C,
698 // V extends VectorPayload>
699 // void store(Class<?> vClass, Class<?> eClass,
700 // int length,
701 // Object base, long offset, // Unsafe addressing
702 // boolean fromSegment,
703 // V v, C container, long index, // Arguments for default implementation
704 // StoreVectorOperation<C, V> defaultImpl) {
705 bool LibraryCallKit::inline_vector_mem_operation(bool is_store) {
706 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
707 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
708 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
709 const TypeInt* from_ms = gvn().type(argument(6))->isa_int();
710
711 if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr || !from_ms->is_con() ||
712 vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
713 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s from_ms=%s",
714 NodeClassNames[argument(0)->Opcode()],
715 NodeClassNames[argument(1)->Opcode()],
716 NodeClassNames[argument(2)->Opcode()],
717 NodeClassNames[argument(6)->Opcode()]);
718 return false; // not enough info for intrinsification
719 }
720 if (!is_klass_initialized(vector_klass)) {
721 log_if_needed(" ** klass argument not initialized");
722 return false;
723 }
724
725 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
726 if (!elem_type->is_primitive_type()) {
727 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
728 return false; // should be primitive type
729 }
730 BasicType elem_bt = elem_type->basic_type();
731 int num_elem = vlen->get_con();
732
733 // TODO When mask usage is supported, VecMaskNotUsed needs to be VecMaskUseLoad.
734 if (!arch_supports_vector(is_store ? Op_StoreVector : Op_LoadVector, num_elem, elem_bt, VecMaskNotUsed)) {
735 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d etype=%s ismask=no",
736 is_store, is_store ? "store" : "load",
737 num_elem, type2name(elem_bt));
738 return false; // not supported
739 }
740
741 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
742 bool is_mask = is_vector_mask(vbox_klass);
743
744 Node* base = argument(3);
745 Node* offset = ConvL2X(argument(4));
746
747 // Save state and restore on bailout
748 uint old_sp = sp();
749 SafePointNode* old_map = clone_map();
750
751 Node* addr = make_unsafe_address(base, offset, (is_mask ? T_BOOLEAN : elem_bt), true);
752
753 // The memory barrier checks are based on ones for unsafe access.
754 // This is not 1-1 implementation.
755 const Type *const base_type = gvn().type(base);
756
757 const TypePtr *addr_type = gvn().type(addr)->isa_ptr();
758 const TypeAryPtr* arr_type = addr_type->isa_aryptr();
759
760 const bool in_native = TypePtr::NULL_PTR == base_type; // base always null
761 const bool in_heap = !TypePtr::NULL_PTR->higher_equal(base_type); // base never null
762
763 const bool is_mixed_access = !in_heap && !in_native;
764
765 const bool is_mismatched_access = in_heap && (addr_type->isa_aryptr() == nullptr);
766
767 const bool needs_cpu_membar = is_mixed_access || is_mismatched_access;
768
769 // For non-masked mismatched memory segment vector read/write accesses, intrinsification can continue
770 // with unknown backing storage type and compiler can skip inserting explicit reinterpretation IR after
771 // loading from or before storing to backing storage which is mandatory for semantic correctness of
772 // big-endian memory layout.
773 bool mismatched_ms = LITTLE_ENDIAN_ONLY(false)
774 BIG_ENDIAN_ONLY(from_ms->get_con() && !is_mask && arr_type != nullptr &&
775 arr_type->elem()->array_element_basic_type() != elem_bt);
776 BasicType mem_elem_bt = mismatched_ms ? arr_type->elem()->array_element_basic_type() : elem_bt;
777 if (!is_java_primitive(mem_elem_bt)) {
778 log_if_needed(" ** non-primitive array element type");
779 return false;
780 }
781 int mem_num_elem = mismatched_ms ? (num_elem * type2aelembytes(elem_bt)) / type2aelembytes(mem_elem_bt) : num_elem;
782 if (arr_type != nullptr && !is_mask && !elem_consistent_with_arr(elem_bt, arr_type, mismatched_ms)) {
783 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d etype=%s atype=%s ismask=no",
784 is_store, is_store ? "store" : "load",
785 num_elem, type2name(elem_bt), type2name(arr_type->elem()->array_element_basic_type()));
786 set_map(old_map);
787 set_sp(old_sp);
788 return false;
789 }
790
791 // In case of mismatched memory segment accesses, we need to double check that the source type memory operations are supported by backend.
792 if (mismatched_ms) {
793 if (is_store) {
794 if (!arch_supports_vector(Op_StoreVector, num_elem, elem_bt, VecMaskNotUsed)
795 || !arch_supports_vector(Op_VectorReinterpret, mem_num_elem, mem_elem_bt, VecMaskNotUsed)) {
796 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d*8 etype=%s/8 ismask=no",
797 is_store, "store",
798 num_elem, type2name(elem_bt));
799 set_map(old_map);
800 set_sp(old_sp);
801 return false; // not supported
802 }
803 } else {
804 if (!arch_supports_vector(Op_LoadVector, mem_num_elem, mem_elem_bt, VecMaskNotUsed)
805 || !arch_supports_vector(Op_VectorReinterpret, num_elem, elem_bt, VecMaskNotUsed)) {
806 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d*8 etype=%s/8 ismask=no",
807 is_store, "load",
808 mem_num_elem, type2name(mem_elem_bt));
809 set_map(old_map);
810 set_sp(old_sp);
811 return false; // not supported
812 }
813 }
814 }
815 if (is_mask) {
816 if (!is_store) {
817 if (!arch_supports_vector(Op_LoadVector, num_elem, elem_bt, VecMaskUseLoad)) {
818 set_map(old_map);
819 set_sp(old_sp);
820 return false; // not supported
821 }
822 } else {
823 if (!arch_supports_vector(Op_StoreVector, num_elem, elem_bt, VecMaskUseStore)) {
824 set_map(old_map);
825 set_sp(old_sp);
826 return false; // not supported
827 }
828 }
829 }
830
831 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
832
833 if (needs_cpu_membar) {
834 insert_mem_bar(Op_MemBarCPUOrder);
835 }
836
837 if (is_store) {
838 Node* val = unbox_vector(argument(7), vbox_type, elem_bt, num_elem);
839 if (val == nullptr) {
840 set_map(old_map);
841 set_sp(old_sp);
842 return false; // operand unboxing failed
843 }
844 set_all_memory(reset_memory());
845
846 // In case the store needs to happen to byte array, reinterpret the incoming vector to byte vector.
847 int store_num_elem = num_elem;
848 if (mismatched_ms) {
849 store_num_elem = mem_num_elem;
850 const TypeVect* to_vect_type = TypeVect::make(mem_elem_bt, store_num_elem);
851 val = gvn().transform(new VectorReinterpretNode(val, val->bottom_type()->is_vect(), to_vect_type));
852 }
853 if (is_mask) {
854 val = gvn().transform(VectorStoreMaskNode::make(gvn(), val, elem_bt, num_elem));
855 }
856 Node* vstore = gvn().transform(StoreVectorNode::make(0, control(), memory(addr), addr, addr_type, val, store_num_elem));
857 set_memory(vstore, addr_type);
858 } else {
859 // When using byte array, we need to load as byte then reinterpret the value. Otherwise, do a simple vector load.
860 Node* vload = nullptr;
861 if (mismatched_ms) {
862 vload = gvn().transform(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, mem_num_elem, mem_elem_bt));
863 const TypeVect* to_vect_type = TypeVect::make(elem_bt, num_elem);
864 vload = gvn().transform(new VectorReinterpretNode(vload, vload->bottom_type()->is_vect(), to_vect_type));
865 } else {
866 // Special handle for masks
867 if (is_mask) {
868 vload = gvn().transform(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, num_elem, T_BOOLEAN));
869 vload = gvn().transform(new VectorLoadMaskNode(vload, TypeVect::makemask(elem_bt, num_elem)));
870 } else {
871 vload = gvn().transform(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, num_elem, elem_bt));
872 }
873 }
874 Node* box = box_vector(vload, vbox_type, elem_bt, num_elem);
875 set_result(box);
876 }
877
878 destruct_map_clone(old_map);
879
880 if (needs_cpu_membar) {
881 insert_mem_bar(Op_MemBarCPUOrder);
882 }
883
884 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
885 return true;
886 }
887
888 // public static
889 // <C,
890 // V extends Vector<?>,
891 // E,
892 // S extends VectorSpecies<E>,
893 // M extends VectorMask<E>>
894 // V loadMasked(Class<? extends V> vClass, Class<M> mClass, Class<E> eClass,
895 // int length, Object base, long offset, // Unsafe addressing
896 // boolean fromSegment,
897 // M m, int offsetInRange,
898 // C container, long index, S s, // Arguments for default implementation
899 // LoadVectorMaskedOperation<C, V, S, M> defaultImpl) {
900 // public static
901 // <C,
902 // V extends Vector<E>,
903 // M extends VectorMask<E>,
904 // E>
905 // void storeMasked(Class<? extends V> vClass, Class<M> mClass, Class<E> eClass,
906 // int length,
907 // Object base, long offset, // Unsafe addressing
908 // boolean fromSegment,
909 // V v, M m, C container, long index, // Arguments for default implementation
910 // StoreVectorMaskedOperation<C, V, M> defaultImpl) {
911
912 bool LibraryCallKit::inline_vector_mem_masked_operation(bool is_store) {
913 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
914 const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
915 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
916 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
917 const TypeInt* from_ms = gvn().type(argument(7))->isa_int();
918
919 if (vector_klass == nullptr || mask_klass == nullptr || elem_klass == nullptr || vlen == nullptr ||
920 vector_klass->const_oop() == nullptr || mask_klass->const_oop() == nullptr || from_ms == nullptr ||
921 elem_klass->const_oop() == nullptr || !vlen->is_con() || !from_ms->is_con()) {
922 log_if_needed(" ** missing constant: vclass=%s mclass=%s etype=%s vlen=%s from_ms=%s",
923 NodeClassNames[argument(0)->Opcode()],
924 NodeClassNames[argument(1)->Opcode()],
925 NodeClassNames[argument(2)->Opcode()],
926 NodeClassNames[argument(3)->Opcode()],
927 NodeClassNames[argument(7)->Opcode()]);
928 return false; // not enough info for intrinsification
929 }
930 if (!is_klass_initialized(vector_klass)) {
931 log_if_needed(" ** klass argument not initialized");
932 return false;
933 }
934
935 if (!is_klass_initialized(mask_klass)) {
936 log_if_needed(" ** mask klass argument not initialized");
937 return false;
938 }
939
940 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
941 if (!elem_type->is_primitive_type()) {
942 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
943 return false; // should be primitive type
944 }
945
946 BasicType elem_bt = elem_type->basic_type();
947 int num_elem = vlen->get_con();
948
949 Node* base = argument(4);
950 Node* offset = ConvL2X(argument(5));
951
952 // Save state and restore on bailout
953 uint old_sp = sp();
954 SafePointNode* old_map = clone_map();
955
956 Node* addr = make_unsafe_address(base, offset, elem_bt, true);
957 const TypePtr *addr_type = gvn().type(addr)->isa_ptr();
958 const TypeAryPtr* arr_type = addr_type->isa_aryptr();
959
960 bool mismatched_ms = from_ms->get_con() && arr_type != nullptr && arr_type->elem()->array_element_basic_type() != elem_bt;
961 BIG_ENDIAN_ONLY(if (mismatched_ms) return false;)
962 // If there is no consistency between array and vector element types, it must be special byte array case
963 if (arr_type != nullptr && !elem_consistent_with_arr(elem_bt, arr_type, mismatched_ms)) {
964 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d etype=%s atype=%s",
965 is_store, is_store ? "storeMasked" : "loadMasked",
966 num_elem, type2name(elem_bt), type2name(arr_type->elem()->array_element_basic_type()));
967 set_map(old_map);
968 set_sp(old_sp);
969 return false;
970 }
971
972 int mem_num_elem = mismatched_ms ? num_elem * type2aelembytes(elem_bt) : num_elem;
973 BasicType mem_elem_bt = mismatched_ms ? T_BYTE : elem_bt;
974 bool supports_predicate = arch_supports_vector(is_store ? Op_StoreVectorMasked : Op_LoadVectorMasked,
975 mem_num_elem, mem_elem_bt, VecMaskUseLoad);
976
977 // If current arch does not support the predicated operations, we have to bail
978 // out when current case uses the predicate feature.
979 if (!supports_predicate) {
980 bool needs_predicate = false;
981 if (is_store) {
982 // Masked vector store always uses the predicated store.
983 needs_predicate = true;
984 } else {
985 // Masked vector load with IOOBE always uses the predicated load.
986 const TypeInt* offset_in_range = gvn().type(argument(9))->isa_int();
987 if (!offset_in_range->is_con()) {
988 log_if_needed(" ** missing constant: offsetInRange=%s",
989 NodeClassNames[argument(8)->Opcode()]);
990 set_map(old_map);
991 set_sp(old_sp);
992 return false;
993 }
994 needs_predicate = (offset_in_range->get_con() == 0);
995 }
996
997 if (needs_predicate) {
998 log_if_needed(" ** not supported: op=%s vlen=%d etype=%s mismatched_ms=%d",
999 is_store ? "storeMasked" : "loadMasked",
1000 num_elem, type2name(elem_bt), mismatched_ms ? 1 : 0);
1001 set_map(old_map);
1002 set_sp(old_sp);
1003 return false;
1004 }
1005 }
1006
1007 // This only happens for masked vector load. If predicate is not supported, then check whether
1008 // the normal vector load and blend operations are supported by backend.
1009 if (!supports_predicate && (!arch_supports_vector(Op_LoadVector, mem_num_elem, mem_elem_bt, VecMaskNotUsed) ||
1010 !arch_supports_vector(Op_VectorBlend, mem_num_elem, mem_elem_bt, VecMaskUseLoad))) {
1011 log_if_needed(" ** not supported: op=loadMasked vlen=%d etype=%s mismatched_ms=%d",
1012 num_elem, type2name(elem_bt), mismatched_ms ? 1 : 0);
1013 set_map(old_map);
1014 set_sp(old_sp);
1015 return false;
1016 }
1017
1018 // Since we are using byte array, we need to double check that the vector reinterpret operation
1019 // with byte type is supported by backend.
1020 if (mismatched_ms) {
1021 if (!arch_supports_vector(Op_VectorReinterpret, mem_num_elem, T_BYTE, VecMaskNotUsed)) {
1022 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d etype=%s mismatched_ms=1",
1023 is_store, is_store ? "storeMasked" : "loadMasked",
1024 num_elem, type2name(elem_bt));
1025 set_map(old_map);
1026 set_sp(old_sp);
1027 return false;
1028 }
1029 }
1030
1031 // Since it needs to unbox the mask, we need to double check that the related load operations
1032 // for mask are supported by backend.
1033 if (!arch_supports_vector(Op_LoadVector, num_elem, elem_bt, VecMaskUseLoad)) {
1034 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d etype=%s",
1035 is_store, is_store ? "storeMasked" : "loadMasked",
1036 num_elem, type2name(elem_bt));
1037 set_map(old_map);
1038 set_sp(old_sp);
1039 return false;
1040 }
1041
1042 // Can base be null? Otherwise, always on-heap access.
1043 bool can_access_non_heap = TypePtr::NULL_PTR->higher_equal(gvn().type(base));
1044 if (can_access_non_heap) {
1045 insert_mem_bar(Op_MemBarCPUOrder);
1046 }
1047
1048 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1049 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1050 assert(!is_vector_mask(vbox_klass) && is_vector_mask(mbox_klass), "Invalid class type");
1051 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1052 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1053
1054 Node* mask = unbox_vector(is_store ? argument(9) : argument(8), mbox_type, elem_bt, num_elem);
1055 if (mask == nullptr) {
1056 log_if_needed(" ** unbox failed mask=%s",
1057 is_store ? NodeClassNames[argument(9)->Opcode()]
1058 : NodeClassNames[argument(8)->Opcode()]);
1059 set_map(old_map);
1060 set_sp(old_sp);
1061 return false;
1062 }
1063
1064 if (is_store) {
1065 Node* val = unbox_vector(argument(8), vbox_type, elem_bt, num_elem);
1066 if (val == nullptr) {
1067 log_if_needed(" ** unbox failed vector=%s",
1068 NodeClassNames[argument(8)->Opcode()]);
1069 set_map(old_map);
1070 set_sp(old_sp);
1071 return false; // operand unboxing failed
1072 }
1073 set_all_memory(reset_memory());
1074
1075 if (mismatched_ms) {
1076 // Reinterpret the incoming vector to byte vector.
1077 const TypeVect* to_vect_type = TypeVect::make(mem_elem_bt, mem_num_elem);
1078 val = gvn().transform(new VectorReinterpretNode(val, val->bottom_type()->is_vect(), to_vect_type));
1079 // Reinterpret the vector mask to byte type.
1080 const TypeVect* from_mask_type = TypeVect::makemask(elem_bt, num_elem);
1081 const TypeVect* to_mask_type = TypeVect::makemask(mem_elem_bt, mem_num_elem);
1082 mask = gvn().transform(new VectorReinterpretNode(mask, from_mask_type, to_mask_type));
1083 }
1084 Node* vstore = gvn().transform(new StoreVectorMaskedNode(control(), memory(addr), addr, val, addr_type, mask));
1085 set_memory(vstore, addr_type);
1086 } else {
1087 Node* vload = nullptr;
1088
1089 if (mismatched_ms) {
1090 // Reinterpret the vector mask to byte type.
1091 const TypeVect* from_mask_type = TypeVect::makemask(elem_bt, num_elem);
1092 const TypeVect* to_mask_type = TypeVect::makemask(mem_elem_bt, mem_num_elem);
1093 mask = gvn().transform(new VectorReinterpretNode(mask, from_mask_type, to_mask_type));
1094 }
1095
1096 if (supports_predicate) {
1097 // Generate masked load vector node if predicate feature is supported.
1098 const TypeVect* vt = TypeVect::make(mem_elem_bt, mem_num_elem);
1099 vload = gvn().transform(new LoadVectorMaskedNode(control(), memory(addr), addr, addr_type, vt, mask));
1100 } else {
1101 // Use the vector blend to implement the masked load vector. The biased elements are zeros.
1102 Node* zero = gvn().transform(gvn().zerocon(mem_elem_bt));
1103 zero = gvn().transform(VectorNode::scalar2vector(zero, mem_num_elem, mem_elem_bt));
1104 vload = gvn().transform(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, mem_num_elem, mem_elem_bt));
1105 vload = gvn().transform(new VectorBlendNode(zero, vload, mask));
1106 }
1107
1108 if (mismatched_ms) {
1109 const TypeVect* to_vect_type = TypeVect::make(elem_bt, num_elem);
1110 vload = gvn().transform(new VectorReinterpretNode(vload, vload->bottom_type()->is_vect(), to_vect_type));
1111 }
1112
1113 Node* box = box_vector(vload, vbox_type, elem_bt, num_elem);
1114 set_result(box);
1115 }
1116
1117 destruct_map_clone(old_map);
1118
1119 if (can_access_non_heap) {
1120 insert_mem_bar(Op_MemBarCPUOrder);
1121 }
1122
1123 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1124 return true;
1125 }
1126
1127 // <C,
1128 // V extends Vector<?>,
1129 // W extends Vector<Integer>,
1130 // S extends VectorSpecies<E>,
1131 // M extends VectorMask<E>,
1132 // E>
1133 // V loadWithMap(Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType, int length,
1134 // Class<? extends Vector<Integer>> vectorIndexClass,
1135 // Object base, long offset, // Unsafe addressing
1136 // W index_vector, M m,
1137 // C container, int index, int[] indexMap, int indexM, S s, // Arguments for default implementation
1138 // LoadVectorOperationWithMap<C, V, E, S, M> defaultImpl)
1139 //
1140 // <C,
1141 // V extends Vector<E>,
1142 // W extends Vector<Integer>,
1143 // M extends VectorMask<E>,
1144 // E>
1145 // void storeWithMap(Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType,
1146 // int length, Class<? extends Vector<Integer>> vectorIndexClass, Object base, long offset, // Unsafe addressing
1147 // W index_vector, V v, M m,
1148 // C container, int index, int[] indexMap, int indexM, // Arguments for default implementation
1149 // StoreVectorOperationWithMap<C, V, M, E> defaultImpl)
1150 //
1151 bool LibraryCallKit::inline_vector_gather_scatter(bool is_scatter) {
1152 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
1153 const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
1154 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
1155 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
1156 const TypeInstPtr* vector_idx_klass = gvn().type(argument(4))->isa_instptr();
1157
1158 if (vector_klass == nullptr || elem_klass == nullptr || vector_idx_klass == nullptr || vlen == nullptr ||
1159 vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || vector_idx_klass->const_oop() == nullptr || !vlen->is_con()) {
1160 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s viclass=%s",
1161 NodeClassNames[argument(0)->Opcode()],
1162 NodeClassNames[argument(2)->Opcode()],
1163 NodeClassNames[argument(3)->Opcode()],
1164 NodeClassNames[argument(4)->Opcode()]);
1165 return false; // not enough info for intrinsification
1166 }
1167
1168 if (!is_klass_initialized(vector_klass) || !is_klass_initialized(vector_idx_klass)) {
1169 log_if_needed(" ** klass argument not initialized");
1170 return false;
1171 }
1172
1173 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1174 if (!elem_type->is_primitive_type()) {
1175 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1176 return false; // should be primitive type
1177 }
1178
1179 BasicType elem_bt = elem_type->basic_type();
1180 int num_elem = vlen->get_con();
1181
1182 const Type* vmask_type = gvn().type(is_scatter ? argument(10) : argument(9));
1183 bool is_masked_op = vmask_type != TypePtr::NULL_PTR;
1184 if (is_masked_op) {
1185 if (mask_klass == nullptr || mask_klass->const_oop() == nullptr) {
1186 log_if_needed(" ** missing constant: maskclass=%s", NodeClassNames[argument(1)->Opcode()]);
1187 return false; // not enough info for intrinsification
1188 }
1189
1190 if (!is_klass_initialized(mask_klass)) {
1191 log_if_needed(" ** mask klass argument not initialized");
1192 return false;
1193 }
1194
1195 if (vmask_type->maybe_null()) {
1196 log_if_needed(" ** null mask values are not allowed for masked op");
1197 return false;
1198 }
1199
1200 // Check whether the predicated gather/scatter node is supported by architecture.
1201 VectorMaskUseType mask = (VectorMaskUseType) (VecMaskUseLoad | VecMaskUsePred);
1202 if (!arch_supports_vector(is_scatter ? Op_StoreVectorScatterMasked : Op_LoadVectorGatherMasked, num_elem, elem_bt, mask)) {
1203 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d etype=%s is_masked_op=1",
1204 is_scatter, is_scatter ? "scatterMasked" : "gatherMasked",
1205 num_elem, type2name(elem_bt));
1206 return false; // not supported
1207 }
1208 } else {
1209 // Check whether the normal gather/scatter node is supported for non-masked operation.
1210 if (!arch_supports_vector(is_scatter ? Op_StoreVectorScatter : Op_LoadVectorGather, num_elem, elem_bt, VecMaskNotUsed)) {
1211 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d etype=%s is_masked_op=0",
1212 is_scatter, is_scatter ? "scatter" : "gather",
1213 num_elem, type2name(elem_bt));
1214 return false; // not supported
1215 }
1216 }
1217
1218 // Check that the vector holding indices is supported by architecture
1219 // For sub-word gathers expander receive index array.
1220 if (!is_subword_type(elem_bt) && !arch_supports_vector(Op_LoadVector, num_elem, T_INT, VecMaskNotUsed)) {
1221 log_if_needed(" ** not supported: arity=%d op=%s/loadindex vlen=%d etype=int is_masked_op=%d",
1222 is_scatter, is_scatter ? "scatter" : "gather",
1223 num_elem, is_masked_op ? 1 : 0);
1224 return false; // not supported
1225 }
1226
1227 Node* base = argument(5);
1228 Node* offset = ConvL2X(argument(6));
1229
1230 // Save state and restore on bailout
1231 uint old_sp = sp();
1232 SafePointNode* old_map = clone_map();
1233
1234 Node* addr = make_unsafe_address(base, offset, elem_bt, true);
1235
1236 const TypePtr *addr_type = gvn().type(addr)->isa_ptr();
1237 const TypeAryPtr* arr_type = addr_type->isa_aryptr();
1238
1239 // The array must be consistent with vector type
1240 if (arr_type == nullptr || (arr_type != nullptr && !elem_consistent_with_arr(elem_bt, arr_type, false))) {
1241 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d etype=%s atype=%s ismask=no",
1242 is_scatter, is_scatter ? "scatter" : "gather",
1243 num_elem, type2name(elem_bt), type2name(arr_type->elem()->array_element_basic_type()));
1244 set_map(old_map);
1245 set_sp(old_sp);
1246 return false;
1247 }
1248
1249 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1250 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1251 ciKlass* vbox_idx_klass = vector_idx_klass->const_oop()->as_instance()->java_lang_Class_klass();
1252 if (vbox_idx_klass == nullptr) {
1253 set_map(old_map);
1254 set_sp(old_sp);
1255 return false;
1256 }
1257
1258 Node* index_vect = nullptr;
1259 const TypeInstPtr* vbox_idx_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_idx_klass);
1260 if (!is_subword_type(elem_bt)) {
1261 index_vect = unbox_vector(argument(8), vbox_idx_type, T_INT, num_elem);
1262 if (index_vect == nullptr) {
1263 set_map(old_map);
1264 set_sp(old_sp);
1265 return false;
1266 }
1267 }
1268
1269 Node* mask = nullptr;
1270 if (is_masked_op) {
1271 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1272 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1273 mask = unbox_vector(is_scatter ? argument(10) : argument(9), mbox_type, elem_bt, num_elem);
1274 if (mask == nullptr) {
1275 log_if_needed(" ** unbox failed mask=%s",
1276 is_scatter ? NodeClassNames[argument(10)->Opcode()]
1277 : NodeClassNames[argument(9)->Opcode()]);
1278 set_map(old_map);
1279 set_sp(old_sp);
1280 return false;
1281 }
1282 }
1283
1284 const TypeVect* vector_type = TypeVect::make(elem_bt, num_elem);
1285 if (is_scatter) {
1286 Node* val = unbox_vector(argument(9), vbox_type, elem_bt, num_elem);
1287 if (val == nullptr) {
1288 set_map(old_map);
1289 set_sp(old_sp);
1290 return false; // operand unboxing failed
1291 }
1292 set_all_memory(reset_memory());
1293
1294 Node* vstore = nullptr;
1295 if (mask != nullptr) {
1296 vstore = gvn().transform(new StoreVectorScatterMaskedNode(control(), memory(addr), addr, addr_type, val, index_vect, mask));
1297 } else {
1298 vstore = gvn().transform(new StoreVectorScatterNode(control(), memory(addr), addr, addr_type, val, index_vect));
1299 }
1300 set_memory(vstore, addr_type);
1301 } else {
1302 Node* vload = nullptr;
1303 Node* index = argument(11);
1304 Node* indexMap = argument(12);
1305 Node* indexM = argument(13);
1306 if (mask != nullptr) {
1307 if (is_subword_type(elem_bt)) {
1308 Node* index_arr_base = array_element_address(indexMap, indexM, T_INT);
1309 vload = gvn().transform(new LoadVectorGatherMaskedNode(control(), memory(addr), addr, addr_type, vector_type, index_arr_base, mask, index));
1310 } else {
1311 vload = gvn().transform(new LoadVectorGatherMaskedNode(control(), memory(addr), addr, addr_type, vector_type, index_vect, mask));
1312 }
1313 } else {
1314 if (is_subword_type(elem_bt)) {
1315 Node* index_arr_base = array_element_address(indexMap, indexM, T_INT);
1316 vload = gvn().transform(new LoadVectorGatherNode(control(), memory(addr), addr, addr_type, vector_type, index_arr_base, index));
1317 } else {
1318 vload = gvn().transform(new LoadVectorGatherNode(control(), memory(addr), addr, addr_type, vector_type, index_vect));
1319 }
1320 }
1321 Node* box = box_vector(vload, vbox_type, elem_bt, num_elem);
1322 set_result(box);
1323 }
1324
1325 destruct_map_clone(old_map);
1326
1327 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1328 return true;
1329 }
1330
1331 // public static
1332 // <V extends Vector<E>,
1333 // M extends VectorMask<E>,
1334 // E>
1335 // long reductionCoerced(int oprId, Class<? extends V> vectorClass, Class<? extends M> maskClass,
1336 // Class<E> elementType, int length, V v, M m,
1337 // ReductionOperation<V, M> defaultImpl)
1338 bool LibraryCallKit::inline_vector_reduction() {
1339 const TypeInt* opr = gvn().type(argument(0))->isa_int();
1340 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
1341 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
1342 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
1343 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
1344
1345 if (opr == nullptr || vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr ||
1346 !opr->is_con() || vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
1347 log_if_needed(" ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
1348 NodeClassNames[argument(0)->Opcode()],
1349 NodeClassNames[argument(1)->Opcode()],
1350 NodeClassNames[argument(3)->Opcode()],
1351 NodeClassNames[argument(4)->Opcode()]);
1352 return false; // not enough info for intrinsification
1353 }
1354 if (!is_klass_initialized(vector_klass)) {
1355 log_if_needed(" ** klass argument not initialized");
1356 return false;
1357 }
1358 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1359 if (!elem_type->is_primitive_type()) {
1360 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1361 return false; // should be primitive type
1362 }
1363
1364 const Type* vmask_type = gvn().type(argument(6));
1365 bool is_masked_op = vmask_type != TypePtr::NULL_PTR;
1366 if (is_masked_op) {
1367 if (mask_klass == nullptr || mask_klass->const_oop() == nullptr) {
1368 log_if_needed(" ** missing constant: maskclass=%s", NodeClassNames[argument(2)->Opcode()]);
1369 return false; // not enough info for intrinsification
1370 }
1371
1372 if (!is_klass_initialized(mask_klass)) {
1373 log_if_needed(" ** mask klass argument not initialized");
1374 return false;
1375 }
1376
1377 if (vmask_type->maybe_null()) {
1378 log_if_needed(" ** null mask values are not allowed for masked op");
1379 return false;
1380 }
1381 }
1382
1383 BasicType elem_bt = elem_type->basic_type();
1384 int num_elem = vlen->get_con();
1385 int opc = VectorSupport::vop2ideal(opr->get_con(), elem_bt);
1386 int sopc = ReductionNode::opcode(opc, elem_bt);
1387
1388 // Ensure reduction operation for lanewise operation
1389 // When using mask, mask use type needs to be VecMaskUseLoad.
1390 if (sopc == opc || !arch_supports_vector(sopc, num_elem, elem_bt, is_masked_op ? VecMaskUseLoad : VecMaskNotUsed)) {
1391 log_if_needed(" ** not supported: arity=1 op=%d/reduce vlen=%d etype=%s is_masked_op=%d",
1392 sopc, num_elem, type2name(elem_bt), is_masked_op ? 1 : 0);
1393 return false;
1394 }
1395
1396 // Return true if current platform has implemented the masked operation with predicate feature.
1397 bool use_predicate = is_masked_op && arch_supports_vector(sopc, num_elem, elem_bt, VecMaskUsePred);
1398 if (is_masked_op && !use_predicate && !arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad)) {
1399 log_if_needed(" ** not supported: arity=1 op=%d/reduce vlen=%d etype=%s is_masked_op=1",
1400 sopc, num_elem, type2name(elem_bt));
1401 return false;
1402 }
1403
1404 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1405 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1406
1407 Node* opd = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
1408 if (opd == nullptr) {
1409 return false; // operand unboxing failed
1410 }
1411
1412 Node* mask = nullptr;
1413 if (is_masked_op) {
1414 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1415 assert(is_vector_mask(mbox_klass), "argument(2) should be a mask class");
1416 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1417 mask = unbox_vector(argument(6), mbox_type, elem_bt, num_elem);
1418 if (mask == nullptr) {
1419 log_if_needed(" ** unbox failed mask=%s",
1420 NodeClassNames[argument(6)->Opcode()]);
1421 return false;
1422 }
1423 }
1424
1425 Node* init = ReductionNode::make_identity_con_scalar(gvn(), opc, elem_bt);
1426 Node* value = opd;
1427
1428 assert(mask != nullptr || !is_masked_op, "Masked op needs the mask value never null");
1429 if (mask != nullptr && !use_predicate) {
1430 Node* reduce_identity = gvn().transform(VectorNode::scalar2vector(init, num_elem, elem_bt));
1431 value = gvn().transform(new VectorBlendNode(reduce_identity, value, mask));
1432 }
1433
1434 // Make an unordered Reduction node. This affects only AddReductionVF/VD and MulReductionVF/VD,
1435 // as these operations are allowed to be associative (not requiring strict order) in VectorAPI.
1436 value = ReductionNode::make(opc, nullptr, init, value, elem_bt, /* requires_strict_order */ false);
1437
1438 if (mask != nullptr && use_predicate) {
1439 value->add_req(mask);
1440 value->add_flag(Node::Flag_is_predicated_vector);
1441 }
1442
1443 value = gvn().transform(value);
1444
1445 Node* bits = nullptr;
1446 switch (elem_bt) {
1447 case T_BYTE:
1448 case T_SHORT:
1449 case T_INT: {
1450 bits = gvn().transform(new ConvI2LNode(value));
1451 break;
1452 }
1453 case T_FLOAT: {
1454 value = gvn().transform(new MoveF2INode(value));
1455 bits = gvn().transform(new ConvI2LNode(value));
1456 break;
1457 }
1458 case T_DOUBLE: {
1459 bits = gvn().transform(new MoveD2LNode(value));
1460 break;
1461 }
1462 case T_LONG: {
1463 bits = value; // no conversion needed
1464 break;
1465 }
1466 default: fatal("%s", type2name(elem_bt));
1467 }
1468 set_result(bits);
1469 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1470 return true;
1471 }
1472
1473 // public static <V> boolean test(int cond, Class<?> vectorClass, Class<?> elementType, int vlen,
1474 // V v1, V v2,
1475 // BiFunction<V, V, Boolean> defaultImpl)
1476 //
1477 bool LibraryCallKit::inline_vector_test() {
1478 const TypeInt* cond = gvn().type(argument(0))->isa_int();
1479 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
1480 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
1481 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
1482
1483 if (cond == nullptr || vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr ||
1484 !cond->is_con() || vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
1485 log_if_needed(" ** missing constant: cond=%s vclass=%s etype=%s vlen=%s",
1486 NodeClassNames[argument(0)->Opcode()],
1487 NodeClassNames[argument(1)->Opcode()],
1488 NodeClassNames[argument(2)->Opcode()],
1489 NodeClassNames[argument(3)->Opcode()]);
1490 return false; // not enough info for intrinsification
1491 }
1492 if (!is_klass_initialized(vector_klass)) {
1493 log_if_needed(" ** klass argument not initialized");
1494 return false;
1495 }
1496 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1497 if (!elem_type->is_primitive_type()) {
1498 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1499 return false; // should be primitive type
1500 }
1501 BasicType elem_bt = elem_type->basic_type();
1502 int num_elem = vlen->get_con();
1503 BoolTest::mask booltest = (BoolTest::mask)cond->get_con();
1504 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1505 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1506
1507 if (!arch_supports_vector(Op_VectorTest, num_elem, elem_bt, is_vector_mask(vbox_klass) ? VecMaskUseLoad : VecMaskNotUsed)) {
1508 log_if_needed(" ** not supported: arity=2 op=test/%d vlen=%d etype=%s ismask=%d",
1509 cond->get_con(), num_elem, type2name(elem_bt),
1510 is_vector_mask(vbox_klass));
1511 return false;
1512 }
1513
1514 Node* opd1 = unbox_vector(argument(4), vbox_type, elem_bt, num_elem);
1515 Node* opd2;
1516 if (Matcher::vectortest_needs_second_argument(booltest == BoolTest::overflow,
1517 opd1->bottom_type()->isa_vectmask())) {
1518 opd2 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
1519 } else {
1520 opd2 = opd1;
1521 }
1522 if (opd1 == nullptr || opd2 == nullptr) {
1523 return false; // operand unboxing failed
1524 }
1525
1526 Node* cmp = gvn().transform(new VectorTestNode(opd1, opd2, booltest));
1527 BoolTest::mask test = Matcher::vectortest_mask(booltest == BoolTest::overflow,
1528 opd1->bottom_type()->isa_vectmask(), num_elem);
1529 Node* bol = gvn().transform(new BoolNode(cmp, test));
1530 Node* res = gvn().transform(new CMoveINode(bol, gvn().intcon(0), gvn().intcon(1), TypeInt::BOOL));
1531
1532 set_result(res);
1533 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1534 return true;
1535 }
1536
1537 // public static
1538 // <V extends Vector<E>,
1539 // M extends VectorMask<E>,
1540 // E>
1541 // V blend(Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType, int vlen,
1542 // V v1, V v2, M m,
1543 // VectorBlendOp<V, M, E> defaultImpl)
1544 bool LibraryCallKit::inline_vector_blend() {
1545 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
1546 const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
1547 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
1548 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
1549
1550 if (mask_klass == nullptr || vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr) {
1551 return false; // dead code
1552 }
1553 if (mask_klass->const_oop() == nullptr || vector_klass->const_oop() == nullptr ||
1554 elem_klass->const_oop() == nullptr || !vlen->is_con()) {
1555 log_if_needed(" ** missing constant: vclass=%s mclass=%s etype=%s vlen=%s",
1556 NodeClassNames[argument(0)->Opcode()],
1557 NodeClassNames[argument(1)->Opcode()],
1558 NodeClassNames[argument(2)->Opcode()],
1559 NodeClassNames[argument(3)->Opcode()]);
1560 return false; // not enough info for intrinsification
1561 }
1562 if (!is_klass_initialized(vector_klass) || !is_klass_initialized(mask_klass)) {
1563 log_if_needed(" ** klass argument not initialized");
1564 return false;
1565 }
1566 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1567 if (!elem_type->is_primitive_type()) {
1568 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1569 return false; // should be primitive type
1570 }
1571 BasicType elem_bt = elem_type->basic_type();
1572 BasicType mask_bt = elem_bt;
1573 int num_elem = vlen->get_con();
1574
1575 if (!arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad)) {
1576 log_if_needed(" ** not supported: arity=2 op=blend vlen=%d etype=%s ismask=useload",
1577 num_elem, type2name(elem_bt));
1578 return false; // not supported
1579 }
1580 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1581 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1582
1583 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1584 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1585
1586 Node* v1 = unbox_vector(argument(4), vbox_type, elem_bt, num_elem);
1587 Node* v2 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
1588 Node* mask = unbox_vector(argument(6), mbox_type, mask_bt, num_elem);
1589
1590 if (v1 == nullptr || v2 == nullptr || mask == nullptr) {
1591 return false; // operand unboxing failed
1592 }
1593
1594 Node* blend = gvn().transform(new VectorBlendNode(v1, v2, mask));
1595
1596 Node* box = box_vector(blend, vbox_type, elem_bt, num_elem);
1597 set_result(box);
1598 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1599 return true;
1600 }
1601
1602 // public static
1603 // <V extends Vector<E>,
1604 // M extends VectorMask<E>,
1605 // E>
1606 // M compare(int cond, Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType, int vlen,
1607 // V v1, V v2, M m,
1608 // VectorCompareOp<V,M> defaultImpl)
1609 bool LibraryCallKit::inline_vector_compare() {
1610 const TypeInt* cond = gvn().type(argument(0))->isa_int();
1611 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
1612 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
1613 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
1614 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
1615
1616 if (cond == nullptr || vector_klass == nullptr || mask_klass == nullptr || elem_klass == nullptr || vlen == nullptr) {
1617 return false; // dead code
1618 }
1619 if (!cond->is_con() || vector_klass->const_oop() == nullptr || mask_klass->const_oop() == nullptr ||
1620 elem_klass->const_oop() == nullptr || !vlen->is_con()) {
1621 log_if_needed(" ** missing constant: cond=%s vclass=%s mclass=%s etype=%s vlen=%s",
1622 NodeClassNames[argument(0)->Opcode()],
1623 NodeClassNames[argument(1)->Opcode()],
1624 NodeClassNames[argument(2)->Opcode()],
1625 NodeClassNames[argument(3)->Opcode()],
1626 NodeClassNames[argument(4)->Opcode()]);
1627 return false; // not enough info for intrinsification
1628 }
1629 if (!is_klass_initialized(vector_klass) || !is_klass_initialized(mask_klass)) {
1630 log_if_needed(" ** klass argument not initialized");
1631 return false;
1632 }
1633 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1634 if (!elem_type->is_primitive_type()) {
1635 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1636 return false; // should be primitive type
1637 }
1638
1639 int num_elem = vlen->get_con();
1640 BasicType elem_bt = elem_type->basic_type();
1641 BasicType mask_bt = elem_bt;
1642
1643 if ((cond->get_con() & BoolTest::unsigned_compare) != 0) {
1644 if (!Matcher::supports_vector_comparison_unsigned(num_elem, elem_bt)) {
1645 log_if_needed(" ** not supported: unsigned comparison op=comp/%d vlen=%d etype=%s ismask=usestore",
1646 cond->get_con() & (BoolTest::unsigned_compare - 1), num_elem, type2name(elem_bt));
1647 return false;
1648 }
1649 }
1650
1651 if (!arch_supports_vector(Op_VectorMaskCmp, num_elem, elem_bt, VecMaskUseStore)) {
1652 log_if_needed(" ** not supported: arity=2 op=comp/%d vlen=%d etype=%s ismask=usestore",
1653 cond->get_con(), num_elem, type2name(elem_bt));
1654 return false;
1655 }
1656
1657 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1658 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1659
1660 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1661 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1662
1663 Node* v1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
1664 Node* v2 = unbox_vector(argument(6), vbox_type, elem_bt, num_elem);
1665
1666 bool is_masked_op = argument(7)->bottom_type() != TypePtr::NULL_PTR;
1667 Node* mask = is_masked_op ? unbox_vector(argument(7), mbox_type, elem_bt, num_elem) : nullptr;
1668 if (is_masked_op && mask == nullptr) {
1669 log_if_needed(" ** not supported: mask = null arity=2 op=comp/%d vlen=%d etype=%s ismask=usestore is_masked_op=1",
1670 cond->get_con(), num_elem, type2name(elem_bt));
1671 return false;
1672 }
1673
1674 bool use_predicate = is_masked_op && arch_supports_vector(Op_VectorMaskCmp, num_elem, elem_bt, VecMaskUsePred);
1675 if (is_masked_op && !use_predicate && !arch_supports_vector(Op_AndV, num_elem, elem_bt, VecMaskUseLoad)) {
1676 log_if_needed(" ** not supported: arity=2 op=comp/%d vlen=%d etype=%s ismask=usestore is_masked_op=1",
1677 cond->get_con(), num_elem, type2name(elem_bt));
1678 return false;
1679 }
1680
1681 if (v1 == nullptr || v2 == nullptr) {
1682 return false; // operand unboxing failed
1683 }
1684 BoolTest::mask pred = (BoolTest::mask)cond->get_con();
1685 ConINode* pred_node = (ConINode*)gvn().makecon(cond);
1686
1687 const TypeVect* vmask_type = TypeVect::makemask(mask_bt, num_elem);
1688 Node* operation = new VectorMaskCmpNode(pred, v1, v2, pred_node, vmask_type);
1689
1690 if (is_masked_op) {
1691 if (use_predicate) {
1692 operation->add_req(mask);
1693 operation->add_flag(Node::Flag_is_predicated_vector);
1694 } else {
1695 operation = gvn().transform(operation);
1696 operation = VectorNode::make(Op_AndV, operation, mask, vmask_type);
1697 }
1698 }
1699
1700 operation = gvn().transform(operation);
1701
1702 Node* box = box_vector(operation, mbox_type, mask_bt, num_elem);
1703 set_result(box);
1704 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1705 return true;
1706 }
1707
1708 // public static
1709 // <V extends Vector<E>,
1710 // Sh extends VectorShuffle<E>,
1711 // M extends VectorMask<E>,
1712 // E>
1713 // V rearrangeOp(Class<? extends V> vectorClass, Class<Sh> shuffleClass, Class<M> maskClass, Class<E> elementType, int vlen,
1714 // V v1, Sh sh, M m,
1715 // VectorRearrangeOp<V, Sh, M, E> defaultImpl)
1716 bool LibraryCallKit::inline_vector_rearrange() {
1717 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
1718 const TypeInstPtr* shuffle_klass = gvn().type(argument(1))->isa_instptr();
1719 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
1720 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
1721 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
1722
1723 if (vector_klass == nullptr || shuffle_klass == nullptr || elem_klass == nullptr || vlen == nullptr) {
1724 return false; // dead code
1725 }
1726 if (shuffle_klass->const_oop() == nullptr ||
1727 vector_klass->const_oop() == nullptr ||
1728 elem_klass->const_oop() == nullptr ||
1729 !vlen->is_con()) {
1730 log_if_needed(" ** missing constant: vclass=%s sclass=%s etype=%s vlen=%s",
1731 NodeClassNames[argument(0)->Opcode()],
1732 NodeClassNames[argument(1)->Opcode()],
1733 NodeClassNames[argument(3)->Opcode()],
1734 NodeClassNames[argument(4)->Opcode()]);
1735 return false; // not enough info for intrinsification
1736 }
1737 if (!is_klass_initialized(vector_klass) ||
1738 !is_klass_initialized(shuffle_klass)) {
1739 log_if_needed(" ** klass argument not initialized");
1740 return false;
1741 }
1742 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1743 if (!elem_type->is_primitive_type()) {
1744 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1745 return false; // should be primitive type
1746 }
1747
1748 BasicType elem_bt = elem_type->basic_type();
1749 BasicType shuffle_bt = elem_bt;
1750 if (shuffle_bt == T_FLOAT) {
1751 shuffle_bt = T_INT;
1752 } else if (shuffle_bt == T_DOUBLE) {
1753 shuffle_bt = T_LONG;
1754 }
1755
1756 int num_elem = vlen->get_con();
1757 bool need_load_shuffle = Matcher::vector_rearrange_requires_load_shuffle(shuffle_bt, num_elem);
1758
1759 if (need_load_shuffle && !arch_supports_vector(Op_VectorLoadShuffle, num_elem, shuffle_bt, VecMaskNotUsed)) {
1760 if (C->print_intrinsics()) {
1761 tty->print_cr(" ** not supported: arity=0 op=load/shuffle vlen=%d etype=%s ismask=no",
1762 num_elem, type2name(shuffle_bt));
1763 }
1764 return false; // not supported
1765 }
1766
1767 bool is_masked_op = argument(7)->bottom_type() != TypePtr::NULL_PTR;
1768 bool use_predicate = is_masked_op;
1769 if (is_masked_op &&
1770 (mask_klass == nullptr ||
1771 mask_klass->const_oop() == nullptr ||
1772 !is_klass_initialized(mask_klass))) {
1773 log_if_needed(" ** mask_klass argument not initialized");
1774 }
1775 if (!arch_supports_vector(Op_AndV, num_elem, elem_bt, VecMaskNotUsed)) {
1776 log_if_needed(" ** not supported: arity=2 op=and vlen=%d etype=%s ismask=no",
1777 num_elem, type2name(elem_bt));
1778 return false;
1779 }
1780 VectorMaskUseType checkFlags = (VectorMaskUseType)(is_masked_op ? (VecMaskUseLoad | VecMaskUsePred) : VecMaskNotUsed);
1781 if (!arch_supports_vector(Op_VectorRearrange, num_elem, elem_bt, checkFlags)) {
1782 use_predicate = false;
1783 if(!is_masked_op ||
1784 (!arch_supports_vector(Op_VectorRearrange, num_elem, elem_bt, VecMaskNotUsed) ||
1785 !arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad) ||
1786 !arch_supports_vector(Op_Replicate, num_elem, elem_bt, VecMaskNotUsed))) {
1787 log_if_needed(" ** not supported: arity=2 op=shuffle/rearrange vlen=%d etype=%s ismask=no",
1788 num_elem, type2name(elem_bt));
1789 return false; // not supported
1790 }
1791 }
1792 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1793 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1794
1795 ciKlass* shbox_klass = shuffle_klass->const_oop()->as_instance()->java_lang_Class_klass();
1796 const TypeInstPtr* shbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, shbox_klass);
1797
1798 Node* v1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
1799 Node* shuffle = unbox_vector(argument(6), shbox_type, shuffle_bt, num_elem);
1800 const TypeVect* st = TypeVect::make(shuffle_bt, num_elem);
1801
1802 if (v1 == nullptr || shuffle == nullptr) {
1803 return false; // operand unboxing failed
1804 }
1805
1806 assert(is_power_of_2(num_elem), "wrapping invalid");
1807 Node* wrapping_mask_elem = gvn().makecon(TypeInteger::make(num_elem - 1, num_elem - 1, Type::WidenMin, shuffle_bt == T_LONG ? T_LONG : T_INT));
1808 Node* wrapping_mask = gvn().transform(VectorNode::scalar2vector(wrapping_mask_elem, num_elem, shuffle_bt));
1809 shuffle = gvn().transform(new AndVNode(shuffle, wrapping_mask, st));
1810
1811 Node* mask = nullptr;
1812 if (is_masked_op) {
1813 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1814 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1815 mask = unbox_vector(argument(7), mbox_type, elem_bt, num_elem);
1816 if (mask == nullptr) {
1817 log_if_needed(" ** not supported: arity=3 op=shuffle/rearrange vlen=%d etype=%s ismask=useload is_masked_op=1",
1818 num_elem, type2name(elem_bt));
1819 return false;
1820 }
1821 }
1822
1823 if (need_load_shuffle) {
1824 shuffle = gvn().transform(new VectorLoadShuffleNode(shuffle, st));
1825 }
1826
1827 Node* rearrange = new VectorRearrangeNode(v1, shuffle);
1828 if (is_masked_op) {
1829 if (use_predicate) {
1830 rearrange->add_req(mask);
1831 rearrange->add_flag(Node::Flag_is_predicated_vector);
1832 } else {
1833 rearrange = gvn().transform(rearrange);
1834 Node* zero = gvn().makecon(Type::get_zero_type(elem_bt));
1835 Node* zerovec = gvn().transform(VectorNode::scalar2vector(zero, num_elem, elem_bt));
1836 rearrange = new VectorBlendNode(zerovec, rearrange, mask);
1837 }
1838 }
1839 rearrange = gvn().transform(rearrange);
1840
1841 Node* box = box_vector(rearrange, vbox_type, elem_bt, num_elem);
1842 set_result(box);
1843 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1844 return true;
1845 }
1846
1847 static address get_vector_math_address(int vop, int bits, BasicType bt, char* name_ptr, int name_len) {
1848 address addr = nullptr;
1849 assert(UseVectorStubs, "sanity");
1850 assert(name_ptr != nullptr, "unexpected");
1851 assert((vop >= VectorSupport::VECTOR_OP_MATH_START) && (vop <= VectorSupport::VECTOR_OP_MATH_END), "unexpected");
1852 int op = vop - VectorSupport::VECTOR_OP_MATH_START;
1853
1854 switch(bits) {
1855 case 64: //fallthough
1856 case 128: //fallthough
1857 case 256: //fallthough
1858 case 512:
1859 if (bt == T_FLOAT) {
1860 snprintf(name_ptr, name_len, "vector_%s_float_%dbits_fixed", VectorSupport::mathname[op], bits);
1861 addr = StubRoutines::_vector_f_math[exact_log2(bits/64)][op];
1862 } else {
1863 assert(bt == T_DOUBLE, "must be FP type only");
1864 snprintf(name_ptr, name_len, "vector_%s_double_%dbits_fixed", VectorSupport::mathname[op], bits);
1865 addr = StubRoutines::_vector_d_math[exact_log2(bits/64)][op];
1866 }
1867 break;
1868 default:
1869 if (!Matcher::supports_scalable_vector() || !Matcher::vector_size_supported(bt, bits/type2aelembytes(bt)) ) {
1870 snprintf(name_ptr, name_len, "invalid");
1871 addr = nullptr;
1872 Unimplemented();
1873 }
1874 break;
1875 }
1876
1877 if (addr == nullptr && Matcher::supports_scalable_vector()) {
1878 if (bt == T_FLOAT) {
1879 snprintf(name_ptr, name_len, "vector_%s_float_%dbits_scalable", VectorSupport::mathname[op], bits);
1880 addr = StubRoutines::_vector_f_math[VectorSupport::VEC_SIZE_SCALABLE][op];
1881 } else {
1882 assert(bt == T_DOUBLE, "must be FP type only");
1883 snprintf(name_ptr, name_len, "vector_%s_double_%dbits_scalable", VectorSupport::mathname[op], bits);
1884 addr = StubRoutines::_vector_d_math[VectorSupport::VEC_SIZE_SCALABLE][op];
1885 }
1886 }
1887
1888 return addr;
1889 }
1890
1891 // public static
1892 // <V extends Vector<E>,
1893 // M extends VectorMask<E>,
1894 // E>
1895 // V selectFromOp(Class<? extends V> vClass, Class<M> mClass, Class<E> eClass,
1896 // int length, V v1, V v2, M m,
1897 // VectorSelectFromOp<V, M> defaultImpl)
1898 bool LibraryCallKit::inline_vector_select_from() {
1899 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
1900 const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
1901 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
1902 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
1903
1904 if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr ||
1905 vector_klass->const_oop() == nullptr ||
1906 elem_klass->const_oop() == nullptr ||
1907 !vlen->is_con()) {
1908 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s",
1909 NodeClassNames[argument(0)->Opcode()],
1910 NodeClassNames[argument(2)->Opcode()],
1911 NodeClassNames[argument(3)->Opcode()]);
1912 return false; // not enough info for intrinsification
1913 }
1914 if (!is_klass_initialized(vector_klass)) {
1915 log_if_needed(" ** klass argument not initialized");
1916 return false;
1917 }
1918 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1919 if (!elem_type->is_primitive_type()) {
1920 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1921 return false; // should be primitive type
1922 }
1923 BasicType elem_bt = elem_type->basic_type();
1924 int num_elem = vlen->get_con();
1925 if (!is_power_of_2(num_elem)) {
1926 log_if_needed(" ** vlen not power of two=%d", num_elem);
1927 return false;
1928 }
1929
1930 BasicType shuffle_bt = elem_bt;
1931 if (shuffle_bt == T_FLOAT) {
1932 shuffle_bt = T_INT;
1933 } else if (shuffle_bt == T_DOUBLE) {
1934 shuffle_bt = T_LONG;
1935 }
1936 bool need_load_shuffle = Matcher::vector_rearrange_requires_load_shuffle(shuffle_bt, num_elem);
1937
1938 int cast_vopc = VectorCastNode::opcode(-1, elem_bt); // from vector of type elem_bt
1939 if ((need_load_shuffle && !arch_supports_vector(Op_VectorLoadShuffle, num_elem, elem_bt, VecMaskNotUsed)) ||
1940 (elem_bt != shuffle_bt && !arch_supports_vector(cast_vopc, num_elem, shuffle_bt, VecMaskNotUsed)) ||
1941 !arch_supports_vector(Op_AndV, num_elem, shuffle_bt, VecMaskNotUsed) ||
1942 !arch_supports_vector(Op_Replicate, num_elem, shuffle_bt, VecMaskNotUsed)) {
1943 log_if_needed(" ** not supported: arity=0 op=selectFrom vlen=%d etype=%s ismask=no",
1944 num_elem, type2name(elem_bt));
1945 return false; // not supported
1946 }
1947
1948 bool is_masked_op = argument(6)->bottom_type() != TypePtr::NULL_PTR;
1949 bool use_predicate = is_masked_op;
1950 if (is_masked_op &&
1951 (mask_klass == nullptr ||
1952 mask_klass->const_oop() == nullptr ||
1953 !is_klass_initialized(mask_klass))) {
1954 log_if_needed(" ** mask_klass argument not initialized");
1955 return false; // not supported
1956 }
1957 VectorMaskUseType checkFlags = (VectorMaskUseType)(is_masked_op ? (VecMaskUseLoad | VecMaskUsePred) : VecMaskNotUsed);
1958 if (!arch_supports_vector(Op_VectorRearrange, num_elem, elem_bt, checkFlags)) {
1959 use_predicate = false;
1960 if(!is_masked_op ||
1961 (!arch_supports_vector(Op_VectorRearrange, num_elem, elem_bt, VecMaskNotUsed) ||
1962 !arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad) ||
1963 !arch_supports_vector(Op_Replicate, num_elem, elem_bt, VecMaskNotUsed))) {
1964 log_if_needed(" ** not supported: op=selectFrom vlen=%d etype=%s is_masked_op=%d",
1965 num_elem, type2name(elem_bt), is_masked_op);
1966 return false; // not supported
1967 }
1968 }
1969 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1970 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1971
1972 // v1 is the index vector
1973 Node* v1 = unbox_vector(argument(4), vbox_type, elem_bt, num_elem);
1974 // v2 is the vector being rearranged
1975 Node* v2 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
1976
1977 if (v1 == nullptr) {
1978 log_if_needed(" ** unbox failed v1=%s", NodeClassNames[argument(4)->Opcode()]);
1979 return false; // operand unboxing failed
1980 }
1981
1982 if (v2 == nullptr) {
1983 log_if_needed(" ** unbox failed v2=%s", NodeClassNames[argument(5)->Opcode()]);
1984 return false; // operand unboxing failed
1985 }
1986
1987 Node* mask = nullptr;
1988 if (is_masked_op) {
1989 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1990 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1991 mask = unbox_vector(argument(6), mbox_type, elem_bt, num_elem);
1992 if (mask == nullptr) {
1993 log_if_needed(" ** unbox failed mask=%s", NodeClassNames[argument(6)->Opcode()]);
1994 return false;
1995 }
1996 }
1997
1998 // cast index vector from elem_bt vector to byte vector
1999 const TypeVect* shuffle_vt = TypeVect::make(shuffle_bt, num_elem);
2000 Node* shuffle = v1;
2001
2002 if (shuffle_bt != elem_bt) {
2003 shuffle = gvn().transform(VectorCastNode::make(cast_vopc, v1, shuffle_bt, num_elem));
2004 }
2005
2006 // wrap the byte vector lanes to (num_elem - 1) to form the shuffle vector where num_elem is vector length
2007 // this is a simple AND operation as we come here only for power of two vector length
2008 Node* mod_val = gvn().makecon(TypeInteger::make(num_elem - 1, num_elem - 1, Type::WidenMin, shuffle_bt == T_LONG ? T_LONG : T_INT));
2009 Node* bcast_mod = gvn().transform(VectorNode::scalar2vector(mod_val, num_elem, shuffle_bt));
2010 shuffle = gvn().transform(VectorNode::make(Op_AndV, shuffle, bcast_mod, shuffle_vt));
2011
2012 // load the shuffle to use in rearrange
2013 if (need_load_shuffle) {
2014 shuffle = gvn().transform(new VectorLoadShuffleNode(shuffle, shuffle_vt));
2015 }
2016
2017 // and finally rearrange
2018 Node* rearrange = new VectorRearrangeNode(v2, shuffle);
2019 if (is_masked_op) {
2020 if (use_predicate) {
2021 // masked rearrange is supported so use that directly
2022 rearrange->add_req(mask);
2023 rearrange->add_flag(Node::Flag_is_predicated_vector);
2024 } else {
2025 // masked rearrange is not supported so emulate usig blend
2026 const TypeVect* vt = v1->bottom_type()->is_vect();
2027 rearrange = gvn().transform(rearrange);
2028
2029 // create a zero vector with each lane element set as zero
2030 Node* zero = gvn().makecon(Type::get_zero_type(elem_bt));
2031 Node* zerovec = gvn().transform(VectorNode::scalar2vector(zero, num_elem, elem_bt));
2032
2033 // For each lane for which mask is set, blend in the rearranged lane into zero vector
2034 rearrange = new VectorBlendNode(zerovec, rearrange, mask);
2035 }
2036 }
2037 rearrange = gvn().transform(rearrange);
2038
2039 // box the result
2040 Node* box = box_vector(rearrange, vbox_type, elem_bt, num_elem);
2041 set_result(box);
2042
2043 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2044 return true;
2045 }
2046
2047 Node* LibraryCallKit::gen_call_to_vector_math(int vector_api_op_id, BasicType bt, int num_elem, Node* opd1, Node* opd2) {
2048 assert(UseVectorStubs, "sanity");
2049 assert(vector_api_op_id >= VectorSupport::VECTOR_OP_MATH_START && vector_api_op_id <= VectorSupport::VECTOR_OP_MATH_END, "need valid op id");
2050 assert(opd1 != nullptr, "must not be null");
2051 const TypeVect* vt = TypeVect::make(bt, num_elem);
2052 const TypeFunc* call_type = OptoRuntime::Math_Vector_Vector_Type(opd2 != nullptr ? 2 : 1, vt, vt);
2053 char name[100] = "";
2054
2055 // Get address for vector math method.
2056 address addr = get_vector_math_address(vector_api_op_id, vt->length_in_bytes() * BitsPerByte, bt, name, 100);
2057
2058 if (addr == nullptr) {
2059 return nullptr;
2060 }
2061
2062 assert(name[0] != '\0', "name must not be null");
2063 Node* operation = make_runtime_call(RC_VECTOR,
2064 call_type,
2065 addr,
2066 name,
2067 TypePtr::BOTTOM,
2068 opd1,
2069 opd2);
2070 return gvn().transform(new ProjNode(gvn().transform(operation), TypeFunc::Parms));
2071 }
2072
2073 // public static
2074 // <V extends Vector<E>,
2075 // M extends VectorMask<E>,
2076 // E>
2077 // V broadcastInt(int opr, Class<? extends V> vectorClass, Class<? extends M> maskClass,
2078 // Class<E> elementType, int length,
2079 // V v, int n, M m,
2080 // VectorBroadcastIntOp<V, M> defaultImpl)
2081 bool LibraryCallKit::inline_vector_broadcast_int() {
2082 const TypeInt* opr = gvn().type(argument(0))->isa_int();
2083 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
2084 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
2085 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
2086 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
2087
2088 if (opr == nullptr || vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr) {
2089 return false; // dead code
2090 }
2091 if (!opr->is_con() || vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
2092 log_if_needed(" ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
2093 NodeClassNames[argument(0)->Opcode()],
2094 NodeClassNames[argument(1)->Opcode()],
2095 NodeClassNames[argument(3)->Opcode()],
2096 NodeClassNames[argument(4)->Opcode()]);
2097 return false; // not enough info for intrinsification
2098 }
2099 if (!is_klass_initialized(vector_klass)) {
2100 log_if_needed(" ** klass argument not initialized");
2101 return false;
2102 }
2103
2104 const Type* vmask_type = gvn().type(argument(7));
2105 bool is_masked_op = vmask_type != TypePtr::NULL_PTR;
2106 if (is_masked_op) {
2107 if (mask_klass == nullptr || mask_klass->const_oop() == nullptr) {
2108 log_if_needed(" ** missing constant: maskclass=%s", NodeClassNames[argument(2)->Opcode()]);
2109 return false; // not enough info for intrinsification
2110 }
2111
2112 if (!is_klass_initialized(mask_klass)) {
2113 log_if_needed(" ** mask klass argument not initialized");
2114 return false;
2115 }
2116
2117 if (vmask_type->maybe_null()) {
2118 log_if_needed(" ** null mask values are not allowed for masked op");
2119 return false;
2120 }
2121 }
2122
2123 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2124 if (!elem_type->is_primitive_type()) {
2125 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2126 return false; // should be primitive type
2127 }
2128
2129 int num_elem = vlen->get_con();
2130 BasicType elem_bt = elem_type->basic_type();
2131 int opc = VectorSupport::vop2ideal(opr->get_con(), elem_bt);
2132
2133 bool is_shift = VectorNode::is_shift_opcode(opc);
2134 bool is_rotate = VectorNode::is_rotate_opcode(opc);
2135
2136 if (opc == 0 || (!is_shift && !is_rotate)) {
2137 log_if_needed(" ** operation not supported: op=%d bt=%s", opr->get_con(), type2name(elem_bt));
2138 return false; // operation not supported
2139 }
2140
2141 int sopc = VectorNode::opcode(opc, elem_bt);
2142 if (sopc == 0) {
2143 log_if_needed(" ** operation not supported: opc=%s bt=%s", NodeClassNames[opc], type2name(elem_bt));
2144 return false; // operation not supported
2145 }
2146
2147 Node* cnt = argument(6);
2148 const TypeInt* cnt_type = cnt->bottom_type()->isa_int();
2149
2150 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2151 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2152
2153 // If CPU supports vector constant rotate instructions pass it directly
2154 bool is_const_rotate = is_rotate && cnt_type && cnt_type->is_con() &&
2155 Matcher::supports_vector_constant_rotates(cnt_type->get_con());
2156 bool has_scalar_args = is_rotate ? !is_const_rotate : true;
2157
2158 VectorMaskUseType checkFlags = (VectorMaskUseType)(is_masked_op ? (VecMaskUseLoad | VecMaskUsePred) : VecMaskNotUsed);
2159 bool use_predicate = is_masked_op;
2160
2161 if (!arch_supports_vector(sopc, num_elem, elem_bt, checkFlags, has_scalar_args)) {
2162 use_predicate = false;
2163 if (!is_masked_op ||
2164 (!arch_supports_vector(sopc, num_elem, elem_bt, VecMaskNotUsed, has_scalar_args) ||
2165 !arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad))) {
2166
2167 log_if_needed(" ** not supported: arity=0 op=int/%d vlen=%d etype=%s is_masked_op=%d",
2168 sopc, num_elem, type2name(elem_bt), is_masked_op ? 1 : 0);
2169 return false; // not supported
2170 }
2171 }
2172
2173 Node* opd1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
2174 Node* opd2 = nullptr;
2175 if (is_shift) {
2176 opd2 = vector_shift_count(cnt, opc, elem_bt, num_elem);
2177 } else {
2178 assert(is_rotate, "unexpected operation");
2179 if (!is_const_rotate) {
2180 cnt = elem_bt == T_LONG ? gvn().transform(new ConvI2LNode(cnt)) : cnt;
2181 opd2 = gvn().transform(VectorNode::scalar2vector(cnt, num_elem, elem_bt));
2182 } else {
2183 // Constant shift value.
2184 opd2 = cnt;
2185 }
2186 }
2187
2188 if (opd1 == nullptr || opd2 == nullptr) {
2189 return false;
2190 }
2191
2192 Node* mask = nullptr;
2193 if (is_masked_op) {
2194 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
2195 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
2196 mask = unbox_vector(argument(7), mbox_type, elem_bt, num_elem);
2197 if (mask == nullptr) {
2198 log_if_needed(" ** unbox failed mask=%s", NodeClassNames[argument(7)->Opcode()]);
2199 return false;
2200 }
2201 }
2202
2203 Node* operation = VectorNode::make(opc, opd1, opd2, num_elem, elem_bt);
2204 if (is_masked_op && mask != nullptr) {
2205 if (use_predicate) {
2206 operation->add_req(mask);
2207 operation->add_flag(Node::Flag_is_predicated_vector);
2208 } else {
2209 operation = gvn().transform(operation);
2210 operation = new VectorBlendNode(opd1, operation, mask);
2211 }
2212 }
2213 operation = gvn().transform(operation);
2214 Node* vbox = box_vector(operation, vbox_type, elem_bt, num_elem);
2215 set_result(vbox);
2216 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2217 return true;
2218 }
2219
2220 // public static <VOUT extends VectorPayload,
2221 // VIN extends VectorPayload,
2222 // S extends VectorSpecies>
2223 // VOUT convert(int oprId,
2224 // Class<?> fromVectorClass, Class<?> fromElementType, int fromVLen,
2225 // Class<?> toVectorClass, Class<?> toElementType, int toVLen,
2226 // VIN v, S s,
2227 // VectorConvertOp<VOUT, VIN, S> defaultImpl)
2228 //
2229 bool LibraryCallKit::inline_vector_convert() {
2230 const TypeInt* opr = gvn().type(argument(0))->isa_int();
2231
2232 const TypeInstPtr* vector_klass_from = gvn().type(argument(1))->isa_instptr();
2233 const TypeInstPtr* elem_klass_from = gvn().type(argument(2))->isa_instptr();
2234 const TypeInt* vlen_from = gvn().type(argument(3))->isa_int();
2235
2236 const TypeInstPtr* vector_klass_to = gvn().type(argument(4))->isa_instptr();
2237 const TypeInstPtr* elem_klass_to = gvn().type(argument(5))->isa_instptr();
2238 const TypeInt* vlen_to = gvn().type(argument(6))->isa_int();
2239
2240 if (opr == nullptr ||
2241 vector_klass_from == nullptr || elem_klass_from == nullptr || vlen_from == nullptr ||
2242 vector_klass_to == nullptr || elem_klass_to == nullptr || vlen_to == nullptr) {
2243 return false; // dead code
2244 }
2245 if (!opr->is_con() ||
2246 vector_klass_from->const_oop() == nullptr || elem_klass_from->const_oop() == nullptr || !vlen_from->is_con() ||
2247 vector_klass_to->const_oop() == nullptr || elem_klass_to->const_oop() == nullptr || !vlen_to->is_con()) {
2248 log_if_needed(" ** missing constant: opr=%s vclass_from=%s etype_from=%s vlen_from=%s vclass_to=%s etype_to=%s vlen_to=%s",
2249 NodeClassNames[argument(0)->Opcode()],
2250 NodeClassNames[argument(1)->Opcode()],
2251 NodeClassNames[argument(2)->Opcode()],
2252 NodeClassNames[argument(3)->Opcode()],
2253 NodeClassNames[argument(4)->Opcode()],
2254 NodeClassNames[argument(5)->Opcode()],
2255 NodeClassNames[argument(6)->Opcode()]);
2256 return false; // not enough info for intrinsification
2257 }
2258 if (!is_klass_initialized(vector_klass_from) || !is_klass_initialized(vector_klass_to)) {
2259 log_if_needed(" ** klass argument not initialized");
2260 return false;
2261 }
2262
2263 assert(opr->get_con() == VectorSupport::VECTOR_OP_CAST ||
2264 opr->get_con() == VectorSupport::VECTOR_OP_UCAST ||
2265 opr->get_con() == VectorSupport::VECTOR_OP_REINTERPRET, "wrong opcode");
2266 bool is_cast = (opr->get_con() == VectorSupport::VECTOR_OP_CAST || opr->get_con() == VectorSupport::VECTOR_OP_UCAST);
2267 bool is_ucast = (opr->get_con() == VectorSupport::VECTOR_OP_UCAST);
2268
2269 ciKlass* vbox_klass_from = vector_klass_from->const_oop()->as_instance()->java_lang_Class_klass();
2270 ciKlass* vbox_klass_to = vector_klass_to->const_oop()->as_instance()->java_lang_Class_klass();
2271
2272 bool is_mask = is_vector_mask(vbox_klass_from);
2273
2274 ciType* elem_type_from = elem_klass_from->const_oop()->as_instance()->java_mirror_type();
2275 if (!elem_type_from->is_primitive_type()) {
2276 return false; // should be primitive type
2277 }
2278 BasicType elem_bt_from = elem_type_from->basic_type();
2279 ciType* elem_type_to = elem_klass_to->const_oop()->as_instance()->java_mirror_type();
2280 if (!elem_type_to->is_primitive_type()) {
2281 return false; // should be primitive type
2282 }
2283 BasicType elem_bt_to = elem_type_to->basic_type();
2284
2285 int num_elem_from = vlen_from->get_con();
2286 int num_elem_to = vlen_to->get_con();
2287
2288 // Check whether we can unbox to appropriate size. Even with casting, checking for reinterpret is needed
2289 // since we may need to change size.
2290 if (!arch_supports_vector(Op_VectorReinterpret,
2291 num_elem_from,
2292 elem_bt_from,
2293 is_mask ? VecMaskUseAll : VecMaskNotUsed)) {
2294 log_if_needed(" ** not supported: arity=1 op=%s/1 vlen1=%d etype1=%s ismask=%d",
2295 is_cast ? "cast" : "reinterpret",
2296 num_elem_from, type2name(elem_bt_from), is_mask);
2297 return false;
2298 }
2299
2300 // Check whether we can support resizing/reinterpreting to the new size.
2301 if (!arch_supports_vector(Op_VectorReinterpret,
2302 num_elem_to,
2303 elem_bt_to,
2304 is_mask ? VecMaskUseAll : VecMaskNotUsed)) {
2305 log_if_needed(" ** not supported: arity=1 op=%s/2 vlen2=%d etype2=%s ismask=%d",
2306 is_cast ? "cast" : "reinterpret",
2307 num_elem_to, type2name(elem_bt_to), is_mask);
2308 return false;
2309 }
2310
2311 // At this point, we know that both input and output vector registers are supported
2312 // by the architecture. Next check if the casted type is simply to same type - which means
2313 // that it is actually a resize and not a cast.
2314 if (is_cast && elem_bt_from == elem_bt_to) {
2315 is_cast = false;
2316 }
2317
2318 const TypeInstPtr* vbox_type_from = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass_from);
2319
2320 Node* opd1 = unbox_vector(argument(7), vbox_type_from, elem_bt_from, num_elem_from);
2321 if (opd1 == nullptr) {
2322 return false;
2323 }
2324
2325 const TypeVect* src_type = TypeVect::make(elem_bt_from, num_elem_from, is_mask);
2326 const TypeVect* dst_type = TypeVect::make(elem_bt_to, num_elem_to, is_mask);
2327
2328 // Safety check to prevent casting if source mask is of type vector
2329 // and destination mask of type predicate vector and vice-versa.
2330 // From X86 standpoint, this case will only arise over KNL target,
2331 // where certain masks (depending on the species) are either propagated
2332 // through a vector or predicate register.
2333 if (is_mask &&
2334 ((src_type->isa_vectmask() == nullptr && dst_type->isa_vectmask()) ||
2335 (dst_type->isa_vectmask() == nullptr && src_type->isa_vectmask()))) {
2336 return false;
2337 }
2338
2339 Node* op = opd1;
2340 if (is_cast) {
2341 assert(!is_mask || num_elem_from == num_elem_to, "vector mask cast needs the same elem num");
2342 int cast_vopc = VectorCastNode::opcode(-1, elem_bt_from, !is_ucast);
2343
2344 // Make sure that vector cast is implemented to particular type/size combination if it is
2345 // not a mask casting.
2346 if (!is_mask && !arch_supports_vector(cast_vopc, num_elem_to, elem_bt_to, VecMaskNotUsed)) {
2347 log_if_needed(" ** not supported: arity=1 op=cast#%d/3 vlen2=%d etype2=%s ismask=%d",
2348 cast_vopc, num_elem_to, type2name(elem_bt_to), is_mask);
2349 return false;
2350 }
2351
2352 if (num_elem_from < num_elem_to) {
2353 // Since input and output number of elements are not consistent, we need to make sure we
2354 // properly size. Thus, first make a cast that retains the number of elements from source.
2355 int num_elem_for_cast = num_elem_from;
2356
2357 // It is possible that arch does not support this intermediate vector size
2358 // TODO More complex logic required here to handle this corner case for the sizes.
2359 if (!arch_supports_vector(cast_vopc, num_elem_for_cast, elem_bt_to, VecMaskNotUsed)) {
2360 log_if_needed(" ** not supported: arity=1 op=cast#%d/4 vlen1=%d etype2=%s ismask=%d",
2361 cast_vopc,
2362 num_elem_for_cast, type2name(elem_bt_to), is_mask);
2363 return false;
2364 }
2365
2366 op = gvn().transform(VectorCastNode::make(cast_vopc, op, elem_bt_to, num_elem_for_cast));
2367 // Now ensure that the destination gets properly resized to needed size.
2368 op = gvn().transform(new VectorReinterpretNode(op, op->bottom_type()->is_vect(), dst_type));
2369 } else if (num_elem_from > num_elem_to) {
2370 // Since number of elements from input is larger than output, simply reduce size of input
2371 // (we are supposed to drop top elements anyway).
2372 int num_elem_for_resize = num_elem_to;
2373
2374 // It is possible that arch does not support this intermediate vector size
2375 // TODO More complex logic required here to handle this corner case for the sizes.
2376 if (!arch_supports_vector(Op_VectorReinterpret,
2377 num_elem_for_resize,
2378 elem_bt_from,
2379 VecMaskNotUsed)) {
2380 log_if_needed(" ** not supported: arity=1 op=cast/5 vlen2=%d etype1=%s ismask=%d",
2381 num_elem_for_resize, type2name(elem_bt_from), is_mask);
2382 return false;
2383 }
2384
2385 const TypeVect* resize_type = TypeVect::make(elem_bt_from, num_elem_for_resize);
2386 op = gvn().transform(new VectorReinterpretNode(op, src_type, resize_type));
2387 op = gvn().transform(VectorCastNode::make(cast_vopc, op, elem_bt_to, num_elem_to));
2388 } else { // num_elem_from == num_elem_to
2389 if (is_mask) {
2390 // Make sure that cast for vector mask is implemented to particular type/size combination.
2391 if (!arch_supports_vector(Op_VectorMaskCast, num_elem_to, elem_bt_to, VecMaskNotUsed)) {
2392 log_if_needed(" ** not supported: arity=1 op=maskcast vlen2=%d etype2=%s ismask=%d",
2393 num_elem_to, type2name(elem_bt_to), is_mask);
2394 return false;
2395 }
2396 op = gvn().transform(new VectorMaskCastNode(op, dst_type));
2397 } else {
2398 // Since input and output number of elements match, and since we know this vector size is
2399 // supported, simply do a cast with no resize needed.
2400 op = gvn().transform(VectorCastNode::make(cast_vopc, op, elem_bt_to, num_elem_to));
2401 }
2402 }
2403 } else if (!Type::equals(src_type, dst_type)) {
2404 assert(!is_cast, "must be reinterpret");
2405 op = gvn().transform(new VectorReinterpretNode(op, src_type, dst_type));
2406 }
2407
2408 const TypeInstPtr* vbox_type_to = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass_to);
2409 Node* vbox = box_vector(op, vbox_type_to, elem_bt_to, num_elem_to);
2410 set_result(vbox);
2411 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem_to * type2aelembytes(elem_bt_to))));
2412 return true;
2413 }
2414
2415 // public static
2416 // <V extends Vector<E>,
2417 // E>
2418 // V insert(Class<? extends V> vectorClass, Class<E> elementType, int vlen,
2419 // V vec, int ix, long val,
2420 // VecInsertOp<V> defaultImpl)
2421 bool LibraryCallKit::inline_vector_insert() {
2422 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
2423 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
2424 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
2425 const TypeInt* idx = gvn().type(argument(4))->isa_int();
2426
2427 if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr || idx == nullptr) {
2428 return false; // dead code
2429 }
2430 if (vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con() || !idx->is_con()) {
2431 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s idx=%s",
2432 NodeClassNames[argument(0)->Opcode()],
2433 NodeClassNames[argument(1)->Opcode()],
2434 NodeClassNames[argument(2)->Opcode()],
2435 NodeClassNames[argument(4)->Opcode()]);
2436 return false; // not enough info for intrinsification
2437 }
2438 if (!is_klass_initialized(vector_klass)) {
2439 log_if_needed(" ** klass argument not initialized");
2440 return false;
2441 }
2442 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2443 if (!elem_type->is_primitive_type()) {
2444 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2445 return false; // should be primitive type
2446 }
2447 BasicType elem_bt = elem_type->basic_type();
2448 int num_elem = vlen->get_con();
2449 if (!arch_supports_vector(Op_VectorInsert, num_elem, elem_bt, VecMaskNotUsed)) {
2450 log_if_needed(" ** not supported: arity=1 op=insert vlen=%d etype=%s ismask=no",
2451 num_elem, type2name(elem_bt));
2452 return false; // not supported
2453 }
2454
2455 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2456 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2457
2458 Node* opd = unbox_vector(argument(3), vbox_type, elem_bt, num_elem);
2459 if (opd == nullptr) {
2460 return false;
2461 }
2462
2463 Node* insert_val = argument(5);
2464 assert(gvn().type(insert_val)->isa_long() != nullptr, "expected to be long");
2465
2466 // Convert insert value back to its appropriate type.
2467 switch (elem_bt) {
2468 case T_BYTE:
2469 insert_val = gvn().transform(new ConvL2INode(insert_val, TypeInt::BYTE));
2470 break;
2471 case T_SHORT:
2472 insert_val = gvn().transform(new ConvL2INode(insert_val, TypeInt::SHORT));
2473 break;
2474 case T_INT:
2475 insert_val = gvn().transform(new ConvL2INode(insert_val));
2476 break;
2477 case T_FLOAT:
2478 insert_val = gvn().transform(new ConvL2INode(insert_val));
2479 insert_val = gvn().transform(new MoveI2FNode(insert_val));
2480 break;
2481 case T_DOUBLE:
2482 insert_val = gvn().transform(new MoveL2DNode(insert_val));
2483 break;
2484 case T_LONG:
2485 // no conversion needed
2486 break;
2487 default: fatal("%s", type2name(elem_bt)); break;
2488 }
2489
2490 Node* operation = gvn().transform(VectorInsertNode::make(opd, insert_val, idx->get_con(), gvn()));
2491
2492 Node* vbox = box_vector(operation, vbox_type, elem_bt, num_elem);
2493 set_result(vbox);
2494 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2495 return true;
2496 }
2497
2498 // public static
2499 // <VM extends VectorPayload,
2500 // E>
2501 // long extract(Class<? extends VM> vClass, Class<E> eClass,
2502 // int length,
2503 // VM vm, int i,
2504 // VecExtractOp<VM> defaultImpl)
2505 bool LibraryCallKit::inline_vector_extract() {
2506 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
2507 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
2508 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
2509 const TypeInt* idx = gvn().type(argument(4))->isa_int();
2510
2511 if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr || idx == nullptr) {
2512 return false; // dead code
2513 }
2514 if (vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
2515 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s",
2516 NodeClassNames[argument(0)->Opcode()],
2517 NodeClassNames[argument(1)->Opcode()],
2518 NodeClassNames[argument(2)->Opcode()]);
2519 return false; // not enough info for intrinsification
2520 }
2521 if (!is_klass_initialized(vector_klass)) {
2522 log_if_needed(" ** klass argument not initialized");
2523 return false;
2524 }
2525 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2526 if (!elem_type->is_primitive_type()) {
2527 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2528 return false; // should be primitive type
2529 }
2530 BasicType elem_bt = elem_type->basic_type();
2531 int num_elem = vlen->get_con();
2532
2533 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2534 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2535
2536 Node* opd = nullptr;
2537
2538 if (is_vector_mask(vbox_klass)) {
2539 // vbox_klass is mask. This is used for VectorMask.laneIsSet(int).
2540
2541 Node* pos = argument(4); // can be variable
2542 if (arch_supports_vector(Op_ExtractUB, num_elem, elem_bt, VecMaskUseAll)) {
2543 // Transform mask to vector with type of boolean and utilize ExtractUB node.
2544 opd = unbox_vector(argument(3), vbox_type, elem_bt, num_elem);
2545 if (opd == nullptr) {
2546 return false;
2547 }
2548 opd = gvn().transform(VectorStoreMaskNode::make(gvn(), opd, elem_bt, num_elem));
2549 opd = gvn().transform(new ExtractUBNode(opd, pos));
2550 opd = gvn().transform(new ConvI2LNode(opd));
2551 } else if (arch_supports_vector(Op_VectorMaskToLong, num_elem, elem_bt, VecMaskUseLoad)) {
2552 opd = unbox_vector(argument(3), vbox_type, elem_bt, num_elem);
2553 if (opd == nullptr) {
2554 return false;
2555 }
2556 // VectorMaskToLongNode requires the input is either a mask or a vector with BOOLEAN type.
2557 if (opd->bottom_type()->isa_vectmask() == nullptr) {
2558 opd = gvn().transform(VectorStoreMaskNode::make(gvn(), opd, elem_bt, num_elem));
2559 }
2560 // ((toLong() >>> pos) & 1L
2561 opd = gvn().transform(new VectorMaskToLongNode(opd, TypeLong::LONG));
2562 opd = gvn().transform(new URShiftLNode(opd, pos));
2563 opd = gvn().transform(new AndLNode(opd, gvn().makecon(TypeLong::ONE)));
2564 } else {
2565 log_if_needed(" ** Rejected mask extraction because architecture does not support it");
2566 return false; // not supported
2567 }
2568 } else {
2569 // vbox_klass is vector. This is used for Vector.lane(int).
2570 if (!idx->is_con()) {
2571 log_if_needed(" ** missing constant: idx=%s", NodeClassNames[argument(4)->Opcode()]);
2572 return false; // not enough info for intrinsification
2573 }
2574
2575 int vopc = ExtractNode::opcode(elem_bt);
2576 if (!arch_supports_vector(vopc, num_elem, elem_bt, VecMaskNotUsed)) {
2577 log_if_needed(" ** not supported: arity=1 op=extract vlen=%d etype=%s ismask=no",
2578 num_elem, type2name(elem_bt));
2579 return false; // not supported
2580 }
2581
2582 opd = unbox_vector(argument(3), vbox_type, elem_bt, num_elem);
2583 if (opd == nullptr) {
2584 return false;
2585 }
2586 ConINode* idx_con = gvn().intcon(idx->get_con())->as_ConI();
2587
2588 opd = gvn().transform(ExtractNode::make(opd, idx_con, elem_bt));
2589 switch (elem_bt) {
2590 case T_BYTE:
2591 case T_SHORT:
2592 case T_INT: {
2593 opd = gvn().transform(new ConvI2LNode(opd));
2594 break;
2595 }
2596 case T_FLOAT: {
2597 opd = gvn().transform(new MoveF2INode(opd));
2598 opd = gvn().transform(new ConvI2LNode(opd));
2599 break;
2600 }
2601 case T_DOUBLE: {
2602 opd = gvn().transform(new MoveD2LNode(opd));
2603 break;
2604 }
2605 case T_LONG: {
2606 // no conversion needed
2607 break;
2608 }
2609 default: fatal("%s", type2name(elem_bt));
2610 }
2611 }
2612 set_result(opd);
2613 return true;
2614 }
2615
2616 static Node* LowerSelectFromTwoVectorOperation(PhaseGVN& phase, Node* index_vec, Node* src1, Node* src2, const TypeVect* vt) {
2617 int num_elem = vt->length();
2618 BasicType elem_bt = vt->element_basic_type();
2619
2620 // Lower selectFrom operation into its constituent operations.
2621 // SelectFromTwoVectorNode =
2622 // (VectorBlend
2623 // (VectorRearrange SRC1 (WRAPED_INDEX AND (VLEN-1))
2624 // (VectorRearrange SRC2 (WRAPED_INDEX AND (VLEN-1))
2625 // MASK)
2626 // Where
2627 // WRAPED_INDEX are computed by wrapping incoming indexes
2628 // to two vector index range [0, VLEN*2) and
2629 // MASK = WRAPED_INDEX < VLEN
2630 //
2631 // IR lowering prevents intrinsification failure and associated argument
2632 // boxing penalties.
2633 //
2634
2635 BasicType shuffle_bt = elem_bt;
2636 if (shuffle_bt == T_FLOAT) {
2637 shuffle_bt = T_INT;
2638 } else if (shuffle_bt == T_DOUBLE) {
2639 shuffle_bt = T_LONG;
2640 }
2641 const TypeVect* st = TypeVect::make(shuffle_bt, num_elem);
2642
2643 // Cast index vector to the corresponding bit type
2644 if (elem_bt != shuffle_bt) {
2645 int cast_vopc = VectorCastNode::opcode(0, elem_bt, true);
2646 index_vec = phase.transform(VectorCastNode::make(cast_vopc, index_vec, shuffle_bt, num_elem));
2647 }
2648
2649 // Wrap indexes into two vector index range [0, VLEN * 2)
2650 Node* two_vect_lane_cnt_m1 = phase.makecon(TypeInteger::make(2 * num_elem - 1, 2 * num_elem - 1, Type::WidenMin, shuffle_bt == T_LONG ? T_LONG : T_INT));
2651 Node* bcast_two_vect_lane_cnt_m1_vec = phase.transform(VectorNode::scalar2vector(two_vect_lane_cnt_m1, num_elem,
2652 shuffle_bt, false));
2653 index_vec = phase.transform(VectorNode::make(Op_AndV, index_vec, bcast_two_vect_lane_cnt_m1_vec, st));
2654
2655 // Compute the blend mask for merging two independently permitted vectors
2656 // using shuffle index in two vector index range [0, VLEN * 2).
2657 BoolTest::mask pred = BoolTest::le;
2658 ConINode* pred_node = phase.makecon(TypeInt::make(pred))->as_ConI();
2659 const TypeVect* vmask_type = TypeVect::makemask(shuffle_bt, num_elem);
2660 Node* lane_cnt_m1 = phase.makecon(TypeInteger::make(num_elem - 1, num_elem - 1, Type::WidenMin, shuffle_bt == T_LONG ? T_LONG : T_INT));
2661 Node* bcast_lane_cnt_m1_vec = phase.transform(VectorNode::scalar2vector(lane_cnt_m1, num_elem, shuffle_bt, false));
2662 Node* mask = phase.transform(new VectorMaskCmpNode(pred, index_vec, bcast_lane_cnt_m1_vec, pred_node, vmask_type));
2663
2664 // Rearrange expects the indexes to lie within single vector index range [0, VLEN).
2665 Node* wrapped_index_vec = phase.transform(VectorNode::make(Op_AndV, index_vec, bcast_lane_cnt_m1_vec, st));
2666
2667 // Load indexes from byte vector and appropriately transform them to target
2668 // specific permutation index format.
2669 if (Matcher::vector_rearrange_requires_load_shuffle(shuffle_bt, num_elem)) {
2670 wrapped_index_vec = phase.transform(new VectorLoadShuffleNode(wrapped_index_vec, st));
2671 }
2672
2673 vmask_type = TypeVect::makemask(elem_bt, num_elem);
2674 mask = phase.transform(new VectorMaskCastNode(mask, vmask_type));
2675
2676 Node* p1 = phase.transform(new VectorRearrangeNode(src1, wrapped_index_vec));
2677 Node* p2 = phase.transform(new VectorRearrangeNode(src2, wrapped_index_vec));
2678
2679 return new VectorBlendNode(p2, p1, mask);
2680 }
2681
2682 // public static
2683 // <V extends Vector<E>,
2684 // E>
2685 // V selectFromTwoVectorOp(Class<? extends V> vClass, Class<E> eClass, int length,
2686 // V v1, V v2, V v3,
2687 // SelectFromTwoVector<V> defaultImpl)
2688 bool LibraryCallKit::inline_vector_select_from_two_vectors() {
2689 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
2690 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
2691 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
2692
2693 if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr || vector_klass->const_oop() == nullptr ||
2694 elem_klass->const_oop() == nullptr ||!vlen->is_con()) {
2695 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s",
2696 NodeClassNames[argument(0)->Opcode()],
2697 NodeClassNames[argument(1)->Opcode()],
2698 NodeClassNames[argument(2)->Opcode()]);
2699 return false; // not enough info for intrinsification
2700 }
2701
2702 if (!is_klass_initialized(vector_klass)) {
2703 log_if_needed(" ** klass argument not initialized");
2704 return false;
2705 }
2706
2707 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2708 if (!elem_type->is_primitive_type()) {
2709 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2710 return false; // should be primitive type
2711 }
2712
2713 int num_elem = vlen->get_con();
2714 if (!is_power_of_2(num_elem)) {
2715 log_if_needed(" ** vlen is not power of two=%d", num_elem);
2716 return false;
2717 }
2718
2719 BasicType elem_bt = elem_type->basic_type();
2720 BasicType index_elem_bt = elem_bt;
2721 if (elem_bt == T_FLOAT) {
2722 index_elem_bt = T_INT;
2723 } else if (elem_bt == T_DOUBLE) {
2724 index_elem_bt = T_LONG;
2725 }
2726
2727 bool lowerSelectFromOp = false;
2728 if (!arch_supports_vector(Op_SelectFromTwoVector, num_elem, elem_bt, VecMaskNotUsed)) {
2729 int cast_vopc = VectorCastNode::opcode(-1, elem_bt, true);
2730 if ((elem_bt != index_elem_bt && !arch_supports_vector(cast_vopc, num_elem, index_elem_bt, VecMaskNotUsed)) ||
2731 !arch_supports_vector(Op_VectorMaskCmp, num_elem, index_elem_bt, VecMaskNotUsed) ||
2732 !arch_supports_vector(Op_AndV, num_elem, index_elem_bt, VecMaskNotUsed) ||
2733 !arch_supports_vector(Op_VectorMaskCast, num_elem, elem_bt, VecMaskNotUsed) ||
2734 !arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad) ||
2735 !arch_supports_vector(Op_VectorRearrange, num_elem, elem_bt, VecMaskNotUsed) ||
2736 !arch_supports_vector(Op_VectorLoadShuffle, num_elem, index_elem_bt, VecMaskNotUsed) ||
2737 !arch_supports_vector(Op_Replicate, num_elem, index_elem_bt, VecMaskNotUsed)) {
2738 log_if_needed(" ** not supported: opc=%d vlen=%d etype=%s ismask=useload",
2739 Op_SelectFromTwoVector, num_elem, type2name(elem_bt));
2740 return false; // not supported
2741 }
2742 lowerSelectFromOp = true;
2743 }
2744
2745 int cast_vopc = VectorCastNode::opcode(-1, elem_bt, true);
2746 if (!lowerSelectFromOp) {
2747 if (!arch_supports_vector(Op_AndV, num_elem, index_elem_bt, VecMaskNotUsed) ||
2748 !arch_supports_vector(Op_Replicate, num_elem, index_elem_bt, VecMaskNotUsed) ||
2749 (is_floating_point_type(elem_bt) &&
2750 !arch_supports_vector(cast_vopc, num_elem, index_elem_bt, VecMaskNotUsed))) {
2751 log_if_needed(" ** index wrapping not supported: vlen=%d etype=%s" ,
2752 num_elem, type2name(elem_bt));
2753 return false; // not supported
2754 }
2755 }
2756
2757 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2758 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2759
2760 Node* opd1 = unbox_vector(argument(3), vbox_type, elem_bt, num_elem);
2761 if (opd1 == nullptr) {
2762 log_if_needed(" ** unbox failed v1=%s",
2763 NodeClassNames[argument(3)->Opcode()]);
2764 return false;
2765 }
2766 Node* opd2 = unbox_vector(argument(4), vbox_type, elem_bt, num_elem);
2767 if (opd2 == nullptr) {
2768 log_if_needed(" ** unbox failed v2=%s",
2769 NodeClassNames[argument(4)->Opcode()]);
2770 return false;
2771 }
2772 Node* opd3 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
2773 if (opd3 == nullptr) {
2774 log_if_needed(" ** unbox failed v3=%s",
2775 NodeClassNames[argument(5)->Opcode()]);
2776 return false;
2777 }
2778
2779 const TypeVect* vt = TypeVect::make(elem_bt, num_elem);
2780
2781 Node* operation = nullptr;
2782 if (lowerSelectFromOp) {
2783 operation = gvn().transform(LowerSelectFromTwoVectorOperation(gvn(), opd1, opd2, opd3, vt));
2784 } else {
2785 if (index_elem_bt != elem_bt) {
2786 opd1 = gvn().transform(VectorCastNode::make(cast_vopc, opd1, index_elem_bt, num_elem));
2787 }
2788 int indexRangeMask = 2 * num_elem - 1;
2789 Node* wrap_mask = gvn().makecon(TypeInteger::make(indexRangeMask, indexRangeMask, Type::WidenMin, index_elem_bt != T_LONG ? T_INT : index_elem_bt));
2790 Node* wrap_mask_vec = gvn().transform(VectorNode::scalar2vector(wrap_mask, num_elem, index_elem_bt, false));
2791 opd1 = gvn().transform(VectorNode::make(Op_AndV, opd1, wrap_mask_vec, opd1->bottom_type()->is_vect()));
2792 operation = gvn().transform(VectorNode::make(Op_SelectFromTwoVector, opd1, opd2, opd3, vt));
2793 }
2794
2795 // Wrap it up in VectorBox to keep object type information.
2796 Node* vbox = box_vector(operation, vbox_type, elem_bt, num_elem);
2797 set_result(vbox);
2798 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2799 return true;
2800 }
2801
2802 // public static
2803 // <V extends Vector<E>,
2804 // M extends VectorMask<E>,
2805 // E>
2806 // V compressExpandOp(int opr,
2807 // Class<? extends V> vClass, Class<? extends M> mClass, Class<E> eClass,
2808 // int length, V v, M m,
2809 // CompressExpandOperation<V, M> defaultImpl)
2810 bool LibraryCallKit::inline_vector_compress_expand() {
2811 const TypeInt* opr = gvn().type(argument(0))->isa_int();
2812 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
2813 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
2814 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
2815 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
2816
2817 if (vector_klass == nullptr || elem_klass == nullptr || mask_klass == nullptr || vlen == nullptr ||
2818 vector_klass->const_oop() == nullptr || mask_klass->const_oop() == nullptr ||
2819 elem_klass->const_oop() == nullptr || !vlen->is_con() || !opr->is_con()) {
2820 log_if_needed(" ** missing constant: opr=%s vclass=%s mclass=%s etype=%s vlen=%s",
2821 NodeClassNames[argument(0)->Opcode()],
2822 NodeClassNames[argument(1)->Opcode()],
2823 NodeClassNames[argument(2)->Opcode()],
2824 NodeClassNames[argument(3)->Opcode()],
2825 NodeClassNames[argument(4)->Opcode()]);
2826 return false; // not enough info for intrinsification
2827 }
2828
2829 if (!is_klass_initialized(vector_klass) || !is_klass_initialized(mask_klass)) {
2830 log_if_needed(" ** klass argument not initialized");
2831 return false;
2832 }
2833
2834 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2835 if (!elem_type->is_primitive_type()) {
2836 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2837 return false; // should be primitive type
2838 }
2839
2840 int num_elem = vlen->get_con();
2841 BasicType elem_bt = elem_type->basic_type();
2842 int opc = VectorSupport::vop2ideal(opr->get_con(), elem_bt);
2843
2844 if (!arch_supports_vector(opc, num_elem, elem_bt, VecMaskUseLoad)) {
2845 log_if_needed(" ** not supported: opc=%d vlen=%d etype=%s ismask=useload",
2846 opc, num_elem, type2name(elem_bt));
2847 return false; // not supported
2848 }
2849
2850 Node* opd1 = nullptr;
2851 const TypeInstPtr* vbox_type = nullptr;
2852 if (opc != Op_CompressM) {
2853 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2854 vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2855 opd1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
2856 if (opd1 == nullptr) {
2857 log_if_needed(" ** unbox failed vector=%s",
2858 NodeClassNames[argument(5)->Opcode()]);
2859 return false;
2860 }
2861 }
2862
2863 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
2864 assert(is_vector_mask(mbox_klass), "argument(6) should be a mask class");
2865 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
2866
2867 Node* mask = unbox_vector(argument(6), mbox_type, elem_bt, num_elem);
2868 if (mask == nullptr) {
2869 log_if_needed(" ** unbox failed mask=%s",
2870 NodeClassNames[argument(6)->Opcode()]);
2871 return false;
2872 }
2873
2874 const TypeVect* vt = TypeVect::make(elem_bt, num_elem, opc == Op_CompressM);
2875 Node* operation = gvn().transform(VectorNode::make(opc, opd1, mask, vt));
2876
2877 // Wrap it up in VectorBox to keep object type information.
2878 const TypeInstPtr* box_type = opc == Op_CompressM ? mbox_type : vbox_type;
2879 Node* vbox = box_vector(operation, box_type, elem_bt, num_elem);
2880 set_result(vbox);
2881 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2882 return true;
2883 }
2884
2885 // public static
2886 // <V extends Vector<E>,
2887 // E,
2888 // S extends VectorSpecies<E>>
2889 // V indexVector(Class<? extends V> vClass, Class<E> eClass,
2890 // int length,
2891 // V v, int step, S s,
2892 // IndexOperation<V, S> defaultImpl)
2893 bool LibraryCallKit::inline_index_vector() {
2894 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
2895 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
2896 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
2897
2898 if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr ||
2899 vector_klass->const_oop() == nullptr || !vlen->is_con() ||
2900 elem_klass->const_oop() == nullptr) {
2901 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s",
2902 NodeClassNames[argument(0)->Opcode()],
2903 NodeClassNames[argument(1)->Opcode()],
2904 NodeClassNames[argument(2)->Opcode()]);
2905 return false; // not enough info for intrinsification
2906 }
2907
2908 if (!is_klass_initialized(vector_klass)) {
2909 log_if_needed(" ** klass argument not initialized");
2910 return false;
2911 }
2912
2913 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2914 if (!elem_type->is_primitive_type()) {
2915 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2916 return false; // should be primitive type
2917 }
2918
2919 int num_elem = vlen->get_con();
2920 BasicType elem_bt = elem_type->basic_type();
2921
2922 // Check whether the iota index generation op is supported by the current hardware
2923 if (!arch_supports_vector(Op_VectorLoadConst, num_elem, elem_bt, VecMaskNotUsed)) {
2924 log_if_needed(" ** not supported: vlen=%d etype=%s", num_elem, type2name(elem_bt));
2925 return false; // not supported
2926 }
2927
2928 int mul_op = VectorSupport::vop2ideal(VectorSupport::VECTOR_OP_MUL, elem_bt);
2929 int vmul_op = VectorNode::opcode(mul_op, elem_bt);
2930 bool needs_mul = true;
2931 Node* scale = argument(4);
2932 const TypeInt* scale_type = gvn().type(scale)->isa_int();
2933 // Multiply is not needed if the scale is a constant "1".
2934 if (scale_type && scale_type->is_con() && scale_type->get_con() == 1) {
2935 needs_mul = false;
2936 } else {
2937 // Check whether the vector multiply op is supported by the current hardware
2938 if (!arch_supports_vector(vmul_op, num_elem, elem_bt, VecMaskNotUsed)) {
2939 log_if_needed(" ** not supported: vlen=%d etype=%s", num_elem, type2name(elem_bt));
2940 return false; // not supported
2941 }
2942
2943 // Check whether the scalar cast op is supported by the current hardware
2944 if (is_floating_point_type(elem_bt) || elem_bt == T_LONG) {
2945 int cast_op = elem_bt == T_LONG ? Op_ConvI2L :
2946 elem_bt == T_FLOAT? Op_ConvI2F : Op_ConvI2D;
2947 if (!Matcher::match_rule_supported(cast_op)) {
2948 log_if_needed(" ** Rejected op (%s) because architecture does not support it",
2949 NodeClassNames[cast_op]);
2950 return false; // not supported
2951 }
2952 }
2953 }
2954
2955 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2956 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2957 Node* opd = unbox_vector(argument(3), vbox_type, elem_bt, num_elem);
2958 if (opd == nullptr) {
2959 log_if_needed(" ** unbox failed vector=%s",
2960 NodeClassNames[argument(3)->Opcode()]);
2961 return false;
2962 }
2963
2964 int add_op = VectorSupport::vop2ideal(VectorSupport::VECTOR_OP_ADD, elem_bt);
2965 int vadd_op = VectorNode::opcode(add_op, elem_bt);
2966 bool needs_add = true;
2967 // The addition is not needed if all the element values of "opd" are zero
2968 if (VectorNode::is_all_zeros_vector(opd)) {
2969 needs_add = false;
2970 } else {
2971 // Check whether the vector addition op is supported by the current hardware
2972 if (!arch_supports_vector(vadd_op, num_elem, elem_bt, VecMaskNotUsed)) {
2973 log_if_needed(" ** not supported: vlen=%d etype=%s", num_elem, type2name(elem_bt));
2974 return false; // not supported
2975 }
2976 }
2977
2978 // Compute the iota indice vector
2979 const TypeVect* vt = TypeVect::make(elem_bt, num_elem);
2980 Node* index = gvn().transform(new VectorLoadConstNode(gvn().makecon(TypeInt::ZERO), vt));
2981
2982 // Broadcast the "scale" to a vector, and multiply the "scale" with iota indice vector.
2983 if (needs_mul) {
2984 switch (elem_bt) {
2985 case T_BOOLEAN: // fall-through
2986 case T_BYTE: // fall-through
2987 case T_SHORT: // fall-through
2988 case T_CHAR: // fall-through
2989 case T_INT: {
2990 // no conversion needed
2991 break;
2992 }
2993 case T_LONG: {
2994 scale = gvn().transform(new ConvI2LNode(scale));
2995 break;
2996 }
2997 case T_FLOAT: {
2998 scale = gvn().transform(new ConvI2FNode(scale));
2999 break;
3000 }
3001 case T_DOUBLE: {
3002 scale = gvn().transform(new ConvI2DNode(scale));
3003 break;
3004 }
3005 default: fatal("%s", type2name(elem_bt));
3006 }
3007 scale = gvn().transform(VectorNode::scalar2vector(scale, num_elem, elem_bt));
3008 index = gvn().transform(VectorNode::make(vmul_op, index, scale, vt));
3009 }
3010
3011 // Add "opd" if addition is needed.
3012 if (needs_add) {
3013 index = gvn().transform(VectorNode::make(vadd_op, opd, index, vt));
3014 }
3015 Node* vbox = box_vector(index, vbox_type, elem_bt, num_elem);
3016 set_result(vbox);
3017 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
3018 return true;
3019 }
3020
3021 // public static
3022 // <E,
3023 // M extends VectorMask<E>>
3024 // M indexPartiallyInUpperRange(Class<? extends M> mClass, Class<E> eClass, int length,
3025 // long offset, long limit,
3026 // IndexPartiallyInUpperRangeOperation<E, M> defaultImpl)
3027 bool LibraryCallKit::inline_index_partially_in_upper_range() {
3028 const TypeInstPtr* mask_klass = gvn().type(argument(0))->isa_instptr();
3029 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
3030 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
3031
3032 if (mask_klass == nullptr || elem_klass == nullptr || vlen == nullptr ||
3033 mask_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
3034 log_if_needed(" ** missing constant: mclass=%s etype=%s vlen=%s",
3035 NodeClassNames[argument(0)->Opcode()],
3036 NodeClassNames[argument(1)->Opcode()],
3037 NodeClassNames[argument(2)->Opcode()]);
3038 return false; // not enough info for intrinsification
3039 }
3040
3041 if (!is_klass_initialized(mask_klass)) {
3042 log_if_needed(" ** klass argument not initialized");
3043 return false;
3044 }
3045
3046 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
3047 if (!elem_type->is_primitive_type()) {
3048 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
3049 return false; // should be primitive type
3050 }
3051
3052 int num_elem = vlen->get_con();
3053 BasicType elem_bt = elem_type->basic_type();
3054
3055 // Check whether the necessary ops are supported by current hardware.
3056 bool supports_mask_gen = arch_supports_vector(Op_VectorMaskGen, num_elem, elem_bt, VecMaskUseStore);
3057 if (!supports_mask_gen) {
3058 if (!arch_supports_vector(Op_VectorLoadConst, num_elem, elem_bt, VecMaskNotUsed) ||
3059 !arch_supports_vector(Op_Replicate, num_elem, elem_bt, VecMaskNotUsed) ||
3060 !arch_supports_vector(Op_VectorMaskCmp, num_elem, elem_bt, VecMaskUseStore)) {
3061 log_if_needed(" ** not supported: vlen=%d etype=%s", num_elem, type2name(elem_bt));
3062 return false; // not supported
3063 }
3064
3065 // Check whether the scalar cast operation is supported by current hardware.
3066 if (elem_bt != T_LONG) {
3067 int cast_op = is_integral_type(elem_bt) ? Op_ConvL2I
3068 : (elem_bt == T_FLOAT ? Op_ConvL2F : Op_ConvL2D);
3069 if (!Matcher::match_rule_supported(cast_op)) {
3070 log_if_needed(" ** Rejected op (%s) because architecture does not support it",
3071 NodeClassNames[cast_op]);
3072 return false; // not supported
3073 }
3074 }
3075 }
3076
3077 Node* offset = argument(3);
3078 Node* limit = argument(5);
3079 if (offset == nullptr || limit == nullptr) {
3080 log_if_needed(" ** offset or limit argument is null");
3081 return false; // not supported
3082 }
3083
3084 ciKlass* box_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
3085 assert(is_vector_mask(box_klass), "argument(0) should be a mask class");
3086 const TypeInstPtr* box_type = TypeInstPtr::make_exact(TypePtr::NotNull, box_klass);
3087
3088 // We assume "offset > 0 && limit >= offset && limit - offset < num_elem".
3089 // So directly get indexLimit with "indexLimit = limit - offset".
3090 Node* indexLimit = gvn().transform(new SubLNode(limit, offset));
3091 Node* mask = nullptr;
3092 if (supports_mask_gen) {
3093 mask = gvn().transform(VectorMaskGenNode::make(indexLimit, elem_bt, num_elem));
3094 } else {
3095 // Generate the vector mask based on "mask = iota < indexLimit".
3096 // Broadcast "indexLimit" to a vector.
3097 switch (elem_bt) {
3098 case T_BOOLEAN: // fall-through
3099 case T_BYTE: // fall-through
3100 case T_SHORT: // fall-through
3101 case T_CHAR: // fall-through
3102 case T_INT: {
3103 indexLimit = gvn().transform(new ConvL2INode(indexLimit));
3104 break;
3105 }
3106 case T_DOUBLE: {
3107 indexLimit = gvn().transform(new ConvL2DNode(indexLimit));
3108 break;
3109 }
3110 case T_FLOAT: {
3111 indexLimit = gvn().transform(new ConvL2FNode(indexLimit));
3112 break;
3113 }
3114 case T_LONG: {
3115 // no conversion needed
3116 break;
3117 }
3118 default: fatal("%s", type2name(elem_bt));
3119 }
3120 indexLimit = gvn().transform(VectorNode::scalar2vector(indexLimit, num_elem, elem_bt));
3121
3122 // Load the "iota" vector.
3123 const TypeVect* vt = TypeVect::make(elem_bt, num_elem);
3124 Node* iota = gvn().transform(new VectorLoadConstNode(gvn().makecon(TypeInt::ZERO), vt));
3125
3126 // Compute the vector mask with "mask = iota < indexLimit".
3127 ConINode* pred_node = (ConINode*)gvn().makecon(TypeInt::make(BoolTest::lt));
3128 const TypeVect* vmask_type = TypeVect::makemask(elem_bt, num_elem);
3129 mask = gvn().transform(new VectorMaskCmpNode(BoolTest::lt, iota, indexLimit, pred_node, vmask_type));
3130 }
3131 Node* vbox = box_vector(mask, box_type, elem_bt, num_elem);
3132 set_result(vbox);
3133 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
3134 return true;
3135 }
3136
3137 #undef non_product_log_if_needed
3138 #undef log_if_needed