1 /*
2 * Copyright (c) 2020, 2024, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "ci/ciSymbols.hpp"
27 #include "classfile/vmSymbols.hpp"
28 #include "opto/library_call.hpp"
29 #include "opto/runtime.hpp"
30 #include "opto/vectornode.hpp"
31 #include "prims/vectorSupport.hpp"
32 #include "runtime/stubRoutines.hpp"
33
34 #ifdef ASSERT
35 static bool is_vector(ciKlass* klass) {
36 return klass->is_subclass_of(ciEnv::current()->vector_VectorPayload_klass());
37 }
38
39 static bool check_vbox(const TypeInstPtr* vbox_type) {
40 assert(vbox_type->klass_is_exact(), "");
41
42 ciInstanceKlass* ik = vbox_type->instance_klass();
43 assert(is_vector(ik), "not a vector");
44
45 ciField* fd1 = ik->get_field_by_name(ciSymbols::ETYPE_name(), ciSymbols::class_signature(), /* is_static */ true);
46 assert(fd1 != nullptr, "element type info is missing");
47
48 ciConstant val1 = fd1->constant_value();
49 BasicType elem_bt = val1.as_object()->as_instance()->java_mirror_type()->basic_type();
50 assert(is_java_primitive(elem_bt), "element type info is missing");
51
52 ciField* fd2 = ik->get_field_by_name(ciSymbols::VLENGTH_name(), ciSymbols::int_signature(), /* is_static */ true);
53 assert(fd2 != nullptr, "vector length info is missing");
54
55 ciConstant val2 = fd2->constant_value();
56 assert(val2.as_int() > 0, "vector length info is missing");
57
58 return true;
59 }
60 #endif
61
62 #define log_if_needed(...) \
63 if (C->print_intrinsics()) { \
64 tty->print_cr(__VA_ARGS__); \
65 }
66
67 #ifndef PRODUCT
68 #define non_product_log_if_needed(...) log_if_needed(__VA_ARGS__)
69 #else
70 #define non_product_log_if_needed(...)
71 #endif
72
73 static bool is_vector_mask(ciKlass* klass) {
74 return klass->is_subclass_of(ciEnv::current()->vector_VectorMask_klass());
75 }
76
77 static bool is_vector_shuffle(ciKlass* klass) {
78 return klass->is_subclass_of(ciEnv::current()->vector_VectorShuffle_klass());
79 }
80
81 bool LibraryCallKit::arch_supports_vector_rotate(int opc, int num_elem, BasicType elem_bt,
82 VectorMaskUseType mask_use_type, bool has_scalar_args) {
83 bool is_supported = true;
84
85 // has_scalar_args flag is true only for non-constant scalar shift count,
86 // since in this case shift needs to be broadcasted.
87 if (!Matcher::match_rule_supported_vector(opc, num_elem, elem_bt) ||
88 (has_scalar_args && !arch_supports_vector(Op_Replicate, num_elem, elem_bt, VecMaskNotUsed))) {
89 is_supported = false;
90 }
91
92 if (is_supported) {
93 // Check if mask unboxing is supported, this is a two step process which first loads the contents
94 // of boolean array into vector followed by either lane expansion to match the lane size of masked
95 // vector operation or populate the predicate register.
96 if ((mask_use_type & VecMaskUseLoad) != 0) {
97 if (!Matcher::match_rule_supported_vector(Op_VectorLoadMask, num_elem, elem_bt) ||
98 !Matcher::match_rule_supported_vector(Op_LoadVector, num_elem, T_BOOLEAN)) {
99 non_product_log_if_needed(" ** Rejected vector mask loading (%s,%s,%d) because architecture does not support it",
100 NodeClassNames[Op_VectorLoadMask], type2name(elem_bt), num_elem);
101 return false;
102 }
103 }
104
105 if ((mask_use_type & VecMaskUsePred) != 0) {
106 if (!Matcher::has_predicated_vectors() ||
107 !Matcher::match_rule_supported_vector_masked(opc, num_elem, elem_bt)) {
108 non_product_log_if_needed("Rejected vector mask predicate using (%s,%s,%d) because architecture does not support it",
109 NodeClassNames[opc], type2name(elem_bt), num_elem);
110 return false;
111 }
112 }
113 }
114
115 int lshiftopc, rshiftopc;
116 switch(elem_bt) {
117 case T_BYTE:
118 lshiftopc = Op_LShiftI;
119 rshiftopc = Op_URShiftB;
120 break;
121 case T_SHORT:
122 lshiftopc = Op_LShiftI;
123 rshiftopc = Op_URShiftS;
124 break;
125 case T_INT:
126 lshiftopc = Op_LShiftI;
127 rshiftopc = Op_URShiftI;
128 break;
129 case T_LONG:
130 lshiftopc = Op_LShiftL;
131 rshiftopc = Op_URShiftL;
132 break;
133 default: fatal("Unexpected type: %s", type2name(elem_bt));
134 }
135 int lshiftvopc = VectorNode::opcode(lshiftopc, elem_bt);
136 int rshiftvopc = VectorNode::opcode(rshiftopc, elem_bt);
137 if (!is_supported &&
138 arch_supports_vector(lshiftvopc, num_elem, elem_bt, VecMaskNotUsed, has_scalar_args) &&
139 arch_supports_vector(rshiftvopc, num_elem, elem_bt, VecMaskNotUsed, has_scalar_args) &&
140 arch_supports_vector(Op_OrV, num_elem, elem_bt, VecMaskNotUsed)) {
141 is_supported = true;
142 }
143 return is_supported;
144 }
145
146 Node* GraphKit::box_vector(Node* vector, const TypeInstPtr* vbox_type, BasicType elem_bt, int num_elem, bool deoptimize_on_exception) {
147 assert(EnableVectorSupport, "");
148
149 PreserveReexecuteState preexecs(this);
150 jvms()->set_should_reexecute(true);
151
152 VectorBoxAllocateNode* alloc = new VectorBoxAllocateNode(C, vbox_type);
153 set_edges_for_java_call(alloc, /*must_throw=*/false, /*separate_io_proj=*/true);
154 make_slow_call_ex(alloc, env()->Throwable_klass(), /*separate_io_proj=*/true, deoptimize_on_exception);
155 set_i_o(gvn().transform( new ProjNode(alloc, TypeFunc::I_O) ));
156 set_all_memory(gvn().transform( new ProjNode(alloc, TypeFunc::Memory) ));
157 Node* ret = gvn().transform(new ProjNode(alloc, TypeFunc::Parms));
158
159 assert(check_vbox(vbox_type), "");
160 const TypeVect* vt = TypeVect::make(elem_bt, num_elem, is_vector_mask(vbox_type->instance_klass()));
161 VectorBoxNode* vbox = new VectorBoxNode(C, ret, vector, vbox_type, vt);
162 return gvn().transform(vbox);
163 }
164
165 Node* GraphKit::unbox_vector(Node* v, const TypeInstPtr* vbox_type, BasicType elem_bt, int num_elem, bool shuffle_to_vector) {
166 assert(EnableVectorSupport, "");
167 const TypeInstPtr* vbox_type_v = gvn().type(v)->isa_instptr();
168 if (vbox_type_v == nullptr || vbox_type->instance_klass() != vbox_type_v->instance_klass()) {
169 return nullptr; // arguments don't agree on vector shapes
170 }
171 if (vbox_type_v->maybe_null()) {
172 return nullptr; // no nulls are allowed
173 }
174 assert(check_vbox(vbox_type), "");
175 const TypeVect* vt = TypeVect::make(elem_bt, num_elem, is_vector_mask(vbox_type->instance_klass()));
176 Node* unbox = gvn().transform(new VectorUnboxNode(C, vt, v, merged_memory(), shuffle_to_vector));
177 return unbox;
178 }
179
180 Node* GraphKit::vector_shift_count(Node* cnt, int shift_op, BasicType bt, int num_elem) {
181 assert(bt == T_INT || bt == T_LONG || bt == T_SHORT || bt == T_BYTE, "byte, short, long and int are supported");
182 juint mask = (type2aelembytes(bt) * BitsPerByte - 1);
183 Node* nmask = gvn().transform(ConNode::make(TypeInt::make(mask)));
184 Node* mcnt = gvn().transform(new AndINode(cnt, nmask));
185 return gvn().transform(VectorNode::shift_count(shift_op, mcnt, num_elem, bt));
186 }
187
188 bool LibraryCallKit::arch_supports_vector(int sopc, int num_elem, BasicType type, VectorMaskUseType mask_use_type, bool has_scalar_args) {
189 // Check that the operation is valid.
190 if (sopc <= 0) {
191 non_product_log_if_needed(" ** Rejected intrinsification because no valid vector op could be extracted");
192 return false;
193 }
194
195 if (VectorNode::is_vector_rotate(sopc)) {
196 if(!arch_supports_vector_rotate(sopc, num_elem, type, mask_use_type, has_scalar_args)) {
197 non_product_log_if_needed(" ** Rejected vector op (%s,%s,%d) because architecture does not support variable vector shifts",
198 NodeClassNames[sopc], type2name(type), num_elem);
199 return false;
200 }
201 } else if (VectorNode::is_vector_integral_negate(sopc)) {
202 if (!VectorNode::is_vector_integral_negate_supported(sopc, num_elem, type, false)) {
203 non_product_log_if_needed(" ** Rejected vector op (%s,%s,%d) because architecture does not support integral vector negate",
204 NodeClassNames[sopc], type2name(type), num_elem);
205 return false;
206 }
207 } else {
208 // Check that architecture supports this op-size-type combination.
209 if (!Matcher::match_rule_supported_vector(sopc, num_elem, type)) {
210 non_product_log_if_needed(" ** Rejected vector op (%s,%s,%d) because architecture does not support it",
211 NodeClassNames[sopc], type2name(type), num_elem);
212 return false;
213 } else {
214 assert(Matcher::match_rule_supported(sopc), "must be supported");
215 }
216 }
217
218 if (num_elem == 1) {
219 if (mask_use_type != VecMaskNotUsed) {
220 non_product_log_if_needed(" ** Rejected vector mask op (%s,%s,%d) because architecture does not support it",
221 NodeClassNames[sopc], type2name(type), num_elem);
222 return false;
223 }
224
225 if (sopc != 0) {
226 if (sopc != Op_LoadVector && sopc != Op_StoreVector) {
227 non_product_log_if_needed(" ** Not a svml call or load/store vector op (%s,%s,%d)",
228 NodeClassNames[sopc], type2name(type), num_elem);
229 return false;
230 }
231 }
232 }
233
234 if (!has_scalar_args && VectorNode::is_vector_shift(sopc) &&
235 Matcher::supports_vector_variable_shifts() == false) {
236 log_if_needed(" ** Rejected vector op (%s,%s,%d) because architecture does not support variable vector shifts",
237 NodeClassNames[sopc], type2name(type), num_elem);
238 return false;
239 }
240
241 // Check if mask unboxing is supported, this is a two step process which first loads the contents
242 // of boolean array into vector followed by either lane expansion to match the lane size of masked
243 // vector operation or populate the predicate register.
244 if ((mask_use_type & VecMaskUseLoad) != 0) {
245 if (!Matcher::match_rule_supported_vector(Op_VectorLoadMask, num_elem, type) ||
246 !Matcher::match_rule_supported_vector(Op_LoadVector, num_elem, T_BOOLEAN)) {
247 non_product_log_if_needed(" ** Rejected vector mask loading (%s,%s,%d) because architecture does not support it",
248 NodeClassNames[Op_VectorLoadMask], type2name(type), num_elem);
249 return false;
250 }
251 }
252
253 // Check if mask boxing is supported, this is a two step process which first stores the contents
254 // of mask vector / predicate register into a boolean vector followed by vector store operation to
255 // transfer the contents to underlined storage of mask boxes which is a boolean array.
256 if ((mask_use_type & VecMaskUseStore) != 0) {
257 if (!Matcher::match_rule_supported_vector(Op_VectorStoreMask, num_elem, type) ||
258 !Matcher::match_rule_supported_vector(Op_StoreVector, num_elem, T_BOOLEAN)) {
259 non_product_log_if_needed("Rejected vector mask storing (%s,%s,%d) because architecture does not support it",
260 NodeClassNames[Op_VectorStoreMask], type2name(type), num_elem);
261 return false;
262 }
263 }
264
265 if ((mask_use_type & VecMaskUsePred) != 0) {
266 bool is_supported = false;
267 if (Matcher::has_predicated_vectors()) {
268 if (VectorNode::is_vector_integral_negate(sopc)) {
269 is_supported = VectorNode::is_vector_integral_negate_supported(sopc, num_elem, type, true);
270 } else {
271 is_supported = Matcher::match_rule_supported_vector_masked(sopc, num_elem, type);
272 }
273 }
274 is_supported |= Matcher::supports_vector_predicate_op_emulation(sopc, num_elem, type);
275
276 if (!is_supported) {
277 non_product_log_if_needed("Rejected vector mask predicate using (%s,%s,%d) because architecture does not support it",
278 NodeClassNames[sopc], type2name(type), num_elem);
279 return false;
280 }
281 }
282
283 return true;
284 }
285
286 static bool is_klass_initialized(const TypeInstPtr* vec_klass) {
287 if (vec_klass->const_oop() == nullptr) {
288 return false; // uninitialized or some kind of unsafe access
289 }
290 assert(vec_klass->const_oop()->as_instance()->java_lang_Class_klass() != nullptr, "klass instance expected");
291 ciInstanceKlass* klass = vec_klass->const_oop()->as_instance()->java_lang_Class_klass()->as_instance_klass();
292 return klass->is_initialized();
293 }
294
295 // public static
296 // <V extends Vector<E>,
297 // M extends VectorMask<E>,
298 // E>
299 // V unaryOp(int oprId, Class<? extends V> vmClass, Class<? extends M> maskClass, Class<E> elementType,
300 // int length, V v, M m,
301 // UnaryOperation<V, M> defaultImpl)
302 //
303 // public static
304 // <V,
305 // M extends VectorMask<E>,
306 // E>
307 // V binaryOp(int oprId, Class<? extends V> vmClass, Class<? extends M> maskClass, Class<E> elementType,
308 // int length, V v1, V v2, M m,
309 // BinaryOperation<V, M> defaultImpl)
310 //
311 // public static
312 // <V extends Vector<E>,
313 // M extends VectorMask<E>,
314 // E>
315 // V ternaryOp(int oprId, Class<? extends V> vmClass, Class<? extends M> maskClass, Class<E> elementType,
316 // int length, V v1, V v2, V v3, M m,
317 // TernaryOperation<V, M> defaultImpl)
318 //
319 bool LibraryCallKit::inline_vector_nary_operation(int n) {
320 const TypeInt* opr = gvn().type(argument(0))->isa_int();
321 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
322 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
323 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
324 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
325
326 if (opr == nullptr || !opr->is_con() ||
327 vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
328 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
329 vlen == nullptr || !vlen->is_con()) {
330 log_if_needed(" ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
331 NodeClassNames[argument(0)->Opcode()],
332 NodeClassNames[argument(1)->Opcode()],
333 NodeClassNames[argument(3)->Opcode()],
334 NodeClassNames[argument(4)->Opcode()]);
335 return false; // not enough info for intrinsification
336 }
337
338 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
339 if (!elem_type->is_primitive_type()) {
340 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
341 return false; // should be primitive type
342 }
343 if (!is_klass_initialized(vector_klass)) {
344 log_if_needed(" ** klass argument not initialized");
345 return false;
346 }
347
348 // "argument(n + 5)" should be the mask object. We assume it is "null" when no mask
349 // is used to control this operation.
350 const Type* vmask_type = gvn().type(argument(n + 5));
351 bool is_masked_op = vmask_type != TypePtr::NULL_PTR;
352 if (is_masked_op) {
353 if (mask_klass == nullptr || mask_klass->const_oop() == nullptr) {
354 log_if_needed(" ** missing constant: maskclass=%s", NodeClassNames[argument(2)->Opcode()]);
355 return false; // not enough info for intrinsification
356 }
357
358 if (!is_klass_initialized(mask_klass)) {
359 log_if_needed(" ** mask klass argument not initialized");
360 return false;
361 }
362
363 if (vmask_type->maybe_null()) {
364 log_if_needed(" ** null mask values are not allowed for masked op");
365 return false;
366 }
367 }
368
369 BasicType elem_bt = elem_type->basic_type();
370 bool has_scalar_op = VectorSupport::has_scalar_op(opr->get_con());
371 bool is_unsigned = VectorSupport::is_unsigned_op(opr->get_con());
372
373 int num_elem = vlen->get_con();
374 int opc = VectorSupport::vop2ideal(opr->get_con(), elem_bt);
375 int sopc = has_scalar_op ? VectorNode::opcode(opc, elem_bt) : opc;
376 if ((opc != Op_CallLeafVector) && (sopc == 0)) {
377 log_if_needed(" ** operation not supported: opc=%s bt=%s", NodeClassNames[opc], type2name(elem_bt));
378 return false; // operation not supported
379 }
380 if (num_elem == 1) {
381 if (opc != Op_CallLeafVector || elem_bt != T_DOUBLE) {
382 log_if_needed(" ** not a svml call: arity=%d opc=%d vlen=%d etype=%s",
383 n, opc, num_elem, type2name(elem_bt));
384 return false;
385 }
386 }
387 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
388 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
389
390 if (is_vector_mask(vbox_klass)) {
391 assert(!is_masked_op, "mask operations do not need mask to control");
392 }
393
394 if (opc == Op_CallLeafVector) {
395 if (!UseVectorStubs) {
396 log_if_needed(" ** vector stubs support is disabled");
397 return false;
398 }
399 if (!Matcher::supports_vector_calling_convention()) {
400 log_if_needed(" ** no vector calling conventions supported");
401 return false;
402 }
403 if (!Matcher::vector_size_supported(elem_bt, num_elem)) {
404 log_if_needed(" ** vector size (vlen=%d, etype=%s) is not supported",
405 num_elem, type2name(elem_bt));
406 return false;
407 }
408 }
409
410 // When using mask, mask use type needs to be VecMaskUseLoad.
411 VectorMaskUseType mask_use_type = is_vector_mask(vbox_klass) ? VecMaskUseAll
412 : is_masked_op ? VecMaskUseLoad : VecMaskNotUsed;
413 if ((sopc != 0) && !arch_supports_vector(sopc, num_elem, elem_bt, mask_use_type)) {
414 log_if_needed(" ** not supported: arity=%d opc=%d vlen=%d etype=%s ismask=%d is_masked_op=%d",
415 n, sopc, num_elem, type2name(elem_bt),
416 is_vector_mask(vbox_klass) ? 1 : 0, is_masked_op ? 1 : 0);
417 return false; // not supported
418 }
419
420 // Return true if current platform has implemented the masked operation with predicate feature.
421 bool use_predicate = is_masked_op && sopc != 0 && arch_supports_vector(sopc, num_elem, elem_bt, VecMaskUsePred);
422 if (is_masked_op && !use_predicate && !arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad)) {
423 log_if_needed(" ** not supported: arity=%d opc=%d vlen=%d etype=%s ismask=0 is_masked_op=1",
424 n, sopc, num_elem, type2name(elem_bt));
425 return false;
426 }
427
428 Node* opd1 = nullptr; Node* opd2 = nullptr; Node* opd3 = nullptr;
429 switch (n) {
430 case 3: {
431 opd3 = unbox_vector(argument(7), vbox_type, elem_bt, num_elem);
432 if (opd3 == nullptr) {
433 log_if_needed(" ** unbox failed v3=%s",
434 NodeClassNames[argument(7)->Opcode()]);
435 return false;
436 }
437 // fall-through
438 }
439 case 2: {
440 opd2 = unbox_vector(argument(6), vbox_type, elem_bt, num_elem);
441 if (opd2 == nullptr) {
442 log_if_needed(" ** unbox failed v2=%s",
443 NodeClassNames[argument(6)->Opcode()]);
444 return false;
445 }
446 // fall-through
447 }
448 case 1: {
449 opd1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
450 if (opd1 == nullptr) {
451 log_if_needed(" ** unbox failed v1=%s",
452 NodeClassNames[argument(5)->Opcode()]);
453 return false;
454 }
455 break;
456 }
457 default: fatal("unsupported arity: %d", n);
458 }
459
460 Node* mask = nullptr;
461 if (is_masked_op) {
462 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
463 assert(is_vector_mask(mbox_klass), "argument(2) should be a mask class");
464 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
465 mask = unbox_vector(argument(n + 5), mbox_type, elem_bt, num_elem);
466 if (mask == nullptr) {
467 log_if_needed(" ** unbox failed mask=%s",
468 NodeClassNames[argument(n + 5)->Opcode()]);
469 return false;
470 }
471 }
472
473 Node* operation = nullptr;
474 if (opc == Op_CallLeafVector) {
475 assert(UseVectorStubs, "sanity");
476 operation = gen_call_to_vector_math(opr->get_con(), elem_bt, num_elem, opd1, opd2);
477 if (operation == nullptr) {
478 log_if_needed(" ** Vector math call failed for %s_%s_%d",
479 (elem_bt == T_FLOAT) ? "float" : "double",
480 VectorSupport::mathname[opr->get_con() - VectorSupport::VECTOR_OP_MATH_START],
481 num_elem * type2aelembytes(elem_bt));
482 return false;
483 }
484 } else {
485 const TypeVect* vt = TypeVect::make(elem_bt, num_elem, is_vector_mask(vbox_klass));
486 switch (n) {
487 case 1:
488 case 2: {
489 operation = VectorNode::make(sopc, opd1, opd2, vt, is_vector_mask(vbox_klass), VectorNode::is_shift_opcode(opc), is_unsigned);
490 break;
491 }
492 case 3: {
493 operation = VectorNode::make(sopc, opd1, opd2, opd3, vt);
494 break;
495 }
496 default: fatal("unsupported arity: %d", n);
497 }
498 }
499
500 if (is_masked_op && mask != nullptr) {
501 if (use_predicate) {
502 operation->add_req(mask);
503 operation->add_flag(Node::Flag_is_predicated_vector);
504 } else {
505 operation->add_flag(Node::Flag_is_predicated_using_blend);
506 operation = gvn().transform(operation);
507 operation = new VectorBlendNode(opd1, operation, mask);
508 }
509 }
510 operation = gvn().transform(operation);
511
512 // Wrap it up in VectorBox to keep object type information.
513 Node* vbox = box_vector(operation, vbox_type, elem_bt, num_elem);
514 set_result(vbox);
515 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
516 return true;
517 }
518
519 // Following routine generates IR corresponding to AbstractShuffle::partiallyWrapIndex method,
520 // which partially wraps index by modulo VEC_LENGTH and generates a negative index value if original
521 // index is out of valid index range [0, VEC_LENGTH)
522 //
523 // wrapped_index = (VEC_LENGTH - 1) & index
524 // if (index u> VEC_LENGTH) {
525 // wrapped_index -= VEC_LENGTH;
526 //
527 // Note: Unsigned greater than comparison treat both <0 and >VEC_LENGTH indices as out-of-bound
528 // indexes.
529 Node* LibraryCallKit::partially_wrap_indexes(Node* index_vec, int num_elem, BasicType elem_bt) {
530 assert(elem_bt == T_BYTE, "Shuffles use byte array based backing storage.");
531 const TypeVect* vt = TypeVect::make(elem_bt, num_elem);
532
533 Node* mod_mask = gvn().makecon(TypeInt::make(num_elem-1));
534 Node* bcast_mod_mask = gvn().transform(VectorNode::scalar2vector(mod_mask, num_elem, elem_bt));
535
536 BoolTest::mask pred = BoolTest::ugt;
537 ConINode* pred_node = (ConINode*)gvn().makecon(TypeInt::make(pred));
538 Node* lane_cnt = gvn().makecon(TypeInt::make(num_elem));
539 Node* bcast_lane_cnt = gvn().transform(VectorNode::scalar2vector(lane_cnt, num_elem, elem_bt));
540 const TypeVect* vmask_type = TypeVect::makemask(elem_bt, num_elem);
541 Node* mask = gvn().transform(new VectorMaskCmpNode(pred, bcast_lane_cnt, index_vec, pred_node, vmask_type));
542
543 // Make the indices greater than lane count as -ve values to match the java side implementation.
544 index_vec = gvn().transform(VectorNode::make(Op_AndV, index_vec, bcast_mod_mask, vt));
545 Node* biased_val = gvn().transform(VectorNode::make(Op_SubVB, index_vec, bcast_lane_cnt, vt));
546 return gvn().transform(new VectorBlendNode(biased_val, index_vec, mask));
547 }
548
549 // <Sh extends VectorShuffle<E>, E>
550 // Sh ShuffleIota(Class<?> E, Class<?> shuffleClass, Vector.Species<E> s, int length,
551 // int start, int step, int wrap, ShuffleIotaOperation<Sh, E> defaultImpl)
552 bool LibraryCallKit::inline_vector_shuffle_iota() {
553 const TypeInstPtr* shuffle_klass = gvn().type(argument(1))->isa_instptr();
554 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
555 const TypeInt* start_val = gvn().type(argument(4))->isa_int();
556 const TypeInt* step_val = gvn().type(argument(5))->isa_int();
557 const TypeInt* wrap = gvn().type(argument(6))->isa_int();
558
559 if (shuffle_klass == nullptr || shuffle_klass->const_oop() == nullptr ||
560 vlen == nullptr || !vlen->is_con() ||
561 start_val == nullptr ||
562 step_val == nullptr ||
563 wrap == nullptr || !wrap->is_con()) {
564 return false; // not enough info for intrinsification
565 }
566
567 if (!is_klass_initialized(shuffle_klass)) {
568 log_if_needed(" ** klass argument not initialized");
569 return false;
570 }
571
572 int do_wrap = wrap->get_con();
573 int num_elem = vlen->get_con();
574 BasicType elem_bt = T_BYTE;
575
576 bool effective_indices_in_range = false;
577 if (start_val->is_con() && step_val->is_con()) {
578 int effective_min_index = start_val->get_con();
579 int effective_max_index = start_val->get_con() + step_val->get_con() * (num_elem - 1);
580 effective_indices_in_range = effective_max_index >= effective_min_index && effective_min_index >= -128 && effective_max_index <= 127;
581 }
582
583 if (!do_wrap && !effective_indices_in_range) {
584 // Disable instrinsification for unwrapped shuffle iota if start/step
585 // values are non-constant OR if intermediate result overflows byte value range.
586 return false;
587 }
588
589 if (!arch_supports_vector(Op_AddVB, num_elem, elem_bt, VecMaskNotUsed) ||
590 !arch_supports_vector(Op_AndV, num_elem, elem_bt, VecMaskNotUsed) ||
591 !arch_supports_vector(Op_VectorLoadConst, num_elem, elem_bt, VecMaskNotUsed) ||
592 !arch_supports_vector(Op_Replicate, num_elem, elem_bt, VecMaskNotUsed)) {
593 return false;
594 }
595
596 if (!do_wrap &&
597 (!arch_supports_vector(Op_SubVB, num_elem, elem_bt, VecMaskNotUsed) ||
598 !arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskNotUsed) ||
599 !arch_supports_vector(Op_VectorMaskCmp, num_elem, elem_bt, VecMaskNotUsed))) {
600 return false;
601 }
602
603 bool step_multiply = !step_val->is_con() || !is_power_of_2(step_val->get_con());
604 if ((step_multiply && !arch_supports_vector(Op_MulVB, num_elem, elem_bt, VecMaskNotUsed)) ||
605 (!step_multiply && !arch_supports_vector(Op_LShiftVB, num_elem, elem_bt, VecMaskNotUsed))) {
606 return false;
607 }
608
609 const TypeVect* vt = TypeVect::make(elem_bt, num_elem);
610
611 Node* res = gvn().transform(new VectorLoadConstNode(gvn().makecon(TypeInt::ZERO), vt));
612
613 Node* start = argument(4);
614 Node* step = argument(5);
615
616 if (step_multiply) {
617 Node* bcast_step = gvn().transform(VectorNode::scalar2vector(step, num_elem, elem_bt));
618 res = gvn().transform(VectorNode::make(Op_MulVB, res, bcast_step, vt));
619 } else if (step_val->get_con() > 1) {
620 Node* cnt = gvn().makecon(TypeInt::make(log2i_exact(step_val->get_con())));
621 Node* shift_cnt = vector_shift_count(cnt, Op_LShiftI, elem_bt, num_elem);
622 res = gvn().transform(VectorNode::make(Op_LShiftVB, res, shift_cnt, vt));
623 }
624
625 if (!start_val->is_con() || start_val->get_con() != 0) {
626 Node* bcast_start = gvn().transform(VectorNode::scalar2vector(start, num_elem, elem_bt));
627 res = gvn().transform(VectorNode::make(Op_AddVB, res, bcast_start, vt));
628 }
629
630 Node* mod_val = gvn().makecon(TypeInt::make(num_elem-1));
631 Node* bcast_mod = gvn().transform(VectorNode::scalar2vector(mod_val, num_elem, elem_bt));
632
633 if (do_wrap) {
634 // Wrap the indices greater than lane count.
635 res = gvn().transform(VectorNode::make(Op_AndV, res, bcast_mod, vt));
636 } else {
637 res = partially_wrap_indexes(res, num_elem, elem_bt);
638 }
639
640 ciKlass* sbox_klass = shuffle_klass->const_oop()->as_instance()->java_lang_Class_klass();
641 const TypeInstPtr* shuffle_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, sbox_klass);
642
643 // Wrap it up in VectorBox to keep object type information.
644 res = box_vector(res, shuffle_box_type, elem_bt, num_elem);
645 set_result(res);
646 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
647 return true;
648 }
649
650 // <E, M>
651 // long maskReductionCoerced(int oper, Class<? extends M> maskClass, Class<?> elemClass,
652 // int length, M m, VectorMaskOp<M> defaultImpl)
653 bool LibraryCallKit::inline_vector_mask_operation() {
654 const TypeInt* oper = gvn().type(argument(0))->isa_int();
655 const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
656 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
657 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
658 Node* mask = argument(4);
659
660 if (mask_klass == nullptr || mask_klass->const_oop() == nullptr ||
661 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
662 vlen == nullptr || !vlen->is_con() ||
663 oper == nullptr || !oper->is_con() ||
664 mask->is_top()) {
665 return false; // dead code
666 }
667
668 if (!is_klass_initialized(mask_klass)) {
669 log_if_needed(" ** klass argument not initialized");
670 return false;
671 }
672
673 int num_elem = vlen->get_con();
674 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
675 BasicType elem_bt = elem_type->basic_type();
676
677 int mopc = VectorSupport::vop2ideal(oper->get_con(), elem_bt);
678 if (!arch_supports_vector(mopc, num_elem, elem_bt, VecMaskUseLoad)) {
679 log_if_needed(" ** not supported: arity=1 op=cast#%d/3 vlen2=%d etype2=%s",
680 mopc, num_elem, type2name(elem_bt));
681 return false; // not supported
682 }
683
684 const Type* elem_ty = Type::get_const_basic_type(elem_bt);
685 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
686 const TypeInstPtr* mask_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
687 Node* mask_vec = unbox_vector(mask, mask_box_type, elem_bt, num_elem, true);
688 if (mask_vec == nullptr) {
689 log_if_needed(" ** unbox failed mask=%s",
690 NodeClassNames[argument(4)->Opcode()]);
691 return false;
692 }
693
694 if (mask_vec->bottom_type()->isa_vectmask() == nullptr) {
695 mask_vec = gvn().transform(VectorStoreMaskNode::make(gvn(), mask_vec, elem_bt, num_elem));
696 }
697 const Type* maskoper_ty = mopc == Op_VectorMaskToLong ? (const Type*)TypeLong::LONG : (const Type*)TypeInt::INT;
698 Node* maskoper = gvn().transform(VectorMaskOpNode::make(mask_vec, maskoper_ty, mopc));
699 if (mopc != Op_VectorMaskToLong) {
700 maskoper = ConvI2L(maskoper);
701 }
702 set_result(maskoper);
703
704 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
705 return true;
706 }
707
708 // public static
709 // <V,
710 // Sh extends VectorShuffle<E>,
711 // E>
712 // V shuffleToVector(Class<? extends Vector<E>> vclass, Class<E> elementType,
713 // Class<? extends Sh> shuffleClass, Sh s, int length,
714 // ShuffleToVectorOperation<V, Sh, E> defaultImpl)
715 bool LibraryCallKit::inline_vector_shuffle_to_vector() {
716 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
717 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
718 const TypeInstPtr* shuffle_klass = gvn().type(argument(2))->isa_instptr();
719 Node* shuffle = argument(3);
720 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
721
722 if (vector_klass == nullptr || elem_klass == nullptr || shuffle_klass == nullptr || shuffle->is_top() || vlen == nullptr) {
723 return false; // dead code
724 }
725 if (vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || shuffle_klass->const_oop() == nullptr || !vlen->is_con()) {
726 return false; // not enough info for intrinsification
727 }
728 if (!is_klass_initialized(shuffle_klass) || !is_klass_initialized(vector_klass) ) {
729 log_if_needed(" ** klass argument not initialized");
730 return false;
731 }
732
733 int num_elem = vlen->get_con();
734 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
735 BasicType elem_bt = elem_type->basic_type();
736
737 if (num_elem < 4) {
738 return false;
739 }
740
741 int cast_vopc = VectorCastNode::opcode(-1, T_BYTE); // from shuffle of type T_BYTE
742 // Make sure that cast is implemented to particular type/size combination.
743 if (!arch_supports_vector(cast_vopc, num_elem, elem_bt, VecMaskNotUsed)) {
744 log_if_needed(" ** not supported: arity=1 op=cast#%d/3 vlen2=%d etype2=%s",
745 cast_vopc, num_elem, type2name(elem_bt));
746 return false;
747 }
748
749 ciKlass* sbox_klass = shuffle_klass->const_oop()->as_instance()->java_lang_Class_klass();
750 const TypeInstPtr* shuffle_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, sbox_klass);
751
752 // Unbox shuffle with true flag to indicate its load shuffle to vector
753 // shuffle is a byte array
754 Node* shuffle_vec = unbox_vector(shuffle, shuffle_box_type, T_BYTE, num_elem, true);
755
756 // cast byte to target element type
757 shuffle_vec = gvn().transform(VectorCastNode::make(cast_vopc, shuffle_vec, elem_bt, num_elem));
758
759 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
760 const TypeInstPtr* vec_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
761
762 // Box vector
763 Node* res = box_vector(shuffle_vec, vec_box_type, elem_bt, num_elem);
764 set_result(res);
765 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
766 return true;
767 }
768
769 // public static
770 // <E,
771 // SH extends VectorShuffle<E>>
772 // SH wrapShuffleIndexes(Class<E> eClass, Class<? extends SH> shClass, SH sh, int length,
773 // ShuffleWrapIndexesOperation<SH> defaultImpl)
774 bool LibraryCallKit::inline_vector_wrap_shuffle_indexes() {
775 const TypeInstPtr* elem_klass = gvn().type(argument(0))->isa_instptr();
776 const TypeInstPtr* shuffle_klass = gvn().type(argument(1))->isa_instptr();
777 Node* shuffle = argument(2);
778 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
779
780 if (elem_klass == nullptr || shuffle_klass == nullptr || shuffle->is_top() || vlen == nullptr ||
781 !vlen->is_con() || shuffle_klass->const_oop() == nullptr) {
782 // not enough info for intrinsification
783 return false;
784 }
785
786 if (!is_klass_initialized(shuffle_klass)) {
787 log_if_needed(" ** klass argument not initialized");
788 return false;
789 }
790
791 int num_elem = vlen->get_con();
792 if ((num_elem < 4) || !is_power_of_2(num_elem)) {
793 log_if_needed(" ** vlen < 4 or not power of two=%d", num_elem);
794 return false;
795 }
796
797 // Shuffles use byte array based backing storage
798 BasicType shuffle_bt = T_BYTE;
799 if (!arch_supports_vector(Op_AndV, num_elem, shuffle_bt, VecMaskNotUsed) ||
800 !arch_supports_vector(Op_Replicate, num_elem, shuffle_bt, VecMaskNotUsed)) {
801 log_if_needed(" ** not supported: op=wrapShuffleIndexes vlen=%d etype=%s",
802 num_elem, type2name(shuffle_bt));
803 return false;
804 }
805
806 ciKlass* sbox_klass = shuffle_klass->const_oop()->as_instance()->java_lang_Class_klass();
807 const TypeInstPtr* shuffle_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, sbox_klass);
808
809 // Unbox shuffle with true flag to indicate its load shuffle to vector
810 // shuffle is a byte array
811 Node* shuffle_vec = unbox_vector(shuffle, shuffle_box_type, shuffle_bt, num_elem, true);
812
813 const TypeVect* vt = TypeVect::make(shuffle_bt, num_elem);
814 Node* mod_mask = gvn().makecon(TypeInt::make(num_elem - 1));
815 Node* bcast_mod_mask = gvn().transform(VectorNode::scalar2vector(mod_mask, num_elem, shuffle_bt));
816 // Wrap the indices greater than lane count.
817 Node* res = gvn().transform(VectorNode::make(Op_AndV, shuffle_vec, bcast_mod_mask, vt));
818
819 // Wrap it up in VectorBox to keep object type information.
820 res = box_vector(res, shuffle_box_type, shuffle_bt, num_elem);
821 set_result(res);
822 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(shuffle_bt))));
823 return true;
824 }
825
826 // public static
827 // <M,
828 // S extends VectorSpecies<E>,
829 // E>
830 // M fromBitsCoerced(Class<? extends M> vmClass, Class<E> elementType, int length,
831 // long bits, int mode, S s,
832 // BroadcastOperation<M, E, S> defaultImpl)
833 bool LibraryCallKit::inline_vector_frombits_coerced() {
834 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
835 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
836 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
837 const TypeLong* bits_type = gvn().type(argument(3))->isa_long();
838 // Mode argument determines the mode of operation it can take following values:-
839 // MODE_BROADCAST for vector Vector.broadcast and VectorMask.maskAll operations.
840 // MODE_BITS_COERCED_LONG_TO_MASK for VectorMask.fromLong operation.
841 const TypeInt* mode = gvn().type(argument(5))->isa_int();
842
843 if (vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
844 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
845 vlen == nullptr || !vlen->is_con() ||
846 bits_type == nullptr ||
847 mode == nullptr || !mode->is_con()) {
848 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s bitwise=%s",
849 NodeClassNames[argument(0)->Opcode()],
850 NodeClassNames[argument(1)->Opcode()],
851 NodeClassNames[argument(2)->Opcode()],
852 NodeClassNames[argument(5)->Opcode()]);
853 return false; // not enough info for intrinsification
854 }
855
856 if (!is_klass_initialized(vector_klass)) {
857 log_if_needed(" ** klass argument not initialized");
858 return false;
859 }
860 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
861 if (!elem_type->is_primitive_type()) {
862 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
863 return false; // should be primitive type
864 }
865 BasicType elem_bt = elem_type->basic_type();
866 int num_elem = vlen->get_con();
867 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
868 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
869
870 bool is_mask = is_vector_mask(vbox_klass);
871 int bcast_mode = mode->get_con();
872 VectorMaskUseType checkFlags = (VectorMaskUseType)(is_mask ? VecMaskUseAll : VecMaskNotUsed);
873 int opc = bcast_mode == VectorSupport::MODE_BITS_COERCED_LONG_TO_MASK ? Op_VectorLongToMask : Op_Replicate;
874
875 if (!arch_supports_vector(opc, num_elem, elem_bt, checkFlags, true /*has_scalar_args*/)) {
876 log_if_needed(" ** not supported: arity=0 op=broadcast vlen=%d etype=%s ismask=%d bcast_mode=%d",
877 num_elem, type2name(elem_bt),
878 is_mask ? 1 : 0,
879 bcast_mode);
880 return false; // not supported
881 }
882
883 Node* broadcast = nullptr;
884 Node* bits = argument(3);
885 Node* elem = bits;
886
887 if (opc == Op_VectorLongToMask) {
888 const TypeVect* vt = TypeVect::makemask(elem_bt, num_elem);
889 if (vt->isa_vectmask()) {
890 broadcast = gvn().transform(new VectorLongToMaskNode(elem, vt));
891 } else {
892 const TypeVect* mvt = TypeVect::make(T_BOOLEAN, num_elem);
893 broadcast = gvn().transform(new VectorLongToMaskNode(elem, mvt));
894 broadcast = gvn().transform(new VectorLoadMaskNode(broadcast, vt));
895 }
896 } else {
897 switch (elem_bt) {
898 case T_BOOLEAN: // fall-through
899 case T_BYTE: // fall-through
900 case T_SHORT: // fall-through
901 case T_CHAR: // fall-through
902 case T_INT: {
903 elem = gvn().transform(new ConvL2INode(bits));
904 break;
905 }
906 case T_DOUBLE: {
907 elem = gvn().transform(new MoveL2DNode(bits));
908 break;
909 }
910 case T_FLOAT: {
911 bits = gvn().transform(new ConvL2INode(bits));
912 elem = gvn().transform(new MoveI2FNode(bits));
913 break;
914 }
915 case T_LONG: {
916 // no conversion needed
917 break;
918 }
919 default: fatal("%s", type2name(elem_bt));
920 }
921 broadcast = VectorNode::scalar2vector(elem, num_elem, elem_bt, is_mask);
922 broadcast = gvn().transform(broadcast);
923 }
924
925 Node* box = box_vector(broadcast, vbox_type, elem_bt, num_elem);
926 set_result(box);
927 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
928 return true;
929 }
930
931 static bool elem_consistent_with_arr(BasicType elem_bt, const TypeAryPtr* arr_type, bool mismatched_ms) {
932 assert(arr_type != nullptr, "unexpected");
933 BasicType arr_elem_bt = arr_type->elem()->array_element_basic_type();
934 if (elem_bt == arr_elem_bt) {
935 return true;
936 } else if (elem_bt == T_SHORT && arr_elem_bt == T_CHAR) {
937 // Load/store of short vector from/to char[] is supported
938 return true;
939 } else if (elem_bt == T_BYTE && arr_elem_bt == T_BOOLEAN) {
940 // Load/store of byte vector from/to boolean[] is supported
941 return true;
942 } else {
943 return mismatched_ms;
944 }
945 }
946
947 // public static
948 // <C,
949 // VM extends VectorPayload,
950 // E,
951 // S extends VectorSpecies<E>>
952 // VM load(Class<? extends VM> vmClass, Class<E> eClass,
953 // int length,
954 // Object base, long offset, // Unsafe addressing
955 // boolean fromSegment,
956 // C container, long index, S s, // Arguments for default implementation
957 // LoadOperation<C, VM, S> defaultImpl) {
958 // public static
959 // <C,
960 // V extends VectorPayload>
961 // void store(Class<?> vClass, Class<?> eClass,
962 // int length,
963 // Object base, long offset, // Unsafe addressing
964 // boolean fromSegment,
965 // V v, C container, long index, // Arguments for default implementation
966 // StoreVectorOperation<C, V> defaultImpl) {
967 bool LibraryCallKit::inline_vector_mem_operation(bool is_store) {
968 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
969 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
970 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
971 const TypeInt* from_ms = gvn().type(argument(6))->isa_int();
972
973 if (vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
974 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
975 vlen == nullptr || !vlen->is_con() ||
976 from_ms == nullptr || !from_ms->is_con()) {
977 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s from_ms=%s",
978 NodeClassNames[argument(0)->Opcode()],
979 NodeClassNames[argument(1)->Opcode()],
980 NodeClassNames[argument(2)->Opcode()],
981 NodeClassNames[argument(6)->Opcode()]);
982 return false; // not enough info for intrinsification
983 }
984 if (!is_klass_initialized(vector_klass)) {
985 log_if_needed(" ** klass argument not initialized");
986 return false;
987 }
988
989 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
990 if (!elem_type->is_primitive_type()) {
991 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
992 return false; // should be primitive type
993 }
994 BasicType elem_bt = elem_type->basic_type();
995 int num_elem = vlen->get_con();
996
997 // TODO When mask usage is supported, VecMaskNotUsed needs to be VecMaskUseLoad.
998 if (!arch_supports_vector(is_store ? Op_StoreVector : Op_LoadVector, num_elem, elem_bt, VecMaskNotUsed)) {
999 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d etype=%s ismask=no",
1000 is_store, is_store ? "store" : "load",
1001 num_elem, type2name(elem_bt));
1002 return false; // not supported
1003 }
1004
1005 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1006 bool is_mask = is_vector_mask(vbox_klass);
1007
1008 Node* base = argument(3);
1009 Node* offset = ConvL2X(argument(4));
1010
1011 // Save state and restore on bailout
1012 uint old_sp = sp();
1013 SafePointNode* old_map = clone_map();
1014
1015 Node* addr = make_unsafe_address(base, offset, (is_mask ? T_BOOLEAN : elem_bt), true);
1016
1017 // The memory barrier checks are based on ones for unsafe access.
1018 // This is not 1-1 implementation.
1019 const Type *const base_type = gvn().type(base);
1020
1021 const TypePtr *addr_type = gvn().type(addr)->isa_ptr();
1022 const TypeAryPtr* arr_type = addr_type->isa_aryptr();
1023
1024 const bool in_native = TypePtr::NULL_PTR == base_type; // base always null
1025 const bool in_heap = !TypePtr::NULL_PTR->higher_equal(base_type); // base never null
1026
1027 const bool is_mixed_access = !in_heap && !in_native;
1028
1029 const bool is_mismatched_access = in_heap && (addr_type->isa_aryptr() == nullptr);
1030
1031 const bool needs_cpu_membar = is_mixed_access || is_mismatched_access;
1032
1033 // For non-masked mismatched memory segment vector read/write accesses, intrinsification can continue
1034 // with unknown backing storage type and compiler can skip inserting explicit reinterpretation IR after
1035 // loading from or before storing to backing storage which is mandatory for semantic correctness of
1036 // big-endian memory layout.
1037 bool mismatched_ms = LITTLE_ENDIAN_ONLY(false)
1038 BIG_ENDIAN_ONLY(from_ms->get_con() && !is_mask && arr_type != nullptr &&
1039 arr_type->elem()->array_element_basic_type() != elem_bt);
1040 BasicType mem_elem_bt = mismatched_ms ? arr_type->elem()->array_element_basic_type() : elem_bt;
1041 if (!is_java_primitive(mem_elem_bt)) {
1042 log_if_needed(" ** non-primitive array element type");
1043 return false;
1044 }
1045 int mem_num_elem = mismatched_ms ? (num_elem * type2aelembytes(elem_bt)) / type2aelembytes(mem_elem_bt) : num_elem;
1046 if (arr_type != nullptr && !is_mask && !elem_consistent_with_arr(elem_bt, arr_type, mismatched_ms)) {
1047 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d etype=%s atype=%s ismask=no",
1048 is_store, is_store ? "store" : "load",
1049 num_elem, type2name(elem_bt), type2name(arr_type->elem()->array_element_basic_type()));
1050 set_map(old_map);
1051 set_sp(old_sp);
1052 return false;
1053 }
1054
1055 // In case of mismatched memory segment accesses, we need to double check that the source type memory operations are supported by backend.
1056 if (mismatched_ms) {
1057 if (is_store) {
1058 if (!arch_supports_vector(Op_StoreVector, num_elem, elem_bt, VecMaskNotUsed)
1059 || !arch_supports_vector(Op_VectorReinterpret, mem_num_elem, mem_elem_bt, VecMaskNotUsed)) {
1060 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d*8 etype=%s/8 ismask=no",
1061 is_store, "store",
1062 num_elem, type2name(elem_bt));
1063 set_map(old_map);
1064 set_sp(old_sp);
1065 return false; // not supported
1066 }
1067 } else {
1068 if (!arch_supports_vector(Op_LoadVector, mem_num_elem, mem_elem_bt, VecMaskNotUsed)
1069 || !arch_supports_vector(Op_VectorReinterpret, num_elem, elem_bt, VecMaskNotUsed)) {
1070 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d*8 etype=%s/8 ismask=no",
1071 is_store, "load",
1072 mem_num_elem, type2name(mem_elem_bt));
1073 set_map(old_map);
1074 set_sp(old_sp);
1075 return false; // not supported
1076 }
1077 }
1078 }
1079 if (is_mask) {
1080 if (!is_store) {
1081 if (!arch_supports_vector(Op_LoadVector, num_elem, elem_bt, VecMaskUseLoad)) {
1082 set_map(old_map);
1083 set_sp(old_sp);
1084 return false; // not supported
1085 }
1086 } else {
1087 if (!arch_supports_vector(Op_StoreVector, num_elem, elem_bt, VecMaskUseStore)) {
1088 set_map(old_map);
1089 set_sp(old_sp);
1090 return false; // not supported
1091 }
1092 }
1093 }
1094
1095 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1096
1097 if (needs_cpu_membar) {
1098 insert_mem_bar(Op_MemBarCPUOrder);
1099 }
1100
1101 if (is_store) {
1102 Node* val = unbox_vector(argument(7), vbox_type, elem_bt, num_elem);
1103 if (val == nullptr) {
1104 set_map(old_map);
1105 set_sp(old_sp);
1106 return false; // operand unboxing failed
1107 }
1108 set_all_memory(reset_memory());
1109
1110 // In case the store needs to happen to byte array, reinterpret the incoming vector to byte vector.
1111 int store_num_elem = num_elem;
1112 if (mismatched_ms) {
1113 store_num_elem = mem_num_elem;
1114 const TypeVect* to_vect_type = TypeVect::make(mem_elem_bt, store_num_elem);
1115 val = gvn().transform(new VectorReinterpretNode(val, val->bottom_type()->is_vect(), to_vect_type));
1116 }
1117 if (is_mask) {
1118 val = gvn().transform(VectorStoreMaskNode::make(gvn(), val, elem_bt, num_elem));
1119 }
1120 Node* vstore = gvn().transform(StoreVectorNode::make(0, control(), memory(addr), addr, addr_type, val, store_num_elem));
1121 set_memory(vstore, addr_type);
1122 } else {
1123 // When using byte array, we need to load as byte then reinterpret the value. Otherwise, do a simple vector load.
1124 Node* vload = nullptr;
1125 if (mismatched_ms) {
1126 vload = gvn().transform(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, mem_num_elem, mem_elem_bt));
1127 const TypeVect* to_vect_type = TypeVect::make(elem_bt, num_elem);
1128 vload = gvn().transform(new VectorReinterpretNode(vload, vload->bottom_type()->is_vect(), to_vect_type));
1129 } else {
1130 // Special handle for masks
1131 if (is_mask) {
1132 vload = gvn().transform(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, num_elem, T_BOOLEAN));
1133 vload = gvn().transform(new VectorLoadMaskNode(vload, TypeVect::makemask(elem_bt, num_elem)));
1134 } else {
1135 vload = gvn().transform(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, num_elem, elem_bt));
1136 }
1137 }
1138 Node* box = box_vector(vload, vbox_type, elem_bt, num_elem);
1139 set_result(box);
1140 }
1141
1142 destruct_map_clone(old_map);
1143
1144 if (needs_cpu_membar) {
1145 insert_mem_bar(Op_MemBarCPUOrder);
1146 }
1147
1148 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1149 return true;
1150 }
1151
1152 // public static
1153 // <C,
1154 // V extends Vector<?>,
1155 // E,
1156 // S extends VectorSpecies<E>,
1157 // M extends VectorMask<E>>
1158 // V loadMasked(Class<? extends V> vClass, Class<M> mClass, Class<E> eClass,
1159 // int length, Object base, long offset, // Unsafe addressing
1160 // boolean fromSegment,
1161 // M m, int offsetInRange,
1162 // C container, long index, S s, // Arguments for default implementation
1163 // LoadVectorMaskedOperation<C, V, S, M> defaultImpl) {
1164 // public static
1165 // <C,
1166 // V extends Vector<E>,
1167 // M extends VectorMask<E>,
1168 // E>
1169 // void storeMasked(Class<? extends V> vClass, Class<M> mClass, Class<E> eClass,
1170 // int length,
1171 // Object base, long offset, // Unsafe addressing
1172 // boolean fromSegment,
1173 // V v, M m, C container, long index, // Arguments for default implementation
1174 // StoreVectorMaskedOperation<C, V, M> defaultImpl) {
1175
1176 bool LibraryCallKit::inline_vector_mem_masked_operation(bool is_store) {
1177 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
1178 const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
1179 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
1180 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
1181 const TypeInt* from_ms = gvn().type(argument(7))->isa_int();
1182
1183 if (vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
1184 mask_klass == nullptr || mask_klass->const_oop() == nullptr ||
1185 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
1186 vlen == nullptr || !vlen->is_con() ||
1187 from_ms == nullptr || !from_ms->is_con()) {
1188 log_if_needed(" ** missing constant: vclass=%s mclass=%s etype=%s vlen=%s from_ms=%s",
1189 NodeClassNames[argument(0)->Opcode()],
1190 NodeClassNames[argument(1)->Opcode()],
1191 NodeClassNames[argument(2)->Opcode()],
1192 NodeClassNames[argument(3)->Opcode()],
1193 NodeClassNames[argument(7)->Opcode()]);
1194 return false; // not enough info for intrinsification
1195 }
1196 if (!is_klass_initialized(vector_klass)) {
1197 log_if_needed(" ** klass argument not initialized");
1198 return false;
1199 }
1200
1201 if (!is_klass_initialized(mask_klass)) {
1202 log_if_needed(" ** mask klass argument not initialized");
1203 return false;
1204 }
1205
1206 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1207 if (!elem_type->is_primitive_type()) {
1208 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1209 return false; // should be primitive type
1210 }
1211
1212 BasicType elem_bt = elem_type->basic_type();
1213 int num_elem = vlen->get_con();
1214
1215 Node* base = argument(4);
1216 Node* offset = ConvL2X(argument(5));
1217
1218 // Save state and restore on bailout
1219 uint old_sp = sp();
1220 SafePointNode* old_map = clone_map();
1221
1222 Node* addr = make_unsafe_address(base, offset, elem_bt, true);
1223 const TypePtr *addr_type = gvn().type(addr)->isa_ptr();
1224 const TypeAryPtr* arr_type = addr_type->isa_aryptr();
1225
1226 bool mismatched_ms = from_ms->get_con() && arr_type != nullptr && arr_type->elem()->array_element_basic_type() != elem_bt;
1227 BIG_ENDIAN_ONLY(if (mismatched_ms) return false;)
1228 // If there is no consistency between array and vector element types, it must be special byte array case
1229 if (arr_type != nullptr && !elem_consistent_with_arr(elem_bt, arr_type, mismatched_ms)) {
1230 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d etype=%s atype=%s",
1231 is_store, is_store ? "storeMasked" : "loadMasked",
1232 num_elem, type2name(elem_bt), type2name(arr_type->elem()->array_element_basic_type()));
1233 set_map(old_map);
1234 set_sp(old_sp);
1235 return false;
1236 }
1237
1238 int mem_num_elem = mismatched_ms ? num_elem * type2aelembytes(elem_bt) : num_elem;
1239 BasicType mem_elem_bt = mismatched_ms ? T_BYTE : elem_bt;
1240 bool supports_predicate = arch_supports_vector(is_store ? Op_StoreVectorMasked : Op_LoadVectorMasked,
1241 mem_num_elem, mem_elem_bt, VecMaskUseLoad);
1242
1243 // If current arch does not support the predicated operations, we have to bail
1244 // out when current case uses the predicate feature.
1245 if (!supports_predicate) {
1246 bool needs_predicate = false;
1247 if (is_store) {
1248 // Masked vector store always uses the predicated store.
1249 needs_predicate = true;
1250 } else {
1251 // Masked vector load with IOOBE always uses the predicated load.
1252 const TypeInt* offset_in_range = gvn().type(argument(9))->isa_int();
1253 if (!offset_in_range->is_con()) {
1254 log_if_needed(" ** missing constant: offsetInRange=%s",
1255 NodeClassNames[argument(8)->Opcode()]);
1256 set_map(old_map);
1257 set_sp(old_sp);
1258 return false;
1259 }
1260 needs_predicate = (offset_in_range->get_con() == 0);
1261 }
1262
1263 if (needs_predicate) {
1264 log_if_needed(" ** not supported: op=%s vlen=%d etype=%s mismatched_ms=%d",
1265 is_store ? "storeMasked" : "loadMasked",
1266 num_elem, type2name(elem_bt), mismatched_ms ? 1 : 0);
1267 set_map(old_map);
1268 set_sp(old_sp);
1269 return false;
1270 }
1271 }
1272
1273 // This only happens for masked vector load. If predicate is not supported, then check whether
1274 // the normal vector load and blend operations are supported by backend.
1275 if (!supports_predicate && (!arch_supports_vector(Op_LoadVector, mem_num_elem, mem_elem_bt, VecMaskNotUsed) ||
1276 !arch_supports_vector(Op_VectorBlend, mem_num_elem, mem_elem_bt, VecMaskUseLoad))) {
1277 log_if_needed(" ** not supported: op=loadMasked vlen=%d etype=%s mismatched_ms=%d",
1278 num_elem, type2name(elem_bt), mismatched_ms ? 1 : 0);
1279 set_map(old_map);
1280 set_sp(old_sp);
1281 return false;
1282 }
1283
1284 // Since we are using byte array, we need to double check that the vector reinterpret operation
1285 // with byte type is supported by backend.
1286 if (mismatched_ms) {
1287 if (!arch_supports_vector(Op_VectorReinterpret, mem_num_elem, T_BYTE, VecMaskNotUsed)) {
1288 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d etype=%s mismatched_ms=1",
1289 is_store, is_store ? "storeMasked" : "loadMasked",
1290 num_elem, type2name(elem_bt));
1291 set_map(old_map);
1292 set_sp(old_sp);
1293 return false;
1294 }
1295 }
1296
1297 // Since it needs to unbox the mask, we need to double check that the related load operations
1298 // for mask are supported by backend.
1299 if (!arch_supports_vector(Op_LoadVector, num_elem, elem_bt, VecMaskUseLoad)) {
1300 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d etype=%s",
1301 is_store, is_store ? "storeMasked" : "loadMasked",
1302 num_elem, type2name(elem_bt));
1303 set_map(old_map);
1304 set_sp(old_sp);
1305 return false;
1306 }
1307
1308 // Can base be null? Otherwise, always on-heap access.
1309 bool can_access_non_heap = TypePtr::NULL_PTR->higher_equal(gvn().type(base));
1310 if (can_access_non_heap) {
1311 insert_mem_bar(Op_MemBarCPUOrder);
1312 }
1313
1314 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1315 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1316 assert(!is_vector_mask(vbox_klass) && is_vector_mask(mbox_klass), "Invalid class type");
1317 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1318 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1319
1320 Node* mask = unbox_vector(is_store ? argument(9) : argument(8), mbox_type, elem_bt, num_elem);
1321 if (mask == nullptr) {
1322 log_if_needed(" ** unbox failed mask=%s",
1323 is_store ? NodeClassNames[argument(9)->Opcode()]
1324 : NodeClassNames[argument(8)->Opcode()]);
1325 set_map(old_map);
1326 set_sp(old_sp);
1327 return false;
1328 }
1329
1330 if (is_store) {
1331 Node* val = unbox_vector(argument(8), vbox_type, elem_bt, num_elem);
1332 if (val == nullptr) {
1333 log_if_needed(" ** unbox failed vector=%s",
1334 NodeClassNames[argument(8)->Opcode()]);
1335 set_map(old_map);
1336 set_sp(old_sp);
1337 return false; // operand unboxing failed
1338 }
1339 set_all_memory(reset_memory());
1340
1341 if (mismatched_ms) {
1342 // Reinterpret the incoming vector to byte vector.
1343 const TypeVect* to_vect_type = TypeVect::make(mem_elem_bt, mem_num_elem);
1344 val = gvn().transform(new VectorReinterpretNode(val, val->bottom_type()->is_vect(), to_vect_type));
1345 // Reinterpret the vector mask to byte type.
1346 const TypeVect* from_mask_type = TypeVect::makemask(elem_bt, num_elem);
1347 const TypeVect* to_mask_type = TypeVect::makemask(mem_elem_bt, mem_num_elem);
1348 mask = gvn().transform(new VectorReinterpretNode(mask, from_mask_type, to_mask_type));
1349 }
1350 Node* vstore = gvn().transform(new StoreVectorMaskedNode(control(), memory(addr), addr, val, addr_type, mask));
1351 set_memory(vstore, addr_type);
1352 } else {
1353 Node* vload = nullptr;
1354
1355 if (mismatched_ms) {
1356 // Reinterpret the vector mask to byte type.
1357 const TypeVect* from_mask_type = TypeVect::makemask(elem_bt, num_elem);
1358 const TypeVect* to_mask_type = TypeVect::makemask(mem_elem_bt, mem_num_elem);
1359 mask = gvn().transform(new VectorReinterpretNode(mask, from_mask_type, to_mask_type));
1360 }
1361
1362 if (supports_predicate) {
1363 // Generate masked load vector node if predicate feature is supported.
1364 const TypeVect* vt = TypeVect::make(mem_elem_bt, mem_num_elem);
1365 vload = gvn().transform(new LoadVectorMaskedNode(control(), memory(addr), addr, addr_type, vt, mask));
1366 } else {
1367 // Use the vector blend to implement the masked load vector. The biased elements are zeros.
1368 Node* zero = gvn().transform(gvn().zerocon(mem_elem_bt));
1369 zero = gvn().transform(VectorNode::scalar2vector(zero, mem_num_elem, mem_elem_bt));
1370 vload = gvn().transform(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, mem_num_elem, mem_elem_bt));
1371 vload = gvn().transform(new VectorBlendNode(zero, vload, mask));
1372 }
1373
1374 if (mismatched_ms) {
1375 const TypeVect* to_vect_type = TypeVect::make(elem_bt, num_elem);
1376 vload = gvn().transform(new VectorReinterpretNode(vload, vload->bottom_type()->is_vect(), to_vect_type));
1377 }
1378
1379 Node* box = box_vector(vload, vbox_type, elem_bt, num_elem);
1380 set_result(box);
1381 }
1382
1383 destruct_map_clone(old_map);
1384
1385 if (can_access_non_heap) {
1386 insert_mem_bar(Op_MemBarCPUOrder);
1387 }
1388
1389 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1390 return true;
1391 }
1392
1393 // <C,
1394 // V extends Vector<?>,
1395 // W extends Vector<Integer>,
1396 // S extends VectorSpecies<E>,
1397 // M extends VectorMask<E>,
1398 // E>
1399 // V loadWithMap(Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType, int length,
1400 // Class<? extends Vector<Integer>> vectorIndexClass,
1401 // Object base, long offset, // Unsafe addressing
1402 // W index_vector, M m,
1403 // C container, int index, int[] indexMap, int indexM, S s, // Arguments for default implementation
1404 // LoadVectorOperationWithMap<C, V, E, S, M> defaultImpl)
1405 //
1406 // <C,
1407 // V extends Vector<E>,
1408 // W extends Vector<Integer>,
1409 // M extends VectorMask<E>,
1410 // E>
1411 // void storeWithMap(Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType,
1412 // int length, Class<? extends Vector<Integer>> vectorIndexClass, Object base, long offset, // Unsafe addressing
1413 // W index_vector, V v, M m,
1414 // C container, int index, int[] indexMap, int indexM, // Arguments for default implementation
1415 // StoreVectorOperationWithMap<C, V, M, E> defaultImpl)
1416 //
1417 bool LibraryCallKit::inline_vector_gather_scatter(bool is_scatter) {
1418 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
1419 const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
1420 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
1421 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
1422 const TypeInstPtr* vector_idx_klass = gvn().type(argument(4))->isa_instptr();
1423
1424 if (vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
1425 // mask_klass == nullptr || mask_klass->const_oop() == nullptr ||
1426 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
1427 vlen == nullptr || !vlen->is_con() ||
1428 vector_idx_klass == nullptr || vector_idx_klass->const_oop() == nullptr) {
1429 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s viclass=%s",
1430 NodeClassNames[argument(0)->Opcode()],
1431 NodeClassNames[argument(2)->Opcode()],
1432 NodeClassNames[argument(3)->Opcode()],
1433 NodeClassNames[argument(4)->Opcode()]);
1434 return false; // not enough info for intrinsification
1435 }
1436
1437 if (!is_klass_initialized(vector_klass) || !is_klass_initialized(vector_idx_klass)) {
1438 log_if_needed(" ** klass argument not initialized");
1439 return false;
1440 }
1441
1442 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1443 if (!elem_type->is_primitive_type()) {
1444 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1445 return false; // should be primitive type
1446 }
1447
1448 BasicType elem_bt = elem_type->basic_type();
1449 int num_elem = vlen->get_con();
1450
1451 const Type* vmask_type = gvn().type(is_scatter ? argument(10) : argument(9));
1452 bool is_masked_op = vmask_type != TypePtr::NULL_PTR;
1453 if (is_masked_op) {
1454 if (mask_klass == nullptr || mask_klass->const_oop() == nullptr) {
1455 log_if_needed(" ** missing constant: maskclass=%s", NodeClassNames[argument(1)->Opcode()]);
1456 return false; // not enough info for intrinsification
1457 }
1458
1459 if (!is_klass_initialized(mask_klass)) {
1460 log_if_needed(" ** mask klass argument not initialized");
1461 return false;
1462 }
1463
1464 if (vmask_type->maybe_null()) {
1465 log_if_needed(" ** null mask values are not allowed for masked op");
1466 return false;
1467 }
1468
1469 // Check whether the predicated gather/scatter node is supported by architecture.
1470 VectorMaskUseType mask = (VectorMaskUseType) (VecMaskUseLoad | VecMaskUsePred);
1471 if (!arch_supports_vector(is_scatter ? Op_StoreVectorScatterMasked : Op_LoadVectorGatherMasked, num_elem, elem_bt, mask)) {
1472 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d etype=%s is_masked_op=1",
1473 is_scatter, is_scatter ? "scatterMasked" : "gatherMasked",
1474 num_elem, type2name(elem_bt));
1475 return false; // not supported
1476 }
1477 } else {
1478 // Check whether the normal gather/scatter node is supported for non-masked operation.
1479 if (!arch_supports_vector(is_scatter ? Op_StoreVectorScatter : Op_LoadVectorGather, num_elem, elem_bt, VecMaskNotUsed)) {
1480 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d etype=%s is_masked_op=0",
1481 is_scatter, is_scatter ? "scatter" : "gather",
1482 num_elem, type2name(elem_bt));
1483 return false; // not supported
1484 }
1485 }
1486
1487 // Check that the vector holding indices is supported by architecture
1488 // For sub-word gathers expander receive index array.
1489 if (!is_subword_type(elem_bt) && !arch_supports_vector(Op_LoadVector, num_elem, T_INT, VecMaskNotUsed)) {
1490 log_if_needed(" ** not supported: arity=%d op=%s/loadindex vlen=%d etype=int is_masked_op=%d",
1491 is_scatter, is_scatter ? "scatter" : "gather",
1492 num_elem, is_masked_op ? 1 : 0);
1493 return false; // not supported
1494 }
1495
1496 Node* base = argument(5);
1497 Node* offset = ConvL2X(argument(6));
1498
1499 // Save state and restore on bailout
1500 uint old_sp = sp();
1501 SafePointNode* old_map = clone_map();
1502
1503 Node* addr = make_unsafe_address(base, offset, elem_bt, true);
1504
1505 const TypePtr *addr_type = gvn().type(addr)->isa_ptr();
1506 const TypeAryPtr* arr_type = addr_type->isa_aryptr();
1507
1508 // The array must be consistent with vector type
1509 if (arr_type == nullptr || (arr_type != nullptr && !elem_consistent_with_arr(elem_bt, arr_type, false))) {
1510 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d etype=%s atype=%s ismask=no",
1511 is_scatter, is_scatter ? "scatter" : "gather",
1512 num_elem, type2name(elem_bt), type2name(arr_type->elem()->array_element_basic_type()));
1513 set_map(old_map);
1514 set_sp(old_sp);
1515 return false;
1516 }
1517
1518 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1519 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1520 ciKlass* vbox_idx_klass = vector_idx_klass->const_oop()->as_instance()->java_lang_Class_klass();
1521 if (vbox_idx_klass == nullptr) {
1522 set_map(old_map);
1523 set_sp(old_sp);
1524 return false;
1525 }
1526
1527 Node* index_vect = nullptr;
1528 const TypeInstPtr* vbox_idx_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_idx_klass);
1529 if (!is_subword_type(elem_bt)) {
1530 index_vect = unbox_vector(argument(8), vbox_idx_type, T_INT, num_elem);
1531 if (index_vect == nullptr) {
1532 set_map(old_map);
1533 set_sp(old_sp);
1534 return false;
1535 }
1536 }
1537
1538 Node* mask = nullptr;
1539 if (is_masked_op) {
1540 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1541 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1542 mask = unbox_vector(is_scatter ? argument(10) : argument(9), mbox_type, elem_bt, num_elem);
1543 if (mask == nullptr) {
1544 log_if_needed(" ** unbox failed mask=%s",
1545 is_scatter ? NodeClassNames[argument(10)->Opcode()]
1546 : NodeClassNames[argument(9)->Opcode()]);
1547 set_map(old_map);
1548 set_sp(old_sp);
1549 return false;
1550 }
1551 }
1552
1553 const TypeVect* vector_type = TypeVect::make(elem_bt, num_elem);
1554 if (is_scatter) {
1555 Node* val = unbox_vector(argument(9), vbox_type, elem_bt, num_elem);
1556 if (val == nullptr) {
1557 set_map(old_map);
1558 set_sp(old_sp);
1559 return false; // operand unboxing failed
1560 }
1561 set_all_memory(reset_memory());
1562
1563 Node* vstore = nullptr;
1564 if (mask != nullptr) {
1565 vstore = gvn().transform(new StoreVectorScatterMaskedNode(control(), memory(addr), addr, addr_type, val, index_vect, mask));
1566 } else {
1567 vstore = gvn().transform(new StoreVectorScatterNode(control(), memory(addr), addr, addr_type, val, index_vect));
1568 }
1569 set_memory(vstore, addr_type);
1570 } else {
1571 Node* vload = nullptr;
1572 Node* index = argument(11);
1573 Node* indexMap = argument(12);
1574 Node* indexM = argument(13);
1575 if (mask != nullptr) {
1576 if (is_subword_type(elem_bt)) {
1577 Node* index_arr_base = array_element_address(indexMap, indexM, T_INT);
1578 vload = gvn().transform(new LoadVectorGatherMaskedNode(control(), memory(addr), addr, addr_type, vector_type, index_arr_base, mask, index));
1579 } else {
1580 vload = gvn().transform(new LoadVectorGatherMaskedNode(control(), memory(addr), addr, addr_type, vector_type, index_vect, mask));
1581 }
1582 } else {
1583 if (is_subword_type(elem_bt)) {
1584 Node* index_arr_base = array_element_address(indexMap, indexM, T_INT);
1585 vload = gvn().transform(new LoadVectorGatherNode(control(), memory(addr), addr, addr_type, vector_type, index_arr_base, index));
1586 } else {
1587 vload = gvn().transform(new LoadVectorGatherNode(control(), memory(addr), addr, addr_type, vector_type, index_vect));
1588 }
1589 }
1590 Node* box = box_vector(vload, vbox_type, elem_bt, num_elem);
1591 set_result(box);
1592 }
1593
1594 destruct_map_clone(old_map);
1595
1596 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1597 return true;
1598 }
1599
1600 // public static
1601 // <V extends Vector<E>,
1602 // M extends VectorMask<E>,
1603 // E>
1604 // long reductionCoerced(int oprId, Class<? extends V> vectorClass, Class<? extends M> maskClass,
1605 // Class<E> elementType, int length, V v, M m,
1606 // ReductionOperation<V, M> defaultImpl)
1607 bool LibraryCallKit::inline_vector_reduction() {
1608 const TypeInt* opr = gvn().type(argument(0))->isa_int();
1609 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
1610 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
1611 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
1612 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
1613
1614 if (opr == nullptr || !opr->is_con() ||
1615 vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
1616 // mask_klass == nullptr || mask_klass->const_oop() == nullptr ||
1617 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
1618 vlen == nullptr || !vlen->is_con()) {
1619 log_if_needed(" ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
1620 NodeClassNames[argument(0)->Opcode()],
1621 NodeClassNames[argument(1)->Opcode()],
1622 NodeClassNames[argument(3)->Opcode()],
1623 NodeClassNames[argument(4)->Opcode()]);
1624 return false; // not enough info for intrinsification
1625 }
1626 if (!is_klass_initialized(vector_klass)) {
1627 log_if_needed(" ** klass argument not initialized");
1628 return false;
1629 }
1630 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1631 if (!elem_type->is_primitive_type()) {
1632 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1633 return false; // should be primitive type
1634 }
1635
1636 const Type* vmask_type = gvn().type(argument(6));
1637 bool is_masked_op = vmask_type != TypePtr::NULL_PTR;
1638 if (is_masked_op) {
1639 if (mask_klass == nullptr || mask_klass->const_oop() == nullptr) {
1640 log_if_needed(" ** missing constant: maskclass=%s", NodeClassNames[argument(2)->Opcode()]);
1641 return false; // not enough info for intrinsification
1642 }
1643
1644 if (!is_klass_initialized(mask_klass)) {
1645 log_if_needed(" ** mask klass argument not initialized");
1646 return false;
1647 }
1648
1649 if (vmask_type->maybe_null()) {
1650 log_if_needed(" ** null mask values are not allowed for masked op");
1651 return false;
1652 }
1653 }
1654
1655 BasicType elem_bt = elem_type->basic_type();
1656 int num_elem = vlen->get_con();
1657 int opc = VectorSupport::vop2ideal(opr->get_con(), elem_bt);
1658 int sopc = ReductionNode::opcode(opc, elem_bt);
1659
1660 // When using mask, mask use type needs to be VecMaskUseLoad.
1661 if (!arch_supports_vector(sopc, num_elem, elem_bt, is_masked_op ? VecMaskUseLoad : VecMaskNotUsed)) {
1662 log_if_needed(" ** not supported: arity=1 op=%d/reduce vlen=%d etype=%s is_masked_op=%d",
1663 sopc, num_elem, type2name(elem_bt), is_masked_op ? 1 : 0);
1664 return false;
1665 }
1666
1667 // Return true if current platform has implemented the masked operation with predicate feature.
1668 bool use_predicate = is_masked_op && arch_supports_vector(sopc, num_elem, elem_bt, VecMaskUsePred);
1669 if (is_masked_op && !use_predicate && !arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad)) {
1670 log_if_needed(" ** not supported: arity=1 op=%d/reduce vlen=%d etype=%s is_masked_op=1",
1671 sopc, num_elem, type2name(elem_bt));
1672 return false;
1673 }
1674
1675 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1676 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1677
1678 Node* opd = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
1679 if (opd == nullptr) {
1680 return false; // operand unboxing failed
1681 }
1682
1683 Node* mask = nullptr;
1684 if (is_masked_op) {
1685 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1686 assert(is_vector_mask(mbox_klass), "argument(2) should be a mask class");
1687 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1688 mask = unbox_vector(argument(6), mbox_type, elem_bt, num_elem);
1689 if (mask == nullptr) {
1690 log_if_needed(" ** unbox failed mask=%s",
1691 NodeClassNames[argument(6)->Opcode()]);
1692 return false;
1693 }
1694 }
1695
1696 Node* init = ReductionNode::make_identity_con_scalar(gvn(), opc, elem_bt);
1697 Node* value = opd;
1698
1699 assert(mask != nullptr || !is_masked_op, "Masked op needs the mask value never null");
1700 if (mask != nullptr && !use_predicate) {
1701 Node* reduce_identity = gvn().transform(VectorNode::scalar2vector(init, num_elem, elem_bt));
1702 value = gvn().transform(new VectorBlendNode(reduce_identity, value, mask));
1703 }
1704
1705 // Make an unordered Reduction node. This affects only AddReductionVF/VD and MulReductionVF/VD,
1706 // as these operations are allowed to be associative (not requiring strict order) in VectorAPI.
1707 value = ReductionNode::make(opc, nullptr, init, value, elem_bt, /* requires_strict_order */ false);
1708
1709 if (mask != nullptr && use_predicate) {
1710 value->add_req(mask);
1711 value->add_flag(Node::Flag_is_predicated_vector);
1712 }
1713
1714 value = gvn().transform(value);
1715
1716 Node* bits = nullptr;
1717 switch (elem_bt) {
1718 case T_BYTE:
1719 case T_SHORT:
1720 case T_INT: {
1721 bits = gvn().transform(new ConvI2LNode(value));
1722 break;
1723 }
1724 case T_FLOAT: {
1725 value = gvn().transform(new MoveF2INode(value));
1726 bits = gvn().transform(new ConvI2LNode(value));
1727 break;
1728 }
1729 case T_DOUBLE: {
1730 bits = gvn().transform(new MoveD2LNode(value));
1731 break;
1732 }
1733 case T_LONG: {
1734 bits = value; // no conversion needed
1735 break;
1736 }
1737 default: fatal("%s", type2name(elem_bt));
1738 }
1739 set_result(bits);
1740 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1741 return true;
1742 }
1743
1744 // public static <V> boolean test(int cond, Class<?> vectorClass, Class<?> elementType, int vlen,
1745 // V v1, V v2,
1746 // BiFunction<V, V, Boolean> defaultImpl)
1747 //
1748 bool LibraryCallKit::inline_vector_test() {
1749 const TypeInt* cond = gvn().type(argument(0))->isa_int();
1750 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
1751 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
1752 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
1753
1754 if (cond == nullptr || !cond->is_con() ||
1755 vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
1756 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
1757 vlen == nullptr || !vlen->is_con()) {
1758 log_if_needed(" ** missing constant: cond=%s vclass=%s etype=%s vlen=%s",
1759 NodeClassNames[argument(0)->Opcode()],
1760 NodeClassNames[argument(1)->Opcode()],
1761 NodeClassNames[argument(2)->Opcode()],
1762 NodeClassNames[argument(3)->Opcode()]);
1763 return false; // not enough info for intrinsification
1764 }
1765 if (!is_klass_initialized(vector_klass)) {
1766 log_if_needed(" ** klass argument not initialized");
1767 return false;
1768 }
1769 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1770 if (!elem_type->is_primitive_type()) {
1771 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1772 return false; // should be primitive type
1773 }
1774 BasicType elem_bt = elem_type->basic_type();
1775 int num_elem = vlen->get_con();
1776 BoolTest::mask booltest = (BoolTest::mask)cond->get_con();
1777 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1778 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1779
1780 if (!arch_supports_vector(Op_VectorTest, num_elem, elem_bt, is_vector_mask(vbox_klass) ? VecMaskUseLoad : VecMaskNotUsed)) {
1781 log_if_needed(" ** not supported: arity=2 op=test/%d vlen=%d etype=%s ismask=%d",
1782 cond->get_con(), num_elem, type2name(elem_bt),
1783 is_vector_mask(vbox_klass));
1784 return false;
1785 }
1786
1787 Node* opd1 = unbox_vector(argument(4), vbox_type, elem_bt, num_elem);
1788 Node* opd2;
1789 if (Matcher::vectortest_needs_second_argument(booltest == BoolTest::overflow,
1790 opd1->bottom_type()->isa_vectmask())) {
1791 opd2 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
1792 } else {
1793 opd2 = opd1;
1794 }
1795 if (opd1 == nullptr || opd2 == nullptr) {
1796 return false; // operand unboxing failed
1797 }
1798
1799 Node* cmp = gvn().transform(new VectorTestNode(opd1, opd2, booltest));
1800 BoolTest::mask test = Matcher::vectortest_mask(booltest == BoolTest::overflow,
1801 opd1->bottom_type()->isa_vectmask(), num_elem);
1802 Node* bol = gvn().transform(new BoolNode(cmp, test));
1803 Node* res = gvn().transform(new CMoveINode(bol, gvn().intcon(0), gvn().intcon(1), TypeInt::BOOL));
1804
1805 set_result(res);
1806 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1807 return true;
1808 }
1809
1810 // public static
1811 // <V extends Vector<E>,
1812 // M extends VectorMask<E>,
1813 // E>
1814 // V blend(Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType, int vlen,
1815 // V v1, V v2, M m,
1816 // VectorBlendOp<V, M, E> defaultImpl)
1817 bool LibraryCallKit::inline_vector_blend() {
1818 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
1819 const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
1820 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
1821 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
1822
1823 if (mask_klass == nullptr || vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr) {
1824 return false; // dead code
1825 }
1826 if (mask_klass->const_oop() == nullptr || vector_klass->const_oop() == nullptr ||
1827 elem_klass->const_oop() == nullptr || !vlen->is_con()) {
1828 log_if_needed(" ** missing constant: vclass=%s mclass=%s etype=%s vlen=%s",
1829 NodeClassNames[argument(0)->Opcode()],
1830 NodeClassNames[argument(1)->Opcode()],
1831 NodeClassNames[argument(2)->Opcode()],
1832 NodeClassNames[argument(3)->Opcode()]);
1833 return false; // not enough info for intrinsification
1834 }
1835 if (!is_klass_initialized(vector_klass) || !is_klass_initialized(mask_klass)) {
1836 log_if_needed(" ** klass argument not initialized");
1837 return false;
1838 }
1839 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1840 if (!elem_type->is_primitive_type()) {
1841 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1842 return false; // should be primitive type
1843 }
1844 BasicType elem_bt = elem_type->basic_type();
1845 BasicType mask_bt = elem_bt;
1846 int num_elem = vlen->get_con();
1847
1848 if (!arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad)) {
1849 log_if_needed(" ** not supported: arity=2 op=blend vlen=%d etype=%s ismask=useload",
1850 num_elem, type2name(elem_bt));
1851 return false; // not supported
1852 }
1853 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1854 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1855
1856 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1857 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1858
1859 Node* v1 = unbox_vector(argument(4), vbox_type, elem_bt, num_elem);
1860 Node* v2 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
1861 Node* mask = unbox_vector(argument(6), mbox_type, mask_bt, num_elem);
1862
1863 if (v1 == nullptr || v2 == nullptr || mask == nullptr) {
1864 return false; // operand unboxing failed
1865 }
1866
1867 Node* blend = gvn().transform(new VectorBlendNode(v1, v2, mask));
1868
1869 Node* box = box_vector(blend, vbox_type, elem_bt, num_elem);
1870 set_result(box);
1871 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1872 return true;
1873 }
1874
1875 // public static
1876 // <V extends Vector<E>,
1877 // M extends VectorMask<E>,
1878 // E>
1879 // M compare(int cond, Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType, int vlen,
1880 // V v1, V v2, M m,
1881 // VectorCompareOp<V,M> defaultImpl)
1882 bool LibraryCallKit::inline_vector_compare() {
1883 const TypeInt* cond = gvn().type(argument(0))->isa_int();
1884 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
1885 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
1886 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
1887 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
1888
1889 if (cond == nullptr || vector_klass == nullptr || mask_klass == nullptr || elem_klass == nullptr || vlen == nullptr) {
1890 return false; // dead code
1891 }
1892 if (!cond->is_con() || vector_klass->const_oop() == nullptr || mask_klass->const_oop() == nullptr ||
1893 elem_klass->const_oop() == nullptr || !vlen->is_con()) {
1894 log_if_needed(" ** missing constant: cond=%s vclass=%s mclass=%s etype=%s vlen=%s",
1895 NodeClassNames[argument(0)->Opcode()],
1896 NodeClassNames[argument(1)->Opcode()],
1897 NodeClassNames[argument(2)->Opcode()],
1898 NodeClassNames[argument(3)->Opcode()],
1899 NodeClassNames[argument(4)->Opcode()]);
1900 return false; // not enough info for intrinsification
1901 }
1902 if (!is_klass_initialized(vector_klass) || !is_klass_initialized(mask_klass)) {
1903 log_if_needed(" ** klass argument not initialized");
1904 return false;
1905 }
1906 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1907 if (!elem_type->is_primitive_type()) {
1908 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1909 return false; // should be primitive type
1910 }
1911
1912 int num_elem = vlen->get_con();
1913 BasicType elem_bt = elem_type->basic_type();
1914 BasicType mask_bt = elem_bt;
1915
1916 if ((cond->get_con() & BoolTest::unsigned_compare) != 0) {
1917 if (!Matcher::supports_vector_comparison_unsigned(num_elem, elem_bt)) {
1918 log_if_needed(" ** not supported: unsigned comparison op=comp/%d vlen=%d etype=%s ismask=usestore",
1919 cond->get_con() & (BoolTest::unsigned_compare - 1), num_elem, type2name(elem_bt));
1920 return false;
1921 }
1922 }
1923
1924 if (!arch_supports_vector(Op_VectorMaskCmp, num_elem, elem_bt, VecMaskUseStore)) {
1925 log_if_needed(" ** not supported: arity=2 op=comp/%d vlen=%d etype=%s ismask=usestore",
1926 cond->get_con(), num_elem, type2name(elem_bt));
1927 return false;
1928 }
1929
1930 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1931 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1932
1933 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1934 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1935
1936 Node* v1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
1937 Node* v2 = unbox_vector(argument(6), vbox_type, elem_bt, num_elem);
1938
1939 bool is_masked_op = argument(7)->bottom_type() != TypePtr::NULL_PTR;
1940 Node* mask = is_masked_op ? unbox_vector(argument(7), mbox_type, elem_bt, num_elem) : nullptr;
1941 if (is_masked_op && mask == nullptr) {
1942 log_if_needed(" ** not supported: mask = null arity=2 op=comp/%d vlen=%d etype=%s ismask=usestore is_masked_op=1",
1943 cond->get_con(), num_elem, type2name(elem_bt));
1944 return false;
1945 }
1946
1947 bool use_predicate = is_masked_op && arch_supports_vector(Op_VectorMaskCmp, num_elem, elem_bt, VecMaskUsePred);
1948 if (is_masked_op && !use_predicate && !arch_supports_vector(Op_AndV, num_elem, elem_bt, VecMaskUseLoad)) {
1949 log_if_needed(" ** not supported: arity=2 op=comp/%d vlen=%d etype=%s ismask=usestore is_masked_op=1",
1950 cond->get_con(), num_elem, type2name(elem_bt));
1951 return false;
1952 }
1953
1954 if (v1 == nullptr || v2 == nullptr) {
1955 return false; // operand unboxing failed
1956 }
1957 BoolTest::mask pred = (BoolTest::mask)cond->get_con();
1958 ConINode* pred_node = (ConINode*)gvn().makecon(cond);
1959
1960 const TypeVect* vmask_type = TypeVect::makemask(mask_bt, num_elem);
1961 Node* operation = new VectorMaskCmpNode(pred, v1, v2, pred_node, vmask_type);
1962
1963 if (is_masked_op) {
1964 if (use_predicate) {
1965 operation->add_req(mask);
1966 operation->add_flag(Node::Flag_is_predicated_vector);
1967 } else {
1968 operation = gvn().transform(operation);
1969 operation = VectorNode::make(Op_AndV, operation, mask, vmask_type);
1970 }
1971 }
1972
1973 operation = gvn().transform(operation);
1974
1975 Node* box = box_vector(operation, mbox_type, mask_bt, num_elem);
1976 set_result(box);
1977 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1978 return true;
1979 }
1980
1981 // public static
1982 // <V extends Vector<E>,
1983 // Sh extends VectorShuffle<E>,
1984 // M extends VectorMask<E>,
1985 // E>
1986 // V rearrangeOp(Class<? extends V> vectorClass, Class<Sh> shuffleClass, Class<M> maskClass, Class<E> elementType, int vlen,
1987 // V v1, Sh sh, M m,
1988 // VectorRearrangeOp<V, Sh, M, E> defaultImpl)
1989 bool LibraryCallKit::inline_vector_rearrange() {
1990 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
1991 const TypeInstPtr* shuffle_klass = gvn().type(argument(1))->isa_instptr();
1992 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
1993 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
1994 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
1995
1996 if (vector_klass == nullptr || shuffle_klass == nullptr || elem_klass == nullptr || vlen == nullptr) {
1997 return false; // dead code
1998 }
1999 if (shuffle_klass->const_oop() == nullptr ||
2000 vector_klass->const_oop() == nullptr ||
2001 elem_klass->const_oop() == nullptr ||
2002 !vlen->is_con()) {
2003 log_if_needed(" ** missing constant: vclass=%s sclass=%s etype=%s vlen=%s",
2004 NodeClassNames[argument(0)->Opcode()],
2005 NodeClassNames[argument(1)->Opcode()],
2006 NodeClassNames[argument(3)->Opcode()],
2007 NodeClassNames[argument(4)->Opcode()]);
2008 return false; // not enough info for intrinsification
2009 }
2010 if (!is_klass_initialized(vector_klass) ||
2011 !is_klass_initialized(shuffle_klass)) {
2012 log_if_needed(" ** klass argument not initialized");
2013 return false;
2014 }
2015 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2016 if (!elem_type->is_primitive_type()) {
2017 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2018 return false; // should be primitive type
2019 }
2020 BasicType elem_bt = elem_type->basic_type();
2021 BasicType shuffle_bt = elem_bt;
2022 int num_elem = vlen->get_con();
2023
2024 if (!arch_supports_vector(Op_VectorLoadShuffle, num_elem, elem_bt, VecMaskNotUsed)) {
2025 log_if_needed(" ** not supported: arity=0 op=load/shuffle vlen=%d etype=%s ismask=no",
2026 num_elem, type2name(elem_bt));
2027 return false; // not supported
2028 }
2029
2030 bool is_masked_op = argument(7)->bottom_type() != TypePtr::NULL_PTR;
2031 bool use_predicate = is_masked_op;
2032 if (is_masked_op &&
2033 (mask_klass == nullptr ||
2034 mask_klass->const_oop() == nullptr ||
2035 !is_klass_initialized(mask_klass))) {
2036 log_if_needed(" ** mask_klass argument not initialized");
2037 }
2038 VectorMaskUseType checkFlags = (VectorMaskUseType)(is_masked_op ? (VecMaskUseLoad | VecMaskUsePred) : VecMaskNotUsed);
2039 if (!arch_supports_vector(Op_VectorRearrange, num_elem, elem_bt, checkFlags)) {
2040 use_predicate = false;
2041 if(!is_masked_op ||
2042 (!arch_supports_vector(Op_VectorRearrange, num_elem, elem_bt, VecMaskNotUsed) ||
2043 !arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad) ||
2044 !arch_supports_vector(Op_Replicate, num_elem, elem_bt, VecMaskNotUsed))) {
2045 log_if_needed(" ** not supported: arity=2 op=shuffle/rearrange vlen=%d etype=%s ismask=no",
2046 num_elem, type2name(elem_bt));
2047 return false; // not supported
2048 }
2049 }
2050 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2051 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2052
2053 ciKlass* shbox_klass = shuffle_klass->const_oop()->as_instance()->java_lang_Class_klass();
2054 const TypeInstPtr* shbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, shbox_klass);
2055
2056 Node* v1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
2057 Node* shuffle = unbox_vector(argument(6), shbox_type, shuffle_bt, num_elem);
2058
2059 if (v1 == nullptr || shuffle == nullptr) {
2060 return false; // operand unboxing failed
2061 }
2062
2063 Node* mask = nullptr;
2064 if (is_masked_op) {
2065 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
2066 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
2067 mask = unbox_vector(argument(7), mbox_type, elem_bt, num_elem);
2068 if (mask == nullptr) {
2069 log_if_needed(" ** not supported: arity=3 op=shuffle/rearrange vlen=%d etype=%s ismask=useload is_masked_op=1",
2070 num_elem, type2name(elem_bt));
2071 return false;
2072 }
2073 }
2074
2075 Node* rearrange = new VectorRearrangeNode(v1, shuffle);
2076 if (is_masked_op) {
2077 if (use_predicate) {
2078 rearrange->add_req(mask);
2079 rearrange->add_flag(Node::Flag_is_predicated_vector);
2080 } else {
2081 const TypeVect* vt = v1->bottom_type()->is_vect();
2082 rearrange = gvn().transform(rearrange);
2083 Node* zero = gvn().makecon(Type::get_zero_type(elem_bt));
2084 Node* zerovec = gvn().transform(VectorNode::scalar2vector(zero, num_elem, elem_bt));
2085 rearrange = new VectorBlendNode(zerovec, rearrange, mask);
2086 }
2087 }
2088 rearrange = gvn().transform(rearrange);
2089
2090 Node* box = box_vector(rearrange, vbox_type, elem_bt, num_elem);
2091 set_result(box);
2092 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2093 return true;
2094 }
2095
2096 static address get_vector_math_address(int vop, int bits, BasicType bt, char* name_ptr, int name_len) {
2097 address addr = nullptr;
2098 assert(UseVectorStubs, "sanity");
2099 assert(name_ptr != nullptr, "unexpected");
2100 assert((vop >= VectorSupport::VECTOR_OP_MATH_START) && (vop <= VectorSupport::VECTOR_OP_MATH_END), "unexpected");
2101 int op = vop - VectorSupport::VECTOR_OP_MATH_START;
2102
2103 switch(bits) {
2104 case 64: //fallthough
2105 case 128: //fallthough
2106 case 256: //fallthough
2107 case 512:
2108 if (bt == T_FLOAT) {
2109 snprintf(name_ptr, name_len, "vector_%s_float_%dbits_fixed", VectorSupport::mathname[op], bits);
2110 addr = StubRoutines::_vector_f_math[exact_log2(bits/64)][op];
2111 } else {
2112 assert(bt == T_DOUBLE, "must be FP type only");
2113 snprintf(name_ptr, name_len, "vector_%s_double_%dbits_fixed", VectorSupport::mathname[op], bits);
2114 addr = StubRoutines::_vector_d_math[exact_log2(bits/64)][op];
2115 }
2116 break;
2117 default:
2118 if (!Matcher::supports_scalable_vector() || !Matcher::vector_size_supported(bt, bits/type2aelembytes(bt)) ) {
2119 snprintf(name_ptr, name_len, "invalid");
2120 addr = nullptr;
2121 Unimplemented();
2122 }
2123 break;
2124 }
2125
2126 if (addr == nullptr && Matcher::supports_scalable_vector()) {
2127 if (bt == T_FLOAT) {
2128 snprintf(name_ptr, name_len, "vector_%s_float_%dbits_scalable", VectorSupport::mathname[op], bits);
2129 addr = StubRoutines::_vector_f_math[VectorSupport::VEC_SIZE_SCALABLE][op];
2130 } else {
2131 assert(bt == T_DOUBLE, "must be FP type only");
2132 snprintf(name_ptr, name_len, "vector_%s_double_%dbits_scalable", VectorSupport::mathname[op], bits);
2133 addr = StubRoutines::_vector_d_math[VectorSupport::VEC_SIZE_SCALABLE][op];
2134 }
2135 }
2136
2137 return addr;
2138 }
2139
2140 // public static
2141 // <V extends Vector<E>,
2142 // M extends VectorMask<E>,
2143 // E>
2144 // V selectFromOp(Class<? extends V> vClass, Class<M> mClass, Class<E> eClass,
2145 // int length, V v1, V v2, M m,
2146 // VectorSelectFromOp<V, M> defaultImpl)
2147 bool LibraryCallKit::inline_vector_select_from() {
2148 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
2149 const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
2150 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
2151 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
2152
2153 if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr ||
2154 vector_klass->const_oop() == nullptr ||
2155 elem_klass->const_oop() == nullptr ||
2156 !vlen->is_con()) {
2157 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s",
2158 NodeClassNames[argument(0)->Opcode()],
2159 NodeClassNames[argument(2)->Opcode()],
2160 NodeClassNames[argument(3)->Opcode()]);
2161 return false; // not enough info for intrinsification
2162 }
2163 if (!is_klass_initialized(vector_klass)) {
2164 log_if_needed(" ** klass argument not initialized");
2165 return false;
2166 }
2167 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2168 if (!elem_type->is_primitive_type()) {
2169 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2170 return false; // should be primitive type
2171 }
2172 BasicType elem_bt = elem_type->basic_type();
2173 int num_elem = vlen->get_con();
2174 if (!is_power_of_2(num_elem)) {
2175 log_if_needed(" ** vlen not power of two=%d", num_elem);
2176 return false;
2177 }
2178
2179 int cast_vopc = VectorCastNode::opcode(-1, elem_bt); // from vector of type elem_bt
2180 if (!arch_supports_vector(Op_VectorLoadShuffle, num_elem, elem_bt, VecMaskNotUsed)||
2181 !arch_supports_vector(Op_AndV, num_elem, T_BYTE, VecMaskNotUsed) ||
2182 !arch_supports_vector(Op_Replicate, num_elem, T_BYTE, VecMaskNotUsed) ||
2183 !arch_supports_vector(cast_vopc, num_elem, T_BYTE, VecMaskNotUsed)) {
2184 log_if_needed(" ** not supported: arity=0 op=selectFrom vlen=%d etype=%s ismask=no",
2185 num_elem, type2name(elem_bt));
2186 return false; // not supported
2187 }
2188
2189 bool is_masked_op = argument(6)->bottom_type() != TypePtr::NULL_PTR;
2190 bool use_predicate = is_masked_op;
2191 if (is_masked_op &&
2192 (mask_klass == nullptr ||
2193 mask_klass->const_oop() == nullptr ||
2194 !is_klass_initialized(mask_klass))) {
2195 log_if_needed(" ** mask_klass argument not initialized");
2196 return false; // not supported
2197 }
2198 VectorMaskUseType checkFlags = (VectorMaskUseType)(is_masked_op ? (VecMaskUseLoad | VecMaskUsePred) : VecMaskNotUsed);
2199 if (!arch_supports_vector(Op_VectorRearrange, num_elem, elem_bt, checkFlags)) {
2200 use_predicate = false;
2201 if(!is_masked_op ||
2202 (!arch_supports_vector(Op_VectorRearrange, num_elem, elem_bt, VecMaskNotUsed) ||
2203 !arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad) ||
2204 !arch_supports_vector(Op_Replicate, num_elem, elem_bt, VecMaskNotUsed))) {
2205 log_if_needed(" ** not supported: op=selectFrom vlen=%d etype=%s is_masked_op=%d",
2206 num_elem, type2name(elem_bt), is_masked_op);
2207 return false; // not supported
2208 }
2209 }
2210 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2211 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2212
2213 // v1 is the index vector
2214 Node* v1 = unbox_vector(argument(4), vbox_type, elem_bt, num_elem);
2215 // v2 is the vector being rearranged
2216 Node* v2 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
2217
2218 if (v1 == nullptr) {
2219 log_if_needed(" ** unbox failed v1=%s", NodeClassNames[argument(4)->Opcode()]);
2220 return false; // operand unboxing failed
2221 }
2222
2223 if (v2 == nullptr) {
2224 log_if_needed(" ** unbox failed v2=%s", NodeClassNames[argument(5)->Opcode()]);
2225 return false; // operand unboxing failed
2226 }
2227
2228 Node* mask = nullptr;
2229 if (is_masked_op) {
2230 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
2231 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
2232 mask = unbox_vector(argument(6), mbox_type, elem_bt, num_elem);
2233 if (mask == nullptr) {
2234 log_if_needed(" ** unbox failed mask=%s", NodeClassNames[argument(6)->Opcode()]);
2235 return false;
2236 }
2237 }
2238
2239 // cast index vector from elem_bt vector to byte vector
2240 const TypeVect* byte_vt = TypeVect::make(T_BYTE, num_elem);
2241 Node* byte_shuffle = gvn().transform(VectorCastNode::make(cast_vopc, v1, T_BYTE, num_elem));
2242
2243 // wrap the byte vector lanes to (num_elem - 1) to form the shuffle vector where num_elem is vector length
2244 // this is a simple AND operation as we come here only for power of two vector length
2245 Node* mod_val = gvn().makecon(TypeInt::make(num_elem-1));
2246 Node* bcast_mod = gvn().transform(VectorNode::scalar2vector(mod_val, num_elem, T_BYTE));
2247 byte_shuffle = gvn().transform(VectorNode::make(Op_AndV, byte_shuffle, bcast_mod, byte_vt));
2248
2249 // load the shuffle to use in rearrange
2250 const TypeVect* shuffle_vt = TypeVect::make(elem_bt, num_elem);
2251 Node* load_shuffle = gvn().transform(new VectorLoadShuffleNode(byte_shuffle, shuffle_vt));
2252
2253 // and finally rearrange
2254 Node* rearrange = new VectorRearrangeNode(v2, load_shuffle);
2255 if (is_masked_op) {
2256 if (use_predicate) {
2257 // masked rearrange is supported so use that directly
2258 rearrange->add_req(mask);
2259 rearrange->add_flag(Node::Flag_is_predicated_vector);
2260 } else {
2261 // masked rearrange is not supported so emulate usig blend
2262 const TypeVect* vt = v1->bottom_type()->is_vect();
2263 rearrange = gvn().transform(rearrange);
2264
2265 // create a zero vector with each lane element set as zero
2266 Node* zero = gvn().makecon(Type::get_zero_type(elem_bt));
2267 Node* zerovec = gvn().transform(VectorNode::scalar2vector(zero, num_elem, elem_bt));
2268
2269 // For each lane for which mask is set, blend in the rearranged lane into zero vector
2270 rearrange = new VectorBlendNode(zerovec, rearrange, mask);
2271 }
2272 }
2273 rearrange = gvn().transform(rearrange);
2274
2275 // box the result
2276 Node* box = box_vector(rearrange, vbox_type, elem_bt, num_elem);
2277 set_result(box);
2278
2279 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2280 return true;
2281 }
2282
2283 Node* LibraryCallKit::gen_call_to_vector_math(int vector_api_op_id, BasicType bt, int num_elem, Node* opd1, Node* opd2) {
2284 assert(UseVectorStubs, "sanity");
2285 assert(vector_api_op_id >= VectorSupport::VECTOR_OP_MATH_START && vector_api_op_id <= VectorSupport::VECTOR_OP_MATH_END, "need valid op id");
2286 assert(opd1 != nullptr, "must not be null");
2287 const TypeVect* vt = TypeVect::make(bt, num_elem);
2288 const TypeFunc* call_type = OptoRuntime::Math_Vector_Vector_Type(opd2 != nullptr ? 2 : 1, vt, vt);
2289 char name[100] = "";
2290
2291 // Get address for vector math method.
2292 address addr = get_vector_math_address(vector_api_op_id, vt->length_in_bytes() * BitsPerByte, bt, name, 100);
2293
2294 if (addr == nullptr) {
2295 return nullptr;
2296 }
2297
2298 assert(name[0] != '\0', "name must not be null");
2299 Node* operation = make_runtime_call(RC_VECTOR,
2300 call_type,
2301 addr,
2302 name,
2303 TypePtr::BOTTOM,
2304 opd1,
2305 opd2);
2306 return gvn().transform(new ProjNode(gvn().transform(operation), TypeFunc::Parms));
2307 }
2308
2309 // public static
2310 // <V extends Vector<E>,
2311 // M extends VectorMask<E>,
2312 // E>
2313 // V broadcastInt(int opr, Class<? extends V> vectorClass, Class<? extends M> maskClass,
2314 // Class<E> elementType, int length,
2315 // V v, int n, M m,
2316 // VectorBroadcastIntOp<V, M> defaultImpl)
2317 bool LibraryCallKit::inline_vector_broadcast_int() {
2318 const TypeInt* opr = gvn().type(argument(0))->isa_int();
2319 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
2320 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
2321 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
2322 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
2323
2324 if (opr == nullptr || vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr) {
2325 return false; // dead code
2326 }
2327 if (!opr->is_con() || vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
2328 log_if_needed(" ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
2329 NodeClassNames[argument(0)->Opcode()],
2330 NodeClassNames[argument(1)->Opcode()],
2331 NodeClassNames[argument(3)->Opcode()],
2332 NodeClassNames[argument(4)->Opcode()]);
2333 return false; // not enough info for intrinsification
2334 }
2335 if (!is_klass_initialized(vector_klass)) {
2336 log_if_needed(" ** klass argument not initialized");
2337 return false;
2338 }
2339
2340 const Type* vmask_type = gvn().type(argument(7));
2341 bool is_masked_op = vmask_type != TypePtr::NULL_PTR;
2342 if (is_masked_op) {
2343 if (mask_klass == nullptr || mask_klass->const_oop() == nullptr) {
2344 log_if_needed(" ** missing constant: maskclass=%s", NodeClassNames[argument(2)->Opcode()]);
2345 return false; // not enough info for intrinsification
2346 }
2347
2348 if (!is_klass_initialized(mask_klass)) {
2349 log_if_needed(" ** mask klass argument not initialized");
2350 return false;
2351 }
2352
2353 if (vmask_type->maybe_null()) {
2354 log_if_needed(" ** null mask values are not allowed for masked op");
2355 return false;
2356 }
2357 }
2358
2359 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2360 if (!elem_type->is_primitive_type()) {
2361 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2362 return false; // should be primitive type
2363 }
2364
2365 int num_elem = vlen->get_con();
2366 BasicType elem_bt = elem_type->basic_type();
2367 int opc = VectorSupport::vop2ideal(opr->get_con(), elem_bt);
2368
2369 bool is_shift = VectorNode::is_shift_opcode(opc);
2370 bool is_rotate = VectorNode::is_rotate_opcode(opc);
2371
2372 if (opc == 0 || (!is_shift && !is_rotate)) {
2373 log_if_needed(" ** operation not supported: op=%d bt=%s", opr->get_con(), type2name(elem_bt));
2374 return false; // operation not supported
2375 }
2376
2377 int sopc = VectorNode::opcode(opc, elem_bt);
2378 if (sopc == 0) {
2379 log_if_needed(" ** operation not supported: opc=%s bt=%s", NodeClassNames[opc], type2name(elem_bt));
2380 return false; // operation not supported
2381 }
2382
2383 Node* cnt = argument(6);
2384 const TypeInt* cnt_type = cnt->bottom_type()->isa_int();
2385
2386 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2387 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2388
2389 // If CPU supports vector constant rotate instructions pass it directly
2390 bool is_const_rotate = is_rotate && cnt_type && cnt_type->is_con() &&
2391 Matcher::supports_vector_constant_rotates(cnt_type->get_con());
2392 bool has_scalar_args = is_rotate ? !is_const_rotate : true;
2393
2394 VectorMaskUseType checkFlags = (VectorMaskUseType)(is_masked_op ? (VecMaskUseLoad | VecMaskUsePred) : VecMaskNotUsed);
2395 bool use_predicate = is_masked_op;
2396
2397 if (!arch_supports_vector(sopc, num_elem, elem_bt, checkFlags, has_scalar_args)) {
2398 use_predicate = false;
2399 if (!is_masked_op ||
2400 (!arch_supports_vector(sopc, num_elem, elem_bt, VecMaskNotUsed, has_scalar_args) ||
2401 !arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad))) {
2402
2403 log_if_needed(" ** not supported: arity=0 op=int/%d vlen=%d etype=%s is_masked_op=%d",
2404 sopc, num_elem, type2name(elem_bt), is_masked_op ? 1 : 0);
2405 return false; // not supported
2406 }
2407 }
2408
2409 Node* opd1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
2410 Node* opd2 = nullptr;
2411 if (is_shift) {
2412 opd2 = vector_shift_count(cnt, opc, elem_bt, num_elem);
2413 } else {
2414 assert(is_rotate, "unexpected operation");
2415 if (!is_const_rotate) {
2416 cnt = elem_bt == T_LONG ? gvn().transform(new ConvI2LNode(cnt)) : cnt;
2417 opd2 = gvn().transform(VectorNode::scalar2vector(cnt, num_elem, elem_bt));
2418 } else {
2419 // Constant shift value.
2420 opd2 = cnt;
2421 }
2422 }
2423
2424 if (opd1 == nullptr || opd2 == nullptr) {
2425 return false;
2426 }
2427
2428 Node* mask = nullptr;
2429 if (is_masked_op) {
2430 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
2431 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
2432 mask = unbox_vector(argument(7), mbox_type, elem_bt, num_elem);
2433 if (mask == nullptr) {
2434 log_if_needed(" ** unbox failed mask=%s", NodeClassNames[argument(7)->Opcode()]);
2435 return false;
2436 }
2437 }
2438
2439 Node* operation = VectorNode::make(opc, opd1, opd2, num_elem, elem_bt);
2440 if (is_masked_op && mask != nullptr) {
2441 if (use_predicate) {
2442 operation->add_req(mask);
2443 operation->add_flag(Node::Flag_is_predicated_vector);
2444 } else {
2445 operation = gvn().transform(operation);
2446 operation = new VectorBlendNode(opd1, operation, mask);
2447 }
2448 }
2449 operation = gvn().transform(operation);
2450 Node* vbox = box_vector(operation, vbox_type, elem_bt, num_elem);
2451 set_result(vbox);
2452 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2453 return true;
2454 }
2455
2456 // public static <VOUT extends VectorPayload,
2457 // VIN extends VectorPayload,
2458 // S extends VectorSpecies>
2459 // VOUT convert(int oprId,
2460 // Class<?> fromVectorClass, Class<?> fromElementType, int fromVLen,
2461 // Class<?> toVectorClass, Class<?> toElementType, int toVLen,
2462 // VIN v, S s,
2463 // VectorConvertOp<VOUT, VIN, S> defaultImpl)
2464 //
2465 bool LibraryCallKit::inline_vector_convert() {
2466 const TypeInt* opr = gvn().type(argument(0))->isa_int();
2467
2468 const TypeInstPtr* vector_klass_from = gvn().type(argument(1))->isa_instptr();
2469 const TypeInstPtr* elem_klass_from = gvn().type(argument(2))->isa_instptr();
2470 const TypeInt* vlen_from = gvn().type(argument(3))->isa_int();
2471
2472 const TypeInstPtr* vector_klass_to = gvn().type(argument(4))->isa_instptr();
2473 const TypeInstPtr* elem_klass_to = gvn().type(argument(5))->isa_instptr();
2474 const TypeInt* vlen_to = gvn().type(argument(6))->isa_int();
2475
2476 if (opr == nullptr ||
2477 vector_klass_from == nullptr || elem_klass_from == nullptr || vlen_from == nullptr ||
2478 vector_klass_to == nullptr || elem_klass_to == nullptr || vlen_to == nullptr) {
2479 return false; // dead code
2480 }
2481 if (!opr->is_con() ||
2482 vector_klass_from->const_oop() == nullptr || elem_klass_from->const_oop() == nullptr || !vlen_from->is_con() ||
2483 vector_klass_to->const_oop() == nullptr || elem_klass_to->const_oop() == nullptr || !vlen_to->is_con()) {
2484 log_if_needed(" ** missing constant: opr=%s vclass_from=%s etype_from=%s vlen_from=%s vclass_to=%s etype_to=%s vlen_to=%s",
2485 NodeClassNames[argument(0)->Opcode()],
2486 NodeClassNames[argument(1)->Opcode()],
2487 NodeClassNames[argument(2)->Opcode()],
2488 NodeClassNames[argument(3)->Opcode()],
2489 NodeClassNames[argument(4)->Opcode()],
2490 NodeClassNames[argument(5)->Opcode()],
2491 NodeClassNames[argument(6)->Opcode()]);
2492 return false; // not enough info for intrinsification
2493 }
2494 if (!is_klass_initialized(vector_klass_from) || !is_klass_initialized(vector_klass_to)) {
2495 log_if_needed(" ** klass argument not initialized");
2496 return false;
2497 }
2498
2499 assert(opr->get_con() == VectorSupport::VECTOR_OP_CAST ||
2500 opr->get_con() == VectorSupport::VECTOR_OP_UCAST ||
2501 opr->get_con() == VectorSupport::VECTOR_OP_REINTERPRET, "wrong opcode");
2502 bool is_cast = (opr->get_con() == VectorSupport::VECTOR_OP_CAST || opr->get_con() == VectorSupport::VECTOR_OP_UCAST);
2503 bool is_ucast = (opr->get_con() == VectorSupport::VECTOR_OP_UCAST);
2504
2505 ciKlass* vbox_klass_from = vector_klass_from->const_oop()->as_instance()->java_lang_Class_klass();
2506 ciKlass* vbox_klass_to = vector_klass_to->const_oop()->as_instance()->java_lang_Class_klass();
2507 if (is_vector_shuffle(vbox_klass_from)) {
2508 return false; // vector shuffles aren't supported
2509 }
2510 bool is_mask = is_vector_mask(vbox_klass_from);
2511
2512 ciType* elem_type_from = elem_klass_from->const_oop()->as_instance()->java_mirror_type();
2513 if (!elem_type_from->is_primitive_type()) {
2514 return false; // should be primitive type
2515 }
2516 BasicType elem_bt_from = elem_type_from->basic_type();
2517 ciType* elem_type_to = elem_klass_to->const_oop()->as_instance()->java_mirror_type();
2518 if (!elem_type_to->is_primitive_type()) {
2519 return false; // should be primitive type
2520 }
2521 BasicType elem_bt_to = elem_type_to->basic_type();
2522
2523 int num_elem_from = vlen_from->get_con();
2524 int num_elem_to = vlen_to->get_con();
2525
2526 // Check whether we can unbox to appropriate size. Even with casting, checking for reinterpret is needed
2527 // since we may need to change size.
2528 if (!arch_supports_vector(Op_VectorReinterpret,
2529 num_elem_from,
2530 elem_bt_from,
2531 is_mask ? VecMaskUseAll : VecMaskNotUsed)) {
2532 log_if_needed(" ** not supported: arity=1 op=%s/1 vlen1=%d etype1=%s ismask=%d",
2533 is_cast ? "cast" : "reinterpret",
2534 num_elem_from, type2name(elem_bt_from), is_mask);
2535 return false;
2536 }
2537
2538 // Check whether we can support resizing/reinterpreting to the new size.
2539 if (!arch_supports_vector(Op_VectorReinterpret,
2540 num_elem_to,
2541 elem_bt_to,
2542 is_mask ? VecMaskUseAll : VecMaskNotUsed)) {
2543 log_if_needed(" ** not supported: arity=1 op=%s/2 vlen2=%d etype2=%s ismask=%d",
2544 is_cast ? "cast" : "reinterpret",
2545 num_elem_to, type2name(elem_bt_to), is_mask);
2546 return false;
2547 }
2548
2549
2550 if (is_vector_shuffle(vbox_klass_to) &&
2551 (!arch_supports_vector(Op_SubVB, num_elem_to, elem_bt_to, VecMaskNotUsed) ||
2552 !arch_supports_vector(Op_VectorBlend, num_elem_to, elem_bt_to, VecMaskNotUsed) ||
2553 !arch_supports_vector(Op_VectorMaskCmp, num_elem_to, elem_bt_to, VecMaskNotUsed) ||
2554 !arch_supports_vector(Op_AndV, num_elem_to, elem_bt_to, VecMaskNotUsed) ||
2555 !arch_supports_vector(Op_Replicate, num_elem_to, elem_bt_to, VecMaskNotUsed))) {
2556 log_if_needed(" ** not supported: arity=1 op=shuffle_index_wrap vlen2=%d etype2=%s",
2557 num_elem_to, type2name(elem_bt_to));
2558 return false;
2559 }
2560
2561 // At this point, we know that both input and output vector registers are supported
2562 // by the architecture. Next check if the casted type is simply to same type - which means
2563 // that it is actually a resize and not a cast.
2564 if (is_cast && elem_bt_from == elem_bt_to) {
2565 is_cast = false;
2566 }
2567
2568 const TypeInstPtr* vbox_type_from = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass_from);
2569
2570 Node* opd1 = unbox_vector(argument(7), vbox_type_from, elem_bt_from, num_elem_from);
2571 if (opd1 == nullptr) {
2572 return false;
2573 }
2574
2575 const TypeVect* src_type = TypeVect::make(elem_bt_from, num_elem_from, is_mask);
2576 const TypeVect* dst_type = TypeVect::make(elem_bt_to, num_elem_to, is_mask);
2577
2578 // Safety check to prevent casting if source mask is of type vector
2579 // and destination mask of type predicate vector and vice-versa.
2580 // From X86 standpoint, this case will only arise over KNL target,
2581 // where certain masks (depending on the species) are either propagated
2582 // through a vector or predicate register.
2583 if (is_mask &&
2584 ((src_type->isa_vectmask() == nullptr && dst_type->isa_vectmask()) ||
2585 (dst_type->isa_vectmask() == nullptr && src_type->isa_vectmask()))) {
2586 return false;
2587 }
2588
2589 Node* op = opd1;
2590 if (is_cast) {
2591 assert(!is_mask || num_elem_from == num_elem_to, "vector mask cast needs the same elem num");
2592 int cast_vopc = VectorCastNode::opcode(-1, elem_bt_from, !is_ucast);
2593
2594 // Make sure that vector cast is implemented to particular type/size combination if it is
2595 // not a mask casting.
2596 if (!is_mask && !arch_supports_vector(cast_vopc, num_elem_to, elem_bt_to, VecMaskNotUsed)) {
2597 log_if_needed(" ** not supported: arity=1 op=cast#%d/3 vlen2=%d etype2=%s ismask=%d",
2598 cast_vopc, num_elem_to, type2name(elem_bt_to), is_mask);
2599 return false;
2600 }
2601
2602 if (num_elem_from < num_elem_to) {
2603 // Since input and output number of elements are not consistent, we need to make sure we
2604 // properly size. Thus, first make a cast that retains the number of elements from source.
2605 int num_elem_for_cast = num_elem_from;
2606
2607 // It is possible that arch does not support this intermediate vector size
2608 // TODO More complex logic required here to handle this corner case for the sizes.
2609 if (!arch_supports_vector(cast_vopc, num_elem_for_cast, elem_bt_to, VecMaskNotUsed)) {
2610 log_if_needed(" ** not supported: arity=1 op=cast#%d/4 vlen1=%d etype2=%s ismask=%d",
2611 cast_vopc,
2612 num_elem_for_cast, type2name(elem_bt_to), is_mask);
2613 return false;
2614 }
2615
2616 op = gvn().transform(VectorCastNode::make(cast_vopc, op, elem_bt_to, num_elem_for_cast));
2617 // Now ensure that the destination gets properly resized to needed size.
2618 op = gvn().transform(new VectorReinterpretNode(op, op->bottom_type()->is_vect(), dst_type));
2619 } else if (num_elem_from > num_elem_to) {
2620 // Since number of elements from input is larger than output, simply reduce size of input
2621 // (we are supposed to drop top elements anyway).
2622 int num_elem_for_resize = num_elem_to;
2623
2624 // It is possible that arch does not support this intermediate vector size
2625 // TODO More complex logic required here to handle this corner case for the sizes.
2626 if (!arch_supports_vector(Op_VectorReinterpret,
2627 num_elem_for_resize,
2628 elem_bt_from,
2629 VecMaskNotUsed)) {
2630 log_if_needed(" ** not supported: arity=1 op=cast/5 vlen2=%d etype1=%s ismask=%d",
2631 num_elem_for_resize, type2name(elem_bt_from), is_mask);
2632 return false;
2633 }
2634
2635 const TypeVect* resize_type = TypeVect::make(elem_bt_from, num_elem_for_resize);
2636 op = gvn().transform(new VectorReinterpretNode(op, src_type, resize_type));
2637 op = gvn().transform(VectorCastNode::make(cast_vopc, op, elem_bt_to, num_elem_to));
2638 } else { // num_elem_from == num_elem_to
2639 if (is_mask) {
2640 // Make sure that cast for vector mask is implemented to particular type/size combination.
2641 if (!arch_supports_vector(Op_VectorMaskCast, num_elem_to, elem_bt_to, VecMaskNotUsed)) {
2642 log_if_needed(" ** not supported: arity=1 op=maskcast vlen2=%d etype2=%s ismask=%d",
2643 num_elem_to, type2name(elem_bt_to), is_mask);
2644 return false;
2645 }
2646 op = gvn().transform(new VectorMaskCastNode(op, dst_type));
2647 } else {
2648 // Since input and output number of elements match, and since we know this vector size is
2649 // supported, simply do a cast with no resize needed.
2650 op = gvn().transform(VectorCastNode::make(cast_vopc, op, elem_bt_to, num_elem_to));
2651 }
2652 }
2653 } else if (!Type::equals(src_type, dst_type)) {
2654 assert(!is_cast, "must be reinterpret");
2655 op = gvn().transform(new VectorReinterpretNode(op, src_type, dst_type));
2656 }
2657
2658 if (is_vector_shuffle(vbox_klass_to)) {
2659 op = partially_wrap_indexes(op, num_elem_to, elem_bt_to);
2660 }
2661
2662 const TypeInstPtr* vbox_type_to = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass_to);
2663 Node* vbox = box_vector(op, vbox_type_to, elem_bt_to, num_elem_to);
2664 set_result(vbox);
2665 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem_to * type2aelembytes(elem_bt_to))));
2666 return true;
2667 }
2668
2669 // public static
2670 // <V extends Vector<E>,
2671 // E>
2672 // V insert(Class<? extends V> vectorClass, Class<E> elementType, int vlen,
2673 // V vec, int ix, long val,
2674 // VecInsertOp<V> defaultImpl)
2675 bool LibraryCallKit::inline_vector_insert() {
2676 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
2677 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
2678 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
2679 const TypeInt* idx = gvn().type(argument(4))->isa_int();
2680
2681 if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr || idx == nullptr) {
2682 return false; // dead code
2683 }
2684 if (vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con() || !idx->is_con()) {
2685 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s idx=%s",
2686 NodeClassNames[argument(0)->Opcode()],
2687 NodeClassNames[argument(1)->Opcode()],
2688 NodeClassNames[argument(2)->Opcode()],
2689 NodeClassNames[argument(4)->Opcode()]);
2690 return false; // not enough info for intrinsification
2691 }
2692 if (!is_klass_initialized(vector_klass)) {
2693 log_if_needed(" ** klass argument not initialized");
2694 return false;
2695 }
2696 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2697 if (!elem_type->is_primitive_type()) {
2698 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2699 return false; // should be primitive type
2700 }
2701 BasicType elem_bt = elem_type->basic_type();
2702 int num_elem = vlen->get_con();
2703 if (!arch_supports_vector(Op_VectorInsert, num_elem, elem_bt, VecMaskNotUsed)) {
2704 log_if_needed(" ** not supported: arity=1 op=insert vlen=%d etype=%s ismask=no",
2705 num_elem, type2name(elem_bt));
2706 return false; // not supported
2707 }
2708
2709 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2710 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2711
2712 Node* opd = unbox_vector(argument(3), vbox_type, elem_bt, num_elem);
2713 if (opd == nullptr) {
2714 return false;
2715 }
2716
2717 Node* insert_val = argument(5);
2718 assert(gvn().type(insert_val)->isa_long() != nullptr, "expected to be long");
2719
2720 // Convert insert value back to its appropriate type.
2721 switch (elem_bt) {
2722 case T_BYTE:
2723 insert_val = gvn().transform(new ConvL2INode(insert_val, TypeInt::BYTE));
2724 break;
2725 case T_SHORT:
2726 insert_val = gvn().transform(new ConvL2INode(insert_val, TypeInt::SHORT));
2727 break;
2728 case T_INT:
2729 insert_val = gvn().transform(new ConvL2INode(insert_val));
2730 break;
2731 case T_FLOAT:
2732 insert_val = gvn().transform(new ConvL2INode(insert_val));
2733 insert_val = gvn().transform(new MoveI2FNode(insert_val));
2734 break;
2735 case T_DOUBLE:
2736 insert_val = gvn().transform(new MoveL2DNode(insert_val));
2737 break;
2738 case T_LONG:
2739 // no conversion needed
2740 break;
2741 default: fatal("%s", type2name(elem_bt)); break;
2742 }
2743
2744 Node* operation = gvn().transform(VectorInsertNode::make(opd, insert_val, idx->get_con(), gvn()));
2745
2746 Node* vbox = box_vector(operation, vbox_type, elem_bt, num_elem);
2747 set_result(vbox);
2748 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2749 return true;
2750 }
2751
2752 // public static
2753 // <VM extends VectorPayload,
2754 // E>
2755 // long extract(Class<? extends VM> vClass, Class<E> eClass,
2756 // int length,
2757 // VM vm, int i,
2758 // VecExtractOp<VM> defaultImpl)
2759 bool LibraryCallKit::inline_vector_extract() {
2760 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
2761 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
2762 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
2763 const TypeInt* idx = gvn().type(argument(4))->isa_int();
2764
2765 if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr || idx == nullptr) {
2766 return false; // dead code
2767 }
2768 if (vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con() || !idx->is_con()) {
2769 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s",
2770 NodeClassNames[argument(0)->Opcode()],
2771 NodeClassNames[argument(1)->Opcode()],
2772 NodeClassNames[argument(2)->Opcode()]);
2773 return false; // not enough info for intrinsification
2774 }
2775 if (!is_klass_initialized(vector_klass)) {
2776 log_if_needed(" ** klass argument not initialized");
2777 return false;
2778 }
2779 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2780 if (!elem_type->is_primitive_type()) {
2781 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2782 return false; // should be primitive type
2783 }
2784 BasicType elem_bt = elem_type->basic_type();
2785 int num_elem = vlen->get_con();
2786
2787 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2788 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2789
2790 Node* opd = nullptr;
2791
2792 if (is_vector_mask(vbox_klass)) {
2793 // vbox_klass is mask. This is used for VectorMask.laneIsSet(int).
2794
2795 Node* pos = argument(4); // can be variable
2796 if (arch_supports_vector(Op_ExtractUB, num_elem, elem_bt, VecMaskUseAll)) {
2797 // Transform mask to vector with type of boolean and utilize ExtractUB node.
2798 opd = unbox_vector(argument(3), vbox_type, elem_bt, num_elem);
2799 if (opd == nullptr) {
2800 return false;
2801 }
2802 opd = gvn().transform(VectorStoreMaskNode::make(gvn(), opd, elem_bt, num_elem));
2803 opd = gvn().transform(new ExtractUBNode(opd, pos));
2804 opd = gvn().transform(new ConvI2LNode(opd));
2805 } else if (arch_supports_vector(Op_VectorMaskToLong, num_elem, elem_bt, VecMaskUseLoad)) {
2806 opd = unbox_vector(argument(3), vbox_type, elem_bt, num_elem);
2807 if (opd == nullptr) {
2808 return false;
2809 }
2810 // VectorMaskToLongNode requires the input is either a mask or a vector with BOOLEAN type.
2811 if (opd->bottom_type()->isa_vectmask() == nullptr) {
2812 opd = gvn().transform(VectorStoreMaskNode::make(gvn(), opd, elem_bt, num_elem));
2813 }
2814 // ((toLong() >>> pos) & 1L
2815 opd = gvn().transform(new VectorMaskToLongNode(opd, TypeLong::LONG));
2816 opd = gvn().transform(new URShiftLNode(opd, pos));
2817 opd = gvn().transform(new AndLNode(opd, gvn().makecon(TypeLong::ONE)));
2818 } else {
2819 log_if_needed(" ** Rejected mask extraction because architecture does not support it");
2820 return false; // not supported
2821 }
2822 } else {
2823 // vbox_klass is vector. This is used for Vector.lane(int).
2824 if (!idx->is_con()) {
2825 log_if_needed(" ** missing constant: idx=%s", NodeClassNames[argument(4)->Opcode()]);
2826 return false; // not enough info for intrinsification
2827 }
2828
2829 int vopc = ExtractNode::opcode(elem_bt);
2830 if (!arch_supports_vector(vopc, num_elem, elem_bt, VecMaskNotUsed)) {
2831 log_if_needed(" ** not supported: arity=1 op=extract vlen=%d etype=%s ismask=no",
2832 num_elem, type2name(elem_bt));
2833 return false; // not supported
2834 }
2835
2836 opd = unbox_vector(argument(3), vbox_type, elem_bt, num_elem);
2837 if (opd == nullptr) {
2838 return false;
2839 }
2840 ConINode* idx_con = gvn().intcon(idx->get_con())->as_ConI();
2841
2842 opd = gvn().transform(ExtractNode::make(opd, idx_con, elem_bt));
2843 switch (elem_bt) {
2844 case T_BYTE:
2845 case T_SHORT:
2846 case T_INT: {
2847 opd = gvn().transform(new ConvI2LNode(opd));
2848 break;
2849 }
2850 case T_FLOAT: {
2851 opd = gvn().transform(new MoveF2INode(opd));
2852 opd = gvn().transform(new ConvI2LNode(opd));
2853 break;
2854 }
2855 case T_DOUBLE: {
2856 opd = gvn().transform(new MoveD2LNode(opd));
2857 break;
2858 }
2859 case T_LONG: {
2860 // no conversion needed
2861 break;
2862 }
2863 default: fatal("%s", type2name(elem_bt));
2864 }
2865 }
2866 set_result(opd);
2867 return true;
2868 }
2869
2870 static Node* LowerSelectFromTwoVectorOperation(PhaseGVN& phase, Node* index_vec, Node* src1, Node* src2, const TypeVect* vt) {
2871 int num_elem = vt->length();
2872 BasicType elem_bt = vt->element_basic_type();
2873
2874 // Lower selectFrom operation into its constituent operations.
2875 // SelectFromTwoVectorNode =
2876 // (VectorBlend
2877 // (VectorRearrange SRC1 (WRAPED_INDEX AND (VLEN-1))
2878 // (VectorRearrange SRC2 (WRAPED_INDEX AND (VLEN-1))
2879 // MASK)
2880 // Where
2881 // WRAPED_INDEX are computed by wrapping incoming indexes
2882 // to two vector index range [0, VLEN*2) and
2883 // MASK = WRAPED_INDEX < VLEN
2884 //
2885 // IR lowering prevents intrinsification failure and associated argument
2886 // boxing penalties.
2887 //
2888
2889 const TypeVect* index_vect_type = index_vec->bottom_type()->is_vect();
2890 BasicType index_elem_bt = index_vect_type->element_basic_type();
2891
2892 // Downcast index vector to a type agnostic shuffle representation, shuffle
2893 // indices are held in a byte vector which are later transformed to target
2894 // specific permutation index format by subsequent VectorLoadShuffle.
2895 int cast_vopc = VectorCastNode::opcode(0, index_elem_bt, true);
2896 Node* index_byte_vec = phase.transform(VectorCastNode::make(cast_vopc, index_vec, T_BYTE, num_elem));
2897
2898 // Wrap indexes into two vector index range [0, VLEN * 2)
2899 Node* two_vect_lane_cnt_m1 = phase.makecon(TypeInt::make(2 * num_elem - 1));
2900 Node* bcast_two_vect_lane_cnt_m1_vec = phase.transform(VectorNode::scalar2vector(two_vect_lane_cnt_m1, num_elem,
2901 T_BYTE, false));
2902 index_byte_vec = phase.transform(VectorNode::make(Op_AndV, index_byte_vec, bcast_two_vect_lane_cnt_m1_vec,
2903 index_byte_vec->bottom_type()->is_vect()));
2904
2905 // Compute the blend mask for merging two independently permitted vectors
2906 // using shuffle index in two vector index range [0, VLEN * 2).
2907 BoolTest::mask pred = BoolTest::le;
2908 ConINode* pred_node = phase.makecon(TypeInt::make(pred))->as_ConI();
2909 const TypeVect* vmask_type = TypeVect::makemask(T_BYTE, num_elem);
2910 Node* lane_cnt_m1 = phase.makecon(TypeInt::make(num_elem - 1));
2911 Node* bcast_lane_cnt_m1_vec = phase.transform(VectorNode::scalar2vector(lane_cnt_m1, num_elem,
2912 T_BYTE, false));
2913 Node* mask = phase.transform(new VectorMaskCmpNode(pred, index_byte_vec, bcast_lane_cnt_m1_vec, pred_node, vmask_type));
2914
2915 // Rearrange expects the indexes to lie within single vector index range [0, VLEN).
2916 index_byte_vec = phase.transform(VectorNode::make(Op_AndV, index_byte_vec, bcast_lane_cnt_m1_vec,
2917 index_byte_vec->bottom_type()->is_vect()));
2918
2919 // Load indexes from byte vector and appropriately transform them to target
2920 // specific permutation index format.
2921 index_vec = phase.transform(new VectorLoadShuffleNode(index_byte_vec, index_vect_type));
2922
2923 vmask_type = TypeVect::makemask(elem_bt, num_elem);
2924 mask = phase.transform(new VectorMaskCastNode(mask, vmask_type));
2925
2926 Node* p1 = phase.transform(new VectorRearrangeNode(src1, index_vec));
2927 Node* p2 = phase.transform(new VectorRearrangeNode(src2, index_vec));
2928
2929 return new VectorBlendNode(p2, p1, mask);
2930 }
2931
2932 // public static
2933 // <V extends Vector<E>,
2934 // E>
2935 // V selectFromTwoVectorOp(Class<? extends V> vClass, Class<E> eClass, int length,
2936 // V v1, V v2, V v3,
2937 // SelectFromTwoVector<V> defaultImpl)
2938 bool LibraryCallKit::inline_vector_select_from_two_vectors() {
2939 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
2940 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
2941 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
2942
2943 if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr || vector_klass->const_oop() == nullptr ||
2944 elem_klass->const_oop() == nullptr ||!vlen->is_con()) {
2945 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s",
2946 NodeClassNames[argument(0)->Opcode()],
2947 NodeClassNames[argument(1)->Opcode()],
2948 NodeClassNames[argument(2)->Opcode()]);
2949 return false; // not enough info for intrinsification
2950 }
2951
2952 if (!is_klass_initialized(vector_klass)) {
2953 log_if_needed(" ** klass argument not initialized");
2954 return false;
2955 }
2956
2957 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2958 if (!elem_type->is_primitive_type()) {
2959 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2960 return false; // should be primitive type
2961 }
2962
2963 int num_elem = vlen->get_con();
2964 if (!is_power_of_2(num_elem)) {
2965 log_if_needed(" ** vlen is not power of two=%d", num_elem);
2966 return false;
2967 }
2968
2969 BasicType elem_bt = elem_type->basic_type();
2970 BasicType index_elem_bt = elem_bt;
2971 if (elem_bt == T_FLOAT) {
2972 index_elem_bt = T_INT;
2973 } else if (elem_bt == T_DOUBLE) {
2974 index_elem_bt = T_LONG;
2975 }
2976
2977 bool lowerSelectFromOp = false;
2978 if (!arch_supports_vector(Op_SelectFromTwoVector, num_elem, elem_bt, VecMaskNotUsed)) {
2979 int cast_vopc = VectorCastNode::opcode(-1, elem_bt, true);
2980 if (!arch_supports_vector(Op_VectorMaskCmp, num_elem, T_BYTE, VecMaskNotUsed) ||
2981 !arch_supports_vector(Op_AndV, num_elem, T_BYTE, VecMaskNotUsed) ||
2982 !arch_supports_vector(Op_VectorMaskCast, num_elem, elem_bt, VecMaskNotUsed) ||
2983 !arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad) ||
2984 !arch_supports_vector(Op_VectorRearrange, num_elem, elem_bt, VecMaskNotUsed) ||
2985 !arch_supports_vector(cast_vopc, num_elem, T_BYTE, VecMaskNotUsed) ||
2986 !arch_supports_vector(Op_VectorLoadShuffle, num_elem, index_elem_bt, VecMaskNotUsed) ||
2987 !arch_supports_vector(Op_Replicate, num_elem, T_BYTE, VecMaskNotUsed)) {
2988 log_if_needed(" ** not supported: opc=%d vlen=%d etype=%s ismask=useload",
2989 Op_SelectFromTwoVector, num_elem, type2name(elem_bt));
2990 return false; // not supported
2991 }
2992 lowerSelectFromOp = true;
2993 }
2994
2995 int cast_vopc = VectorCastNode::opcode(-1, elem_bt, true);
2996 if (!lowerSelectFromOp) {
2997 if (!arch_supports_vector(Op_AndV, num_elem, index_elem_bt, VecMaskNotUsed) ||
2998 !arch_supports_vector(Op_Replicate, num_elem, index_elem_bt, VecMaskNotUsed) ||
2999 (is_floating_point_type(elem_bt) &&
3000 !arch_supports_vector(cast_vopc, num_elem, index_elem_bt, VecMaskNotUsed))) {
3001 log_if_needed(" ** index wrapping not supported: vlen=%d etype=%s" ,
3002 num_elem, type2name(elem_bt));
3003 return false; // not supported
3004 }
3005 }
3006
3007 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
3008 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
3009
3010 Node* opd1 = unbox_vector(argument(3), vbox_type, elem_bt, num_elem);
3011 if (opd1 == nullptr) {
3012 log_if_needed(" ** unbox failed v1=%s",
3013 NodeClassNames[argument(3)->Opcode()]);
3014 return false;
3015 }
3016 Node* opd2 = unbox_vector(argument(4), vbox_type, elem_bt, num_elem);
3017 if (opd2 == nullptr) {
3018 log_if_needed(" ** unbox failed v2=%s",
3019 NodeClassNames[argument(4)->Opcode()]);
3020 return false;
3021 }
3022 Node* opd3 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
3023 if (opd3 == nullptr) {
3024 log_if_needed(" ** unbox failed v3=%s",
3025 NodeClassNames[argument(5)->Opcode()]);
3026 return false;
3027 }
3028
3029 const TypeVect* vt = TypeVect::make(elem_bt, num_elem);
3030
3031 Node* operation = nullptr;
3032 if (lowerSelectFromOp) {
3033 operation = gvn().transform(LowerSelectFromTwoVectorOperation(gvn(), opd1, opd2, opd3, vt));
3034 } else {
3035 if (index_elem_bt != elem_bt) {
3036 opd1 = gvn().transform(VectorCastNode::make(cast_vopc, opd1, index_elem_bt, num_elem));
3037 }
3038 int indexRangeMask = 2 * num_elem - 1;
3039 Node* wrap_mask = gvn().makecon(TypeInteger::make(indexRangeMask, indexRangeMask, Type::WidenMin, index_elem_bt != T_LONG ? T_INT : index_elem_bt));
3040 Node* wrap_mask_vec = gvn().transform(VectorNode::scalar2vector(wrap_mask, num_elem, index_elem_bt, false));
3041 opd1 = gvn().transform(VectorNode::make(Op_AndV, opd1, wrap_mask_vec, opd1->bottom_type()->is_vect()));
3042 operation = gvn().transform(VectorNode::make(Op_SelectFromTwoVector, opd1, opd2, opd3, vt));
3043 }
3044
3045 // Wrap it up in VectorBox to keep object type information.
3046 Node* vbox = box_vector(operation, vbox_type, elem_bt, num_elem);
3047 set_result(vbox);
3048 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
3049 return true;
3050 }
3051
3052 // public static
3053 // <V extends Vector<E>,
3054 // M extends VectorMask<E>,
3055 // E>
3056 // V compressExpandOp(int opr,
3057 // Class<? extends V> vClass, Class<? extends M> mClass, Class<E> eClass,
3058 // int length, V v, M m,
3059 // CompressExpandOperation<V, M> defaultImpl)
3060 bool LibraryCallKit::inline_vector_compress_expand() {
3061 const TypeInt* opr = gvn().type(argument(0))->isa_int();
3062 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
3063 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
3064 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
3065 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
3066
3067 if (opr == nullptr || !opr->is_con() ||
3068 vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
3069 mask_klass == nullptr || mask_klass->const_oop() == nullptr ||
3070 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
3071 vlen == nullptr || !vlen->is_con()) {
3072 log_if_needed(" ** missing constant: opr=%s vclass=%s mclass=%s etype=%s vlen=%s",
3073 NodeClassNames[argument(0)->Opcode()],
3074 NodeClassNames[argument(1)->Opcode()],
3075 NodeClassNames[argument(2)->Opcode()],
3076 NodeClassNames[argument(3)->Opcode()],
3077 NodeClassNames[argument(4)->Opcode()]);
3078 return false; // not enough info for intrinsification
3079 }
3080
3081 if (!is_klass_initialized(vector_klass) || !is_klass_initialized(mask_klass)) {
3082 log_if_needed(" ** klass argument not initialized");
3083 return false;
3084 }
3085
3086 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
3087 if (!elem_type->is_primitive_type()) {
3088 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
3089 return false; // should be primitive type
3090 }
3091
3092 int num_elem = vlen->get_con();
3093 BasicType elem_bt = elem_type->basic_type();
3094 int opc = VectorSupport::vop2ideal(opr->get_con(), elem_bt);
3095
3096 if (!arch_supports_vector(opc, num_elem, elem_bt, VecMaskUseLoad)) {
3097 log_if_needed(" ** not supported: opc=%d vlen=%d etype=%s ismask=useload",
3098 opc, num_elem, type2name(elem_bt));
3099 return false; // not supported
3100 }
3101
3102 Node* opd1 = nullptr;
3103 const TypeInstPtr* vbox_type = nullptr;
3104 if (opc != Op_CompressM) {
3105 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
3106 vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
3107 opd1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
3108 if (opd1 == nullptr) {
3109 log_if_needed(" ** unbox failed vector=%s",
3110 NodeClassNames[argument(5)->Opcode()]);
3111 return false;
3112 }
3113 }
3114
3115 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
3116 assert(is_vector_mask(mbox_klass), "argument(6) should be a mask class");
3117 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
3118
3119 Node* mask = unbox_vector(argument(6), mbox_type, elem_bt, num_elem);
3120 if (mask == nullptr) {
3121 log_if_needed(" ** unbox failed mask=%s",
3122 NodeClassNames[argument(6)->Opcode()]);
3123 return false;
3124 }
3125
3126 const TypeVect* vt = TypeVect::make(elem_bt, num_elem, opc == Op_CompressM);
3127 Node* operation = gvn().transform(VectorNode::make(opc, opd1, mask, vt));
3128
3129 // Wrap it up in VectorBox to keep object type information.
3130 const TypeInstPtr* box_type = opc == Op_CompressM ? mbox_type : vbox_type;
3131 Node* vbox = box_vector(operation, box_type, elem_bt, num_elem);
3132 set_result(vbox);
3133 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
3134 return true;
3135 }
3136
3137 // public static
3138 // <V extends Vector<E>,
3139 // E,
3140 // S extends VectorSpecies<E>>
3141 // V indexVector(Class<? extends V> vClass, Class<E> eClass,
3142 // int length,
3143 // V v, int step, S s,
3144 // IndexOperation<V, S> defaultImpl)
3145 bool LibraryCallKit::inline_index_vector() {
3146 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
3147 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
3148 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
3149
3150 if (vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
3151 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
3152 vlen == nullptr || !vlen->is_con() ) {
3153 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s",
3154 NodeClassNames[argument(0)->Opcode()],
3155 NodeClassNames[argument(1)->Opcode()],
3156 NodeClassNames[argument(2)->Opcode()]);
3157 return false; // not enough info for intrinsification
3158 }
3159
3160 if (!is_klass_initialized(vector_klass)) {
3161 log_if_needed(" ** klass argument not initialized");
3162 return false;
3163 }
3164
3165 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
3166 if (!elem_type->is_primitive_type()) {
3167 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
3168 return false; // should be primitive type
3169 }
3170
3171 int num_elem = vlen->get_con();
3172 BasicType elem_bt = elem_type->basic_type();
3173
3174 // Check whether the iota index generation op is supported by the current hardware
3175 if (!arch_supports_vector(Op_VectorLoadConst, num_elem, elem_bt, VecMaskNotUsed)) {
3176 log_if_needed(" ** not supported: vlen=%d etype=%s", num_elem, type2name(elem_bt));
3177 return false; // not supported
3178 }
3179
3180 int mul_op = VectorSupport::vop2ideal(VectorSupport::VECTOR_OP_MUL, elem_bt);
3181 int vmul_op = VectorNode::opcode(mul_op, elem_bt);
3182 bool needs_mul = true;
3183 Node* scale = argument(4);
3184 const TypeInt* scale_type = gvn().type(scale)->isa_int();
3185 // Multiply is not needed if the scale is a constant "1".
3186 if (scale_type && scale_type->is_con() && scale_type->get_con() == 1) {
3187 needs_mul = false;
3188 } else {
3189 // Check whether the vector multiply op is supported by the current hardware
3190 if (!arch_supports_vector(vmul_op, num_elem, elem_bt, VecMaskNotUsed)) {
3191 log_if_needed(" ** not supported: vlen=%d etype=%s", num_elem, type2name(elem_bt));
3192 return false; // not supported
3193 }
3194
3195 // Check whether the scalar cast op is supported by the current hardware
3196 if (is_floating_point_type(elem_bt) || elem_bt == T_LONG) {
3197 int cast_op = elem_bt == T_LONG ? Op_ConvI2L :
3198 elem_bt == T_FLOAT? Op_ConvI2F : Op_ConvI2D;
3199 if (!Matcher::match_rule_supported(cast_op)) {
3200 log_if_needed(" ** Rejected op (%s) because architecture does not support it",
3201 NodeClassNames[cast_op]);
3202 return false; // not supported
3203 }
3204 }
3205 }
3206
3207 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
3208 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
3209 Node* opd = unbox_vector(argument(3), vbox_type, elem_bt, num_elem);
3210 if (opd == nullptr) {
3211 log_if_needed(" ** unbox failed vector=%s",
3212 NodeClassNames[argument(3)->Opcode()]);
3213 return false;
3214 }
3215
3216 int add_op = VectorSupport::vop2ideal(VectorSupport::VECTOR_OP_ADD, elem_bt);
3217 int vadd_op = VectorNode::opcode(add_op, elem_bt);
3218 bool needs_add = true;
3219 // The addition is not needed if all the element values of "opd" are zero
3220 if (VectorNode::is_all_zeros_vector(opd)) {
3221 needs_add = false;
3222 } else {
3223 // Check whether the vector addition op is supported by the current hardware
3224 if (!arch_supports_vector(vadd_op, num_elem, elem_bt, VecMaskNotUsed)) {
3225 log_if_needed(" ** not supported: vlen=%d etype=%s", num_elem, type2name(elem_bt));
3226 return false; // not supported
3227 }
3228 }
3229
3230 // Compute the iota indice vector
3231 const TypeVect* vt = TypeVect::make(elem_bt, num_elem);
3232 Node* index = gvn().transform(new VectorLoadConstNode(gvn().makecon(TypeInt::ZERO), vt));
3233
3234 // Broadcast the "scale" to a vector, and multiply the "scale" with iota indice vector.
3235 if (needs_mul) {
3236 switch (elem_bt) {
3237 case T_BOOLEAN: // fall-through
3238 case T_BYTE: // fall-through
3239 case T_SHORT: // fall-through
3240 case T_CHAR: // fall-through
3241 case T_INT: {
3242 // no conversion needed
3243 break;
3244 }
3245 case T_LONG: {
3246 scale = gvn().transform(new ConvI2LNode(scale));
3247 break;
3248 }
3249 case T_FLOAT: {
3250 scale = gvn().transform(new ConvI2FNode(scale));
3251 break;
3252 }
3253 case T_DOUBLE: {
3254 scale = gvn().transform(new ConvI2DNode(scale));
3255 break;
3256 }
3257 default: fatal("%s", type2name(elem_bt));
3258 }
3259 scale = gvn().transform(VectorNode::scalar2vector(scale, num_elem, elem_bt));
3260 index = gvn().transform(VectorNode::make(vmul_op, index, scale, vt));
3261 }
3262
3263 // Add "opd" if addition is needed.
3264 if (needs_add) {
3265 index = gvn().transform(VectorNode::make(vadd_op, opd, index, vt));
3266 }
3267 Node* vbox = box_vector(index, vbox_type, elem_bt, num_elem);
3268 set_result(vbox);
3269 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
3270 return true;
3271 }
3272
3273 // public static
3274 // <E,
3275 // M extends VectorMask<E>>
3276 // M indexPartiallyInUpperRange(Class<? extends M> mClass, Class<E> eClass, int length,
3277 // long offset, long limit,
3278 // IndexPartiallyInUpperRangeOperation<E, M> defaultImpl)
3279 bool LibraryCallKit::inline_index_partially_in_upper_range() {
3280 const TypeInstPtr* mask_klass = gvn().type(argument(0))->isa_instptr();
3281 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
3282 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
3283
3284 if (mask_klass == nullptr || mask_klass->const_oop() == nullptr ||
3285 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
3286 vlen == nullptr || !vlen->is_con()) {
3287 log_if_needed(" ** missing constant: mclass=%s etype=%s vlen=%s",
3288 NodeClassNames[argument(0)->Opcode()],
3289 NodeClassNames[argument(1)->Opcode()],
3290 NodeClassNames[argument(2)->Opcode()]);
3291 return false; // not enough info for intrinsification
3292 }
3293
3294 if (!is_klass_initialized(mask_klass)) {
3295 log_if_needed(" ** klass argument not initialized");
3296 return false;
3297 }
3298
3299 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
3300 if (!elem_type->is_primitive_type()) {
3301 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
3302 return false; // should be primitive type
3303 }
3304
3305 int num_elem = vlen->get_con();
3306 BasicType elem_bt = elem_type->basic_type();
3307
3308 // Check whether the necessary ops are supported by current hardware.
3309 bool supports_mask_gen = arch_supports_vector(Op_VectorMaskGen, num_elem, elem_bt, VecMaskUseStore);
3310 if (!supports_mask_gen) {
3311 if (!arch_supports_vector(Op_VectorLoadConst, num_elem, elem_bt, VecMaskNotUsed) ||
3312 !arch_supports_vector(Op_Replicate, num_elem, elem_bt, VecMaskNotUsed) ||
3313 !arch_supports_vector(Op_VectorMaskCmp, num_elem, elem_bt, VecMaskUseStore)) {
3314 log_if_needed(" ** not supported: vlen=%d etype=%s", num_elem, type2name(elem_bt));
3315 return false; // not supported
3316 }
3317
3318 // Check whether the scalar cast operation is supported by current hardware.
3319 if (elem_bt != T_LONG) {
3320 int cast_op = is_integral_type(elem_bt) ? Op_ConvL2I
3321 : (elem_bt == T_FLOAT ? Op_ConvL2F : Op_ConvL2D);
3322 if (!Matcher::match_rule_supported(cast_op)) {
3323 log_if_needed(" ** Rejected op (%s) because architecture does not support it",
3324 NodeClassNames[cast_op]);
3325 return false; // not supported
3326 }
3327 }
3328 }
3329
3330 Node* offset = argument(3);
3331 Node* limit = argument(5);
3332 if (offset == nullptr || limit == nullptr) {
3333 log_if_needed(" ** offset or limit argument is null");
3334 return false; // not supported
3335 }
3336
3337 ciKlass* box_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
3338 assert(is_vector_mask(box_klass), "argument(0) should be a mask class");
3339 const TypeInstPtr* box_type = TypeInstPtr::make_exact(TypePtr::NotNull, box_klass);
3340
3341 // We assume "offset > 0 && limit >= offset && limit - offset < num_elem".
3342 // So directly get indexLimit with "indexLimit = limit - offset".
3343 Node* indexLimit = gvn().transform(new SubLNode(limit, offset));
3344 Node* mask = nullptr;
3345 if (supports_mask_gen) {
3346 mask = gvn().transform(VectorMaskGenNode::make(indexLimit, elem_bt, num_elem));
3347 } else {
3348 // Generate the vector mask based on "mask = iota < indexLimit".
3349 // Broadcast "indexLimit" to a vector.
3350 switch (elem_bt) {
3351 case T_BOOLEAN: // fall-through
3352 case T_BYTE: // fall-through
3353 case T_SHORT: // fall-through
3354 case T_CHAR: // fall-through
3355 case T_INT: {
3356 indexLimit = gvn().transform(new ConvL2INode(indexLimit));
3357 break;
3358 }
3359 case T_DOUBLE: {
3360 indexLimit = gvn().transform(new ConvL2DNode(indexLimit));
3361 break;
3362 }
3363 case T_FLOAT: {
3364 indexLimit = gvn().transform(new ConvL2FNode(indexLimit));
3365 break;
3366 }
3367 case T_LONG: {
3368 // no conversion needed
3369 break;
3370 }
3371 default: fatal("%s", type2name(elem_bt));
3372 }
3373 indexLimit = gvn().transform(VectorNode::scalar2vector(indexLimit, num_elem, elem_bt));
3374
3375 // Load the "iota" vector.
3376 const TypeVect* vt = TypeVect::make(elem_bt, num_elem);
3377 Node* iota = gvn().transform(new VectorLoadConstNode(gvn().makecon(TypeInt::ZERO), vt));
3378
3379 // Compute the vector mask with "mask = iota < indexLimit".
3380 ConINode* pred_node = (ConINode*)gvn().makecon(TypeInt::make(BoolTest::lt));
3381 const TypeVect* vmask_type = TypeVect::makemask(elem_bt, num_elem);
3382 mask = gvn().transform(new VectorMaskCmpNode(BoolTest::lt, iota, indexLimit, pred_node, vmask_type));
3383 }
3384 Node* vbox = box_vector(mask, box_type, elem_bt, num_elem);
3385 set_result(vbox);
3386 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
3387 return true;
3388 }
3389
3390 #undef non_product_log_if_needed
3391 #undef log_if_needed