1 /*
2 * Copyright (c) 2020, 2024, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "ci/ciSymbols.hpp"
27 #include "classfile/vmSymbols.hpp"
28 #include "opto/library_call.hpp"
29 #include "opto/runtime.hpp"
30 #include "opto/vectornode.hpp"
31 #include "prims/vectorSupport.hpp"
32 #include "runtime/stubRoutines.hpp"
33
34 #ifdef ASSERT
35 static bool is_vector(ciKlass* klass) {
36 return klass->is_subclass_of(ciEnv::current()->vector_VectorPayload_klass());
37 }
38
39 static bool check_vbox(const TypeInstPtr* vbox_type) {
40 assert(vbox_type->klass_is_exact(), "");
41
42 ciInstanceKlass* ik = vbox_type->instance_klass();
43 assert(is_vector(ik), "not a vector");
44
45 ciField* fd1 = ik->get_field_by_name(ciSymbols::ETYPE_name(), ciSymbols::class_signature(), /* is_static */ true);
46 assert(fd1 != nullptr, "element type info is missing");
47
48 ciConstant val1 = fd1->constant_value();
49 BasicType elem_bt = val1.as_object()->as_instance()->java_mirror_type()->basic_type();
50 assert(is_java_primitive(elem_bt), "element type info is missing");
51
52 ciField* fd2 = ik->get_field_by_name(ciSymbols::VLENGTH_name(), ciSymbols::int_signature(), /* is_static */ true);
53 assert(fd2 != nullptr, "vector length info is missing");
54
55 ciConstant val2 = fd2->constant_value();
56 assert(val2.as_int() > 0, "vector length info is missing");
57
58 return true;
59 }
60 #endif
61
62 #define log_if_needed(...) \
63 if (C->print_intrinsics()) { \
64 tty->print_cr(__VA_ARGS__); \
65 }
66
67 #ifndef PRODUCT
68 #define non_product_log_if_needed(...) log_if_needed(__VA_ARGS__)
69 #else
70 #define non_product_log_if_needed(...)
71 #endif
72
73 static bool is_vector_mask(ciKlass* klass) {
74 return klass->is_subclass_of(ciEnv::current()->vector_VectorMask_klass());
75 }
76
77 static bool is_vector_shuffle(ciKlass* klass) {
78 return klass->is_subclass_of(ciEnv::current()->vector_VectorShuffle_klass());
79 }
80
81 bool LibraryCallKit::arch_supports_vector_rotate(int opc, int num_elem, BasicType elem_bt,
82 VectorMaskUseType mask_use_type, bool has_scalar_args) {
83 bool is_supported = true;
84
85 // has_scalar_args flag is true only for non-constant scalar shift count,
86 // since in this case shift needs to be broadcasted.
87 if (!Matcher::match_rule_supported_vector(opc, num_elem, elem_bt) ||
88 (has_scalar_args && !arch_supports_vector(Op_Replicate, num_elem, elem_bt, VecMaskNotUsed))) {
89 is_supported = false;
90 }
91
92 if (is_supported) {
93 // Check if mask unboxing is supported, this is a two step process which first loads the contents
94 // of boolean array into vector followed by either lane expansion to match the lane size of masked
95 // vector operation or populate the predicate register.
96 if ((mask_use_type & VecMaskUseLoad) != 0) {
97 if (!Matcher::match_rule_supported_vector(Op_VectorLoadMask, num_elem, elem_bt) ||
98 !Matcher::match_rule_supported_vector(Op_LoadVector, num_elem, T_BOOLEAN)) {
99 non_product_log_if_needed(" ** Rejected vector mask loading (%s,%s,%d) because architecture does not support it",
100 NodeClassNames[Op_VectorLoadMask], type2name(elem_bt), num_elem);
101 return false;
102 }
103 }
104
105 if ((mask_use_type & VecMaskUsePred) != 0) {
106 if (!Matcher::has_predicated_vectors() ||
107 !Matcher::match_rule_supported_vector_masked(opc, num_elem, elem_bt)) {
108 non_product_log_if_needed("Rejected vector mask predicate using (%s,%s,%d) because architecture does not support it",
109 NodeClassNames[opc], type2name(elem_bt), num_elem);
110 return false;
111 }
112 }
113 }
114
115 int lshiftopc, rshiftopc;
116 switch(elem_bt) {
117 case T_BYTE:
118 lshiftopc = Op_LShiftI;
119 rshiftopc = Op_URShiftB;
120 break;
121 case T_SHORT:
122 lshiftopc = Op_LShiftI;
123 rshiftopc = Op_URShiftS;
124 break;
125 case T_INT:
126 lshiftopc = Op_LShiftI;
127 rshiftopc = Op_URShiftI;
128 break;
129 case T_LONG:
130 lshiftopc = Op_LShiftL;
131 rshiftopc = Op_URShiftL;
132 break;
133 default: fatal("Unexpected type: %s", type2name(elem_bt));
134 }
135 int lshiftvopc = VectorNode::opcode(lshiftopc, elem_bt);
136 int rshiftvopc = VectorNode::opcode(rshiftopc, elem_bt);
137 if (!is_supported &&
138 arch_supports_vector(lshiftvopc, num_elem, elem_bt, VecMaskNotUsed, has_scalar_args) &&
139 arch_supports_vector(rshiftvopc, num_elem, elem_bt, VecMaskNotUsed, has_scalar_args) &&
140 arch_supports_vector(Op_OrV, num_elem, elem_bt, VecMaskNotUsed)) {
141 is_supported = true;
142 }
143 return is_supported;
144 }
145
146 Node* GraphKit::box_vector(Node* vector, const TypeInstPtr* vbox_type, BasicType elem_bt, int num_elem, bool deoptimize_on_exception) {
147 assert(EnableVectorSupport, "");
148
149 PreserveReexecuteState preexecs(this);
150 jvms()->set_should_reexecute(true);
151
152 VectorBoxAllocateNode* alloc = new VectorBoxAllocateNode(C, vbox_type);
153 set_edges_for_java_call(alloc, /*must_throw=*/false, /*separate_io_proj=*/true);
154 make_slow_call_ex(alloc, env()->Throwable_klass(), /*separate_io_proj=*/true, deoptimize_on_exception);
155 set_i_o(gvn().transform( new ProjNode(alloc, TypeFunc::I_O) ));
156 set_all_memory(gvn().transform( new ProjNode(alloc, TypeFunc::Memory) ));
157 Node* ret = gvn().transform(new ProjNode(alloc, TypeFunc::Parms));
158
159 assert(check_vbox(vbox_type), "");
160 const TypeVect* vt = TypeVect::make(elem_bt, num_elem, is_vector_mask(vbox_type->instance_klass()));
161 VectorBoxNode* vbox = new VectorBoxNode(C, ret, vector, vbox_type, vt);
162 return gvn().transform(vbox);
163 }
164
165 Node* GraphKit::unbox_vector(Node* v, const TypeInstPtr* vbox_type, BasicType elem_bt, int num_elem, bool shuffle_to_vector) {
166 assert(EnableVectorSupport, "");
167 const TypeInstPtr* vbox_type_v = gvn().type(v)->isa_instptr();
168 if (vbox_type_v == nullptr || vbox_type->instance_klass() != vbox_type_v->instance_klass()) {
169 return nullptr; // arguments don't agree on vector shapes
170 }
171 if (vbox_type_v->maybe_null()) {
172 return nullptr; // no nulls are allowed
173 }
174 assert(check_vbox(vbox_type), "");
175 const TypeVect* vt = TypeVect::make(elem_bt, num_elem, is_vector_mask(vbox_type->instance_klass()));
176 Node* unbox = gvn().transform(new VectorUnboxNode(C, vt, v, merged_memory(), shuffle_to_vector));
177 return unbox;
178 }
179
180 Node* GraphKit::vector_shift_count(Node* cnt, int shift_op, BasicType bt, int num_elem) {
181 assert(bt == T_INT || bt == T_LONG || bt == T_SHORT || bt == T_BYTE, "byte, short, long and int are supported");
182 juint mask = (type2aelembytes(bt) * BitsPerByte - 1);
183 Node* nmask = gvn().transform(ConNode::make(TypeInt::make(mask)));
184 Node* mcnt = gvn().transform(new AndINode(cnt, nmask));
185 return gvn().transform(VectorNode::shift_count(shift_op, mcnt, num_elem, bt));
186 }
187
188 bool LibraryCallKit::arch_supports_vector(int sopc, int num_elem, BasicType type, VectorMaskUseType mask_use_type, bool has_scalar_args) {
189 // Check that the operation is valid.
190 if (sopc <= 0) {
191 non_product_log_if_needed(" ** Rejected intrinsification because no valid vector op could be extracted");
192 return false;
193 }
194
195 if (VectorNode::is_vector_rotate(sopc)) {
196 if(!arch_supports_vector_rotate(sopc, num_elem, type, mask_use_type, has_scalar_args)) {
197 non_product_log_if_needed(" ** Rejected vector op (%s,%s,%d) because architecture does not support variable vector shifts",
198 NodeClassNames[sopc], type2name(type), num_elem);
199 return false;
200 }
201 } else if (VectorNode::is_vector_integral_negate(sopc)) {
202 if (!VectorNode::is_vector_integral_negate_supported(sopc, num_elem, type, false)) {
203 non_product_log_if_needed(" ** Rejected vector op (%s,%s,%d) because architecture does not support integral vector negate",
204 NodeClassNames[sopc], type2name(type), num_elem);
205 return false;
206 }
207 } else {
208 // Check that architecture supports this op-size-type combination.
209 if (!Matcher::match_rule_supported_vector(sopc, num_elem, type)) {
210 non_product_log_if_needed(" ** Rejected vector op (%s,%s,%d) because architecture does not support it",
211 NodeClassNames[sopc], type2name(type), num_elem);
212 return false;
213 } else {
214 assert(Matcher::match_rule_supported(sopc), "must be supported");
215 }
216 }
217
218 if (num_elem == 1) {
219 if (mask_use_type != VecMaskNotUsed) {
220 non_product_log_if_needed(" ** Rejected vector mask op (%s,%s,%d) because architecture does not support it",
221 NodeClassNames[sopc], type2name(type), num_elem);
222 return false;
223 }
224
225 if (sopc != 0) {
226 if (sopc != Op_LoadVector && sopc != Op_StoreVector) {
227 non_product_log_if_needed(" ** Not a svml call or load/store vector op (%s,%s,%d)",
228 NodeClassNames[sopc], type2name(type), num_elem);
229 return false;
230 }
231 }
232 }
233
234 if (!has_scalar_args && VectorNode::is_vector_shift(sopc) &&
235 Matcher::supports_vector_variable_shifts() == false) {
236 log_if_needed(" ** Rejected vector op (%s,%s,%d) because architecture does not support variable vector shifts",
237 NodeClassNames[sopc], type2name(type), num_elem);
238 return false;
239 }
240
241 // Check if mask unboxing is supported, this is a two step process which first loads the contents
242 // of boolean array into vector followed by either lane expansion to match the lane size of masked
243 // vector operation or populate the predicate register.
244 if ((mask_use_type & VecMaskUseLoad) != 0) {
245 if (!Matcher::match_rule_supported_vector(Op_VectorLoadMask, num_elem, type) ||
246 !Matcher::match_rule_supported_vector(Op_LoadVector, num_elem, T_BOOLEAN)) {
247 non_product_log_if_needed(" ** Rejected vector mask loading (%s,%s,%d) because architecture does not support it",
248 NodeClassNames[Op_VectorLoadMask], type2name(type), num_elem);
249 return false;
250 }
251 }
252
253 // Check if mask boxing is supported, this is a two step process which first stores the contents
254 // of mask vector / predicate register into a boolean vector followed by vector store operation to
255 // transfer the contents to underlined storage of mask boxes which is a boolean array.
256 if ((mask_use_type & VecMaskUseStore) != 0) {
257 if (!Matcher::match_rule_supported_vector(Op_VectorStoreMask, num_elem, type) ||
258 !Matcher::match_rule_supported_vector(Op_StoreVector, num_elem, T_BOOLEAN)) {
259 non_product_log_if_needed("Rejected vector mask storing (%s,%s,%d) because architecture does not support it",
260 NodeClassNames[Op_VectorStoreMask], type2name(type), num_elem);
261 return false;
262 }
263 }
264
265 if ((mask_use_type & VecMaskUsePred) != 0) {
266 bool is_supported = false;
267 if (Matcher::has_predicated_vectors()) {
268 if (VectorNode::is_vector_integral_negate(sopc)) {
269 is_supported = VectorNode::is_vector_integral_negate_supported(sopc, num_elem, type, true);
270 } else {
271 is_supported = Matcher::match_rule_supported_vector_masked(sopc, num_elem, type);
272 }
273 }
274 is_supported |= Matcher::supports_vector_predicate_op_emulation(sopc, num_elem, type);
275
276 if (!is_supported) {
277 non_product_log_if_needed("Rejected vector mask predicate using (%s,%s,%d) because architecture does not support it",
278 NodeClassNames[sopc], type2name(type), num_elem);
279 return false;
280 }
281 }
282
283 return true;
284 }
285
286 static bool is_klass_initialized(const TypeInstPtr* vec_klass) {
287 if (vec_klass->const_oop() == nullptr) {
288 return false; // uninitialized or some kind of unsafe access
289 }
290 assert(vec_klass->const_oop()->as_instance()->java_lang_Class_klass() != nullptr, "klass instance expected");
291 ciInstanceKlass* klass = vec_klass->const_oop()->as_instance()->java_lang_Class_klass()->as_instance_klass();
292 return klass->is_initialized();
293 }
294
295 // public static
296 // <V extends Vector<E>,
297 // M extends VectorMask<E>,
298 // E>
299 // V unaryOp(int oprId, Class<? extends V> vmClass, Class<? extends M> maskClass, Class<E> elementType,
300 // int length, V v, M m,
301 // UnaryOperation<V, M> defaultImpl)
302 //
303 // public static
304 // <V,
305 // M extends VectorMask<E>,
306 // E>
307 // V binaryOp(int oprId, Class<? extends V> vmClass, Class<? extends M> maskClass, Class<E> elementType,
308 // int length, V v1, V v2, M m,
309 // BinaryOperation<V, M> defaultImpl)
310 //
311 // public static
312 // <V extends Vector<E>,
313 // M extends VectorMask<E>,
314 // E>
315 // V ternaryOp(int oprId, Class<? extends V> vmClass, Class<? extends M> maskClass, Class<E> elementType,
316 // int length, V v1, V v2, V v3, M m,
317 // TernaryOperation<V, M> defaultImpl)
318 //
319 bool LibraryCallKit::inline_vector_nary_operation(int n) {
320 const TypeInt* opr = gvn().type(argument(0))->isa_int();
321 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
322 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
323 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
324 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
325
326 if (opr == nullptr || vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr ||
327 !opr->is_con() || vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
328 log_if_needed(" ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
329 NodeClassNames[argument(0)->Opcode()],
330 NodeClassNames[argument(1)->Opcode()],
331 NodeClassNames[argument(3)->Opcode()],
332 NodeClassNames[argument(4)->Opcode()]);
333 return false; // not enough info for intrinsification
334 }
335
336 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
337 if (!elem_type->is_primitive_type()) {
338 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
339 return false; // should be primitive type
340 }
341 if (!is_klass_initialized(vector_klass)) {
342 log_if_needed(" ** klass argument not initialized");
343 return false;
344 }
345
346 // "argument(n + 5)" should be the mask object. We assume it is "null" when no mask
347 // is used to control this operation.
348 const Type* vmask_type = gvn().type(argument(n + 5));
349 bool is_masked_op = vmask_type != TypePtr::NULL_PTR;
350 if (is_masked_op) {
351 if (mask_klass == nullptr || mask_klass->const_oop() == nullptr) {
352 log_if_needed(" ** missing constant: maskclass=%s", NodeClassNames[argument(2)->Opcode()]);
353 return false; // not enough info for intrinsification
354 }
355
356 if (!is_klass_initialized(mask_klass)) {
357 log_if_needed(" ** mask klass argument not initialized");
358 return false;
359 }
360
361 if (vmask_type->maybe_null()) {
362 log_if_needed(" ** null mask values are not allowed for masked op");
363 return false;
364 }
365 }
366
367 BasicType elem_bt = elem_type->basic_type();
368 bool has_scalar_op = VectorSupport::has_scalar_op(opr->get_con());
369 bool is_unsigned = VectorSupport::is_unsigned_op(opr->get_con());
370
371 int num_elem = vlen->get_con();
372 int opc = VectorSupport::vop2ideal(opr->get_con(), elem_bt);
373 int sopc = has_scalar_op ? VectorNode::opcode(opc, elem_bt) : opc;
374 if ((opc != Op_CallLeafVector) && (sopc == 0)) {
375 log_if_needed(" ** operation not supported: opc=%s bt=%s", NodeClassNames[opc], type2name(elem_bt));
376 return false; // operation not supported
377 }
378 if (num_elem == 1) {
379 if (opc != Op_CallLeafVector || elem_bt != T_DOUBLE) {
380 log_if_needed(" ** not a svml call: arity=%d opc=%d vlen=%d etype=%s",
381 n, opc, num_elem, type2name(elem_bt));
382 return false;
383 }
384 }
385 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
386 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
387
388 if (is_vector_mask(vbox_klass)) {
389 assert(!is_masked_op, "mask operations do not need mask to control");
390 }
391
392 if (opc == Op_CallLeafVector) {
393 if (!UseVectorStubs) {
394 log_if_needed(" ** vector stubs support is disabled");
395 return false;
396 }
397 if (!Matcher::supports_vector_calling_convention()) {
398 log_if_needed(" ** no vector calling conventions supported");
399 return false;
400 }
401 if (!Matcher::vector_size_supported(elem_bt, num_elem)) {
402 log_if_needed(" ** vector size (vlen=%d, etype=%s) is not supported",
403 num_elem, type2name(elem_bt));
404 return false;
405 }
406 }
407
408 // When using mask, mask use type needs to be VecMaskUseLoad.
409 VectorMaskUseType mask_use_type = is_vector_mask(vbox_klass) ? VecMaskUseAll
410 : is_masked_op ? VecMaskUseLoad : VecMaskNotUsed;
411 if ((sopc != 0) && !arch_supports_vector(sopc, num_elem, elem_bt, mask_use_type)) {
412 log_if_needed(" ** not supported: arity=%d opc=%d vlen=%d etype=%s ismask=%d is_masked_op=%d",
413 n, sopc, num_elem, type2name(elem_bt),
414 is_vector_mask(vbox_klass) ? 1 : 0, is_masked_op ? 1 : 0);
415 return false; // not supported
416 }
417
418 // Return true if current platform has implemented the masked operation with predicate feature.
419 bool use_predicate = is_masked_op && sopc != 0 && arch_supports_vector(sopc, num_elem, elem_bt, VecMaskUsePred);
420 if (is_masked_op && !use_predicate && !arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad)) {
421 log_if_needed(" ** not supported: arity=%d opc=%d vlen=%d etype=%s ismask=0 is_masked_op=1",
422 n, sopc, num_elem, type2name(elem_bt));
423 return false;
424 }
425
426 Node* opd1 = nullptr; Node* opd2 = nullptr; Node* opd3 = nullptr;
427 switch (n) {
428 case 3: {
429 opd3 = unbox_vector(argument(7), vbox_type, elem_bt, num_elem);
430 if (opd3 == nullptr) {
431 log_if_needed(" ** unbox failed v3=%s",
432 NodeClassNames[argument(7)->Opcode()]);
433 return false;
434 }
435 // fall-through
436 }
437 case 2: {
438 opd2 = unbox_vector(argument(6), vbox_type, elem_bt, num_elem);
439 if (opd2 == nullptr) {
440 log_if_needed(" ** unbox failed v2=%s",
441 NodeClassNames[argument(6)->Opcode()]);
442 return false;
443 }
444 // fall-through
445 }
446 case 1: {
447 opd1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
448 if (opd1 == nullptr) {
449 log_if_needed(" ** unbox failed v1=%s",
450 NodeClassNames[argument(5)->Opcode()]);
451 return false;
452 }
453 break;
454 }
455 default: fatal("unsupported arity: %d", n);
456 }
457
458 Node* mask = nullptr;
459 if (is_masked_op) {
460 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
461 assert(is_vector_mask(mbox_klass), "argument(2) should be a mask class");
462 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
463 mask = unbox_vector(argument(n + 5), mbox_type, elem_bt, num_elem);
464 if (mask == nullptr) {
465 log_if_needed(" ** unbox failed mask=%s",
466 NodeClassNames[argument(n + 5)->Opcode()]);
467 return false;
468 }
469 }
470
471 Node* operation = nullptr;
472 if (opc == Op_CallLeafVector) {
473 assert(UseVectorStubs, "sanity");
474 operation = gen_call_to_vector_math(opr->get_con(), elem_bt, num_elem, opd1, opd2);
475 if (operation == nullptr) {
476 log_if_needed(" ** Vector math call failed for %s_%s_%d",
477 (elem_bt == T_FLOAT) ? "float" : "double",
478 VectorSupport::mathname[opr->get_con() - VectorSupport::VECTOR_OP_MATH_START],
479 num_elem * type2aelembytes(elem_bt));
480 return false;
481 }
482 } else {
483 const TypeVect* vt = TypeVect::make(elem_bt, num_elem, is_vector_mask(vbox_klass));
484 switch (n) {
485 case 1:
486 case 2: {
487 operation = VectorNode::make(sopc, opd1, opd2, vt, is_vector_mask(vbox_klass), VectorNode::is_shift_opcode(opc), is_unsigned);
488 break;
489 }
490 case 3: {
491 operation = VectorNode::make(sopc, opd1, opd2, opd3, vt);
492 break;
493 }
494 default: fatal("unsupported arity: %d", n);
495 }
496 }
497
498 if (is_masked_op && mask != nullptr) {
499 if (use_predicate) {
500 operation->add_req(mask);
501 operation->add_flag(Node::Flag_is_predicated_vector);
502 } else {
503 operation->add_flag(Node::Flag_is_predicated_using_blend);
504 operation = gvn().transform(operation);
505 operation = new VectorBlendNode(opd1, operation, mask);
506 }
507 }
508 operation = gvn().transform(operation);
509
510 // Wrap it up in VectorBox to keep object type information.
511 Node* vbox = box_vector(operation, vbox_type, elem_bt, num_elem);
512 set_result(vbox);
513 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
514 return true;
515 }
516
517 // Following routine generates IR corresponding to AbstractShuffle::partiallyWrapIndex method,
518 // which partially wraps index by modulo VEC_LENGTH and generates a negative index value if original
519 // index is out of valid index range [0, VEC_LENGTH)
520 //
521 // wrapped_index = (VEC_LENGTH - 1) & index
522 // if (index u> VEC_LENGTH) {
523 // wrapped_index -= VEC_LENGTH;
524 //
525 // Note: Unsigned greater than comparison treat both <0 and >VEC_LENGTH indices as out-of-bound
526 // indexes.
527 Node* LibraryCallKit::partially_wrap_indexes(Node* index_vec, int num_elem, BasicType elem_bt) {
528 assert(elem_bt == T_BYTE, "Shuffles use byte array based backing storage.");
529 const TypeVect* vt = TypeVect::make(elem_bt, num_elem);
530
531 Node* mod_mask = gvn().makecon(TypeInt::make(num_elem-1));
532 Node* bcast_mod_mask = gvn().transform(VectorNode::scalar2vector(mod_mask, num_elem, elem_bt));
533
534 BoolTest::mask pred = BoolTest::ugt;
535 ConINode* pred_node = (ConINode*)gvn().makecon(TypeInt::make(pred));
536 Node* lane_cnt = gvn().makecon(TypeInt::make(num_elem));
537 Node* bcast_lane_cnt = gvn().transform(VectorNode::scalar2vector(lane_cnt, num_elem, elem_bt));
538 const TypeVect* vmask_type = TypeVect::makemask(elem_bt, num_elem);
539 Node* mask = gvn().transform(new VectorMaskCmpNode(pred, bcast_lane_cnt, index_vec, pred_node, vmask_type));
540
541 // Make the indices greater than lane count as -ve values to match the java side implementation.
542 index_vec = gvn().transform(VectorNode::make(Op_AndV, index_vec, bcast_mod_mask, vt));
543 Node* biased_val = gvn().transform(VectorNode::make(Op_SubVB, index_vec, bcast_lane_cnt, vt));
544 return gvn().transform(new VectorBlendNode(biased_val, index_vec, mask));
545 }
546
547 // <Sh extends VectorShuffle<E>, E>
548 // Sh ShuffleIota(Class<?> E, Class<?> shuffleClass, Vector.Species<E> s, int length,
549 // int start, int step, int wrap, ShuffleIotaOperation<Sh, E> defaultImpl)
550 bool LibraryCallKit::inline_vector_shuffle_iota() {
551 const TypeInstPtr* shuffle_klass = gvn().type(argument(1))->isa_instptr();
552 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
553 const TypeInt* start_val = gvn().type(argument(4))->isa_int();
554 const TypeInt* step_val = gvn().type(argument(5))->isa_int();
555 const TypeInt* wrap = gvn().type(argument(6))->isa_int();
556
557 if (shuffle_klass == nullptr || shuffle_klass->const_oop() == nullptr ||
558 vlen == nullptr || !vlen->is_con() || start_val == nullptr || step_val == nullptr ||
559 wrap == nullptr || !wrap->is_con()) {
560 return false; // not enough info for intrinsification
561 }
562
563 if (!is_klass_initialized(shuffle_klass)) {
564 log_if_needed(" ** klass argument not initialized");
565 return false;
566 }
567
568 int do_wrap = wrap->get_con();
569 int num_elem = vlen->get_con();
570 BasicType elem_bt = T_BYTE;
571
572 bool effective_indices_in_range = false;
573 if (start_val->is_con() && step_val->is_con()) {
574 int effective_min_index = start_val->get_con();
575 int effective_max_index = start_val->get_con() + step_val->get_con() * (num_elem - 1);
576 effective_indices_in_range = effective_max_index >= effective_min_index && effective_min_index >= -128 && effective_max_index <= 127;
577 }
578
579 if (!do_wrap && !effective_indices_in_range) {
580 // Disable instrinsification for unwrapped shuffle iota if start/step
581 // values are non-constant OR if intermediate result overflows byte value range.
582 return false;
583 }
584
585 if (!arch_supports_vector(Op_AddVB, num_elem, elem_bt, VecMaskNotUsed) ||
586 !arch_supports_vector(Op_AndV, num_elem, elem_bt, VecMaskNotUsed) ||
587 !arch_supports_vector(Op_VectorLoadConst, num_elem, elem_bt, VecMaskNotUsed) ||
588 !arch_supports_vector(Op_Replicate, num_elem, elem_bt, VecMaskNotUsed)) {
589 return false;
590 }
591
592 if (!do_wrap &&
593 (!arch_supports_vector(Op_SubVB, num_elem, elem_bt, VecMaskNotUsed) ||
594 !arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskNotUsed) ||
595 !arch_supports_vector(Op_VectorMaskCmp, num_elem, elem_bt, VecMaskNotUsed))) {
596 return false;
597 }
598
599 bool step_multiply = !step_val->is_con() || !is_power_of_2(step_val->get_con());
600 if ((step_multiply && !arch_supports_vector(Op_MulVB, num_elem, elem_bt, VecMaskNotUsed)) ||
601 (!step_multiply && !arch_supports_vector(Op_LShiftVB, num_elem, elem_bt, VecMaskNotUsed))) {
602 return false;
603 }
604
605 const TypeVect* vt = TypeVect::make(elem_bt, num_elem);
606
607 Node* res = gvn().transform(new VectorLoadConstNode(gvn().makecon(TypeInt::ZERO), vt));
608
609 Node* start = argument(4);
610 Node* step = argument(5);
611
612 if (step_multiply) {
613 Node* bcast_step = gvn().transform(VectorNode::scalar2vector(step, num_elem, elem_bt));
614 res = gvn().transform(VectorNode::make(Op_MulVB, res, bcast_step, vt));
615 } else if (step_val->get_con() > 1) {
616 Node* cnt = gvn().makecon(TypeInt::make(log2i_exact(step_val->get_con())));
617 Node* shift_cnt = vector_shift_count(cnt, Op_LShiftI, elem_bt, num_elem);
618 res = gvn().transform(VectorNode::make(Op_LShiftVB, res, shift_cnt, vt));
619 }
620
621 if (!start_val->is_con() || start_val->get_con() != 0) {
622 Node* bcast_start = gvn().transform(VectorNode::scalar2vector(start, num_elem, elem_bt));
623 res = gvn().transform(VectorNode::make(Op_AddVB, res, bcast_start, vt));
624 }
625
626 Node* mod_val = gvn().makecon(TypeInt::make(num_elem-1));
627 Node* bcast_mod = gvn().transform(VectorNode::scalar2vector(mod_val, num_elem, elem_bt));
628
629 if (do_wrap) {
630 // Wrap the indices greater than lane count.
631 res = gvn().transform(VectorNode::make(Op_AndV, res, bcast_mod, vt));
632 } else {
633 res = partially_wrap_indexes(res, num_elem, elem_bt);
634 }
635
636 ciKlass* sbox_klass = shuffle_klass->const_oop()->as_instance()->java_lang_Class_klass();
637 const TypeInstPtr* shuffle_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, sbox_klass);
638
639 // Wrap it up in VectorBox to keep object type information.
640 res = box_vector(res, shuffle_box_type, elem_bt, num_elem);
641 set_result(res);
642 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
643 return true;
644 }
645
646 // <E, M>
647 // long maskReductionCoerced(int oper, Class<? extends M> maskClass, Class<?> elemClass,
648 // int length, M m, VectorMaskOp<M> defaultImpl)
649 bool LibraryCallKit::inline_vector_mask_operation() {
650 const TypeInt* oper = gvn().type(argument(0))->isa_int();
651 const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
652 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
653 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
654 Node* mask = argument(4);
655
656 if (mask_klass == nullptr || elem_klass == nullptr || mask->is_top() || vlen == nullptr) {
657 return false; // dead code
658 }
659
660 if (!is_klass_initialized(mask_klass)) {
661 log_if_needed(" ** klass argument not initialized");
662 return false;
663 }
664
665 int num_elem = vlen->get_con();
666 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
667 BasicType elem_bt = elem_type->basic_type();
668
669 int mopc = VectorSupport::vop2ideal(oper->get_con(), elem_bt);
670 if (!arch_supports_vector(mopc, num_elem, elem_bt, VecMaskUseLoad)) {
671 log_if_needed(" ** not supported: arity=1 op=cast#%d/3 vlen2=%d etype2=%s",
672 mopc, num_elem, type2name(elem_bt));
673 return false; // not supported
674 }
675
676 const Type* elem_ty = Type::get_const_basic_type(elem_bt);
677 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
678 const TypeInstPtr* mask_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
679 Node* mask_vec = unbox_vector(mask, mask_box_type, elem_bt, num_elem, true);
680 if (mask_vec == nullptr) {
681 log_if_needed(" ** unbox failed mask=%s",
682 NodeClassNames[argument(4)->Opcode()]);
683 return false;
684 }
685
686 if (mask_vec->bottom_type()->isa_vectmask() == nullptr) {
687 mask_vec = gvn().transform(VectorStoreMaskNode::make(gvn(), mask_vec, elem_bt, num_elem));
688 }
689 const Type* maskoper_ty = mopc == Op_VectorMaskToLong ? (const Type*)TypeLong::LONG : (const Type*)TypeInt::INT;
690 Node* maskoper = gvn().transform(VectorMaskOpNode::make(mask_vec, maskoper_ty, mopc));
691 if (mopc != Op_VectorMaskToLong) {
692 maskoper = ConvI2L(maskoper);
693 }
694 set_result(maskoper);
695
696 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
697 return true;
698 }
699
700 // public static
701 // <V,
702 // Sh extends VectorShuffle<E>,
703 // E>
704 // V shuffleToVector(Class<? extends Vector<E>> vclass, Class<E> elementType,
705 // Class<? extends Sh> shuffleClass, Sh s, int length,
706 // ShuffleToVectorOperation<V, Sh, E> defaultImpl)
707 bool LibraryCallKit::inline_vector_shuffle_to_vector() {
708 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
709 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
710 const TypeInstPtr* shuffle_klass = gvn().type(argument(2))->isa_instptr();
711 Node* shuffle = argument(3);
712 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
713
714 if (vector_klass == nullptr || elem_klass == nullptr || shuffle_klass == nullptr || shuffle->is_top() || vlen == nullptr) {
715 return false; // dead code
716 }
717 if (!vlen->is_con() || vector_klass->const_oop() == nullptr || shuffle_klass->const_oop() == nullptr) {
718 return false; // not enough info for intrinsification
719 }
720 if (!is_klass_initialized(shuffle_klass) || !is_klass_initialized(vector_klass) ) {
721 log_if_needed(" ** klass argument not initialized");
722 return false;
723 }
724
725 int num_elem = vlen->get_con();
726 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
727 BasicType elem_bt = elem_type->basic_type();
728
729 if (num_elem < 4) {
730 return false;
731 }
732
733 int cast_vopc = VectorCastNode::opcode(-1, T_BYTE); // from shuffle of type T_BYTE
734 // Make sure that cast is implemented to particular type/size combination.
735 if (!arch_supports_vector(cast_vopc, num_elem, elem_bt, VecMaskNotUsed)) {
736 log_if_needed(" ** not supported: arity=1 op=cast#%d/3 vlen2=%d etype2=%s",
737 cast_vopc, num_elem, type2name(elem_bt));
738 return false;
739 }
740
741 ciKlass* sbox_klass = shuffle_klass->const_oop()->as_instance()->java_lang_Class_klass();
742 const TypeInstPtr* shuffle_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, sbox_klass);
743
744 // Unbox shuffle with true flag to indicate its load shuffle to vector
745 // shuffle is a byte array
746 Node* shuffle_vec = unbox_vector(shuffle, shuffle_box_type, T_BYTE, num_elem, true);
747
748 // cast byte to target element type
749 shuffle_vec = gvn().transform(VectorCastNode::make(cast_vopc, shuffle_vec, elem_bt, num_elem));
750
751 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
752 const TypeInstPtr* vec_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
753
754 // Box vector
755 Node* res = box_vector(shuffle_vec, vec_box_type, elem_bt, num_elem);
756 set_result(res);
757 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
758 return true;
759 }
760
761 // public static
762 // <E,
763 // SH extends VectorShuffle<E>>
764 // SH wrapShuffleIndexes(Class<E> eClass, Class<? extends SH> shClass, SH sh, int length,
765 // ShuffleWrapIndexesOperation<SH> defaultImpl)
766 bool LibraryCallKit::inline_vector_wrap_shuffle_indexes() {
767 const TypeInstPtr* elem_klass = gvn().type(argument(0))->isa_instptr();
768 const TypeInstPtr* shuffle_klass = gvn().type(argument(1))->isa_instptr();
769 Node* shuffle = argument(2);
770 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
771
772 if (elem_klass == nullptr || shuffle_klass == nullptr || shuffle->is_top() || vlen == nullptr ||
773 !vlen->is_con() || shuffle_klass->const_oop() == nullptr) {
774 // not enough info for intrinsification
775 return false;
776 }
777
778 if (!is_klass_initialized(shuffle_klass)) {
779 log_if_needed(" ** klass argument not initialized");
780 return false;
781 }
782
783 int num_elem = vlen->get_con();
784 if ((num_elem < 4) || !is_power_of_2(num_elem)) {
785 log_if_needed(" ** vlen < 4 or not power of two=%d", num_elem);
786 return false;
787 }
788
789 // Shuffles use byte array based backing storage
790 BasicType shuffle_bt = T_BYTE;
791 if (!arch_supports_vector(Op_AndV, num_elem, shuffle_bt, VecMaskNotUsed) ||
792 !arch_supports_vector(Op_Replicate, num_elem, shuffle_bt, VecMaskNotUsed)) {
793 log_if_needed(" ** not supported: op=wrapShuffleIndexes vlen=%d etype=%s",
794 num_elem, type2name(shuffle_bt));
795 return false;
796 }
797
798 ciKlass* sbox_klass = shuffle_klass->const_oop()->as_instance()->java_lang_Class_klass();
799 const TypeInstPtr* shuffle_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, sbox_klass);
800
801 // Unbox shuffle with true flag to indicate its load shuffle to vector
802 // shuffle is a byte array
803 Node* shuffle_vec = unbox_vector(shuffle, shuffle_box_type, shuffle_bt, num_elem, true);
804
805 const TypeVect* vt = TypeVect::make(shuffle_bt, num_elem);
806 Node* mod_mask = gvn().makecon(TypeInt::make(num_elem - 1));
807 Node* bcast_mod_mask = gvn().transform(VectorNode::scalar2vector(mod_mask, num_elem, shuffle_bt));
808 // Wrap the indices greater than lane count.
809 Node* res = gvn().transform(VectorNode::make(Op_AndV, shuffle_vec, bcast_mod_mask, vt));
810
811 // Wrap it up in VectorBox to keep object type information.
812 res = box_vector(res, shuffle_box_type, shuffle_bt, num_elem);
813 set_result(res);
814 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(shuffle_bt))));
815 return true;
816 }
817
818 // public static
819 // <M,
820 // S extends VectorSpecies<E>,
821 // E>
822 // M fromBitsCoerced(Class<? extends M> vmClass, Class<E> elementType, int length,
823 // long bits, int mode, S s,
824 // BroadcastOperation<M, E, S> defaultImpl)
825 bool LibraryCallKit::inline_vector_frombits_coerced() {
826 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
827 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
828 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
829 const TypeLong* bits_type = gvn().type(argument(3))->isa_long();
830 // Mode argument determines the mode of operation it can take following values:-
831 // MODE_BROADCAST for vector Vector.broadcast and VectorMask.maskAll operations.
832 // MODE_BITS_COERCED_LONG_TO_MASK for VectorMask.fromLong operation.
833 const TypeInt* mode = gvn().type(argument(5))->isa_int();
834
835 if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr || mode == nullptr ||
836 bits_type == nullptr || vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr ||
837 !vlen->is_con() || !mode->is_con()) {
838 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s bitwise=%s",
839 NodeClassNames[argument(0)->Opcode()],
840 NodeClassNames[argument(1)->Opcode()],
841 NodeClassNames[argument(2)->Opcode()],
842 NodeClassNames[argument(5)->Opcode()]);
843 return false; // not enough info for intrinsification
844 }
845
846 if (!is_klass_initialized(vector_klass)) {
847 log_if_needed(" ** klass argument not initialized");
848 return false;
849 }
850 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
851 if (!elem_type->is_primitive_type()) {
852 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
853 return false; // should be primitive type
854 }
855 BasicType elem_bt = elem_type->basic_type();
856 int num_elem = vlen->get_con();
857 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
858 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
859
860 bool is_mask = is_vector_mask(vbox_klass);
861 int bcast_mode = mode->get_con();
862 VectorMaskUseType checkFlags = (VectorMaskUseType)(is_mask ? VecMaskUseAll : VecMaskNotUsed);
863 int opc = bcast_mode == VectorSupport::MODE_BITS_COERCED_LONG_TO_MASK ? Op_VectorLongToMask : Op_Replicate;
864
865 if (!arch_supports_vector(opc, num_elem, elem_bt, checkFlags, true /*has_scalar_args*/)) {
866 log_if_needed(" ** not supported: arity=0 op=broadcast vlen=%d etype=%s ismask=%d bcast_mode=%d",
867 num_elem, type2name(elem_bt),
868 is_mask ? 1 : 0,
869 bcast_mode);
870 return false; // not supported
871 }
872
873 Node* broadcast = nullptr;
874 Node* bits = argument(3);
875 Node* elem = bits;
876
877 if (opc == Op_VectorLongToMask) {
878 const TypeVect* vt = TypeVect::makemask(elem_bt, num_elem);
879 if (vt->isa_vectmask()) {
880 broadcast = gvn().transform(new VectorLongToMaskNode(elem, vt));
881 } else {
882 const TypeVect* mvt = TypeVect::make(T_BOOLEAN, num_elem);
883 broadcast = gvn().transform(new VectorLongToMaskNode(elem, mvt));
884 broadcast = gvn().transform(new VectorLoadMaskNode(broadcast, vt));
885 }
886 } else {
887 switch (elem_bt) {
888 case T_BOOLEAN: // fall-through
889 case T_BYTE: // fall-through
890 case T_SHORT: // fall-through
891 case T_CHAR: // fall-through
892 case T_INT: {
893 elem = gvn().transform(new ConvL2INode(bits));
894 break;
895 }
896 case T_DOUBLE: {
897 elem = gvn().transform(new MoveL2DNode(bits));
898 break;
899 }
900 case T_FLOAT: {
901 bits = gvn().transform(new ConvL2INode(bits));
902 elem = gvn().transform(new MoveI2FNode(bits));
903 break;
904 }
905 case T_LONG: {
906 // no conversion needed
907 break;
908 }
909 default: fatal("%s", type2name(elem_bt));
910 }
911 broadcast = VectorNode::scalar2vector(elem, num_elem, elem_bt, is_mask);
912 broadcast = gvn().transform(broadcast);
913 }
914
915 Node* box = box_vector(broadcast, vbox_type, elem_bt, num_elem);
916 set_result(box);
917 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
918 return true;
919 }
920
921 static bool elem_consistent_with_arr(BasicType elem_bt, const TypeAryPtr* arr_type, bool mismatched_ms) {
922 assert(arr_type != nullptr, "unexpected");
923 BasicType arr_elem_bt = arr_type->elem()->array_element_basic_type();
924 if (elem_bt == arr_elem_bt) {
925 return true;
926 } else if (elem_bt == T_SHORT && arr_elem_bt == T_CHAR) {
927 // Load/store of short vector from/to char[] is supported
928 return true;
929 } else if (elem_bt == T_BYTE && arr_elem_bt == T_BOOLEAN) {
930 // Load/store of byte vector from/to boolean[] is supported
931 return true;
932 } else {
933 return mismatched_ms;
934 }
935 }
936
937 // public static
938 // <C,
939 // VM extends VectorPayload,
940 // E,
941 // S extends VectorSpecies<E>>
942 // VM load(Class<? extends VM> vmClass, Class<E> eClass,
943 // int length,
944 // Object base, long offset, // Unsafe addressing
945 // boolean fromSegment,
946 // C container, long index, S s, // Arguments for default implementation
947 // LoadOperation<C, VM, S> defaultImpl) {
948 // public static
949 // <C,
950 // V extends VectorPayload>
951 // void store(Class<?> vClass, Class<?> eClass,
952 // int length,
953 // Object base, long offset, // Unsafe addressing
954 // boolean fromSegment,
955 // V v, C container, long index, // Arguments for default implementation
956 // StoreVectorOperation<C, V> defaultImpl) {
957 bool LibraryCallKit::inline_vector_mem_operation(bool is_store) {
958 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
959 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
960 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
961 const TypeInt* from_ms = gvn().type(argument(6))->isa_int();
962
963 if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr || !from_ms->is_con() ||
964 vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
965 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s from_ms=%s",
966 NodeClassNames[argument(0)->Opcode()],
967 NodeClassNames[argument(1)->Opcode()],
968 NodeClassNames[argument(2)->Opcode()],
969 NodeClassNames[argument(6)->Opcode()]);
970 return false; // not enough info for intrinsification
971 }
972 if (!is_klass_initialized(vector_klass)) {
973 log_if_needed(" ** klass argument not initialized");
974 return false;
975 }
976
977 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
978 if (!elem_type->is_primitive_type()) {
979 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
980 return false; // should be primitive type
981 }
982 BasicType elem_bt = elem_type->basic_type();
983 int num_elem = vlen->get_con();
984
985 // TODO When mask usage is supported, VecMaskNotUsed needs to be VecMaskUseLoad.
986 if (!arch_supports_vector(is_store ? Op_StoreVector : Op_LoadVector, num_elem, elem_bt, VecMaskNotUsed)) {
987 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d etype=%s ismask=no",
988 is_store, is_store ? "store" : "load",
989 num_elem, type2name(elem_bt));
990 return false; // not supported
991 }
992
993 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
994 bool is_mask = is_vector_mask(vbox_klass);
995
996 Node* base = argument(3);
997 Node* offset = ConvL2X(argument(4));
998
999 // Save state and restore on bailout
1000 uint old_sp = sp();
1001 SafePointNode* old_map = clone_map();
1002
1003 Node* addr = make_unsafe_address(base, offset, (is_mask ? T_BOOLEAN : elem_bt), true);
1004
1005 // The memory barrier checks are based on ones for unsafe access.
1006 // This is not 1-1 implementation.
1007 const Type *const base_type = gvn().type(base);
1008
1009 const TypePtr *addr_type = gvn().type(addr)->isa_ptr();
1010 const TypeAryPtr* arr_type = addr_type->isa_aryptr();
1011
1012 const bool in_native = TypePtr::NULL_PTR == base_type; // base always null
1013 const bool in_heap = !TypePtr::NULL_PTR->higher_equal(base_type); // base never null
1014
1015 const bool is_mixed_access = !in_heap && !in_native;
1016
1017 const bool is_mismatched_access = in_heap && (addr_type->isa_aryptr() == nullptr);
1018
1019 const bool needs_cpu_membar = is_mixed_access || is_mismatched_access;
1020
1021 // For non-masked mismatched memory segment vector read/write accesses, intrinsification can continue
1022 // with unknown backing storage type and compiler can skip inserting explicit reinterpretation IR after
1023 // loading from or before storing to backing storage which is mandatory for semantic correctness of
1024 // big-endian memory layout.
1025 bool mismatched_ms = LITTLE_ENDIAN_ONLY(false)
1026 BIG_ENDIAN_ONLY(from_ms->get_con() && !is_mask && arr_type != nullptr &&
1027 arr_type->elem()->array_element_basic_type() != elem_bt);
1028 BasicType mem_elem_bt = mismatched_ms ? arr_type->elem()->array_element_basic_type() : elem_bt;
1029 if (!is_java_primitive(mem_elem_bt)) {
1030 log_if_needed(" ** non-primitive array element type");
1031 return false;
1032 }
1033 int mem_num_elem = mismatched_ms ? (num_elem * type2aelembytes(elem_bt)) / type2aelembytes(mem_elem_bt) : num_elem;
1034 if (arr_type != nullptr && !is_mask && !elem_consistent_with_arr(elem_bt, arr_type, mismatched_ms)) {
1035 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d etype=%s atype=%s ismask=no",
1036 is_store, is_store ? "store" : "load",
1037 num_elem, type2name(elem_bt), type2name(arr_type->elem()->array_element_basic_type()));
1038 set_map(old_map);
1039 set_sp(old_sp);
1040 return false;
1041 }
1042
1043 // In case of mismatched memory segment accesses, we need to double check that the source type memory operations are supported by backend.
1044 if (mismatched_ms) {
1045 if (is_store) {
1046 if (!arch_supports_vector(Op_StoreVector, num_elem, elem_bt, VecMaskNotUsed)
1047 || !arch_supports_vector(Op_VectorReinterpret, mem_num_elem, mem_elem_bt, VecMaskNotUsed)) {
1048 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d*8 etype=%s/8 ismask=no",
1049 is_store, "store",
1050 num_elem, type2name(elem_bt));
1051 set_map(old_map);
1052 set_sp(old_sp);
1053 return false; // not supported
1054 }
1055 } else {
1056 if (!arch_supports_vector(Op_LoadVector, mem_num_elem, mem_elem_bt, VecMaskNotUsed)
1057 || !arch_supports_vector(Op_VectorReinterpret, num_elem, elem_bt, VecMaskNotUsed)) {
1058 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d*8 etype=%s/8 ismask=no",
1059 is_store, "load",
1060 mem_num_elem, type2name(mem_elem_bt));
1061 set_map(old_map);
1062 set_sp(old_sp);
1063 return false; // not supported
1064 }
1065 }
1066 }
1067 if (is_mask) {
1068 if (!is_store) {
1069 if (!arch_supports_vector(Op_LoadVector, num_elem, elem_bt, VecMaskUseLoad)) {
1070 set_map(old_map);
1071 set_sp(old_sp);
1072 return false; // not supported
1073 }
1074 } else {
1075 if (!arch_supports_vector(Op_StoreVector, num_elem, elem_bt, VecMaskUseStore)) {
1076 set_map(old_map);
1077 set_sp(old_sp);
1078 return false; // not supported
1079 }
1080 }
1081 }
1082
1083 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1084
1085 if (needs_cpu_membar) {
1086 insert_mem_bar(Op_MemBarCPUOrder);
1087 }
1088
1089 if (is_store) {
1090 Node* val = unbox_vector(argument(7), vbox_type, elem_bt, num_elem);
1091 if (val == nullptr) {
1092 set_map(old_map);
1093 set_sp(old_sp);
1094 return false; // operand unboxing failed
1095 }
1096 set_all_memory(reset_memory());
1097
1098 // In case the store needs to happen to byte array, reinterpret the incoming vector to byte vector.
1099 int store_num_elem = num_elem;
1100 if (mismatched_ms) {
1101 store_num_elem = mem_num_elem;
1102 const TypeVect* to_vect_type = TypeVect::make(mem_elem_bt, store_num_elem);
1103 val = gvn().transform(new VectorReinterpretNode(val, val->bottom_type()->is_vect(), to_vect_type));
1104 }
1105 if (is_mask) {
1106 val = gvn().transform(VectorStoreMaskNode::make(gvn(), val, elem_bt, num_elem));
1107 }
1108 Node* vstore = gvn().transform(StoreVectorNode::make(0, control(), memory(addr), addr, addr_type, val, store_num_elem));
1109 set_memory(vstore, addr_type);
1110 } else {
1111 // When using byte array, we need to load as byte then reinterpret the value. Otherwise, do a simple vector load.
1112 Node* vload = nullptr;
1113 if (mismatched_ms) {
1114 vload = gvn().transform(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, mem_num_elem, mem_elem_bt));
1115 const TypeVect* to_vect_type = TypeVect::make(elem_bt, num_elem);
1116 vload = gvn().transform(new VectorReinterpretNode(vload, vload->bottom_type()->is_vect(), to_vect_type));
1117 } else {
1118 // Special handle for masks
1119 if (is_mask) {
1120 vload = gvn().transform(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, num_elem, T_BOOLEAN));
1121 vload = gvn().transform(new VectorLoadMaskNode(vload, TypeVect::makemask(elem_bt, num_elem)));
1122 } else {
1123 vload = gvn().transform(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, num_elem, elem_bt));
1124 }
1125 }
1126 Node* box = box_vector(vload, vbox_type, elem_bt, num_elem);
1127 set_result(box);
1128 }
1129
1130 destruct_map_clone(old_map);
1131
1132 if (needs_cpu_membar) {
1133 insert_mem_bar(Op_MemBarCPUOrder);
1134 }
1135
1136 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1137 return true;
1138 }
1139
1140 // public static
1141 // <C,
1142 // V extends Vector<?>,
1143 // E,
1144 // S extends VectorSpecies<E>,
1145 // M extends VectorMask<E>>
1146 // V loadMasked(Class<? extends V> vClass, Class<M> mClass, Class<E> eClass,
1147 // int length, Object base, long offset, // Unsafe addressing
1148 // boolean fromSegment,
1149 // M m, int offsetInRange,
1150 // C container, long index, S s, // Arguments for default implementation
1151 // LoadVectorMaskedOperation<C, V, S, M> defaultImpl) {
1152 // public static
1153 // <C,
1154 // V extends Vector<E>,
1155 // M extends VectorMask<E>,
1156 // E>
1157 // void storeMasked(Class<? extends V> vClass, Class<M> mClass, Class<E> eClass,
1158 // int length,
1159 // Object base, long offset, // Unsafe addressing
1160 // boolean fromSegment,
1161 // V v, M m, C container, long index, // Arguments for default implementation
1162 // StoreVectorMaskedOperation<C, V, M> defaultImpl) {
1163
1164 bool LibraryCallKit::inline_vector_mem_masked_operation(bool is_store) {
1165 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
1166 const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
1167 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
1168 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
1169 const TypeInt* from_ms = gvn().type(argument(7))->isa_int();
1170
1171 if (vector_klass == nullptr || mask_klass == nullptr || elem_klass == nullptr || vlen == nullptr ||
1172 vector_klass->const_oop() == nullptr || mask_klass->const_oop() == nullptr || from_ms == nullptr ||
1173 elem_klass->const_oop() == nullptr || !vlen->is_con() || !from_ms->is_con()) {
1174 log_if_needed(" ** missing constant: vclass=%s mclass=%s etype=%s vlen=%s from_ms=%s",
1175 NodeClassNames[argument(0)->Opcode()],
1176 NodeClassNames[argument(1)->Opcode()],
1177 NodeClassNames[argument(2)->Opcode()],
1178 NodeClassNames[argument(3)->Opcode()],
1179 NodeClassNames[argument(7)->Opcode()]);
1180 return false; // not enough info for intrinsification
1181 }
1182 if (!is_klass_initialized(vector_klass)) {
1183 log_if_needed(" ** klass argument not initialized");
1184 return false;
1185 }
1186
1187 if (!is_klass_initialized(mask_klass)) {
1188 log_if_needed(" ** mask klass argument not initialized");
1189 return false;
1190 }
1191
1192 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1193 if (!elem_type->is_primitive_type()) {
1194 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1195 return false; // should be primitive type
1196 }
1197
1198 BasicType elem_bt = elem_type->basic_type();
1199 int num_elem = vlen->get_con();
1200
1201 Node* base = argument(4);
1202 Node* offset = ConvL2X(argument(5));
1203
1204 // Save state and restore on bailout
1205 uint old_sp = sp();
1206 SafePointNode* old_map = clone_map();
1207
1208 Node* addr = make_unsafe_address(base, offset, elem_bt, true);
1209 const TypePtr *addr_type = gvn().type(addr)->isa_ptr();
1210 const TypeAryPtr* arr_type = addr_type->isa_aryptr();
1211
1212 bool mismatched_ms = from_ms->get_con() && arr_type != nullptr && arr_type->elem()->array_element_basic_type() != elem_bt;
1213 BIG_ENDIAN_ONLY(if (mismatched_ms) return false;)
1214 // If there is no consistency between array and vector element types, it must be special byte array case
1215 if (arr_type != nullptr && !elem_consistent_with_arr(elem_bt, arr_type, mismatched_ms)) {
1216 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d etype=%s atype=%s",
1217 is_store, is_store ? "storeMasked" : "loadMasked",
1218 num_elem, type2name(elem_bt), type2name(arr_type->elem()->array_element_basic_type()));
1219 set_map(old_map);
1220 set_sp(old_sp);
1221 return false;
1222 }
1223
1224 int mem_num_elem = mismatched_ms ? num_elem * type2aelembytes(elem_bt) : num_elem;
1225 BasicType mem_elem_bt = mismatched_ms ? T_BYTE : elem_bt;
1226 bool supports_predicate = arch_supports_vector(is_store ? Op_StoreVectorMasked : Op_LoadVectorMasked,
1227 mem_num_elem, mem_elem_bt, VecMaskUseLoad);
1228
1229 // If current arch does not support the predicated operations, we have to bail
1230 // out when current case uses the predicate feature.
1231 if (!supports_predicate) {
1232 bool needs_predicate = false;
1233 if (is_store) {
1234 // Masked vector store always uses the predicated store.
1235 needs_predicate = true;
1236 } else {
1237 // Masked vector load with IOOBE always uses the predicated load.
1238 const TypeInt* offset_in_range = gvn().type(argument(9))->isa_int();
1239 if (!offset_in_range->is_con()) {
1240 log_if_needed(" ** missing constant: offsetInRange=%s",
1241 NodeClassNames[argument(8)->Opcode()]);
1242 set_map(old_map);
1243 set_sp(old_sp);
1244 return false;
1245 }
1246 needs_predicate = (offset_in_range->get_con() == 0);
1247 }
1248
1249 if (needs_predicate) {
1250 log_if_needed(" ** not supported: op=%s vlen=%d etype=%s mismatched_ms=%d",
1251 is_store ? "storeMasked" : "loadMasked",
1252 num_elem, type2name(elem_bt), mismatched_ms ? 1 : 0);
1253 set_map(old_map);
1254 set_sp(old_sp);
1255 return false;
1256 }
1257 }
1258
1259 // This only happens for masked vector load. If predicate is not supported, then check whether
1260 // the normal vector load and blend operations are supported by backend.
1261 if (!supports_predicate && (!arch_supports_vector(Op_LoadVector, mem_num_elem, mem_elem_bt, VecMaskNotUsed) ||
1262 !arch_supports_vector(Op_VectorBlend, mem_num_elem, mem_elem_bt, VecMaskUseLoad))) {
1263 log_if_needed(" ** not supported: op=loadMasked vlen=%d etype=%s mismatched_ms=%d",
1264 num_elem, type2name(elem_bt), mismatched_ms ? 1 : 0);
1265 set_map(old_map);
1266 set_sp(old_sp);
1267 return false;
1268 }
1269
1270 // Since we are using byte array, we need to double check that the vector reinterpret operation
1271 // with byte type is supported by backend.
1272 if (mismatched_ms) {
1273 if (!arch_supports_vector(Op_VectorReinterpret, mem_num_elem, T_BYTE, VecMaskNotUsed)) {
1274 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d etype=%s mismatched_ms=1",
1275 is_store, is_store ? "storeMasked" : "loadMasked",
1276 num_elem, type2name(elem_bt));
1277 set_map(old_map);
1278 set_sp(old_sp);
1279 return false;
1280 }
1281 }
1282
1283 // Since it needs to unbox the mask, we need to double check that the related load operations
1284 // for mask are supported by backend.
1285 if (!arch_supports_vector(Op_LoadVector, num_elem, elem_bt, VecMaskUseLoad)) {
1286 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d etype=%s",
1287 is_store, is_store ? "storeMasked" : "loadMasked",
1288 num_elem, type2name(elem_bt));
1289 set_map(old_map);
1290 set_sp(old_sp);
1291 return false;
1292 }
1293
1294 // Can base be null? Otherwise, always on-heap access.
1295 bool can_access_non_heap = TypePtr::NULL_PTR->higher_equal(gvn().type(base));
1296 if (can_access_non_heap) {
1297 insert_mem_bar(Op_MemBarCPUOrder);
1298 }
1299
1300 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1301 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1302 assert(!is_vector_mask(vbox_klass) && is_vector_mask(mbox_klass), "Invalid class type");
1303 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1304 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1305
1306 Node* mask = unbox_vector(is_store ? argument(9) : argument(8), mbox_type, elem_bt, num_elem);
1307 if (mask == nullptr) {
1308 log_if_needed(" ** unbox failed mask=%s",
1309 is_store ? NodeClassNames[argument(9)->Opcode()]
1310 : NodeClassNames[argument(8)->Opcode()]);
1311 set_map(old_map);
1312 set_sp(old_sp);
1313 return false;
1314 }
1315
1316 if (is_store) {
1317 Node* val = unbox_vector(argument(8), vbox_type, elem_bt, num_elem);
1318 if (val == nullptr) {
1319 log_if_needed(" ** unbox failed vector=%s",
1320 NodeClassNames[argument(8)->Opcode()]);
1321 set_map(old_map);
1322 set_sp(old_sp);
1323 return false; // operand unboxing failed
1324 }
1325 set_all_memory(reset_memory());
1326
1327 if (mismatched_ms) {
1328 // Reinterpret the incoming vector to byte vector.
1329 const TypeVect* to_vect_type = TypeVect::make(mem_elem_bt, mem_num_elem);
1330 val = gvn().transform(new VectorReinterpretNode(val, val->bottom_type()->is_vect(), to_vect_type));
1331 // Reinterpret the vector mask to byte type.
1332 const TypeVect* from_mask_type = TypeVect::makemask(elem_bt, num_elem);
1333 const TypeVect* to_mask_type = TypeVect::makemask(mem_elem_bt, mem_num_elem);
1334 mask = gvn().transform(new VectorReinterpretNode(mask, from_mask_type, to_mask_type));
1335 }
1336 Node* vstore = gvn().transform(new StoreVectorMaskedNode(control(), memory(addr), addr, val, addr_type, mask));
1337 set_memory(vstore, addr_type);
1338 } else {
1339 Node* vload = nullptr;
1340
1341 if (mismatched_ms) {
1342 // Reinterpret the vector mask to byte type.
1343 const TypeVect* from_mask_type = TypeVect::makemask(elem_bt, num_elem);
1344 const TypeVect* to_mask_type = TypeVect::makemask(mem_elem_bt, mem_num_elem);
1345 mask = gvn().transform(new VectorReinterpretNode(mask, from_mask_type, to_mask_type));
1346 }
1347
1348 if (supports_predicate) {
1349 // Generate masked load vector node if predicate feature is supported.
1350 const TypeVect* vt = TypeVect::make(mem_elem_bt, mem_num_elem);
1351 vload = gvn().transform(new LoadVectorMaskedNode(control(), memory(addr), addr, addr_type, vt, mask));
1352 } else {
1353 // Use the vector blend to implement the masked load vector. The biased elements are zeros.
1354 Node* zero = gvn().transform(gvn().zerocon(mem_elem_bt));
1355 zero = gvn().transform(VectorNode::scalar2vector(zero, mem_num_elem, mem_elem_bt));
1356 vload = gvn().transform(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, mem_num_elem, mem_elem_bt));
1357 vload = gvn().transform(new VectorBlendNode(zero, vload, mask));
1358 }
1359
1360 if (mismatched_ms) {
1361 const TypeVect* to_vect_type = TypeVect::make(elem_bt, num_elem);
1362 vload = gvn().transform(new VectorReinterpretNode(vload, vload->bottom_type()->is_vect(), to_vect_type));
1363 }
1364
1365 Node* box = box_vector(vload, vbox_type, elem_bt, num_elem);
1366 set_result(box);
1367 }
1368
1369 destruct_map_clone(old_map);
1370
1371 if (can_access_non_heap) {
1372 insert_mem_bar(Op_MemBarCPUOrder);
1373 }
1374
1375 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1376 return true;
1377 }
1378
1379 // <C,
1380 // V extends Vector<?>,
1381 // W extends Vector<Integer>,
1382 // S extends VectorSpecies<E>,
1383 // M extends VectorMask<E>,
1384 // E>
1385 // V loadWithMap(Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType, int length,
1386 // Class<? extends Vector<Integer>> vectorIndexClass,
1387 // Object base, long offset, // Unsafe addressing
1388 // W index_vector, M m,
1389 // C container, int index, int[] indexMap, int indexM, S s, // Arguments for default implementation
1390 // LoadVectorOperationWithMap<C, V, E, S, M> defaultImpl)
1391 //
1392 // <C,
1393 // V extends Vector<E>,
1394 // W extends Vector<Integer>,
1395 // M extends VectorMask<E>,
1396 // E>
1397 // void storeWithMap(Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType,
1398 // int length, Class<? extends Vector<Integer>> vectorIndexClass, Object base, long offset, // Unsafe addressing
1399 // W index_vector, V v, M m,
1400 // C container, int index, int[] indexMap, int indexM, // Arguments for default implementation
1401 // StoreVectorOperationWithMap<C, V, M, E> defaultImpl)
1402 //
1403 bool LibraryCallKit::inline_vector_gather_scatter(bool is_scatter) {
1404 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
1405 const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
1406 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
1407 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
1408 const TypeInstPtr* vector_idx_klass = gvn().type(argument(4))->isa_instptr();
1409
1410 if (vector_klass == nullptr || elem_klass == nullptr || vector_idx_klass == nullptr || vlen == nullptr ||
1411 vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || vector_idx_klass->const_oop() == nullptr || !vlen->is_con()) {
1412 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s viclass=%s",
1413 NodeClassNames[argument(0)->Opcode()],
1414 NodeClassNames[argument(2)->Opcode()],
1415 NodeClassNames[argument(3)->Opcode()],
1416 NodeClassNames[argument(4)->Opcode()]);
1417 return false; // not enough info for intrinsification
1418 }
1419
1420 if (!is_klass_initialized(vector_klass) || !is_klass_initialized(vector_idx_klass)) {
1421 log_if_needed(" ** klass argument not initialized");
1422 return false;
1423 }
1424
1425 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1426 if (!elem_type->is_primitive_type()) {
1427 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1428 return false; // should be primitive type
1429 }
1430
1431 BasicType elem_bt = elem_type->basic_type();
1432 int num_elem = vlen->get_con();
1433
1434 const Type* vmask_type = gvn().type(is_scatter ? argument(10) : argument(9));
1435 bool is_masked_op = vmask_type != TypePtr::NULL_PTR;
1436 if (is_masked_op) {
1437 if (mask_klass == nullptr || mask_klass->const_oop() == nullptr) {
1438 log_if_needed(" ** missing constant: maskclass=%s", NodeClassNames[argument(1)->Opcode()]);
1439 return false; // not enough info for intrinsification
1440 }
1441
1442 if (!is_klass_initialized(mask_klass)) {
1443 log_if_needed(" ** mask klass argument not initialized");
1444 return false;
1445 }
1446
1447 if (vmask_type->maybe_null()) {
1448 log_if_needed(" ** null mask values are not allowed for masked op");
1449 return false;
1450 }
1451
1452 // Check whether the predicated gather/scatter node is supported by architecture.
1453 VectorMaskUseType mask = (VectorMaskUseType) (VecMaskUseLoad | VecMaskUsePred);
1454 if (!arch_supports_vector(is_scatter ? Op_StoreVectorScatterMasked : Op_LoadVectorGatherMasked, num_elem, elem_bt, mask)) {
1455 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d etype=%s is_masked_op=1",
1456 is_scatter, is_scatter ? "scatterMasked" : "gatherMasked",
1457 num_elem, type2name(elem_bt));
1458 return false; // not supported
1459 }
1460 } else {
1461 // Check whether the normal gather/scatter node is supported for non-masked operation.
1462 if (!arch_supports_vector(is_scatter ? Op_StoreVectorScatter : Op_LoadVectorGather, num_elem, elem_bt, VecMaskNotUsed)) {
1463 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d etype=%s is_masked_op=0",
1464 is_scatter, is_scatter ? "scatter" : "gather",
1465 num_elem, type2name(elem_bt));
1466 return false; // not supported
1467 }
1468 }
1469
1470 // Check that the vector holding indices is supported by architecture
1471 // For sub-word gathers expander receive index array.
1472 if (!is_subword_type(elem_bt) && !arch_supports_vector(Op_LoadVector, num_elem, T_INT, VecMaskNotUsed)) {
1473 log_if_needed(" ** not supported: arity=%d op=%s/loadindex vlen=%d etype=int is_masked_op=%d",
1474 is_scatter, is_scatter ? "scatter" : "gather",
1475 num_elem, is_masked_op ? 1 : 0);
1476 return false; // not supported
1477 }
1478
1479 Node* base = argument(5);
1480 Node* offset = ConvL2X(argument(6));
1481
1482 // Save state and restore on bailout
1483 uint old_sp = sp();
1484 SafePointNode* old_map = clone_map();
1485
1486 Node* addr = make_unsafe_address(base, offset, elem_bt, true);
1487
1488 const TypePtr *addr_type = gvn().type(addr)->isa_ptr();
1489 const TypeAryPtr* arr_type = addr_type->isa_aryptr();
1490
1491 // The array must be consistent with vector type
1492 if (arr_type == nullptr || (arr_type != nullptr && !elem_consistent_with_arr(elem_bt, arr_type, false))) {
1493 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d etype=%s atype=%s ismask=no",
1494 is_scatter, is_scatter ? "scatter" : "gather",
1495 num_elem, type2name(elem_bt), type2name(arr_type->elem()->array_element_basic_type()));
1496 set_map(old_map);
1497 set_sp(old_sp);
1498 return false;
1499 }
1500
1501 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1502 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1503 ciKlass* vbox_idx_klass = vector_idx_klass->const_oop()->as_instance()->java_lang_Class_klass();
1504 if (vbox_idx_klass == nullptr) {
1505 set_map(old_map);
1506 set_sp(old_sp);
1507 return false;
1508 }
1509
1510 Node* index_vect = nullptr;
1511 const TypeInstPtr* vbox_idx_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_idx_klass);
1512 if (!is_subword_type(elem_bt)) {
1513 index_vect = unbox_vector(argument(8), vbox_idx_type, T_INT, num_elem);
1514 if (index_vect == nullptr) {
1515 set_map(old_map);
1516 set_sp(old_sp);
1517 return false;
1518 }
1519 }
1520
1521 Node* mask = nullptr;
1522 if (is_masked_op) {
1523 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1524 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1525 mask = unbox_vector(is_scatter ? argument(10) : argument(9), mbox_type, elem_bt, num_elem);
1526 if (mask == nullptr) {
1527 log_if_needed(" ** unbox failed mask=%s",
1528 is_scatter ? NodeClassNames[argument(10)->Opcode()]
1529 : NodeClassNames[argument(9)->Opcode()]);
1530 set_map(old_map);
1531 set_sp(old_sp);
1532 return false;
1533 }
1534 }
1535
1536 const TypeVect* vector_type = TypeVect::make(elem_bt, num_elem);
1537 if (is_scatter) {
1538 Node* val = unbox_vector(argument(9), vbox_type, elem_bt, num_elem);
1539 if (val == nullptr) {
1540 set_map(old_map);
1541 set_sp(old_sp);
1542 return false; // operand unboxing failed
1543 }
1544 set_all_memory(reset_memory());
1545
1546 Node* vstore = nullptr;
1547 if (mask != nullptr) {
1548 vstore = gvn().transform(new StoreVectorScatterMaskedNode(control(), memory(addr), addr, addr_type, val, index_vect, mask));
1549 } else {
1550 vstore = gvn().transform(new StoreVectorScatterNode(control(), memory(addr), addr, addr_type, val, index_vect));
1551 }
1552 set_memory(vstore, addr_type);
1553 } else {
1554 Node* vload = nullptr;
1555 Node* index = argument(11);
1556 Node* indexMap = argument(12);
1557 Node* indexM = argument(13);
1558 if (mask != nullptr) {
1559 if (is_subword_type(elem_bt)) {
1560 Node* index_arr_base = array_element_address(indexMap, indexM, T_INT);
1561 vload = gvn().transform(new LoadVectorGatherMaskedNode(control(), memory(addr), addr, addr_type, vector_type, index_arr_base, mask, index));
1562 } else {
1563 vload = gvn().transform(new LoadVectorGatherMaskedNode(control(), memory(addr), addr, addr_type, vector_type, index_vect, mask));
1564 }
1565 } else {
1566 if (is_subword_type(elem_bt)) {
1567 Node* index_arr_base = array_element_address(indexMap, indexM, T_INT);
1568 vload = gvn().transform(new LoadVectorGatherNode(control(), memory(addr), addr, addr_type, vector_type, index_arr_base, index));
1569 } else {
1570 vload = gvn().transform(new LoadVectorGatherNode(control(), memory(addr), addr, addr_type, vector_type, index_vect));
1571 }
1572 }
1573 Node* box = box_vector(vload, vbox_type, elem_bt, num_elem);
1574 set_result(box);
1575 }
1576
1577 destruct_map_clone(old_map);
1578
1579 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1580 return true;
1581 }
1582
1583 // public static
1584 // <V extends Vector<E>,
1585 // M extends VectorMask<E>,
1586 // E>
1587 // long reductionCoerced(int oprId, Class<? extends V> vectorClass, Class<? extends M> maskClass,
1588 // Class<E> elementType, int length, V v, M m,
1589 // ReductionOperation<V, M> defaultImpl)
1590 bool LibraryCallKit::inline_vector_reduction() {
1591 const TypeInt* opr = gvn().type(argument(0))->isa_int();
1592 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
1593 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
1594 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
1595 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
1596
1597 if (opr == nullptr || vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr ||
1598 !opr->is_con() || vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
1599 log_if_needed(" ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
1600 NodeClassNames[argument(0)->Opcode()],
1601 NodeClassNames[argument(1)->Opcode()],
1602 NodeClassNames[argument(3)->Opcode()],
1603 NodeClassNames[argument(4)->Opcode()]);
1604 return false; // not enough info for intrinsification
1605 }
1606 if (!is_klass_initialized(vector_klass)) {
1607 log_if_needed(" ** klass argument not initialized");
1608 return false;
1609 }
1610 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1611 if (!elem_type->is_primitive_type()) {
1612 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1613 return false; // should be primitive type
1614 }
1615
1616 const Type* vmask_type = gvn().type(argument(6));
1617 bool is_masked_op = vmask_type != TypePtr::NULL_PTR;
1618 if (is_masked_op) {
1619 if (mask_klass == nullptr || mask_klass->const_oop() == nullptr) {
1620 log_if_needed(" ** missing constant: maskclass=%s", NodeClassNames[argument(2)->Opcode()]);
1621 return false; // not enough info for intrinsification
1622 }
1623
1624 if (!is_klass_initialized(mask_klass)) {
1625 log_if_needed(" ** mask klass argument not initialized");
1626 return false;
1627 }
1628
1629 if (vmask_type->maybe_null()) {
1630 log_if_needed(" ** null mask values are not allowed for masked op");
1631 return false;
1632 }
1633 }
1634
1635 BasicType elem_bt = elem_type->basic_type();
1636 int num_elem = vlen->get_con();
1637 int opc = VectorSupport::vop2ideal(opr->get_con(), elem_bt);
1638 int sopc = ReductionNode::opcode(opc, elem_bt);
1639
1640 // When using mask, mask use type needs to be VecMaskUseLoad.
1641 if (!arch_supports_vector(sopc, num_elem, elem_bt, is_masked_op ? VecMaskUseLoad : VecMaskNotUsed)) {
1642 log_if_needed(" ** not supported: arity=1 op=%d/reduce vlen=%d etype=%s is_masked_op=%d",
1643 sopc, num_elem, type2name(elem_bt), is_masked_op ? 1 : 0);
1644 return false;
1645 }
1646
1647 // Return true if current platform has implemented the masked operation with predicate feature.
1648 bool use_predicate = is_masked_op && arch_supports_vector(sopc, num_elem, elem_bt, VecMaskUsePred);
1649 if (is_masked_op && !use_predicate && !arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad)) {
1650 log_if_needed(" ** not supported: arity=1 op=%d/reduce vlen=%d etype=%s is_masked_op=1",
1651 sopc, num_elem, type2name(elem_bt));
1652 return false;
1653 }
1654
1655 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1656 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1657
1658 Node* opd = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
1659 if (opd == nullptr) {
1660 return false; // operand unboxing failed
1661 }
1662
1663 Node* mask = nullptr;
1664 if (is_masked_op) {
1665 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1666 assert(is_vector_mask(mbox_klass), "argument(2) should be a mask class");
1667 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1668 mask = unbox_vector(argument(6), mbox_type, elem_bt, num_elem);
1669 if (mask == nullptr) {
1670 log_if_needed(" ** unbox failed mask=%s",
1671 NodeClassNames[argument(6)->Opcode()]);
1672 return false;
1673 }
1674 }
1675
1676 Node* init = ReductionNode::make_identity_con_scalar(gvn(), opc, elem_bt);
1677 Node* value = opd;
1678
1679 assert(mask != nullptr || !is_masked_op, "Masked op needs the mask value never null");
1680 if (mask != nullptr && !use_predicate) {
1681 Node* reduce_identity = gvn().transform(VectorNode::scalar2vector(init, num_elem, elem_bt));
1682 value = gvn().transform(new VectorBlendNode(reduce_identity, value, mask));
1683 }
1684
1685 // Make an unordered Reduction node. This affects only AddReductionVF/VD and MulReductionVF/VD,
1686 // as these operations are allowed to be associative (not requiring strict order) in VectorAPI.
1687 value = ReductionNode::make(opc, nullptr, init, value, elem_bt, /* requires_strict_order */ false);
1688
1689 if (mask != nullptr && use_predicate) {
1690 value->add_req(mask);
1691 value->add_flag(Node::Flag_is_predicated_vector);
1692 }
1693
1694 value = gvn().transform(value);
1695
1696 Node* bits = nullptr;
1697 switch (elem_bt) {
1698 case T_BYTE:
1699 case T_SHORT:
1700 case T_INT: {
1701 bits = gvn().transform(new ConvI2LNode(value));
1702 break;
1703 }
1704 case T_FLOAT: {
1705 value = gvn().transform(new MoveF2INode(value));
1706 bits = gvn().transform(new ConvI2LNode(value));
1707 break;
1708 }
1709 case T_DOUBLE: {
1710 bits = gvn().transform(new MoveD2LNode(value));
1711 break;
1712 }
1713 case T_LONG: {
1714 bits = value; // no conversion needed
1715 break;
1716 }
1717 default: fatal("%s", type2name(elem_bt));
1718 }
1719 set_result(bits);
1720 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1721 return true;
1722 }
1723
1724 // public static <V> boolean test(int cond, Class<?> vectorClass, Class<?> elementType, int vlen,
1725 // V v1, V v2,
1726 // BiFunction<V, V, Boolean> defaultImpl)
1727 //
1728 bool LibraryCallKit::inline_vector_test() {
1729 const TypeInt* cond = gvn().type(argument(0))->isa_int();
1730 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
1731 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
1732 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
1733
1734 if (cond == nullptr || vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr ||
1735 !cond->is_con() || vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
1736 log_if_needed(" ** missing constant: cond=%s vclass=%s etype=%s vlen=%s",
1737 NodeClassNames[argument(0)->Opcode()],
1738 NodeClassNames[argument(1)->Opcode()],
1739 NodeClassNames[argument(2)->Opcode()],
1740 NodeClassNames[argument(3)->Opcode()]);
1741 return false; // not enough info for intrinsification
1742 }
1743 if (!is_klass_initialized(vector_klass)) {
1744 log_if_needed(" ** klass argument not initialized");
1745 return false;
1746 }
1747 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1748 if (!elem_type->is_primitive_type()) {
1749 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1750 return false; // should be primitive type
1751 }
1752 BasicType elem_bt = elem_type->basic_type();
1753 int num_elem = vlen->get_con();
1754 BoolTest::mask booltest = (BoolTest::mask)cond->get_con();
1755 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1756 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1757
1758 if (!arch_supports_vector(Op_VectorTest, num_elem, elem_bt, is_vector_mask(vbox_klass) ? VecMaskUseLoad : VecMaskNotUsed)) {
1759 log_if_needed(" ** not supported: arity=2 op=test/%d vlen=%d etype=%s ismask=%d",
1760 cond->get_con(), num_elem, type2name(elem_bt),
1761 is_vector_mask(vbox_klass));
1762 return false;
1763 }
1764
1765 Node* opd1 = unbox_vector(argument(4), vbox_type, elem_bt, num_elem);
1766 Node* opd2;
1767 if (Matcher::vectortest_needs_second_argument(booltest == BoolTest::overflow,
1768 opd1->bottom_type()->isa_vectmask())) {
1769 opd2 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
1770 } else {
1771 opd2 = opd1;
1772 }
1773 if (opd1 == nullptr || opd2 == nullptr) {
1774 return false; // operand unboxing failed
1775 }
1776
1777 Node* cmp = gvn().transform(new VectorTestNode(opd1, opd2, booltest));
1778 BoolTest::mask test = Matcher::vectortest_mask(booltest == BoolTest::overflow,
1779 opd1->bottom_type()->isa_vectmask(), num_elem);
1780 Node* bol = gvn().transform(new BoolNode(cmp, test));
1781 Node* res = gvn().transform(new CMoveINode(bol, gvn().intcon(0), gvn().intcon(1), TypeInt::BOOL));
1782
1783 set_result(res);
1784 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1785 return true;
1786 }
1787
1788 // public static
1789 // <V extends Vector<E>,
1790 // M extends VectorMask<E>,
1791 // E>
1792 // V blend(Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType, int vlen,
1793 // V v1, V v2, M m,
1794 // VectorBlendOp<V, M, E> defaultImpl)
1795 bool LibraryCallKit::inline_vector_blend() {
1796 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
1797 const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
1798 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
1799 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
1800
1801 if (mask_klass == nullptr || vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr) {
1802 return false; // dead code
1803 }
1804 if (mask_klass->const_oop() == nullptr || vector_klass->const_oop() == nullptr ||
1805 elem_klass->const_oop() == nullptr || !vlen->is_con()) {
1806 log_if_needed(" ** missing constant: vclass=%s mclass=%s etype=%s vlen=%s",
1807 NodeClassNames[argument(0)->Opcode()],
1808 NodeClassNames[argument(1)->Opcode()],
1809 NodeClassNames[argument(2)->Opcode()],
1810 NodeClassNames[argument(3)->Opcode()]);
1811 return false; // not enough info for intrinsification
1812 }
1813 if (!is_klass_initialized(vector_klass) || !is_klass_initialized(mask_klass)) {
1814 log_if_needed(" ** klass argument not initialized");
1815 return false;
1816 }
1817 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1818 if (!elem_type->is_primitive_type()) {
1819 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1820 return false; // should be primitive type
1821 }
1822 BasicType elem_bt = elem_type->basic_type();
1823 BasicType mask_bt = elem_bt;
1824 int num_elem = vlen->get_con();
1825
1826 if (!arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad)) {
1827 log_if_needed(" ** not supported: arity=2 op=blend vlen=%d etype=%s ismask=useload",
1828 num_elem, type2name(elem_bt));
1829 return false; // not supported
1830 }
1831 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1832 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1833
1834 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1835 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1836
1837 Node* v1 = unbox_vector(argument(4), vbox_type, elem_bt, num_elem);
1838 Node* v2 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
1839 Node* mask = unbox_vector(argument(6), mbox_type, mask_bt, num_elem);
1840
1841 if (v1 == nullptr || v2 == nullptr || mask == nullptr) {
1842 return false; // operand unboxing failed
1843 }
1844
1845 Node* blend = gvn().transform(new VectorBlendNode(v1, v2, mask));
1846
1847 Node* box = box_vector(blend, vbox_type, elem_bt, num_elem);
1848 set_result(box);
1849 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1850 return true;
1851 }
1852
1853 // public static
1854 // <V extends Vector<E>,
1855 // M extends VectorMask<E>,
1856 // E>
1857 // M compare(int cond, Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType, int vlen,
1858 // V v1, V v2, M m,
1859 // VectorCompareOp<V,M> defaultImpl)
1860 bool LibraryCallKit::inline_vector_compare() {
1861 const TypeInt* cond = gvn().type(argument(0))->isa_int();
1862 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
1863 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
1864 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
1865 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
1866
1867 if (cond == nullptr || vector_klass == nullptr || mask_klass == nullptr || elem_klass == nullptr || vlen == nullptr) {
1868 return false; // dead code
1869 }
1870 if (!cond->is_con() || vector_klass->const_oop() == nullptr || mask_klass->const_oop() == nullptr ||
1871 elem_klass->const_oop() == nullptr || !vlen->is_con()) {
1872 log_if_needed(" ** missing constant: cond=%s vclass=%s mclass=%s etype=%s vlen=%s",
1873 NodeClassNames[argument(0)->Opcode()],
1874 NodeClassNames[argument(1)->Opcode()],
1875 NodeClassNames[argument(2)->Opcode()],
1876 NodeClassNames[argument(3)->Opcode()],
1877 NodeClassNames[argument(4)->Opcode()]);
1878 return false; // not enough info for intrinsification
1879 }
1880 if (!is_klass_initialized(vector_klass) || !is_klass_initialized(mask_klass)) {
1881 log_if_needed(" ** klass argument not initialized");
1882 return false;
1883 }
1884 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1885 if (!elem_type->is_primitive_type()) {
1886 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1887 return false; // should be primitive type
1888 }
1889
1890 int num_elem = vlen->get_con();
1891 BasicType elem_bt = elem_type->basic_type();
1892 BasicType mask_bt = elem_bt;
1893
1894 if ((cond->get_con() & BoolTest::unsigned_compare) != 0) {
1895 if (!Matcher::supports_vector_comparison_unsigned(num_elem, elem_bt)) {
1896 log_if_needed(" ** not supported: unsigned comparison op=comp/%d vlen=%d etype=%s ismask=usestore",
1897 cond->get_con() & (BoolTest::unsigned_compare - 1), num_elem, type2name(elem_bt));
1898 return false;
1899 }
1900 }
1901
1902 if (!arch_supports_vector(Op_VectorMaskCmp, num_elem, elem_bt, VecMaskUseStore)) {
1903 log_if_needed(" ** not supported: arity=2 op=comp/%d vlen=%d etype=%s ismask=usestore",
1904 cond->get_con(), num_elem, type2name(elem_bt));
1905 return false;
1906 }
1907
1908 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1909 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1910
1911 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1912 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1913
1914 Node* v1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
1915 Node* v2 = unbox_vector(argument(6), vbox_type, elem_bt, num_elem);
1916
1917 bool is_masked_op = argument(7)->bottom_type() != TypePtr::NULL_PTR;
1918 Node* mask = is_masked_op ? unbox_vector(argument(7), mbox_type, elem_bt, num_elem) : nullptr;
1919 if (is_masked_op && mask == nullptr) {
1920 log_if_needed(" ** not supported: mask = null arity=2 op=comp/%d vlen=%d etype=%s ismask=usestore is_masked_op=1",
1921 cond->get_con(), num_elem, type2name(elem_bt));
1922 return false;
1923 }
1924
1925 bool use_predicate = is_masked_op && arch_supports_vector(Op_VectorMaskCmp, num_elem, elem_bt, VecMaskUsePred);
1926 if (is_masked_op && !use_predicate && !arch_supports_vector(Op_AndV, num_elem, elem_bt, VecMaskUseLoad)) {
1927 log_if_needed(" ** not supported: arity=2 op=comp/%d vlen=%d etype=%s ismask=usestore is_masked_op=1",
1928 cond->get_con(), num_elem, type2name(elem_bt));
1929 return false;
1930 }
1931
1932 if (v1 == nullptr || v2 == nullptr) {
1933 return false; // operand unboxing failed
1934 }
1935 BoolTest::mask pred = (BoolTest::mask)cond->get_con();
1936 ConINode* pred_node = (ConINode*)gvn().makecon(cond);
1937
1938 const TypeVect* vmask_type = TypeVect::makemask(mask_bt, num_elem);
1939 Node* operation = new VectorMaskCmpNode(pred, v1, v2, pred_node, vmask_type);
1940
1941 if (is_masked_op) {
1942 if (use_predicate) {
1943 operation->add_req(mask);
1944 operation->add_flag(Node::Flag_is_predicated_vector);
1945 } else {
1946 operation = gvn().transform(operation);
1947 operation = VectorNode::make(Op_AndV, operation, mask, vmask_type);
1948 }
1949 }
1950
1951 operation = gvn().transform(operation);
1952
1953 Node* box = box_vector(operation, mbox_type, mask_bt, num_elem);
1954 set_result(box);
1955 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1956 return true;
1957 }
1958
1959 // public static
1960 // <V extends Vector<E>,
1961 // Sh extends VectorShuffle<E>,
1962 // M extends VectorMask<E>,
1963 // E>
1964 // V rearrangeOp(Class<? extends V> vectorClass, Class<Sh> shuffleClass, Class<M> maskClass, Class<E> elementType, int vlen,
1965 // V v1, Sh sh, M m,
1966 // VectorRearrangeOp<V, Sh, M, E> defaultImpl)
1967 bool LibraryCallKit::inline_vector_rearrange() {
1968 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
1969 const TypeInstPtr* shuffle_klass = gvn().type(argument(1))->isa_instptr();
1970 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
1971 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
1972 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
1973
1974 if (vector_klass == nullptr || shuffle_klass == nullptr || elem_klass == nullptr || vlen == nullptr) {
1975 return false; // dead code
1976 }
1977 if (shuffle_klass->const_oop() == nullptr ||
1978 vector_klass->const_oop() == nullptr ||
1979 elem_klass->const_oop() == nullptr ||
1980 !vlen->is_con()) {
1981 log_if_needed(" ** missing constant: vclass=%s sclass=%s etype=%s vlen=%s",
1982 NodeClassNames[argument(0)->Opcode()],
1983 NodeClassNames[argument(1)->Opcode()],
1984 NodeClassNames[argument(3)->Opcode()],
1985 NodeClassNames[argument(4)->Opcode()]);
1986 return false; // not enough info for intrinsification
1987 }
1988 if (!is_klass_initialized(vector_klass) ||
1989 !is_klass_initialized(shuffle_klass)) {
1990 log_if_needed(" ** klass argument not initialized");
1991 return false;
1992 }
1993 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1994 if (!elem_type->is_primitive_type()) {
1995 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1996 return false; // should be primitive type
1997 }
1998 BasicType elem_bt = elem_type->basic_type();
1999 BasicType shuffle_bt = elem_bt;
2000 int num_elem = vlen->get_con();
2001
2002 if (!arch_supports_vector(Op_VectorLoadShuffle, num_elem, elem_bt, VecMaskNotUsed)) {
2003 log_if_needed(" ** not supported: arity=0 op=load/shuffle vlen=%d etype=%s ismask=no",
2004 num_elem, type2name(elem_bt));
2005 return false; // not supported
2006 }
2007
2008 bool is_masked_op = argument(7)->bottom_type() != TypePtr::NULL_PTR;
2009 bool use_predicate = is_masked_op;
2010 if (is_masked_op &&
2011 (mask_klass == nullptr ||
2012 mask_klass->const_oop() == nullptr ||
2013 !is_klass_initialized(mask_klass))) {
2014 log_if_needed(" ** mask_klass argument not initialized");
2015 }
2016 VectorMaskUseType checkFlags = (VectorMaskUseType)(is_masked_op ? (VecMaskUseLoad | VecMaskUsePred) : VecMaskNotUsed);
2017 if (!arch_supports_vector(Op_VectorRearrange, num_elem, elem_bt, checkFlags)) {
2018 use_predicate = false;
2019 if(!is_masked_op ||
2020 (!arch_supports_vector(Op_VectorRearrange, num_elem, elem_bt, VecMaskNotUsed) ||
2021 !arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad) ||
2022 !arch_supports_vector(Op_Replicate, num_elem, elem_bt, VecMaskNotUsed))) {
2023 log_if_needed(" ** not supported: arity=2 op=shuffle/rearrange vlen=%d etype=%s ismask=no",
2024 num_elem, type2name(elem_bt));
2025 return false; // not supported
2026 }
2027 }
2028 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2029 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2030
2031 ciKlass* shbox_klass = shuffle_klass->const_oop()->as_instance()->java_lang_Class_klass();
2032 const TypeInstPtr* shbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, shbox_klass);
2033
2034 Node* v1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
2035 Node* shuffle = unbox_vector(argument(6), shbox_type, shuffle_bt, num_elem);
2036
2037 if (v1 == nullptr || shuffle == nullptr) {
2038 return false; // operand unboxing failed
2039 }
2040
2041 Node* mask = nullptr;
2042 if (is_masked_op) {
2043 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
2044 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
2045 mask = unbox_vector(argument(7), mbox_type, elem_bt, num_elem);
2046 if (mask == nullptr) {
2047 log_if_needed(" ** not supported: arity=3 op=shuffle/rearrange vlen=%d etype=%s ismask=useload is_masked_op=1",
2048 num_elem, type2name(elem_bt));
2049 return false;
2050 }
2051 }
2052
2053 Node* rearrange = new VectorRearrangeNode(v1, shuffle);
2054 if (is_masked_op) {
2055 if (use_predicate) {
2056 rearrange->add_req(mask);
2057 rearrange->add_flag(Node::Flag_is_predicated_vector);
2058 } else {
2059 const TypeVect* vt = v1->bottom_type()->is_vect();
2060 rearrange = gvn().transform(rearrange);
2061 Node* zero = gvn().makecon(Type::get_zero_type(elem_bt));
2062 Node* zerovec = gvn().transform(VectorNode::scalar2vector(zero, num_elem, elem_bt));
2063 rearrange = new VectorBlendNode(zerovec, rearrange, mask);
2064 }
2065 }
2066 rearrange = gvn().transform(rearrange);
2067
2068 Node* box = box_vector(rearrange, vbox_type, elem_bt, num_elem);
2069 set_result(box);
2070 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2071 return true;
2072 }
2073
2074 static address get_vector_math_address(int vop, int bits, BasicType bt, char* name_ptr, int name_len) {
2075 address addr = nullptr;
2076 assert(UseVectorStubs, "sanity");
2077 assert(name_ptr != nullptr, "unexpected");
2078 assert((vop >= VectorSupport::VECTOR_OP_MATH_START) && (vop <= VectorSupport::VECTOR_OP_MATH_END), "unexpected");
2079 int op = vop - VectorSupport::VECTOR_OP_MATH_START;
2080
2081 switch(bits) {
2082 case 64: //fallthough
2083 case 128: //fallthough
2084 case 256: //fallthough
2085 case 512:
2086 if (bt == T_FLOAT) {
2087 snprintf(name_ptr, name_len, "vector_%s_float_%dbits_fixed", VectorSupport::mathname[op], bits);
2088 addr = StubRoutines::_vector_f_math[exact_log2(bits/64)][op];
2089 } else {
2090 assert(bt == T_DOUBLE, "must be FP type only");
2091 snprintf(name_ptr, name_len, "vector_%s_double_%dbits_fixed", VectorSupport::mathname[op], bits);
2092 addr = StubRoutines::_vector_d_math[exact_log2(bits/64)][op];
2093 }
2094 break;
2095 default:
2096 if (!Matcher::supports_scalable_vector() || !Matcher::vector_size_supported(bt, bits/type2aelembytes(bt)) ) {
2097 snprintf(name_ptr, name_len, "invalid");
2098 addr = nullptr;
2099 Unimplemented();
2100 }
2101 break;
2102 }
2103
2104 if (addr == nullptr && Matcher::supports_scalable_vector()) {
2105 if (bt == T_FLOAT) {
2106 snprintf(name_ptr, name_len, "vector_%s_float_%dbits_scalable", VectorSupport::mathname[op], bits);
2107 addr = StubRoutines::_vector_f_math[VectorSupport::VEC_SIZE_SCALABLE][op];
2108 } else {
2109 assert(bt == T_DOUBLE, "must be FP type only");
2110 snprintf(name_ptr, name_len, "vector_%s_double_%dbits_scalable", VectorSupport::mathname[op], bits);
2111 addr = StubRoutines::_vector_d_math[VectorSupport::VEC_SIZE_SCALABLE][op];
2112 }
2113 }
2114
2115 return addr;
2116 }
2117
2118 // public static
2119 // <V extends Vector<E>,
2120 // M extends VectorMask<E>,
2121 // E>
2122 // V selectFromOp(Class<? extends V> vClass, Class<M> mClass, Class<E> eClass,
2123 // int length, V v1, V v2, M m,
2124 // VectorSelectFromOp<V, M> defaultImpl)
2125 bool LibraryCallKit::inline_vector_select_from() {
2126 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
2127 const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
2128 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
2129 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
2130
2131 if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr ||
2132 vector_klass->const_oop() == nullptr ||
2133 elem_klass->const_oop() == nullptr ||
2134 !vlen->is_con()) {
2135 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s",
2136 NodeClassNames[argument(0)->Opcode()],
2137 NodeClassNames[argument(2)->Opcode()],
2138 NodeClassNames[argument(3)->Opcode()]);
2139 return false; // not enough info for intrinsification
2140 }
2141 if (!is_klass_initialized(vector_klass)) {
2142 log_if_needed(" ** klass argument not initialized");
2143 return false;
2144 }
2145 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2146 if (!elem_type->is_primitive_type()) {
2147 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2148 return false; // should be primitive type
2149 }
2150 BasicType elem_bt = elem_type->basic_type();
2151 int num_elem = vlen->get_con();
2152 if (!is_power_of_2(num_elem)) {
2153 log_if_needed(" ** vlen not power of two=%d", num_elem);
2154 return false;
2155 }
2156
2157 int cast_vopc = VectorCastNode::opcode(-1, elem_bt); // from vector of type elem_bt
2158 if (!arch_supports_vector(Op_VectorLoadShuffle, num_elem, elem_bt, VecMaskNotUsed)||
2159 !arch_supports_vector(Op_AndV, num_elem, T_BYTE, VecMaskNotUsed) ||
2160 !arch_supports_vector(Op_Replicate, num_elem, T_BYTE, VecMaskNotUsed) ||
2161 !arch_supports_vector(cast_vopc, num_elem, T_BYTE, VecMaskNotUsed)) {
2162 log_if_needed(" ** not supported: arity=0 op=selectFrom vlen=%d etype=%s ismask=no",
2163 num_elem, type2name(elem_bt));
2164 return false; // not supported
2165 }
2166
2167 bool is_masked_op = argument(6)->bottom_type() != TypePtr::NULL_PTR;
2168 bool use_predicate = is_masked_op;
2169 if (is_masked_op &&
2170 (mask_klass == nullptr ||
2171 mask_klass->const_oop() == nullptr ||
2172 !is_klass_initialized(mask_klass))) {
2173 log_if_needed(" ** mask_klass argument not initialized");
2174 return false; // not supported
2175 }
2176 VectorMaskUseType checkFlags = (VectorMaskUseType)(is_masked_op ? (VecMaskUseLoad | VecMaskUsePred) : VecMaskNotUsed);
2177 if (!arch_supports_vector(Op_VectorRearrange, num_elem, elem_bt, checkFlags)) {
2178 use_predicate = false;
2179 if(!is_masked_op ||
2180 (!arch_supports_vector(Op_VectorRearrange, num_elem, elem_bt, VecMaskNotUsed) ||
2181 !arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad) ||
2182 !arch_supports_vector(Op_Replicate, num_elem, elem_bt, VecMaskNotUsed))) {
2183 log_if_needed(" ** not supported: op=selectFrom vlen=%d etype=%s is_masked_op=%d",
2184 num_elem, type2name(elem_bt), is_masked_op);
2185 return false; // not supported
2186 }
2187 }
2188 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2189 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2190
2191 // v1 is the index vector
2192 Node* v1 = unbox_vector(argument(4), vbox_type, elem_bt, num_elem);
2193 // v2 is the vector being rearranged
2194 Node* v2 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
2195
2196 if (v1 == nullptr) {
2197 log_if_needed(" ** unbox failed v1=%s", NodeClassNames[argument(4)->Opcode()]);
2198 return false; // operand unboxing failed
2199 }
2200
2201 if (v2 == nullptr) {
2202 log_if_needed(" ** unbox failed v2=%s", NodeClassNames[argument(5)->Opcode()]);
2203 return false; // operand unboxing failed
2204 }
2205
2206 Node* mask = nullptr;
2207 if (is_masked_op) {
2208 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
2209 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
2210 mask = unbox_vector(argument(6), mbox_type, elem_bt, num_elem);
2211 if (mask == nullptr) {
2212 log_if_needed(" ** unbox failed mask=%s", NodeClassNames[argument(6)->Opcode()]);
2213 return false;
2214 }
2215 }
2216
2217 // cast index vector from elem_bt vector to byte vector
2218 const TypeVect* byte_vt = TypeVect::make(T_BYTE, num_elem);
2219 Node* byte_shuffle = gvn().transform(VectorCastNode::make(cast_vopc, v1, T_BYTE, num_elem));
2220
2221 // wrap the byte vector lanes to (num_elem - 1) to form the shuffle vector where num_elem is vector length
2222 // this is a simple AND operation as we come here only for power of two vector length
2223 Node* mod_val = gvn().makecon(TypeInt::make(num_elem-1));
2224 Node* bcast_mod = gvn().transform(VectorNode::scalar2vector(mod_val, num_elem, T_BYTE));
2225 byte_shuffle = gvn().transform(VectorNode::make(Op_AndV, byte_shuffle, bcast_mod, byte_vt));
2226
2227 // load the shuffle to use in rearrange
2228 const TypeVect* shuffle_vt = TypeVect::make(elem_bt, num_elem);
2229 Node* load_shuffle = gvn().transform(new VectorLoadShuffleNode(byte_shuffle, shuffle_vt));
2230
2231 // and finally rearrange
2232 Node* rearrange = new VectorRearrangeNode(v2, load_shuffle);
2233 if (is_masked_op) {
2234 if (use_predicate) {
2235 // masked rearrange is supported so use that directly
2236 rearrange->add_req(mask);
2237 rearrange->add_flag(Node::Flag_is_predicated_vector);
2238 } else {
2239 // masked rearrange is not supported so emulate usig blend
2240 const TypeVect* vt = v1->bottom_type()->is_vect();
2241 rearrange = gvn().transform(rearrange);
2242
2243 // create a zero vector with each lane element set as zero
2244 Node* zero = gvn().makecon(Type::get_zero_type(elem_bt));
2245 Node* zerovec = gvn().transform(VectorNode::scalar2vector(zero, num_elem, elem_bt));
2246
2247 // For each lane for which mask is set, blend in the rearranged lane into zero vector
2248 rearrange = new VectorBlendNode(zerovec, rearrange, mask);
2249 }
2250 }
2251 rearrange = gvn().transform(rearrange);
2252
2253 // box the result
2254 Node* box = box_vector(rearrange, vbox_type, elem_bt, num_elem);
2255 set_result(box);
2256
2257 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2258 return true;
2259 }
2260
2261 Node* LibraryCallKit::gen_call_to_vector_math(int vector_api_op_id, BasicType bt, int num_elem, Node* opd1, Node* opd2) {
2262 assert(UseVectorStubs, "sanity");
2263 assert(vector_api_op_id >= VectorSupport::VECTOR_OP_MATH_START && vector_api_op_id <= VectorSupport::VECTOR_OP_MATH_END, "need valid op id");
2264 assert(opd1 != nullptr, "must not be null");
2265 const TypeVect* vt = TypeVect::make(bt, num_elem);
2266 const TypeFunc* call_type = OptoRuntime::Math_Vector_Vector_Type(opd2 != nullptr ? 2 : 1, vt, vt);
2267 char name[100] = "";
2268
2269 // Get address for vector math method.
2270 address addr = get_vector_math_address(vector_api_op_id, vt->length_in_bytes() * BitsPerByte, bt, name, 100);
2271
2272 if (addr == nullptr) {
2273 return nullptr;
2274 }
2275
2276 assert(name[0] != '\0', "name must not be null");
2277 Node* operation = make_runtime_call(RC_VECTOR,
2278 call_type,
2279 addr,
2280 name,
2281 TypePtr::BOTTOM,
2282 opd1,
2283 opd2);
2284 return gvn().transform(new ProjNode(gvn().transform(operation), TypeFunc::Parms));
2285 }
2286
2287 // public static
2288 // <V extends Vector<E>,
2289 // M extends VectorMask<E>,
2290 // E>
2291 // V broadcastInt(int opr, Class<? extends V> vectorClass, Class<? extends M> maskClass,
2292 // Class<E> elementType, int length,
2293 // V v, int n, M m,
2294 // VectorBroadcastIntOp<V, M> defaultImpl)
2295 bool LibraryCallKit::inline_vector_broadcast_int() {
2296 const TypeInt* opr = gvn().type(argument(0))->isa_int();
2297 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
2298 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
2299 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
2300 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
2301
2302 if (opr == nullptr || vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr) {
2303 return false; // dead code
2304 }
2305 if (!opr->is_con() || vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
2306 log_if_needed(" ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
2307 NodeClassNames[argument(0)->Opcode()],
2308 NodeClassNames[argument(1)->Opcode()],
2309 NodeClassNames[argument(3)->Opcode()],
2310 NodeClassNames[argument(4)->Opcode()]);
2311 return false; // not enough info for intrinsification
2312 }
2313 if (!is_klass_initialized(vector_klass)) {
2314 log_if_needed(" ** klass argument not initialized");
2315 return false;
2316 }
2317
2318 const Type* vmask_type = gvn().type(argument(7));
2319 bool is_masked_op = vmask_type != TypePtr::NULL_PTR;
2320 if (is_masked_op) {
2321 if (mask_klass == nullptr || mask_klass->const_oop() == nullptr) {
2322 log_if_needed(" ** missing constant: maskclass=%s", NodeClassNames[argument(2)->Opcode()]);
2323 return false; // not enough info for intrinsification
2324 }
2325
2326 if (!is_klass_initialized(mask_klass)) {
2327 log_if_needed(" ** mask klass argument not initialized");
2328 return false;
2329 }
2330
2331 if (vmask_type->maybe_null()) {
2332 log_if_needed(" ** null mask values are not allowed for masked op");
2333 return false;
2334 }
2335 }
2336
2337 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2338 if (!elem_type->is_primitive_type()) {
2339 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2340 return false; // should be primitive type
2341 }
2342
2343 int num_elem = vlen->get_con();
2344 BasicType elem_bt = elem_type->basic_type();
2345 int opc = VectorSupport::vop2ideal(opr->get_con(), elem_bt);
2346
2347 bool is_shift = VectorNode::is_shift_opcode(opc);
2348 bool is_rotate = VectorNode::is_rotate_opcode(opc);
2349
2350 if (opc == 0 || (!is_shift && !is_rotate)) {
2351 log_if_needed(" ** operation not supported: op=%d bt=%s", opr->get_con(), type2name(elem_bt));
2352 return false; // operation not supported
2353 }
2354
2355 int sopc = VectorNode::opcode(opc, elem_bt);
2356 if (sopc == 0) {
2357 log_if_needed(" ** operation not supported: opc=%s bt=%s", NodeClassNames[opc], type2name(elem_bt));
2358 return false; // operation not supported
2359 }
2360
2361 Node* cnt = argument(6);
2362 const TypeInt* cnt_type = cnt->bottom_type()->isa_int();
2363
2364 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2365 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2366
2367 // If CPU supports vector constant rotate instructions pass it directly
2368 bool is_const_rotate = is_rotate && cnt_type && cnt_type->is_con() &&
2369 Matcher::supports_vector_constant_rotates(cnt_type->get_con());
2370 bool has_scalar_args = is_rotate ? !is_const_rotate : true;
2371
2372 VectorMaskUseType checkFlags = (VectorMaskUseType)(is_masked_op ? (VecMaskUseLoad | VecMaskUsePred) : VecMaskNotUsed);
2373 bool use_predicate = is_masked_op;
2374
2375 if (!arch_supports_vector(sopc, num_elem, elem_bt, checkFlags, has_scalar_args)) {
2376 use_predicate = false;
2377 if (!is_masked_op ||
2378 (!arch_supports_vector(sopc, num_elem, elem_bt, VecMaskNotUsed, has_scalar_args) ||
2379 !arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad))) {
2380
2381 log_if_needed(" ** not supported: arity=0 op=int/%d vlen=%d etype=%s is_masked_op=%d",
2382 sopc, num_elem, type2name(elem_bt), is_masked_op ? 1 : 0);
2383 return false; // not supported
2384 }
2385 }
2386
2387 Node* opd1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
2388 Node* opd2 = nullptr;
2389 if (is_shift) {
2390 opd2 = vector_shift_count(cnt, opc, elem_bt, num_elem);
2391 } else {
2392 assert(is_rotate, "unexpected operation");
2393 if (!is_const_rotate) {
2394 cnt = elem_bt == T_LONG ? gvn().transform(new ConvI2LNode(cnt)) : cnt;
2395 opd2 = gvn().transform(VectorNode::scalar2vector(cnt, num_elem, elem_bt));
2396 } else {
2397 // Constant shift value.
2398 opd2 = cnt;
2399 }
2400 }
2401
2402 if (opd1 == nullptr || opd2 == nullptr) {
2403 return false;
2404 }
2405
2406 Node* mask = nullptr;
2407 if (is_masked_op) {
2408 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
2409 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
2410 mask = unbox_vector(argument(7), mbox_type, elem_bt, num_elem);
2411 if (mask == nullptr) {
2412 log_if_needed(" ** unbox failed mask=%s", NodeClassNames[argument(7)->Opcode()]);
2413 return false;
2414 }
2415 }
2416
2417 Node* operation = VectorNode::make(opc, opd1, opd2, num_elem, elem_bt);
2418 if (is_masked_op && mask != nullptr) {
2419 if (use_predicate) {
2420 operation->add_req(mask);
2421 operation->add_flag(Node::Flag_is_predicated_vector);
2422 } else {
2423 operation = gvn().transform(operation);
2424 operation = new VectorBlendNode(opd1, operation, mask);
2425 }
2426 }
2427 operation = gvn().transform(operation);
2428 Node* vbox = box_vector(operation, vbox_type, elem_bt, num_elem);
2429 set_result(vbox);
2430 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2431 return true;
2432 }
2433
2434 // public static <VOUT extends VectorPayload,
2435 // VIN extends VectorPayload,
2436 // S extends VectorSpecies>
2437 // VOUT convert(int oprId,
2438 // Class<?> fromVectorClass, Class<?> fromElementType, int fromVLen,
2439 // Class<?> toVectorClass, Class<?> toElementType, int toVLen,
2440 // VIN v, S s,
2441 // VectorConvertOp<VOUT, VIN, S> defaultImpl)
2442 //
2443 bool LibraryCallKit::inline_vector_convert() {
2444 const TypeInt* opr = gvn().type(argument(0))->isa_int();
2445
2446 const TypeInstPtr* vector_klass_from = gvn().type(argument(1))->isa_instptr();
2447 const TypeInstPtr* elem_klass_from = gvn().type(argument(2))->isa_instptr();
2448 const TypeInt* vlen_from = gvn().type(argument(3))->isa_int();
2449
2450 const TypeInstPtr* vector_klass_to = gvn().type(argument(4))->isa_instptr();
2451 const TypeInstPtr* elem_klass_to = gvn().type(argument(5))->isa_instptr();
2452 const TypeInt* vlen_to = gvn().type(argument(6))->isa_int();
2453
2454 if (opr == nullptr ||
2455 vector_klass_from == nullptr || elem_klass_from == nullptr || vlen_from == nullptr ||
2456 vector_klass_to == nullptr || elem_klass_to == nullptr || vlen_to == nullptr) {
2457 return false; // dead code
2458 }
2459 if (!opr->is_con() ||
2460 vector_klass_from->const_oop() == nullptr || elem_klass_from->const_oop() == nullptr || !vlen_from->is_con() ||
2461 vector_klass_to->const_oop() == nullptr || elem_klass_to->const_oop() == nullptr || !vlen_to->is_con()) {
2462 log_if_needed(" ** missing constant: opr=%s vclass_from=%s etype_from=%s vlen_from=%s vclass_to=%s etype_to=%s vlen_to=%s",
2463 NodeClassNames[argument(0)->Opcode()],
2464 NodeClassNames[argument(1)->Opcode()],
2465 NodeClassNames[argument(2)->Opcode()],
2466 NodeClassNames[argument(3)->Opcode()],
2467 NodeClassNames[argument(4)->Opcode()],
2468 NodeClassNames[argument(5)->Opcode()],
2469 NodeClassNames[argument(6)->Opcode()]);
2470 return false; // not enough info for intrinsification
2471 }
2472 if (!is_klass_initialized(vector_klass_from) || !is_klass_initialized(vector_klass_to)) {
2473 log_if_needed(" ** klass argument not initialized");
2474 return false;
2475 }
2476
2477 assert(opr->get_con() == VectorSupport::VECTOR_OP_CAST ||
2478 opr->get_con() == VectorSupport::VECTOR_OP_UCAST ||
2479 opr->get_con() == VectorSupport::VECTOR_OP_REINTERPRET, "wrong opcode");
2480 bool is_cast = (opr->get_con() == VectorSupport::VECTOR_OP_CAST || opr->get_con() == VectorSupport::VECTOR_OP_UCAST);
2481 bool is_ucast = (opr->get_con() == VectorSupport::VECTOR_OP_UCAST);
2482
2483 ciKlass* vbox_klass_from = vector_klass_from->const_oop()->as_instance()->java_lang_Class_klass();
2484 ciKlass* vbox_klass_to = vector_klass_to->const_oop()->as_instance()->java_lang_Class_klass();
2485 if (is_vector_shuffle(vbox_klass_from)) {
2486 return false; // vector shuffles aren't supported
2487 }
2488 bool is_mask = is_vector_mask(vbox_klass_from);
2489
2490 ciType* elem_type_from = elem_klass_from->const_oop()->as_instance()->java_mirror_type();
2491 if (!elem_type_from->is_primitive_type()) {
2492 return false; // should be primitive type
2493 }
2494 BasicType elem_bt_from = elem_type_from->basic_type();
2495 ciType* elem_type_to = elem_klass_to->const_oop()->as_instance()->java_mirror_type();
2496 if (!elem_type_to->is_primitive_type()) {
2497 return false; // should be primitive type
2498 }
2499 BasicType elem_bt_to = elem_type_to->basic_type();
2500
2501 int num_elem_from = vlen_from->get_con();
2502 int num_elem_to = vlen_to->get_con();
2503
2504 // Check whether we can unbox to appropriate size. Even with casting, checking for reinterpret is needed
2505 // since we may need to change size.
2506 if (!arch_supports_vector(Op_VectorReinterpret,
2507 num_elem_from,
2508 elem_bt_from,
2509 is_mask ? VecMaskUseAll : VecMaskNotUsed)) {
2510 log_if_needed(" ** not supported: arity=1 op=%s/1 vlen1=%d etype1=%s ismask=%d",
2511 is_cast ? "cast" : "reinterpret",
2512 num_elem_from, type2name(elem_bt_from), is_mask);
2513 return false;
2514 }
2515
2516 // Check whether we can support resizing/reinterpreting to the new size.
2517 if (!arch_supports_vector(Op_VectorReinterpret,
2518 num_elem_to,
2519 elem_bt_to,
2520 is_mask ? VecMaskUseAll : VecMaskNotUsed)) {
2521 log_if_needed(" ** not supported: arity=1 op=%s/2 vlen2=%d etype2=%s ismask=%d",
2522 is_cast ? "cast" : "reinterpret",
2523 num_elem_to, type2name(elem_bt_to), is_mask);
2524 return false;
2525 }
2526
2527
2528 if (is_vector_shuffle(vbox_klass_to) &&
2529 (!arch_supports_vector(Op_SubVB, num_elem_to, elem_bt_to, VecMaskNotUsed) ||
2530 !arch_supports_vector(Op_VectorBlend, num_elem_to, elem_bt_to, VecMaskNotUsed) ||
2531 !arch_supports_vector(Op_VectorMaskCmp, num_elem_to, elem_bt_to, VecMaskNotUsed) ||
2532 !arch_supports_vector(Op_AndV, num_elem_to, elem_bt_to, VecMaskNotUsed) ||
2533 !arch_supports_vector(Op_Replicate, num_elem_to, elem_bt_to, VecMaskNotUsed))) {
2534 log_if_needed(" ** not supported: arity=1 op=shuffle_index_wrap vlen2=%d etype2=%s",
2535 num_elem_to, type2name(elem_bt_to));
2536 return false;
2537 }
2538
2539 // At this point, we know that both input and output vector registers are supported
2540 // by the architecture. Next check if the casted type is simply to same type - which means
2541 // that it is actually a resize and not a cast.
2542 if (is_cast && elem_bt_from == elem_bt_to) {
2543 is_cast = false;
2544 }
2545
2546 const TypeInstPtr* vbox_type_from = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass_from);
2547
2548 Node* opd1 = unbox_vector(argument(7), vbox_type_from, elem_bt_from, num_elem_from);
2549 if (opd1 == nullptr) {
2550 return false;
2551 }
2552
2553 const TypeVect* src_type = TypeVect::make(elem_bt_from, num_elem_from, is_mask);
2554 const TypeVect* dst_type = TypeVect::make(elem_bt_to, num_elem_to, is_mask);
2555
2556 // Safety check to prevent casting if source mask is of type vector
2557 // and destination mask of type predicate vector and vice-versa.
2558 // From X86 standpoint, this case will only arise over KNL target,
2559 // where certain masks (depending on the species) are either propagated
2560 // through a vector or predicate register.
2561 if (is_mask &&
2562 ((src_type->isa_vectmask() == nullptr && dst_type->isa_vectmask()) ||
2563 (dst_type->isa_vectmask() == nullptr && src_type->isa_vectmask()))) {
2564 return false;
2565 }
2566
2567 Node* op = opd1;
2568 if (is_cast) {
2569 assert(!is_mask || num_elem_from == num_elem_to, "vector mask cast needs the same elem num");
2570 int cast_vopc = VectorCastNode::opcode(-1, elem_bt_from, !is_ucast);
2571
2572 // Make sure that vector cast is implemented to particular type/size combination if it is
2573 // not a mask casting.
2574 if (!is_mask && !arch_supports_vector(cast_vopc, num_elem_to, elem_bt_to, VecMaskNotUsed)) {
2575 log_if_needed(" ** not supported: arity=1 op=cast#%d/3 vlen2=%d etype2=%s ismask=%d",
2576 cast_vopc, num_elem_to, type2name(elem_bt_to), is_mask);
2577 return false;
2578 }
2579
2580 if (num_elem_from < num_elem_to) {
2581 // Since input and output number of elements are not consistent, we need to make sure we
2582 // properly size. Thus, first make a cast that retains the number of elements from source.
2583 int num_elem_for_cast = num_elem_from;
2584
2585 // It is possible that arch does not support this intermediate vector size
2586 // TODO More complex logic required here to handle this corner case for the sizes.
2587 if (!arch_supports_vector(cast_vopc, num_elem_for_cast, elem_bt_to, VecMaskNotUsed)) {
2588 log_if_needed(" ** not supported: arity=1 op=cast#%d/4 vlen1=%d etype2=%s ismask=%d",
2589 cast_vopc,
2590 num_elem_for_cast, type2name(elem_bt_to), is_mask);
2591 return false;
2592 }
2593
2594 op = gvn().transform(VectorCastNode::make(cast_vopc, op, elem_bt_to, num_elem_for_cast));
2595 // Now ensure that the destination gets properly resized to needed size.
2596 op = gvn().transform(new VectorReinterpretNode(op, op->bottom_type()->is_vect(), dst_type));
2597 } else if (num_elem_from > num_elem_to) {
2598 // Since number of elements from input is larger than output, simply reduce size of input
2599 // (we are supposed to drop top elements anyway).
2600 int num_elem_for_resize = num_elem_to;
2601
2602 // It is possible that arch does not support this intermediate vector size
2603 // TODO More complex logic required here to handle this corner case for the sizes.
2604 if (!arch_supports_vector(Op_VectorReinterpret,
2605 num_elem_for_resize,
2606 elem_bt_from,
2607 VecMaskNotUsed)) {
2608 log_if_needed(" ** not supported: arity=1 op=cast/5 vlen2=%d etype1=%s ismask=%d",
2609 num_elem_for_resize, type2name(elem_bt_from), is_mask);
2610 return false;
2611 }
2612
2613 const TypeVect* resize_type = TypeVect::make(elem_bt_from, num_elem_for_resize);
2614 op = gvn().transform(new VectorReinterpretNode(op, src_type, resize_type));
2615 op = gvn().transform(VectorCastNode::make(cast_vopc, op, elem_bt_to, num_elem_to));
2616 } else { // num_elem_from == num_elem_to
2617 if (is_mask) {
2618 // Make sure that cast for vector mask is implemented to particular type/size combination.
2619 if (!arch_supports_vector(Op_VectorMaskCast, num_elem_to, elem_bt_to, VecMaskNotUsed)) {
2620 log_if_needed(" ** not supported: arity=1 op=maskcast vlen2=%d etype2=%s ismask=%d",
2621 num_elem_to, type2name(elem_bt_to), is_mask);
2622 return false;
2623 }
2624 op = gvn().transform(new VectorMaskCastNode(op, dst_type));
2625 } else {
2626 // Since input and output number of elements match, and since we know this vector size is
2627 // supported, simply do a cast with no resize needed.
2628 op = gvn().transform(VectorCastNode::make(cast_vopc, op, elem_bt_to, num_elem_to));
2629 }
2630 }
2631 } else if (!Type::equals(src_type, dst_type)) {
2632 assert(!is_cast, "must be reinterpret");
2633 op = gvn().transform(new VectorReinterpretNode(op, src_type, dst_type));
2634 }
2635
2636 if (is_vector_shuffle(vbox_klass_to)) {
2637 op = partially_wrap_indexes(op, num_elem_to, elem_bt_to);
2638 }
2639
2640 const TypeInstPtr* vbox_type_to = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass_to);
2641 Node* vbox = box_vector(op, vbox_type_to, elem_bt_to, num_elem_to);
2642 set_result(vbox);
2643 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem_to * type2aelembytes(elem_bt_to))));
2644 return true;
2645 }
2646
2647 // public static
2648 // <V extends Vector<E>,
2649 // E>
2650 // V insert(Class<? extends V> vectorClass, Class<E> elementType, int vlen,
2651 // V vec, int ix, long val,
2652 // VecInsertOp<V> defaultImpl)
2653 bool LibraryCallKit::inline_vector_insert() {
2654 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
2655 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
2656 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
2657 const TypeInt* idx = gvn().type(argument(4))->isa_int();
2658
2659 if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr || idx == nullptr) {
2660 return false; // dead code
2661 }
2662 if (vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con() || !idx->is_con()) {
2663 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s idx=%s",
2664 NodeClassNames[argument(0)->Opcode()],
2665 NodeClassNames[argument(1)->Opcode()],
2666 NodeClassNames[argument(2)->Opcode()],
2667 NodeClassNames[argument(4)->Opcode()]);
2668 return false; // not enough info for intrinsification
2669 }
2670 if (!is_klass_initialized(vector_klass)) {
2671 log_if_needed(" ** klass argument not initialized");
2672 return false;
2673 }
2674 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2675 if (!elem_type->is_primitive_type()) {
2676 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2677 return false; // should be primitive type
2678 }
2679 BasicType elem_bt = elem_type->basic_type();
2680 int num_elem = vlen->get_con();
2681 if (!arch_supports_vector(Op_VectorInsert, num_elem, elem_bt, VecMaskNotUsed)) {
2682 log_if_needed(" ** not supported: arity=1 op=insert vlen=%d etype=%s ismask=no",
2683 num_elem, type2name(elem_bt));
2684 return false; // not supported
2685 }
2686
2687 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2688 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2689
2690 Node* opd = unbox_vector(argument(3), vbox_type, elem_bt, num_elem);
2691 if (opd == nullptr) {
2692 return false;
2693 }
2694
2695 Node* insert_val = argument(5);
2696 assert(gvn().type(insert_val)->isa_long() != nullptr, "expected to be long");
2697
2698 // Convert insert value back to its appropriate type.
2699 switch (elem_bt) {
2700 case T_BYTE:
2701 insert_val = gvn().transform(new ConvL2INode(insert_val, TypeInt::BYTE));
2702 break;
2703 case T_SHORT:
2704 insert_val = gvn().transform(new ConvL2INode(insert_val, TypeInt::SHORT));
2705 break;
2706 case T_INT:
2707 insert_val = gvn().transform(new ConvL2INode(insert_val));
2708 break;
2709 case T_FLOAT:
2710 insert_val = gvn().transform(new ConvL2INode(insert_val));
2711 insert_val = gvn().transform(new MoveI2FNode(insert_val));
2712 break;
2713 case T_DOUBLE:
2714 insert_val = gvn().transform(new MoveL2DNode(insert_val));
2715 break;
2716 case T_LONG:
2717 // no conversion needed
2718 break;
2719 default: fatal("%s", type2name(elem_bt)); break;
2720 }
2721
2722 Node* operation = gvn().transform(VectorInsertNode::make(opd, insert_val, idx->get_con(), gvn()));
2723
2724 Node* vbox = box_vector(operation, vbox_type, elem_bt, num_elem);
2725 set_result(vbox);
2726 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2727 return true;
2728 }
2729
2730 // public static
2731 // <VM extends VectorPayload,
2732 // E>
2733 // long extract(Class<? extends VM> vClass, Class<E> eClass,
2734 // int length,
2735 // VM vm, int i,
2736 // VecExtractOp<VM> defaultImpl)
2737 bool LibraryCallKit::inline_vector_extract() {
2738 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
2739 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
2740 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
2741 const TypeInt* idx = gvn().type(argument(4))->isa_int();
2742
2743 if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr || idx == nullptr) {
2744 return false; // dead code
2745 }
2746 if (vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
2747 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s",
2748 NodeClassNames[argument(0)->Opcode()],
2749 NodeClassNames[argument(1)->Opcode()],
2750 NodeClassNames[argument(2)->Opcode()]);
2751 return false; // not enough info for intrinsification
2752 }
2753 if (!is_klass_initialized(vector_klass)) {
2754 log_if_needed(" ** klass argument not initialized");
2755 return false;
2756 }
2757 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2758 if (!elem_type->is_primitive_type()) {
2759 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2760 return false; // should be primitive type
2761 }
2762 BasicType elem_bt = elem_type->basic_type();
2763 int num_elem = vlen->get_con();
2764
2765 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2766 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2767
2768 Node* opd = nullptr;
2769
2770 if (is_vector_mask(vbox_klass)) {
2771 // vbox_klass is mask. This is used for VectorMask.laneIsSet(int).
2772
2773 Node* pos = argument(4); // can be variable
2774 if (arch_supports_vector(Op_ExtractUB, num_elem, elem_bt, VecMaskUseAll)) {
2775 // Transform mask to vector with type of boolean and utilize ExtractUB node.
2776 opd = unbox_vector(argument(3), vbox_type, elem_bt, num_elem);
2777 if (opd == nullptr) {
2778 return false;
2779 }
2780 opd = gvn().transform(VectorStoreMaskNode::make(gvn(), opd, elem_bt, num_elem));
2781 opd = gvn().transform(new ExtractUBNode(opd, pos));
2782 opd = gvn().transform(new ConvI2LNode(opd));
2783 } else if (arch_supports_vector(Op_VectorMaskToLong, num_elem, elem_bt, VecMaskUseLoad)) {
2784 opd = unbox_vector(argument(3), vbox_type, elem_bt, num_elem);
2785 if (opd == nullptr) {
2786 return false;
2787 }
2788 // VectorMaskToLongNode requires the input is either a mask or a vector with BOOLEAN type.
2789 if (opd->bottom_type()->isa_vectmask() == nullptr) {
2790 opd = gvn().transform(VectorStoreMaskNode::make(gvn(), opd, elem_bt, num_elem));
2791 }
2792 // ((toLong() >>> pos) & 1L
2793 opd = gvn().transform(new VectorMaskToLongNode(opd, TypeLong::LONG));
2794 opd = gvn().transform(new URShiftLNode(opd, pos));
2795 opd = gvn().transform(new AndLNode(opd, gvn().makecon(TypeLong::ONE)));
2796 } else {
2797 log_if_needed(" ** Rejected mask extraction because architecture does not support it");
2798 return false; // not supported
2799 }
2800 } else {
2801 // vbox_klass is vector. This is used for Vector.lane(int).
2802 if (!idx->is_con()) {
2803 log_if_needed(" ** missing constant: idx=%s", NodeClassNames[argument(4)->Opcode()]);
2804 return false; // not enough info for intrinsification
2805 }
2806
2807 int vopc = ExtractNode::opcode(elem_bt);
2808 if (!arch_supports_vector(vopc, num_elem, elem_bt, VecMaskNotUsed)) {
2809 log_if_needed(" ** not supported: arity=1 op=extract vlen=%d etype=%s ismask=no",
2810 num_elem, type2name(elem_bt));
2811 return false; // not supported
2812 }
2813
2814 opd = unbox_vector(argument(3), vbox_type, elem_bt, num_elem);
2815 if (opd == nullptr) {
2816 return false;
2817 }
2818 ConINode* idx_con = gvn().intcon(idx->get_con())->as_ConI();
2819
2820 opd = gvn().transform(ExtractNode::make(opd, idx_con, elem_bt));
2821 switch (elem_bt) {
2822 case T_BYTE:
2823 case T_SHORT:
2824 case T_INT: {
2825 opd = gvn().transform(new ConvI2LNode(opd));
2826 break;
2827 }
2828 case T_FLOAT: {
2829 opd = gvn().transform(new MoveF2INode(opd));
2830 opd = gvn().transform(new ConvI2LNode(opd));
2831 break;
2832 }
2833 case T_DOUBLE: {
2834 opd = gvn().transform(new MoveD2LNode(opd));
2835 break;
2836 }
2837 case T_LONG: {
2838 // no conversion needed
2839 break;
2840 }
2841 default: fatal("%s", type2name(elem_bt));
2842 }
2843 }
2844 set_result(opd);
2845 return true;
2846 }
2847
2848 static Node* LowerSelectFromTwoVectorOperation(PhaseGVN& phase, Node* index_vec, Node* src1, Node* src2, const TypeVect* vt) {
2849 int num_elem = vt->length();
2850 BasicType elem_bt = vt->element_basic_type();
2851
2852 // Lower selectFrom operation into its constituent operations.
2853 // SelectFromTwoVectorNode =
2854 // (VectorBlend
2855 // (VectorRearrange SRC1 (WRAPED_INDEX AND (VLEN-1))
2856 // (VectorRearrange SRC2 (WRAPED_INDEX AND (VLEN-1))
2857 // MASK)
2858 // Where
2859 // WRAPED_INDEX are computed by wrapping incoming indexes
2860 // to two vector index range [0, VLEN*2) and
2861 // MASK = WRAPED_INDEX < VLEN
2862 //
2863 // IR lowering prevents intrinsification failure and associated argument
2864 // boxing penalties.
2865 //
2866
2867 const TypeVect* index_vect_type = index_vec->bottom_type()->is_vect();
2868 BasicType index_elem_bt = index_vect_type->element_basic_type();
2869
2870 // Downcast index vector to a type agnostic shuffle representation, shuffle
2871 // indices are held in a byte vector which are later transformed to target
2872 // specific permutation index format by subsequent VectorLoadShuffle.
2873 int cast_vopc = VectorCastNode::opcode(0, index_elem_bt, true);
2874 Node* index_byte_vec = phase.transform(VectorCastNode::make(cast_vopc, index_vec, T_BYTE, num_elem));
2875
2876 // Wrap indexes into two vector index range [0, VLEN * 2)
2877 Node* two_vect_lane_cnt_m1 = phase.makecon(TypeInt::make(2 * num_elem - 1));
2878 Node* bcast_two_vect_lane_cnt_m1_vec = phase.transform(VectorNode::scalar2vector(two_vect_lane_cnt_m1, num_elem,
2879 T_BYTE, false));
2880 index_byte_vec = phase.transform(VectorNode::make(Op_AndV, index_byte_vec, bcast_two_vect_lane_cnt_m1_vec,
2881 index_byte_vec->bottom_type()->is_vect()));
2882
2883 // Compute the blend mask for merging two independently permitted vectors
2884 // using shuffle index in two vector index range [0, VLEN * 2).
2885 BoolTest::mask pred = BoolTest::le;
2886 ConINode* pred_node = phase.makecon(TypeInt::make(pred))->as_ConI();
2887 const TypeVect* vmask_type = TypeVect::makemask(T_BYTE, num_elem);
2888 Node* lane_cnt_m1 = phase.makecon(TypeInt::make(num_elem - 1));
2889 Node* bcast_lane_cnt_m1_vec = phase.transform(VectorNode::scalar2vector(lane_cnt_m1, num_elem,
2890 T_BYTE, false));
2891 Node* mask = phase.transform(new VectorMaskCmpNode(pred, index_byte_vec, bcast_lane_cnt_m1_vec, pred_node, vmask_type));
2892
2893 // Rearrange expects the indexes to lie within single vector index range [0, VLEN).
2894 index_byte_vec = phase.transform(VectorNode::make(Op_AndV, index_byte_vec, bcast_lane_cnt_m1_vec,
2895 index_byte_vec->bottom_type()->is_vect()));
2896
2897 // Load indexes from byte vector and appropriately transform them to target
2898 // specific permutation index format.
2899 index_vec = phase.transform(new VectorLoadShuffleNode(index_byte_vec, index_vect_type));
2900
2901 vmask_type = TypeVect::makemask(elem_bt, num_elem);
2902 mask = phase.transform(new VectorMaskCastNode(mask, vmask_type));
2903
2904 Node* p1 = phase.transform(new VectorRearrangeNode(src1, index_vec));
2905 Node* p2 = phase.transform(new VectorRearrangeNode(src2, index_vec));
2906
2907 return new VectorBlendNode(p2, p1, mask);
2908 }
2909
2910 // public static
2911 // <V extends Vector<E>,
2912 // E>
2913 // V selectFromTwoVectorOp(Class<? extends V> vClass, Class<E> eClass, int length,
2914 // V v1, V v2, V v3,
2915 // SelectFromTwoVector<V> defaultImpl)
2916 bool LibraryCallKit::inline_vector_select_from_two_vectors() {
2917 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
2918 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
2919 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
2920
2921 if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr || vector_klass->const_oop() == nullptr ||
2922 elem_klass->const_oop() == nullptr ||!vlen->is_con()) {
2923 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s",
2924 NodeClassNames[argument(0)->Opcode()],
2925 NodeClassNames[argument(1)->Opcode()],
2926 NodeClassNames[argument(2)->Opcode()]);
2927 return false; // not enough info for intrinsification
2928 }
2929
2930 if (!is_klass_initialized(vector_klass)) {
2931 log_if_needed(" ** klass argument not initialized");
2932 return false;
2933 }
2934
2935 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2936 if (!elem_type->is_primitive_type()) {
2937 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2938 return false; // should be primitive type
2939 }
2940
2941 int num_elem = vlen->get_con();
2942 if (!is_power_of_2(num_elem)) {
2943 log_if_needed(" ** vlen is not power of two=%d", num_elem);
2944 return false;
2945 }
2946
2947 BasicType elem_bt = elem_type->basic_type();
2948 BasicType index_elem_bt = elem_bt;
2949 if (elem_bt == T_FLOAT) {
2950 index_elem_bt = T_INT;
2951 } else if (elem_bt == T_DOUBLE) {
2952 index_elem_bt = T_LONG;
2953 }
2954
2955 bool lowerSelectFromOp = false;
2956 if (!arch_supports_vector(Op_SelectFromTwoVector, num_elem, elem_bt, VecMaskNotUsed)) {
2957 int cast_vopc = VectorCastNode::opcode(-1, elem_bt, true);
2958 if (!arch_supports_vector(Op_VectorMaskCmp, num_elem, T_BYTE, VecMaskNotUsed) ||
2959 !arch_supports_vector(Op_AndV, num_elem, T_BYTE, VecMaskNotUsed) ||
2960 !arch_supports_vector(Op_VectorMaskCast, num_elem, elem_bt, VecMaskNotUsed) ||
2961 !arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad) ||
2962 !arch_supports_vector(Op_VectorRearrange, num_elem, elem_bt, VecMaskNotUsed) ||
2963 !arch_supports_vector(cast_vopc, num_elem, T_BYTE, VecMaskNotUsed) ||
2964 !arch_supports_vector(Op_VectorLoadShuffle, num_elem, index_elem_bt, VecMaskNotUsed) ||
2965 !arch_supports_vector(Op_Replicate, num_elem, T_BYTE, VecMaskNotUsed)) {
2966 log_if_needed(" ** not supported: opc=%d vlen=%d etype=%s ismask=useload",
2967 Op_SelectFromTwoVector, num_elem, type2name(elem_bt));
2968 return false; // not supported
2969 }
2970 lowerSelectFromOp = true;
2971 }
2972
2973 int cast_vopc = VectorCastNode::opcode(-1, elem_bt, true);
2974 if (!lowerSelectFromOp) {
2975 if (!arch_supports_vector(Op_AndV, num_elem, index_elem_bt, VecMaskNotUsed) ||
2976 !arch_supports_vector(Op_Replicate, num_elem, index_elem_bt, VecMaskNotUsed) ||
2977 (is_floating_point_type(elem_bt) &&
2978 !arch_supports_vector(cast_vopc, num_elem, index_elem_bt, VecMaskNotUsed))) {
2979 log_if_needed(" ** index wrapping not supported: vlen=%d etype=%s" ,
2980 num_elem, type2name(elem_bt));
2981 return false; // not supported
2982 }
2983 }
2984
2985 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2986 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2987
2988 Node* opd1 = unbox_vector(argument(3), vbox_type, elem_bt, num_elem);
2989 if (opd1 == nullptr) {
2990 log_if_needed(" ** unbox failed v1=%s",
2991 NodeClassNames[argument(3)->Opcode()]);
2992 return false;
2993 }
2994 Node* opd2 = unbox_vector(argument(4), vbox_type, elem_bt, num_elem);
2995 if (opd2 == nullptr) {
2996 log_if_needed(" ** unbox failed v2=%s",
2997 NodeClassNames[argument(4)->Opcode()]);
2998 return false;
2999 }
3000 Node* opd3 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
3001 if (opd3 == nullptr) {
3002 log_if_needed(" ** unbox failed v3=%s",
3003 NodeClassNames[argument(5)->Opcode()]);
3004 return false;
3005 }
3006
3007 const TypeVect* vt = TypeVect::make(elem_bt, num_elem);
3008
3009 Node* operation = nullptr;
3010 if (lowerSelectFromOp) {
3011 operation = gvn().transform(LowerSelectFromTwoVectorOperation(gvn(), opd1, opd2, opd3, vt));
3012 } else {
3013 if (index_elem_bt != elem_bt) {
3014 opd1 = gvn().transform(VectorCastNode::make(cast_vopc, opd1, index_elem_bt, num_elem));
3015 }
3016 int indexRangeMask = 2 * num_elem - 1;
3017 Node* wrap_mask = gvn().makecon(TypeInteger::make(indexRangeMask, indexRangeMask, Type::WidenMin, index_elem_bt != T_LONG ? T_INT : index_elem_bt));
3018 Node* wrap_mask_vec = gvn().transform(VectorNode::scalar2vector(wrap_mask, num_elem, index_elem_bt, false));
3019 opd1 = gvn().transform(VectorNode::make(Op_AndV, opd1, wrap_mask_vec, opd1->bottom_type()->is_vect()));
3020 operation = gvn().transform(VectorNode::make(Op_SelectFromTwoVector, opd1, opd2, opd3, vt));
3021 }
3022
3023 // Wrap it up in VectorBox to keep object type information.
3024 Node* vbox = box_vector(operation, vbox_type, elem_bt, num_elem);
3025 set_result(vbox);
3026 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
3027 return true;
3028 }
3029
3030 // public static
3031 // <V extends Vector<E>,
3032 // M extends VectorMask<E>,
3033 // E>
3034 // V compressExpandOp(int opr,
3035 // Class<? extends V> vClass, Class<? extends M> mClass, Class<E> eClass,
3036 // int length, V v, M m,
3037 // CompressExpandOperation<V, M> defaultImpl)
3038 bool LibraryCallKit::inline_vector_compress_expand() {
3039 const TypeInt* opr = gvn().type(argument(0))->isa_int();
3040 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
3041 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
3042 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
3043 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
3044
3045 if (vector_klass == nullptr || elem_klass == nullptr || mask_klass == nullptr || vlen == nullptr ||
3046 vector_klass->const_oop() == nullptr || mask_klass->const_oop() == nullptr ||
3047 elem_klass->const_oop() == nullptr || !vlen->is_con() || !opr->is_con()) {
3048 log_if_needed(" ** missing constant: opr=%s vclass=%s mclass=%s etype=%s vlen=%s",
3049 NodeClassNames[argument(0)->Opcode()],
3050 NodeClassNames[argument(1)->Opcode()],
3051 NodeClassNames[argument(2)->Opcode()],
3052 NodeClassNames[argument(3)->Opcode()],
3053 NodeClassNames[argument(4)->Opcode()]);
3054 return false; // not enough info for intrinsification
3055 }
3056
3057 if (!is_klass_initialized(vector_klass) || !is_klass_initialized(mask_klass)) {
3058 log_if_needed(" ** klass argument not initialized");
3059 return false;
3060 }
3061
3062 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
3063 if (!elem_type->is_primitive_type()) {
3064 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
3065 return false; // should be primitive type
3066 }
3067
3068 int num_elem = vlen->get_con();
3069 BasicType elem_bt = elem_type->basic_type();
3070 int opc = VectorSupport::vop2ideal(opr->get_con(), elem_bt);
3071
3072 if (!arch_supports_vector(opc, num_elem, elem_bt, VecMaskUseLoad)) {
3073 log_if_needed(" ** not supported: opc=%d vlen=%d etype=%s ismask=useload",
3074 opc, num_elem, type2name(elem_bt));
3075 return false; // not supported
3076 }
3077
3078 Node* opd1 = nullptr;
3079 const TypeInstPtr* vbox_type = nullptr;
3080 if (opc != Op_CompressM) {
3081 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
3082 vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
3083 opd1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
3084 if (opd1 == nullptr) {
3085 log_if_needed(" ** unbox failed vector=%s",
3086 NodeClassNames[argument(5)->Opcode()]);
3087 return false;
3088 }
3089 }
3090
3091 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
3092 assert(is_vector_mask(mbox_klass), "argument(6) should be a mask class");
3093 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
3094
3095 Node* mask = unbox_vector(argument(6), mbox_type, elem_bt, num_elem);
3096 if (mask == nullptr) {
3097 log_if_needed(" ** unbox failed mask=%s",
3098 NodeClassNames[argument(6)->Opcode()]);
3099 return false;
3100 }
3101
3102 const TypeVect* vt = TypeVect::make(elem_bt, num_elem, opc == Op_CompressM);
3103 Node* operation = gvn().transform(VectorNode::make(opc, opd1, mask, vt));
3104
3105 // Wrap it up in VectorBox to keep object type information.
3106 const TypeInstPtr* box_type = opc == Op_CompressM ? mbox_type : vbox_type;
3107 Node* vbox = box_vector(operation, box_type, elem_bt, num_elem);
3108 set_result(vbox);
3109 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
3110 return true;
3111 }
3112
3113 // public static
3114 // <V extends Vector<E>,
3115 // E,
3116 // S extends VectorSpecies<E>>
3117 // V indexVector(Class<? extends V> vClass, Class<E> eClass,
3118 // int length,
3119 // V v, int step, S s,
3120 // IndexOperation<V, S> defaultImpl)
3121 bool LibraryCallKit::inline_index_vector() {
3122 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
3123 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
3124 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
3125
3126 if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr ||
3127 vector_klass->const_oop() == nullptr || !vlen->is_con() ||
3128 elem_klass->const_oop() == nullptr) {
3129 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s",
3130 NodeClassNames[argument(0)->Opcode()],
3131 NodeClassNames[argument(1)->Opcode()],
3132 NodeClassNames[argument(2)->Opcode()]);
3133 return false; // not enough info for intrinsification
3134 }
3135
3136 if (!is_klass_initialized(vector_klass)) {
3137 log_if_needed(" ** klass argument not initialized");
3138 return false;
3139 }
3140
3141 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
3142 if (!elem_type->is_primitive_type()) {
3143 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
3144 return false; // should be primitive type
3145 }
3146
3147 int num_elem = vlen->get_con();
3148 BasicType elem_bt = elem_type->basic_type();
3149
3150 // Check whether the iota index generation op is supported by the current hardware
3151 if (!arch_supports_vector(Op_VectorLoadConst, num_elem, elem_bt, VecMaskNotUsed)) {
3152 log_if_needed(" ** not supported: vlen=%d etype=%s", num_elem, type2name(elem_bt));
3153 return false; // not supported
3154 }
3155
3156 int mul_op = VectorSupport::vop2ideal(VectorSupport::VECTOR_OP_MUL, elem_bt);
3157 int vmul_op = VectorNode::opcode(mul_op, elem_bt);
3158 bool needs_mul = true;
3159 Node* scale = argument(4);
3160 const TypeInt* scale_type = gvn().type(scale)->isa_int();
3161 // Multiply is not needed if the scale is a constant "1".
3162 if (scale_type && scale_type->is_con() && scale_type->get_con() == 1) {
3163 needs_mul = false;
3164 } else {
3165 // Check whether the vector multiply op is supported by the current hardware
3166 if (!arch_supports_vector(vmul_op, num_elem, elem_bt, VecMaskNotUsed)) {
3167 log_if_needed(" ** not supported: vlen=%d etype=%s", num_elem, type2name(elem_bt));
3168 return false; // not supported
3169 }
3170
3171 // Check whether the scalar cast op is supported by the current hardware
3172 if (is_floating_point_type(elem_bt) || elem_bt == T_LONG) {
3173 int cast_op = elem_bt == T_LONG ? Op_ConvI2L :
3174 elem_bt == T_FLOAT? Op_ConvI2F : Op_ConvI2D;
3175 if (!Matcher::match_rule_supported(cast_op)) {
3176 log_if_needed(" ** Rejected op (%s) because architecture does not support it",
3177 NodeClassNames[cast_op]);
3178 return false; // not supported
3179 }
3180 }
3181 }
3182
3183 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
3184 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
3185 Node* opd = unbox_vector(argument(3), vbox_type, elem_bt, num_elem);
3186 if (opd == nullptr) {
3187 log_if_needed(" ** unbox failed vector=%s",
3188 NodeClassNames[argument(3)->Opcode()]);
3189 return false;
3190 }
3191
3192 int add_op = VectorSupport::vop2ideal(VectorSupport::VECTOR_OP_ADD, elem_bt);
3193 int vadd_op = VectorNode::opcode(add_op, elem_bt);
3194 bool needs_add = true;
3195 // The addition is not needed if all the element values of "opd" are zero
3196 if (VectorNode::is_all_zeros_vector(opd)) {
3197 needs_add = false;
3198 } else {
3199 // Check whether the vector addition op is supported by the current hardware
3200 if (!arch_supports_vector(vadd_op, num_elem, elem_bt, VecMaskNotUsed)) {
3201 log_if_needed(" ** not supported: vlen=%d etype=%s", num_elem, type2name(elem_bt));
3202 return false; // not supported
3203 }
3204 }
3205
3206 // Compute the iota indice vector
3207 const TypeVect* vt = TypeVect::make(elem_bt, num_elem);
3208 Node* index = gvn().transform(new VectorLoadConstNode(gvn().makecon(TypeInt::ZERO), vt));
3209
3210 // Broadcast the "scale" to a vector, and multiply the "scale" with iota indice vector.
3211 if (needs_mul) {
3212 switch (elem_bt) {
3213 case T_BOOLEAN: // fall-through
3214 case T_BYTE: // fall-through
3215 case T_SHORT: // fall-through
3216 case T_CHAR: // fall-through
3217 case T_INT: {
3218 // no conversion needed
3219 break;
3220 }
3221 case T_LONG: {
3222 scale = gvn().transform(new ConvI2LNode(scale));
3223 break;
3224 }
3225 case T_FLOAT: {
3226 scale = gvn().transform(new ConvI2FNode(scale));
3227 break;
3228 }
3229 case T_DOUBLE: {
3230 scale = gvn().transform(new ConvI2DNode(scale));
3231 break;
3232 }
3233 default: fatal("%s", type2name(elem_bt));
3234 }
3235 scale = gvn().transform(VectorNode::scalar2vector(scale, num_elem, elem_bt));
3236 index = gvn().transform(VectorNode::make(vmul_op, index, scale, vt));
3237 }
3238
3239 // Add "opd" if addition is needed.
3240 if (needs_add) {
3241 index = gvn().transform(VectorNode::make(vadd_op, opd, index, vt));
3242 }
3243 Node* vbox = box_vector(index, vbox_type, elem_bt, num_elem);
3244 set_result(vbox);
3245 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
3246 return true;
3247 }
3248
3249 // public static
3250 // <E,
3251 // M extends VectorMask<E>>
3252 // M indexPartiallyInUpperRange(Class<? extends M> mClass, Class<E> eClass, int length,
3253 // long offset, long limit,
3254 // IndexPartiallyInUpperRangeOperation<E, M> defaultImpl)
3255 bool LibraryCallKit::inline_index_partially_in_upper_range() {
3256 const TypeInstPtr* mask_klass = gvn().type(argument(0))->isa_instptr();
3257 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
3258 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
3259
3260 if (mask_klass == nullptr || elem_klass == nullptr || vlen == nullptr ||
3261 mask_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
3262 log_if_needed(" ** missing constant: mclass=%s etype=%s vlen=%s",
3263 NodeClassNames[argument(0)->Opcode()],
3264 NodeClassNames[argument(1)->Opcode()],
3265 NodeClassNames[argument(2)->Opcode()]);
3266 return false; // not enough info for intrinsification
3267 }
3268
3269 if (!is_klass_initialized(mask_klass)) {
3270 log_if_needed(" ** klass argument not initialized");
3271 return false;
3272 }
3273
3274 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
3275 if (!elem_type->is_primitive_type()) {
3276 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
3277 return false; // should be primitive type
3278 }
3279
3280 int num_elem = vlen->get_con();
3281 BasicType elem_bt = elem_type->basic_type();
3282
3283 // Check whether the necessary ops are supported by current hardware.
3284 bool supports_mask_gen = arch_supports_vector(Op_VectorMaskGen, num_elem, elem_bt, VecMaskUseStore);
3285 if (!supports_mask_gen) {
3286 if (!arch_supports_vector(Op_VectorLoadConst, num_elem, elem_bt, VecMaskNotUsed) ||
3287 !arch_supports_vector(Op_Replicate, num_elem, elem_bt, VecMaskNotUsed) ||
3288 !arch_supports_vector(Op_VectorMaskCmp, num_elem, elem_bt, VecMaskUseStore)) {
3289 log_if_needed(" ** not supported: vlen=%d etype=%s", num_elem, type2name(elem_bt));
3290 return false; // not supported
3291 }
3292
3293 // Check whether the scalar cast operation is supported by current hardware.
3294 if (elem_bt != T_LONG) {
3295 int cast_op = is_integral_type(elem_bt) ? Op_ConvL2I
3296 : (elem_bt == T_FLOAT ? Op_ConvL2F : Op_ConvL2D);
3297 if (!Matcher::match_rule_supported(cast_op)) {
3298 log_if_needed(" ** Rejected op (%s) because architecture does not support it",
3299 NodeClassNames[cast_op]);
3300 return false; // not supported
3301 }
3302 }
3303 }
3304
3305 Node* offset = argument(3);
3306 Node* limit = argument(5);
3307 if (offset == nullptr || limit == nullptr) {
3308 log_if_needed(" ** offset or limit argument is null");
3309 return false; // not supported
3310 }
3311
3312 ciKlass* box_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
3313 assert(is_vector_mask(box_klass), "argument(0) should be a mask class");
3314 const TypeInstPtr* box_type = TypeInstPtr::make_exact(TypePtr::NotNull, box_klass);
3315
3316 // We assume "offset > 0 && limit >= offset && limit - offset < num_elem".
3317 // So directly get indexLimit with "indexLimit = limit - offset".
3318 Node* indexLimit = gvn().transform(new SubLNode(limit, offset));
3319 Node* mask = nullptr;
3320 if (supports_mask_gen) {
3321 mask = gvn().transform(VectorMaskGenNode::make(indexLimit, elem_bt, num_elem));
3322 } else {
3323 // Generate the vector mask based on "mask = iota < indexLimit".
3324 // Broadcast "indexLimit" to a vector.
3325 switch (elem_bt) {
3326 case T_BOOLEAN: // fall-through
3327 case T_BYTE: // fall-through
3328 case T_SHORT: // fall-through
3329 case T_CHAR: // fall-through
3330 case T_INT: {
3331 indexLimit = gvn().transform(new ConvL2INode(indexLimit));
3332 break;
3333 }
3334 case T_DOUBLE: {
3335 indexLimit = gvn().transform(new ConvL2DNode(indexLimit));
3336 break;
3337 }
3338 case T_FLOAT: {
3339 indexLimit = gvn().transform(new ConvL2FNode(indexLimit));
3340 break;
3341 }
3342 case T_LONG: {
3343 // no conversion needed
3344 break;
3345 }
3346 default: fatal("%s", type2name(elem_bt));
3347 }
3348 indexLimit = gvn().transform(VectorNode::scalar2vector(indexLimit, num_elem, elem_bt));
3349
3350 // Load the "iota" vector.
3351 const TypeVect* vt = TypeVect::make(elem_bt, num_elem);
3352 Node* iota = gvn().transform(new VectorLoadConstNode(gvn().makecon(TypeInt::ZERO), vt));
3353
3354 // Compute the vector mask with "mask = iota < indexLimit".
3355 ConINode* pred_node = (ConINode*)gvn().makecon(TypeInt::make(BoolTest::lt));
3356 const TypeVect* vmask_type = TypeVect::makemask(elem_bt, num_elem);
3357 mask = gvn().transform(new VectorMaskCmpNode(BoolTest::lt, iota, indexLimit, pred_node, vmask_type));
3358 }
3359 Node* vbox = box_vector(mask, box_type, elem_bt, num_elem);
3360 set_result(vbox);
3361 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
3362 return true;
3363 }
3364
3365 #undef non_product_log_if_needed
3366 #undef log_if_needed