1 /*
2 * Copyright (c) 2020, 2024, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "ci/ciSymbols.hpp"
27 #include "classfile/vmSymbols.hpp"
28 #include "opto/library_call.hpp"
29 #include "opto/runtime.hpp"
30 #include "opto/vectornode.hpp"
31 #include "prims/vectorSupport.hpp"
32 #include "runtime/stubRoutines.hpp"
33
34 #ifdef ASSERT
35 static bool is_vector(ciKlass* klass) {
36 return klass->is_subclass_of(ciEnv::current()->vector_VectorPayload_klass());
37 }
38
39 static bool check_vbox(const TypeInstPtr* vbox_type) {
40 assert(vbox_type->klass_is_exact(), "");
41
42 ciInstanceKlass* ik = vbox_type->instance_klass();
43 assert(is_vector(ik), "not a vector");
44
45 ciField* fd1 = ik->get_field_by_name(ciSymbols::ETYPE_name(), ciSymbols::class_signature(), /* is_static */ true);
46 assert(fd1 != nullptr, "element type info is missing");
47
48 ciConstant val1 = fd1->constant_value();
49 BasicType elem_bt = val1.as_object()->as_instance()->java_mirror_type()->basic_type();
50 assert(is_java_primitive(elem_bt), "element type info is missing");
51
52 ciField* fd2 = ik->get_field_by_name(ciSymbols::VLENGTH_name(), ciSymbols::int_signature(), /* is_static */ true);
53 assert(fd2 != nullptr, "vector length info is missing");
54
55 ciConstant val2 = fd2->constant_value();
56 assert(val2.as_int() > 0, "vector length info is missing");
57
58 return true;
59 }
60 #endif
61
62 #define log_if_needed(...) \
63 if (C->print_intrinsics()) { \
64 tty->print_cr(__VA_ARGS__); \
65 }
66
67 #ifndef PRODUCT
68 #define non_product_log_if_needed(...) log_if_needed(__VA_ARGS__)
69 #else
70 #define non_product_log_if_needed(...)
71 #endif
72
73 static bool is_vector_mask(ciKlass* klass) {
74 return klass->is_subclass_of(ciEnv::current()->vector_VectorMask_klass());
75 }
76
77 static bool is_vector_shuffle(ciKlass* klass) {
78 return klass->is_subclass_of(ciEnv::current()->vector_VectorShuffle_klass());
79 }
80
81 bool LibraryCallKit::arch_supports_vector_rotate(int opc, int num_elem, BasicType elem_bt,
82 VectorMaskUseType mask_use_type, bool has_scalar_args) {
83 bool is_supported = true;
84
85 // has_scalar_args flag is true only for non-constant scalar shift count,
86 // since in this case shift needs to be broadcasted.
87 if (!Matcher::match_rule_supported_vector(opc, num_elem, elem_bt) ||
88 (has_scalar_args && !arch_supports_vector(Op_Replicate, num_elem, elem_bt, VecMaskNotUsed))) {
89 is_supported = false;
90 }
91
92 if (is_supported) {
93 // Check if mask unboxing is supported, this is a two step process which first loads the contents
94 // of boolean array into vector followed by either lane expansion to match the lane size of masked
95 // vector operation or populate the predicate register.
96 if ((mask_use_type & VecMaskUseLoad) != 0) {
97 if (!Matcher::match_rule_supported_vector(Op_VectorLoadMask, num_elem, elem_bt) ||
98 !Matcher::match_rule_supported_vector(Op_LoadVector, num_elem, T_BOOLEAN)) {
99 non_product_log_if_needed(" ** Rejected vector mask loading (%s,%s,%d) because architecture does not support it",
100 NodeClassNames[Op_VectorLoadMask], type2name(elem_bt), num_elem);
101 return false;
102 }
103 }
104
105 if ((mask_use_type & VecMaskUsePred) != 0) {
106 if (!Matcher::has_predicated_vectors() ||
107 !Matcher::match_rule_supported_vector_masked(opc, num_elem, elem_bt)) {
108 non_product_log_if_needed("Rejected vector mask predicate using (%s,%s,%d) because architecture does not support it",
109 NodeClassNames[opc], type2name(elem_bt), num_elem);
110 return false;
111 }
112 }
113 }
114
115 int lshiftopc, rshiftopc;
116 switch(elem_bt) {
117 case T_BYTE:
118 lshiftopc = Op_LShiftI;
119 rshiftopc = Op_URShiftB;
120 break;
121 case T_SHORT:
122 lshiftopc = Op_LShiftI;
123 rshiftopc = Op_URShiftS;
124 break;
125 case T_INT:
126 lshiftopc = Op_LShiftI;
127 rshiftopc = Op_URShiftI;
128 break;
129 case T_LONG:
130 lshiftopc = Op_LShiftL;
131 rshiftopc = Op_URShiftL;
132 break;
133 default: fatal("Unexpected type: %s", type2name(elem_bt));
134 }
135 int lshiftvopc = VectorNode::opcode(lshiftopc, elem_bt);
136 int rshiftvopc = VectorNode::opcode(rshiftopc, elem_bt);
137 if (!is_supported &&
138 arch_supports_vector(lshiftvopc, num_elem, elem_bt, VecMaskNotUsed, has_scalar_args) &&
139 arch_supports_vector(rshiftvopc, num_elem, elem_bt, VecMaskNotUsed, has_scalar_args) &&
140 arch_supports_vector(Op_OrV, num_elem, elem_bt, VecMaskNotUsed)) {
141 is_supported = true;
142 }
143 return is_supported;
144 }
145
146 Node* GraphKit::box_vector(Node* vector, const TypeInstPtr* vbox_type, BasicType elem_bt, int num_elem, bool deoptimize_on_exception) {
147 assert(EnableVectorSupport, "");
148
149 PreserveReexecuteState preexecs(this);
150 jvms()->set_should_reexecute(true);
151
152 VectorBoxAllocateNode* alloc = new VectorBoxAllocateNode(C, vbox_type);
153 set_edges_for_java_call(alloc, /*must_throw=*/false, /*separate_io_proj=*/true);
154 make_slow_call_ex(alloc, env()->Throwable_klass(), /*separate_io_proj=*/true, deoptimize_on_exception);
155 set_i_o(gvn().transform( new ProjNode(alloc, TypeFunc::I_O) ));
156 set_all_memory(gvn().transform( new ProjNode(alloc, TypeFunc::Memory) ));
157 Node* ret = gvn().transform(new ProjNode(alloc, TypeFunc::Parms));
158
159 assert(check_vbox(vbox_type), "");
160 const TypeVect* vt = TypeVect::make(elem_bt, num_elem, is_vector_mask(vbox_type->instance_klass()));
161 VectorBoxNode* vbox = new VectorBoxNode(C, ret, vector, vbox_type, vt);
162 return gvn().transform(vbox);
163 }
164
165 Node* GraphKit::unbox_vector(Node* v, const TypeInstPtr* vbox_type, BasicType elem_bt, int num_elem, bool shuffle_to_vector) {
166 assert(EnableVectorSupport, "");
167 const TypeInstPtr* vbox_type_v = gvn().type(v)->isa_instptr();
168 if (vbox_type_v == nullptr || vbox_type->instance_klass() != vbox_type_v->instance_klass()) {
169 return nullptr; // arguments don't agree on vector shapes
170 }
171 if (vbox_type_v->maybe_null()) {
172 return nullptr; // no nulls are allowed
173 }
174 assert(check_vbox(vbox_type), "");
175 const TypeVect* vt = TypeVect::make(elem_bt, num_elem, is_vector_mask(vbox_type->instance_klass()));
176 Node* unbox = gvn().transform(new VectorUnboxNode(C, vt, v, merged_memory(), shuffle_to_vector));
177 return unbox;
178 }
179
180 Node* GraphKit::vector_shift_count(Node* cnt, int shift_op, BasicType bt, int num_elem) {
181 assert(bt == T_INT || bt == T_LONG || bt == T_SHORT || bt == T_BYTE, "byte, short, long and int are supported");
182 juint mask = (type2aelembytes(bt) * BitsPerByte - 1);
183 Node* nmask = gvn().transform(ConNode::make(TypeInt::make(mask)));
184 Node* mcnt = gvn().transform(new AndINode(cnt, nmask));
185 return gvn().transform(VectorNode::shift_count(shift_op, mcnt, num_elem, bt));
186 }
187
188 bool LibraryCallKit::arch_supports_vector(int sopc, int num_elem, BasicType type, VectorMaskUseType mask_use_type, bool has_scalar_args) {
189 // Check that the operation is valid.
190 if (sopc <= 0) {
191 non_product_log_if_needed(" ** Rejected intrinsification because no valid vector op could be extracted");
192 return false;
193 }
194
195 if (VectorNode::is_vector_rotate(sopc)) {
196 if(!arch_supports_vector_rotate(sopc, num_elem, type, mask_use_type, has_scalar_args)) {
197 non_product_log_if_needed(" ** Rejected vector op (%s,%s,%d) because architecture does not support variable vector shifts",
198 NodeClassNames[sopc], type2name(type), num_elem);
199 return false;
200 }
201 } else if (VectorNode::is_vector_integral_negate(sopc)) {
202 if (!VectorNode::is_vector_integral_negate_supported(sopc, num_elem, type, false)) {
203 non_product_log_if_needed(" ** Rejected vector op (%s,%s,%d) because architecture does not support integral vector negate",
204 NodeClassNames[sopc], type2name(type), num_elem);
205 return false;
206 }
207 } else {
208 // Check that architecture supports this op-size-type combination.
209 if (!Matcher::match_rule_supported_vector(sopc, num_elem, type)) {
210 non_product_log_if_needed(" ** Rejected vector op (%s,%s,%d) because architecture does not support it",
211 NodeClassNames[sopc], type2name(type), num_elem);
212 return false;
213 } else {
214 assert(Matcher::match_rule_supported(sopc), "must be supported");
215 }
216 }
217
218 if (num_elem == 1) {
219 if (mask_use_type != VecMaskNotUsed) {
220 non_product_log_if_needed(" ** Rejected vector mask op (%s,%s,%d) because architecture does not support it",
221 NodeClassNames[sopc], type2name(type), num_elem);
222 return false;
223 }
224
225 if (sopc != 0) {
226 if (sopc != Op_LoadVector && sopc != Op_StoreVector) {
227 non_product_log_if_needed(" ** Not a svml call or load/store vector op (%s,%s,%d)",
228 NodeClassNames[sopc], type2name(type), num_elem);
229 return false;
230 }
231 }
232 }
233
234 if (!has_scalar_args && VectorNode::is_vector_shift(sopc) &&
235 Matcher::supports_vector_variable_shifts() == false) {
236 log_if_needed(" ** Rejected vector op (%s,%s,%d) because architecture does not support variable vector shifts",
237 NodeClassNames[sopc], type2name(type), num_elem);
238 return false;
239 }
240
241 // Check if mask unboxing is supported, this is a two step process which first loads the contents
242 // of boolean array into vector followed by either lane expansion to match the lane size of masked
243 // vector operation or populate the predicate register.
244 if ((mask_use_type & VecMaskUseLoad) != 0) {
245 if (!Matcher::match_rule_supported_vector(Op_VectorLoadMask, num_elem, type) ||
246 !Matcher::match_rule_supported_vector(Op_LoadVector, num_elem, T_BOOLEAN)) {
247 non_product_log_if_needed(" ** Rejected vector mask loading (%s,%s,%d) because architecture does not support it",
248 NodeClassNames[Op_VectorLoadMask], type2name(type), num_elem);
249 return false;
250 }
251 }
252
253 // Check if mask boxing is supported, this is a two step process which first stores the contents
254 // of mask vector / predicate register into a boolean vector followed by vector store operation to
255 // transfer the contents to underlined storage of mask boxes which is a boolean array.
256 if ((mask_use_type & VecMaskUseStore) != 0) {
257 if (!Matcher::match_rule_supported_vector(Op_VectorStoreMask, num_elem, type) ||
258 !Matcher::match_rule_supported_vector(Op_StoreVector, num_elem, T_BOOLEAN)) {
259 non_product_log_if_needed("Rejected vector mask storing (%s,%s,%d) because architecture does not support it",
260 NodeClassNames[Op_VectorStoreMask], type2name(type), num_elem);
261 return false;
262 }
263 }
264
265 if ((mask_use_type & VecMaskUsePred) != 0) {
266 bool is_supported = false;
267 if (Matcher::has_predicated_vectors()) {
268 if (VectorNode::is_vector_integral_negate(sopc)) {
269 is_supported = VectorNode::is_vector_integral_negate_supported(sopc, num_elem, type, true);
270 } else {
271 is_supported = Matcher::match_rule_supported_vector_masked(sopc, num_elem, type);
272 }
273 }
274 is_supported |= Matcher::supports_vector_predicate_op_emulation(sopc, num_elem, type);
275
276 if (!is_supported) {
277 non_product_log_if_needed("Rejected vector mask predicate using (%s,%s,%d) because architecture does not support it",
278 NodeClassNames[sopc], type2name(type), num_elem);
279 return false;
280 }
281 }
282
283 return true;
284 }
285
286 static bool is_klass_initialized(const TypeInstPtr* vec_klass) {
287 if (vec_klass->const_oop() == nullptr) {
288 return false; // uninitialized or some kind of unsafe access
289 }
290 assert(vec_klass->const_oop()->as_instance()->java_lang_Class_klass() != nullptr, "klass instance expected");
291 ciInstanceKlass* klass = vec_klass->const_oop()->as_instance()->java_lang_Class_klass()->as_instance_klass();
292 return klass->is_initialized();
293 }
294
295 // public static
296 // <V extends Vector<E>,
297 // M extends VectorMask<E>,
298 // E>
299 // V unaryOp(int oprId, Class<? extends V> vmClass, Class<? extends M> maskClass, Class<E> elementType,
300 // int length, V v, M m,
301 // UnaryOperation<V, M> defaultImpl)
302 //
303 // public static
304 // <V,
305 // M extends VectorMask<E>,
306 // E>
307 // V binaryOp(int oprId, Class<? extends V> vmClass, Class<? extends M> maskClass, Class<E> elementType,
308 // int length, V v1, V v2, M m,
309 // BinaryOperation<V, M> defaultImpl)
310 //
311 // public static
312 // <V extends Vector<E>,
313 // M extends VectorMask<E>,
314 // E>
315 // V ternaryOp(int oprId, Class<? extends V> vmClass, Class<? extends M> maskClass, Class<E> elementType,
316 // int length, V v1, V v2, V v3, M m,
317 // TernaryOperation<V, M> defaultImpl)
318 //
319 bool LibraryCallKit::inline_vector_nary_operation(int n) {
320 const TypeInt* opr = gvn().type(argument(0))->isa_int();
321 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
322 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
323 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
324 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
325
326 if (opr == nullptr || vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr ||
327 !opr->is_con() || vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
328 log_if_needed(" ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
329 NodeClassNames[argument(0)->Opcode()],
330 NodeClassNames[argument(1)->Opcode()],
331 NodeClassNames[argument(3)->Opcode()],
332 NodeClassNames[argument(4)->Opcode()]);
333 return false; // not enough info for intrinsification
334 }
335
336 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
337 if (!elem_type->is_primitive_type()) {
338 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
339 return false; // should be primitive type
340 }
341 if (!is_klass_initialized(vector_klass)) {
342 log_if_needed(" ** klass argument not initialized");
343 return false;
344 }
345
346 // "argument(n + 5)" should be the mask object. We assume it is "null" when no mask
347 // is used to control this operation.
348 const Type* vmask_type = gvn().type(argument(n + 5));
349 bool is_masked_op = vmask_type != TypePtr::NULL_PTR;
350 if (is_masked_op) {
351 if (mask_klass == nullptr || mask_klass->const_oop() == nullptr) {
352 log_if_needed(" ** missing constant: maskclass=%s", NodeClassNames[argument(2)->Opcode()]);
353 return false; // not enough info for intrinsification
354 }
355
356 if (!is_klass_initialized(mask_klass)) {
357 log_if_needed(" ** mask klass argument not initialized");
358 return false;
359 }
360
361 if (vmask_type->maybe_null()) {
362 log_if_needed(" ** null mask values are not allowed for masked op");
363 return false;
364 }
365 }
366
367 BasicType elem_bt = elem_type->basic_type();
368 int num_elem = vlen->get_con();
369 int opc = VectorSupport::vop2ideal(opr->get_con(), elem_bt);
370 int sopc = VectorNode::opcode(opc, elem_bt);
371 if ((opc != Op_CallLeafVector) && (sopc == 0)) {
372 log_if_needed(" ** operation not supported: opc=%s bt=%s", NodeClassNames[opc], type2name(elem_bt));
373 return false; // operation not supported
374 }
375 if (num_elem == 1) {
376 if (opc != Op_CallLeafVector || elem_bt != T_DOUBLE) {
377 log_if_needed(" ** not a svml call: arity=%d opc=%d vlen=%d etype=%s",
378 n, opc, num_elem, type2name(elem_bt));
379 return false;
380 }
381 }
382 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
383 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
384
385 if (is_vector_mask(vbox_klass)) {
386 assert(!is_masked_op, "mask operations do not need mask to control");
387 }
388
389 if (opc == Op_CallLeafVector) {
390 if (!UseVectorStubs) {
391 log_if_needed(" ** vector stubs support is disabled");
392 return false;
393 }
394 if (!Matcher::supports_vector_calling_convention()) {
395 log_if_needed(" ** no vector calling conventions supported");
396 return false;
397 }
398 if (!Matcher::vector_size_supported(elem_bt, num_elem)) {
399 log_if_needed(" ** vector size (vlen=%d, etype=%s) is not supported",
400 num_elem, type2name(elem_bt));
401 return false;
402 }
403 }
404
405 // When using mask, mask use type needs to be VecMaskUseLoad.
406 VectorMaskUseType mask_use_type = is_vector_mask(vbox_klass) ? VecMaskUseAll
407 : is_masked_op ? VecMaskUseLoad : VecMaskNotUsed;
408 if ((sopc != 0) && !arch_supports_vector(sopc, num_elem, elem_bt, mask_use_type)) {
409 log_if_needed(" ** not supported: arity=%d opc=%d vlen=%d etype=%s ismask=%d is_masked_op=%d",
410 n, sopc, num_elem, type2name(elem_bt),
411 is_vector_mask(vbox_klass) ? 1 : 0, is_masked_op ? 1 : 0);
412 return false; // not supported
413 }
414
415 // Return true if current platform has implemented the masked operation with predicate feature.
416 bool use_predicate = is_masked_op && sopc != 0 && arch_supports_vector(sopc, num_elem, elem_bt, VecMaskUsePred);
417 if (is_masked_op && !use_predicate && !arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad)) {
418 log_if_needed(" ** not supported: arity=%d opc=%d vlen=%d etype=%s ismask=0 is_masked_op=1",
419 n, sopc, num_elem, type2name(elem_bt));
420 return false;
421 }
422
423 Node* opd1 = nullptr; Node* opd2 = nullptr; Node* opd3 = nullptr;
424 switch (n) {
425 case 3: {
426 opd3 = unbox_vector(argument(7), vbox_type, elem_bt, num_elem);
427 if (opd3 == nullptr) {
428 log_if_needed(" ** unbox failed v3=%s",
429 NodeClassNames[argument(7)->Opcode()]);
430 return false;
431 }
432 // fall-through
433 }
434 case 2: {
435 opd2 = unbox_vector(argument(6), vbox_type, elem_bt, num_elem);
436 if (opd2 == nullptr) {
437 log_if_needed(" ** unbox failed v2=%s",
438 NodeClassNames[argument(6)->Opcode()]);
439 return false;
440 }
441 // fall-through
442 }
443 case 1: {
444 opd1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
445 if (opd1 == nullptr) {
446 log_if_needed(" ** unbox failed v1=%s",
447 NodeClassNames[argument(5)->Opcode()]);
448 return false;
449 }
450 break;
451 }
452 default: fatal("unsupported arity: %d", n);
453 }
454
455 Node* mask = nullptr;
456 if (is_masked_op) {
457 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
458 assert(is_vector_mask(mbox_klass), "argument(2) should be a mask class");
459 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
460 mask = unbox_vector(argument(n + 5), mbox_type, elem_bt, num_elem);
461 if (mask == nullptr) {
462 log_if_needed(" ** unbox failed mask=%s",
463 NodeClassNames[argument(n + 5)->Opcode()]);
464 return false;
465 }
466 }
467
468 Node* operation = nullptr;
469 if (opc == Op_CallLeafVector) {
470 assert(UseVectorStubs, "sanity");
471 operation = gen_call_to_svml(opr->get_con(), elem_bt, num_elem, opd1, opd2);
472 if (operation == nullptr) {
473 log_if_needed(" ** svml call failed for %s_%s_%d",
474 (elem_bt == T_FLOAT)?"float":"double",
475 VectorSupport::svmlname[opr->get_con() - VectorSupport::VECTOR_OP_SVML_START],
476 num_elem * type2aelembytes(elem_bt));
477 return false;
478 }
479 } else {
480 const TypeVect* vt = TypeVect::make(elem_bt, num_elem, is_vector_mask(vbox_klass));
481 switch (n) {
482 case 1:
483 case 2: {
484 operation = VectorNode::make(sopc, opd1, opd2, vt, is_vector_mask(vbox_klass), VectorNode::is_shift_opcode(opc));
485 break;
486 }
487 case 3: {
488 operation = VectorNode::make(sopc, opd1, opd2, opd3, vt);
489 break;
490 }
491 default: fatal("unsupported arity: %d", n);
492 }
493 }
494
495 if (is_masked_op && mask != nullptr) {
496 if (use_predicate) {
497 operation->add_req(mask);
498 operation->add_flag(Node::Flag_is_predicated_vector);
499 } else {
500 operation->add_flag(Node::Flag_is_predicated_using_blend);
501 operation = gvn().transform(operation);
502 operation = new VectorBlendNode(opd1, operation, mask);
503 }
504 }
505 operation = gvn().transform(operation);
506
507 // Wrap it up in VectorBox to keep object type information.
508 Node* vbox = box_vector(operation, vbox_type, elem_bt, num_elem);
509 set_result(vbox);
510 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
511 return true;
512 }
513
514 // Following routine generates IR corresponding to AbstractShuffle::partiallyWrapIndex method,
515 // which partially wraps index by modulo VEC_LENGTH and generates a negative index value if original
516 // index is out of valid index range [0, VEC_LENGTH)
517 //
518 // wrapped_index = (VEC_LENGTH - 1) & index
519 // if (index u> VEC_LENGTH) {
520 // wrapped_index -= VEC_LENGTH;
521 //
522 // Note: Unsigned greater than comparison treat both <0 and >VEC_LENGTH indices as out-of-bound
523 // indexes.
524 Node* LibraryCallKit::partially_wrap_indexes(Node* index_vec, int num_elem, BasicType elem_bt) {
525 assert(elem_bt == T_BYTE, "Shuffles use byte array based backing storage.");
526 const TypeVect* vt = TypeVect::make(elem_bt, num_elem);
527 const Type* type_bt = Type::get_const_basic_type(elem_bt);
528
529 Node* mod_mask = gvn().makecon(TypeInt::make(num_elem-1));
530 Node* bcast_mod_mask = gvn().transform(VectorNode::scalar2vector(mod_mask, num_elem, type_bt));
531
532 BoolTest::mask pred = BoolTest::ugt;
533 ConINode* pred_node = (ConINode*)gvn().makecon(TypeInt::make(pred));
534 Node* lane_cnt = gvn().makecon(TypeInt::make(num_elem));
535 Node* bcast_lane_cnt = gvn().transform(VectorNode::scalar2vector(lane_cnt, num_elem, type_bt));
536 const TypeVect* vmask_type = TypeVect::makemask(type_bt, num_elem);
537 Node* mask = gvn().transform(new VectorMaskCmpNode(pred, bcast_lane_cnt, index_vec, pred_node, vmask_type));
538
539 // Make the indices greater than lane count as -ve values to match the java side implementation.
540 index_vec = gvn().transform(VectorNode::make(Op_AndV, index_vec, bcast_mod_mask, vt));
541 Node* biased_val = gvn().transform(VectorNode::make(Op_SubVB, index_vec, bcast_lane_cnt, vt));
542 return gvn().transform(new VectorBlendNode(biased_val, index_vec, mask));
543 }
544
545 // <Sh extends VectorShuffle<E>, E>
546 // Sh ShuffleIota(Class<?> E, Class<?> shuffleClass, Vector.Species<E> s, int length,
547 // int start, int step, int wrap, ShuffleIotaOperation<Sh, E> defaultImpl)
548 bool LibraryCallKit::inline_vector_shuffle_iota() {
549 const TypeInstPtr* shuffle_klass = gvn().type(argument(1))->isa_instptr();
550 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
551 const TypeInt* start_val = gvn().type(argument(4))->isa_int();
552 const TypeInt* step_val = gvn().type(argument(5))->isa_int();
553 const TypeInt* wrap = gvn().type(argument(6))->isa_int();
554
555 if (shuffle_klass == nullptr || shuffle_klass->const_oop() == nullptr ||
556 vlen == nullptr || !vlen->is_con() || start_val == nullptr || step_val == nullptr ||
557 wrap == nullptr || !wrap->is_con()) {
558 return false; // not enough info for intrinsification
559 }
560
561 if (!is_klass_initialized(shuffle_klass)) {
562 log_if_needed(" ** klass argument not initialized");
563 return false;
564 }
565
566 int do_wrap = wrap->get_con();
567 int num_elem = vlen->get_con();
568 BasicType elem_bt = T_BYTE;
569
570 bool effective_indices_in_range = false;
571 if (start_val->is_con() && step_val->is_con()) {
572 int effective_min_index = start_val->get_con();
573 int effective_max_index = start_val->get_con() + step_val->get_con() * (num_elem - 1);
574 effective_indices_in_range = effective_max_index >= effective_min_index && effective_min_index >= -128 && effective_max_index <= 127;
575 }
576
577 if (!do_wrap && !effective_indices_in_range) {
578 // Disable instrinsification for unwrapped shuffle iota if start/step
579 // values are non-constant OR if intermediate result overflows byte value range.
580 return false;
581 }
582
583 if (!arch_supports_vector(Op_AddVB, num_elem, elem_bt, VecMaskNotUsed) ||
584 !arch_supports_vector(Op_AndV, num_elem, elem_bt, VecMaskNotUsed) ||
585 !arch_supports_vector(Op_VectorLoadConst, num_elem, elem_bt, VecMaskNotUsed) ||
586 !arch_supports_vector(Op_Replicate, num_elem, elem_bt, VecMaskNotUsed)) {
587 return false;
588 }
589
590 if (!do_wrap &&
591 (!arch_supports_vector(Op_SubVB, num_elem, elem_bt, VecMaskNotUsed) ||
592 !arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskNotUsed) ||
593 !arch_supports_vector(Op_VectorMaskCmp, num_elem, elem_bt, VecMaskNotUsed))) {
594 return false;
595 }
596
597 bool step_multiply = !step_val->is_con() || !is_power_of_2(step_val->get_con());
598 if ((step_multiply && !arch_supports_vector(Op_MulVB, num_elem, elem_bt, VecMaskNotUsed)) ||
599 (!step_multiply && !arch_supports_vector(Op_LShiftVB, num_elem, elem_bt, VecMaskNotUsed))) {
600 return false;
601 }
602
603 const Type * type_bt = Type::get_const_basic_type(elem_bt);
604 const TypeVect * vt = TypeVect::make(type_bt, num_elem);
605
606 Node* res = gvn().transform(new VectorLoadConstNode(gvn().makecon(TypeInt::ZERO), vt));
607
608 Node* start = argument(4);
609 Node* step = argument(5);
610
611 if (step_multiply) {
612 Node* bcast_step = gvn().transform(VectorNode::scalar2vector(step, num_elem, type_bt));
613 res = gvn().transform(VectorNode::make(Op_MulVB, res, bcast_step, vt));
614 } else if (step_val->get_con() > 1) {
615 Node* cnt = gvn().makecon(TypeInt::make(log2i_exact(step_val->get_con())));
616 Node* shift_cnt = vector_shift_count(cnt, Op_LShiftI, elem_bt, num_elem);
617 res = gvn().transform(VectorNode::make(Op_LShiftVB, res, shift_cnt, vt));
618 }
619
620 if (!start_val->is_con() || start_val->get_con() != 0) {
621 Node* bcast_start = gvn().transform(VectorNode::scalar2vector(start, num_elem, type_bt));
622 res = gvn().transform(VectorNode::make(Op_AddVB, res, bcast_start, vt));
623 }
624
625 Node * mod_val = gvn().makecon(TypeInt::make(num_elem-1));
626 Node * bcast_mod = gvn().transform(VectorNode::scalar2vector(mod_val, num_elem, type_bt));
627
628 if (do_wrap) {
629 // Wrap the indices greater than lane count.
630 res = gvn().transform(VectorNode::make(Op_AndV, res, bcast_mod, vt));
631 } else {
632 res = partially_wrap_indexes(res, num_elem, elem_bt);
633 }
634
635 ciKlass* sbox_klass = shuffle_klass->const_oop()->as_instance()->java_lang_Class_klass();
636 const TypeInstPtr* shuffle_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, sbox_klass);
637
638 // Wrap it up in VectorBox to keep object type information.
639 res = box_vector(res, shuffle_box_type, elem_bt, num_elem);
640 set_result(res);
641 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
642 return true;
643 }
644
645 // <E, M>
646 // long maskReductionCoerced(int oper, Class<? extends M> maskClass, Class<?> elemClass,
647 // int length, M m, VectorMaskOp<M> defaultImpl)
648 bool LibraryCallKit::inline_vector_mask_operation() {
649 const TypeInt* oper = gvn().type(argument(0))->isa_int();
650 const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
651 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
652 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
653 Node* mask = argument(4);
654
655 if (mask_klass == nullptr || elem_klass == nullptr || mask->is_top() || vlen == nullptr) {
656 return false; // dead code
657 }
658
659 if (!is_klass_initialized(mask_klass)) {
660 log_if_needed(" ** klass argument not initialized");
661 return false;
662 }
663
664 int num_elem = vlen->get_con();
665 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
666 BasicType elem_bt = elem_type->basic_type();
667
668 int mopc = VectorSupport::vop2ideal(oper->get_con(), elem_bt);
669 if (!arch_supports_vector(mopc, num_elem, elem_bt, VecMaskUseLoad)) {
670 log_if_needed(" ** not supported: arity=1 op=cast#%d/3 vlen2=%d etype2=%s",
671 mopc, num_elem, type2name(elem_bt));
672 return false; // not supported
673 }
674
675 const Type* elem_ty = Type::get_const_basic_type(elem_bt);
676 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
677 const TypeInstPtr* mask_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
678 Node* mask_vec = unbox_vector(mask, mask_box_type, elem_bt, num_elem, true);
679 if (mask_vec == nullptr) {
680 log_if_needed(" ** unbox failed mask=%s",
681 NodeClassNames[argument(4)->Opcode()]);
682 return false;
683 }
684
685 if (mask_vec->bottom_type()->isa_vectmask() == nullptr) {
686 mask_vec = gvn().transform(VectorStoreMaskNode::make(gvn(), mask_vec, elem_bt, num_elem));
687 }
688 const Type* maskoper_ty = mopc == Op_VectorMaskToLong ? (const Type*)TypeLong::LONG : (const Type*)TypeInt::INT;
689 Node* maskoper = gvn().transform(VectorMaskOpNode::make(mask_vec, maskoper_ty, mopc));
690 if (mopc != Op_VectorMaskToLong) {
691 maskoper = ConvI2L(maskoper);
692 }
693 set_result(maskoper);
694
695 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
696 return true;
697 }
698
699 // public static
700 // <V,
701 // Sh extends VectorShuffle<E>,
702 // E>
703 // V shuffleToVector(Class<? extends Vector<E>> vclass, Class<E> elementType,
704 // Class<? extends Sh> shuffleClass, Sh s, int length,
705 // ShuffleToVectorOperation<V, Sh, E> defaultImpl)
706 bool LibraryCallKit::inline_vector_shuffle_to_vector() {
707 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
708 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
709 const TypeInstPtr* shuffle_klass = gvn().type(argument(2))->isa_instptr();
710 Node* shuffle = argument(3);
711 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
712
713 if (vector_klass == nullptr || elem_klass == nullptr || shuffle_klass == nullptr || shuffle->is_top() || vlen == nullptr) {
714 return false; // dead code
715 }
716 if (!vlen->is_con() || vector_klass->const_oop() == nullptr || shuffle_klass->const_oop() == nullptr) {
717 return false; // not enough info for intrinsification
718 }
719 if (!is_klass_initialized(shuffle_klass) || !is_klass_initialized(vector_klass) ) {
720 log_if_needed(" ** klass argument not initialized");
721 return false;
722 }
723
724 int num_elem = vlen->get_con();
725 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
726 BasicType elem_bt = elem_type->basic_type();
727
728 if (num_elem < 4) {
729 return false;
730 }
731
732 int cast_vopc = VectorCastNode::opcode(-1, T_BYTE); // from shuffle of type T_BYTE
733 // Make sure that cast is implemented to particular type/size combination.
734 if (!arch_supports_vector(cast_vopc, num_elem, elem_bt, VecMaskNotUsed)) {
735 log_if_needed(" ** not supported: arity=1 op=cast#%d/3 vlen2=%d etype2=%s",
736 cast_vopc, num_elem, type2name(elem_bt));
737 return false;
738 }
739
740 ciKlass* sbox_klass = shuffle_klass->const_oop()->as_instance()->java_lang_Class_klass();
741 const TypeInstPtr* shuffle_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, sbox_klass);
742
743 // Unbox shuffle with true flag to indicate its load shuffle to vector
744 // shuffle is a byte array
745 Node* shuffle_vec = unbox_vector(shuffle, shuffle_box_type, T_BYTE, num_elem, true);
746
747 // cast byte to target element type
748 shuffle_vec = gvn().transform(VectorCastNode::make(cast_vopc, shuffle_vec, elem_bt, num_elem));
749
750 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
751 const TypeInstPtr* vec_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
752
753 // Box vector
754 Node* res = box_vector(shuffle_vec, vec_box_type, elem_bt, num_elem);
755 set_result(res);
756 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
757 return true;
758 }
759
760 // public static
761 // <M,
762 // S extends VectorSpecies<E>,
763 // E>
764 // M fromBitsCoerced(Class<? extends M> vmClass, Class<E> elementType, int length,
765 // long bits, int mode, S s,
766 // BroadcastOperation<M, E, S> defaultImpl)
767 bool LibraryCallKit::inline_vector_frombits_coerced() {
768 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
769 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
770 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
771 const TypeLong* bits_type = gvn().type(argument(3))->isa_long();
772 // Mode argument determines the mode of operation it can take following values:-
773 // MODE_BROADCAST for vector Vector.broadcast and VectorMask.maskAll operations.
774 // MODE_BITS_COERCED_LONG_TO_MASK for VectorMask.fromLong operation.
775 const TypeInt* mode = gvn().type(argument(5))->isa_int();
776
777 if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr || mode == nullptr ||
778 bits_type == nullptr || vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr ||
779 !vlen->is_con() || !mode->is_con()) {
780 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s bitwise=%s",
781 NodeClassNames[argument(0)->Opcode()],
782 NodeClassNames[argument(1)->Opcode()],
783 NodeClassNames[argument(2)->Opcode()],
784 NodeClassNames[argument(5)->Opcode()]);
785 return false; // not enough info for intrinsification
786 }
787
788 if (!is_klass_initialized(vector_klass)) {
789 log_if_needed(" ** klass argument not initialized");
790 return false;
791 }
792 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
793 if (!elem_type->is_primitive_type()) {
794 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
795 return false; // should be primitive type
796 }
797 BasicType elem_bt = elem_type->basic_type();
798 int num_elem = vlen->get_con();
799 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
800 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
801
802 bool is_mask = is_vector_mask(vbox_klass);
803 int bcast_mode = mode->get_con();
804 VectorMaskUseType checkFlags = (VectorMaskUseType)(is_mask ? VecMaskUseAll : VecMaskNotUsed);
805 int opc = bcast_mode == VectorSupport::MODE_BITS_COERCED_LONG_TO_MASK ? Op_VectorLongToMask : Op_Replicate;
806
807 if (!arch_supports_vector(opc, num_elem, elem_bt, checkFlags, true /*has_scalar_args*/)) {
808 log_if_needed(" ** not supported: arity=0 op=broadcast vlen=%d etype=%s ismask=%d bcast_mode=%d",
809 num_elem, type2name(elem_bt),
810 is_mask ? 1 : 0,
811 bcast_mode);
812 return false; // not supported
813 }
814
815 Node* broadcast = nullptr;
816 Node* bits = argument(3);
817 Node* elem = bits;
818
819 if (opc == Op_VectorLongToMask) {
820 const TypeVect* vt = TypeVect::makemask(elem_bt, num_elem);
821 if (vt->isa_vectmask()) {
822 broadcast = gvn().transform(new VectorLongToMaskNode(elem, vt));
823 } else {
824 const TypeVect* mvt = TypeVect::make(T_BOOLEAN, num_elem);
825 broadcast = gvn().transform(new VectorLongToMaskNode(elem, mvt));
826 broadcast = gvn().transform(new VectorLoadMaskNode(broadcast, vt));
827 }
828 } else {
829 switch (elem_bt) {
830 case T_BOOLEAN: // fall-through
831 case T_BYTE: // fall-through
832 case T_SHORT: // fall-through
833 case T_CHAR: // fall-through
834 case T_INT: {
835 elem = gvn().transform(new ConvL2INode(bits));
836 break;
837 }
838 case T_DOUBLE: {
839 elem = gvn().transform(new MoveL2DNode(bits));
840 break;
841 }
842 case T_FLOAT: {
843 bits = gvn().transform(new ConvL2INode(bits));
844 elem = gvn().transform(new MoveI2FNode(bits));
845 break;
846 }
847 case T_LONG: {
848 // no conversion needed
849 break;
850 }
851 default: fatal("%s", type2name(elem_bt));
852 }
853 broadcast = VectorNode::scalar2vector(elem, num_elem, Type::get_const_basic_type(elem_bt), is_mask);
854 broadcast = gvn().transform(broadcast);
855 }
856
857 Node* box = box_vector(broadcast, vbox_type, elem_bt, num_elem);
858 set_result(box);
859 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
860 return true;
861 }
862
863 static bool elem_consistent_with_arr(BasicType elem_bt, const TypeAryPtr* arr_type, bool mismatched_ms) {
864 assert(arr_type != nullptr, "unexpected");
865 BasicType arr_elem_bt = arr_type->elem()->array_element_basic_type();
866 if (elem_bt == arr_elem_bt) {
867 return true;
868 } else if (elem_bt == T_SHORT && arr_elem_bt == T_CHAR) {
869 // Load/store of short vector from/to char[] is supported
870 return true;
871 } else if (elem_bt == T_BYTE && arr_elem_bt == T_BOOLEAN) {
872 // Load/store of byte vector from/to boolean[] is supported
873 return true;
874 } else {
875 return mismatched_ms;
876 }
877 }
878
879 // public static
880 // <C,
881 // VM extends VectorPayload,
882 // E,
883 // S extends VectorSpecies<E>>
884 // VM load(Class<? extends VM> vmClass, Class<E> eClass,
885 // int length,
886 // Object base, long offset, // Unsafe addressing
887 // boolean fromSegment,
888 // C container, long index, S s, // Arguments for default implementation
889 // LoadOperation<C, VM, S> defaultImpl) {
890 // public static
891 // <C,
892 // V extends VectorPayload>
893 // void store(Class<?> vClass, Class<?> eClass,
894 // int length,
895 // Object base, long offset, // Unsafe addressing
896 // boolean fromSegment,
897 // V v, C container, long index, // Arguments for default implementation
898 // StoreVectorOperation<C, V> defaultImpl) {
899 bool LibraryCallKit::inline_vector_mem_operation(bool is_store) {
900 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
901 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
902 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
903 const TypeInt* from_ms = gvn().type(argument(6))->isa_int();
904
905 if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr || !from_ms->is_con() ||
906 vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
907 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s from_ms=%s",
908 NodeClassNames[argument(0)->Opcode()],
909 NodeClassNames[argument(1)->Opcode()],
910 NodeClassNames[argument(2)->Opcode()],
911 NodeClassNames[argument(6)->Opcode()]);
912 return false; // not enough info for intrinsification
913 }
914 if (!is_klass_initialized(vector_klass)) {
915 log_if_needed(" ** klass argument not initialized");
916 return false;
917 }
918
919 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
920 if (!elem_type->is_primitive_type()) {
921 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
922 return false; // should be primitive type
923 }
924 BasicType elem_bt = elem_type->basic_type();
925 int num_elem = vlen->get_con();
926
927 // TODO When mask usage is supported, VecMaskNotUsed needs to be VecMaskUseLoad.
928 if (!arch_supports_vector(is_store ? Op_StoreVector : Op_LoadVector, num_elem, elem_bt, VecMaskNotUsed)) {
929 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d etype=%s ismask=no",
930 is_store, is_store ? "store" : "load",
931 num_elem, type2name(elem_bt));
932 return false; // not supported
933 }
934
935 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
936 bool is_mask = is_vector_mask(vbox_klass);
937
938 Node* base = argument(3);
939 Node* offset = ConvL2X(argument(4));
940
941 // Save state and restore on bailout
942 uint old_sp = sp();
943 SafePointNode* old_map = clone_map();
944
945 Node* addr = make_unsafe_address(base, offset, (is_mask ? T_BOOLEAN : elem_bt), true);
946
947 // The memory barrier checks are based on ones for unsafe access.
948 // This is not 1-1 implementation.
949 const Type *const base_type = gvn().type(base);
950
951 const TypePtr *addr_type = gvn().type(addr)->isa_ptr();
952 const TypeAryPtr* arr_type = addr_type->isa_aryptr();
953
954 const bool in_native = TypePtr::NULL_PTR == base_type; // base always null
955 const bool in_heap = !TypePtr::NULL_PTR->higher_equal(base_type); // base never null
956
957 const bool is_mixed_access = !in_heap && !in_native;
958
959 const bool is_mismatched_access = in_heap && (addr_type->isa_aryptr() == nullptr);
960
961 const bool needs_cpu_membar = is_mixed_access || is_mismatched_access;
962
963 // For non-masked mismatched memory segment vector read/write accesses, intrinsification can continue
964 // with unknown backing storage type and compiler can skip inserting explicit reinterpretation IR after
965 // loading from or before storing to backing storage which is mandatory for semantic correctness of
966 // big-endian memory layout.
967 bool mismatched_ms = LITTLE_ENDIAN_ONLY(false)
968 BIG_ENDIAN_ONLY(from_ms->get_con() && !is_mask && arr_type != nullptr &&
969 arr_type->elem()->array_element_basic_type() != elem_bt);
970 BasicType mem_elem_bt = mismatched_ms ? arr_type->elem()->array_element_basic_type() : elem_bt;
971 if (!is_java_primitive(mem_elem_bt)) {
972 log_if_needed(" ** non-primitive array element type");
973 return false;
974 }
975 int mem_num_elem = mismatched_ms ? (num_elem * type2aelembytes(elem_bt)) / type2aelembytes(mem_elem_bt) : num_elem;
976 if (arr_type != nullptr && !is_mask && !elem_consistent_with_arr(elem_bt, arr_type, mismatched_ms)) {
977 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d etype=%s atype=%s ismask=no",
978 is_store, is_store ? "store" : "load",
979 num_elem, type2name(elem_bt), type2name(arr_type->elem()->array_element_basic_type()));
980 set_map(old_map);
981 set_sp(old_sp);
982 return false;
983 }
984
985 // In case of mismatched memory segment accesses, we need to double check that the source type memory operations are supported by backend.
986 if (mismatched_ms) {
987 if (is_store) {
988 if (!arch_supports_vector(Op_StoreVector, num_elem, elem_bt, VecMaskNotUsed)
989 || !arch_supports_vector(Op_VectorReinterpret, mem_num_elem, mem_elem_bt, VecMaskNotUsed)) {
990 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d*8 etype=%s/8 ismask=no",
991 is_store, "store",
992 num_elem, type2name(elem_bt));
993 set_map(old_map);
994 set_sp(old_sp);
995 return false; // not supported
996 }
997 } else {
998 if (!arch_supports_vector(Op_LoadVector, mem_num_elem, mem_elem_bt, VecMaskNotUsed)
999 || !arch_supports_vector(Op_VectorReinterpret, num_elem, elem_bt, VecMaskNotUsed)) {
1000 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d*8 etype=%s/8 ismask=no",
1001 is_store, "load",
1002 mem_num_elem, type2name(mem_elem_bt));
1003 set_map(old_map);
1004 set_sp(old_sp);
1005 return false; // not supported
1006 }
1007 }
1008 }
1009 if (is_mask) {
1010 if (!is_store) {
1011 if (!arch_supports_vector(Op_LoadVector, num_elem, elem_bt, VecMaskUseLoad)) {
1012 set_map(old_map);
1013 set_sp(old_sp);
1014 return false; // not supported
1015 }
1016 } else {
1017 if (!arch_supports_vector(Op_StoreVector, num_elem, elem_bt, VecMaskUseStore)) {
1018 set_map(old_map);
1019 set_sp(old_sp);
1020 return false; // not supported
1021 }
1022 }
1023 }
1024
1025 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1026
1027 if (needs_cpu_membar) {
1028 insert_mem_bar(Op_MemBarCPUOrder);
1029 }
1030
1031 if (is_store) {
1032 Node* val = unbox_vector(argument(7), vbox_type, elem_bt, num_elem);
1033 if (val == nullptr) {
1034 set_map(old_map);
1035 set_sp(old_sp);
1036 return false; // operand unboxing failed
1037 }
1038 set_all_memory(reset_memory());
1039
1040 // In case the store needs to happen to byte array, reinterpret the incoming vector to byte vector.
1041 int store_num_elem = num_elem;
1042 if (mismatched_ms) {
1043 store_num_elem = mem_num_elem;
1044 const TypeVect* to_vect_type = TypeVect::make(mem_elem_bt, store_num_elem);
1045 val = gvn().transform(new VectorReinterpretNode(val, val->bottom_type()->is_vect(), to_vect_type));
1046 }
1047 if (is_mask) {
1048 val = gvn().transform(VectorStoreMaskNode::make(gvn(), val, elem_bt, num_elem));
1049 }
1050 Node* vstore = gvn().transform(StoreVectorNode::make(0, control(), memory(addr), addr, addr_type, val, store_num_elem));
1051 set_memory(vstore, addr_type);
1052 } else {
1053 // When using byte array, we need to load as byte then reinterpret the value. Otherwise, do a simple vector load.
1054 Node* vload = nullptr;
1055 if (mismatched_ms) {
1056 vload = gvn().transform(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, mem_num_elem, mem_elem_bt));
1057 const TypeVect* to_vect_type = TypeVect::make(elem_bt, num_elem);
1058 vload = gvn().transform(new VectorReinterpretNode(vload, vload->bottom_type()->is_vect(), to_vect_type));
1059 } else {
1060 // Special handle for masks
1061 if (is_mask) {
1062 vload = gvn().transform(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, num_elem, T_BOOLEAN));
1063 vload = gvn().transform(new VectorLoadMaskNode(vload, TypeVect::makemask(elem_bt, num_elem)));
1064 } else {
1065 vload = gvn().transform(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, num_elem, elem_bt));
1066 }
1067 }
1068 Node* box = box_vector(vload, vbox_type, elem_bt, num_elem);
1069 set_result(box);
1070 }
1071
1072 destruct_map_clone(old_map);
1073
1074 if (needs_cpu_membar) {
1075 insert_mem_bar(Op_MemBarCPUOrder);
1076 }
1077
1078 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1079 return true;
1080 }
1081
1082 // public static
1083 // <C,
1084 // V extends Vector<?>,
1085 // E,
1086 // S extends VectorSpecies<E>,
1087 // M extends VectorMask<E>>
1088 // V loadMasked(Class<? extends V> vClass, Class<M> mClass, Class<E> eClass,
1089 // int length, Object base, long offset, // Unsafe addressing
1090 // boolean fromSegment,
1091 // M m, int offsetInRange,
1092 // C container, long index, S s, // Arguments for default implementation
1093 // LoadVectorMaskedOperation<C, V, S, M> defaultImpl) {
1094 // public static
1095 // <C,
1096 // V extends Vector<E>,
1097 // M extends VectorMask<E>,
1098 // E>
1099 // void storeMasked(Class<? extends V> vClass, Class<M> mClass, Class<E> eClass,
1100 // int length,
1101 // Object base, long offset, // Unsafe addressing
1102 // boolean fromSegment,
1103 // V v, M m, C container, long index, // Arguments for default implementation
1104 // StoreVectorMaskedOperation<C, V, M> defaultImpl) {
1105
1106 bool LibraryCallKit::inline_vector_mem_masked_operation(bool is_store) {
1107 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
1108 const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
1109 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
1110 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
1111 const TypeInt* from_ms = gvn().type(argument(7))->isa_int();
1112
1113 if (vector_klass == nullptr || mask_klass == nullptr || elem_klass == nullptr || vlen == nullptr ||
1114 vector_klass->const_oop() == nullptr || mask_klass->const_oop() == nullptr || from_ms == nullptr ||
1115 elem_klass->const_oop() == nullptr || !vlen->is_con() || !from_ms->is_con()) {
1116 log_if_needed(" ** missing constant: vclass=%s mclass=%s etype=%s vlen=%s from_ms=%s",
1117 NodeClassNames[argument(0)->Opcode()],
1118 NodeClassNames[argument(1)->Opcode()],
1119 NodeClassNames[argument(2)->Opcode()],
1120 NodeClassNames[argument(3)->Opcode()],
1121 NodeClassNames[argument(7)->Opcode()]);
1122 return false; // not enough info for intrinsification
1123 }
1124 if (!is_klass_initialized(vector_klass)) {
1125 log_if_needed(" ** klass argument not initialized");
1126 return false;
1127 }
1128
1129 if (!is_klass_initialized(mask_klass)) {
1130 log_if_needed(" ** mask klass argument not initialized");
1131 return false;
1132 }
1133
1134 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1135 if (!elem_type->is_primitive_type()) {
1136 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1137 return false; // should be primitive type
1138 }
1139
1140 BasicType elem_bt = elem_type->basic_type();
1141 int num_elem = vlen->get_con();
1142
1143 Node* base = argument(4);
1144 Node* offset = ConvL2X(argument(5));
1145
1146 // Save state and restore on bailout
1147 uint old_sp = sp();
1148 SafePointNode* old_map = clone_map();
1149
1150 Node* addr = make_unsafe_address(base, offset, elem_bt, true);
1151 const TypePtr *addr_type = gvn().type(addr)->isa_ptr();
1152 const TypeAryPtr* arr_type = addr_type->isa_aryptr();
1153
1154 bool mismatched_ms = from_ms->get_con() && arr_type != nullptr && arr_type->elem()->array_element_basic_type() != elem_bt;
1155 BIG_ENDIAN_ONLY(if (mismatched_ms) return false;)
1156 // If there is no consistency between array and vector element types, it must be special byte array case
1157 if (arr_type != nullptr && !elem_consistent_with_arr(elem_bt, arr_type, mismatched_ms)) {
1158 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d etype=%s atype=%s",
1159 is_store, is_store ? "storeMasked" : "loadMasked",
1160 num_elem, type2name(elem_bt), type2name(arr_type->elem()->array_element_basic_type()));
1161 set_map(old_map);
1162 set_sp(old_sp);
1163 return false;
1164 }
1165
1166 int mem_num_elem = mismatched_ms ? num_elem * type2aelembytes(elem_bt) : num_elem;
1167 BasicType mem_elem_bt = mismatched_ms ? T_BYTE : elem_bt;
1168 bool supports_predicate = arch_supports_vector(is_store ? Op_StoreVectorMasked : Op_LoadVectorMasked,
1169 mem_num_elem, mem_elem_bt, VecMaskUseLoad);
1170
1171 // If current arch does not support the predicated operations, we have to bail
1172 // out when current case uses the predicate feature.
1173 if (!supports_predicate) {
1174 bool needs_predicate = false;
1175 if (is_store) {
1176 // Masked vector store always uses the predicated store.
1177 needs_predicate = true;
1178 } else {
1179 // Masked vector load with IOOBE always uses the predicated load.
1180 const TypeInt* offset_in_range = gvn().type(argument(9))->isa_int();
1181 if (!offset_in_range->is_con()) {
1182 log_if_needed(" ** missing constant: offsetInRange=%s",
1183 NodeClassNames[argument(8)->Opcode()]);
1184 set_map(old_map);
1185 set_sp(old_sp);
1186 return false;
1187 }
1188 needs_predicate = (offset_in_range->get_con() == 0);
1189 }
1190
1191 if (needs_predicate) {
1192 log_if_needed(" ** not supported: op=%s vlen=%d etype=%s mismatched_ms=%d",
1193 is_store ? "storeMasked" : "loadMasked",
1194 num_elem, type2name(elem_bt), mismatched_ms ? 1 : 0);
1195 set_map(old_map);
1196 set_sp(old_sp);
1197 return false;
1198 }
1199 }
1200
1201 // This only happens for masked vector load. If predicate is not supported, then check whether
1202 // the normal vector load and blend operations are supported by backend.
1203 if (!supports_predicate && (!arch_supports_vector(Op_LoadVector, mem_num_elem, mem_elem_bt, VecMaskNotUsed) ||
1204 !arch_supports_vector(Op_VectorBlend, mem_num_elem, mem_elem_bt, VecMaskUseLoad))) {
1205 log_if_needed(" ** not supported: op=loadMasked vlen=%d etype=%s mismatched_ms=%d",
1206 num_elem, type2name(elem_bt), mismatched_ms ? 1 : 0);
1207 set_map(old_map);
1208 set_sp(old_sp);
1209 return false;
1210 }
1211
1212 // Since we are using byte array, we need to double check that the vector reinterpret operation
1213 // with byte type is supported by backend.
1214 if (mismatched_ms) {
1215 if (!arch_supports_vector(Op_VectorReinterpret, mem_num_elem, T_BYTE, VecMaskNotUsed)) {
1216 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d etype=%s mismatched_ms=1",
1217 is_store, is_store ? "storeMasked" : "loadMasked",
1218 num_elem, type2name(elem_bt));
1219 set_map(old_map);
1220 set_sp(old_sp);
1221 return false;
1222 }
1223 }
1224
1225 // Since it needs to unbox the mask, we need to double check that the related load operations
1226 // for mask are supported by backend.
1227 if (!arch_supports_vector(Op_LoadVector, num_elem, elem_bt, VecMaskUseLoad)) {
1228 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d etype=%s",
1229 is_store, is_store ? "storeMasked" : "loadMasked",
1230 num_elem, type2name(elem_bt));
1231 set_map(old_map);
1232 set_sp(old_sp);
1233 return false;
1234 }
1235
1236 // Can base be null? Otherwise, always on-heap access.
1237 bool can_access_non_heap = TypePtr::NULL_PTR->higher_equal(gvn().type(base));
1238 if (can_access_non_heap) {
1239 insert_mem_bar(Op_MemBarCPUOrder);
1240 }
1241
1242 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1243 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1244 assert(!is_vector_mask(vbox_klass) && is_vector_mask(mbox_klass), "Invalid class type");
1245 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1246 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1247
1248 Node* mask = unbox_vector(is_store ? argument(9) : argument(8), mbox_type, elem_bt, num_elem);
1249 if (mask == nullptr) {
1250 log_if_needed(" ** unbox failed mask=%s",
1251 is_store ? NodeClassNames[argument(9)->Opcode()]
1252 : NodeClassNames[argument(8)->Opcode()]);
1253 set_map(old_map);
1254 set_sp(old_sp);
1255 return false;
1256 }
1257
1258 if (is_store) {
1259 Node* val = unbox_vector(argument(8), vbox_type, elem_bt, num_elem);
1260 if (val == nullptr) {
1261 log_if_needed(" ** unbox failed vector=%s",
1262 NodeClassNames[argument(8)->Opcode()]);
1263 set_map(old_map);
1264 set_sp(old_sp);
1265 return false; // operand unboxing failed
1266 }
1267 set_all_memory(reset_memory());
1268
1269 if (mismatched_ms) {
1270 // Reinterpret the incoming vector to byte vector.
1271 const TypeVect* to_vect_type = TypeVect::make(mem_elem_bt, mem_num_elem);
1272 val = gvn().transform(new VectorReinterpretNode(val, val->bottom_type()->is_vect(), to_vect_type));
1273 // Reinterpret the vector mask to byte type.
1274 const TypeVect* from_mask_type = TypeVect::makemask(elem_bt, num_elem);
1275 const TypeVect* to_mask_type = TypeVect::makemask(mem_elem_bt, mem_num_elem);
1276 mask = gvn().transform(new VectorReinterpretNode(mask, from_mask_type, to_mask_type));
1277 }
1278 Node* vstore = gvn().transform(new StoreVectorMaskedNode(control(), memory(addr), addr, val, addr_type, mask));
1279 set_memory(vstore, addr_type);
1280 } else {
1281 Node* vload = nullptr;
1282
1283 if (mismatched_ms) {
1284 // Reinterpret the vector mask to byte type.
1285 const TypeVect* from_mask_type = TypeVect::makemask(elem_bt, num_elem);
1286 const TypeVect* to_mask_type = TypeVect::makemask(mem_elem_bt, mem_num_elem);
1287 mask = gvn().transform(new VectorReinterpretNode(mask, from_mask_type, to_mask_type));
1288 }
1289
1290 if (supports_predicate) {
1291 // Generate masked load vector node if predicate feature is supported.
1292 const TypeVect* vt = TypeVect::make(mem_elem_bt, mem_num_elem);
1293 vload = gvn().transform(new LoadVectorMaskedNode(control(), memory(addr), addr, addr_type, vt, mask));
1294 } else {
1295 // Use the vector blend to implement the masked load vector. The biased elements are zeros.
1296 Node* zero = gvn().transform(gvn().zerocon(mem_elem_bt));
1297 zero = gvn().transform(VectorNode::scalar2vector(zero, mem_num_elem, Type::get_const_basic_type(mem_elem_bt)));
1298 vload = gvn().transform(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, mem_num_elem, mem_elem_bt));
1299 vload = gvn().transform(new VectorBlendNode(zero, vload, mask));
1300 }
1301
1302 if (mismatched_ms) {
1303 const TypeVect* to_vect_type = TypeVect::make(elem_bt, num_elem);
1304 vload = gvn().transform(new VectorReinterpretNode(vload, vload->bottom_type()->is_vect(), to_vect_type));
1305 }
1306
1307 Node* box = box_vector(vload, vbox_type, elem_bt, num_elem);
1308 set_result(box);
1309 }
1310
1311 destruct_map_clone(old_map);
1312
1313 if (can_access_non_heap) {
1314 insert_mem_bar(Op_MemBarCPUOrder);
1315 }
1316
1317 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1318 return true;
1319 }
1320
1321 // <C,
1322 // V extends Vector<?>,
1323 // W extends Vector<Integer>,
1324 // S extends VectorSpecies<E>,
1325 // M extends VectorMask<E>,
1326 // E>
1327 // V loadWithMap(Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType, int length,
1328 // Class<? extends Vector<Integer>> vectorIndexClass,
1329 // Object base, long offset, // Unsafe addressing
1330 // W index_vector, M m,
1331 // C container, int index, int[] indexMap, int indexM, S s, // Arguments for default implementation
1332 // LoadVectorOperationWithMap<C, V, E, S, M> defaultImpl)
1333 //
1334 // <C,
1335 // V extends Vector<E>,
1336 // W extends Vector<Integer>,
1337 // M extends VectorMask<E>,
1338 // E>
1339 // void storeWithMap(Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType,
1340 // int length, Class<? extends Vector<Integer>> vectorIndexClass, Object base, long offset, // Unsafe addressing
1341 // W index_vector, V v, M m,
1342 // C container, int index, int[] indexMap, int indexM, // Arguments for default implementation
1343 // StoreVectorOperationWithMap<C, V, M, E> defaultImpl)
1344 //
1345 bool LibraryCallKit::inline_vector_gather_scatter(bool is_scatter) {
1346 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
1347 const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
1348 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
1349 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
1350 const TypeInstPtr* vector_idx_klass = gvn().type(argument(4))->isa_instptr();
1351
1352 if (vector_klass == nullptr || elem_klass == nullptr || vector_idx_klass == nullptr || vlen == nullptr ||
1353 vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || vector_idx_klass->const_oop() == nullptr || !vlen->is_con()) {
1354 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s viclass=%s",
1355 NodeClassNames[argument(0)->Opcode()],
1356 NodeClassNames[argument(2)->Opcode()],
1357 NodeClassNames[argument(3)->Opcode()],
1358 NodeClassNames[argument(4)->Opcode()]);
1359 return false; // not enough info for intrinsification
1360 }
1361
1362 if (!is_klass_initialized(vector_klass) || !is_klass_initialized(vector_idx_klass)) {
1363 log_if_needed(" ** klass argument not initialized");
1364 return false;
1365 }
1366
1367 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1368 if (!elem_type->is_primitive_type()) {
1369 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1370 return false; // should be primitive type
1371 }
1372
1373 BasicType elem_bt = elem_type->basic_type();
1374 int num_elem = vlen->get_con();
1375
1376 const Type* vmask_type = gvn().type(is_scatter ? argument(10) : argument(9));
1377 bool is_masked_op = vmask_type != TypePtr::NULL_PTR;
1378 if (is_masked_op) {
1379 if (mask_klass == nullptr || mask_klass->const_oop() == nullptr) {
1380 log_if_needed(" ** missing constant: maskclass=%s", NodeClassNames[argument(1)->Opcode()]);
1381 return false; // not enough info for intrinsification
1382 }
1383
1384 if (!is_klass_initialized(mask_klass)) {
1385 log_if_needed(" ** mask klass argument not initialized");
1386 return false;
1387 }
1388
1389 if (vmask_type->maybe_null()) {
1390 log_if_needed(" ** null mask values are not allowed for masked op");
1391 return false;
1392 }
1393
1394 // Check whether the predicated gather/scatter node is supported by architecture.
1395 VectorMaskUseType mask = (VectorMaskUseType) (VecMaskUseLoad | VecMaskUsePred);
1396 if (!arch_supports_vector(is_scatter ? Op_StoreVectorScatterMasked : Op_LoadVectorGatherMasked, num_elem, elem_bt, mask)) {
1397 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d etype=%s is_masked_op=1",
1398 is_scatter, is_scatter ? "scatterMasked" : "gatherMasked",
1399 num_elem, type2name(elem_bt));
1400 return false; // not supported
1401 }
1402 } else {
1403 // Check whether the normal gather/scatter node is supported for non-masked operation.
1404 if (!arch_supports_vector(is_scatter ? Op_StoreVectorScatter : Op_LoadVectorGather, num_elem, elem_bt, VecMaskNotUsed)) {
1405 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d etype=%s is_masked_op=0",
1406 is_scatter, is_scatter ? "scatter" : "gather",
1407 num_elem, type2name(elem_bt));
1408 return false; // not supported
1409 }
1410 }
1411
1412 // Check that the vector holding indices is supported by architecture
1413 // For sub-word gathers expander receive index array.
1414 if (!is_subword_type(elem_bt) && !arch_supports_vector(Op_LoadVector, num_elem, T_INT, VecMaskNotUsed)) {
1415 log_if_needed(" ** not supported: arity=%d op=%s/loadindex vlen=%d etype=int is_masked_op=%d",
1416 is_scatter, is_scatter ? "scatter" : "gather",
1417 num_elem, is_masked_op ? 1 : 0);
1418 return false; // not supported
1419 }
1420
1421 Node* base = argument(5);
1422 Node* offset = ConvL2X(argument(6));
1423
1424 // Save state and restore on bailout
1425 uint old_sp = sp();
1426 SafePointNode* old_map = clone_map();
1427
1428 Node* addr = make_unsafe_address(base, offset, elem_bt, true);
1429
1430 const TypePtr *addr_type = gvn().type(addr)->isa_ptr();
1431 const TypeAryPtr* arr_type = addr_type->isa_aryptr();
1432
1433 // The array must be consistent with vector type
1434 if (arr_type == nullptr || (arr_type != nullptr && !elem_consistent_with_arr(elem_bt, arr_type, false))) {
1435 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d etype=%s atype=%s ismask=no",
1436 is_scatter, is_scatter ? "scatter" : "gather",
1437 num_elem, type2name(elem_bt), type2name(arr_type->elem()->array_element_basic_type()));
1438 set_map(old_map);
1439 set_sp(old_sp);
1440 return false;
1441 }
1442
1443 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1444 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1445 ciKlass* vbox_idx_klass = vector_idx_klass->const_oop()->as_instance()->java_lang_Class_klass();
1446 if (vbox_idx_klass == nullptr) {
1447 set_map(old_map);
1448 set_sp(old_sp);
1449 return false;
1450 }
1451
1452 Node* index_vect = nullptr;
1453 const TypeInstPtr* vbox_idx_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_idx_klass);
1454 if (!is_subword_type(elem_bt)) {
1455 index_vect = unbox_vector(argument(8), vbox_idx_type, T_INT, num_elem);
1456 if (index_vect == nullptr) {
1457 set_map(old_map);
1458 set_sp(old_sp);
1459 return false;
1460 }
1461 }
1462
1463 Node* mask = nullptr;
1464 if (is_masked_op) {
1465 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1466 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1467 mask = unbox_vector(is_scatter ? argument(10) : argument(9), mbox_type, elem_bt, num_elem);
1468 if (mask == nullptr) {
1469 log_if_needed(" ** unbox failed mask=%s",
1470 is_scatter ? NodeClassNames[argument(10)->Opcode()]
1471 : NodeClassNames[argument(9)->Opcode()]);
1472 set_map(old_map);
1473 set_sp(old_sp);
1474 return false;
1475 }
1476 }
1477
1478 const TypeVect* vector_type = TypeVect::make(elem_bt, num_elem);
1479 if (is_scatter) {
1480 Node* val = unbox_vector(argument(9), vbox_type, elem_bt, num_elem);
1481 if (val == nullptr) {
1482 set_map(old_map);
1483 set_sp(old_sp);
1484 return false; // operand unboxing failed
1485 }
1486 set_all_memory(reset_memory());
1487
1488 Node* vstore = nullptr;
1489 if (mask != nullptr) {
1490 vstore = gvn().transform(new StoreVectorScatterMaskedNode(control(), memory(addr), addr, addr_type, val, index_vect, mask));
1491 } else {
1492 vstore = gvn().transform(new StoreVectorScatterNode(control(), memory(addr), addr, addr_type, val, index_vect));
1493 }
1494 set_memory(vstore, addr_type);
1495 } else {
1496 Node* vload = nullptr;
1497 Node* index = argument(11);
1498 Node* indexMap = argument(12);
1499 Node* indexM = argument(13);
1500 if (mask != nullptr) {
1501 if (is_subword_type(elem_bt)) {
1502 Node* index_arr_base = array_element_address(indexMap, indexM, T_INT);
1503 vload = gvn().transform(new LoadVectorGatherMaskedNode(control(), memory(addr), addr, addr_type, vector_type, index_arr_base, mask, index));
1504 } else {
1505 vload = gvn().transform(new LoadVectorGatherMaskedNode(control(), memory(addr), addr, addr_type, vector_type, index_vect, mask));
1506 }
1507 } else {
1508 if (is_subword_type(elem_bt)) {
1509 Node* index_arr_base = array_element_address(indexMap, indexM, T_INT);
1510 vload = gvn().transform(new LoadVectorGatherNode(control(), memory(addr), addr, addr_type, vector_type, index_arr_base, index));
1511 } else {
1512 vload = gvn().transform(new LoadVectorGatherNode(control(), memory(addr), addr, addr_type, vector_type, index_vect));
1513 }
1514 }
1515 Node* box = box_vector(vload, vbox_type, elem_bt, num_elem);
1516 set_result(box);
1517 }
1518
1519 destruct_map_clone(old_map);
1520
1521 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1522 return true;
1523 }
1524
1525 // public static
1526 // <V extends Vector<E>,
1527 // M extends VectorMask<E>,
1528 // E>
1529 // long reductionCoerced(int oprId, Class<? extends V> vectorClass, Class<? extends M> maskClass,
1530 // Class<E> elementType, int length, V v, M m,
1531 // ReductionOperation<V, M> defaultImpl)
1532 bool LibraryCallKit::inline_vector_reduction() {
1533 const TypeInt* opr = gvn().type(argument(0))->isa_int();
1534 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
1535 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
1536 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
1537 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
1538
1539 if (opr == nullptr || vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr ||
1540 !opr->is_con() || vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
1541 log_if_needed(" ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
1542 NodeClassNames[argument(0)->Opcode()],
1543 NodeClassNames[argument(1)->Opcode()],
1544 NodeClassNames[argument(3)->Opcode()],
1545 NodeClassNames[argument(4)->Opcode()]);
1546 return false; // not enough info for intrinsification
1547 }
1548 if (!is_klass_initialized(vector_klass)) {
1549 log_if_needed(" ** klass argument not initialized");
1550 return false;
1551 }
1552 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1553 if (!elem_type->is_primitive_type()) {
1554 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1555 return false; // should be primitive type
1556 }
1557
1558 const Type* vmask_type = gvn().type(argument(6));
1559 bool is_masked_op = vmask_type != TypePtr::NULL_PTR;
1560 if (is_masked_op) {
1561 if (mask_klass == nullptr || mask_klass->const_oop() == nullptr) {
1562 log_if_needed(" ** missing constant: maskclass=%s", NodeClassNames[argument(2)->Opcode()]);
1563 return false; // not enough info for intrinsification
1564 }
1565
1566 if (!is_klass_initialized(mask_klass)) {
1567 log_if_needed(" ** mask klass argument not initialized");
1568 return false;
1569 }
1570
1571 if (vmask_type->maybe_null()) {
1572 log_if_needed(" ** null mask values are not allowed for masked op");
1573 return false;
1574 }
1575 }
1576
1577 BasicType elem_bt = elem_type->basic_type();
1578 int num_elem = vlen->get_con();
1579 int opc = VectorSupport::vop2ideal(opr->get_con(), elem_bt);
1580 int sopc = ReductionNode::opcode(opc, elem_bt);
1581
1582 // When using mask, mask use type needs to be VecMaskUseLoad.
1583 if (!arch_supports_vector(sopc, num_elem, elem_bt, is_masked_op ? VecMaskUseLoad : VecMaskNotUsed)) {
1584 log_if_needed(" ** not supported: arity=1 op=%d/reduce vlen=%d etype=%s is_masked_op=%d",
1585 sopc, num_elem, type2name(elem_bt), is_masked_op ? 1 : 0);
1586 return false;
1587 }
1588
1589 // Return true if current platform has implemented the masked operation with predicate feature.
1590 bool use_predicate = is_masked_op && arch_supports_vector(sopc, num_elem, elem_bt, VecMaskUsePred);
1591 if (is_masked_op && !use_predicate && !arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad)) {
1592 log_if_needed(" ** not supported: arity=1 op=%d/reduce vlen=%d etype=%s is_masked_op=1",
1593 sopc, num_elem, type2name(elem_bt));
1594 return false;
1595 }
1596
1597 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1598 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1599
1600 Node* opd = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
1601 if (opd == nullptr) {
1602 return false; // operand unboxing failed
1603 }
1604
1605 Node* mask = nullptr;
1606 if (is_masked_op) {
1607 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1608 assert(is_vector_mask(mbox_klass), "argument(2) should be a mask class");
1609 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1610 mask = unbox_vector(argument(6), mbox_type, elem_bt, num_elem);
1611 if (mask == nullptr) {
1612 log_if_needed(" ** unbox failed mask=%s",
1613 NodeClassNames[argument(6)->Opcode()]);
1614 return false;
1615 }
1616 }
1617
1618 Node* init = ReductionNode::make_identity_con_scalar(gvn(), opc, elem_bt);
1619 Node* value = opd;
1620
1621 assert(mask != nullptr || !is_masked_op, "Masked op needs the mask value never null");
1622 if (mask != nullptr && !use_predicate) {
1623 Node* reduce_identity = gvn().transform(VectorNode::scalar2vector(init, num_elem, Type::get_const_basic_type(elem_bt)));
1624 value = gvn().transform(new VectorBlendNode(reduce_identity, value, mask));
1625 }
1626
1627 // Make an unordered Reduction node. This affects only AddReductionVF/VD and MulReductionVF/VD,
1628 // as these operations are allowed to be associative (not requiring strict order) in VectorAPI.
1629 value = ReductionNode::make(opc, nullptr, init, value, elem_bt, /* requires_strict_order */ false);
1630
1631 if (mask != nullptr && use_predicate) {
1632 value->add_req(mask);
1633 value->add_flag(Node::Flag_is_predicated_vector);
1634 }
1635
1636 value = gvn().transform(value);
1637
1638 Node* bits = nullptr;
1639 switch (elem_bt) {
1640 case T_BYTE:
1641 case T_SHORT:
1642 case T_INT: {
1643 bits = gvn().transform(new ConvI2LNode(value));
1644 break;
1645 }
1646 case T_FLOAT: {
1647 value = gvn().transform(new MoveF2INode(value));
1648 bits = gvn().transform(new ConvI2LNode(value));
1649 break;
1650 }
1651 case T_DOUBLE: {
1652 bits = gvn().transform(new MoveD2LNode(value));
1653 break;
1654 }
1655 case T_LONG: {
1656 bits = value; // no conversion needed
1657 break;
1658 }
1659 default: fatal("%s", type2name(elem_bt));
1660 }
1661 set_result(bits);
1662 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1663 return true;
1664 }
1665
1666 // public static <V> boolean test(int cond, Class<?> vectorClass, Class<?> elementType, int vlen,
1667 // V v1, V v2,
1668 // BiFunction<V, V, Boolean> defaultImpl)
1669 //
1670 bool LibraryCallKit::inline_vector_test() {
1671 const TypeInt* cond = gvn().type(argument(0))->isa_int();
1672 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
1673 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
1674 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
1675
1676 if (cond == nullptr || vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr ||
1677 !cond->is_con() || vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
1678 log_if_needed(" ** missing constant: cond=%s vclass=%s etype=%s vlen=%s",
1679 NodeClassNames[argument(0)->Opcode()],
1680 NodeClassNames[argument(1)->Opcode()],
1681 NodeClassNames[argument(2)->Opcode()],
1682 NodeClassNames[argument(3)->Opcode()]);
1683 return false; // not enough info for intrinsification
1684 }
1685 if (!is_klass_initialized(vector_klass)) {
1686 log_if_needed(" ** klass argument not initialized");
1687 return false;
1688 }
1689 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1690 if (!elem_type->is_primitive_type()) {
1691 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1692 return false; // should be primitive type
1693 }
1694 BasicType elem_bt = elem_type->basic_type();
1695 int num_elem = vlen->get_con();
1696 BoolTest::mask booltest = (BoolTest::mask)cond->get_con();
1697 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1698 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1699
1700 if (!arch_supports_vector(Op_VectorTest, num_elem, elem_bt, is_vector_mask(vbox_klass) ? VecMaskUseLoad : VecMaskNotUsed)) {
1701 log_if_needed(" ** not supported: arity=2 op=test/%d vlen=%d etype=%s ismask=%d",
1702 cond->get_con(), num_elem, type2name(elem_bt),
1703 is_vector_mask(vbox_klass));
1704 return false;
1705 }
1706
1707 Node* opd1 = unbox_vector(argument(4), vbox_type, elem_bt, num_elem);
1708 Node* opd2;
1709 if (Matcher::vectortest_needs_second_argument(booltest == BoolTest::overflow,
1710 opd1->bottom_type()->isa_vectmask())) {
1711 opd2 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
1712 } else {
1713 opd2 = opd1;
1714 }
1715 if (opd1 == nullptr || opd2 == nullptr) {
1716 return false; // operand unboxing failed
1717 }
1718
1719 Node* cmp = gvn().transform(new VectorTestNode(opd1, opd2, booltest));
1720 BoolTest::mask test = Matcher::vectortest_mask(booltest == BoolTest::overflow,
1721 opd1->bottom_type()->isa_vectmask(), num_elem);
1722 Node* bol = gvn().transform(new BoolNode(cmp, test));
1723 Node* res = gvn().transform(new CMoveINode(bol, gvn().intcon(0), gvn().intcon(1), TypeInt::BOOL));
1724
1725 set_result(res);
1726 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1727 return true;
1728 }
1729
1730 // public static
1731 // <V extends Vector<E>,
1732 // M extends VectorMask<E>,
1733 // E>
1734 // V blend(Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType, int vlen,
1735 // V v1, V v2, M m,
1736 // VectorBlendOp<V, M, E> defaultImpl)
1737 bool LibraryCallKit::inline_vector_blend() {
1738 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
1739 const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
1740 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
1741 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
1742
1743 if (mask_klass == nullptr || vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr) {
1744 return false; // dead code
1745 }
1746 if (mask_klass->const_oop() == nullptr || vector_klass->const_oop() == nullptr ||
1747 elem_klass->const_oop() == nullptr || !vlen->is_con()) {
1748 log_if_needed(" ** missing constant: vclass=%s mclass=%s etype=%s vlen=%s",
1749 NodeClassNames[argument(0)->Opcode()],
1750 NodeClassNames[argument(1)->Opcode()],
1751 NodeClassNames[argument(2)->Opcode()],
1752 NodeClassNames[argument(3)->Opcode()]);
1753 return false; // not enough info for intrinsification
1754 }
1755 if (!is_klass_initialized(vector_klass) || !is_klass_initialized(mask_klass)) {
1756 log_if_needed(" ** klass argument not initialized");
1757 return false;
1758 }
1759 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1760 if (!elem_type->is_primitive_type()) {
1761 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1762 return false; // should be primitive type
1763 }
1764 BasicType elem_bt = elem_type->basic_type();
1765 BasicType mask_bt = elem_bt;
1766 int num_elem = vlen->get_con();
1767
1768 if (!arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad)) {
1769 log_if_needed(" ** not supported: arity=2 op=blend vlen=%d etype=%s ismask=useload",
1770 num_elem, type2name(elem_bt));
1771 return false; // not supported
1772 }
1773 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1774 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1775
1776 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1777 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1778
1779 Node* v1 = unbox_vector(argument(4), vbox_type, elem_bt, num_elem);
1780 Node* v2 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
1781 Node* mask = unbox_vector(argument(6), mbox_type, mask_bt, num_elem);
1782
1783 if (v1 == nullptr || v2 == nullptr || mask == nullptr) {
1784 return false; // operand unboxing failed
1785 }
1786
1787 Node* blend = gvn().transform(new VectorBlendNode(v1, v2, mask));
1788
1789 Node* box = box_vector(blend, vbox_type, elem_bt, num_elem);
1790 set_result(box);
1791 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1792 return true;
1793 }
1794
1795 // public static
1796 // <V extends Vector<E>,
1797 // M extends VectorMask<E>,
1798 // E>
1799 // M compare(int cond, Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType, int vlen,
1800 // V v1, V v2, M m,
1801 // VectorCompareOp<V,M> defaultImpl)
1802 bool LibraryCallKit::inline_vector_compare() {
1803 const TypeInt* cond = gvn().type(argument(0))->isa_int();
1804 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
1805 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
1806 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
1807 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
1808
1809 if (cond == nullptr || vector_klass == nullptr || mask_klass == nullptr || elem_klass == nullptr || vlen == nullptr) {
1810 return false; // dead code
1811 }
1812 if (!cond->is_con() || vector_klass->const_oop() == nullptr || mask_klass->const_oop() == nullptr ||
1813 elem_klass->const_oop() == nullptr || !vlen->is_con()) {
1814 log_if_needed(" ** missing constant: cond=%s vclass=%s mclass=%s etype=%s vlen=%s",
1815 NodeClassNames[argument(0)->Opcode()],
1816 NodeClassNames[argument(1)->Opcode()],
1817 NodeClassNames[argument(2)->Opcode()],
1818 NodeClassNames[argument(3)->Opcode()],
1819 NodeClassNames[argument(4)->Opcode()]);
1820 return false; // not enough info for intrinsification
1821 }
1822 if (!is_klass_initialized(vector_klass) || !is_klass_initialized(mask_klass)) {
1823 log_if_needed(" ** klass argument not initialized");
1824 return false;
1825 }
1826 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1827 if (!elem_type->is_primitive_type()) {
1828 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1829 return false; // should be primitive type
1830 }
1831
1832 int num_elem = vlen->get_con();
1833 BasicType elem_bt = elem_type->basic_type();
1834 BasicType mask_bt = elem_bt;
1835
1836 if ((cond->get_con() & BoolTest::unsigned_compare) != 0) {
1837 if (!Matcher::supports_vector_comparison_unsigned(num_elem, elem_bt)) {
1838 log_if_needed(" ** not supported: unsigned comparison op=comp/%d vlen=%d etype=%s ismask=usestore",
1839 cond->get_con() & (BoolTest::unsigned_compare - 1), num_elem, type2name(elem_bt));
1840 return false;
1841 }
1842 }
1843
1844 if (!arch_supports_vector(Op_VectorMaskCmp, num_elem, elem_bt, VecMaskUseStore)) {
1845 log_if_needed(" ** not supported: arity=2 op=comp/%d vlen=%d etype=%s ismask=usestore",
1846 cond->get_con(), num_elem, type2name(elem_bt));
1847 return false;
1848 }
1849
1850 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1851 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1852
1853 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1854 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1855
1856 Node* v1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
1857 Node* v2 = unbox_vector(argument(6), vbox_type, elem_bt, num_elem);
1858
1859 bool is_masked_op = argument(7)->bottom_type() != TypePtr::NULL_PTR;
1860 Node* mask = is_masked_op ? unbox_vector(argument(7), mbox_type, elem_bt, num_elem) : nullptr;
1861 if (is_masked_op && mask == nullptr) {
1862 log_if_needed(" ** not supported: mask = null arity=2 op=comp/%d vlen=%d etype=%s ismask=usestore is_masked_op=1",
1863 cond->get_con(), num_elem, type2name(elem_bt));
1864 return false;
1865 }
1866
1867 bool use_predicate = is_masked_op && arch_supports_vector(Op_VectorMaskCmp, num_elem, elem_bt, VecMaskUsePred);
1868 if (is_masked_op && !use_predicate && !arch_supports_vector(Op_AndV, num_elem, elem_bt, VecMaskUseLoad)) {
1869 log_if_needed(" ** not supported: arity=2 op=comp/%d vlen=%d etype=%s ismask=usestore is_masked_op=1",
1870 cond->get_con(), num_elem, type2name(elem_bt));
1871 return false;
1872 }
1873
1874 if (v1 == nullptr || v2 == nullptr) {
1875 return false; // operand unboxing failed
1876 }
1877 BoolTest::mask pred = (BoolTest::mask)cond->get_con();
1878 ConINode* pred_node = (ConINode*)gvn().makecon(cond);
1879
1880 const TypeVect* vmask_type = TypeVect::makemask(mask_bt, num_elem);
1881 Node* operation = new VectorMaskCmpNode(pred, v1, v2, pred_node, vmask_type);
1882
1883 if (is_masked_op) {
1884 if (use_predicate) {
1885 operation->add_req(mask);
1886 operation->add_flag(Node::Flag_is_predicated_vector);
1887 } else {
1888 operation = gvn().transform(operation);
1889 operation = VectorNode::make(Op_AndV, operation, mask, vmask_type);
1890 }
1891 }
1892
1893 operation = gvn().transform(operation);
1894
1895 Node* box = box_vector(operation, mbox_type, mask_bt, num_elem);
1896 set_result(box);
1897 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1898 return true;
1899 }
1900
1901 // public static
1902 // <V extends Vector<E>,
1903 // Sh extends VectorShuffle<E>,
1904 // M extends VectorMask<E>,
1905 // E>
1906 // V rearrangeOp(Class<? extends V> vectorClass, Class<Sh> shuffleClass, Class<M> maskClass, Class<E> elementType, int vlen,
1907 // V v1, Sh sh, M m,
1908 // VectorRearrangeOp<V, Sh, M, E> defaultImpl)
1909 bool LibraryCallKit::inline_vector_rearrange() {
1910 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
1911 const TypeInstPtr* shuffle_klass = gvn().type(argument(1))->isa_instptr();
1912 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
1913 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
1914 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
1915
1916 if (vector_klass == nullptr || shuffle_klass == nullptr || elem_klass == nullptr || vlen == nullptr) {
1917 return false; // dead code
1918 }
1919 if (shuffle_klass->const_oop() == nullptr ||
1920 vector_klass->const_oop() == nullptr ||
1921 elem_klass->const_oop() == nullptr ||
1922 !vlen->is_con()) {
1923 log_if_needed(" ** missing constant: vclass=%s sclass=%s etype=%s vlen=%s",
1924 NodeClassNames[argument(0)->Opcode()],
1925 NodeClassNames[argument(1)->Opcode()],
1926 NodeClassNames[argument(3)->Opcode()],
1927 NodeClassNames[argument(4)->Opcode()]);
1928 return false; // not enough info for intrinsification
1929 }
1930 if (!is_klass_initialized(vector_klass) ||
1931 !is_klass_initialized(shuffle_klass)) {
1932 log_if_needed(" ** klass argument not initialized");
1933 return false;
1934 }
1935 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1936 if (!elem_type->is_primitive_type()) {
1937 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1938 return false; // should be primitive type
1939 }
1940 BasicType elem_bt = elem_type->basic_type();
1941 BasicType shuffle_bt = elem_bt;
1942 int num_elem = vlen->get_con();
1943
1944 if (!arch_supports_vector(Op_VectorLoadShuffle, num_elem, elem_bt, VecMaskNotUsed)) {
1945 log_if_needed(" ** not supported: arity=0 op=load/shuffle vlen=%d etype=%s ismask=no",
1946 num_elem, type2name(elem_bt));
1947 return false; // not supported
1948 }
1949
1950 bool is_masked_op = argument(7)->bottom_type() != TypePtr::NULL_PTR;
1951 bool use_predicate = is_masked_op;
1952 if (is_masked_op &&
1953 (mask_klass == nullptr ||
1954 mask_klass->const_oop() == nullptr ||
1955 !is_klass_initialized(mask_klass))) {
1956 log_if_needed(" ** mask_klass argument not initialized");
1957 }
1958 VectorMaskUseType checkFlags = (VectorMaskUseType)(is_masked_op ? (VecMaskUseLoad | VecMaskUsePred) : VecMaskNotUsed);
1959 if (!arch_supports_vector(Op_VectorRearrange, num_elem, elem_bt, checkFlags)) {
1960 use_predicate = false;
1961 if(!is_masked_op ||
1962 (!arch_supports_vector(Op_VectorRearrange, num_elem, elem_bt, VecMaskNotUsed) ||
1963 !arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad) ||
1964 !arch_supports_vector(Op_Replicate, num_elem, elem_bt, VecMaskNotUsed))) {
1965 log_if_needed(" ** not supported: arity=2 op=shuffle/rearrange vlen=%d etype=%s ismask=no",
1966 num_elem, type2name(elem_bt));
1967 return false; // not supported
1968 }
1969 }
1970 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1971 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1972
1973 ciKlass* shbox_klass = shuffle_klass->const_oop()->as_instance()->java_lang_Class_klass();
1974 const TypeInstPtr* shbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, shbox_klass);
1975
1976 Node* v1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
1977 Node* shuffle = unbox_vector(argument(6), shbox_type, shuffle_bt, num_elem);
1978
1979 if (v1 == nullptr || shuffle == nullptr) {
1980 return false; // operand unboxing failed
1981 }
1982
1983 Node* mask = nullptr;
1984 if (is_masked_op) {
1985 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1986 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1987 mask = unbox_vector(argument(7), mbox_type, elem_bt, num_elem);
1988 if (mask == nullptr) {
1989 log_if_needed(" ** not supported: arity=3 op=shuffle/rearrange vlen=%d etype=%s ismask=useload is_masked_op=1",
1990 num_elem, type2name(elem_bt));
1991 return false;
1992 }
1993 }
1994
1995 Node* rearrange = new VectorRearrangeNode(v1, shuffle);
1996 if (is_masked_op) {
1997 if (use_predicate) {
1998 rearrange->add_req(mask);
1999 rearrange->add_flag(Node::Flag_is_predicated_vector);
2000 } else {
2001 const TypeVect* vt = v1->bottom_type()->is_vect();
2002 rearrange = gvn().transform(rearrange);
2003 Node* zero = gvn().makecon(Type::get_zero_type(elem_bt));
2004 Node* zerovec = gvn().transform(VectorNode::scalar2vector(zero, num_elem, Type::get_const_basic_type(elem_bt)));
2005 rearrange = new VectorBlendNode(zerovec, rearrange, mask);
2006 }
2007 }
2008 rearrange = gvn().transform(rearrange);
2009
2010 Node* box = box_vector(rearrange, vbox_type, elem_bt, num_elem);
2011 set_result(box);
2012 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2013 return true;
2014 }
2015
2016 static address get_svml_address(int vop, int bits, BasicType bt, char* name_ptr, int name_len) {
2017 address addr = nullptr;
2018 assert(UseVectorStubs, "sanity");
2019 assert(name_ptr != nullptr, "unexpected");
2020 assert((vop >= VectorSupport::VECTOR_OP_SVML_START) && (vop <= VectorSupport::VECTOR_OP_SVML_END), "unexpected");
2021 int op = vop - VectorSupport::VECTOR_OP_SVML_START;
2022
2023 switch(bits) {
2024 case 64: //fallthough
2025 case 128: //fallthough
2026 case 256: //fallthough
2027 case 512:
2028 if (bt == T_FLOAT) {
2029 snprintf(name_ptr, name_len, "vector_%s_float%d", VectorSupport::svmlname[op], bits);
2030 addr = StubRoutines::_vector_f_math[exact_log2(bits/64)][op];
2031 } else {
2032 assert(bt == T_DOUBLE, "must be FP type only");
2033 snprintf(name_ptr, name_len, "vector_%s_double%d", VectorSupport::svmlname[op], bits);
2034 addr = StubRoutines::_vector_d_math[exact_log2(bits/64)][op];
2035 }
2036 break;
2037 default:
2038 snprintf(name_ptr, name_len, "invalid");
2039 addr = nullptr;
2040 Unimplemented();
2041 break;
2042 }
2043
2044 return addr;
2045 }
2046
2047 Node* LibraryCallKit::gen_call_to_svml(int vector_api_op_id, BasicType bt, int num_elem, Node* opd1, Node* opd2) {
2048 assert(UseVectorStubs, "sanity");
2049 assert(vector_api_op_id >= VectorSupport::VECTOR_OP_SVML_START && vector_api_op_id <= VectorSupport::VECTOR_OP_SVML_END, "need valid op id");
2050 assert(opd1 != nullptr, "must not be null");
2051 const TypeVect* vt = TypeVect::make(bt, num_elem);
2052 const TypeFunc* call_type = OptoRuntime::Math_Vector_Vector_Type(opd2 != nullptr ? 2 : 1, vt, vt);
2053 char name[100] = "";
2054
2055 // Get address for svml method.
2056 address addr = get_svml_address(vector_api_op_id, vt->length_in_bytes() * BitsPerByte, bt, name, 100);
2057
2058 if (addr == nullptr) {
2059 return nullptr;
2060 }
2061
2062 assert(name[0] != '\0', "name must not be null");
2063 Node* operation = make_runtime_call(RC_VECTOR,
2064 call_type,
2065 addr,
2066 name,
2067 TypePtr::BOTTOM,
2068 opd1,
2069 opd2);
2070 return gvn().transform(new ProjNode(gvn().transform(operation), TypeFunc::Parms));
2071 }
2072
2073 // public static
2074 // <V extends Vector<E>,
2075 // M extends VectorMask<E>,
2076 // E>
2077 // V broadcastInt(int opr, Class<? extends V> vectorClass, Class<? extends M> maskClass,
2078 // Class<E> elementType, int length,
2079 // V v, int n, M m,
2080 // VectorBroadcastIntOp<V, M> defaultImpl)
2081 bool LibraryCallKit::inline_vector_broadcast_int() {
2082 const TypeInt* opr = gvn().type(argument(0))->isa_int();
2083 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
2084 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
2085 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
2086 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
2087
2088 if (opr == nullptr || vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr) {
2089 return false; // dead code
2090 }
2091 if (!opr->is_con() || vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
2092 log_if_needed(" ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
2093 NodeClassNames[argument(0)->Opcode()],
2094 NodeClassNames[argument(1)->Opcode()],
2095 NodeClassNames[argument(3)->Opcode()],
2096 NodeClassNames[argument(4)->Opcode()]);
2097 return false; // not enough info for intrinsification
2098 }
2099 if (!is_klass_initialized(vector_klass)) {
2100 log_if_needed(" ** klass argument not initialized");
2101 return false;
2102 }
2103
2104 const Type* vmask_type = gvn().type(argument(7));
2105 bool is_masked_op = vmask_type != TypePtr::NULL_PTR;
2106 if (is_masked_op) {
2107 if (mask_klass == nullptr || mask_klass->const_oop() == nullptr) {
2108 log_if_needed(" ** missing constant: maskclass=%s", NodeClassNames[argument(2)->Opcode()]);
2109 return false; // not enough info for intrinsification
2110 }
2111
2112 if (!is_klass_initialized(mask_klass)) {
2113 log_if_needed(" ** mask klass argument not initialized");
2114 return false;
2115 }
2116
2117 if (vmask_type->maybe_null()) {
2118 log_if_needed(" ** null mask values are not allowed for masked op");
2119 return false;
2120 }
2121 }
2122
2123 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2124 if (!elem_type->is_primitive_type()) {
2125 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2126 return false; // should be primitive type
2127 }
2128
2129 int num_elem = vlen->get_con();
2130 BasicType elem_bt = elem_type->basic_type();
2131 int opc = VectorSupport::vop2ideal(opr->get_con(), elem_bt);
2132
2133 bool is_shift = VectorNode::is_shift_opcode(opc);
2134 bool is_rotate = VectorNode::is_rotate_opcode(opc);
2135
2136 if (opc == 0 || (!is_shift && !is_rotate)) {
2137 log_if_needed(" ** operation not supported: op=%d bt=%s", opr->get_con(), type2name(elem_bt));
2138 return false; // operation not supported
2139 }
2140
2141 int sopc = VectorNode::opcode(opc, elem_bt);
2142 if (sopc == 0) {
2143 log_if_needed(" ** operation not supported: opc=%s bt=%s", NodeClassNames[opc], type2name(elem_bt));
2144 return false; // operation not supported
2145 }
2146
2147 Node* cnt = argument(6);
2148 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2149 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2150 const TypeInt* cnt_type = cnt->bottom_type()->isa_int();
2151
2152 // If CPU supports vector constant rotate instructions pass it directly
2153 bool is_const_rotate = is_rotate && cnt_type && cnt_type->is_con() &&
2154 Matcher::supports_vector_constant_rotates(cnt_type->get_con());
2155 bool has_scalar_args = is_rotate ? !is_const_rotate : true;
2156
2157 VectorMaskUseType checkFlags = (VectorMaskUseType)(is_masked_op ? (VecMaskUseLoad | VecMaskUsePred) : VecMaskNotUsed);
2158 bool use_predicate = is_masked_op;
2159
2160 if (!arch_supports_vector(sopc, num_elem, elem_bt, checkFlags, has_scalar_args)) {
2161 use_predicate = false;
2162 if (!is_masked_op ||
2163 (!arch_supports_vector(sopc, num_elem, elem_bt, VecMaskNotUsed, has_scalar_args) ||
2164 !arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad))) {
2165
2166 log_if_needed(" ** not supported: arity=0 op=int/%d vlen=%d etype=%s is_masked_op=%d",
2167 sopc, num_elem, type2name(elem_bt), is_masked_op ? 1 : 0);
2168 return false; // not supported
2169 }
2170 }
2171
2172 Node* opd1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
2173 Node* opd2 = nullptr;
2174 if (is_shift) {
2175 opd2 = vector_shift_count(cnt, opc, elem_bt, num_elem);
2176 } else {
2177 assert(is_rotate, "unexpected operation");
2178 if (!is_const_rotate) {
2179 const Type * type_bt = Type::get_const_basic_type(elem_bt);
2180 cnt = elem_bt == T_LONG ? gvn().transform(new ConvI2LNode(cnt)) : cnt;
2181 opd2 = gvn().transform(VectorNode::scalar2vector(cnt, num_elem, type_bt));
2182 } else {
2183 // Constant shift value.
2184 opd2 = cnt;
2185 }
2186 }
2187
2188 if (opd1 == nullptr || opd2 == nullptr) {
2189 return false;
2190 }
2191
2192 Node* mask = nullptr;
2193 if (is_masked_op) {
2194 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
2195 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
2196 mask = unbox_vector(argument(7), mbox_type, elem_bt, num_elem);
2197 if (mask == nullptr) {
2198 log_if_needed(" ** unbox failed mask=%s", NodeClassNames[argument(7)->Opcode()]);
2199 return false;
2200 }
2201 }
2202
2203 Node* operation = VectorNode::make(opc, opd1, opd2, num_elem, elem_bt);
2204 if (is_masked_op && mask != nullptr) {
2205 if (use_predicate) {
2206 operation->add_req(mask);
2207 operation->add_flag(Node::Flag_is_predicated_vector);
2208 } else {
2209 operation = gvn().transform(operation);
2210 operation = new VectorBlendNode(opd1, operation, mask);
2211 }
2212 }
2213 operation = gvn().transform(operation);
2214 Node* vbox = box_vector(operation, vbox_type, elem_bt, num_elem);
2215 set_result(vbox);
2216 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2217 return true;
2218 }
2219
2220 // public static <VOUT extends VectorPayload,
2221 // VIN extends VectorPayload,
2222 // S extends VectorSpecies>
2223 // VOUT convert(int oprId,
2224 // Class<?> fromVectorClass, Class<?> fromElementType, int fromVLen,
2225 // Class<?> toVectorClass, Class<?> toElementType, int toVLen,
2226 // VIN v, S s,
2227 // VectorConvertOp<VOUT, VIN, S> defaultImpl)
2228 //
2229 bool LibraryCallKit::inline_vector_convert() {
2230 const TypeInt* opr = gvn().type(argument(0))->isa_int();
2231
2232 const TypeInstPtr* vector_klass_from = gvn().type(argument(1))->isa_instptr();
2233 const TypeInstPtr* elem_klass_from = gvn().type(argument(2))->isa_instptr();
2234 const TypeInt* vlen_from = gvn().type(argument(3))->isa_int();
2235
2236 const TypeInstPtr* vector_klass_to = gvn().type(argument(4))->isa_instptr();
2237 const TypeInstPtr* elem_klass_to = gvn().type(argument(5))->isa_instptr();
2238 const TypeInt* vlen_to = gvn().type(argument(6))->isa_int();
2239
2240 if (opr == nullptr ||
2241 vector_klass_from == nullptr || elem_klass_from == nullptr || vlen_from == nullptr ||
2242 vector_klass_to == nullptr || elem_klass_to == nullptr || vlen_to == nullptr) {
2243 return false; // dead code
2244 }
2245 if (!opr->is_con() ||
2246 vector_klass_from->const_oop() == nullptr || elem_klass_from->const_oop() == nullptr || !vlen_from->is_con() ||
2247 vector_klass_to->const_oop() == nullptr || elem_klass_to->const_oop() == nullptr || !vlen_to->is_con()) {
2248 log_if_needed(" ** missing constant: opr=%s vclass_from=%s etype_from=%s vlen_from=%s vclass_to=%s etype_to=%s vlen_to=%s",
2249 NodeClassNames[argument(0)->Opcode()],
2250 NodeClassNames[argument(1)->Opcode()],
2251 NodeClassNames[argument(2)->Opcode()],
2252 NodeClassNames[argument(3)->Opcode()],
2253 NodeClassNames[argument(4)->Opcode()],
2254 NodeClassNames[argument(5)->Opcode()],
2255 NodeClassNames[argument(6)->Opcode()]);
2256 return false; // not enough info for intrinsification
2257 }
2258 if (!is_klass_initialized(vector_klass_from) || !is_klass_initialized(vector_klass_to)) {
2259 log_if_needed(" ** klass argument not initialized");
2260 return false;
2261 }
2262
2263 assert(opr->get_con() == VectorSupport::VECTOR_OP_CAST ||
2264 opr->get_con() == VectorSupport::VECTOR_OP_UCAST ||
2265 opr->get_con() == VectorSupport::VECTOR_OP_REINTERPRET, "wrong opcode");
2266 bool is_cast = (opr->get_con() == VectorSupport::VECTOR_OP_CAST || opr->get_con() == VectorSupport::VECTOR_OP_UCAST);
2267 bool is_ucast = (opr->get_con() == VectorSupport::VECTOR_OP_UCAST);
2268
2269 ciKlass* vbox_klass_from = vector_klass_from->const_oop()->as_instance()->java_lang_Class_klass();
2270 ciKlass* vbox_klass_to = vector_klass_to->const_oop()->as_instance()->java_lang_Class_klass();
2271 if (is_vector_shuffle(vbox_klass_from)) {
2272 return false; // vector shuffles aren't supported
2273 }
2274 bool is_mask = is_vector_mask(vbox_klass_from);
2275
2276 ciType* elem_type_from = elem_klass_from->const_oop()->as_instance()->java_mirror_type();
2277 if (!elem_type_from->is_primitive_type()) {
2278 return false; // should be primitive type
2279 }
2280 BasicType elem_bt_from = elem_type_from->basic_type();
2281 ciType* elem_type_to = elem_klass_to->const_oop()->as_instance()->java_mirror_type();
2282 if (!elem_type_to->is_primitive_type()) {
2283 return false; // should be primitive type
2284 }
2285 BasicType elem_bt_to = elem_type_to->basic_type();
2286
2287 int num_elem_from = vlen_from->get_con();
2288 int num_elem_to = vlen_to->get_con();
2289
2290 // Check whether we can unbox to appropriate size. Even with casting, checking for reinterpret is needed
2291 // since we may need to change size.
2292 if (!arch_supports_vector(Op_VectorReinterpret,
2293 num_elem_from,
2294 elem_bt_from,
2295 is_mask ? VecMaskUseAll : VecMaskNotUsed)) {
2296 log_if_needed(" ** not supported: arity=1 op=%s/1 vlen1=%d etype1=%s ismask=%d",
2297 is_cast ? "cast" : "reinterpret",
2298 num_elem_from, type2name(elem_bt_from), is_mask);
2299 return false;
2300 }
2301
2302 // Check whether we can support resizing/reinterpreting to the new size.
2303 if (!arch_supports_vector(Op_VectorReinterpret,
2304 num_elem_to,
2305 elem_bt_to,
2306 is_mask ? VecMaskUseAll : VecMaskNotUsed)) {
2307 log_if_needed(" ** not supported: arity=1 op=%s/2 vlen2=%d etype2=%s ismask=%d",
2308 is_cast ? "cast" : "reinterpret",
2309 num_elem_to, type2name(elem_bt_to), is_mask);
2310 return false;
2311 }
2312
2313
2314 if (is_vector_shuffle(vbox_klass_to) &&
2315 (!arch_supports_vector(Op_SubVB, num_elem_to, elem_bt_to, VecMaskNotUsed) ||
2316 !arch_supports_vector(Op_VectorBlend, num_elem_to, elem_bt_to, VecMaskNotUsed) ||
2317 !arch_supports_vector(Op_VectorMaskCmp, num_elem_to, elem_bt_to, VecMaskNotUsed) ||
2318 !arch_supports_vector(Op_AndV, num_elem_to, elem_bt_to, VecMaskNotUsed) ||
2319 !arch_supports_vector(Op_Replicate, num_elem_to, elem_bt_to, VecMaskNotUsed))) {
2320 log_if_needed(" ** not supported: arity=1 op=shuffle_index_wrap vlen2=%d etype2=%s",
2321 num_elem_to, type2name(elem_bt_to));
2322 return false;
2323 }
2324
2325 // At this point, we know that both input and output vector registers are supported
2326 // by the architecture. Next check if the casted type is simply to same type - which means
2327 // that it is actually a resize and not a cast.
2328 if (is_cast && elem_bt_from == elem_bt_to) {
2329 is_cast = false;
2330 }
2331
2332 const TypeInstPtr* vbox_type_from = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass_from);
2333
2334 Node* opd1 = unbox_vector(argument(7), vbox_type_from, elem_bt_from, num_elem_from);
2335 if (opd1 == nullptr) {
2336 return false;
2337 }
2338
2339 const TypeVect* src_type = TypeVect::make(elem_bt_from, num_elem_from, is_mask);
2340 const TypeVect* dst_type = TypeVect::make(elem_bt_to, num_elem_to, is_mask);
2341
2342 // Safety check to prevent casting if source mask is of type vector
2343 // and destination mask of type predicate vector and vice-versa.
2344 // From X86 standpoint, this case will only arise over KNL target,
2345 // where certain masks (depending on the species) are either propagated
2346 // through a vector or predicate register.
2347 if (is_mask &&
2348 ((src_type->isa_vectmask() == nullptr && dst_type->isa_vectmask()) ||
2349 (dst_type->isa_vectmask() == nullptr && src_type->isa_vectmask()))) {
2350 return false;
2351 }
2352
2353 Node* op = opd1;
2354 if (is_cast) {
2355 assert(!is_mask || num_elem_from == num_elem_to, "vector mask cast needs the same elem num");
2356 int cast_vopc = VectorCastNode::opcode(-1, elem_bt_from, !is_ucast);
2357
2358 // Make sure that vector cast is implemented to particular type/size combination if it is
2359 // not a mask casting.
2360 if (!is_mask && !arch_supports_vector(cast_vopc, num_elem_to, elem_bt_to, VecMaskNotUsed)) {
2361 log_if_needed(" ** not supported: arity=1 op=cast#%d/3 vlen2=%d etype2=%s ismask=%d",
2362 cast_vopc, num_elem_to, type2name(elem_bt_to), is_mask);
2363 return false;
2364 }
2365
2366 if (num_elem_from < num_elem_to) {
2367 // Since input and output number of elements are not consistent, we need to make sure we
2368 // properly size. Thus, first make a cast that retains the number of elements from source.
2369 int num_elem_for_cast = num_elem_from;
2370
2371 // It is possible that arch does not support this intermediate vector size
2372 // TODO More complex logic required here to handle this corner case for the sizes.
2373 if (!arch_supports_vector(cast_vopc, num_elem_for_cast, elem_bt_to, VecMaskNotUsed)) {
2374 log_if_needed(" ** not supported: arity=1 op=cast#%d/4 vlen1=%d etype2=%s ismask=%d",
2375 cast_vopc,
2376 num_elem_for_cast, type2name(elem_bt_to), is_mask);
2377 return false;
2378 }
2379
2380 op = gvn().transform(VectorCastNode::make(cast_vopc, op, elem_bt_to, num_elem_for_cast));
2381 // Now ensure that the destination gets properly resized to needed size.
2382 op = gvn().transform(new VectorReinterpretNode(op, op->bottom_type()->is_vect(), dst_type));
2383 } else if (num_elem_from > num_elem_to) {
2384 // Since number of elements from input is larger than output, simply reduce size of input
2385 // (we are supposed to drop top elements anyway).
2386 int num_elem_for_resize = num_elem_to;
2387
2388 // It is possible that arch does not support this intermediate vector size
2389 // TODO More complex logic required here to handle this corner case for the sizes.
2390 if (!arch_supports_vector(Op_VectorReinterpret,
2391 num_elem_for_resize,
2392 elem_bt_from,
2393 VecMaskNotUsed)) {
2394 log_if_needed(" ** not supported: arity=1 op=cast/5 vlen2=%d etype1=%s ismask=%d",
2395 num_elem_for_resize, type2name(elem_bt_from), is_mask);
2396 return false;
2397 }
2398
2399 const TypeVect* resize_type = TypeVect::make(elem_bt_from, num_elem_for_resize);
2400 op = gvn().transform(new VectorReinterpretNode(op, src_type, resize_type));
2401 op = gvn().transform(VectorCastNode::make(cast_vopc, op, elem_bt_to, num_elem_to));
2402 } else { // num_elem_from == num_elem_to
2403 if (is_mask) {
2404 // Make sure that cast for vector mask is implemented to particular type/size combination.
2405 if (!arch_supports_vector(Op_VectorMaskCast, num_elem_to, elem_bt_to, VecMaskNotUsed)) {
2406 log_if_needed(" ** not supported: arity=1 op=maskcast vlen2=%d etype2=%s ismask=%d",
2407 num_elem_to, type2name(elem_bt_to), is_mask);
2408 return false;
2409 }
2410 op = gvn().transform(new VectorMaskCastNode(op, dst_type));
2411 } else {
2412 // Since input and output number of elements match, and since we know this vector size is
2413 // supported, simply do a cast with no resize needed.
2414 op = gvn().transform(VectorCastNode::make(cast_vopc, op, elem_bt_to, num_elem_to));
2415 }
2416 }
2417 } else if (!Type::equals(src_type, dst_type)) {
2418 assert(!is_cast, "must be reinterpret");
2419 op = gvn().transform(new VectorReinterpretNode(op, src_type, dst_type));
2420 }
2421
2422 if (is_vector_shuffle(vbox_klass_to)) {
2423 op = partially_wrap_indexes(op, num_elem_to, elem_bt_to);
2424 }
2425
2426 const TypeInstPtr* vbox_type_to = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass_to);
2427 Node* vbox = box_vector(op, vbox_type_to, elem_bt_to, num_elem_to);
2428 set_result(vbox);
2429 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem_to * type2aelembytes(elem_bt_to))));
2430 return true;
2431 }
2432
2433 // public static
2434 // <V extends Vector<E>,
2435 // E>
2436 // V insert(Class<? extends V> vectorClass, Class<E> elementType, int vlen,
2437 // V vec, int ix, long val,
2438 // VecInsertOp<V> defaultImpl)
2439 bool LibraryCallKit::inline_vector_insert() {
2440 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
2441 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
2442 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
2443 const TypeInt* idx = gvn().type(argument(4))->isa_int();
2444
2445 if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr || idx == nullptr) {
2446 return false; // dead code
2447 }
2448 if (vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con() || !idx->is_con()) {
2449 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s idx=%s",
2450 NodeClassNames[argument(0)->Opcode()],
2451 NodeClassNames[argument(1)->Opcode()],
2452 NodeClassNames[argument(2)->Opcode()],
2453 NodeClassNames[argument(4)->Opcode()]);
2454 return false; // not enough info for intrinsification
2455 }
2456 if (!is_klass_initialized(vector_klass)) {
2457 log_if_needed(" ** klass argument not initialized");
2458 return false;
2459 }
2460 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2461 if (!elem_type->is_primitive_type()) {
2462 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2463 return false; // should be primitive type
2464 }
2465 BasicType elem_bt = elem_type->basic_type();
2466 int num_elem = vlen->get_con();
2467 if (!arch_supports_vector(Op_VectorInsert, num_elem, elem_bt, VecMaskNotUsed)) {
2468 log_if_needed(" ** not supported: arity=1 op=insert vlen=%d etype=%s ismask=no",
2469 num_elem, type2name(elem_bt));
2470 return false; // not supported
2471 }
2472
2473 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2474 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2475
2476 Node* opd = unbox_vector(argument(3), vbox_type, elem_bt, num_elem);
2477 if (opd == nullptr) {
2478 return false;
2479 }
2480
2481 Node* insert_val = argument(5);
2482 assert(gvn().type(insert_val)->isa_long() != nullptr, "expected to be long");
2483
2484 // Convert insert value back to its appropriate type.
2485 switch (elem_bt) {
2486 case T_BYTE:
2487 insert_val = gvn().transform(new ConvL2INode(insert_val));
2488 insert_val = gvn().transform(new CastIINode(insert_val, TypeInt::BYTE));
2489 break;
2490 case T_SHORT:
2491 insert_val = gvn().transform(new ConvL2INode(insert_val));
2492 insert_val = gvn().transform(new CastIINode(insert_val, TypeInt::SHORT));
2493 break;
2494 case T_INT:
2495 insert_val = gvn().transform(new ConvL2INode(insert_val));
2496 break;
2497 case T_FLOAT:
2498 insert_val = gvn().transform(new ConvL2INode(insert_val));
2499 insert_val = gvn().transform(new MoveI2FNode(insert_val));
2500 break;
2501 case T_DOUBLE:
2502 insert_val = gvn().transform(new MoveL2DNode(insert_val));
2503 break;
2504 case T_LONG:
2505 // no conversion needed
2506 break;
2507 default: fatal("%s", type2name(elem_bt)); break;
2508 }
2509
2510 Node* operation = gvn().transform(VectorInsertNode::make(opd, insert_val, idx->get_con(), gvn()));
2511
2512 Node* vbox = box_vector(operation, vbox_type, elem_bt, num_elem);
2513 set_result(vbox);
2514 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2515 return true;
2516 }
2517
2518 // public static
2519 // <VM extends VectorPayload,
2520 // E>
2521 // long extract(Class<? extends VM> vClass, Class<E> eClass,
2522 // int length,
2523 // VM vm, int i,
2524 // VecExtractOp<VM> defaultImpl)
2525 bool LibraryCallKit::inline_vector_extract() {
2526 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
2527 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
2528 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
2529 const TypeInt* idx = gvn().type(argument(4))->isa_int();
2530
2531 if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr || idx == nullptr) {
2532 return false; // dead code
2533 }
2534 if (vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
2535 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s",
2536 NodeClassNames[argument(0)->Opcode()],
2537 NodeClassNames[argument(1)->Opcode()],
2538 NodeClassNames[argument(2)->Opcode()]);
2539 return false; // not enough info for intrinsification
2540 }
2541 if (!is_klass_initialized(vector_klass)) {
2542 log_if_needed(" ** klass argument not initialized");
2543 return false;
2544 }
2545 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2546 if (!elem_type->is_primitive_type()) {
2547 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2548 return false; // should be primitive type
2549 }
2550 BasicType elem_bt = elem_type->basic_type();
2551 int num_elem = vlen->get_con();
2552
2553 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2554 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2555
2556 Node* opd = nullptr;
2557
2558 if (is_vector_mask(vbox_klass)) {
2559 // vbox_klass is mask. This is used for VectorMask.laneIsSet(int).
2560
2561 Node* pos = argument(4); // can be variable
2562 if (arch_supports_vector(Op_ExtractUB, num_elem, elem_bt, VecMaskUseAll)) {
2563 // Transform mask to vector with type of boolean and utilize ExtractUB node.
2564 opd = unbox_vector(argument(3), vbox_type, elem_bt, num_elem);
2565 if (opd == nullptr) {
2566 return false;
2567 }
2568 opd = gvn().transform(VectorStoreMaskNode::make(gvn(), opd, elem_bt, num_elem));
2569 opd = gvn().transform(new ExtractUBNode(opd, pos));
2570 opd = gvn().transform(new ConvI2LNode(opd));
2571 } else if (arch_supports_vector(Op_VectorMaskToLong, num_elem, elem_bt, VecMaskUseLoad)) {
2572 opd = unbox_vector(argument(3), vbox_type, elem_bt, num_elem);
2573 if (opd == nullptr) {
2574 return false;
2575 }
2576 // VectorMaskToLongNode requires the input is either a mask or a vector with BOOLEAN type.
2577 if (opd->bottom_type()->isa_vectmask() == nullptr) {
2578 opd = gvn().transform(VectorStoreMaskNode::make(gvn(), opd, elem_bt, num_elem));
2579 }
2580 // ((toLong() >>> pos) & 1L
2581 opd = gvn().transform(new VectorMaskToLongNode(opd, TypeLong::LONG));
2582 opd = gvn().transform(new URShiftLNode(opd, pos));
2583 opd = gvn().transform(new AndLNode(opd, gvn().makecon(TypeLong::ONE)));
2584 } else {
2585 log_if_needed(" ** Rejected mask extraction because architecture does not support it");
2586 return false; // not supported
2587 }
2588 } else {
2589 // vbox_klass is vector. This is used for Vector.lane(int).
2590 if (!idx->is_con()) {
2591 log_if_needed(" ** missing constant: idx=%s", NodeClassNames[argument(4)->Opcode()]);
2592 return false; // not enough info for intrinsification
2593 }
2594
2595 int vopc = ExtractNode::opcode(elem_bt);
2596 if (!arch_supports_vector(vopc, num_elem, elem_bt, VecMaskNotUsed)) {
2597 log_if_needed(" ** not supported: arity=1 op=extract vlen=%d etype=%s ismask=no",
2598 num_elem, type2name(elem_bt));
2599 return false; // not supported
2600 }
2601
2602 opd = unbox_vector(argument(3), vbox_type, elem_bt, num_elem);
2603 if (opd == nullptr) {
2604 return false;
2605 }
2606 ConINode* idx_con = gvn().intcon(idx->get_con())->as_ConI();
2607
2608 opd = gvn().transform(ExtractNode::make(opd, idx_con, elem_bt));
2609 switch (elem_bt) {
2610 case T_BYTE:
2611 case T_SHORT:
2612 case T_INT: {
2613 opd = gvn().transform(new ConvI2LNode(opd));
2614 break;
2615 }
2616 case T_FLOAT: {
2617 opd = gvn().transform(new MoveF2INode(opd));
2618 opd = gvn().transform(new ConvI2LNode(opd));
2619 break;
2620 }
2621 case T_DOUBLE: {
2622 opd = gvn().transform(new MoveD2LNode(opd));
2623 break;
2624 }
2625 case T_LONG: {
2626 // no conversion needed
2627 break;
2628 }
2629 default: fatal("%s", type2name(elem_bt));
2630 }
2631 }
2632 set_result(opd);
2633 return true;
2634 }
2635
2636 // public static
2637 // <V extends Vector<E>,
2638 // M extends VectorMask<E>,
2639 // E>
2640 // V compressExpandOp(int opr,
2641 // Class<? extends V> vClass, Class<? extends M> mClass, Class<E> eClass,
2642 // int length, V v, M m,
2643 // CompressExpandOperation<V, M> defaultImpl)
2644 bool LibraryCallKit::inline_vector_compress_expand() {
2645 const TypeInt* opr = gvn().type(argument(0))->isa_int();
2646 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
2647 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
2648 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
2649 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
2650
2651 if (vector_klass == nullptr || elem_klass == nullptr || mask_klass == nullptr || vlen == nullptr ||
2652 vector_klass->const_oop() == nullptr || mask_klass->const_oop() == nullptr ||
2653 elem_klass->const_oop() == nullptr || !vlen->is_con() || !opr->is_con()) {
2654 log_if_needed(" ** missing constant: opr=%s vclass=%s mclass=%s etype=%s vlen=%s",
2655 NodeClassNames[argument(0)->Opcode()],
2656 NodeClassNames[argument(1)->Opcode()],
2657 NodeClassNames[argument(2)->Opcode()],
2658 NodeClassNames[argument(3)->Opcode()],
2659 NodeClassNames[argument(4)->Opcode()]);
2660 return false; // not enough info for intrinsification
2661 }
2662
2663 if (!is_klass_initialized(vector_klass) || !is_klass_initialized(mask_klass)) {
2664 log_if_needed(" ** klass argument not initialized");
2665 return false;
2666 }
2667
2668 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2669 if (!elem_type->is_primitive_type()) {
2670 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2671 return false; // should be primitive type
2672 }
2673
2674 int num_elem = vlen->get_con();
2675 BasicType elem_bt = elem_type->basic_type();
2676 int opc = VectorSupport::vop2ideal(opr->get_con(), elem_bt);
2677
2678 if (!arch_supports_vector(opc, num_elem, elem_bt, VecMaskUseLoad)) {
2679 log_if_needed(" ** not supported: opc=%d vlen=%d etype=%s ismask=useload",
2680 opc, num_elem, type2name(elem_bt));
2681 return false; // not supported
2682 }
2683
2684 Node* opd1 = nullptr;
2685 const TypeInstPtr* vbox_type = nullptr;
2686 if (opc != Op_CompressM) {
2687 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2688 vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2689 opd1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
2690 if (opd1 == nullptr) {
2691 log_if_needed(" ** unbox failed vector=%s",
2692 NodeClassNames[argument(5)->Opcode()]);
2693 return false;
2694 }
2695 }
2696
2697 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
2698 assert(is_vector_mask(mbox_klass), "argument(6) should be a mask class");
2699 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
2700
2701 Node* mask = unbox_vector(argument(6), mbox_type, elem_bt, num_elem);
2702 if (mask == nullptr) {
2703 log_if_needed(" ** unbox failed mask=%s",
2704 NodeClassNames[argument(6)->Opcode()]);
2705 return false;
2706 }
2707
2708 const TypeVect* vt = TypeVect::make(elem_bt, num_elem, opc == Op_CompressM);
2709 Node* operation = gvn().transform(VectorNode::make(opc, opd1, mask, vt));
2710
2711 // Wrap it up in VectorBox to keep object type information.
2712 const TypeInstPtr* box_type = opc == Op_CompressM ? mbox_type : vbox_type;
2713 Node* vbox = box_vector(operation, box_type, elem_bt, num_elem);
2714 set_result(vbox);
2715 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2716 return true;
2717 }
2718
2719 // public static
2720 // <V extends Vector<E>,
2721 // E,
2722 // S extends VectorSpecies<E>>
2723 // V indexVector(Class<? extends V> vClass, Class<E> eClass,
2724 // int length,
2725 // V v, int step, S s,
2726 // IndexOperation<V, S> defaultImpl)
2727 bool LibraryCallKit::inline_index_vector() {
2728 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
2729 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
2730 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
2731
2732 if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr ||
2733 vector_klass->const_oop() == nullptr || !vlen->is_con() ||
2734 elem_klass->const_oop() == nullptr) {
2735 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s",
2736 NodeClassNames[argument(0)->Opcode()],
2737 NodeClassNames[argument(1)->Opcode()],
2738 NodeClassNames[argument(2)->Opcode()]);
2739 return false; // not enough info for intrinsification
2740 }
2741
2742 if (!is_klass_initialized(vector_klass)) {
2743 log_if_needed(" ** klass argument not initialized");
2744 return false;
2745 }
2746
2747 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2748 if (!elem_type->is_primitive_type()) {
2749 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2750 return false; // should be primitive type
2751 }
2752
2753 int num_elem = vlen->get_con();
2754 BasicType elem_bt = elem_type->basic_type();
2755
2756 // Check whether the iota index generation op is supported by the current hardware
2757 if (!arch_supports_vector(Op_VectorLoadConst, num_elem, elem_bt, VecMaskNotUsed)) {
2758 log_if_needed(" ** not supported: vlen=%d etype=%s", num_elem, type2name(elem_bt));
2759 return false; // not supported
2760 }
2761
2762 int mul_op = VectorSupport::vop2ideal(VectorSupport::VECTOR_OP_MUL, elem_bt);
2763 int vmul_op = VectorNode::opcode(mul_op, elem_bt);
2764 bool needs_mul = true;
2765 Node* scale = argument(4);
2766 const TypeInt* scale_type = gvn().type(scale)->isa_int();
2767 // Multiply is not needed if the scale is a constant "1".
2768 if (scale_type && scale_type->is_con() && scale_type->get_con() == 1) {
2769 needs_mul = false;
2770 } else {
2771 // Check whether the vector multiply op is supported by the current hardware
2772 if (!arch_supports_vector(vmul_op, num_elem, elem_bt, VecMaskNotUsed)) {
2773 log_if_needed(" ** not supported: vlen=%d etype=%s", num_elem, type2name(elem_bt));
2774 return false; // not supported
2775 }
2776
2777 // Check whether the scalar cast op is supported by the current hardware
2778 if (is_floating_point_type(elem_bt) || elem_bt == T_LONG) {
2779 int cast_op = elem_bt == T_LONG ? Op_ConvI2L :
2780 elem_bt == T_FLOAT? Op_ConvI2F : Op_ConvI2D;
2781 if (!Matcher::match_rule_supported(cast_op)) {
2782 log_if_needed(" ** Rejected op (%s) because architecture does not support it",
2783 NodeClassNames[cast_op]);
2784 return false; // not supported
2785 }
2786 }
2787 }
2788
2789 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2790 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2791 Node* opd = unbox_vector(argument(3), vbox_type, elem_bt, num_elem);
2792 if (opd == nullptr) {
2793 log_if_needed(" ** unbox failed vector=%s",
2794 NodeClassNames[argument(3)->Opcode()]);
2795 return false;
2796 }
2797
2798 int add_op = VectorSupport::vop2ideal(VectorSupport::VECTOR_OP_ADD, elem_bt);
2799 int vadd_op = VectorNode::opcode(add_op, elem_bt);
2800 bool needs_add = true;
2801 // The addition is not needed if all the element values of "opd" are zero
2802 if (VectorNode::is_all_zeros_vector(opd)) {
2803 needs_add = false;
2804 } else {
2805 // Check whether the vector addition op is supported by the current hardware
2806 if (!arch_supports_vector(vadd_op, num_elem, elem_bt, VecMaskNotUsed)) {
2807 log_if_needed(" ** not supported: vlen=%d etype=%s", num_elem, type2name(elem_bt));
2808 return false; // not supported
2809 }
2810 }
2811
2812 // Compute the iota indice vector
2813 const TypeVect* vt = TypeVect::make(elem_bt, num_elem);
2814 Node* index = gvn().transform(new VectorLoadConstNode(gvn().makecon(TypeInt::ZERO), vt));
2815
2816 // Broadcast the "scale" to a vector, and multiply the "scale" with iota indice vector.
2817 if (needs_mul) {
2818 switch (elem_bt) {
2819 case T_BOOLEAN: // fall-through
2820 case T_BYTE: // fall-through
2821 case T_SHORT: // fall-through
2822 case T_CHAR: // fall-through
2823 case T_INT: {
2824 // no conversion needed
2825 break;
2826 }
2827 case T_LONG: {
2828 scale = gvn().transform(new ConvI2LNode(scale));
2829 break;
2830 }
2831 case T_FLOAT: {
2832 scale = gvn().transform(new ConvI2FNode(scale));
2833 break;
2834 }
2835 case T_DOUBLE: {
2836 scale = gvn().transform(new ConvI2DNode(scale));
2837 break;
2838 }
2839 default: fatal("%s", type2name(elem_bt));
2840 }
2841 scale = gvn().transform(VectorNode::scalar2vector(scale, num_elem, Type::get_const_basic_type(elem_bt)));
2842 index = gvn().transform(VectorNode::make(vmul_op, index, scale, vt));
2843 }
2844
2845 // Add "opd" if addition is needed.
2846 if (needs_add) {
2847 index = gvn().transform(VectorNode::make(vadd_op, opd, index, vt));
2848 }
2849 Node* vbox = box_vector(index, vbox_type, elem_bt, num_elem);
2850 set_result(vbox);
2851 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2852 return true;
2853 }
2854
2855 // public static
2856 // <E,
2857 // M extends VectorMask<E>>
2858 // M indexPartiallyInUpperRange(Class<? extends M> mClass, Class<E> eClass, int length,
2859 // long offset, long limit,
2860 // IndexPartiallyInUpperRangeOperation<E, M> defaultImpl)
2861 bool LibraryCallKit::inline_index_partially_in_upper_range() {
2862 const TypeInstPtr* mask_klass = gvn().type(argument(0))->isa_instptr();
2863 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
2864 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
2865
2866 if (mask_klass == nullptr || elem_klass == nullptr || vlen == nullptr ||
2867 mask_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
2868 log_if_needed(" ** missing constant: mclass=%s etype=%s vlen=%s",
2869 NodeClassNames[argument(0)->Opcode()],
2870 NodeClassNames[argument(1)->Opcode()],
2871 NodeClassNames[argument(2)->Opcode()]);
2872 return false; // not enough info for intrinsification
2873 }
2874
2875 if (!is_klass_initialized(mask_klass)) {
2876 log_if_needed(" ** klass argument not initialized");
2877 return false;
2878 }
2879
2880 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2881 if (!elem_type->is_primitive_type()) {
2882 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2883 return false; // should be primitive type
2884 }
2885
2886 int num_elem = vlen->get_con();
2887 BasicType elem_bt = elem_type->basic_type();
2888
2889 // Check whether the necessary ops are supported by current hardware.
2890 bool supports_mask_gen = arch_supports_vector(Op_VectorMaskGen, num_elem, elem_bt, VecMaskUseStore);
2891 if (!supports_mask_gen) {
2892 if (!arch_supports_vector(Op_VectorLoadConst, num_elem, elem_bt, VecMaskNotUsed) ||
2893 !arch_supports_vector(Op_Replicate, num_elem, elem_bt, VecMaskNotUsed) ||
2894 !arch_supports_vector(Op_VectorMaskCmp, num_elem, elem_bt, VecMaskUseStore)) {
2895 log_if_needed(" ** not supported: vlen=%d etype=%s", num_elem, type2name(elem_bt));
2896 return false; // not supported
2897 }
2898
2899 // Check whether the scalar cast operation is supported by current hardware.
2900 if (elem_bt != T_LONG) {
2901 int cast_op = is_integral_type(elem_bt) ? Op_ConvL2I
2902 : (elem_bt == T_FLOAT ? Op_ConvL2F : Op_ConvL2D);
2903 if (!Matcher::match_rule_supported(cast_op)) {
2904 log_if_needed(" ** Rejected op (%s) because architecture does not support it",
2905 NodeClassNames[cast_op]);
2906 return false; // not supported
2907 }
2908 }
2909 }
2910
2911 Node* offset = argument(3);
2912 Node* limit = argument(5);
2913 if (offset == nullptr || limit == nullptr) {
2914 log_if_needed(" ** offset or limit argument is null");
2915 return false; // not supported
2916 }
2917
2918 ciKlass* box_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
2919 assert(is_vector_mask(box_klass), "argument(0) should be a mask class");
2920 const TypeInstPtr* box_type = TypeInstPtr::make_exact(TypePtr::NotNull, box_klass);
2921
2922 // We assume "offset > 0 && limit >= offset && limit - offset < num_elem".
2923 // So directly get indexLimit with "indexLimit = limit - offset".
2924 Node* indexLimit = gvn().transform(new SubLNode(limit, offset));
2925 Node* mask = nullptr;
2926 if (supports_mask_gen) {
2927 mask = gvn().transform(VectorMaskGenNode::make(indexLimit, elem_bt, num_elem));
2928 } else {
2929 // Generate the vector mask based on "mask = iota < indexLimit".
2930 // Broadcast "indexLimit" to a vector.
2931 switch (elem_bt) {
2932 case T_BOOLEAN: // fall-through
2933 case T_BYTE: // fall-through
2934 case T_SHORT: // fall-through
2935 case T_CHAR: // fall-through
2936 case T_INT: {
2937 indexLimit = gvn().transform(new ConvL2INode(indexLimit));
2938 break;
2939 }
2940 case T_DOUBLE: {
2941 indexLimit = gvn().transform(new ConvL2DNode(indexLimit));
2942 break;
2943 }
2944 case T_FLOAT: {
2945 indexLimit = gvn().transform(new ConvL2FNode(indexLimit));
2946 break;
2947 }
2948 case T_LONG: {
2949 // no conversion needed
2950 break;
2951 }
2952 default: fatal("%s", type2name(elem_bt));
2953 }
2954 indexLimit = gvn().transform(VectorNode::scalar2vector(indexLimit, num_elem, Type::get_const_basic_type(elem_bt)));
2955
2956 // Load the "iota" vector.
2957 const TypeVect* vt = TypeVect::make(elem_bt, num_elem);
2958 Node* iota = gvn().transform(new VectorLoadConstNode(gvn().makecon(TypeInt::ZERO), vt));
2959
2960 // Compute the vector mask with "mask = iota < indexLimit".
2961 ConINode* pred_node = (ConINode*)gvn().makecon(TypeInt::make(BoolTest::lt));
2962 const TypeVect* vmask_type = TypeVect::makemask(elem_bt, num_elem);
2963 mask = gvn().transform(new VectorMaskCmpNode(BoolTest::lt, iota, indexLimit, pred_node, vmask_type));
2964 }
2965 Node* vbox = box_vector(mask, box_type, elem_bt, num_elem);
2966 set_result(vbox);
2967 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2968 return true;
2969 }
2970
2971 #undef non_product_log_if_needed
2972 #undef log_if_needed