1 /*
2 * Copyright (c) 2020, 2025, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "ci/ciSymbols.hpp"
26 #include "classfile/vmSymbols.hpp"
27 #include "opto/library_call.hpp"
28 #include "opto/runtime.hpp"
29 #include "opto/vectornode.hpp"
30 #include "prims/vectorSupport.hpp"
31 #include "runtime/stubRoutines.hpp"
32
33 #ifdef ASSERT
34 static bool is_vector(ciKlass* klass) {
35 return klass->is_subclass_of(ciEnv::current()->vector_VectorPayload_klass());
36 }
37
38 static bool check_vbox(const TypeInstPtr* vbox_type) {
39 assert(vbox_type->klass_is_exact(), "");
40
41 ciInstanceKlass* ik = vbox_type->instance_klass();
42 assert(is_vector(ik), "not a vector");
43
44 ciField* fd1 = ik->get_field_by_name(ciSymbols::ETYPE_name(), ciSymbols::class_signature(), /* is_static */ true);
45 assert(fd1 != nullptr, "element type info is missing");
46
47 ciConstant val1 = fd1->constant_value();
48 BasicType elem_bt = val1.as_object()->as_instance()->java_mirror_type()->basic_type();
49 assert(is_java_primitive(elem_bt), "element type info is missing");
50
51 ciField* fd2 = ik->get_field_by_name(ciSymbols::VLENGTH_name(), ciSymbols::int_signature(), /* is_static */ true);
52 assert(fd2 != nullptr, "vector length info is missing");
53
54 ciConstant val2 = fd2->constant_value();
55 assert(val2.as_int() > 0, "vector length info is missing");
56
57 return true;
58 }
59 #endif
60
61 #define log_if_needed(...) \
62 if (C->print_intrinsics()) { \
63 tty->print_cr(__VA_ARGS__); \
64 }
65
66 #ifndef PRODUCT
67 #define non_product_log_if_needed(...) log_if_needed(__VA_ARGS__)
68 #else
69 #define non_product_log_if_needed(...)
70 #endif
71
72 static bool is_vector_mask(ciKlass* klass) {
73 return klass->is_subclass_of(ciEnv::current()->vector_VectorMask_klass());
74 }
75
76 bool LibraryCallKit::arch_supports_vector_rotate(int opc, int num_elem, BasicType elem_bt,
77 VectorMaskUseType mask_use_type, bool has_scalar_args) {
78 bool is_supported = true;
79
80 // has_scalar_args flag is true only for non-constant scalar shift count,
81 // since in this case shift needs to be broadcasted.
82 if (!Matcher::match_rule_supported_vector(opc, num_elem, elem_bt) ||
83 (has_scalar_args && !arch_supports_vector(Op_Replicate, num_elem, elem_bt, VecMaskNotUsed))) {
84 is_supported = false;
85 }
86
87 if (is_supported) {
88 // Check if mask unboxing is supported, this is a two step process which first loads the contents
89 // of boolean array into vector followed by either lane expansion to match the lane size of masked
90 // vector operation or populate the predicate register.
91 if ((mask_use_type & VecMaskUseLoad) != 0) {
92 if (!Matcher::match_rule_supported_vector(Op_VectorLoadMask, num_elem, elem_bt) ||
93 !Matcher::match_rule_supported_vector(Op_LoadVector, num_elem, T_BOOLEAN)) {
94 non_product_log_if_needed(" ** Rejected vector mask loading (%s,%s,%d) because architecture does not support it",
95 NodeClassNames[Op_VectorLoadMask], type2name(elem_bt), num_elem);
96 return false;
97 }
98 }
99
100 if ((mask_use_type & VecMaskUsePred) != 0) {
101 if (!Matcher::has_predicated_vectors() ||
102 !Matcher::match_rule_supported_vector_masked(opc, num_elem, elem_bt)) {
103 non_product_log_if_needed("Rejected vector mask predicate using (%s,%s,%d) because architecture does not support it",
104 NodeClassNames[opc], type2name(elem_bt), num_elem);
105 return false;
106 }
107 }
108 }
109
110 int lshiftopc, rshiftopc;
111 switch(elem_bt) {
112 case T_BYTE:
113 lshiftopc = Op_LShiftI;
114 rshiftopc = Op_URShiftB;
115 break;
116 case T_SHORT:
117 lshiftopc = Op_LShiftI;
118 rshiftopc = Op_URShiftS;
119 break;
120 case T_INT:
121 lshiftopc = Op_LShiftI;
122 rshiftopc = Op_URShiftI;
123 break;
124 case T_LONG:
125 lshiftopc = Op_LShiftL;
126 rshiftopc = Op_URShiftL;
127 break;
128 default: fatal("Unexpected type: %s", type2name(elem_bt));
129 }
130 int lshiftvopc = VectorNode::opcode(lshiftopc, elem_bt);
131 int rshiftvopc = VectorNode::opcode(rshiftopc, elem_bt);
132 if (!is_supported &&
133 arch_supports_vector(lshiftvopc, num_elem, elem_bt, VecMaskNotUsed, has_scalar_args) &&
134 arch_supports_vector(rshiftvopc, num_elem, elem_bt, VecMaskNotUsed, has_scalar_args) &&
135 arch_supports_vector(Op_OrV, num_elem, elem_bt, VecMaskNotUsed)) {
136 is_supported = true;
137 }
138 return is_supported;
139 }
140
141 Node* GraphKit::box_vector(Node* vector, const TypeInstPtr* vbox_type, BasicType elem_bt, int num_elem, bool deoptimize_on_exception) {
142 assert(EnableVectorSupport, "");
143
144 PreserveReexecuteState preexecs(this);
145 jvms()->set_should_reexecute(true);
146
147 VectorBoxAllocateNode* alloc = new VectorBoxAllocateNode(C, vbox_type);
148 set_edges_for_java_call(alloc, /*must_throw=*/false, /*separate_io_proj=*/true);
149 make_slow_call_ex(alloc, env()->Throwable_klass(), /*separate_io_proj=*/true, deoptimize_on_exception);
150 set_i_o(gvn().transform( new ProjNode(alloc, TypeFunc::I_O) ));
151 set_all_memory(gvn().transform( new ProjNode(alloc, TypeFunc::Memory) ));
152 Node* ret = gvn().transform(new ProjNode(alloc, TypeFunc::Parms));
153
154 assert(check_vbox(vbox_type), "");
155 const TypeVect* vt = TypeVect::make(elem_bt, num_elem, is_vector_mask(vbox_type->instance_klass()));
156 VectorBoxNode* vbox = new VectorBoxNode(C, ret, vector, vbox_type, vt);
157 return gvn().transform(vbox);
158 }
159
160 Node* GraphKit::unbox_vector(Node* v, const TypeInstPtr* vbox_type, BasicType elem_bt, int num_elem) {
161 assert(EnableVectorSupport, "");
162 const TypeInstPtr* vbox_type_v = gvn().type(v)->isa_instptr();
163 if (vbox_type_v == nullptr || vbox_type->instance_klass() != vbox_type_v->instance_klass()) {
164 return nullptr; // arguments don't agree on vector shapes
165 }
166 if (vbox_type_v->maybe_null()) {
167 return nullptr; // no nulls are allowed
168 }
169 assert(check_vbox(vbox_type), "");
170 const TypeVect* vt = TypeVect::make(elem_bt, num_elem, is_vector_mask(vbox_type->instance_klass()));
171 Node* unbox = gvn().transform(new VectorUnboxNode(C, vt, v, merged_memory()));
172 return unbox;
173 }
174
175 Node* GraphKit::vector_shift_count(Node* cnt, int shift_op, BasicType bt, int num_elem) {
176 assert(bt == T_INT || bt == T_LONG || bt == T_SHORT || bt == T_BYTE, "byte, short, long and int are supported");
177 juint mask = (type2aelembytes(bt) * BitsPerByte - 1);
178 Node* nmask = gvn().transform(ConNode::make(TypeInt::make(mask)));
179 Node* mcnt = gvn().transform(new AndINode(cnt, nmask));
180 return gvn().transform(VectorNode::shift_count(shift_op, mcnt, num_elem, bt));
181 }
182
183 bool LibraryCallKit::arch_supports_vector(int sopc, int num_elem, BasicType type, VectorMaskUseType mask_use_type, bool has_scalar_args) {
184 // Check that the operation is valid.
185 if (sopc <= 0) {
186 non_product_log_if_needed(" ** Rejected intrinsification because no valid vector op could be extracted");
187 return false;
188 }
189
190 if (VectorNode::is_vector_rotate(sopc)) {
191 if(!arch_supports_vector_rotate(sopc, num_elem, type, mask_use_type, has_scalar_args)) {
192 non_product_log_if_needed(" ** Rejected vector op (%s,%s,%d) because architecture does not support variable vector shifts",
193 NodeClassNames[sopc], type2name(type), num_elem);
194 return false;
195 }
196 } else if (VectorNode::is_vector_integral_negate(sopc)) {
197 if (!VectorNode::is_vector_integral_negate_supported(sopc, num_elem, type, false)) {
198 non_product_log_if_needed(" ** Rejected vector op (%s,%s,%d) because architecture does not support integral vector negate",
199 NodeClassNames[sopc], type2name(type), num_elem);
200 return false;
201 }
202 } else {
203 // Check that architecture supports this op-size-type combination.
204 if (!Matcher::match_rule_supported_vector(sopc, num_elem, type)) {
205 non_product_log_if_needed(" ** Rejected vector op (%s,%s,%d) because architecture does not support it",
206 NodeClassNames[sopc], type2name(type), num_elem);
207 return false;
208 } else {
209 assert(Matcher::match_rule_supported(sopc), "must be supported");
210 }
211 }
212
213 if (num_elem == 1) {
214 if (mask_use_type != VecMaskNotUsed) {
215 non_product_log_if_needed(" ** Rejected vector mask op (%s,%s,%d) because architecture does not support it",
216 NodeClassNames[sopc], type2name(type), num_elem);
217 return false;
218 }
219
220 if (sopc != 0) {
221 if (sopc != Op_LoadVector && sopc != Op_StoreVector) {
222 non_product_log_if_needed(" ** Not a svml call or load/store vector op (%s,%s,%d)",
223 NodeClassNames[sopc], type2name(type), num_elem);
224 return false;
225 }
226 }
227 }
228
229 if (!has_scalar_args && VectorNode::is_vector_shift(sopc) &&
230 Matcher::supports_vector_variable_shifts() == false) {
231 log_if_needed(" ** Rejected vector op (%s,%s,%d) because architecture does not support variable vector shifts",
232 NodeClassNames[sopc], type2name(type), num_elem);
233 return false;
234 }
235
236 // Check if mask unboxing is supported, this is a two step process which first loads the contents
237 // of boolean array into vector followed by either lane expansion to match the lane size of masked
238 // vector operation or populate the predicate register.
239 if ((mask_use_type & VecMaskUseLoad) != 0) {
240 if (!Matcher::match_rule_supported_vector(Op_VectorLoadMask, num_elem, type) ||
241 !Matcher::match_rule_supported_vector(Op_LoadVector, num_elem, T_BOOLEAN)) {
242 non_product_log_if_needed(" ** Rejected vector mask loading (%s,%s,%d) because architecture does not support it",
243 NodeClassNames[Op_VectorLoadMask], type2name(type), num_elem);
244 return false;
245 }
246 }
247
248 // Check if mask boxing is supported, this is a two step process which first stores the contents
249 // of mask vector / predicate register into a boolean vector followed by vector store operation to
250 // transfer the contents to underlined storage of mask boxes which is a boolean array.
251 if ((mask_use_type & VecMaskUseStore) != 0) {
252 if (!Matcher::match_rule_supported_vector(Op_VectorStoreMask, num_elem, type) ||
253 !Matcher::match_rule_supported_vector(Op_StoreVector, num_elem, T_BOOLEAN)) {
254 non_product_log_if_needed("Rejected vector mask storing (%s,%s,%d) because architecture does not support it",
255 NodeClassNames[Op_VectorStoreMask], type2name(type), num_elem);
256 return false;
257 }
258 }
259
260 if ((mask_use_type & VecMaskUsePred) != 0) {
261 bool is_supported = false;
262 if (Matcher::has_predicated_vectors()) {
263 if (VectorNode::is_vector_integral_negate(sopc)) {
264 is_supported = VectorNode::is_vector_integral_negate_supported(sopc, num_elem, type, true);
265 } else {
266 is_supported = Matcher::match_rule_supported_vector_masked(sopc, num_elem, type);
267 }
268 }
269 is_supported |= Matcher::supports_vector_predicate_op_emulation(sopc, num_elem, type);
270
271 if (!is_supported) {
272 non_product_log_if_needed("Rejected vector mask predicate using (%s,%s,%d) because architecture does not support it",
273 NodeClassNames[sopc], type2name(type), num_elem);
274 return false;
275 }
276 }
277
278 return true;
279 }
280
281 static bool is_klass_initialized(const TypeInstPtr* vec_klass) {
282 if (vec_klass->const_oop() == nullptr) {
283 return false; // uninitialized or some kind of unsafe access
284 }
285 assert(vec_klass->const_oop()->as_instance()->java_lang_Class_klass() != nullptr, "klass instance expected");
286 ciInstanceKlass* klass = vec_klass->const_oop()->as_instance()->java_lang_Class_klass()->as_instance_klass();
287 return klass->is_initialized();
288 }
289
290 // public static
291 // <V extends Vector<E>,
292 // M extends VectorMask<E>,
293 // E>
294 // V unaryOp(int oprId, Class<? extends V> vmClass, Class<? extends M> maskClass, Class<E> elementType,
295 // int length, V v, M m,
296 // UnaryOperation<V, M> defaultImpl)
297 //
298 // public static
299 // <V,
300 // M extends VectorMask<E>,
301 // E>
302 // V binaryOp(int oprId, Class<? extends V> vmClass, Class<? extends M> maskClass, Class<E> elementType,
303 // int length, V v1, V v2, M m,
304 // BinaryOperation<V, M> defaultImpl)
305 //
306 // public static
307 // <V extends Vector<E>,
308 // M extends VectorMask<E>,
309 // E>
310 // V ternaryOp(int oprId, Class<? extends V> vmClass, Class<? extends M> maskClass, Class<E> elementType,
311 // int length, V v1, V v2, V v3, M m,
312 // TernaryOperation<V, M> defaultImpl)
313 //
314 bool LibraryCallKit::inline_vector_nary_operation(int n) {
315 const TypeInt* opr = gvn().type(argument(0))->isa_int();
316 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
317 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
318 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
319 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
320
321 if (opr == nullptr || !opr->is_con() ||
322 vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
323 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
324 vlen == nullptr || !vlen->is_con()) {
325 log_if_needed(" ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
326 NodeClassNames[argument(0)->Opcode()],
327 NodeClassNames[argument(1)->Opcode()],
328 NodeClassNames[argument(3)->Opcode()],
329 NodeClassNames[argument(4)->Opcode()]);
330 return false; // not enough info for intrinsification
331 }
332
333 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
334 if (!elem_type->is_primitive_type()) {
335 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
336 return false; // should be primitive type
337 }
338 if (!is_klass_initialized(vector_klass)) {
339 log_if_needed(" ** klass argument not initialized");
340 return false;
341 }
342
343 // "argument(n + 5)" should be the mask object. We assume it is "null" when no mask
344 // is used to control this operation.
345 const Type* vmask_type = gvn().type(argument(n + 5));
346 bool is_masked_op = vmask_type != TypePtr::NULL_PTR;
347 if (is_masked_op) {
348 if (mask_klass == nullptr || mask_klass->const_oop() == nullptr) {
349 log_if_needed(" ** missing constant: maskclass=%s", NodeClassNames[argument(2)->Opcode()]);
350 return false; // not enough info for intrinsification
351 }
352
353 if (!is_klass_initialized(mask_klass)) {
354 log_if_needed(" ** mask klass argument not initialized");
355 return false;
356 }
357
358 if (vmask_type->maybe_null()) {
359 log_if_needed(" ** null mask values are not allowed for masked op");
360 return false;
361 }
362 }
363
364 BasicType elem_bt = elem_type->basic_type();
365 bool has_scalar_op = VectorSupport::has_scalar_op(opr->get_con());
366 bool is_unsigned = VectorSupport::is_unsigned_op(opr->get_con());
367
368 int num_elem = vlen->get_con();
369 int opc = VectorSupport::vop2ideal(opr->get_con(), elem_bt);
370 int sopc = has_scalar_op ? VectorNode::opcode(opc, elem_bt) : opc;
371 if ((opc != Op_CallLeafVector) && (sopc == 0)) {
372 log_if_needed(" ** operation not supported: opc=%s bt=%s", NodeClassNames[opc], type2name(elem_bt));
373 return false; // operation not supported
374 }
375 if (num_elem == 1) {
376 if (opc != Op_CallLeafVector || elem_bt != T_DOUBLE) {
377 log_if_needed(" ** not a svml call: arity=%d opc=%d vlen=%d etype=%s",
378 n, opc, num_elem, type2name(elem_bt));
379 return false;
380 }
381 }
382 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
383 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
384
385 if (is_vector_mask(vbox_klass)) {
386 assert(!is_masked_op, "mask operations do not need mask to control");
387 }
388
389 if (opc == Op_CallLeafVector) {
390 if (!UseVectorStubs) {
391 log_if_needed(" ** vector stubs support is disabled");
392 return false;
393 }
394 if (!Matcher::supports_vector_calling_convention()) {
395 log_if_needed(" ** no vector calling conventions supported");
396 return false;
397 }
398 if (!Matcher::vector_size_supported(elem_bt, num_elem)) {
399 log_if_needed(" ** vector size (vlen=%d, etype=%s) is not supported",
400 num_elem, type2name(elem_bt));
401 return false;
402 }
403 }
404
405 // When using mask, mask use type needs to be VecMaskUseLoad.
406 VectorMaskUseType mask_use_type = is_vector_mask(vbox_klass) ? VecMaskUseAll
407 : is_masked_op ? VecMaskUseLoad : VecMaskNotUsed;
408 if ((sopc != 0) && !arch_supports_vector(sopc, num_elem, elem_bt, mask_use_type)) {
409 log_if_needed(" ** not supported: arity=%d opc=%d vlen=%d etype=%s ismask=%d is_masked_op=%d",
410 n, sopc, num_elem, type2name(elem_bt),
411 is_vector_mask(vbox_klass) ? 1 : 0, is_masked_op ? 1 : 0);
412 return false; // not supported
413 }
414
415 // Return true if current platform has implemented the masked operation with predicate feature.
416 bool use_predicate = is_masked_op && sopc != 0 && arch_supports_vector(sopc, num_elem, elem_bt, VecMaskUsePred);
417 if (is_masked_op && !use_predicate && !arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad)) {
418 log_if_needed(" ** not supported: arity=%d opc=%d vlen=%d etype=%s ismask=0 is_masked_op=1",
419 n, sopc, num_elem, type2name(elem_bt));
420 return false;
421 }
422
423 Node* opd1 = nullptr; Node* opd2 = nullptr; Node* opd3 = nullptr;
424 switch (n) {
425 case 3: {
426 opd3 = unbox_vector(argument(7), vbox_type, elem_bt, num_elem);
427 if (opd3 == nullptr) {
428 log_if_needed(" ** unbox failed v3=%s",
429 NodeClassNames[argument(7)->Opcode()]);
430 return false;
431 }
432 // fall-through
433 }
434 case 2: {
435 opd2 = unbox_vector(argument(6), vbox_type, elem_bt, num_elem);
436 if (opd2 == nullptr) {
437 log_if_needed(" ** unbox failed v2=%s",
438 NodeClassNames[argument(6)->Opcode()]);
439 return false;
440 }
441 // fall-through
442 }
443 case 1: {
444 opd1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
445 if (opd1 == nullptr) {
446 log_if_needed(" ** unbox failed v1=%s",
447 NodeClassNames[argument(5)->Opcode()]);
448 return false;
449 }
450 break;
451 }
452 default: fatal("unsupported arity: %d", n);
453 }
454
455 Node* mask = nullptr;
456 if (is_masked_op) {
457 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
458 assert(is_vector_mask(mbox_klass), "argument(2) should be a mask class");
459 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
460 mask = unbox_vector(argument(n + 5), mbox_type, elem_bt, num_elem);
461 if (mask == nullptr) {
462 log_if_needed(" ** unbox failed mask=%s",
463 NodeClassNames[argument(n + 5)->Opcode()]);
464 return false;
465 }
466 }
467
468 Node* operation = nullptr;
469 if (opc == Op_CallLeafVector) {
470 assert(UseVectorStubs, "sanity");
471 operation = gen_call_to_vector_math(opr->get_con(), elem_bt, num_elem, opd1, opd2);
472 if (operation == nullptr) {
473 log_if_needed(" ** Vector math call failed for %s_%s_%d",
474 (elem_bt == T_FLOAT) ? "float" : "double",
475 VectorSupport::mathname[opr->get_con() - VectorSupport::VECTOR_OP_MATH_START],
476 num_elem * type2aelembytes(elem_bt));
477 return false;
478 }
479 } else {
480 const TypeVect* vt = TypeVect::make(elem_bt, num_elem, is_vector_mask(vbox_klass));
481 switch (n) {
482 case 1:
483 case 2: {
484 operation = VectorNode::make(sopc, opd1, opd2, vt, is_vector_mask(vbox_klass), VectorNode::is_shift_opcode(opc), is_unsigned);
485 break;
486 }
487 case 3: {
488 operation = VectorNode::make(sopc, opd1, opd2, opd3, vt);
489 break;
490 }
491 default: fatal("unsupported arity: %d", n);
492 }
493 }
494
495 if (is_masked_op && mask != nullptr) {
496 if (use_predicate) {
497 operation->add_req(mask);
498 operation->add_flag(Node::Flag_is_predicated_vector);
499 } else {
500 operation->add_flag(Node::Flag_is_predicated_using_blend);
501 operation = gvn().transform(operation);
502 operation = new VectorBlendNode(opd1, operation, mask);
503 }
504 }
505 operation = gvn().transform(operation);
506
507 // Wrap it up in VectorBox to keep object type information.
508 Node* vbox = box_vector(operation, vbox_type, elem_bt, num_elem);
509 set_result(vbox);
510 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
511 return true;
512 }
513
514 // public static
515 // <E, M>
516 // long maskReductionCoerced(int oper, Class<? extends M> maskClass, Class<?> elemClass,
517 // int length, M m, VectorMaskOp<M> defaultImpl)
518 bool LibraryCallKit::inline_vector_mask_operation() {
519 const TypeInt* oper = gvn().type(argument(0))->isa_int();
520 const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
521 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
522 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
523 Node* mask = argument(4);
524
525 if (mask_klass == nullptr || mask_klass->const_oop() == nullptr ||
526 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
527 vlen == nullptr || !vlen->is_con() ||
528 oper == nullptr || !oper->is_con() ||
529 mask->is_top()) {
530 return false; // dead code
531 }
532
533 if (!is_klass_initialized(mask_klass)) {
534 log_if_needed(" ** klass argument not initialized");
535 return false;
536 }
537
538 int num_elem = vlen->get_con();
539 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
540 BasicType elem_bt = elem_type->basic_type();
541
542 int mopc = VectorSupport::vop2ideal(oper->get_con(), elem_bt);
543 if (!arch_supports_vector(mopc, num_elem, elem_bt, VecMaskUseLoad)) {
544 log_if_needed(" ** not supported: arity=1 op=cast#%d/3 vlen2=%d etype2=%s",
545 mopc, num_elem, type2name(elem_bt));
546 return false; // not supported
547 }
548
549 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
550 const TypeInstPtr* mask_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
551 Node* mask_vec = unbox_vector(mask, mask_box_type, elem_bt, num_elem);
552 if (mask_vec == nullptr) {
553 log_if_needed(" ** unbox failed mask=%s",
554 NodeClassNames[argument(4)->Opcode()]);
555 return false;
556 }
557
558 if (mask_vec->bottom_type()->isa_vectmask() == nullptr) {
559 mask_vec = gvn().transform(VectorStoreMaskNode::make(gvn(), mask_vec, elem_bt, num_elem));
560 }
561 const Type* maskoper_ty = mopc == Op_VectorMaskToLong ? (const Type*)TypeLong::LONG : (const Type*)TypeInt::INT;
562 Node* maskoper = gvn().transform(VectorMaskOpNode::make(mask_vec, maskoper_ty, mopc));
563 if (mopc != Op_VectorMaskToLong) {
564 maskoper = ConvI2L(maskoper);
565 }
566 set_result(maskoper);
567
568 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
569 return true;
570 }
571
572 // public static
573 // <M,
574 // S extends VectorSpecies<E>,
575 // E>
576 // M fromBitsCoerced(Class<? extends M> vmClass, Class<E> elementType, int length,
577 // long bits, int mode, S s,
578 // BroadcastOperation<M, E, S> defaultImpl)
579 bool LibraryCallKit::inline_vector_frombits_coerced() {
580 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
581 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
582 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
583 const TypeLong* bits_type = gvn().type(argument(3))->isa_long();
584 // Mode argument determines the mode of operation it can take following values:-
585 // MODE_BROADCAST for vector Vector.broadcast and VectorMask.maskAll operations.
586 // MODE_BITS_COERCED_LONG_TO_MASK for VectorMask.fromLong operation.
587 const TypeInt* mode = gvn().type(argument(5))->isa_int();
588
589 if (vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
590 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
591 vlen == nullptr || !vlen->is_con() ||
592 bits_type == nullptr ||
593 mode == nullptr || !mode->is_con()) {
594 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s bitwise=%s",
595 NodeClassNames[argument(0)->Opcode()],
596 NodeClassNames[argument(1)->Opcode()],
597 NodeClassNames[argument(2)->Opcode()],
598 NodeClassNames[argument(5)->Opcode()]);
599 return false; // not enough info for intrinsification
600 }
601
602 if (!is_klass_initialized(vector_klass)) {
603 log_if_needed(" ** klass argument not initialized");
604 return false;
605 }
606 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
607 if (!elem_type->is_primitive_type()) {
608 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
609 return false; // should be primitive type
610 }
611 BasicType elem_bt = elem_type->basic_type();
612 int num_elem = vlen->get_con();
613 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
614 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
615
616 bool is_mask = is_vector_mask(vbox_klass);
617 int bcast_mode = mode->get_con();
618 VectorMaskUseType checkFlags = (VectorMaskUseType)(is_mask ? VecMaskUseAll : VecMaskNotUsed);
619 int opc = bcast_mode == VectorSupport::MODE_BITS_COERCED_LONG_TO_MASK ? Op_VectorLongToMask : Op_Replicate;
620
621 if (!arch_supports_vector(opc, num_elem, elem_bt, checkFlags, true /*has_scalar_args*/)) {
622 log_if_needed(" ** not supported: arity=0 op=broadcast vlen=%d etype=%s ismask=%d bcast_mode=%d",
623 num_elem, type2name(elem_bt),
624 is_mask ? 1 : 0,
625 bcast_mode);
626 return false; // not supported
627 }
628
629 Node* broadcast = nullptr;
630 Node* bits = argument(3);
631 Node* elem = bits;
632
633 if (opc == Op_VectorLongToMask) {
634 const TypeVect* vt = TypeVect::makemask(elem_bt, num_elem);
635 if (vt->isa_vectmask()) {
636 broadcast = gvn().transform(new VectorLongToMaskNode(elem, vt));
637 } else {
638 const TypeVect* mvt = TypeVect::make(T_BOOLEAN, num_elem);
639 broadcast = gvn().transform(new VectorLongToMaskNode(elem, mvt));
640 broadcast = gvn().transform(new VectorLoadMaskNode(broadcast, vt));
641 }
642 } else {
643 switch (elem_bt) {
644 case T_BOOLEAN: // fall-through
645 case T_BYTE: // fall-through
646 case T_SHORT: // fall-through
647 case T_CHAR: // fall-through
648 case T_INT: {
649 elem = gvn().transform(new ConvL2INode(bits));
650 break;
651 }
652 case T_DOUBLE: {
653 elem = gvn().transform(new MoveL2DNode(bits));
654 break;
655 }
656 case T_FLOAT: {
657 bits = gvn().transform(new ConvL2INode(bits));
658 elem = gvn().transform(new MoveI2FNode(bits));
659 break;
660 }
661 case T_LONG: {
662 // no conversion needed
663 break;
664 }
665 default: fatal("%s", type2name(elem_bt));
666 }
667 broadcast = VectorNode::scalar2vector(elem, num_elem, elem_bt, is_mask);
668 broadcast = gvn().transform(broadcast);
669 }
670
671 Node* box = box_vector(broadcast, vbox_type, elem_bt, num_elem);
672 set_result(box);
673 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
674 return true;
675 }
676
677 static bool elem_consistent_with_arr(BasicType elem_bt, const TypeAryPtr* arr_type, bool mismatched_ms) {
678 assert(arr_type != nullptr, "unexpected");
679 BasicType arr_elem_bt = arr_type->elem()->array_element_basic_type();
680 if (elem_bt == arr_elem_bt) {
681 return true;
682 } else if (elem_bt == T_SHORT && arr_elem_bt == T_CHAR) {
683 // Load/store of short vector from/to char[] is supported
684 return true;
685 } else if (elem_bt == T_BYTE && arr_elem_bt == T_BOOLEAN) {
686 // Load/store of byte vector from/to boolean[] is supported
687 return true;
688 } else {
689 return mismatched_ms;
690 }
691 }
692
693 // public static
694 // <C,
695 // VM extends VectorPayload,
696 // E,
697 // S extends VectorSpecies<E>>
698 // VM load(Class<? extends VM> vmClass, Class<E> eClass,
699 // int length,
700 // Object base, long offset, // Unsafe addressing
701 // boolean fromSegment,
702 // C container, long index, S s, // Arguments for default implementation
703 // LoadOperation<C, VM, S> defaultImpl) {
704 // public static
705 // <C,
706 // V extends VectorPayload>
707 // void store(Class<?> vClass, Class<?> eClass,
708 // int length,
709 // Object base, long offset, // Unsafe addressing
710 // boolean fromSegment,
711 // V v, C container, long index, // Arguments for default implementation
712 // StoreVectorOperation<C, V> defaultImpl) {
713 bool LibraryCallKit::inline_vector_mem_operation(bool is_store) {
714 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
715 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
716 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
717 const TypeInt* from_ms = gvn().type(argument(6))->isa_int();
718
719 if (vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
720 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
721 vlen == nullptr || !vlen->is_con() ||
722 from_ms == nullptr || !from_ms->is_con()) {
723 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s from_ms=%s",
724 NodeClassNames[argument(0)->Opcode()],
725 NodeClassNames[argument(1)->Opcode()],
726 NodeClassNames[argument(2)->Opcode()],
727 NodeClassNames[argument(6)->Opcode()]);
728 return false; // not enough info for intrinsification
729 }
730 if (!is_klass_initialized(vector_klass)) {
731 log_if_needed(" ** klass argument not initialized");
732 return false;
733 }
734
735 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
736 if (!elem_type->is_primitive_type()) {
737 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
738 return false; // should be primitive type
739 }
740 BasicType elem_bt = elem_type->basic_type();
741 int num_elem = vlen->get_con();
742
743 // TODO When mask usage is supported, VecMaskNotUsed needs to be VecMaskUseLoad.
744 if (!arch_supports_vector(is_store ? Op_StoreVector : Op_LoadVector, num_elem, elem_bt, VecMaskNotUsed)) {
745 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d etype=%s ismask=no",
746 is_store, is_store ? "store" : "load",
747 num_elem, type2name(elem_bt));
748 return false; // not supported
749 }
750
751 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
752 bool is_mask = is_vector_mask(vbox_klass);
753
754 Node* base = argument(3);
755 Node* offset = ConvL2X(argument(4));
756
757 // Save state and restore on bailout
758 uint old_sp = sp();
759 SafePointNode* old_map = clone_map();
760
761 Node* addr = make_unsafe_address(base, offset, (is_mask ? T_BOOLEAN : elem_bt), true);
762
763 // The memory barrier checks are based on ones for unsafe access.
764 // This is not 1-1 implementation.
765 const Type *const base_type = gvn().type(base);
766
767 const TypePtr *addr_type = gvn().type(addr)->isa_ptr();
768 const TypeAryPtr* arr_type = addr_type->isa_aryptr();
769
770 const bool in_native = TypePtr::NULL_PTR == base_type; // base always null
771 const bool in_heap = !TypePtr::NULL_PTR->higher_equal(base_type); // base never null
772
773 const bool is_mixed_access = !in_heap && !in_native;
774
775 const bool is_mismatched_access = in_heap && (addr_type->isa_aryptr() == nullptr);
776
777 const bool needs_cpu_membar = is_mixed_access || is_mismatched_access;
778
779 // For non-masked mismatched memory segment vector read/write accesses, intrinsification can continue
780 // with unknown backing storage type and compiler can skip inserting explicit reinterpretation IR after
781 // loading from or before storing to backing storage which is mandatory for semantic correctness of
782 // big-endian memory layout.
783 bool mismatched_ms = LITTLE_ENDIAN_ONLY(false)
784 BIG_ENDIAN_ONLY(from_ms->get_con() && !is_mask && arr_type != nullptr &&
785 arr_type->elem()->array_element_basic_type() != elem_bt);
786 BasicType mem_elem_bt = mismatched_ms ? arr_type->elem()->array_element_basic_type() : elem_bt;
787 if (!is_java_primitive(mem_elem_bt)) {
788 log_if_needed(" ** non-primitive array element type");
789 return false;
790 }
791 int mem_num_elem = mismatched_ms ? (num_elem * type2aelembytes(elem_bt)) / type2aelembytes(mem_elem_bt) : num_elem;
792 if (arr_type != nullptr && !is_mask && !elem_consistent_with_arr(elem_bt, arr_type, mismatched_ms)) {
793 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d etype=%s atype=%s ismask=no",
794 is_store, is_store ? "store" : "load",
795 num_elem, type2name(elem_bt), type2name(arr_type->elem()->array_element_basic_type()));
796 set_map(old_map);
797 set_sp(old_sp);
798 return false;
799 }
800
801 // In case of mismatched memory segment accesses, we need to double check that the source type memory operations are supported by backend.
802 if (mismatched_ms) {
803 if (is_store) {
804 if (!arch_supports_vector(Op_StoreVector, num_elem, elem_bt, VecMaskNotUsed)
805 || !arch_supports_vector(Op_VectorReinterpret, mem_num_elem, mem_elem_bt, VecMaskNotUsed)) {
806 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d*8 etype=%s/8 ismask=no",
807 is_store, "store",
808 num_elem, type2name(elem_bt));
809 set_map(old_map);
810 set_sp(old_sp);
811 return false; // not supported
812 }
813 } else {
814 if (!arch_supports_vector(Op_LoadVector, mem_num_elem, mem_elem_bt, VecMaskNotUsed)
815 || !arch_supports_vector(Op_VectorReinterpret, num_elem, elem_bt, VecMaskNotUsed)) {
816 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d*8 etype=%s/8 ismask=no",
817 is_store, "load",
818 mem_num_elem, type2name(mem_elem_bt));
819 set_map(old_map);
820 set_sp(old_sp);
821 return false; // not supported
822 }
823 }
824 }
825 if (is_mask) {
826 if (!is_store) {
827 if (!arch_supports_vector(Op_LoadVector, num_elem, elem_bt, VecMaskUseLoad)) {
828 set_map(old_map);
829 set_sp(old_sp);
830 return false; // not supported
831 }
832 } else {
833 if (!arch_supports_vector(Op_StoreVector, num_elem, elem_bt, VecMaskUseStore)) {
834 set_map(old_map);
835 set_sp(old_sp);
836 return false; // not supported
837 }
838 }
839 }
840
841 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
842
843 if (needs_cpu_membar) {
844 insert_mem_bar(Op_MemBarCPUOrder);
845 }
846
847 if (is_store) {
848 Node* val = unbox_vector(argument(7), vbox_type, elem_bt, num_elem);
849 if (val == nullptr) {
850 set_map(old_map);
851 set_sp(old_sp);
852 return false; // operand unboxing failed
853 }
854 set_all_memory(reset_memory());
855
856 // In case the store needs to happen to byte array, reinterpret the incoming vector to byte vector.
857 int store_num_elem = num_elem;
858 if (mismatched_ms) {
859 store_num_elem = mem_num_elem;
860 const TypeVect* to_vect_type = TypeVect::make(mem_elem_bt, store_num_elem);
861 val = gvn().transform(new VectorReinterpretNode(val, val->bottom_type()->is_vect(), to_vect_type));
862 }
863 if (is_mask) {
864 val = gvn().transform(VectorStoreMaskNode::make(gvn(), val, elem_bt, num_elem));
865 }
866 Node* vstore = gvn().transform(StoreVectorNode::make(0, control(), memory(addr), addr, addr_type, val, store_num_elem));
867 set_memory(vstore, addr_type);
868 } else {
869 // When using byte array, we need to load as byte then reinterpret the value. Otherwise, do a simple vector load.
870 Node* vload = nullptr;
871 if (mismatched_ms) {
872 vload = gvn().transform(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, mem_num_elem, mem_elem_bt));
873 const TypeVect* to_vect_type = TypeVect::make(elem_bt, num_elem);
874 vload = gvn().transform(new VectorReinterpretNode(vload, vload->bottom_type()->is_vect(), to_vect_type));
875 } else {
876 // Special handle for masks
877 if (is_mask) {
878 vload = gvn().transform(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, num_elem, T_BOOLEAN));
879 vload = gvn().transform(new VectorLoadMaskNode(vload, TypeVect::makemask(elem_bt, num_elem)));
880 } else {
881 vload = gvn().transform(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, num_elem, elem_bt));
882 }
883 }
884 Node* box = box_vector(vload, vbox_type, elem_bt, num_elem);
885 set_result(box);
886 }
887
888 destruct_map_clone(old_map);
889
890 if (needs_cpu_membar) {
891 insert_mem_bar(Op_MemBarCPUOrder);
892 }
893
894 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
895 return true;
896 }
897
898 // public static
899 // <C,
900 // V extends Vector<?>,
901 // E,
902 // S extends VectorSpecies<E>,
903 // M extends VectorMask<E>>
904 // V loadMasked(Class<? extends V> vClass, Class<M> mClass, Class<E> eClass,
905 // int length, Object base, long offset, // Unsafe addressing
906 // boolean fromSegment,
907 // M m, int offsetInRange,
908 // C container, long index, S s, // Arguments for default implementation
909 // LoadVectorMaskedOperation<C, V, S, M> defaultImpl) {
910 // public static
911 // <C,
912 // V extends Vector<E>,
913 // M extends VectorMask<E>,
914 // E>
915 // void storeMasked(Class<? extends V> vClass, Class<M> mClass, Class<E> eClass,
916 // int length,
917 // Object base, long offset, // Unsafe addressing
918 // boolean fromSegment,
919 // V v, M m, C container, long index, // Arguments for default implementation
920 // StoreVectorMaskedOperation<C, V, M> defaultImpl) {
921
922 bool LibraryCallKit::inline_vector_mem_masked_operation(bool is_store) {
923 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
924 const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
925 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
926 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
927 const TypeInt* from_ms = gvn().type(argument(7))->isa_int();
928
929 if (vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
930 mask_klass == nullptr || mask_klass->const_oop() == nullptr ||
931 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
932 vlen == nullptr || !vlen->is_con() ||
933 from_ms == nullptr || !from_ms->is_con()) {
934 log_if_needed(" ** missing constant: vclass=%s mclass=%s etype=%s vlen=%s from_ms=%s",
935 NodeClassNames[argument(0)->Opcode()],
936 NodeClassNames[argument(1)->Opcode()],
937 NodeClassNames[argument(2)->Opcode()],
938 NodeClassNames[argument(3)->Opcode()],
939 NodeClassNames[argument(7)->Opcode()]);
940 return false; // not enough info for intrinsification
941 }
942 if (!is_klass_initialized(vector_klass)) {
943 log_if_needed(" ** klass argument not initialized");
944 return false;
945 }
946
947 if (!is_klass_initialized(mask_klass)) {
948 log_if_needed(" ** mask klass argument not initialized");
949 return false;
950 }
951
952 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
953 if (!elem_type->is_primitive_type()) {
954 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
955 return false; // should be primitive type
956 }
957
958 BasicType elem_bt = elem_type->basic_type();
959 int num_elem = vlen->get_con();
960
961 Node* base = argument(4);
962 Node* offset = ConvL2X(argument(5));
963
964 // Save state and restore on bailout
965 uint old_sp = sp();
966 SafePointNode* old_map = clone_map();
967
968 Node* addr = make_unsafe_address(base, offset, elem_bt, true);
969 const TypePtr *addr_type = gvn().type(addr)->isa_ptr();
970 const TypeAryPtr* arr_type = addr_type->isa_aryptr();
971
972 bool mismatched_ms = from_ms->get_con() && arr_type != nullptr && arr_type->elem()->array_element_basic_type() != elem_bt;
973 BIG_ENDIAN_ONLY(if (mismatched_ms) return false;)
974 // If there is no consistency between array and vector element types, it must be special byte array case
975 if (arr_type != nullptr && !elem_consistent_with_arr(elem_bt, arr_type, mismatched_ms)) {
976 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d etype=%s atype=%s",
977 is_store, is_store ? "storeMasked" : "loadMasked",
978 num_elem, type2name(elem_bt), type2name(arr_type->elem()->array_element_basic_type()));
979 set_map(old_map);
980 set_sp(old_sp);
981 return false;
982 }
983
984 int mem_num_elem = mismatched_ms ? num_elem * type2aelembytes(elem_bt) : num_elem;
985 BasicType mem_elem_bt = mismatched_ms ? T_BYTE : elem_bt;
986 bool supports_predicate = arch_supports_vector(is_store ? Op_StoreVectorMasked : Op_LoadVectorMasked,
987 mem_num_elem, mem_elem_bt, VecMaskUseLoad);
988
989 // If current arch does not support the predicated operations, we have to bail
990 // out when current case uses the predicate feature.
991 if (!supports_predicate) {
992 bool needs_predicate = false;
993 if (is_store) {
994 // Masked vector store always uses the predicated store.
995 needs_predicate = true;
996 } else {
997 // Masked vector load with IOOBE always uses the predicated load.
998 const TypeInt* offset_in_range = gvn().type(argument(9))->isa_int();
999 if (!offset_in_range->is_con()) {
1000 log_if_needed(" ** missing constant: offsetInRange=%s",
1001 NodeClassNames[argument(8)->Opcode()]);
1002 set_map(old_map);
1003 set_sp(old_sp);
1004 return false;
1005 }
1006 needs_predicate = (offset_in_range->get_con() == 0);
1007 }
1008
1009 if (needs_predicate) {
1010 log_if_needed(" ** not supported: op=%s vlen=%d etype=%s mismatched_ms=%d",
1011 is_store ? "storeMasked" : "loadMasked",
1012 num_elem, type2name(elem_bt), mismatched_ms ? 1 : 0);
1013 set_map(old_map);
1014 set_sp(old_sp);
1015 return false;
1016 }
1017 }
1018
1019 // This only happens for masked vector load. If predicate is not supported, then check whether
1020 // the normal vector load and blend operations are supported by backend.
1021 if (!supports_predicate && (!arch_supports_vector(Op_LoadVector, mem_num_elem, mem_elem_bt, VecMaskNotUsed) ||
1022 !arch_supports_vector(Op_VectorBlend, mem_num_elem, mem_elem_bt, VecMaskUseLoad))) {
1023 log_if_needed(" ** not supported: op=loadMasked vlen=%d etype=%s mismatched_ms=%d",
1024 num_elem, type2name(elem_bt), mismatched_ms ? 1 : 0);
1025 set_map(old_map);
1026 set_sp(old_sp);
1027 return false;
1028 }
1029
1030 // Since we are using byte array, we need to double check that the vector reinterpret operation
1031 // with byte type is supported by backend.
1032 if (mismatched_ms) {
1033 if (!arch_supports_vector(Op_VectorReinterpret, mem_num_elem, T_BYTE, VecMaskNotUsed)) {
1034 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d etype=%s mismatched_ms=1",
1035 is_store, is_store ? "storeMasked" : "loadMasked",
1036 num_elem, type2name(elem_bt));
1037 set_map(old_map);
1038 set_sp(old_sp);
1039 return false;
1040 }
1041 }
1042
1043 // Since it needs to unbox the mask, we need to double check that the related load operations
1044 // for mask are supported by backend.
1045 if (!arch_supports_vector(Op_LoadVector, num_elem, elem_bt, VecMaskUseLoad)) {
1046 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d etype=%s",
1047 is_store, is_store ? "storeMasked" : "loadMasked",
1048 num_elem, type2name(elem_bt));
1049 set_map(old_map);
1050 set_sp(old_sp);
1051 return false;
1052 }
1053
1054 // Can base be null? Otherwise, always on-heap access.
1055 bool can_access_non_heap = TypePtr::NULL_PTR->higher_equal(gvn().type(base));
1056 if (can_access_non_heap) {
1057 insert_mem_bar(Op_MemBarCPUOrder);
1058 }
1059
1060 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1061 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1062 assert(!is_vector_mask(vbox_klass) && is_vector_mask(mbox_klass), "Invalid class type");
1063 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1064 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1065
1066 Node* mask = unbox_vector(is_store ? argument(9) : argument(8), mbox_type, elem_bt, num_elem);
1067 if (mask == nullptr) {
1068 log_if_needed(" ** unbox failed mask=%s",
1069 is_store ? NodeClassNames[argument(9)->Opcode()]
1070 : NodeClassNames[argument(8)->Opcode()]);
1071 set_map(old_map);
1072 set_sp(old_sp);
1073 return false;
1074 }
1075
1076 if (is_store) {
1077 Node* val = unbox_vector(argument(8), vbox_type, elem_bt, num_elem);
1078 if (val == nullptr) {
1079 log_if_needed(" ** unbox failed vector=%s",
1080 NodeClassNames[argument(8)->Opcode()]);
1081 set_map(old_map);
1082 set_sp(old_sp);
1083 return false; // operand unboxing failed
1084 }
1085 set_all_memory(reset_memory());
1086
1087 if (mismatched_ms) {
1088 // Reinterpret the incoming vector to byte vector.
1089 const TypeVect* to_vect_type = TypeVect::make(mem_elem_bt, mem_num_elem);
1090 val = gvn().transform(new VectorReinterpretNode(val, val->bottom_type()->is_vect(), to_vect_type));
1091 // Reinterpret the vector mask to byte type.
1092 const TypeVect* from_mask_type = TypeVect::makemask(elem_bt, num_elem);
1093 const TypeVect* to_mask_type = TypeVect::makemask(mem_elem_bt, mem_num_elem);
1094 mask = gvn().transform(new VectorReinterpretNode(mask, from_mask_type, to_mask_type));
1095 }
1096 Node* vstore = gvn().transform(new StoreVectorMaskedNode(control(), memory(addr), addr, val, addr_type, mask));
1097 set_memory(vstore, addr_type);
1098 } else {
1099 Node* vload = nullptr;
1100
1101 if (mismatched_ms) {
1102 // Reinterpret the vector mask to byte type.
1103 const TypeVect* from_mask_type = TypeVect::makemask(elem_bt, num_elem);
1104 const TypeVect* to_mask_type = TypeVect::makemask(mem_elem_bt, mem_num_elem);
1105 mask = gvn().transform(new VectorReinterpretNode(mask, from_mask_type, to_mask_type));
1106 }
1107
1108 if (supports_predicate) {
1109 // Generate masked load vector node if predicate feature is supported.
1110 const TypeVect* vt = TypeVect::make(mem_elem_bt, mem_num_elem);
1111 vload = gvn().transform(new LoadVectorMaskedNode(control(), memory(addr), addr, addr_type, vt, mask));
1112 } else {
1113 // Use the vector blend to implement the masked load vector. The biased elements are zeros.
1114 Node* zero = gvn().transform(gvn().zerocon(mem_elem_bt));
1115 zero = gvn().transform(VectorNode::scalar2vector(zero, mem_num_elem, mem_elem_bt));
1116 vload = gvn().transform(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, mem_num_elem, mem_elem_bt));
1117 vload = gvn().transform(new VectorBlendNode(zero, vload, mask));
1118 }
1119
1120 if (mismatched_ms) {
1121 const TypeVect* to_vect_type = TypeVect::make(elem_bt, num_elem);
1122 vload = gvn().transform(new VectorReinterpretNode(vload, vload->bottom_type()->is_vect(), to_vect_type));
1123 }
1124
1125 Node* box = box_vector(vload, vbox_type, elem_bt, num_elem);
1126 set_result(box);
1127 }
1128
1129 destruct_map_clone(old_map);
1130
1131 if (can_access_non_heap) {
1132 insert_mem_bar(Op_MemBarCPUOrder);
1133 }
1134
1135 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1136 return true;
1137 }
1138
1139 // <C,
1140 // V extends Vector<?>,
1141 // W extends Vector<Integer>,
1142 // S extends VectorSpecies<E>,
1143 // M extends VectorMask<E>,
1144 // E>
1145 // V loadWithMap(Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType, int length,
1146 // Class<? extends Vector<Integer>> vectorIndexClass,
1147 // Object base, long offset, // Unsafe addressing
1148 // W index_vector, M m,
1149 // C container, int index, int[] indexMap, int indexM, S s, // Arguments for default implementation
1150 // LoadVectorOperationWithMap<C, V, E, S, M> defaultImpl)
1151 //
1152 // <C,
1153 // V extends Vector<E>,
1154 // W extends Vector<Integer>,
1155 // M extends VectorMask<E>,
1156 // E>
1157 // void storeWithMap(Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType,
1158 // int length, Class<? extends Vector<Integer>> vectorIndexClass, Object base, long offset, // Unsafe addressing
1159 // W index_vector, V v, M m,
1160 // C container, int index, int[] indexMap, int indexM, // Arguments for default implementation
1161 // StoreVectorOperationWithMap<C, V, M, E> defaultImpl)
1162 //
1163 bool LibraryCallKit::inline_vector_gather_scatter(bool is_scatter) {
1164 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
1165 const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
1166 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
1167 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
1168 const TypeInstPtr* vector_idx_klass = gvn().type(argument(4))->isa_instptr();
1169
1170 if (vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
1171 // mask_klass == nullptr || mask_klass->const_oop() == nullptr ||
1172 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
1173 vlen == nullptr || !vlen->is_con() ||
1174 vector_idx_klass == nullptr || vector_idx_klass->const_oop() == nullptr) {
1175 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s viclass=%s",
1176 NodeClassNames[argument(0)->Opcode()],
1177 NodeClassNames[argument(2)->Opcode()],
1178 NodeClassNames[argument(3)->Opcode()],
1179 NodeClassNames[argument(4)->Opcode()]);
1180 return false; // not enough info for intrinsification
1181 }
1182
1183 if (!is_klass_initialized(vector_klass) || !is_klass_initialized(vector_idx_klass)) {
1184 log_if_needed(" ** klass argument not initialized");
1185 return false;
1186 }
1187
1188 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1189 if (!elem_type->is_primitive_type()) {
1190 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1191 return false; // should be primitive type
1192 }
1193
1194 BasicType elem_bt = elem_type->basic_type();
1195 int num_elem = vlen->get_con();
1196
1197 const Type* vmask_type = gvn().type(is_scatter ? argument(10) : argument(9));
1198 bool is_masked_op = vmask_type != TypePtr::NULL_PTR;
1199 if (is_masked_op) {
1200 if (mask_klass == nullptr || mask_klass->const_oop() == nullptr) {
1201 log_if_needed(" ** missing constant: maskclass=%s", NodeClassNames[argument(1)->Opcode()]);
1202 return false; // not enough info for intrinsification
1203 }
1204
1205 if (!is_klass_initialized(mask_klass)) {
1206 log_if_needed(" ** mask klass argument not initialized");
1207 return false;
1208 }
1209
1210 if (vmask_type->maybe_null()) {
1211 log_if_needed(" ** null mask values are not allowed for masked op");
1212 return false;
1213 }
1214
1215 // Check whether the predicated gather/scatter node is supported by architecture.
1216 VectorMaskUseType mask = (VectorMaskUseType) (VecMaskUseLoad | VecMaskUsePred);
1217 if (!arch_supports_vector(is_scatter ? Op_StoreVectorScatterMasked : Op_LoadVectorGatherMasked, num_elem, elem_bt, mask)) {
1218 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d etype=%s is_masked_op=1",
1219 is_scatter, is_scatter ? "scatterMasked" : "gatherMasked",
1220 num_elem, type2name(elem_bt));
1221 return false; // not supported
1222 }
1223 } else {
1224 // Check whether the normal gather/scatter node is supported for non-masked operation.
1225 if (!arch_supports_vector(is_scatter ? Op_StoreVectorScatter : Op_LoadVectorGather, num_elem, elem_bt, VecMaskNotUsed)) {
1226 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d etype=%s is_masked_op=0",
1227 is_scatter, is_scatter ? "scatter" : "gather",
1228 num_elem, type2name(elem_bt));
1229 return false; // not supported
1230 }
1231 }
1232
1233 // Check that the vector holding indices is supported by architecture
1234 // For sub-word gathers expander receive index array.
1235 if (!is_subword_type(elem_bt) && !arch_supports_vector(Op_LoadVector, num_elem, T_INT, VecMaskNotUsed)) {
1236 log_if_needed(" ** not supported: arity=%d op=%s/loadindex vlen=%d etype=int is_masked_op=%d",
1237 is_scatter, is_scatter ? "scatter" : "gather",
1238 num_elem, is_masked_op ? 1 : 0);
1239 return false; // not supported
1240 }
1241
1242 Node* base = argument(5);
1243 Node* offset = ConvL2X(argument(6));
1244
1245 // Save state and restore on bailout
1246 uint old_sp = sp();
1247 SafePointNode* old_map = clone_map();
1248
1249 Node* addr = make_unsafe_address(base, offset, elem_bt, true);
1250
1251 const TypePtr *addr_type = gvn().type(addr)->isa_ptr();
1252 const TypeAryPtr* arr_type = addr_type->isa_aryptr();
1253
1254 // The array must be consistent with vector type
1255 if (arr_type == nullptr || (arr_type != nullptr && !elem_consistent_with_arr(elem_bt, arr_type, false))) {
1256 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d etype=%s atype=%s ismask=no",
1257 is_scatter, is_scatter ? "scatter" : "gather",
1258 num_elem, type2name(elem_bt), type2name(arr_type->elem()->array_element_basic_type()));
1259 set_map(old_map);
1260 set_sp(old_sp);
1261 return false;
1262 }
1263
1264 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1265 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1266 ciKlass* vbox_idx_klass = vector_idx_klass->const_oop()->as_instance()->java_lang_Class_klass();
1267 if (vbox_idx_klass == nullptr) {
1268 set_map(old_map);
1269 set_sp(old_sp);
1270 return false;
1271 }
1272
1273 Node* index_vect = nullptr;
1274 const TypeInstPtr* vbox_idx_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_idx_klass);
1275 if (!is_subword_type(elem_bt)) {
1276 index_vect = unbox_vector(argument(8), vbox_idx_type, T_INT, num_elem);
1277 if (index_vect == nullptr) {
1278 set_map(old_map);
1279 set_sp(old_sp);
1280 return false;
1281 }
1282 }
1283
1284 Node* mask = nullptr;
1285 if (is_masked_op) {
1286 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1287 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1288 mask = unbox_vector(is_scatter ? argument(10) : argument(9), mbox_type, elem_bt, num_elem);
1289 if (mask == nullptr) {
1290 log_if_needed(" ** unbox failed mask=%s",
1291 is_scatter ? NodeClassNames[argument(10)->Opcode()]
1292 : NodeClassNames[argument(9)->Opcode()]);
1293 set_map(old_map);
1294 set_sp(old_sp);
1295 return false;
1296 }
1297 }
1298
1299 const TypeVect* vector_type = TypeVect::make(elem_bt, num_elem);
1300 if (is_scatter) {
1301 Node* val = unbox_vector(argument(9), vbox_type, elem_bt, num_elem);
1302 if (val == nullptr) {
1303 set_map(old_map);
1304 set_sp(old_sp);
1305 return false; // operand unboxing failed
1306 }
1307 set_all_memory(reset_memory());
1308
1309 Node* vstore = nullptr;
1310 if (mask != nullptr) {
1311 vstore = gvn().transform(new StoreVectorScatterMaskedNode(control(), memory(addr), addr, addr_type, val, index_vect, mask));
1312 } else {
1313 vstore = gvn().transform(new StoreVectorScatterNode(control(), memory(addr), addr, addr_type, val, index_vect));
1314 }
1315 set_memory(vstore, addr_type);
1316 } else {
1317 Node* vload = nullptr;
1318 Node* index = argument(11);
1319 Node* indexMap = argument(12);
1320 Node* indexM = argument(13);
1321 if (mask != nullptr) {
1322 if (is_subword_type(elem_bt)) {
1323 Node* index_arr_base = array_element_address(indexMap, indexM, T_INT);
1324 vload = gvn().transform(new LoadVectorGatherMaskedNode(control(), memory(addr), addr, addr_type, vector_type, index_arr_base, mask, index));
1325 } else {
1326 vload = gvn().transform(new LoadVectorGatherMaskedNode(control(), memory(addr), addr, addr_type, vector_type, index_vect, mask));
1327 }
1328 } else {
1329 if (is_subword_type(elem_bt)) {
1330 Node* index_arr_base = array_element_address(indexMap, indexM, T_INT);
1331 vload = gvn().transform(new LoadVectorGatherNode(control(), memory(addr), addr, addr_type, vector_type, index_arr_base, index));
1332 } else {
1333 vload = gvn().transform(new LoadVectorGatherNode(control(), memory(addr), addr, addr_type, vector_type, index_vect));
1334 }
1335 }
1336 Node* box = box_vector(vload, vbox_type, elem_bt, num_elem);
1337 set_result(box);
1338 }
1339
1340 destruct_map_clone(old_map);
1341
1342 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1343 return true;
1344 }
1345
1346 // public static
1347 // <V extends Vector<E>,
1348 // M extends VectorMask<E>,
1349 // E>
1350 // long reductionCoerced(int oprId, Class<? extends V> vectorClass, Class<? extends M> maskClass,
1351 // Class<E> elementType, int length, V v, M m,
1352 // ReductionOperation<V, M> defaultImpl)
1353 bool LibraryCallKit::inline_vector_reduction() {
1354 const TypeInt* opr = gvn().type(argument(0))->isa_int();
1355 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
1356 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
1357 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
1358 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
1359
1360 if (opr == nullptr || !opr->is_con() ||
1361 vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
1362 // mask_klass == nullptr || mask_klass->const_oop() == nullptr ||
1363 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
1364 vlen == nullptr || !vlen->is_con()) {
1365 log_if_needed(" ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
1366 NodeClassNames[argument(0)->Opcode()],
1367 NodeClassNames[argument(1)->Opcode()],
1368 NodeClassNames[argument(3)->Opcode()],
1369 NodeClassNames[argument(4)->Opcode()]);
1370 return false; // not enough info for intrinsification
1371 }
1372 if (!is_klass_initialized(vector_klass)) {
1373 log_if_needed(" ** klass argument not initialized");
1374 return false;
1375 }
1376 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1377 if (!elem_type->is_primitive_type()) {
1378 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1379 return false; // should be primitive type
1380 }
1381
1382 const Type* vmask_type = gvn().type(argument(6));
1383 bool is_masked_op = vmask_type != TypePtr::NULL_PTR;
1384 if (is_masked_op) {
1385 if (mask_klass == nullptr || mask_klass->const_oop() == nullptr) {
1386 log_if_needed(" ** missing constant: maskclass=%s", NodeClassNames[argument(2)->Opcode()]);
1387 return false; // not enough info for intrinsification
1388 }
1389
1390 if (!is_klass_initialized(mask_klass)) {
1391 log_if_needed(" ** mask klass argument not initialized");
1392 return false;
1393 }
1394
1395 if (vmask_type->maybe_null()) {
1396 log_if_needed(" ** null mask values are not allowed for masked op");
1397 return false;
1398 }
1399 }
1400
1401 BasicType elem_bt = elem_type->basic_type();
1402 int num_elem = vlen->get_con();
1403 int opc = VectorSupport::vop2ideal(opr->get_con(), elem_bt);
1404 int sopc = ReductionNode::opcode(opc, elem_bt);
1405
1406 // Ensure reduction operation for lanewise operation
1407 // When using mask, mask use type needs to be VecMaskUseLoad.
1408 if (sopc == opc || !arch_supports_vector(sopc, num_elem, elem_bt, is_masked_op ? VecMaskUseLoad : VecMaskNotUsed)) {
1409 log_if_needed(" ** not supported: arity=1 op=%d/reduce vlen=%d etype=%s is_masked_op=%d",
1410 sopc, num_elem, type2name(elem_bt), is_masked_op ? 1 : 0);
1411 return false;
1412 }
1413
1414 // Return true if current platform has implemented the masked operation with predicate feature.
1415 bool use_predicate = is_masked_op && arch_supports_vector(sopc, num_elem, elem_bt, VecMaskUsePred);
1416 if (is_masked_op && !use_predicate && !arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad)) {
1417 log_if_needed(" ** not supported: arity=1 op=%d/reduce vlen=%d etype=%s is_masked_op=1",
1418 sopc, num_elem, type2name(elem_bt));
1419 return false;
1420 }
1421
1422 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1423 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1424
1425 Node* opd = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
1426 if (opd == nullptr) {
1427 return false; // operand unboxing failed
1428 }
1429
1430 Node* mask = nullptr;
1431 if (is_masked_op) {
1432 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1433 assert(is_vector_mask(mbox_klass), "argument(2) should be a mask class");
1434 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1435 mask = unbox_vector(argument(6), mbox_type, elem_bt, num_elem);
1436 if (mask == nullptr) {
1437 log_if_needed(" ** unbox failed mask=%s",
1438 NodeClassNames[argument(6)->Opcode()]);
1439 return false;
1440 }
1441 }
1442
1443 Node* init = ReductionNode::make_identity_con_scalar(gvn(), opc, elem_bt);
1444 Node* value = opd;
1445
1446 assert(mask != nullptr || !is_masked_op, "Masked op needs the mask value never null");
1447 if (mask != nullptr && !use_predicate) {
1448 Node* reduce_identity = gvn().transform(VectorNode::scalar2vector(init, num_elem, elem_bt));
1449 value = gvn().transform(new VectorBlendNode(reduce_identity, value, mask));
1450 }
1451
1452 // Make an unordered Reduction node. This affects only AddReductionVF/VD and MulReductionVF/VD,
1453 // as these operations are allowed to be associative (not requiring strict order) in VectorAPI.
1454 value = ReductionNode::make(opc, nullptr, init, value, elem_bt, /* requires_strict_order */ false);
1455
1456 if (mask != nullptr && use_predicate) {
1457 value->add_req(mask);
1458 value->add_flag(Node::Flag_is_predicated_vector);
1459 }
1460
1461 value = gvn().transform(value);
1462
1463 Node* bits = nullptr;
1464 switch (elem_bt) {
1465 case T_BYTE:
1466 case T_SHORT:
1467 case T_INT: {
1468 bits = gvn().transform(new ConvI2LNode(value));
1469 break;
1470 }
1471 case T_FLOAT: {
1472 value = gvn().transform(new MoveF2INode(value));
1473 bits = gvn().transform(new ConvI2LNode(value));
1474 break;
1475 }
1476 case T_DOUBLE: {
1477 bits = gvn().transform(new MoveD2LNode(value));
1478 break;
1479 }
1480 case T_LONG: {
1481 bits = value; // no conversion needed
1482 break;
1483 }
1484 default: fatal("%s", type2name(elem_bt));
1485 }
1486 set_result(bits);
1487 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1488 return true;
1489 }
1490
1491 // public static <V> boolean test(int cond, Class<?> vectorClass, Class<?> elementType, int vlen,
1492 // V v1, V v2,
1493 // BiFunction<V, V, Boolean> defaultImpl)
1494 //
1495 bool LibraryCallKit::inline_vector_test() {
1496 const TypeInt* cond = gvn().type(argument(0))->isa_int();
1497 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
1498 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
1499 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
1500
1501 if (cond == nullptr || !cond->is_con() ||
1502 vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
1503 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
1504 vlen == nullptr || !vlen->is_con()) {
1505 log_if_needed(" ** missing constant: cond=%s vclass=%s etype=%s vlen=%s",
1506 NodeClassNames[argument(0)->Opcode()],
1507 NodeClassNames[argument(1)->Opcode()],
1508 NodeClassNames[argument(2)->Opcode()],
1509 NodeClassNames[argument(3)->Opcode()]);
1510 return false; // not enough info for intrinsification
1511 }
1512 if (!is_klass_initialized(vector_klass)) {
1513 log_if_needed(" ** klass argument not initialized");
1514 return false;
1515 }
1516 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1517 if (!elem_type->is_primitive_type()) {
1518 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1519 return false; // should be primitive type
1520 }
1521 BasicType elem_bt = elem_type->basic_type();
1522 int num_elem = vlen->get_con();
1523 BoolTest::mask booltest = (BoolTest::mask)cond->get_con();
1524 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1525 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1526
1527 if (!arch_supports_vector(Op_VectorTest, num_elem, elem_bt, is_vector_mask(vbox_klass) ? VecMaskUseLoad : VecMaskNotUsed)) {
1528 log_if_needed(" ** not supported: arity=2 op=test/%d vlen=%d etype=%s ismask=%d",
1529 cond->get_con(), num_elem, type2name(elem_bt),
1530 is_vector_mask(vbox_klass));
1531 return false;
1532 }
1533
1534 Node* opd1 = unbox_vector(argument(4), vbox_type, elem_bt, num_elem);
1535 Node* opd2;
1536 if (Matcher::vectortest_needs_second_argument(booltest == BoolTest::overflow,
1537 opd1->bottom_type()->isa_vectmask())) {
1538 opd2 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
1539 } else {
1540 opd2 = opd1;
1541 }
1542 if (opd1 == nullptr || opd2 == nullptr) {
1543 return false; // operand unboxing failed
1544 }
1545
1546 Node* cmp = gvn().transform(new VectorTestNode(opd1, opd2, booltest));
1547 BoolTest::mask test = Matcher::vectortest_mask(booltest == BoolTest::overflow,
1548 opd1->bottom_type()->isa_vectmask(), num_elem);
1549 Node* bol = gvn().transform(new BoolNode(cmp, test));
1550 Node* res = gvn().transform(new CMoveINode(bol, gvn().intcon(0), gvn().intcon(1), TypeInt::BOOL));
1551
1552 set_result(res);
1553 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1554 return true;
1555 }
1556
1557 // public static
1558 // <V extends Vector<E>,
1559 // M extends VectorMask<E>,
1560 // E>
1561 // V blend(Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType, int vlen,
1562 // V v1, V v2, M m,
1563 // VectorBlendOp<V, M, E> defaultImpl)
1564 bool LibraryCallKit::inline_vector_blend() {
1565 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
1566 const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
1567 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
1568 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
1569
1570 if (mask_klass == nullptr || vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr) {
1571 return false; // dead code
1572 }
1573 if (mask_klass->const_oop() == nullptr || vector_klass->const_oop() == nullptr ||
1574 elem_klass->const_oop() == nullptr || !vlen->is_con()) {
1575 log_if_needed(" ** missing constant: vclass=%s mclass=%s etype=%s vlen=%s",
1576 NodeClassNames[argument(0)->Opcode()],
1577 NodeClassNames[argument(1)->Opcode()],
1578 NodeClassNames[argument(2)->Opcode()],
1579 NodeClassNames[argument(3)->Opcode()]);
1580 return false; // not enough info for intrinsification
1581 }
1582 if (!is_klass_initialized(vector_klass) || !is_klass_initialized(mask_klass)) {
1583 log_if_needed(" ** klass argument not initialized");
1584 return false;
1585 }
1586 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1587 if (!elem_type->is_primitive_type()) {
1588 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1589 return false; // should be primitive type
1590 }
1591 BasicType elem_bt = elem_type->basic_type();
1592 BasicType mask_bt = elem_bt;
1593 int num_elem = vlen->get_con();
1594
1595 if (!arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad)) {
1596 log_if_needed(" ** not supported: arity=2 op=blend vlen=%d etype=%s ismask=useload",
1597 num_elem, type2name(elem_bt));
1598 return false; // not supported
1599 }
1600 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1601 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1602
1603 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1604 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1605
1606 Node* v1 = unbox_vector(argument(4), vbox_type, elem_bt, num_elem);
1607 Node* v2 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
1608 Node* mask = unbox_vector(argument(6), mbox_type, mask_bt, num_elem);
1609
1610 if (v1 == nullptr || v2 == nullptr || mask == nullptr) {
1611 return false; // operand unboxing failed
1612 }
1613
1614 Node* blend = gvn().transform(new VectorBlendNode(v1, v2, mask));
1615
1616 Node* box = box_vector(blend, vbox_type, elem_bt, num_elem);
1617 set_result(box);
1618 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1619 return true;
1620 }
1621
1622 // public static
1623 // <V extends Vector<E>,
1624 // M extends VectorMask<E>,
1625 // E>
1626 // M compare(int cond, Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType, int vlen,
1627 // V v1, V v2, M m,
1628 // VectorCompareOp<V,M> defaultImpl)
1629 bool LibraryCallKit::inline_vector_compare() {
1630 const TypeInt* cond = gvn().type(argument(0))->isa_int();
1631 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
1632 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
1633 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
1634 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
1635
1636 if (cond == nullptr || vector_klass == nullptr || mask_klass == nullptr || elem_klass == nullptr || vlen == nullptr) {
1637 return false; // dead code
1638 }
1639 if (!cond->is_con() || vector_klass->const_oop() == nullptr || mask_klass->const_oop() == nullptr ||
1640 elem_klass->const_oop() == nullptr || !vlen->is_con()) {
1641 log_if_needed(" ** missing constant: cond=%s vclass=%s mclass=%s etype=%s vlen=%s",
1642 NodeClassNames[argument(0)->Opcode()],
1643 NodeClassNames[argument(1)->Opcode()],
1644 NodeClassNames[argument(2)->Opcode()],
1645 NodeClassNames[argument(3)->Opcode()],
1646 NodeClassNames[argument(4)->Opcode()]);
1647 return false; // not enough info for intrinsification
1648 }
1649 if (!is_klass_initialized(vector_klass) || !is_klass_initialized(mask_klass)) {
1650 log_if_needed(" ** klass argument not initialized");
1651 return false;
1652 }
1653 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1654 if (!elem_type->is_primitive_type()) {
1655 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1656 return false; // should be primitive type
1657 }
1658
1659 int num_elem = vlen->get_con();
1660 BasicType elem_bt = elem_type->basic_type();
1661 BasicType mask_bt = elem_bt;
1662
1663 if ((cond->get_con() & BoolTest::unsigned_compare) != 0) {
1664 if (!Matcher::supports_vector_comparison_unsigned(num_elem, elem_bt)) {
1665 log_if_needed(" ** not supported: unsigned comparison op=comp/%d vlen=%d etype=%s ismask=usestore",
1666 cond->get_con() & (BoolTest::unsigned_compare - 1), num_elem, type2name(elem_bt));
1667 return false;
1668 }
1669 }
1670
1671 if (!arch_supports_vector(Op_VectorMaskCmp, num_elem, elem_bt, VecMaskUseStore)) {
1672 log_if_needed(" ** not supported: arity=2 op=comp/%d vlen=%d etype=%s ismask=usestore",
1673 cond->get_con(), num_elem, type2name(elem_bt));
1674 return false;
1675 }
1676
1677 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1678 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1679
1680 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1681 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1682
1683 Node* v1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
1684 Node* v2 = unbox_vector(argument(6), vbox_type, elem_bt, num_elem);
1685
1686 bool is_masked_op = argument(7)->bottom_type() != TypePtr::NULL_PTR;
1687 Node* mask = is_masked_op ? unbox_vector(argument(7), mbox_type, elem_bt, num_elem) : nullptr;
1688 if (is_masked_op && mask == nullptr) {
1689 log_if_needed(" ** not supported: mask = null arity=2 op=comp/%d vlen=%d etype=%s ismask=usestore is_masked_op=1",
1690 cond->get_con(), num_elem, type2name(elem_bt));
1691 return false;
1692 }
1693
1694 bool use_predicate = is_masked_op && arch_supports_vector(Op_VectorMaskCmp, num_elem, elem_bt, VecMaskUsePred);
1695 if (is_masked_op && !use_predicate && !arch_supports_vector(Op_AndV, num_elem, elem_bt, VecMaskUseLoad)) {
1696 log_if_needed(" ** not supported: arity=2 op=comp/%d vlen=%d etype=%s ismask=usestore is_masked_op=1",
1697 cond->get_con(), num_elem, type2name(elem_bt));
1698 return false;
1699 }
1700
1701 if (v1 == nullptr || v2 == nullptr) {
1702 return false; // operand unboxing failed
1703 }
1704 BoolTest::mask pred = (BoolTest::mask)cond->get_con();
1705 ConINode* pred_node = (ConINode*)gvn().makecon(cond);
1706
1707 const TypeVect* vmask_type = TypeVect::makemask(mask_bt, num_elem);
1708 Node* operation = new VectorMaskCmpNode(pred, v1, v2, pred_node, vmask_type);
1709
1710 if (is_masked_op) {
1711 if (use_predicate) {
1712 operation->add_req(mask);
1713 operation->add_flag(Node::Flag_is_predicated_vector);
1714 } else {
1715 operation = gvn().transform(operation);
1716 operation = VectorNode::make(Op_AndV, operation, mask, vmask_type);
1717 }
1718 }
1719
1720 operation = gvn().transform(operation);
1721
1722 Node* box = box_vector(operation, mbox_type, mask_bt, num_elem);
1723 set_result(box);
1724 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1725 return true;
1726 }
1727
1728 // public static
1729 // <V extends Vector<E>,
1730 // Sh extends VectorShuffle<E>,
1731 // M extends VectorMask<E>,
1732 // E>
1733 // V rearrangeOp(Class<? extends V> vectorClass, Class<Sh> shuffleClass, Class<M> maskClass, Class<E> elementType, int vlen,
1734 // V v1, Sh sh, M m,
1735 // VectorRearrangeOp<V, Sh, M, E> defaultImpl)
1736 bool LibraryCallKit::inline_vector_rearrange() {
1737 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
1738 const TypeInstPtr* shuffle_klass = gvn().type(argument(1))->isa_instptr();
1739 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
1740 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
1741 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
1742
1743 if (vector_klass == nullptr || shuffle_klass == nullptr || elem_klass == nullptr || vlen == nullptr) {
1744 return false; // dead code
1745 }
1746 if (shuffle_klass->const_oop() == nullptr ||
1747 vector_klass->const_oop() == nullptr ||
1748 elem_klass->const_oop() == nullptr ||
1749 !vlen->is_con()) {
1750 log_if_needed(" ** missing constant: vclass=%s sclass=%s etype=%s vlen=%s",
1751 NodeClassNames[argument(0)->Opcode()],
1752 NodeClassNames[argument(1)->Opcode()],
1753 NodeClassNames[argument(3)->Opcode()],
1754 NodeClassNames[argument(4)->Opcode()]);
1755 return false; // not enough info for intrinsification
1756 }
1757 if (!is_klass_initialized(vector_klass) ||
1758 !is_klass_initialized(shuffle_klass)) {
1759 log_if_needed(" ** klass argument not initialized");
1760 return false;
1761 }
1762 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1763 if (!elem_type->is_primitive_type()) {
1764 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1765 return false; // should be primitive type
1766 }
1767
1768 BasicType elem_bt = elem_type->basic_type();
1769 BasicType shuffle_bt = elem_bt;
1770 if (shuffle_bt == T_FLOAT) {
1771 shuffle_bt = T_INT;
1772 } else if (shuffle_bt == T_DOUBLE) {
1773 shuffle_bt = T_LONG;
1774 }
1775
1776 int num_elem = vlen->get_con();
1777 bool need_load_shuffle = Matcher::vector_rearrange_requires_load_shuffle(shuffle_bt, num_elem);
1778
1779 if (need_load_shuffle && !arch_supports_vector(Op_VectorLoadShuffle, num_elem, shuffle_bt, VecMaskNotUsed)) {
1780 if (C->print_intrinsics()) {
1781 tty->print_cr(" ** not supported: arity=0 op=load/shuffle vlen=%d etype=%s ismask=no",
1782 num_elem, type2name(shuffle_bt));
1783 }
1784 return false; // not supported
1785 }
1786
1787 bool is_masked_op = argument(7)->bottom_type() != TypePtr::NULL_PTR;
1788 bool use_predicate = is_masked_op;
1789 if (is_masked_op &&
1790 (mask_klass == nullptr ||
1791 mask_klass->const_oop() == nullptr ||
1792 !is_klass_initialized(mask_klass))) {
1793 log_if_needed(" ** mask_klass argument not initialized");
1794 }
1795 if (!arch_supports_vector(Op_AndV, num_elem, elem_bt, VecMaskNotUsed)) {
1796 log_if_needed(" ** not supported: arity=2 op=and vlen=%d etype=%s ismask=no",
1797 num_elem, type2name(elem_bt));
1798 return false;
1799 }
1800 VectorMaskUseType checkFlags = (VectorMaskUseType)(is_masked_op ? (VecMaskUseLoad | VecMaskUsePred) : VecMaskNotUsed);
1801 if (!arch_supports_vector(Op_VectorRearrange, num_elem, elem_bt, checkFlags)) {
1802 use_predicate = false;
1803 if(!is_masked_op ||
1804 (!arch_supports_vector(Op_VectorRearrange, num_elem, elem_bt, VecMaskNotUsed) ||
1805 !arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad) ||
1806 !arch_supports_vector(Op_Replicate, num_elem, elem_bt, VecMaskNotUsed))) {
1807 log_if_needed(" ** not supported: arity=2 op=shuffle/rearrange vlen=%d etype=%s ismask=no",
1808 num_elem, type2name(elem_bt));
1809 return false; // not supported
1810 }
1811 }
1812 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1813 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1814
1815 ciKlass* shbox_klass = shuffle_klass->const_oop()->as_instance()->java_lang_Class_klass();
1816 const TypeInstPtr* shbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, shbox_klass);
1817
1818 Node* v1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
1819 Node* shuffle = unbox_vector(argument(6), shbox_type, shuffle_bt, num_elem);
1820 const TypeVect* st = TypeVect::make(shuffle_bt, num_elem);
1821
1822 if (v1 == nullptr || shuffle == nullptr) {
1823 return false; // operand unboxing failed
1824 }
1825
1826 assert(is_power_of_2(num_elem), "wrapping invalid");
1827 Node* wrapping_mask_elem = gvn().makecon(TypeInteger::make(num_elem - 1, num_elem - 1, Type::WidenMin, shuffle_bt == T_LONG ? T_LONG : T_INT));
1828 Node* wrapping_mask = gvn().transform(VectorNode::scalar2vector(wrapping_mask_elem, num_elem, shuffle_bt));
1829 shuffle = gvn().transform(new AndVNode(shuffle, wrapping_mask, st));
1830
1831 Node* mask = nullptr;
1832 if (is_masked_op) {
1833 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1834 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1835 mask = unbox_vector(argument(7), mbox_type, elem_bt, num_elem);
1836 if (mask == nullptr) {
1837 log_if_needed(" ** not supported: arity=3 op=shuffle/rearrange vlen=%d etype=%s ismask=useload is_masked_op=1",
1838 num_elem, type2name(elem_bt));
1839 return false;
1840 }
1841 }
1842
1843 if (need_load_shuffle) {
1844 shuffle = gvn().transform(new VectorLoadShuffleNode(shuffle, st));
1845 }
1846
1847 Node* rearrange = new VectorRearrangeNode(v1, shuffle);
1848 if (is_masked_op) {
1849 if (use_predicate) {
1850 rearrange->add_req(mask);
1851 rearrange->add_flag(Node::Flag_is_predicated_vector);
1852 } else {
1853 rearrange = gvn().transform(rearrange);
1854 Node* zero = gvn().makecon(Type::get_zero_type(elem_bt));
1855 Node* zerovec = gvn().transform(VectorNode::scalar2vector(zero, num_elem, elem_bt));
1856 rearrange = new VectorBlendNode(zerovec, rearrange, mask);
1857 }
1858 }
1859 rearrange = gvn().transform(rearrange);
1860
1861 Node* box = box_vector(rearrange, vbox_type, elem_bt, num_elem);
1862 set_result(box);
1863 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1864 return true;
1865 }
1866
1867 static address get_vector_math_address(int vop, int bits, BasicType bt, char* name_ptr, int name_len) {
1868 address addr = nullptr;
1869 assert(UseVectorStubs, "sanity");
1870 assert(name_ptr != nullptr, "unexpected");
1871 assert((vop >= VectorSupport::VECTOR_OP_MATH_START) && (vop <= VectorSupport::VECTOR_OP_MATH_END), "unexpected");
1872 int op = vop - VectorSupport::VECTOR_OP_MATH_START;
1873
1874 switch(bits) {
1875 case 64: //fallthough
1876 case 128: //fallthough
1877 case 256: //fallthough
1878 case 512:
1879 if (bt == T_FLOAT) {
1880 snprintf(name_ptr, name_len, "vector_%s_float_%dbits_fixed", VectorSupport::mathname[op], bits);
1881 addr = StubRoutines::_vector_f_math[exact_log2(bits/64)][op];
1882 } else {
1883 assert(bt == T_DOUBLE, "must be FP type only");
1884 snprintf(name_ptr, name_len, "vector_%s_double_%dbits_fixed", VectorSupport::mathname[op], bits);
1885 addr = StubRoutines::_vector_d_math[exact_log2(bits/64)][op];
1886 }
1887 break;
1888 default:
1889 if (!Matcher::supports_scalable_vector() || !Matcher::vector_size_supported(bt, bits/type2aelembytes(bt)) ) {
1890 snprintf(name_ptr, name_len, "invalid");
1891 addr = nullptr;
1892 Unimplemented();
1893 }
1894 break;
1895 }
1896
1897 if (addr == nullptr && Matcher::supports_scalable_vector()) {
1898 if (bt == T_FLOAT) {
1899 snprintf(name_ptr, name_len, "vector_%s_float_%dbits_scalable", VectorSupport::mathname[op], bits);
1900 addr = StubRoutines::_vector_f_math[VectorSupport::VEC_SIZE_SCALABLE][op];
1901 } else {
1902 assert(bt == T_DOUBLE, "must be FP type only");
1903 snprintf(name_ptr, name_len, "vector_%s_double_%dbits_scalable", VectorSupport::mathname[op], bits);
1904 addr = StubRoutines::_vector_d_math[VectorSupport::VEC_SIZE_SCALABLE][op];
1905 }
1906 }
1907
1908 return addr;
1909 }
1910
1911 // public static
1912 // <V extends Vector<E>,
1913 // M extends VectorMask<E>,
1914 // E>
1915 // V selectFromOp(Class<? extends V> vClass, Class<M> mClass, Class<E> eClass,
1916 // int length, V v1, V v2, M m,
1917 // VectorSelectFromOp<V, M> defaultImpl)
1918 bool LibraryCallKit::inline_vector_select_from() {
1919 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
1920 const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
1921 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
1922 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
1923
1924 if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr ||
1925 vector_klass->const_oop() == nullptr ||
1926 elem_klass->const_oop() == nullptr ||
1927 !vlen->is_con()) {
1928 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s",
1929 NodeClassNames[argument(0)->Opcode()],
1930 NodeClassNames[argument(2)->Opcode()],
1931 NodeClassNames[argument(3)->Opcode()]);
1932 return false; // not enough info for intrinsification
1933 }
1934 if (!is_klass_initialized(vector_klass)) {
1935 log_if_needed(" ** klass argument not initialized");
1936 return false;
1937 }
1938 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1939 if (!elem_type->is_primitive_type()) {
1940 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1941 return false; // should be primitive type
1942 }
1943 BasicType elem_bt = elem_type->basic_type();
1944 int num_elem = vlen->get_con();
1945 if (!is_power_of_2(num_elem)) {
1946 log_if_needed(" ** vlen not power of two=%d", num_elem);
1947 return false;
1948 }
1949
1950 BasicType shuffle_bt = elem_bt;
1951 if (shuffle_bt == T_FLOAT) {
1952 shuffle_bt = T_INT;
1953 } else if (shuffle_bt == T_DOUBLE) {
1954 shuffle_bt = T_LONG;
1955 }
1956 bool need_load_shuffle = Matcher::vector_rearrange_requires_load_shuffle(shuffle_bt, num_elem);
1957
1958 int cast_vopc = VectorCastNode::opcode(-1, elem_bt); // from vector of type elem_bt
1959 if ((need_load_shuffle && !arch_supports_vector(Op_VectorLoadShuffle, num_elem, elem_bt, VecMaskNotUsed)) ||
1960 (elem_bt != shuffle_bt && !arch_supports_vector(cast_vopc, num_elem, shuffle_bt, VecMaskNotUsed)) ||
1961 !arch_supports_vector(Op_AndV, num_elem, shuffle_bt, VecMaskNotUsed) ||
1962 !arch_supports_vector(Op_Replicate, num_elem, shuffle_bt, VecMaskNotUsed)) {
1963 log_if_needed(" ** not supported: arity=0 op=selectFrom vlen=%d etype=%s ismask=no",
1964 num_elem, type2name(elem_bt));
1965 return false; // not supported
1966 }
1967
1968 bool is_masked_op = argument(6)->bottom_type() != TypePtr::NULL_PTR;
1969 bool use_predicate = is_masked_op;
1970 if (is_masked_op &&
1971 (mask_klass == nullptr ||
1972 mask_klass->const_oop() == nullptr ||
1973 !is_klass_initialized(mask_klass))) {
1974 log_if_needed(" ** mask_klass argument not initialized");
1975 return false; // not supported
1976 }
1977 VectorMaskUseType checkFlags = (VectorMaskUseType)(is_masked_op ? (VecMaskUseLoad | VecMaskUsePred) : VecMaskNotUsed);
1978 if (!arch_supports_vector(Op_VectorRearrange, num_elem, elem_bt, checkFlags)) {
1979 use_predicate = false;
1980 if(!is_masked_op ||
1981 (!arch_supports_vector(Op_VectorRearrange, num_elem, elem_bt, VecMaskNotUsed) ||
1982 !arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad) ||
1983 !arch_supports_vector(Op_Replicate, num_elem, elem_bt, VecMaskNotUsed))) {
1984 log_if_needed(" ** not supported: op=selectFrom vlen=%d etype=%s is_masked_op=%d",
1985 num_elem, type2name(elem_bt), is_masked_op);
1986 return false; // not supported
1987 }
1988 }
1989 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1990 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1991
1992 // v1 is the index vector
1993 Node* v1 = unbox_vector(argument(4), vbox_type, elem_bt, num_elem);
1994 // v2 is the vector being rearranged
1995 Node* v2 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
1996
1997 if (v1 == nullptr) {
1998 log_if_needed(" ** unbox failed v1=%s", NodeClassNames[argument(4)->Opcode()]);
1999 return false; // operand unboxing failed
2000 }
2001
2002 if (v2 == nullptr) {
2003 log_if_needed(" ** unbox failed v2=%s", NodeClassNames[argument(5)->Opcode()]);
2004 return false; // operand unboxing failed
2005 }
2006
2007 Node* mask = nullptr;
2008 if (is_masked_op) {
2009 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
2010 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
2011 mask = unbox_vector(argument(6), mbox_type, elem_bt, num_elem);
2012 if (mask == nullptr) {
2013 log_if_needed(" ** unbox failed mask=%s", NodeClassNames[argument(6)->Opcode()]);
2014 return false;
2015 }
2016 }
2017
2018 // cast index vector from elem_bt vector to byte vector
2019 const TypeVect* shuffle_vt = TypeVect::make(shuffle_bt, num_elem);
2020 Node* shuffle = v1;
2021
2022 if (shuffle_bt != elem_bt) {
2023 shuffle = gvn().transform(VectorCastNode::make(cast_vopc, v1, shuffle_bt, num_elem));
2024 }
2025
2026 // wrap the byte vector lanes to (num_elem - 1) to form the shuffle vector where num_elem is vector length
2027 // this is a simple AND operation as we come here only for power of two vector length
2028 Node* mod_val = gvn().makecon(TypeInteger::make(num_elem - 1, num_elem - 1, Type::WidenMin, shuffle_bt == T_LONG ? T_LONG : T_INT));
2029 Node* bcast_mod = gvn().transform(VectorNode::scalar2vector(mod_val, num_elem, shuffle_bt));
2030 shuffle = gvn().transform(VectorNode::make(Op_AndV, shuffle, bcast_mod, shuffle_vt));
2031
2032 // load the shuffle to use in rearrange
2033 if (need_load_shuffle) {
2034 shuffle = gvn().transform(new VectorLoadShuffleNode(shuffle, shuffle_vt));
2035 }
2036
2037 // and finally rearrange
2038 Node* rearrange = new VectorRearrangeNode(v2, shuffle);
2039 if (is_masked_op) {
2040 if (use_predicate) {
2041 // masked rearrange is supported so use that directly
2042 rearrange->add_req(mask);
2043 rearrange->add_flag(Node::Flag_is_predicated_vector);
2044 } else {
2045 // masked rearrange is not supported so emulate usig blend
2046 const TypeVect* vt = v1->bottom_type()->is_vect();
2047 rearrange = gvn().transform(rearrange);
2048
2049 // create a zero vector with each lane element set as zero
2050 Node* zero = gvn().makecon(Type::get_zero_type(elem_bt));
2051 Node* zerovec = gvn().transform(VectorNode::scalar2vector(zero, num_elem, elem_bt));
2052
2053 // For each lane for which mask is set, blend in the rearranged lane into zero vector
2054 rearrange = new VectorBlendNode(zerovec, rearrange, mask);
2055 }
2056 }
2057 rearrange = gvn().transform(rearrange);
2058
2059 // box the result
2060 Node* box = box_vector(rearrange, vbox_type, elem_bt, num_elem);
2061 set_result(box);
2062
2063 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2064 return true;
2065 }
2066
2067 Node* LibraryCallKit::gen_call_to_vector_math(int vector_api_op_id, BasicType bt, int num_elem, Node* opd1, Node* opd2) {
2068 assert(UseVectorStubs, "sanity");
2069 assert(vector_api_op_id >= VectorSupport::VECTOR_OP_MATH_START && vector_api_op_id <= VectorSupport::VECTOR_OP_MATH_END, "need valid op id");
2070 assert(opd1 != nullptr, "must not be null");
2071 const TypeVect* vt = TypeVect::make(bt, num_elem);
2072 const TypeFunc* call_type = OptoRuntime::Math_Vector_Vector_Type(opd2 != nullptr ? 2 : 1, vt, vt);
2073 char name[100] = "";
2074
2075 // Get address for vector math method.
2076 address addr = get_vector_math_address(vector_api_op_id, vt->length_in_bytes() * BitsPerByte, bt, name, 100);
2077
2078 if (addr == nullptr) {
2079 return nullptr;
2080 }
2081
2082 assert(name[0] != '\0', "name must not be null");
2083 Node* operation = make_runtime_call(RC_VECTOR,
2084 call_type,
2085 addr,
2086 name,
2087 TypePtr::BOTTOM,
2088 opd1,
2089 opd2);
2090 return gvn().transform(new ProjNode(gvn().transform(operation), TypeFunc::Parms));
2091 }
2092
2093 // public static
2094 // <V extends Vector<E>,
2095 // M extends VectorMask<E>,
2096 // E>
2097 // V broadcastInt(int opr, Class<? extends V> vectorClass, Class<? extends M> maskClass,
2098 // Class<E> elementType, int length,
2099 // V v, int n, M m,
2100 // VectorBroadcastIntOp<V, M> defaultImpl)
2101 bool LibraryCallKit::inline_vector_broadcast_int() {
2102 const TypeInt* opr = gvn().type(argument(0))->isa_int();
2103 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
2104 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
2105 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
2106 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
2107
2108 if (opr == nullptr || vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr) {
2109 return false; // dead code
2110 }
2111 if (!opr->is_con() || vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
2112 log_if_needed(" ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
2113 NodeClassNames[argument(0)->Opcode()],
2114 NodeClassNames[argument(1)->Opcode()],
2115 NodeClassNames[argument(3)->Opcode()],
2116 NodeClassNames[argument(4)->Opcode()]);
2117 return false; // not enough info for intrinsification
2118 }
2119 if (!is_klass_initialized(vector_klass)) {
2120 log_if_needed(" ** klass argument not initialized");
2121 return false;
2122 }
2123
2124 const Type* vmask_type = gvn().type(argument(7));
2125 bool is_masked_op = vmask_type != TypePtr::NULL_PTR;
2126 if (is_masked_op) {
2127 if (mask_klass == nullptr || mask_klass->const_oop() == nullptr) {
2128 log_if_needed(" ** missing constant: maskclass=%s", NodeClassNames[argument(2)->Opcode()]);
2129 return false; // not enough info for intrinsification
2130 }
2131
2132 if (!is_klass_initialized(mask_klass)) {
2133 log_if_needed(" ** mask klass argument not initialized");
2134 return false;
2135 }
2136
2137 if (vmask_type->maybe_null()) {
2138 log_if_needed(" ** null mask values are not allowed for masked op");
2139 return false;
2140 }
2141 }
2142
2143 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2144 if (!elem_type->is_primitive_type()) {
2145 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2146 return false; // should be primitive type
2147 }
2148
2149 int num_elem = vlen->get_con();
2150 BasicType elem_bt = elem_type->basic_type();
2151 int opc = VectorSupport::vop2ideal(opr->get_con(), elem_bt);
2152
2153 bool is_shift = VectorNode::is_shift_opcode(opc);
2154 bool is_rotate = VectorNode::is_rotate_opcode(opc);
2155
2156 if (opc == 0 || (!is_shift && !is_rotate)) {
2157 log_if_needed(" ** operation not supported: op=%d bt=%s", opr->get_con(), type2name(elem_bt));
2158 return false; // operation not supported
2159 }
2160
2161 int sopc = VectorNode::opcode(opc, elem_bt);
2162 if (sopc == 0) {
2163 log_if_needed(" ** operation not supported: opc=%s bt=%s", NodeClassNames[opc], type2name(elem_bt));
2164 return false; // operation not supported
2165 }
2166
2167 Node* cnt = argument(6);
2168 const TypeInt* cnt_type = cnt->bottom_type()->isa_int();
2169
2170 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2171 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2172
2173 // If CPU supports vector constant rotate instructions pass it directly
2174 bool is_const_rotate = is_rotate && cnt_type && cnt_type->is_con() &&
2175 Matcher::supports_vector_constant_rotates(cnt_type->get_con());
2176 bool has_scalar_args = is_rotate ? !is_const_rotate : true;
2177
2178 VectorMaskUseType checkFlags = (VectorMaskUseType)(is_masked_op ? (VecMaskUseLoad | VecMaskUsePred) : VecMaskNotUsed);
2179 bool use_predicate = is_masked_op;
2180
2181 if (!arch_supports_vector(sopc, num_elem, elem_bt, checkFlags, has_scalar_args)) {
2182 use_predicate = false;
2183 if (!is_masked_op ||
2184 (!arch_supports_vector(sopc, num_elem, elem_bt, VecMaskNotUsed, has_scalar_args) ||
2185 !arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad))) {
2186
2187 log_if_needed(" ** not supported: arity=0 op=int/%d vlen=%d etype=%s is_masked_op=%d",
2188 sopc, num_elem, type2name(elem_bt), is_masked_op ? 1 : 0);
2189 return false; // not supported
2190 }
2191 }
2192
2193 Node* opd1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
2194 Node* opd2 = nullptr;
2195 if (is_shift) {
2196 opd2 = vector_shift_count(cnt, opc, elem_bt, num_elem);
2197 } else {
2198 assert(is_rotate, "unexpected operation");
2199 if (!is_const_rotate) {
2200 cnt = elem_bt == T_LONG ? gvn().transform(new ConvI2LNode(cnt)) : cnt;
2201 opd2 = gvn().transform(VectorNode::scalar2vector(cnt, num_elem, elem_bt));
2202 } else {
2203 // Constant shift value.
2204 opd2 = cnt;
2205 }
2206 }
2207
2208 if (opd1 == nullptr || opd2 == nullptr) {
2209 return false;
2210 }
2211
2212 Node* mask = nullptr;
2213 if (is_masked_op) {
2214 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
2215 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
2216 mask = unbox_vector(argument(7), mbox_type, elem_bt, num_elem);
2217 if (mask == nullptr) {
2218 log_if_needed(" ** unbox failed mask=%s", NodeClassNames[argument(7)->Opcode()]);
2219 return false;
2220 }
2221 }
2222
2223 Node* operation = VectorNode::make(opc, opd1, opd2, num_elem, elem_bt);
2224 if (is_masked_op && mask != nullptr) {
2225 if (use_predicate) {
2226 operation->add_req(mask);
2227 operation->add_flag(Node::Flag_is_predicated_vector);
2228 } else {
2229 operation = gvn().transform(operation);
2230 operation = new VectorBlendNode(opd1, operation, mask);
2231 }
2232 }
2233 operation = gvn().transform(operation);
2234 Node* vbox = box_vector(operation, vbox_type, elem_bt, num_elem);
2235 set_result(vbox);
2236 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2237 return true;
2238 }
2239
2240 // public static <VOUT extends VectorPayload,
2241 // VIN extends VectorPayload,
2242 // S extends VectorSpecies>
2243 // VOUT convert(int oprId,
2244 // Class<?> fromVectorClass, Class<?> fromElementType, int fromVLen,
2245 // Class<?> toVectorClass, Class<?> toElementType, int toVLen,
2246 // VIN v, S s,
2247 // VectorConvertOp<VOUT, VIN, S> defaultImpl)
2248 //
2249 bool LibraryCallKit::inline_vector_convert() {
2250 const TypeInt* opr = gvn().type(argument(0))->isa_int();
2251
2252 const TypeInstPtr* vector_klass_from = gvn().type(argument(1))->isa_instptr();
2253 const TypeInstPtr* elem_klass_from = gvn().type(argument(2))->isa_instptr();
2254 const TypeInt* vlen_from = gvn().type(argument(3))->isa_int();
2255
2256 const TypeInstPtr* vector_klass_to = gvn().type(argument(4))->isa_instptr();
2257 const TypeInstPtr* elem_klass_to = gvn().type(argument(5))->isa_instptr();
2258 const TypeInt* vlen_to = gvn().type(argument(6))->isa_int();
2259
2260 if (opr == nullptr ||
2261 vector_klass_from == nullptr || elem_klass_from == nullptr || vlen_from == nullptr ||
2262 vector_klass_to == nullptr || elem_klass_to == nullptr || vlen_to == nullptr) {
2263 return false; // dead code
2264 }
2265 if (!opr->is_con() ||
2266 vector_klass_from->const_oop() == nullptr || elem_klass_from->const_oop() == nullptr || !vlen_from->is_con() ||
2267 vector_klass_to->const_oop() == nullptr || elem_klass_to->const_oop() == nullptr || !vlen_to->is_con()) {
2268 log_if_needed(" ** missing constant: opr=%s vclass_from=%s etype_from=%s vlen_from=%s vclass_to=%s etype_to=%s vlen_to=%s",
2269 NodeClassNames[argument(0)->Opcode()],
2270 NodeClassNames[argument(1)->Opcode()],
2271 NodeClassNames[argument(2)->Opcode()],
2272 NodeClassNames[argument(3)->Opcode()],
2273 NodeClassNames[argument(4)->Opcode()],
2274 NodeClassNames[argument(5)->Opcode()],
2275 NodeClassNames[argument(6)->Opcode()]);
2276 return false; // not enough info for intrinsification
2277 }
2278 if (!is_klass_initialized(vector_klass_from) || !is_klass_initialized(vector_klass_to)) {
2279 log_if_needed(" ** klass argument not initialized");
2280 return false;
2281 }
2282
2283 assert(opr->get_con() == VectorSupport::VECTOR_OP_CAST ||
2284 opr->get_con() == VectorSupport::VECTOR_OP_UCAST ||
2285 opr->get_con() == VectorSupport::VECTOR_OP_REINTERPRET, "wrong opcode");
2286 bool is_cast = (opr->get_con() == VectorSupport::VECTOR_OP_CAST || opr->get_con() == VectorSupport::VECTOR_OP_UCAST);
2287 bool is_ucast = (opr->get_con() == VectorSupport::VECTOR_OP_UCAST);
2288
2289 ciKlass* vbox_klass_from = vector_klass_from->const_oop()->as_instance()->java_lang_Class_klass();
2290 ciKlass* vbox_klass_to = vector_klass_to->const_oop()->as_instance()->java_lang_Class_klass();
2291
2292 bool is_mask = is_vector_mask(vbox_klass_from);
2293
2294 ciType* elem_type_from = elem_klass_from->const_oop()->as_instance()->java_mirror_type();
2295 if (!elem_type_from->is_primitive_type()) {
2296 return false; // should be primitive type
2297 }
2298 BasicType elem_bt_from = elem_type_from->basic_type();
2299 ciType* elem_type_to = elem_klass_to->const_oop()->as_instance()->java_mirror_type();
2300 if (!elem_type_to->is_primitive_type()) {
2301 return false; // should be primitive type
2302 }
2303 BasicType elem_bt_to = elem_type_to->basic_type();
2304
2305 int num_elem_from = vlen_from->get_con();
2306 int num_elem_to = vlen_to->get_con();
2307
2308 // Check whether we can unbox to appropriate size. Even with casting, checking for reinterpret is needed
2309 // since we may need to change size.
2310 if (!arch_supports_vector(Op_VectorReinterpret,
2311 num_elem_from,
2312 elem_bt_from,
2313 is_mask ? VecMaskUseAll : VecMaskNotUsed)) {
2314 log_if_needed(" ** not supported: arity=1 op=%s/1 vlen1=%d etype1=%s ismask=%d",
2315 is_cast ? "cast" : "reinterpret",
2316 num_elem_from, type2name(elem_bt_from), is_mask);
2317 return false;
2318 }
2319
2320 // Check whether we can support resizing/reinterpreting to the new size.
2321 if (!arch_supports_vector(Op_VectorReinterpret,
2322 num_elem_to,
2323 elem_bt_to,
2324 is_mask ? VecMaskUseAll : VecMaskNotUsed)) {
2325 log_if_needed(" ** not supported: arity=1 op=%s/2 vlen2=%d etype2=%s ismask=%d",
2326 is_cast ? "cast" : "reinterpret",
2327 num_elem_to, type2name(elem_bt_to), is_mask);
2328 return false;
2329 }
2330
2331 // At this point, we know that both input and output vector registers are supported
2332 // by the architecture. Next check if the casted type is simply to same type - which means
2333 // that it is actually a resize and not a cast.
2334 if (is_cast && elem_bt_from == elem_bt_to) {
2335 is_cast = false;
2336 }
2337
2338 const TypeInstPtr* vbox_type_from = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass_from);
2339
2340 Node* opd1 = unbox_vector(argument(7), vbox_type_from, elem_bt_from, num_elem_from);
2341 if (opd1 == nullptr) {
2342 return false;
2343 }
2344
2345 const TypeVect* src_type = TypeVect::make(elem_bt_from, num_elem_from, is_mask);
2346 const TypeVect* dst_type = TypeVect::make(elem_bt_to, num_elem_to, is_mask);
2347
2348 // Safety check to prevent casting if source mask is of type vector
2349 // and destination mask of type predicate vector and vice-versa.
2350 // From X86 standpoint, this case will only arise over KNL target,
2351 // where certain masks (depending on the species) are either propagated
2352 // through a vector or predicate register.
2353 if (is_mask &&
2354 ((src_type->isa_vectmask() == nullptr && dst_type->isa_vectmask()) ||
2355 (dst_type->isa_vectmask() == nullptr && src_type->isa_vectmask()))) {
2356 return false;
2357 }
2358
2359 Node* op = opd1;
2360 if (is_cast) {
2361 assert(!is_mask || num_elem_from == num_elem_to, "vector mask cast needs the same elem num");
2362 int cast_vopc = VectorCastNode::opcode(-1, elem_bt_from, !is_ucast);
2363
2364 // Make sure that vector cast is implemented to particular type/size combination if it is
2365 // not a mask casting.
2366 if (!is_mask && !arch_supports_vector(cast_vopc, num_elem_to, elem_bt_to, VecMaskNotUsed)) {
2367 log_if_needed(" ** not supported: arity=1 op=cast#%d/3 vlen2=%d etype2=%s ismask=%d",
2368 cast_vopc, num_elem_to, type2name(elem_bt_to), is_mask);
2369 return false;
2370 }
2371
2372 if (num_elem_from < num_elem_to) {
2373 // Since input and output number of elements are not consistent, we need to make sure we
2374 // properly size. Thus, first make a cast that retains the number of elements from source.
2375 int num_elem_for_cast = num_elem_from;
2376
2377 // It is possible that arch does not support this intermediate vector size
2378 // TODO More complex logic required here to handle this corner case for the sizes.
2379 if (!arch_supports_vector(cast_vopc, num_elem_for_cast, elem_bt_to, VecMaskNotUsed)) {
2380 log_if_needed(" ** not supported: arity=1 op=cast#%d/4 vlen1=%d etype2=%s ismask=%d",
2381 cast_vopc,
2382 num_elem_for_cast, type2name(elem_bt_to), is_mask);
2383 return false;
2384 }
2385
2386 op = gvn().transform(VectorCastNode::make(cast_vopc, op, elem_bt_to, num_elem_for_cast));
2387 // Now ensure that the destination gets properly resized to needed size.
2388 op = gvn().transform(new VectorReinterpretNode(op, op->bottom_type()->is_vect(), dst_type));
2389 } else if (num_elem_from > num_elem_to) {
2390 // Since number of elements from input is larger than output, simply reduce size of input
2391 // (we are supposed to drop top elements anyway).
2392 int num_elem_for_resize = num_elem_to;
2393
2394 // It is possible that arch does not support this intermediate vector size
2395 // TODO More complex logic required here to handle this corner case for the sizes.
2396 if (!arch_supports_vector(Op_VectorReinterpret,
2397 num_elem_for_resize,
2398 elem_bt_from,
2399 VecMaskNotUsed)) {
2400 log_if_needed(" ** not supported: arity=1 op=cast/5 vlen2=%d etype1=%s ismask=%d",
2401 num_elem_for_resize, type2name(elem_bt_from), is_mask);
2402 return false;
2403 }
2404
2405 const TypeVect* resize_type = TypeVect::make(elem_bt_from, num_elem_for_resize);
2406 op = gvn().transform(new VectorReinterpretNode(op, src_type, resize_type));
2407 op = gvn().transform(VectorCastNode::make(cast_vopc, op, elem_bt_to, num_elem_to));
2408 } else { // num_elem_from == num_elem_to
2409 if (is_mask) {
2410 // Make sure that cast for vector mask is implemented to particular type/size combination.
2411 if (!arch_supports_vector(Op_VectorMaskCast, num_elem_to, elem_bt_to, VecMaskNotUsed)) {
2412 log_if_needed(" ** not supported: arity=1 op=maskcast vlen2=%d etype2=%s ismask=%d",
2413 num_elem_to, type2name(elem_bt_to), is_mask);
2414 return false;
2415 }
2416 op = gvn().transform(new VectorMaskCastNode(op, dst_type));
2417 } else {
2418 // Since input and output number of elements match, and since we know this vector size is
2419 // supported, simply do a cast with no resize needed.
2420 op = gvn().transform(VectorCastNode::make(cast_vopc, op, elem_bt_to, num_elem_to));
2421 }
2422 }
2423 } else if (!Type::equals(src_type, dst_type)) {
2424 assert(!is_cast, "must be reinterpret");
2425 op = gvn().transform(new VectorReinterpretNode(op, src_type, dst_type));
2426 }
2427
2428 const TypeInstPtr* vbox_type_to = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass_to);
2429 Node* vbox = box_vector(op, vbox_type_to, elem_bt_to, num_elem_to);
2430 set_result(vbox);
2431 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem_to * type2aelembytes(elem_bt_to))));
2432 return true;
2433 }
2434
2435 // public static
2436 // <V extends Vector<E>,
2437 // E>
2438 // V insert(Class<? extends V> vectorClass, Class<E> elementType, int vlen,
2439 // V vec, int ix, long val,
2440 // VecInsertOp<V> defaultImpl)
2441 bool LibraryCallKit::inline_vector_insert() {
2442 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
2443 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
2444 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
2445 const TypeInt* idx = gvn().type(argument(4))->isa_int();
2446
2447 if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr || idx == nullptr) {
2448 return false; // dead code
2449 }
2450 if (vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con() || !idx->is_con()) {
2451 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s idx=%s",
2452 NodeClassNames[argument(0)->Opcode()],
2453 NodeClassNames[argument(1)->Opcode()],
2454 NodeClassNames[argument(2)->Opcode()],
2455 NodeClassNames[argument(4)->Opcode()]);
2456 return false; // not enough info for intrinsification
2457 }
2458 if (!is_klass_initialized(vector_klass)) {
2459 log_if_needed(" ** klass argument not initialized");
2460 return false;
2461 }
2462 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2463 if (!elem_type->is_primitive_type()) {
2464 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2465 return false; // should be primitive type
2466 }
2467 BasicType elem_bt = elem_type->basic_type();
2468 int num_elem = vlen->get_con();
2469 if (!arch_supports_vector(Op_VectorInsert, num_elem, elem_bt, VecMaskNotUsed)) {
2470 log_if_needed(" ** not supported: arity=1 op=insert vlen=%d etype=%s ismask=no",
2471 num_elem, type2name(elem_bt));
2472 return false; // not supported
2473 }
2474
2475 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2476 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2477
2478 Node* opd = unbox_vector(argument(3), vbox_type, elem_bt, num_elem);
2479 if (opd == nullptr) {
2480 return false;
2481 }
2482
2483 Node* insert_val = argument(5);
2484 assert(gvn().type(insert_val)->isa_long() != nullptr, "expected to be long");
2485
2486 // Convert insert value back to its appropriate type.
2487 switch (elem_bt) {
2488 case T_BYTE:
2489 insert_val = gvn().transform(new ConvL2INode(insert_val, TypeInt::BYTE));
2490 break;
2491 case T_SHORT:
2492 insert_val = gvn().transform(new ConvL2INode(insert_val, TypeInt::SHORT));
2493 break;
2494 case T_INT:
2495 insert_val = gvn().transform(new ConvL2INode(insert_val));
2496 break;
2497 case T_FLOAT:
2498 insert_val = gvn().transform(new ConvL2INode(insert_val));
2499 insert_val = gvn().transform(new MoveI2FNode(insert_val));
2500 break;
2501 case T_DOUBLE:
2502 insert_val = gvn().transform(new MoveL2DNode(insert_val));
2503 break;
2504 case T_LONG:
2505 // no conversion needed
2506 break;
2507 default: fatal("%s", type2name(elem_bt)); break;
2508 }
2509
2510 Node* operation = gvn().transform(VectorInsertNode::make(opd, insert_val, idx->get_con(), gvn()));
2511
2512 Node* vbox = box_vector(operation, vbox_type, elem_bt, num_elem);
2513 set_result(vbox);
2514 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2515 return true;
2516 }
2517
2518 // public static
2519 // <VM extends VectorPayload,
2520 // E>
2521 // long extract(Class<? extends VM> vClass, Class<E> eClass,
2522 // int length,
2523 // VM vm, int i,
2524 // VecExtractOp<VM> defaultImpl)
2525 bool LibraryCallKit::inline_vector_extract() {
2526 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
2527 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
2528 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
2529 const TypeInt* idx = gvn().type(argument(4))->isa_int();
2530
2531 if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr || idx == nullptr) {
2532 return false; // dead code
2533 }
2534 if (vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con() || !idx->is_con()) {
2535 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s",
2536 NodeClassNames[argument(0)->Opcode()],
2537 NodeClassNames[argument(1)->Opcode()],
2538 NodeClassNames[argument(2)->Opcode()]);
2539 return false; // not enough info for intrinsification
2540 }
2541 if (!is_klass_initialized(vector_klass)) {
2542 log_if_needed(" ** klass argument not initialized");
2543 return false;
2544 }
2545 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2546 if (!elem_type->is_primitive_type()) {
2547 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2548 return false; // should be primitive type
2549 }
2550 BasicType elem_bt = elem_type->basic_type();
2551 int num_elem = vlen->get_con();
2552
2553 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2554 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2555
2556 Node* opd = nullptr;
2557
2558 if (is_vector_mask(vbox_klass)) {
2559 // vbox_klass is mask. This is used for VectorMask.laneIsSet(int).
2560
2561 Node* pos = argument(4); // can be variable
2562 if (arch_supports_vector(Op_ExtractUB, num_elem, elem_bt, VecMaskUseAll)) {
2563 // Transform mask to vector with type of boolean and utilize ExtractUB node.
2564 opd = unbox_vector(argument(3), vbox_type, elem_bt, num_elem);
2565 if (opd == nullptr) {
2566 return false;
2567 }
2568 opd = gvn().transform(VectorStoreMaskNode::make(gvn(), opd, elem_bt, num_elem));
2569 opd = gvn().transform(new ExtractUBNode(opd, pos));
2570 opd = gvn().transform(new ConvI2LNode(opd));
2571 } else if (arch_supports_vector(Op_VectorMaskToLong, num_elem, elem_bt, VecMaskUseLoad)) {
2572 opd = unbox_vector(argument(3), vbox_type, elem_bt, num_elem);
2573 if (opd == nullptr) {
2574 return false;
2575 }
2576 // VectorMaskToLongNode requires the input is either a mask or a vector with BOOLEAN type.
2577 if (opd->bottom_type()->isa_vectmask() == nullptr) {
2578 opd = gvn().transform(VectorStoreMaskNode::make(gvn(), opd, elem_bt, num_elem));
2579 }
2580 // ((toLong() >>> pos) & 1L
2581 opd = gvn().transform(new VectorMaskToLongNode(opd, TypeLong::LONG));
2582 opd = gvn().transform(new URShiftLNode(opd, pos));
2583 opd = gvn().transform(new AndLNode(opd, gvn().makecon(TypeLong::ONE)));
2584 } else {
2585 log_if_needed(" ** Rejected mask extraction because architecture does not support it");
2586 return false; // not supported
2587 }
2588 } else {
2589 // vbox_klass is vector. This is used for Vector.lane(int).
2590 if (!idx->is_con()) {
2591 log_if_needed(" ** missing constant: idx=%s", NodeClassNames[argument(4)->Opcode()]);
2592 return false; // not enough info for intrinsification
2593 }
2594
2595 int vopc = ExtractNode::opcode(elem_bt);
2596 if (!arch_supports_vector(vopc, num_elem, elem_bt, VecMaskNotUsed)) {
2597 log_if_needed(" ** not supported: arity=1 op=extract vlen=%d etype=%s ismask=no",
2598 num_elem, type2name(elem_bt));
2599 return false; // not supported
2600 }
2601
2602 opd = unbox_vector(argument(3), vbox_type, elem_bt, num_elem);
2603 if (opd == nullptr) {
2604 return false;
2605 }
2606 ConINode* idx_con = gvn().intcon(idx->get_con())->as_ConI();
2607
2608 opd = gvn().transform(ExtractNode::make(opd, idx_con, elem_bt));
2609 switch (elem_bt) {
2610 case T_BYTE:
2611 case T_SHORT:
2612 case T_INT: {
2613 opd = gvn().transform(new ConvI2LNode(opd));
2614 break;
2615 }
2616 case T_FLOAT: {
2617 opd = gvn().transform(new MoveF2INode(opd));
2618 opd = gvn().transform(new ConvI2LNode(opd));
2619 break;
2620 }
2621 case T_DOUBLE: {
2622 opd = gvn().transform(new MoveD2LNode(opd));
2623 break;
2624 }
2625 case T_LONG: {
2626 // no conversion needed
2627 break;
2628 }
2629 default: fatal("%s", type2name(elem_bt));
2630 }
2631 }
2632 set_result(opd);
2633 return true;
2634 }
2635
2636 static Node* LowerSelectFromTwoVectorOperation(PhaseGVN& phase, Node* index_vec, Node* src1, Node* src2, const TypeVect* vt) {
2637 int num_elem = vt->length();
2638 BasicType elem_bt = vt->element_basic_type();
2639
2640 // Lower selectFrom operation into its constituent operations.
2641 // SelectFromTwoVectorNode =
2642 // (VectorBlend
2643 // (VectorRearrange SRC1 (WRAPED_INDEX AND (VLEN-1))
2644 // (VectorRearrange SRC2 (WRAPED_INDEX AND (VLEN-1))
2645 // MASK)
2646 // Where
2647 // WRAPED_INDEX are computed by wrapping incoming indexes
2648 // to two vector index range [0, VLEN*2) and
2649 // MASK = WRAPED_INDEX < VLEN
2650 //
2651 // IR lowering prevents intrinsification failure and associated argument
2652 // boxing penalties.
2653 //
2654
2655 BasicType shuffle_bt = elem_bt;
2656 if (shuffle_bt == T_FLOAT) {
2657 shuffle_bt = T_INT;
2658 } else if (shuffle_bt == T_DOUBLE) {
2659 shuffle_bt = T_LONG;
2660 }
2661 const TypeVect* st = TypeVect::make(shuffle_bt, num_elem);
2662
2663 // Cast index vector to the corresponding bit type
2664 if (elem_bt != shuffle_bt) {
2665 int cast_vopc = VectorCastNode::opcode(0, elem_bt, true);
2666 index_vec = phase.transform(VectorCastNode::make(cast_vopc, index_vec, shuffle_bt, num_elem));
2667 }
2668
2669 // Wrap indexes into two vector index range [0, VLEN * 2)
2670 Node* two_vect_lane_cnt_m1 = phase.makecon(TypeInteger::make(2 * num_elem - 1, 2 * num_elem - 1, Type::WidenMin, shuffle_bt == T_LONG ? T_LONG : T_INT));
2671 Node* bcast_two_vect_lane_cnt_m1_vec = phase.transform(VectorNode::scalar2vector(two_vect_lane_cnt_m1, num_elem,
2672 shuffle_bt, false));
2673 index_vec = phase.transform(VectorNode::make(Op_AndV, index_vec, bcast_two_vect_lane_cnt_m1_vec, st));
2674
2675 // Compute the blend mask for merging two independently permitted vectors
2676 // using shuffle index in two vector index range [0, VLEN * 2).
2677 BoolTest::mask pred = BoolTest::le;
2678 ConINode* pred_node = phase.makecon(TypeInt::make(pred))->as_ConI();
2679 const TypeVect* vmask_type = TypeVect::makemask(shuffle_bt, num_elem);
2680 Node* lane_cnt_m1 = phase.makecon(TypeInteger::make(num_elem - 1, num_elem - 1, Type::WidenMin, shuffle_bt == T_LONG ? T_LONG : T_INT));
2681 Node* bcast_lane_cnt_m1_vec = phase.transform(VectorNode::scalar2vector(lane_cnt_m1, num_elem, shuffle_bt, false));
2682 Node* mask = phase.transform(new VectorMaskCmpNode(pred, index_vec, bcast_lane_cnt_m1_vec, pred_node, vmask_type));
2683
2684 // Rearrange expects the indexes to lie within single vector index range [0, VLEN).
2685 Node* wrapped_index_vec = phase.transform(VectorNode::make(Op_AndV, index_vec, bcast_lane_cnt_m1_vec, st));
2686
2687 // Load indexes from byte vector and appropriately transform them to target
2688 // specific permutation index format.
2689 if (Matcher::vector_rearrange_requires_load_shuffle(shuffle_bt, num_elem)) {
2690 wrapped_index_vec = phase.transform(new VectorLoadShuffleNode(wrapped_index_vec, st));
2691 }
2692
2693 vmask_type = TypeVect::makemask(elem_bt, num_elem);
2694 mask = phase.transform(new VectorMaskCastNode(mask, vmask_type));
2695
2696 Node* p1 = phase.transform(new VectorRearrangeNode(src1, wrapped_index_vec));
2697 Node* p2 = phase.transform(new VectorRearrangeNode(src2, wrapped_index_vec));
2698
2699 return new VectorBlendNode(p2, p1, mask);
2700 }
2701
2702 // public static
2703 // <V extends Vector<E>,
2704 // E>
2705 // V selectFromTwoVectorOp(Class<? extends V> vClass, Class<E> eClass, int length,
2706 // V v1, V v2, V v3,
2707 // SelectFromTwoVector<V> defaultImpl)
2708 bool LibraryCallKit::inline_vector_select_from_two_vectors() {
2709 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
2710 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
2711 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
2712
2713 if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr || vector_klass->const_oop() == nullptr ||
2714 elem_klass->const_oop() == nullptr ||!vlen->is_con()) {
2715 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s",
2716 NodeClassNames[argument(0)->Opcode()],
2717 NodeClassNames[argument(1)->Opcode()],
2718 NodeClassNames[argument(2)->Opcode()]);
2719 return false; // not enough info for intrinsification
2720 }
2721
2722 if (!is_klass_initialized(vector_klass)) {
2723 log_if_needed(" ** klass argument not initialized");
2724 return false;
2725 }
2726
2727 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2728 if (!elem_type->is_primitive_type()) {
2729 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2730 return false; // should be primitive type
2731 }
2732
2733 int num_elem = vlen->get_con();
2734 if (!is_power_of_2(num_elem)) {
2735 log_if_needed(" ** vlen is not power of two=%d", num_elem);
2736 return false;
2737 }
2738
2739 BasicType elem_bt = elem_type->basic_type();
2740 BasicType index_elem_bt = elem_bt;
2741 if (elem_bt == T_FLOAT) {
2742 index_elem_bt = T_INT;
2743 } else if (elem_bt == T_DOUBLE) {
2744 index_elem_bt = T_LONG;
2745 }
2746
2747 bool lowerSelectFromOp = false;
2748 if (!arch_supports_vector(Op_SelectFromTwoVector, num_elem, elem_bt, VecMaskNotUsed)) {
2749 int cast_vopc = VectorCastNode::opcode(-1, elem_bt, true);
2750 if ((elem_bt != index_elem_bt && !arch_supports_vector(cast_vopc, num_elem, index_elem_bt, VecMaskNotUsed)) ||
2751 !arch_supports_vector(Op_VectorMaskCmp, num_elem, index_elem_bt, VecMaskNotUsed) ||
2752 !arch_supports_vector(Op_AndV, num_elem, index_elem_bt, VecMaskNotUsed) ||
2753 !arch_supports_vector(Op_VectorMaskCast, num_elem, elem_bt, VecMaskNotUsed) ||
2754 !arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad) ||
2755 !arch_supports_vector(Op_VectorRearrange, num_elem, elem_bt, VecMaskNotUsed) ||
2756 !arch_supports_vector(Op_VectorLoadShuffle, num_elem, index_elem_bt, VecMaskNotUsed) ||
2757 !arch_supports_vector(Op_Replicate, num_elem, index_elem_bt, VecMaskNotUsed)) {
2758 log_if_needed(" ** not supported: opc=%d vlen=%d etype=%s ismask=useload",
2759 Op_SelectFromTwoVector, num_elem, type2name(elem_bt));
2760 return false; // not supported
2761 }
2762 lowerSelectFromOp = true;
2763 }
2764
2765 int cast_vopc = VectorCastNode::opcode(-1, elem_bt, true);
2766 if (!lowerSelectFromOp) {
2767 if (!arch_supports_vector(Op_AndV, num_elem, index_elem_bt, VecMaskNotUsed) ||
2768 !arch_supports_vector(Op_Replicate, num_elem, index_elem_bt, VecMaskNotUsed) ||
2769 (is_floating_point_type(elem_bt) &&
2770 !arch_supports_vector(cast_vopc, num_elem, index_elem_bt, VecMaskNotUsed))) {
2771 log_if_needed(" ** index wrapping not supported: vlen=%d etype=%s" ,
2772 num_elem, type2name(elem_bt));
2773 return false; // not supported
2774 }
2775 }
2776
2777 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2778 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2779
2780 Node* opd1 = unbox_vector(argument(3), vbox_type, elem_bt, num_elem);
2781 if (opd1 == nullptr) {
2782 log_if_needed(" ** unbox failed v1=%s",
2783 NodeClassNames[argument(3)->Opcode()]);
2784 return false;
2785 }
2786 Node* opd2 = unbox_vector(argument(4), vbox_type, elem_bt, num_elem);
2787 if (opd2 == nullptr) {
2788 log_if_needed(" ** unbox failed v2=%s",
2789 NodeClassNames[argument(4)->Opcode()]);
2790 return false;
2791 }
2792 Node* opd3 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
2793 if (opd3 == nullptr) {
2794 log_if_needed(" ** unbox failed v3=%s",
2795 NodeClassNames[argument(5)->Opcode()]);
2796 return false;
2797 }
2798
2799 const TypeVect* vt = TypeVect::make(elem_bt, num_elem);
2800
2801 Node* operation = nullptr;
2802 if (lowerSelectFromOp) {
2803 operation = gvn().transform(LowerSelectFromTwoVectorOperation(gvn(), opd1, opd2, opd3, vt));
2804 } else {
2805 if (index_elem_bt != elem_bt) {
2806 opd1 = gvn().transform(VectorCastNode::make(cast_vopc, opd1, index_elem_bt, num_elem));
2807 }
2808 int indexRangeMask = 2 * num_elem - 1;
2809 Node* wrap_mask = gvn().makecon(TypeInteger::make(indexRangeMask, indexRangeMask, Type::WidenMin, index_elem_bt != T_LONG ? T_INT : index_elem_bt));
2810 Node* wrap_mask_vec = gvn().transform(VectorNode::scalar2vector(wrap_mask, num_elem, index_elem_bt, false));
2811 opd1 = gvn().transform(VectorNode::make(Op_AndV, opd1, wrap_mask_vec, opd1->bottom_type()->is_vect()));
2812 operation = gvn().transform(VectorNode::make(Op_SelectFromTwoVector, opd1, opd2, opd3, vt));
2813 }
2814
2815 // Wrap it up in VectorBox to keep object type information.
2816 Node* vbox = box_vector(operation, vbox_type, elem_bt, num_elem);
2817 set_result(vbox);
2818 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2819 return true;
2820 }
2821
2822 // public static
2823 // <V extends Vector<E>,
2824 // M extends VectorMask<E>,
2825 // E>
2826 // V compressExpandOp(int opr,
2827 // Class<? extends V> vClass, Class<? extends M> mClass, Class<E> eClass,
2828 // int length, V v, M m,
2829 // CompressExpandOperation<V, M> defaultImpl)
2830 bool LibraryCallKit::inline_vector_compress_expand() {
2831 const TypeInt* opr = gvn().type(argument(0))->isa_int();
2832 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
2833 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
2834 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
2835 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
2836
2837 if (opr == nullptr || !opr->is_con() ||
2838 vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
2839 mask_klass == nullptr || mask_klass->const_oop() == nullptr ||
2840 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
2841 vlen == nullptr || !vlen->is_con()) {
2842 log_if_needed(" ** missing constant: opr=%s vclass=%s mclass=%s etype=%s vlen=%s",
2843 NodeClassNames[argument(0)->Opcode()],
2844 NodeClassNames[argument(1)->Opcode()],
2845 NodeClassNames[argument(2)->Opcode()],
2846 NodeClassNames[argument(3)->Opcode()],
2847 NodeClassNames[argument(4)->Opcode()]);
2848 return false; // not enough info for intrinsification
2849 }
2850
2851 if (!is_klass_initialized(vector_klass) || !is_klass_initialized(mask_klass)) {
2852 log_if_needed(" ** klass argument not initialized");
2853 return false;
2854 }
2855
2856 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2857 if (!elem_type->is_primitive_type()) {
2858 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2859 return false; // should be primitive type
2860 }
2861
2862 int num_elem = vlen->get_con();
2863 BasicType elem_bt = elem_type->basic_type();
2864 int opc = VectorSupport::vop2ideal(opr->get_con(), elem_bt);
2865
2866 if (!arch_supports_vector(opc, num_elem, elem_bt, VecMaskUseLoad)) {
2867 log_if_needed(" ** not supported: opc=%d vlen=%d etype=%s ismask=useload",
2868 opc, num_elem, type2name(elem_bt));
2869 return false; // not supported
2870 }
2871
2872 Node* opd1 = nullptr;
2873 const TypeInstPtr* vbox_type = nullptr;
2874 if (opc != Op_CompressM) {
2875 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2876 vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2877 opd1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
2878 if (opd1 == nullptr) {
2879 log_if_needed(" ** unbox failed vector=%s",
2880 NodeClassNames[argument(5)->Opcode()]);
2881 return false;
2882 }
2883 }
2884
2885 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
2886 assert(is_vector_mask(mbox_klass), "argument(6) should be a mask class");
2887 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
2888
2889 Node* mask = unbox_vector(argument(6), mbox_type, elem_bt, num_elem);
2890 if (mask == nullptr) {
2891 log_if_needed(" ** unbox failed mask=%s",
2892 NodeClassNames[argument(6)->Opcode()]);
2893 return false;
2894 }
2895
2896 const TypeVect* vt = TypeVect::make(elem_bt, num_elem, opc == Op_CompressM);
2897 Node* operation = gvn().transform(VectorNode::make(opc, opd1, mask, vt));
2898
2899 // Wrap it up in VectorBox to keep object type information.
2900 const TypeInstPtr* box_type = opc == Op_CompressM ? mbox_type : vbox_type;
2901 Node* vbox = box_vector(operation, box_type, elem_bt, num_elem);
2902 set_result(vbox);
2903 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2904 return true;
2905 }
2906
2907 // public static
2908 // <V extends Vector<E>,
2909 // E,
2910 // S extends VectorSpecies<E>>
2911 // V indexVector(Class<? extends V> vClass, Class<E> eClass,
2912 // int length,
2913 // V v, int step, S s,
2914 // IndexOperation<V, S> defaultImpl)
2915 bool LibraryCallKit::inline_index_vector() {
2916 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
2917 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
2918 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
2919
2920 if (vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
2921 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
2922 vlen == nullptr || !vlen->is_con() ) {
2923 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s",
2924 NodeClassNames[argument(0)->Opcode()],
2925 NodeClassNames[argument(1)->Opcode()],
2926 NodeClassNames[argument(2)->Opcode()]);
2927 return false; // not enough info for intrinsification
2928 }
2929
2930 if (!is_klass_initialized(vector_klass)) {
2931 log_if_needed(" ** klass argument not initialized");
2932 return false;
2933 }
2934
2935 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2936 if (!elem_type->is_primitive_type()) {
2937 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2938 return false; // should be primitive type
2939 }
2940
2941 int num_elem = vlen->get_con();
2942 BasicType elem_bt = elem_type->basic_type();
2943
2944 // Check whether the iota index generation op is supported by the current hardware
2945 if (!arch_supports_vector(Op_VectorLoadConst, num_elem, elem_bt, VecMaskNotUsed)) {
2946 log_if_needed(" ** not supported: vlen=%d etype=%s", num_elem, type2name(elem_bt));
2947 return false; // not supported
2948 }
2949
2950 int mul_op = VectorSupport::vop2ideal(VectorSupport::VECTOR_OP_MUL, elem_bt);
2951 int vmul_op = VectorNode::opcode(mul_op, elem_bt);
2952 bool needs_mul = true;
2953 Node* scale = argument(4);
2954 const TypeInt* scale_type = gvn().type(scale)->isa_int();
2955 // Multiply is not needed if the scale is a constant "1".
2956 if (scale_type && scale_type->is_con() && scale_type->get_con() == 1) {
2957 needs_mul = false;
2958 } else {
2959 // Check whether the vector multiply op is supported by the current hardware
2960 if (!arch_supports_vector(vmul_op, num_elem, elem_bt, VecMaskNotUsed)) {
2961 log_if_needed(" ** not supported: vlen=%d etype=%s", num_elem, type2name(elem_bt));
2962 return false; // not supported
2963 }
2964
2965 // Check whether the scalar cast op is supported by the current hardware
2966 if (is_floating_point_type(elem_bt) || elem_bt == T_LONG) {
2967 int cast_op = elem_bt == T_LONG ? Op_ConvI2L :
2968 elem_bt == T_FLOAT? Op_ConvI2F : Op_ConvI2D;
2969 if (!Matcher::match_rule_supported(cast_op)) {
2970 log_if_needed(" ** Rejected op (%s) because architecture does not support it",
2971 NodeClassNames[cast_op]);
2972 return false; // not supported
2973 }
2974 }
2975 }
2976
2977 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2978 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2979 Node* opd = unbox_vector(argument(3), vbox_type, elem_bt, num_elem);
2980 if (opd == nullptr) {
2981 log_if_needed(" ** unbox failed vector=%s",
2982 NodeClassNames[argument(3)->Opcode()]);
2983 return false;
2984 }
2985
2986 int add_op = VectorSupport::vop2ideal(VectorSupport::VECTOR_OP_ADD, elem_bt);
2987 int vadd_op = VectorNode::opcode(add_op, elem_bt);
2988 bool needs_add = true;
2989 // The addition is not needed if all the element values of "opd" are zero
2990 if (VectorNode::is_all_zeros_vector(opd)) {
2991 needs_add = false;
2992 } else {
2993 // Check whether the vector addition op is supported by the current hardware
2994 if (!arch_supports_vector(vadd_op, num_elem, elem_bt, VecMaskNotUsed)) {
2995 log_if_needed(" ** not supported: vlen=%d etype=%s", num_elem, type2name(elem_bt));
2996 return false; // not supported
2997 }
2998 }
2999
3000 // Compute the iota indice vector
3001 const TypeVect* vt = TypeVect::make(elem_bt, num_elem);
3002 Node* index = gvn().transform(new VectorLoadConstNode(gvn().makecon(TypeInt::ZERO), vt));
3003
3004 // Broadcast the "scale" to a vector, and multiply the "scale" with iota indice vector.
3005 if (needs_mul) {
3006 switch (elem_bt) {
3007 case T_BOOLEAN: // fall-through
3008 case T_BYTE: // fall-through
3009 case T_SHORT: // fall-through
3010 case T_CHAR: // fall-through
3011 case T_INT: {
3012 // no conversion needed
3013 break;
3014 }
3015 case T_LONG: {
3016 scale = gvn().transform(new ConvI2LNode(scale));
3017 break;
3018 }
3019 case T_FLOAT: {
3020 scale = gvn().transform(new ConvI2FNode(scale));
3021 break;
3022 }
3023 case T_DOUBLE: {
3024 scale = gvn().transform(new ConvI2DNode(scale));
3025 break;
3026 }
3027 default: fatal("%s", type2name(elem_bt));
3028 }
3029 scale = gvn().transform(VectorNode::scalar2vector(scale, num_elem, elem_bt));
3030 index = gvn().transform(VectorNode::make(vmul_op, index, scale, vt));
3031 }
3032
3033 // Add "opd" if addition is needed.
3034 if (needs_add) {
3035 index = gvn().transform(VectorNode::make(vadd_op, opd, index, vt));
3036 }
3037 Node* vbox = box_vector(index, vbox_type, elem_bt, num_elem);
3038 set_result(vbox);
3039 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
3040 return true;
3041 }
3042
3043 // public static
3044 // <E,
3045 // M extends VectorMask<E>>
3046 // M indexPartiallyInUpperRange(Class<? extends M> mClass, Class<E> eClass, int length,
3047 // long offset, long limit,
3048 // IndexPartiallyInUpperRangeOperation<E, M> defaultImpl)
3049 bool LibraryCallKit::inline_index_partially_in_upper_range() {
3050 const TypeInstPtr* mask_klass = gvn().type(argument(0))->isa_instptr();
3051 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
3052 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
3053
3054 if (mask_klass == nullptr || mask_klass->const_oop() == nullptr ||
3055 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
3056 vlen == nullptr || !vlen->is_con()) {
3057 log_if_needed(" ** missing constant: mclass=%s etype=%s vlen=%s",
3058 NodeClassNames[argument(0)->Opcode()],
3059 NodeClassNames[argument(1)->Opcode()],
3060 NodeClassNames[argument(2)->Opcode()]);
3061 return false; // not enough info for intrinsification
3062 }
3063
3064 if (!is_klass_initialized(mask_klass)) {
3065 log_if_needed(" ** klass argument not initialized");
3066 return false;
3067 }
3068
3069 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
3070 if (!elem_type->is_primitive_type()) {
3071 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
3072 return false; // should be primitive type
3073 }
3074
3075 int num_elem = vlen->get_con();
3076 BasicType elem_bt = elem_type->basic_type();
3077
3078 // Check whether the necessary ops are supported by current hardware.
3079 bool supports_mask_gen = arch_supports_vector(Op_VectorMaskGen, num_elem, elem_bt, VecMaskUseStore);
3080 if (!supports_mask_gen) {
3081 if (!arch_supports_vector(Op_VectorLoadConst, num_elem, elem_bt, VecMaskNotUsed) ||
3082 !arch_supports_vector(Op_Replicate, num_elem, elem_bt, VecMaskNotUsed) ||
3083 !arch_supports_vector(Op_VectorMaskCmp, num_elem, elem_bt, VecMaskUseStore)) {
3084 log_if_needed(" ** not supported: vlen=%d etype=%s", num_elem, type2name(elem_bt));
3085 return false; // not supported
3086 }
3087
3088 // Check whether the scalar cast operation is supported by current hardware.
3089 if (elem_bt != T_LONG) {
3090 int cast_op = is_integral_type(elem_bt) ? Op_ConvL2I
3091 : (elem_bt == T_FLOAT ? Op_ConvL2F : Op_ConvL2D);
3092 if (!Matcher::match_rule_supported(cast_op)) {
3093 log_if_needed(" ** Rejected op (%s) because architecture does not support it",
3094 NodeClassNames[cast_op]);
3095 return false; // not supported
3096 }
3097 }
3098 }
3099
3100 Node* offset = argument(3);
3101 Node* limit = argument(5);
3102 if (offset == nullptr || limit == nullptr) {
3103 log_if_needed(" ** offset or limit argument is null");
3104 return false; // not supported
3105 }
3106
3107 ciKlass* box_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
3108 assert(is_vector_mask(box_klass), "argument(0) should be a mask class");
3109 const TypeInstPtr* box_type = TypeInstPtr::make_exact(TypePtr::NotNull, box_klass);
3110
3111 // We assume "offset > 0 && limit >= offset && limit - offset < num_elem".
3112 // So directly get indexLimit with "indexLimit = limit - offset".
3113 Node* indexLimit = gvn().transform(new SubLNode(limit, offset));
3114 Node* mask = nullptr;
3115 if (supports_mask_gen) {
3116 mask = gvn().transform(VectorMaskGenNode::make(indexLimit, elem_bt, num_elem));
3117 } else {
3118 // Generate the vector mask based on "mask = iota < indexLimit".
3119 // Broadcast "indexLimit" to a vector.
3120 switch (elem_bt) {
3121 case T_BOOLEAN: // fall-through
3122 case T_BYTE: // fall-through
3123 case T_SHORT: // fall-through
3124 case T_CHAR: // fall-through
3125 case T_INT: {
3126 indexLimit = gvn().transform(new ConvL2INode(indexLimit));
3127 break;
3128 }
3129 case T_DOUBLE: {
3130 indexLimit = gvn().transform(new ConvL2DNode(indexLimit));
3131 break;
3132 }
3133 case T_FLOAT: {
3134 indexLimit = gvn().transform(new ConvL2FNode(indexLimit));
3135 break;
3136 }
3137 case T_LONG: {
3138 // no conversion needed
3139 break;
3140 }
3141 default: fatal("%s", type2name(elem_bt));
3142 }
3143 indexLimit = gvn().transform(VectorNode::scalar2vector(indexLimit, num_elem, elem_bt));
3144
3145 // Load the "iota" vector.
3146 const TypeVect* vt = TypeVect::make(elem_bt, num_elem);
3147 Node* iota = gvn().transform(new VectorLoadConstNode(gvn().makecon(TypeInt::ZERO), vt));
3148
3149 // Compute the vector mask with "mask = iota < indexLimit".
3150 ConINode* pred_node = (ConINode*)gvn().makecon(TypeInt::make(BoolTest::lt));
3151 const TypeVect* vmask_type = TypeVect::makemask(elem_bt, num_elem);
3152 mask = gvn().transform(new VectorMaskCmpNode(BoolTest::lt, iota, indexLimit, pred_node, vmask_type));
3153 }
3154 Node* vbox = box_vector(mask, box_type, elem_bt, num_elem);
3155 set_result(vbox);
3156 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
3157 return true;
3158 }
3159
3160 #undef non_product_log_if_needed
3161 #undef log_if_needed