1 /*
2 * Copyright (c) 2020, 2025, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "ci/ciSymbols.hpp"
26 #include "classfile/vmSymbols.hpp"
27 #include "opto/library_call.hpp"
28 #include "opto/runtime.hpp"
29 #include "opto/vectornode.hpp"
30 #include "prims/vectorSupport.hpp"
31 #include "runtime/stubRoutines.hpp"
32
33 #ifdef ASSERT
34 static bool is_vector(ciKlass* klass) {
35 return klass->is_subclass_of(ciEnv::current()->vector_VectorPayload_klass());
36 }
37
38 static bool check_vbox(const TypeInstPtr* vbox_type) {
39 assert(vbox_type->klass_is_exact(), "");
40
41 ciInstanceKlass* ik = vbox_type->instance_klass();
42 assert(is_vector(ik), "not a vector");
43
44 ciField* fd1 = ik->get_field_by_name(ciSymbols::ETYPE_name(), ciSymbols::class_signature(), /* is_static */ true);
45 assert(fd1 != nullptr, "element type info is missing");
46
47 ciConstant val1 = fd1->constant_value();
48 BasicType elem_bt = val1.as_object()->as_instance()->java_mirror_type()->basic_type();
49 assert(is_java_primitive(elem_bt), "element type info is missing");
50
51 ciField* fd2 = ik->get_field_by_name(ciSymbols::VLENGTH_name(), ciSymbols::int_signature(), /* is_static */ true);
52 assert(fd2 != nullptr, "vector length info is missing");
53
54 ciConstant val2 = fd2->constant_value();
55 assert(val2.as_int() > 0, "vector length info is missing");
56
57 return true;
58 }
59 #endif
60
61 #define log_if_needed(...) \
62 if (C->print_intrinsics()) { \
63 tty->print_cr(__VA_ARGS__); \
64 }
65
66 #ifndef PRODUCT
67 #define non_product_log_if_needed(...) log_if_needed(__VA_ARGS__)
68 #else
69 #define non_product_log_if_needed(...)
70 #endif
71
72 static bool is_vector_mask(ciKlass* klass) {
73 return klass->is_subclass_of(ciEnv::current()->vector_VectorMask_klass());
74 }
75
76 bool LibraryCallKit::arch_supports_vector_rotate(int opc, int num_elem, BasicType elem_bt,
77 VectorMaskUseType mask_use_type, bool has_scalar_args) {
78 bool is_supported = true;
79
80 // has_scalar_args flag is true only for non-constant scalar shift count,
81 // since in this case shift needs to be broadcasted.
82 if (!Matcher::match_rule_supported_vector(opc, num_elem, elem_bt) ||
83 (has_scalar_args && !arch_supports_vector(Op_Replicate, num_elem, elem_bt, VecMaskNotUsed))) {
84 is_supported = false;
85 }
86
87 if (is_supported) {
88 // Check if mask unboxing is supported, this is a two step process which first loads the contents
89 // of boolean array into vector followed by either lane expansion to match the lane size of masked
90 // vector operation or populate the predicate register.
91 if ((mask_use_type & VecMaskUseLoad) != 0) {
92 if (!Matcher::match_rule_supported_vector(Op_VectorLoadMask, num_elem, elem_bt) ||
93 !Matcher::match_rule_supported_vector(Op_LoadVector, num_elem, T_BOOLEAN)) {
94 non_product_log_if_needed(" ** Rejected vector mask loading (%s,%s,%d) because architecture does not support it",
95 NodeClassNames[Op_VectorLoadMask], type2name(elem_bt), num_elem);
96 return false;
97 }
98 }
99
100 if ((mask_use_type & VecMaskUsePred) != 0) {
101 if (!Matcher::has_predicated_vectors() ||
102 !Matcher::match_rule_supported_vector_masked(opc, num_elem, elem_bt)) {
103 non_product_log_if_needed("Rejected vector mask predicate using (%s,%s,%d) because architecture does not support it",
104 NodeClassNames[opc], type2name(elem_bt), num_elem);
105 return false;
106 }
107 }
108 }
109
110 int lshiftopc, rshiftopc;
111 switch(elem_bt) {
112 case T_BYTE:
113 lshiftopc = Op_LShiftI;
114 rshiftopc = Op_URShiftB;
115 break;
116 case T_SHORT:
117 lshiftopc = Op_LShiftI;
118 rshiftopc = Op_URShiftS;
119 break;
120 case T_INT:
121 lshiftopc = Op_LShiftI;
122 rshiftopc = Op_URShiftI;
123 break;
124 case T_LONG:
125 lshiftopc = Op_LShiftL;
126 rshiftopc = Op_URShiftL;
127 break;
128 default: fatal("Unexpected type: %s", type2name(elem_bt));
129 }
130 int lshiftvopc = VectorNode::opcode(lshiftopc, elem_bt);
131 int rshiftvopc = VectorNode::opcode(rshiftopc, elem_bt);
132 if (!is_supported &&
133 arch_supports_vector(lshiftvopc, num_elem, elem_bt, VecMaskNotUsed, has_scalar_args) &&
134 arch_supports_vector(rshiftvopc, num_elem, elem_bt, VecMaskNotUsed, has_scalar_args) &&
135 arch_supports_vector(Op_OrV, num_elem, elem_bt, VecMaskNotUsed)) {
136 is_supported = true;
137 }
138 return is_supported;
139 }
140
141 Node* GraphKit::box_vector(Node* vector, const TypeInstPtr* vbox_type, BasicType elem_bt, int num_elem, bool deoptimize_on_exception) {
142 assert(EnableVectorSupport, "");
143
144 PreserveReexecuteState preexecs(this);
145 jvms()->set_should_reexecute(true);
146
147 VectorBoxAllocateNode* alloc = new VectorBoxAllocateNode(C, vbox_type);
148 set_edges_for_java_call(alloc, /*must_throw=*/false, /*separate_io_proj=*/true);
149 make_slow_call_ex(alloc, env()->Throwable_klass(), /*separate_io_proj=*/true, deoptimize_on_exception);
150 set_i_o(gvn().transform( new ProjNode(alloc, TypeFunc::I_O) ));
151 set_all_memory(gvn().transform( new ProjNode(alloc, TypeFunc::Memory) ));
152 Node* ret = gvn().transform(new ProjNode(alloc, TypeFunc::Parms));
153
154 assert(check_vbox(vbox_type), "");
155 const TypeVect* vt = TypeVect::make(elem_bt, num_elem, is_vector_mask(vbox_type->instance_klass()));
156 VectorBoxNode* vbox = new VectorBoxNode(C, ret, vector, vbox_type, vt);
157 return gvn().transform(vbox);
158 }
159
160 Node* GraphKit::unbox_vector(Node* v, const TypeInstPtr* vbox_type, BasicType elem_bt, int num_elem) {
161 assert(EnableVectorSupport, "");
162 const TypeInstPtr* vbox_type_v = gvn().type(v)->isa_instptr();
163 if (vbox_type_v == nullptr || vbox_type->instance_klass() != vbox_type_v->instance_klass()) {
164 return nullptr; // arguments don't agree on vector shapes
165 }
166 if (vbox_type_v->maybe_null()) {
167 return nullptr; // no nulls are allowed
168 }
169 assert(check_vbox(vbox_type), "");
170 const TypeVect* vt = TypeVect::make(elem_bt, num_elem, is_vector_mask(vbox_type->instance_klass()));
171 Node* unbox = gvn().transform(new VectorUnboxNode(C, vt, v, merged_memory()));
172 return unbox;
173 }
174
175 Node* GraphKit::vector_shift_count(Node* cnt, int shift_op, BasicType bt, int num_elem) {
176 assert(bt == T_INT || bt == T_LONG || bt == T_SHORT || bt == T_BYTE, "byte, short, long and int are supported");
177 juint mask = (type2aelembytes(bt) * BitsPerByte - 1);
178 Node* nmask = gvn().transform(ConNode::make(TypeInt::make(mask)));
179 Node* mcnt = gvn().transform(new AndINode(cnt, nmask));
180 return gvn().transform(VectorNode::shift_count(shift_op, mcnt, num_elem, bt));
181 }
182
183 bool LibraryCallKit::arch_supports_vector(int sopc, int num_elem, BasicType type, VectorMaskUseType mask_use_type, bool has_scalar_args) {
184 // Check that the operation is valid.
185 if (sopc <= 0) {
186 non_product_log_if_needed(" ** Rejected intrinsification because no valid vector op could be extracted");
187 return false;
188 }
189
190 if (VectorNode::is_vector_rotate(sopc)) {
191 if(!arch_supports_vector_rotate(sopc, num_elem, type, mask_use_type, has_scalar_args)) {
192 non_product_log_if_needed(" ** Rejected vector op (%s,%s,%d) because architecture does not support variable vector shifts",
193 NodeClassNames[sopc], type2name(type), num_elem);
194 return false;
195 }
196 } else if (VectorNode::is_vector_integral_negate(sopc)) {
197 if (!VectorNode::is_vector_integral_negate_supported(sopc, num_elem, type, false)) {
198 non_product_log_if_needed(" ** Rejected vector op (%s,%s,%d) because architecture does not support integral vector negate",
199 NodeClassNames[sopc], type2name(type), num_elem);
200 return false;
201 }
202 } else {
203 // Check that architecture supports this op-size-type combination.
204 if (!Matcher::match_rule_supported_vector(sopc, num_elem, type)) {
205 non_product_log_if_needed(" ** Rejected vector op (%s,%s,%d) because architecture does not support it",
206 NodeClassNames[sopc], type2name(type), num_elem);
207 return false;
208 } else {
209 assert(Matcher::match_rule_supported(sopc), "must be supported");
210 }
211 }
212
213 if (num_elem == 1) {
214 if (mask_use_type != VecMaskNotUsed) {
215 non_product_log_if_needed(" ** Rejected vector mask op (%s,%s,%d) because architecture does not support it",
216 NodeClassNames[sopc], type2name(type), num_elem);
217 return false;
218 }
219
220 if (sopc != 0) {
221 if (sopc != Op_LoadVector && sopc != Op_StoreVector) {
222 non_product_log_if_needed(" ** Not a svml call or load/store vector op (%s,%s,%d)",
223 NodeClassNames[sopc], type2name(type), num_elem);
224 return false;
225 }
226 }
227 }
228
229 if (!has_scalar_args && VectorNode::is_vector_shift(sopc) &&
230 Matcher::supports_vector_variable_shifts() == false) {
231 log_if_needed(" ** Rejected vector op (%s,%s,%d) because architecture does not support variable vector shifts",
232 NodeClassNames[sopc], type2name(type), num_elem);
233 return false;
234 }
235
236 // Check if mask unboxing is supported, this is a two step process which first loads the contents
237 // of boolean array into vector followed by either lane expansion to match the lane size of masked
238 // vector operation or populate the predicate register.
239 if ((mask_use_type & VecMaskUseLoad) != 0) {
240 if (!Matcher::match_rule_supported_vector(Op_VectorLoadMask, num_elem, type) ||
241 !Matcher::match_rule_supported_vector(Op_LoadVector, num_elem, T_BOOLEAN)) {
242 non_product_log_if_needed(" ** Rejected vector mask loading (%s,%s,%d) because architecture does not support it",
243 NodeClassNames[Op_VectorLoadMask], type2name(type), num_elem);
244 return false;
245 }
246 }
247
248 // Check if mask boxing is supported, this is a two step process which first stores the contents
249 // of mask vector / predicate register into a boolean vector followed by vector store operation to
250 // transfer the contents to underlined storage of mask boxes which is a boolean array.
251 if ((mask_use_type & VecMaskUseStore) != 0) {
252 if (!Matcher::match_rule_supported_vector(Op_VectorStoreMask, num_elem, type) ||
253 !Matcher::match_rule_supported_vector(Op_StoreVector, num_elem, T_BOOLEAN)) {
254 non_product_log_if_needed("Rejected vector mask storing (%s,%s,%d) because architecture does not support it",
255 NodeClassNames[Op_VectorStoreMask], type2name(type), num_elem);
256 return false;
257 }
258 }
259
260 if ((mask_use_type & VecMaskUsePred) != 0) {
261 bool is_supported = false;
262 if (Matcher::has_predicated_vectors()) {
263 if (VectorNode::is_vector_integral_negate(sopc)) {
264 is_supported = VectorNode::is_vector_integral_negate_supported(sopc, num_elem, type, true);
265 } else {
266 is_supported = Matcher::match_rule_supported_vector_masked(sopc, num_elem, type);
267 }
268 }
269 is_supported |= Matcher::supports_vector_predicate_op_emulation(sopc, num_elem, type);
270
271 if (!is_supported) {
272 non_product_log_if_needed("Rejected vector mask predicate using (%s,%s,%d) because architecture does not support it",
273 NodeClassNames[sopc], type2name(type), num_elem);
274 return false;
275 }
276 }
277
278 return true;
279 }
280
281 static bool is_klass_initialized(const TypeInstPtr* vec_klass) {
282 if (vec_klass->const_oop() == nullptr) {
283 return false; // uninitialized or some kind of unsafe access
284 }
285 assert(vec_klass->const_oop()->as_instance()->java_lang_Class_klass() != nullptr, "klass instance expected");
286 ciInstanceKlass* klass = vec_klass->const_oop()->as_instance()->java_lang_Class_klass()->as_instance_klass();
287 return klass->is_initialized();
288 }
289
290 // public static
291 // <V extends Vector<E>,
292 // M extends VectorMask<E>,
293 // E>
294 // V unaryOp(int oprId, Class<? extends V> vmClass, Class<? extends M> maskClass, Class<E> elementType,
295 // int length, V v, M m,
296 // UnaryOperation<V, M> defaultImpl)
297 //
298 // public static
299 // <V,
300 // M extends VectorMask<E>,
301 // E>
302 // V binaryOp(int oprId, Class<? extends V> vmClass, Class<? extends M> maskClass, Class<E> elementType,
303 // int length, V v1, V v2, M m,
304 // BinaryOperation<V, M> defaultImpl)
305 //
306 // public static
307 // <V extends Vector<E>,
308 // M extends VectorMask<E>,
309 // E>
310 // V ternaryOp(int oprId, Class<? extends V> vmClass, Class<? extends M> maskClass, Class<E> elementType,
311 // int length, V v1, V v2, V v3, M m,
312 // TernaryOperation<V, M> defaultImpl)
313 //
314 bool LibraryCallKit::inline_vector_nary_operation(int n) {
315 const TypeInt* opr = gvn().type(argument(0))->isa_int();
316 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
317 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
318 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
319 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
320
321 if (opr == nullptr || vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr ||
322 !opr->is_con() || vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
323 log_if_needed(" ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
324 NodeClassNames[argument(0)->Opcode()],
325 NodeClassNames[argument(1)->Opcode()],
326 NodeClassNames[argument(3)->Opcode()],
327 NodeClassNames[argument(4)->Opcode()]);
328 return false; // not enough info for intrinsification
329 }
330
331 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
332 if (!elem_type->is_primitive_type()) {
333 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
334 return false; // should be primitive type
335 }
336 if (!is_klass_initialized(vector_klass)) {
337 log_if_needed(" ** klass argument not initialized");
338 return false;
339 }
340
341 // "argument(n + 5)" should be the mask object. We assume it is "null" when no mask
342 // is used to control this operation.
343 const Type* vmask_type = gvn().type(argument(n + 5));
344 bool is_masked_op = vmask_type != TypePtr::NULL_PTR;
345 if (is_masked_op) {
346 if (mask_klass == nullptr || mask_klass->const_oop() == nullptr) {
347 log_if_needed(" ** missing constant: maskclass=%s", NodeClassNames[argument(2)->Opcode()]);
348 return false; // not enough info for intrinsification
349 }
350
351 if (!is_klass_initialized(mask_klass)) {
352 log_if_needed(" ** mask klass argument not initialized");
353 return false;
354 }
355
356 if (vmask_type->maybe_null()) {
357 log_if_needed(" ** null mask values are not allowed for masked op");
358 return false;
359 }
360 }
361
362 BasicType elem_bt = elem_type->basic_type();
363 bool has_scalar_op = VectorSupport::has_scalar_op(opr->get_con());
364 bool is_unsigned = VectorSupport::is_unsigned_op(opr->get_con());
365
366 int num_elem = vlen->get_con();
367 int opc = VectorSupport::vop2ideal(opr->get_con(), elem_bt);
368 int sopc = has_scalar_op ? VectorNode::opcode(opc, elem_bt) : opc;
369 if (sopc == 0 || num_elem == 1) {
370 log_if_needed(" ** operation not supported: arity=%d opc=%s[%d] vlen=%d etype=%s",
371 n, NodeClassNames[opc], opc, num_elem, type2name(elem_bt));
372 return false; // operation not supported
373 }
374 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
375 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
376
377 if (is_vector_mask(vbox_klass)) {
378 assert(!is_masked_op, "mask operations do not need mask to control");
379 }
380
381 // When using mask, mask use type needs to be VecMaskUseLoad.
382 VectorMaskUseType mask_use_type = is_vector_mask(vbox_klass) ? VecMaskUseAll
383 : is_masked_op ? VecMaskUseLoad : VecMaskNotUsed;
384 if ((sopc != 0) && !arch_supports_vector(sopc, num_elem, elem_bt, mask_use_type)) {
385 log_if_needed(" ** not supported: arity=%d opc=%d vlen=%d etype=%s ismask=%d is_masked_op=%d",
386 n, sopc, num_elem, type2name(elem_bt),
387 is_vector_mask(vbox_klass) ? 1 : 0, is_masked_op ? 1 : 0);
388 return false; // not supported
389 }
390
391 // Return true if current platform has implemented the masked operation with predicate feature.
392 bool use_predicate = is_masked_op && sopc != 0 && arch_supports_vector(sopc, num_elem, elem_bt, VecMaskUsePred);
393 if (is_masked_op && !use_predicate && !arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad)) {
394 log_if_needed(" ** not supported: arity=%d opc=%d vlen=%d etype=%s ismask=0 is_masked_op=1",
395 n, sopc, num_elem, type2name(elem_bt));
396 return false;
397 }
398
399 Node* opd1 = nullptr; Node* opd2 = nullptr; Node* opd3 = nullptr;
400 switch (n) {
401 case 3: {
402 opd3 = unbox_vector(argument(7), vbox_type, elem_bt, num_elem);
403 if (opd3 == nullptr) {
404 log_if_needed(" ** unbox failed v3=%s",
405 NodeClassNames[argument(7)->Opcode()]);
406 return false;
407 }
408 // fall-through
409 }
410 case 2: {
411 opd2 = unbox_vector(argument(6), vbox_type, elem_bt, num_elem);
412 if (opd2 == nullptr) {
413 log_if_needed(" ** unbox failed v2=%s",
414 NodeClassNames[argument(6)->Opcode()]);
415 return false;
416 }
417 // fall-through
418 }
419 case 1: {
420 opd1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
421 if (opd1 == nullptr) {
422 log_if_needed(" ** unbox failed v1=%s",
423 NodeClassNames[argument(5)->Opcode()]);
424 return false;
425 }
426 break;
427 }
428 default: fatal("unsupported arity: %d", n);
429 }
430
431 Node* mask = nullptr;
432 if (is_masked_op) {
433 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
434 assert(is_vector_mask(mbox_klass), "argument(2) should be a mask class");
435 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
436 mask = unbox_vector(argument(n + 5), mbox_type, elem_bt, num_elem);
437 if (mask == nullptr) {
438 log_if_needed(" ** unbox failed mask=%s",
439 NodeClassNames[argument(n + 5)->Opcode()]);
440 return false;
441 }
442 }
443
444 Node* operation = nullptr;
445 const TypeVect* vt = TypeVect::make(elem_bt, num_elem, is_vector_mask(vbox_klass));
446 switch (n) {
447 case 1:
448 case 2: {
449 operation = VectorNode::make(sopc, opd1, opd2, vt, is_vector_mask(vbox_klass), VectorNode::is_shift_opcode(opc), is_unsigned);
450 break;
451 }
452 case 3: {
453 operation = VectorNode::make(sopc, opd1, opd2, opd3, vt);
454 break;
455 }
456 default: fatal("unsupported arity: %d", n);
457 }
458
459 if (is_masked_op && mask != nullptr) {
460 if (use_predicate) {
461 operation->add_req(mask);
462 operation->add_flag(Node::Flag_is_predicated_vector);
463 } else {
464 operation->add_flag(Node::Flag_is_predicated_using_blend);
465 operation = gvn().transform(operation);
466 operation = new VectorBlendNode(opd1, operation, mask);
467 }
468 }
469 operation = gvn().transform(operation);
470
471 // Wrap it up in VectorBox to keep object type information.
472 Node* vbox = box_vector(operation, vbox_type, elem_bt, num_elem);
473 set_result(vbox);
474 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
475 return true;
476 }
477
478 // public static
479 // <V extends Vector<E>, E>
480 // V libraryUnaryOp(long address, Class<? extends V> vClass, Class<E> elementType, int length, String debugName,
481 // V v,
482 // UnaryOperation<V, ?> defaultImpl)
483 //
484 // public static
485 // <V extends VectorPayload, E>
486 // V libraryBinaryOp(long address, Class<? extends V> vClass, Class<E> elementType, int length, String debugName,
487 // V v1, V v2,
488 // BinaryOperation<V, ?> defaultImpl)
489 bool LibraryCallKit::inline_vector_call(int arity) {
490 assert(Matcher::supports_vector_calling_convention(), "required");
491
492 const TypeLong* entry = gvn().type(argument(0))->isa_long();
493 const TypeInstPtr* vector_klass = gvn().type(argument(2))->isa_instptr();
494 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
495 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
496 const TypeInstPtr* debug_name_oop = gvn().type(argument(5))->isa_instptr();
497
498 if (entry == nullptr || !entry->is_con() ||
499 vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
500 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
501 vlen == nullptr || !vlen->is_con() ||
502 debug_name_oop == nullptr || debug_name_oop->const_oop() == nullptr) {
503 log_if_needed(" ** missing constant: opr=%s vclass=%s etype=%s vlen=%s debug_name=%s",
504 NodeClassNames[argument(0)->Opcode()],
505 NodeClassNames[argument(2)->Opcode()],
506 NodeClassNames[argument(3)->Opcode()],
507 NodeClassNames[argument(4)->Opcode()],
508 NodeClassNames[argument(5)->Opcode()]);
509 return false; // not enough info for intrinsification
510 }
511
512 if (entry->get_con() == 0) {
513 log_if_needed(" ** missing entry point");
514 return false;
515 }
516
517 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
518 if (!elem_type->is_primitive_type()) {
519 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
520 return false; // should be primitive type
521 }
522 if (!is_klass_initialized(vector_klass)) {
523 log_if_needed(" ** klass argument not initialized");
524 return false;
525 }
526
527 BasicType elem_bt = elem_type->basic_type();
528 int num_elem = vlen->get_con();
529 if (!Matcher::vector_size_supported(elem_bt, num_elem)) {
530 log_if_needed(" ** vector size (vlen=%d, etype=%s) is not supported",
531 num_elem, type2name(elem_bt));
532 return false;
533 }
534
535 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
536 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
537
538 Node* opd1 = unbox_vector(argument(6), vbox_type, elem_bt, num_elem);
539 if (opd1 == nullptr) {
540 log_if_needed(" ** unbox failed v1=%s", NodeClassNames[argument(6)->Opcode()]);
541 return false;
542 }
543
544 Node* opd2 = nullptr;
545 if (arity > 1) {
546 opd2 = unbox_vector(argument(7), vbox_type, elem_bt, num_elem);
547 if (opd2 == nullptr) {
548 log_if_needed(" ** unbox failed v2=%s", NodeClassNames[argument(7)->Opcode()]);
549 return false;
550 }
551 }
552 assert(arity == 1 || arity == 2, "arity %d not supported", arity);
553 const TypeVect* vt = TypeVect::make(elem_bt, num_elem);
554 const TypeFunc* call_type = OptoRuntime::Math_Vector_Vector_Type(arity, vt, vt);
555 address entry_addr = (address)entry->get_con();
556
557 const char* debug_name = "<unknown>";
558 if (!debug_name_oop->const_oop()->is_null_object()) {
559 size_t buflen = 100;
560 char* buf = NEW_ARENA_ARRAY(C->comp_arena(), char, buflen);
561 debug_name = debug_name_oop->const_oop()->as_instance()->java_lang_String_str(buf, buflen);
562 }
563 Node* vcall = make_runtime_call(RC_VECTOR,
564 call_type,
565 entry_addr,
566 debug_name,
567 TypePtr::BOTTOM,
568 opd1,
569 opd2);
570
571 vcall = gvn().transform(new ProjNode(gvn().transform(vcall), TypeFunc::Parms));
572
573 // Wrap it up in VectorBox to keep object type information.
574 Node* vbox = box_vector(vcall, vbox_type, elem_bt, num_elem);
575 set_result(vbox);
576 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
577 return true;
578 }
579
580 // <E, M>
581 // long maskReductionCoerced(int oper, Class<? extends M> maskClass, Class<?> elemClass,
582 // int length, M m, VectorMaskOp<M> defaultImpl)
583 bool LibraryCallKit::inline_vector_mask_operation() {
584 const TypeInt* oper = gvn().type(argument(0))->isa_int();
585 const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
586 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
587 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
588 Node* mask = argument(4);
589
590 if (mask_klass == nullptr || elem_klass == nullptr || mask->is_top() || vlen == nullptr) {
591 return false; // dead code
592 }
593
594 if (!is_klass_initialized(mask_klass)) {
595 log_if_needed(" ** klass argument not initialized");
596 return false;
597 }
598
599 int num_elem = vlen->get_con();
600 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
601 BasicType elem_bt = elem_type->basic_type();
602
603 int mopc = VectorSupport::vop2ideal(oper->get_con(), elem_bt);
604 if (!arch_supports_vector(mopc, num_elem, elem_bt, VecMaskUseLoad)) {
605 log_if_needed(" ** not supported: arity=1 op=cast#%d/3 vlen2=%d etype2=%s",
606 mopc, num_elem, type2name(elem_bt));
607 return false; // not supported
608 }
609
610 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
611 const TypeInstPtr* mask_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
612 Node* mask_vec = unbox_vector(mask, mask_box_type, elem_bt, num_elem);
613 if (mask_vec == nullptr) {
614 log_if_needed(" ** unbox failed mask=%s",
615 NodeClassNames[argument(4)->Opcode()]);
616 return false;
617 }
618
619 if (mask_vec->bottom_type()->isa_vectmask() == nullptr) {
620 mask_vec = gvn().transform(VectorStoreMaskNode::make(gvn(), mask_vec, elem_bt, num_elem));
621 }
622 const Type* maskoper_ty = mopc == Op_VectorMaskToLong ? (const Type*)TypeLong::LONG : (const Type*)TypeInt::INT;
623 Node* maskoper = gvn().transform(VectorMaskOpNode::make(mask_vec, maskoper_ty, mopc));
624 if (mopc != Op_VectorMaskToLong) {
625 maskoper = ConvI2L(maskoper);
626 }
627 set_result(maskoper);
628
629 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
630 return true;
631 }
632
633 // public static
634 // <M,
635 // S extends VectorSpecies<E>,
636 // E>
637 // M fromBitsCoerced(Class<? extends M> vmClass, Class<E> elementType, int length,
638 // long bits, int mode, S s,
639 // BroadcastOperation<M, E, S> defaultImpl)
640 bool LibraryCallKit::inline_vector_frombits_coerced() {
641 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
642 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
643 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
644 const TypeLong* bits_type = gvn().type(argument(3))->isa_long();
645 // Mode argument determines the mode of operation it can take following values:-
646 // MODE_BROADCAST for vector Vector.broadcast and VectorMask.maskAll operations.
647 // MODE_BITS_COERCED_LONG_TO_MASK for VectorMask.fromLong operation.
648 const TypeInt* mode = gvn().type(argument(5))->isa_int();
649
650 if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr || mode == nullptr ||
651 bits_type == nullptr || vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr ||
652 !vlen->is_con() || !mode->is_con()) {
653 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s bitwise=%s",
654 NodeClassNames[argument(0)->Opcode()],
655 NodeClassNames[argument(1)->Opcode()],
656 NodeClassNames[argument(2)->Opcode()],
657 NodeClassNames[argument(5)->Opcode()]);
658 return false; // not enough info for intrinsification
659 }
660
661 if (!is_klass_initialized(vector_klass)) {
662 log_if_needed(" ** klass argument not initialized");
663 return false;
664 }
665 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
666 if (!elem_type->is_primitive_type()) {
667 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
668 return false; // should be primitive type
669 }
670 BasicType elem_bt = elem_type->basic_type();
671 int num_elem = vlen->get_con();
672 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
673 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
674
675 bool is_mask = is_vector_mask(vbox_klass);
676 int bcast_mode = mode->get_con();
677 VectorMaskUseType checkFlags = (VectorMaskUseType)(is_mask ? VecMaskUseAll : VecMaskNotUsed);
678 int opc = bcast_mode == VectorSupport::MODE_BITS_COERCED_LONG_TO_MASK ? Op_VectorLongToMask : Op_Replicate;
679
680 if (!arch_supports_vector(opc, num_elem, elem_bt, checkFlags, true /*has_scalar_args*/)) {
681 log_if_needed(" ** not supported: arity=0 op=broadcast vlen=%d etype=%s ismask=%d bcast_mode=%d",
682 num_elem, type2name(elem_bt),
683 is_mask ? 1 : 0,
684 bcast_mode);
685 return false; // not supported
686 }
687
688 Node* broadcast = nullptr;
689 Node* bits = argument(3);
690 Node* elem = bits;
691
692 if (opc == Op_VectorLongToMask) {
693 const TypeVect* vt = TypeVect::makemask(elem_bt, num_elem);
694 if (vt->isa_vectmask()) {
695 broadcast = gvn().transform(new VectorLongToMaskNode(elem, vt));
696 } else {
697 const TypeVect* mvt = TypeVect::make(T_BOOLEAN, num_elem);
698 broadcast = gvn().transform(new VectorLongToMaskNode(elem, mvt));
699 broadcast = gvn().transform(new VectorLoadMaskNode(broadcast, vt));
700 }
701 } else {
702 switch (elem_bt) {
703 case T_BOOLEAN: // fall-through
704 case T_BYTE: // fall-through
705 case T_SHORT: // fall-through
706 case T_CHAR: // fall-through
707 case T_INT: {
708 elem = gvn().transform(new ConvL2INode(bits));
709 break;
710 }
711 case T_DOUBLE: {
712 elem = gvn().transform(new MoveL2DNode(bits));
713 break;
714 }
715 case T_FLOAT: {
716 bits = gvn().transform(new ConvL2INode(bits));
717 elem = gvn().transform(new MoveI2FNode(bits));
718 break;
719 }
720 case T_LONG: {
721 // no conversion needed
722 break;
723 }
724 default: fatal("%s", type2name(elem_bt));
725 }
726 broadcast = VectorNode::scalar2vector(elem, num_elem, elem_bt, is_mask);
727 broadcast = gvn().transform(broadcast);
728 }
729
730 Node* box = box_vector(broadcast, vbox_type, elem_bt, num_elem);
731 set_result(box);
732 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
733 return true;
734 }
735
736 static bool elem_consistent_with_arr(BasicType elem_bt, const TypeAryPtr* arr_type, bool mismatched_ms) {
737 assert(arr_type != nullptr, "unexpected");
738 BasicType arr_elem_bt = arr_type->elem()->array_element_basic_type();
739 if (elem_bt == arr_elem_bt) {
740 return true;
741 } else if (elem_bt == T_SHORT && arr_elem_bt == T_CHAR) {
742 // Load/store of short vector from/to char[] is supported
743 return true;
744 } else if (elem_bt == T_BYTE && arr_elem_bt == T_BOOLEAN) {
745 // Load/store of byte vector from/to boolean[] is supported
746 return true;
747 } else {
748 return mismatched_ms;
749 }
750 }
751
752 // public static
753 // <C,
754 // VM extends VectorPayload,
755 // E,
756 // S extends VectorSpecies<E>>
757 // VM load(Class<? extends VM> vmClass, Class<E> eClass,
758 // int length,
759 // Object base, long offset, // Unsafe addressing
760 // boolean fromSegment,
761 // C container, long index, S s, // Arguments for default implementation
762 // LoadOperation<C, VM, S> defaultImpl) {
763 // public static
764 // <C,
765 // V extends VectorPayload>
766 // void store(Class<?> vClass, Class<?> eClass,
767 // int length,
768 // Object base, long offset, // Unsafe addressing
769 // boolean fromSegment,
770 // V v, C container, long index, // Arguments for default implementation
771 // StoreVectorOperation<C, V> defaultImpl) {
772 bool LibraryCallKit::inline_vector_mem_operation(bool is_store) {
773 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
774 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
775 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
776 const TypeInt* from_ms = gvn().type(argument(6))->isa_int();
777
778 if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr || !from_ms->is_con() ||
779 vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
780 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s from_ms=%s",
781 NodeClassNames[argument(0)->Opcode()],
782 NodeClassNames[argument(1)->Opcode()],
783 NodeClassNames[argument(2)->Opcode()],
784 NodeClassNames[argument(6)->Opcode()]);
785 return false; // not enough info for intrinsification
786 }
787 if (!is_klass_initialized(vector_klass)) {
788 log_if_needed(" ** klass argument not initialized");
789 return false;
790 }
791
792 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
793 if (!elem_type->is_primitive_type()) {
794 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
795 return false; // should be primitive type
796 }
797 BasicType elem_bt = elem_type->basic_type();
798 int num_elem = vlen->get_con();
799
800 // TODO When mask usage is supported, VecMaskNotUsed needs to be VecMaskUseLoad.
801 if (!arch_supports_vector(is_store ? Op_StoreVector : Op_LoadVector, num_elem, elem_bt, VecMaskNotUsed)) {
802 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d etype=%s ismask=no",
803 is_store, is_store ? "store" : "load",
804 num_elem, type2name(elem_bt));
805 return false; // not supported
806 }
807
808 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
809 bool is_mask = is_vector_mask(vbox_klass);
810
811 Node* base = argument(3);
812 Node* offset = ConvL2X(argument(4));
813
814 // Save state and restore on bailout
815 uint old_sp = sp();
816 SafePointNode* old_map = clone_map();
817
818 Node* addr = make_unsafe_address(base, offset, (is_mask ? T_BOOLEAN : elem_bt), true);
819
820 // The memory barrier checks are based on ones for unsafe access.
821 // This is not 1-1 implementation.
822 const Type *const base_type = gvn().type(base);
823
824 const TypePtr *addr_type = gvn().type(addr)->isa_ptr();
825 const TypeAryPtr* arr_type = addr_type->isa_aryptr();
826
827 const bool in_native = TypePtr::NULL_PTR == base_type; // base always null
828 const bool in_heap = !TypePtr::NULL_PTR->higher_equal(base_type); // base never null
829
830 const bool is_mixed_access = !in_heap && !in_native;
831
832 const bool is_mismatched_access = in_heap && (addr_type->isa_aryptr() == nullptr);
833
834 const bool needs_cpu_membar = is_mixed_access || is_mismatched_access;
835
836 // For non-masked mismatched memory segment vector read/write accesses, intrinsification can continue
837 // with unknown backing storage type and compiler can skip inserting explicit reinterpretation IR after
838 // loading from or before storing to backing storage which is mandatory for semantic correctness of
839 // big-endian memory layout.
840 bool mismatched_ms = LITTLE_ENDIAN_ONLY(false)
841 BIG_ENDIAN_ONLY(from_ms->get_con() && !is_mask && arr_type != nullptr &&
842 arr_type->elem()->array_element_basic_type() != elem_bt);
843 BasicType mem_elem_bt = mismatched_ms ? arr_type->elem()->array_element_basic_type() : elem_bt;
844 if (!is_java_primitive(mem_elem_bt)) {
845 log_if_needed(" ** non-primitive array element type");
846 return false;
847 }
848 int mem_num_elem = mismatched_ms ? (num_elem * type2aelembytes(elem_bt)) / type2aelembytes(mem_elem_bt) : num_elem;
849 if (arr_type != nullptr && !is_mask && !elem_consistent_with_arr(elem_bt, arr_type, mismatched_ms)) {
850 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d etype=%s atype=%s ismask=no",
851 is_store, is_store ? "store" : "load",
852 num_elem, type2name(elem_bt), type2name(arr_type->elem()->array_element_basic_type()));
853 set_map(old_map);
854 set_sp(old_sp);
855 return false;
856 }
857
858 // In case of mismatched memory segment accesses, we need to double check that the source type memory operations are supported by backend.
859 if (mismatched_ms) {
860 if (is_store) {
861 if (!arch_supports_vector(Op_StoreVector, num_elem, elem_bt, VecMaskNotUsed)
862 || !arch_supports_vector(Op_VectorReinterpret, mem_num_elem, mem_elem_bt, VecMaskNotUsed)) {
863 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d*8 etype=%s/8 ismask=no",
864 is_store, "store",
865 num_elem, type2name(elem_bt));
866 set_map(old_map);
867 set_sp(old_sp);
868 return false; // not supported
869 }
870 } else {
871 if (!arch_supports_vector(Op_LoadVector, mem_num_elem, mem_elem_bt, VecMaskNotUsed)
872 || !arch_supports_vector(Op_VectorReinterpret, num_elem, elem_bt, VecMaskNotUsed)) {
873 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d*8 etype=%s/8 ismask=no",
874 is_store, "load",
875 mem_num_elem, type2name(mem_elem_bt));
876 set_map(old_map);
877 set_sp(old_sp);
878 return false; // not supported
879 }
880 }
881 }
882 if (is_mask) {
883 if (!is_store) {
884 if (!arch_supports_vector(Op_LoadVector, num_elem, elem_bt, VecMaskUseLoad)) {
885 set_map(old_map);
886 set_sp(old_sp);
887 return false; // not supported
888 }
889 } else {
890 if (!arch_supports_vector(Op_StoreVector, num_elem, elem_bt, VecMaskUseStore)) {
891 set_map(old_map);
892 set_sp(old_sp);
893 return false; // not supported
894 }
895 }
896 }
897
898 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
899
900 if (needs_cpu_membar) {
901 insert_mem_bar(Op_MemBarCPUOrder);
902 }
903
904 if (is_store) {
905 Node* val = unbox_vector(argument(7), vbox_type, elem_bt, num_elem);
906 if (val == nullptr) {
907 set_map(old_map);
908 set_sp(old_sp);
909 return false; // operand unboxing failed
910 }
911 set_all_memory(reset_memory());
912
913 // In case the store needs to happen to byte array, reinterpret the incoming vector to byte vector.
914 int store_num_elem = num_elem;
915 if (mismatched_ms) {
916 store_num_elem = mem_num_elem;
917 const TypeVect* to_vect_type = TypeVect::make(mem_elem_bt, store_num_elem);
918 val = gvn().transform(new VectorReinterpretNode(val, val->bottom_type()->is_vect(), to_vect_type));
919 }
920 if (is_mask) {
921 val = gvn().transform(VectorStoreMaskNode::make(gvn(), val, elem_bt, num_elem));
922 }
923 Node* vstore = gvn().transform(StoreVectorNode::make(0, control(), memory(addr), addr, addr_type, val, store_num_elem));
924 set_memory(vstore, addr_type);
925 } else {
926 // When using byte array, we need to load as byte then reinterpret the value. Otherwise, do a simple vector load.
927 Node* vload = nullptr;
928 if (mismatched_ms) {
929 vload = gvn().transform(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, mem_num_elem, mem_elem_bt));
930 const TypeVect* to_vect_type = TypeVect::make(elem_bt, num_elem);
931 vload = gvn().transform(new VectorReinterpretNode(vload, vload->bottom_type()->is_vect(), to_vect_type));
932 } else {
933 // Special handle for masks
934 if (is_mask) {
935 vload = gvn().transform(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, num_elem, T_BOOLEAN));
936 vload = gvn().transform(new VectorLoadMaskNode(vload, TypeVect::makemask(elem_bt, num_elem)));
937 } else {
938 vload = gvn().transform(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, num_elem, elem_bt));
939 }
940 }
941 Node* box = box_vector(vload, vbox_type, elem_bt, num_elem);
942 set_result(box);
943 }
944
945 destruct_map_clone(old_map);
946
947 if (needs_cpu_membar) {
948 insert_mem_bar(Op_MemBarCPUOrder);
949 }
950
951 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
952 return true;
953 }
954
955 // public static
956 // <C,
957 // V extends Vector<?>,
958 // E,
959 // S extends VectorSpecies<E>,
960 // M extends VectorMask<E>>
961 // V loadMasked(Class<? extends V> vClass, Class<M> mClass, Class<E> eClass,
962 // int length, Object base, long offset, // Unsafe addressing
963 // boolean fromSegment,
964 // M m, int offsetInRange,
965 // C container, long index, S s, // Arguments for default implementation
966 // LoadVectorMaskedOperation<C, V, S, M> defaultImpl) {
967 // public static
968 // <C,
969 // V extends Vector<E>,
970 // M extends VectorMask<E>,
971 // E>
972 // void storeMasked(Class<? extends V> vClass, Class<M> mClass, Class<E> eClass,
973 // int length,
974 // Object base, long offset, // Unsafe addressing
975 // boolean fromSegment,
976 // V v, M m, C container, long index, // Arguments for default implementation
977 // StoreVectorMaskedOperation<C, V, M> defaultImpl) {
978
979 bool LibraryCallKit::inline_vector_mem_masked_operation(bool is_store) {
980 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
981 const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
982 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
983 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
984 const TypeInt* from_ms = gvn().type(argument(7))->isa_int();
985
986 if (vector_klass == nullptr || mask_klass == nullptr || elem_klass == nullptr || vlen == nullptr ||
987 vector_klass->const_oop() == nullptr || mask_klass->const_oop() == nullptr || from_ms == nullptr ||
988 elem_klass->const_oop() == nullptr || !vlen->is_con() || !from_ms->is_con()) {
989 log_if_needed(" ** missing constant: vclass=%s mclass=%s etype=%s vlen=%s from_ms=%s",
990 NodeClassNames[argument(0)->Opcode()],
991 NodeClassNames[argument(1)->Opcode()],
992 NodeClassNames[argument(2)->Opcode()],
993 NodeClassNames[argument(3)->Opcode()],
994 NodeClassNames[argument(7)->Opcode()]);
995 return false; // not enough info for intrinsification
996 }
997 if (!is_klass_initialized(vector_klass)) {
998 log_if_needed(" ** klass argument not initialized");
999 return false;
1000 }
1001
1002 if (!is_klass_initialized(mask_klass)) {
1003 log_if_needed(" ** mask klass argument not initialized");
1004 return false;
1005 }
1006
1007 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1008 if (!elem_type->is_primitive_type()) {
1009 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1010 return false; // should be primitive type
1011 }
1012
1013 BasicType elem_bt = elem_type->basic_type();
1014 int num_elem = vlen->get_con();
1015
1016 Node* base = argument(4);
1017 Node* offset = ConvL2X(argument(5));
1018
1019 // Save state and restore on bailout
1020 uint old_sp = sp();
1021 SafePointNode* old_map = clone_map();
1022
1023 Node* addr = make_unsafe_address(base, offset, elem_bt, true);
1024 const TypePtr *addr_type = gvn().type(addr)->isa_ptr();
1025 const TypeAryPtr* arr_type = addr_type->isa_aryptr();
1026
1027 bool mismatched_ms = from_ms->get_con() && arr_type != nullptr && arr_type->elem()->array_element_basic_type() != elem_bt;
1028 BIG_ENDIAN_ONLY(if (mismatched_ms) return false;)
1029 // If there is no consistency between array and vector element types, it must be special byte array case
1030 if (arr_type != nullptr && !elem_consistent_with_arr(elem_bt, arr_type, mismatched_ms)) {
1031 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d etype=%s atype=%s",
1032 is_store, is_store ? "storeMasked" : "loadMasked",
1033 num_elem, type2name(elem_bt), type2name(arr_type->elem()->array_element_basic_type()));
1034 set_map(old_map);
1035 set_sp(old_sp);
1036 return false;
1037 }
1038
1039 int mem_num_elem = mismatched_ms ? num_elem * type2aelembytes(elem_bt) : num_elem;
1040 BasicType mem_elem_bt = mismatched_ms ? T_BYTE : elem_bt;
1041 bool supports_predicate = arch_supports_vector(is_store ? Op_StoreVectorMasked : Op_LoadVectorMasked,
1042 mem_num_elem, mem_elem_bt, VecMaskUseLoad);
1043
1044 // If current arch does not support the predicated operations, we have to bail
1045 // out when current case uses the predicate feature.
1046 if (!supports_predicate) {
1047 bool needs_predicate = false;
1048 if (is_store) {
1049 // Masked vector store always uses the predicated store.
1050 needs_predicate = true;
1051 } else {
1052 // Masked vector load with IOOBE always uses the predicated load.
1053 const TypeInt* offset_in_range = gvn().type(argument(9))->isa_int();
1054 if (!offset_in_range->is_con()) {
1055 log_if_needed(" ** missing constant: offsetInRange=%s",
1056 NodeClassNames[argument(8)->Opcode()]);
1057 set_map(old_map);
1058 set_sp(old_sp);
1059 return false;
1060 }
1061 needs_predicate = (offset_in_range->get_con() == 0);
1062 }
1063
1064 if (needs_predicate) {
1065 log_if_needed(" ** not supported: op=%s vlen=%d etype=%s mismatched_ms=%d",
1066 is_store ? "storeMasked" : "loadMasked",
1067 num_elem, type2name(elem_bt), mismatched_ms ? 1 : 0);
1068 set_map(old_map);
1069 set_sp(old_sp);
1070 return false;
1071 }
1072 }
1073
1074 // This only happens for masked vector load. If predicate is not supported, then check whether
1075 // the normal vector load and blend operations are supported by backend.
1076 if (!supports_predicate && (!arch_supports_vector(Op_LoadVector, mem_num_elem, mem_elem_bt, VecMaskNotUsed) ||
1077 !arch_supports_vector(Op_VectorBlend, mem_num_elem, mem_elem_bt, VecMaskUseLoad))) {
1078 log_if_needed(" ** not supported: op=loadMasked vlen=%d etype=%s mismatched_ms=%d",
1079 num_elem, type2name(elem_bt), mismatched_ms ? 1 : 0);
1080 set_map(old_map);
1081 set_sp(old_sp);
1082 return false;
1083 }
1084
1085 // Since we are using byte array, we need to double check that the vector reinterpret operation
1086 // with byte type is supported by backend.
1087 if (mismatched_ms) {
1088 if (!arch_supports_vector(Op_VectorReinterpret, mem_num_elem, T_BYTE, VecMaskNotUsed)) {
1089 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d etype=%s mismatched_ms=1",
1090 is_store, is_store ? "storeMasked" : "loadMasked",
1091 num_elem, type2name(elem_bt));
1092 set_map(old_map);
1093 set_sp(old_sp);
1094 return false;
1095 }
1096 }
1097
1098 // Since it needs to unbox the mask, we need to double check that the related load operations
1099 // for mask are supported by backend.
1100 if (!arch_supports_vector(Op_LoadVector, num_elem, elem_bt, VecMaskUseLoad)) {
1101 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d etype=%s",
1102 is_store, is_store ? "storeMasked" : "loadMasked",
1103 num_elem, type2name(elem_bt));
1104 set_map(old_map);
1105 set_sp(old_sp);
1106 return false;
1107 }
1108
1109 // Can base be null? Otherwise, always on-heap access.
1110 bool can_access_non_heap = TypePtr::NULL_PTR->higher_equal(gvn().type(base));
1111 if (can_access_non_heap) {
1112 insert_mem_bar(Op_MemBarCPUOrder);
1113 }
1114
1115 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1116 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1117 assert(!is_vector_mask(vbox_klass) && is_vector_mask(mbox_klass), "Invalid class type");
1118 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1119 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1120
1121 Node* mask = unbox_vector(is_store ? argument(9) : argument(8), mbox_type, elem_bt, num_elem);
1122 if (mask == nullptr) {
1123 log_if_needed(" ** unbox failed mask=%s",
1124 is_store ? NodeClassNames[argument(9)->Opcode()]
1125 : NodeClassNames[argument(8)->Opcode()]);
1126 set_map(old_map);
1127 set_sp(old_sp);
1128 return false;
1129 }
1130
1131 if (is_store) {
1132 Node* val = unbox_vector(argument(8), vbox_type, elem_bt, num_elem);
1133 if (val == nullptr) {
1134 log_if_needed(" ** unbox failed vector=%s",
1135 NodeClassNames[argument(8)->Opcode()]);
1136 set_map(old_map);
1137 set_sp(old_sp);
1138 return false; // operand unboxing failed
1139 }
1140 set_all_memory(reset_memory());
1141
1142 if (mismatched_ms) {
1143 // Reinterpret the incoming vector to byte vector.
1144 const TypeVect* to_vect_type = TypeVect::make(mem_elem_bt, mem_num_elem);
1145 val = gvn().transform(new VectorReinterpretNode(val, val->bottom_type()->is_vect(), to_vect_type));
1146 // Reinterpret the vector mask to byte type.
1147 const TypeVect* from_mask_type = TypeVect::makemask(elem_bt, num_elem);
1148 const TypeVect* to_mask_type = TypeVect::makemask(mem_elem_bt, mem_num_elem);
1149 mask = gvn().transform(new VectorReinterpretNode(mask, from_mask_type, to_mask_type));
1150 }
1151 Node* vstore = gvn().transform(new StoreVectorMaskedNode(control(), memory(addr), addr, val, addr_type, mask));
1152 set_memory(vstore, addr_type);
1153 } else {
1154 Node* vload = nullptr;
1155
1156 if (mismatched_ms) {
1157 // Reinterpret the vector mask to byte type.
1158 const TypeVect* from_mask_type = TypeVect::makemask(elem_bt, num_elem);
1159 const TypeVect* to_mask_type = TypeVect::makemask(mem_elem_bt, mem_num_elem);
1160 mask = gvn().transform(new VectorReinterpretNode(mask, from_mask_type, to_mask_type));
1161 }
1162
1163 if (supports_predicate) {
1164 // Generate masked load vector node if predicate feature is supported.
1165 const TypeVect* vt = TypeVect::make(mem_elem_bt, mem_num_elem);
1166 vload = gvn().transform(new LoadVectorMaskedNode(control(), memory(addr), addr, addr_type, vt, mask));
1167 } else {
1168 // Use the vector blend to implement the masked load vector. The biased elements are zeros.
1169 Node* zero = gvn().transform(gvn().zerocon(mem_elem_bt));
1170 zero = gvn().transform(VectorNode::scalar2vector(zero, mem_num_elem, mem_elem_bt));
1171 vload = gvn().transform(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, mem_num_elem, mem_elem_bt));
1172 vload = gvn().transform(new VectorBlendNode(zero, vload, mask));
1173 }
1174
1175 if (mismatched_ms) {
1176 const TypeVect* to_vect_type = TypeVect::make(elem_bt, num_elem);
1177 vload = gvn().transform(new VectorReinterpretNode(vload, vload->bottom_type()->is_vect(), to_vect_type));
1178 }
1179
1180 Node* box = box_vector(vload, vbox_type, elem_bt, num_elem);
1181 set_result(box);
1182 }
1183
1184 destruct_map_clone(old_map);
1185
1186 if (can_access_non_heap) {
1187 insert_mem_bar(Op_MemBarCPUOrder);
1188 }
1189
1190 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1191 return true;
1192 }
1193
1194 // <C,
1195 // V extends Vector<?>,
1196 // W extends Vector<Integer>,
1197 // S extends VectorSpecies<E>,
1198 // M extends VectorMask<E>,
1199 // E>
1200 // V loadWithMap(Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType, int length,
1201 // Class<? extends Vector<Integer>> vectorIndexClass,
1202 // Object base, long offset, // Unsafe addressing
1203 // W index_vector, M m,
1204 // C container, int index, int[] indexMap, int indexM, S s, // Arguments for default implementation
1205 // LoadVectorOperationWithMap<C, V, E, S, M> defaultImpl)
1206 //
1207 // <C,
1208 // V extends Vector<E>,
1209 // W extends Vector<Integer>,
1210 // M extends VectorMask<E>,
1211 // E>
1212 // void storeWithMap(Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType,
1213 // int length, Class<? extends Vector<Integer>> vectorIndexClass, Object base, long offset, // Unsafe addressing
1214 // W index_vector, V v, M m,
1215 // C container, int index, int[] indexMap, int indexM, // Arguments for default implementation
1216 // StoreVectorOperationWithMap<C, V, M, E> defaultImpl)
1217 //
1218 bool LibraryCallKit::inline_vector_gather_scatter(bool is_scatter) {
1219 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
1220 const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
1221 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
1222 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
1223 const TypeInstPtr* vector_idx_klass = gvn().type(argument(4))->isa_instptr();
1224
1225 if (vector_klass == nullptr || elem_klass == nullptr || vector_idx_klass == nullptr || vlen == nullptr ||
1226 vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || vector_idx_klass->const_oop() == nullptr || !vlen->is_con()) {
1227 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s viclass=%s",
1228 NodeClassNames[argument(0)->Opcode()],
1229 NodeClassNames[argument(2)->Opcode()],
1230 NodeClassNames[argument(3)->Opcode()],
1231 NodeClassNames[argument(4)->Opcode()]);
1232 return false; // not enough info for intrinsification
1233 }
1234
1235 if (!is_klass_initialized(vector_klass) || !is_klass_initialized(vector_idx_klass)) {
1236 log_if_needed(" ** klass argument not initialized");
1237 return false;
1238 }
1239
1240 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1241 if (!elem_type->is_primitive_type()) {
1242 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1243 return false; // should be primitive type
1244 }
1245
1246 BasicType elem_bt = elem_type->basic_type();
1247 int num_elem = vlen->get_con();
1248
1249 const Type* vmask_type = gvn().type(is_scatter ? argument(10) : argument(9));
1250 bool is_masked_op = vmask_type != TypePtr::NULL_PTR;
1251 if (is_masked_op) {
1252 if (mask_klass == nullptr || mask_klass->const_oop() == nullptr) {
1253 log_if_needed(" ** missing constant: maskclass=%s", NodeClassNames[argument(1)->Opcode()]);
1254 return false; // not enough info for intrinsification
1255 }
1256
1257 if (!is_klass_initialized(mask_klass)) {
1258 log_if_needed(" ** mask klass argument not initialized");
1259 return false;
1260 }
1261
1262 if (vmask_type->maybe_null()) {
1263 log_if_needed(" ** null mask values are not allowed for masked op");
1264 return false;
1265 }
1266
1267 // Check whether the predicated gather/scatter node is supported by architecture.
1268 VectorMaskUseType mask = (VectorMaskUseType) (VecMaskUseLoad | VecMaskUsePred);
1269 if (!arch_supports_vector(is_scatter ? Op_StoreVectorScatterMasked : Op_LoadVectorGatherMasked, num_elem, elem_bt, mask)) {
1270 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d etype=%s is_masked_op=1",
1271 is_scatter, is_scatter ? "scatterMasked" : "gatherMasked",
1272 num_elem, type2name(elem_bt));
1273 return false; // not supported
1274 }
1275 } else {
1276 // Check whether the normal gather/scatter node is supported for non-masked operation.
1277 if (!arch_supports_vector(is_scatter ? Op_StoreVectorScatter : Op_LoadVectorGather, num_elem, elem_bt, VecMaskNotUsed)) {
1278 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d etype=%s is_masked_op=0",
1279 is_scatter, is_scatter ? "scatter" : "gather",
1280 num_elem, type2name(elem_bt));
1281 return false; // not supported
1282 }
1283 }
1284
1285 // Check that the vector holding indices is supported by architecture
1286 // For sub-word gathers expander receive index array.
1287 if (!is_subword_type(elem_bt) && !arch_supports_vector(Op_LoadVector, num_elem, T_INT, VecMaskNotUsed)) {
1288 log_if_needed(" ** not supported: arity=%d op=%s/loadindex vlen=%d etype=int is_masked_op=%d",
1289 is_scatter, is_scatter ? "scatter" : "gather",
1290 num_elem, is_masked_op ? 1 : 0);
1291 return false; // not supported
1292 }
1293
1294 Node* base = argument(5);
1295 Node* offset = ConvL2X(argument(6));
1296
1297 // Save state and restore on bailout
1298 uint old_sp = sp();
1299 SafePointNode* old_map = clone_map();
1300
1301 Node* addr = make_unsafe_address(base, offset, elem_bt, true);
1302
1303 const TypePtr *addr_type = gvn().type(addr)->isa_ptr();
1304 const TypeAryPtr* arr_type = addr_type->isa_aryptr();
1305
1306 // The array must be consistent with vector type
1307 if (arr_type == nullptr || (arr_type != nullptr && !elem_consistent_with_arr(elem_bt, arr_type, false))) {
1308 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d etype=%s atype=%s ismask=no",
1309 is_scatter, is_scatter ? "scatter" : "gather",
1310 num_elem, type2name(elem_bt), type2name(arr_type->elem()->array_element_basic_type()));
1311 set_map(old_map);
1312 set_sp(old_sp);
1313 return false;
1314 }
1315
1316 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1317 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1318 ciKlass* vbox_idx_klass = vector_idx_klass->const_oop()->as_instance()->java_lang_Class_klass();
1319 if (vbox_idx_klass == nullptr) {
1320 set_map(old_map);
1321 set_sp(old_sp);
1322 return false;
1323 }
1324
1325 Node* index_vect = nullptr;
1326 const TypeInstPtr* vbox_idx_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_idx_klass);
1327 if (!is_subword_type(elem_bt)) {
1328 index_vect = unbox_vector(argument(8), vbox_idx_type, T_INT, num_elem);
1329 if (index_vect == nullptr) {
1330 set_map(old_map);
1331 set_sp(old_sp);
1332 return false;
1333 }
1334 }
1335
1336 Node* mask = nullptr;
1337 if (is_masked_op) {
1338 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1339 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1340 mask = unbox_vector(is_scatter ? argument(10) : argument(9), mbox_type, elem_bt, num_elem);
1341 if (mask == nullptr) {
1342 log_if_needed(" ** unbox failed mask=%s",
1343 is_scatter ? NodeClassNames[argument(10)->Opcode()]
1344 : NodeClassNames[argument(9)->Opcode()]);
1345 set_map(old_map);
1346 set_sp(old_sp);
1347 return false;
1348 }
1349 }
1350
1351 const TypeVect* vector_type = TypeVect::make(elem_bt, num_elem);
1352 if (is_scatter) {
1353 Node* val = unbox_vector(argument(9), vbox_type, elem_bt, num_elem);
1354 if (val == nullptr) {
1355 set_map(old_map);
1356 set_sp(old_sp);
1357 return false; // operand unboxing failed
1358 }
1359 set_all_memory(reset_memory());
1360
1361 Node* vstore = nullptr;
1362 if (mask != nullptr) {
1363 vstore = gvn().transform(new StoreVectorScatterMaskedNode(control(), memory(addr), addr, addr_type, val, index_vect, mask));
1364 } else {
1365 vstore = gvn().transform(new StoreVectorScatterNode(control(), memory(addr), addr, addr_type, val, index_vect));
1366 }
1367 set_memory(vstore, addr_type);
1368 } else {
1369 Node* vload = nullptr;
1370 Node* index = argument(11);
1371 Node* indexMap = argument(12);
1372 Node* indexM = argument(13);
1373 if (mask != nullptr) {
1374 if (is_subword_type(elem_bt)) {
1375 Node* index_arr_base = array_element_address(indexMap, indexM, T_INT);
1376 vload = gvn().transform(new LoadVectorGatherMaskedNode(control(), memory(addr), addr, addr_type, vector_type, index_arr_base, mask, index));
1377 } else {
1378 vload = gvn().transform(new LoadVectorGatherMaskedNode(control(), memory(addr), addr, addr_type, vector_type, index_vect, mask));
1379 }
1380 } else {
1381 if (is_subword_type(elem_bt)) {
1382 Node* index_arr_base = array_element_address(indexMap, indexM, T_INT);
1383 vload = gvn().transform(new LoadVectorGatherNode(control(), memory(addr), addr, addr_type, vector_type, index_arr_base, index));
1384 } else {
1385 vload = gvn().transform(new LoadVectorGatherNode(control(), memory(addr), addr, addr_type, vector_type, index_vect));
1386 }
1387 }
1388 Node* box = box_vector(vload, vbox_type, elem_bt, num_elem);
1389 set_result(box);
1390 }
1391
1392 destruct_map_clone(old_map);
1393
1394 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1395 return true;
1396 }
1397
1398 // public static
1399 // <V extends Vector<E>,
1400 // M extends VectorMask<E>,
1401 // E>
1402 // long reductionCoerced(int oprId, Class<? extends V> vectorClass, Class<? extends M> maskClass,
1403 // Class<E> elementType, int length, V v, M m,
1404 // ReductionOperation<V, M> defaultImpl)
1405 bool LibraryCallKit::inline_vector_reduction() {
1406 const TypeInt* opr = gvn().type(argument(0))->isa_int();
1407 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
1408 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
1409 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
1410 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
1411
1412 if (opr == nullptr || vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr ||
1413 !opr->is_con() || vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
1414 log_if_needed(" ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
1415 NodeClassNames[argument(0)->Opcode()],
1416 NodeClassNames[argument(1)->Opcode()],
1417 NodeClassNames[argument(3)->Opcode()],
1418 NodeClassNames[argument(4)->Opcode()]);
1419 return false; // not enough info for intrinsification
1420 }
1421 if (!is_klass_initialized(vector_klass)) {
1422 log_if_needed(" ** klass argument not initialized");
1423 return false;
1424 }
1425 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1426 if (!elem_type->is_primitive_type()) {
1427 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1428 return false; // should be primitive type
1429 }
1430
1431 const Type* vmask_type = gvn().type(argument(6));
1432 bool is_masked_op = vmask_type != TypePtr::NULL_PTR;
1433 if (is_masked_op) {
1434 if (mask_klass == nullptr || mask_klass->const_oop() == nullptr) {
1435 log_if_needed(" ** missing constant: maskclass=%s", NodeClassNames[argument(2)->Opcode()]);
1436 return false; // not enough info for intrinsification
1437 }
1438
1439 if (!is_klass_initialized(mask_klass)) {
1440 log_if_needed(" ** mask klass argument not initialized");
1441 return false;
1442 }
1443
1444 if (vmask_type->maybe_null()) {
1445 log_if_needed(" ** null mask values are not allowed for masked op");
1446 return false;
1447 }
1448 }
1449
1450 BasicType elem_bt = elem_type->basic_type();
1451 int num_elem = vlen->get_con();
1452 int opc = VectorSupport::vop2ideal(opr->get_con(), elem_bt);
1453 int sopc = ReductionNode::opcode(opc, elem_bt);
1454
1455 // Ensure reduction operation for lanewise operation
1456 // When using mask, mask use type needs to be VecMaskUseLoad.
1457 if (sopc == opc || !arch_supports_vector(sopc, num_elem, elem_bt, is_masked_op ? VecMaskUseLoad : VecMaskNotUsed)) {
1458 log_if_needed(" ** not supported: arity=1 op=%d/reduce vlen=%d etype=%s is_masked_op=%d",
1459 sopc, num_elem, type2name(elem_bt), is_masked_op ? 1 : 0);
1460 return false;
1461 }
1462
1463 // Return true if current platform has implemented the masked operation with predicate feature.
1464 bool use_predicate = is_masked_op && arch_supports_vector(sopc, num_elem, elem_bt, VecMaskUsePred);
1465 if (is_masked_op && !use_predicate && !arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad)) {
1466 log_if_needed(" ** not supported: arity=1 op=%d/reduce vlen=%d etype=%s is_masked_op=1",
1467 sopc, num_elem, type2name(elem_bt));
1468 return false;
1469 }
1470
1471 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1472 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1473
1474 Node* opd = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
1475 if (opd == nullptr) {
1476 return false; // operand unboxing failed
1477 }
1478
1479 Node* mask = nullptr;
1480 if (is_masked_op) {
1481 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1482 assert(is_vector_mask(mbox_klass), "argument(2) should be a mask class");
1483 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1484 mask = unbox_vector(argument(6), mbox_type, elem_bt, num_elem);
1485 if (mask == nullptr) {
1486 log_if_needed(" ** unbox failed mask=%s",
1487 NodeClassNames[argument(6)->Opcode()]);
1488 return false;
1489 }
1490 }
1491
1492 Node* init = ReductionNode::make_identity_con_scalar(gvn(), opc, elem_bt);
1493 Node* value = opd;
1494
1495 assert(mask != nullptr || !is_masked_op, "Masked op needs the mask value never null");
1496 if (mask != nullptr && !use_predicate) {
1497 Node* reduce_identity = gvn().transform(VectorNode::scalar2vector(init, num_elem, elem_bt));
1498 value = gvn().transform(new VectorBlendNode(reduce_identity, value, mask));
1499 }
1500
1501 // Make an unordered Reduction node. This affects only AddReductionVF/VD and MulReductionVF/VD,
1502 // as these operations are allowed to be associative (not requiring strict order) in VectorAPI.
1503 value = ReductionNode::make(opc, nullptr, init, value, elem_bt, /* requires_strict_order */ false);
1504
1505 if (mask != nullptr && use_predicate) {
1506 value->add_req(mask);
1507 value->add_flag(Node::Flag_is_predicated_vector);
1508 }
1509
1510 value = gvn().transform(value);
1511
1512 Node* bits = nullptr;
1513 switch (elem_bt) {
1514 case T_BYTE:
1515 case T_SHORT:
1516 case T_INT: {
1517 bits = gvn().transform(new ConvI2LNode(value));
1518 break;
1519 }
1520 case T_FLOAT: {
1521 value = gvn().transform(new MoveF2INode(value));
1522 bits = gvn().transform(new ConvI2LNode(value));
1523 break;
1524 }
1525 case T_DOUBLE: {
1526 bits = gvn().transform(new MoveD2LNode(value));
1527 break;
1528 }
1529 case T_LONG: {
1530 bits = value; // no conversion needed
1531 break;
1532 }
1533 default: fatal("%s", type2name(elem_bt));
1534 }
1535 set_result(bits);
1536 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1537 return true;
1538 }
1539
1540 // public static <V> boolean test(int cond, Class<?> vectorClass, Class<?> elementType, int vlen,
1541 // V v1, V v2,
1542 // BiFunction<V, V, Boolean> defaultImpl)
1543 //
1544 bool LibraryCallKit::inline_vector_test() {
1545 const TypeInt* cond = gvn().type(argument(0))->isa_int();
1546 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
1547 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
1548 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
1549
1550 if (cond == nullptr || vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr ||
1551 !cond->is_con() || vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
1552 log_if_needed(" ** missing constant: cond=%s vclass=%s etype=%s vlen=%s",
1553 NodeClassNames[argument(0)->Opcode()],
1554 NodeClassNames[argument(1)->Opcode()],
1555 NodeClassNames[argument(2)->Opcode()],
1556 NodeClassNames[argument(3)->Opcode()]);
1557 return false; // not enough info for intrinsification
1558 }
1559 if (!is_klass_initialized(vector_klass)) {
1560 log_if_needed(" ** klass argument not initialized");
1561 return false;
1562 }
1563 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1564 if (!elem_type->is_primitive_type()) {
1565 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1566 return false; // should be primitive type
1567 }
1568 BasicType elem_bt = elem_type->basic_type();
1569 int num_elem = vlen->get_con();
1570 BoolTest::mask booltest = (BoolTest::mask)cond->get_con();
1571 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1572 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1573
1574 if (!arch_supports_vector(Op_VectorTest, num_elem, elem_bt, is_vector_mask(vbox_klass) ? VecMaskUseLoad : VecMaskNotUsed)) {
1575 log_if_needed(" ** not supported: arity=2 op=test/%d vlen=%d etype=%s ismask=%d",
1576 cond->get_con(), num_elem, type2name(elem_bt),
1577 is_vector_mask(vbox_klass));
1578 return false;
1579 }
1580
1581 Node* opd1 = unbox_vector(argument(4), vbox_type, elem_bt, num_elem);
1582 Node* opd2;
1583 if (Matcher::vectortest_needs_second_argument(booltest == BoolTest::overflow,
1584 opd1->bottom_type()->isa_vectmask())) {
1585 opd2 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
1586 } else {
1587 opd2 = opd1;
1588 }
1589 if (opd1 == nullptr || opd2 == nullptr) {
1590 return false; // operand unboxing failed
1591 }
1592
1593 Node* cmp = gvn().transform(new VectorTestNode(opd1, opd2, booltest));
1594 BoolTest::mask test = Matcher::vectortest_mask(booltest == BoolTest::overflow,
1595 opd1->bottom_type()->isa_vectmask(), num_elem);
1596 Node* bol = gvn().transform(new BoolNode(cmp, test));
1597 Node* res = gvn().transform(new CMoveINode(bol, gvn().intcon(0), gvn().intcon(1), TypeInt::BOOL));
1598
1599 set_result(res);
1600 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1601 return true;
1602 }
1603
1604 // public static
1605 // <V extends Vector<E>,
1606 // M extends VectorMask<E>,
1607 // E>
1608 // V blend(Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType, int vlen,
1609 // V v1, V v2, M m,
1610 // VectorBlendOp<V, M, E> defaultImpl)
1611 bool LibraryCallKit::inline_vector_blend() {
1612 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
1613 const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
1614 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
1615 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
1616
1617 if (mask_klass == nullptr || vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr) {
1618 return false; // dead code
1619 }
1620 if (mask_klass->const_oop() == nullptr || vector_klass->const_oop() == nullptr ||
1621 elem_klass->const_oop() == nullptr || !vlen->is_con()) {
1622 log_if_needed(" ** missing constant: vclass=%s mclass=%s etype=%s vlen=%s",
1623 NodeClassNames[argument(0)->Opcode()],
1624 NodeClassNames[argument(1)->Opcode()],
1625 NodeClassNames[argument(2)->Opcode()],
1626 NodeClassNames[argument(3)->Opcode()]);
1627 return false; // not enough info for intrinsification
1628 }
1629 if (!is_klass_initialized(vector_klass) || !is_klass_initialized(mask_klass)) {
1630 log_if_needed(" ** klass argument not initialized");
1631 return false;
1632 }
1633 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1634 if (!elem_type->is_primitive_type()) {
1635 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1636 return false; // should be primitive type
1637 }
1638 BasicType elem_bt = elem_type->basic_type();
1639 BasicType mask_bt = elem_bt;
1640 int num_elem = vlen->get_con();
1641
1642 if (!arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad)) {
1643 log_if_needed(" ** not supported: arity=2 op=blend vlen=%d etype=%s ismask=useload",
1644 num_elem, type2name(elem_bt));
1645 return false; // not supported
1646 }
1647 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1648 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1649
1650 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1651 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1652
1653 Node* v1 = unbox_vector(argument(4), vbox_type, elem_bt, num_elem);
1654 Node* v2 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
1655 Node* mask = unbox_vector(argument(6), mbox_type, mask_bt, num_elem);
1656
1657 if (v1 == nullptr || v2 == nullptr || mask == nullptr) {
1658 return false; // operand unboxing failed
1659 }
1660
1661 Node* blend = gvn().transform(new VectorBlendNode(v1, v2, mask));
1662
1663 Node* box = box_vector(blend, vbox_type, elem_bt, num_elem);
1664 set_result(box);
1665 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1666 return true;
1667 }
1668
1669 // public static
1670 // <V extends Vector<E>,
1671 // M extends VectorMask<E>,
1672 // E>
1673 // M compare(int cond, Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType, int vlen,
1674 // V v1, V v2, M m,
1675 // VectorCompareOp<V,M> defaultImpl)
1676 bool LibraryCallKit::inline_vector_compare() {
1677 const TypeInt* cond = gvn().type(argument(0))->isa_int();
1678 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
1679 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
1680 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
1681 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
1682
1683 if (cond == nullptr || vector_klass == nullptr || mask_klass == nullptr || elem_klass == nullptr || vlen == nullptr) {
1684 return false; // dead code
1685 }
1686 if (!cond->is_con() || vector_klass->const_oop() == nullptr || mask_klass->const_oop() == nullptr ||
1687 elem_klass->const_oop() == nullptr || !vlen->is_con()) {
1688 log_if_needed(" ** missing constant: cond=%s vclass=%s mclass=%s etype=%s vlen=%s",
1689 NodeClassNames[argument(0)->Opcode()],
1690 NodeClassNames[argument(1)->Opcode()],
1691 NodeClassNames[argument(2)->Opcode()],
1692 NodeClassNames[argument(3)->Opcode()],
1693 NodeClassNames[argument(4)->Opcode()]);
1694 return false; // not enough info for intrinsification
1695 }
1696 if (!is_klass_initialized(vector_klass) || !is_klass_initialized(mask_klass)) {
1697 log_if_needed(" ** klass argument not initialized");
1698 return false;
1699 }
1700 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1701 if (!elem_type->is_primitive_type()) {
1702 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1703 return false; // should be primitive type
1704 }
1705
1706 int num_elem = vlen->get_con();
1707 BasicType elem_bt = elem_type->basic_type();
1708 BasicType mask_bt = elem_bt;
1709
1710 if ((cond->get_con() & BoolTest::unsigned_compare) != 0) {
1711 if (!Matcher::supports_vector_comparison_unsigned(num_elem, elem_bt)) {
1712 log_if_needed(" ** not supported: unsigned comparison op=comp/%d vlen=%d etype=%s ismask=usestore",
1713 cond->get_con() & (BoolTest::unsigned_compare - 1), num_elem, type2name(elem_bt));
1714 return false;
1715 }
1716 }
1717
1718 if (!arch_supports_vector(Op_VectorMaskCmp, num_elem, elem_bt, VecMaskUseStore)) {
1719 log_if_needed(" ** not supported: arity=2 op=comp/%d vlen=%d etype=%s ismask=usestore",
1720 cond->get_con(), num_elem, type2name(elem_bt));
1721 return false;
1722 }
1723
1724 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1725 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1726
1727 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1728 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1729
1730 Node* v1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
1731 Node* v2 = unbox_vector(argument(6), vbox_type, elem_bt, num_elem);
1732
1733 bool is_masked_op = argument(7)->bottom_type() != TypePtr::NULL_PTR;
1734 Node* mask = is_masked_op ? unbox_vector(argument(7), mbox_type, elem_bt, num_elem) : nullptr;
1735 if (is_masked_op && mask == nullptr) {
1736 log_if_needed(" ** not supported: mask = null arity=2 op=comp/%d vlen=%d etype=%s ismask=usestore is_masked_op=1",
1737 cond->get_con(), num_elem, type2name(elem_bt));
1738 return false;
1739 }
1740
1741 bool use_predicate = is_masked_op && arch_supports_vector(Op_VectorMaskCmp, num_elem, elem_bt, VecMaskUsePred);
1742 if (is_masked_op && !use_predicate && !arch_supports_vector(Op_AndV, num_elem, elem_bt, VecMaskUseLoad)) {
1743 log_if_needed(" ** not supported: arity=2 op=comp/%d vlen=%d etype=%s ismask=usestore is_masked_op=1",
1744 cond->get_con(), num_elem, type2name(elem_bt));
1745 return false;
1746 }
1747
1748 if (v1 == nullptr || v2 == nullptr) {
1749 return false; // operand unboxing failed
1750 }
1751 BoolTest::mask pred = (BoolTest::mask)cond->get_con();
1752 ConINode* pred_node = (ConINode*)gvn().makecon(cond);
1753
1754 const TypeVect* vmask_type = TypeVect::makemask(mask_bt, num_elem);
1755 Node* operation = new VectorMaskCmpNode(pred, v1, v2, pred_node, vmask_type);
1756
1757 if (is_masked_op) {
1758 if (use_predicate) {
1759 operation->add_req(mask);
1760 operation->add_flag(Node::Flag_is_predicated_vector);
1761 } else {
1762 operation = gvn().transform(operation);
1763 operation = VectorNode::make(Op_AndV, operation, mask, vmask_type);
1764 }
1765 }
1766
1767 operation = gvn().transform(operation);
1768
1769 Node* box = box_vector(operation, mbox_type, mask_bt, num_elem);
1770 set_result(box);
1771 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1772 return true;
1773 }
1774
1775 // public static
1776 // <V extends Vector<E>,
1777 // Sh extends VectorShuffle<E>,
1778 // M extends VectorMask<E>,
1779 // E>
1780 // V rearrangeOp(Class<? extends V> vectorClass, Class<Sh> shuffleClass, Class<M> maskClass, Class<E> elementType, int vlen,
1781 // V v1, Sh sh, M m,
1782 // VectorRearrangeOp<V, Sh, M, E> defaultImpl)
1783 bool LibraryCallKit::inline_vector_rearrange() {
1784 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
1785 const TypeInstPtr* shuffle_klass = gvn().type(argument(1))->isa_instptr();
1786 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
1787 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
1788 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
1789
1790 if (vector_klass == nullptr || shuffle_klass == nullptr || elem_klass == nullptr || vlen == nullptr) {
1791 return false; // dead code
1792 }
1793 if (shuffle_klass->const_oop() == nullptr ||
1794 vector_klass->const_oop() == nullptr ||
1795 elem_klass->const_oop() == nullptr ||
1796 !vlen->is_con()) {
1797 log_if_needed(" ** missing constant: vclass=%s sclass=%s etype=%s vlen=%s",
1798 NodeClassNames[argument(0)->Opcode()],
1799 NodeClassNames[argument(1)->Opcode()],
1800 NodeClassNames[argument(3)->Opcode()],
1801 NodeClassNames[argument(4)->Opcode()]);
1802 return false; // not enough info for intrinsification
1803 }
1804 if (!is_klass_initialized(vector_klass) ||
1805 !is_klass_initialized(shuffle_klass)) {
1806 log_if_needed(" ** klass argument not initialized");
1807 return false;
1808 }
1809 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1810 if (!elem_type->is_primitive_type()) {
1811 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1812 return false; // should be primitive type
1813 }
1814
1815 BasicType elem_bt = elem_type->basic_type();
1816 BasicType shuffle_bt = elem_bt;
1817 if (shuffle_bt == T_FLOAT) {
1818 shuffle_bt = T_INT;
1819 } else if (shuffle_bt == T_DOUBLE) {
1820 shuffle_bt = T_LONG;
1821 }
1822
1823 int num_elem = vlen->get_con();
1824 bool need_load_shuffle = Matcher::vector_rearrange_requires_load_shuffle(shuffle_bt, num_elem);
1825
1826 if (need_load_shuffle && !arch_supports_vector(Op_VectorLoadShuffle, num_elem, shuffle_bt, VecMaskNotUsed)) {
1827 if (C->print_intrinsics()) {
1828 tty->print_cr(" ** not supported: arity=0 op=load/shuffle vlen=%d etype=%s ismask=no",
1829 num_elem, type2name(shuffle_bt));
1830 }
1831 return false; // not supported
1832 }
1833
1834 bool is_masked_op = argument(7)->bottom_type() != TypePtr::NULL_PTR;
1835 bool use_predicate = is_masked_op;
1836 if (is_masked_op &&
1837 (mask_klass == nullptr ||
1838 mask_klass->const_oop() == nullptr ||
1839 !is_klass_initialized(mask_klass))) {
1840 log_if_needed(" ** mask_klass argument not initialized");
1841 }
1842 if (!arch_supports_vector(Op_AndV, num_elem, elem_bt, VecMaskNotUsed)) {
1843 log_if_needed(" ** not supported: arity=2 op=and vlen=%d etype=%s ismask=no",
1844 num_elem, type2name(elem_bt));
1845 return false;
1846 }
1847 VectorMaskUseType checkFlags = (VectorMaskUseType)(is_masked_op ? (VecMaskUseLoad | VecMaskUsePred) : VecMaskNotUsed);
1848 if (!arch_supports_vector(Op_VectorRearrange, num_elem, elem_bt, checkFlags)) {
1849 use_predicate = false;
1850 if(!is_masked_op ||
1851 (!arch_supports_vector(Op_VectorRearrange, num_elem, elem_bt, VecMaskNotUsed) ||
1852 !arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad) ||
1853 !arch_supports_vector(Op_Replicate, num_elem, elem_bt, VecMaskNotUsed))) {
1854 log_if_needed(" ** not supported: arity=2 op=shuffle/rearrange vlen=%d etype=%s ismask=no",
1855 num_elem, type2name(elem_bt));
1856 return false; // not supported
1857 }
1858 }
1859 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1860 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1861
1862 ciKlass* shbox_klass = shuffle_klass->const_oop()->as_instance()->java_lang_Class_klass();
1863 const TypeInstPtr* shbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, shbox_klass);
1864
1865 Node* v1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
1866 Node* shuffle = unbox_vector(argument(6), shbox_type, shuffle_bt, num_elem);
1867 const TypeVect* st = TypeVect::make(shuffle_bt, num_elem);
1868
1869 if (v1 == nullptr || shuffle == nullptr) {
1870 return false; // operand unboxing failed
1871 }
1872
1873 assert(is_power_of_2(num_elem), "wrapping invalid");
1874 Node* wrapping_mask_elem = gvn().makecon(TypeInteger::make(num_elem - 1, num_elem - 1, Type::WidenMin, shuffle_bt == T_LONG ? T_LONG : T_INT));
1875 Node* wrapping_mask = gvn().transform(VectorNode::scalar2vector(wrapping_mask_elem, num_elem, shuffle_bt));
1876 shuffle = gvn().transform(new AndVNode(shuffle, wrapping_mask, st));
1877
1878 Node* mask = nullptr;
1879 if (is_masked_op) {
1880 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1881 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1882 mask = unbox_vector(argument(7), mbox_type, elem_bt, num_elem);
1883 if (mask == nullptr) {
1884 log_if_needed(" ** not supported: arity=3 op=shuffle/rearrange vlen=%d etype=%s ismask=useload is_masked_op=1",
1885 num_elem, type2name(elem_bt));
1886 return false;
1887 }
1888 }
1889
1890 if (need_load_shuffle) {
1891 shuffle = gvn().transform(new VectorLoadShuffleNode(shuffle, st));
1892 }
1893
1894 Node* rearrange = new VectorRearrangeNode(v1, shuffle);
1895 if (is_masked_op) {
1896 if (use_predicate) {
1897 rearrange->add_req(mask);
1898 rearrange->add_flag(Node::Flag_is_predicated_vector);
1899 } else {
1900 rearrange = gvn().transform(rearrange);
1901 Node* zero = gvn().makecon(Type::get_zero_type(elem_bt));
1902 Node* zerovec = gvn().transform(VectorNode::scalar2vector(zero, num_elem, elem_bt));
1903 rearrange = new VectorBlendNode(zerovec, rearrange, mask);
1904 }
1905 }
1906 rearrange = gvn().transform(rearrange);
1907
1908 Node* box = box_vector(rearrange, vbox_type, elem_bt, num_elem);
1909 set_result(box);
1910 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1911 return true;
1912 }
1913
1914 // public static
1915 // <V extends Vector<E>,
1916 // M extends VectorMask<E>,
1917 // E>
1918 // V selectFromOp(Class<? extends V> vClass, Class<M> mClass, Class<E> eClass,
1919 // int length, V v1, V v2, M m,
1920 // VectorSelectFromOp<V, M> defaultImpl)
1921 bool LibraryCallKit::inline_vector_select_from() {
1922 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
1923 const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
1924 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
1925 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
1926
1927 if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr ||
1928 vector_klass->const_oop() == nullptr ||
1929 elem_klass->const_oop() == nullptr ||
1930 !vlen->is_con()) {
1931 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s",
1932 NodeClassNames[argument(0)->Opcode()],
1933 NodeClassNames[argument(2)->Opcode()],
1934 NodeClassNames[argument(3)->Opcode()]);
1935 return false; // not enough info for intrinsification
1936 }
1937 if (!is_klass_initialized(vector_klass)) {
1938 log_if_needed(" ** klass argument not initialized");
1939 return false;
1940 }
1941 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1942 if (!elem_type->is_primitive_type()) {
1943 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1944 return false; // should be primitive type
1945 }
1946 BasicType elem_bt = elem_type->basic_type();
1947 int num_elem = vlen->get_con();
1948 if (!is_power_of_2(num_elem)) {
1949 log_if_needed(" ** vlen not power of two=%d", num_elem);
1950 return false;
1951 }
1952
1953 BasicType shuffle_bt = elem_bt;
1954 if (shuffle_bt == T_FLOAT) {
1955 shuffle_bt = T_INT;
1956 } else if (shuffle_bt == T_DOUBLE) {
1957 shuffle_bt = T_LONG;
1958 }
1959 bool need_load_shuffle = Matcher::vector_rearrange_requires_load_shuffle(shuffle_bt, num_elem);
1960
1961 int cast_vopc = VectorCastNode::opcode(-1, elem_bt); // from vector of type elem_bt
1962 if ((need_load_shuffle && !arch_supports_vector(Op_VectorLoadShuffle, num_elem, elem_bt, VecMaskNotUsed)) ||
1963 (elem_bt != shuffle_bt && !arch_supports_vector(cast_vopc, num_elem, shuffle_bt, VecMaskNotUsed)) ||
1964 !arch_supports_vector(Op_AndV, num_elem, shuffle_bt, VecMaskNotUsed) ||
1965 !arch_supports_vector(Op_Replicate, num_elem, shuffle_bt, VecMaskNotUsed)) {
1966 log_if_needed(" ** not supported: arity=0 op=selectFrom vlen=%d etype=%s ismask=no",
1967 num_elem, type2name(elem_bt));
1968 return false; // not supported
1969 }
1970
1971 bool is_masked_op = argument(6)->bottom_type() != TypePtr::NULL_PTR;
1972 bool use_predicate = is_masked_op;
1973 if (is_masked_op &&
1974 (mask_klass == nullptr ||
1975 mask_klass->const_oop() == nullptr ||
1976 !is_klass_initialized(mask_klass))) {
1977 log_if_needed(" ** mask_klass argument not initialized");
1978 return false; // not supported
1979 }
1980 VectorMaskUseType checkFlags = (VectorMaskUseType)(is_masked_op ? (VecMaskUseLoad | VecMaskUsePred) : VecMaskNotUsed);
1981 if (!arch_supports_vector(Op_VectorRearrange, num_elem, elem_bt, checkFlags)) {
1982 use_predicate = false;
1983 if(!is_masked_op ||
1984 (!arch_supports_vector(Op_VectorRearrange, num_elem, elem_bt, VecMaskNotUsed) ||
1985 !arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad) ||
1986 !arch_supports_vector(Op_Replicate, num_elem, elem_bt, VecMaskNotUsed))) {
1987 log_if_needed(" ** not supported: op=selectFrom vlen=%d etype=%s is_masked_op=%d",
1988 num_elem, type2name(elem_bt), is_masked_op);
1989 return false; // not supported
1990 }
1991 }
1992 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1993 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1994
1995 // v1 is the index vector
1996 Node* v1 = unbox_vector(argument(4), vbox_type, elem_bt, num_elem);
1997 // v2 is the vector being rearranged
1998 Node* v2 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
1999
2000 if (v1 == nullptr) {
2001 log_if_needed(" ** unbox failed v1=%s", NodeClassNames[argument(4)->Opcode()]);
2002 return false; // operand unboxing failed
2003 }
2004
2005 if (v2 == nullptr) {
2006 log_if_needed(" ** unbox failed v2=%s", NodeClassNames[argument(5)->Opcode()]);
2007 return false; // operand unboxing failed
2008 }
2009
2010 Node* mask = nullptr;
2011 if (is_masked_op) {
2012 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
2013 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
2014 mask = unbox_vector(argument(6), mbox_type, elem_bt, num_elem);
2015 if (mask == nullptr) {
2016 log_if_needed(" ** unbox failed mask=%s", NodeClassNames[argument(6)->Opcode()]);
2017 return false;
2018 }
2019 }
2020
2021 // cast index vector from elem_bt vector to byte vector
2022 const TypeVect* shuffle_vt = TypeVect::make(shuffle_bt, num_elem);
2023 Node* shuffle = v1;
2024
2025 if (shuffle_bt != elem_bt) {
2026 shuffle = gvn().transform(VectorCastNode::make(cast_vopc, v1, shuffle_bt, num_elem));
2027 }
2028
2029 // wrap the byte vector lanes to (num_elem - 1) to form the shuffle vector where num_elem is vector length
2030 // this is a simple AND operation as we come here only for power of two vector length
2031 Node* mod_val = gvn().makecon(TypeInteger::make(num_elem - 1, num_elem - 1, Type::WidenMin, shuffle_bt == T_LONG ? T_LONG : T_INT));
2032 Node* bcast_mod = gvn().transform(VectorNode::scalar2vector(mod_val, num_elem, shuffle_bt));
2033 shuffle = gvn().transform(VectorNode::make(Op_AndV, shuffle, bcast_mod, shuffle_vt));
2034
2035 // load the shuffle to use in rearrange
2036 if (need_load_shuffle) {
2037 shuffle = gvn().transform(new VectorLoadShuffleNode(shuffle, shuffle_vt));
2038 }
2039
2040 // and finally rearrange
2041 Node* rearrange = new VectorRearrangeNode(v2, shuffle);
2042 if (is_masked_op) {
2043 if (use_predicate) {
2044 // masked rearrange is supported so use that directly
2045 rearrange->add_req(mask);
2046 rearrange->add_flag(Node::Flag_is_predicated_vector);
2047 } else {
2048 // masked rearrange is not supported so emulate usig blend
2049 const TypeVect* vt = v1->bottom_type()->is_vect();
2050 rearrange = gvn().transform(rearrange);
2051
2052 // create a zero vector with each lane element set as zero
2053 Node* zero = gvn().makecon(Type::get_zero_type(elem_bt));
2054 Node* zerovec = gvn().transform(VectorNode::scalar2vector(zero, num_elem, elem_bt));
2055
2056 // For each lane for which mask is set, blend in the rearranged lane into zero vector
2057 rearrange = new VectorBlendNode(zerovec, rearrange, mask);
2058 }
2059 }
2060 rearrange = gvn().transform(rearrange);
2061
2062 // box the result
2063 Node* box = box_vector(rearrange, vbox_type, elem_bt, num_elem);
2064 set_result(box);
2065
2066 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2067 return true;
2068 }
2069
2070 // public static
2071 // <V extends Vector<E>,
2072 // M extends VectorMask<E>,
2073 // E>
2074 // V broadcastInt(int opr, Class<? extends V> vectorClass, Class<? extends M> maskClass,
2075 // Class<E> elementType, int length,
2076 // V v, int n, M m,
2077 // VectorBroadcastIntOp<V, M> defaultImpl)
2078 bool LibraryCallKit::inline_vector_broadcast_int() {
2079 const TypeInt* opr = gvn().type(argument(0))->isa_int();
2080 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
2081 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
2082 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
2083 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
2084
2085 if (opr == nullptr || vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr) {
2086 return false; // dead code
2087 }
2088 if (!opr->is_con() || vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
2089 log_if_needed(" ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
2090 NodeClassNames[argument(0)->Opcode()],
2091 NodeClassNames[argument(1)->Opcode()],
2092 NodeClassNames[argument(3)->Opcode()],
2093 NodeClassNames[argument(4)->Opcode()]);
2094 return false; // not enough info for intrinsification
2095 }
2096 if (!is_klass_initialized(vector_klass)) {
2097 log_if_needed(" ** klass argument not initialized");
2098 return false;
2099 }
2100
2101 const Type* vmask_type = gvn().type(argument(7));
2102 bool is_masked_op = vmask_type != TypePtr::NULL_PTR;
2103 if (is_masked_op) {
2104 if (mask_klass == nullptr || mask_klass->const_oop() == nullptr) {
2105 log_if_needed(" ** missing constant: maskclass=%s", NodeClassNames[argument(2)->Opcode()]);
2106 return false; // not enough info for intrinsification
2107 }
2108
2109 if (!is_klass_initialized(mask_klass)) {
2110 log_if_needed(" ** mask klass argument not initialized");
2111 return false;
2112 }
2113
2114 if (vmask_type->maybe_null()) {
2115 log_if_needed(" ** null mask values are not allowed for masked op");
2116 return false;
2117 }
2118 }
2119
2120 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2121 if (!elem_type->is_primitive_type()) {
2122 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2123 return false; // should be primitive type
2124 }
2125
2126 int num_elem = vlen->get_con();
2127 BasicType elem_bt = elem_type->basic_type();
2128 int opc = VectorSupport::vop2ideal(opr->get_con(), elem_bt);
2129
2130 bool is_shift = VectorNode::is_shift_opcode(opc);
2131 bool is_rotate = VectorNode::is_rotate_opcode(opc);
2132
2133 if (opc == 0 || (!is_shift && !is_rotate)) {
2134 log_if_needed(" ** operation not supported: op=%d bt=%s", opr->get_con(), type2name(elem_bt));
2135 return false; // operation not supported
2136 }
2137
2138 int sopc = VectorNode::opcode(opc, elem_bt);
2139 if (sopc == 0) {
2140 log_if_needed(" ** operation not supported: opc=%s bt=%s", NodeClassNames[opc], type2name(elem_bt));
2141 return false; // operation not supported
2142 }
2143
2144 Node* cnt = argument(6);
2145 const TypeInt* cnt_type = cnt->bottom_type()->isa_int();
2146
2147 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2148 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2149
2150 // If CPU supports vector constant rotate instructions pass it directly
2151 bool is_const_rotate = is_rotate && cnt_type && cnt_type->is_con() &&
2152 Matcher::supports_vector_constant_rotates(cnt_type->get_con());
2153 bool has_scalar_args = is_rotate ? !is_const_rotate : true;
2154
2155 VectorMaskUseType checkFlags = (VectorMaskUseType)(is_masked_op ? (VecMaskUseLoad | VecMaskUsePred) : VecMaskNotUsed);
2156 bool use_predicate = is_masked_op;
2157
2158 if (!arch_supports_vector(sopc, num_elem, elem_bt, checkFlags, has_scalar_args)) {
2159 use_predicate = false;
2160 if (!is_masked_op ||
2161 (!arch_supports_vector(sopc, num_elem, elem_bt, VecMaskNotUsed, has_scalar_args) ||
2162 !arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad))) {
2163
2164 log_if_needed(" ** not supported: arity=0 op=int/%d vlen=%d etype=%s is_masked_op=%d",
2165 sopc, num_elem, type2name(elem_bt), is_masked_op ? 1 : 0);
2166 return false; // not supported
2167 }
2168 }
2169
2170 Node* opd1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
2171 Node* opd2 = nullptr;
2172 if (is_shift) {
2173 opd2 = vector_shift_count(cnt, opc, elem_bt, num_elem);
2174 } else {
2175 assert(is_rotate, "unexpected operation");
2176 if (!is_const_rotate) {
2177 cnt = elem_bt == T_LONG ? gvn().transform(new ConvI2LNode(cnt)) : cnt;
2178 opd2 = gvn().transform(VectorNode::scalar2vector(cnt, num_elem, elem_bt));
2179 } else {
2180 // Constant shift value.
2181 opd2 = cnt;
2182 }
2183 }
2184
2185 if (opd1 == nullptr || opd2 == nullptr) {
2186 return false;
2187 }
2188
2189 Node* mask = nullptr;
2190 if (is_masked_op) {
2191 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
2192 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
2193 mask = unbox_vector(argument(7), mbox_type, elem_bt, num_elem);
2194 if (mask == nullptr) {
2195 log_if_needed(" ** unbox failed mask=%s", NodeClassNames[argument(7)->Opcode()]);
2196 return false;
2197 }
2198 }
2199
2200 Node* operation = VectorNode::make(opc, opd1, opd2, num_elem, elem_bt);
2201 if (is_masked_op && mask != nullptr) {
2202 if (use_predicate) {
2203 operation->add_req(mask);
2204 operation->add_flag(Node::Flag_is_predicated_vector);
2205 } else {
2206 operation = gvn().transform(operation);
2207 operation = new VectorBlendNode(opd1, operation, mask);
2208 }
2209 }
2210 operation = gvn().transform(operation);
2211 Node* vbox = box_vector(operation, vbox_type, elem_bt, num_elem);
2212 set_result(vbox);
2213 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2214 return true;
2215 }
2216
2217 // public static <VOUT extends VectorPayload,
2218 // VIN extends VectorPayload,
2219 // S extends VectorSpecies>
2220 // VOUT convert(int oprId,
2221 // Class<?> fromVectorClass, Class<?> fromElementType, int fromVLen,
2222 // Class<?> toVectorClass, Class<?> toElementType, int toVLen,
2223 // VIN v, S s,
2224 // VectorConvertOp<VOUT, VIN, S> defaultImpl)
2225 //
2226 bool LibraryCallKit::inline_vector_convert() {
2227 const TypeInt* opr = gvn().type(argument(0))->isa_int();
2228
2229 const TypeInstPtr* vector_klass_from = gvn().type(argument(1))->isa_instptr();
2230 const TypeInstPtr* elem_klass_from = gvn().type(argument(2))->isa_instptr();
2231 const TypeInt* vlen_from = gvn().type(argument(3))->isa_int();
2232
2233 const TypeInstPtr* vector_klass_to = gvn().type(argument(4))->isa_instptr();
2234 const TypeInstPtr* elem_klass_to = gvn().type(argument(5))->isa_instptr();
2235 const TypeInt* vlen_to = gvn().type(argument(6))->isa_int();
2236
2237 if (opr == nullptr ||
2238 vector_klass_from == nullptr || elem_klass_from == nullptr || vlen_from == nullptr ||
2239 vector_klass_to == nullptr || elem_klass_to == nullptr || vlen_to == nullptr) {
2240 return false; // dead code
2241 }
2242 if (!opr->is_con() ||
2243 vector_klass_from->const_oop() == nullptr || elem_klass_from->const_oop() == nullptr || !vlen_from->is_con() ||
2244 vector_klass_to->const_oop() == nullptr || elem_klass_to->const_oop() == nullptr || !vlen_to->is_con()) {
2245 log_if_needed(" ** missing constant: opr=%s vclass_from=%s etype_from=%s vlen_from=%s vclass_to=%s etype_to=%s vlen_to=%s",
2246 NodeClassNames[argument(0)->Opcode()],
2247 NodeClassNames[argument(1)->Opcode()],
2248 NodeClassNames[argument(2)->Opcode()],
2249 NodeClassNames[argument(3)->Opcode()],
2250 NodeClassNames[argument(4)->Opcode()],
2251 NodeClassNames[argument(5)->Opcode()],
2252 NodeClassNames[argument(6)->Opcode()]);
2253 return false; // not enough info for intrinsification
2254 }
2255 if (!is_klass_initialized(vector_klass_from) || !is_klass_initialized(vector_klass_to)) {
2256 log_if_needed(" ** klass argument not initialized");
2257 return false;
2258 }
2259
2260 assert(opr->get_con() == VectorSupport::VECTOR_OP_CAST ||
2261 opr->get_con() == VectorSupport::VECTOR_OP_UCAST ||
2262 opr->get_con() == VectorSupport::VECTOR_OP_REINTERPRET, "wrong opcode");
2263 bool is_cast = (opr->get_con() == VectorSupport::VECTOR_OP_CAST || opr->get_con() == VectorSupport::VECTOR_OP_UCAST);
2264 bool is_ucast = (opr->get_con() == VectorSupport::VECTOR_OP_UCAST);
2265
2266 ciKlass* vbox_klass_from = vector_klass_from->const_oop()->as_instance()->java_lang_Class_klass();
2267 ciKlass* vbox_klass_to = vector_klass_to->const_oop()->as_instance()->java_lang_Class_klass();
2268
2269 bool is_mask = is_vector_mask(vbox_klass_from);
2270
2271 ciType* elem_type_from = elem_klass_from->const_oop()->as_instance()->java_mirror_type();
2272 if (!elem_type_from->is_primitive_type()) {
2273 return false; // should be primitive type
2274 }
2275 BasicType elem_bt_from = elem_type_from->basic_type();
2276 ciType* elem_type_to = elem_klass_to->const_oop()->as_instance()->java_mirror_type();
2277 if (!elem_type_to->is_primitive_type()) {
2278 return false; // should be primitive type
2279 }
2280 BasicType elem_bt_to = elem_type_to->basic_type();
2281
2282 int num_elem_from = vlen_from->get_con();
2283 int num_elem_to = vlen_to->get_con();
2284
2285 // Check whether we can unbox to appropriate size. Even with casting, checking for reinterpret is needed
2286 // since we may need to change size.
2287 if (!arch_supports_vector(Op_VectorReinterpret,
2288 num_elem_from,
2289 elem_bt_from,
2290 is_mask ? VecMaskUseAll : VecMaskNotUsed)) {
2291 log_if_needed(" ** not supported: arity=1 op=%s/1 vlen1=%d etype1=%s ismask=%d",
2292 is_cast ? "cast" : "reinterpret",
2293 num_elem_from, type2name(elem_bt_from), is_mask);
2294 return false;
2295 }
2296
2297 // Check whether we can support resizing/reinterpreting to the new size.
2298 if (!arch_supports_vector(Op_VectorReinterpret,
2299 num_elem_to,
2300 elem_bt_to,
2301 is_mask ? VecMaskUseAll : VecMaskNotUsed)) {
2302 log_if_needed(" ** not supported: arity=1 op=%s/2 vlen2=%d etype2=%s ismask=%d",
2303 is_cast ? "cast" : "reinterpret",
2304 num_elem_to, type2name(elem_bt_to), is_mask);
2305 return false;
2306 }
2307
2308 // At this point, we know that both input and output vector registers are supported
2309 // by the architecture. Next check if the casted type is simply to same type - which means
2310 // that it is actually a resize and not a cast.
2311 if (is_cast && elem_bt_from == elem_bt_to) {
2312 is_cast = false;
2313 }
2314
2315 const TypeInstPtr* vbox_type_from = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass_from);
2316
2317 Node* opd1 = unbox_vector(argument(7), vbox_type_from, elem_bt_from, num_elem_from);
2318 if (opd1 == nullptr) {
2319 return false;
2320 }
2321
2322 const TypeVect* src_type = TypeVect::make(elem_bt_from, num_elem_from, is_mask);
2323 const TypeVect* dst_type = TypeVect::make(elem_bt_to, num_elem_to, is_mask);
2324
2325 // Safety check to prevent casting if source mask is of type vector
2326 // and destination mask of type predicate vector and vice-versa.
2327 // From X86 standpoint, this case will only arise over KNL target,
2328 // where certain masks (depending on the species) are either propagated
2329 // through a vector or predicate register.
2330 if (is_mask &&
2331 ((src_type->isa_vectmask() == nullptr && dst_type->isa_vectmask()) ||
2332 (dst_type->isa_vectmask() == nullptr && src_type->isa_vectmask()))) {
2333 return false;
2334 }
2335
2336 Node* op = opd1;
2337 if (is_cast) {
2338 assert(!is_mask || num_elem_from == num_elem_to, "vector mask cast needs the same elem num");
2339 int cast_vopc = VectorCastNode::opcode(-1, elem_bt_from, !is_ucast);
2340
2341 // Make sure that vector cast is implemented to particular type/size combination if it is
2342 // not a mask casting.
2343 if (!is_mask && !arch_supports_vector(cast_vopc, num_elem_to, elem_bt_to, VecMaskNotUsed)) {
2344 log_if_needed(" ** not supported: arity=1 op=cast#%d/3 vlen2=%d etype2=%s ismask=%d",
2345 cast_vopc, num_elem_to, type2name(elem_bt_to), is_mask);
2346 return false;
2347 }
2348
2349 if (num_elem_from < num_elem_to) {
2350 // Since input and output number of elements are not consistent, we need to make sure we
2351 // properly size. Thus, first make a cast that retains the number of elements from source.
2352 int num_elem_for_cast = num_elem_from;
2353
2354 // It is possible that arch does not support this intermediate vector size
2355 // TODO More complex logic required here to handle this corner case for the sizes.
2356 if (!arch_supports_vector(cast_vopc, num_elem_for_cast, elem_bt_to, VecMaskNotUsed)) {
2357 log_if_needed(" ** not supported: arity=1 op=cast#%d/4 vlen1=%d etype2=%s ismask=%d",
2358 cast_vopc,
2359 num_elem_for_cast, type2name(elem_bt_to), is_mask);
2360 return false;
2361 }
2362
2363 op = gvn().transform(VectorCastNode::make(cast_vopc, op, elem_bt_to, num_elem_for_cast));
2364 // Now ensure that the destination gets properly resized to needed size.
2365 op = gvn().transform(new VectorReinterpretNode(op, op->bottom_type()->is_vect(), dst_type));
2366 } else if (num_elem_from > num_elem_to) {
2367 // Since number of elements from input is larger than output, simply reduce size of input
2368 // (we are supposed to drop top elements anyway).
2369 int num_elem_for_resize = num_elem_to;
2370
2371 // It is possible that arch does not support this intermediate vector size
2372 // TODO More complex logic required here to handle this corner case for the sizes.
2373 if (!arch_supports_vector(Op_VectorReinterpret,
2374 num_elem_for_resize,
2375 elem_bt_from,
2376 VecMaskNotUsed)) {
2377 log_if_needed(" ** not supported: arity=1 op=cast/5 vlen2=%d etype1=%s ismask=%d",
2378 num_elem_for_resize, type2name(elem_bt_from), is_mask);
2379 return false;
2380 }
2381
2382 const TypeVect* resize_type = TypeVect::make(elem_bt_from, num_elem_for_resize);
2383 op = gvn().transform(new VectorReinterpretNode(op, src_type, resize_type));
2384 op = gvn().transform(VectorCastNode::make(cast_vopc, op, elem_bt_to, num_elem_to));
2385 } else { // num_elem_from == num_elem_to
2386 if (is_mask) {
2387 // Make sure that cast for vector mask is implemented to particular type/size combination.
2388 if (!arch_supports_vector(Op_VectorMaskCast, num_elem_to, elem_bt_to, VecMaskNotUsed)) {
2389 log_if_needed(" ** not supported: arity=1 op=maskcast vlen2=%d etype2=%s ismask=%d",
2390 num_elem_to, type2name(elem_bt_to), is_mask);
2391 return false;
2392 }
2393 op = gvn().transform(new VectorMaskCastNode(op, dst_type));
2394 } else {
2395 // Since input and output number of elements match, and since we know this vector size is
2396 // supported, simply do a cast with no resize needed.
2397 op = gvn().transform(VectorCastNode::make(cast_vopc, op, elem_bt_to, num_elem_to));
2398 }
2399 }
2400 } else if (!Type::equals(src_type, dst_type)) {
2401 assert(!is_cast, "must be reinterpret");
2402 op = gvn().transform(new VectorReinterpretNode(op, src_type, dst_type));
2403 }
2404
2405 const TypeInstPtr* vbox_type_to = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass_to);
2406 Node* vbox = box_vector(op, vbox_type_to, elem_bt_to, num_elem_to);
2407 set_result(vbox);
2408 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem_to * type2aelembytes(elem_bt_to))));
2409 return true;
2410 }
2411
2412 // public static
2413 // <V extends Vector<E>,
2414 // E>
2415 // V insert(Class<? extends V> vectorClass, Class<E> elementType, int vlen,
2416 // V vec, int ix, long val,
2417 // VecInsertOp<V> defaultImpl)
2418 bool LibraryCallKit::inline_vector_insert() {
2419 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
2420 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
2421 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
2422 const TypeInt* idx = gvn().type(argument(4))->isa_int();
2423
2424 if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr || idx == nullptr) {
2425 return false; // dead code
2426 }
2427 if (vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con() || !idx->is_con()) {
2428 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s idx=%s",
2429 NodeClassNames[argument(0)->Opcode()],
2430 NodeClassNames[argument(1)->Opcode()],
2431 NodeClassNames[argument(2)->Opcode()],
2432 NodeClassNames[argument(4)->Opcode()]);
2433 return false; // not enough info for intrinsification
2434 }
2435 if (!is_klass_initialized(vector_klass)) {
2436 log_if_needed(" ** klass argument not initialized");
2437 return false;
2438 }
2439 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2440 if (!elem_type->is_primitive_type()) {
2441 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2442 return false; // should be primitive type
2443 }
2444 BasicType elem_bt = elem_type->basic_type();
2445 int num_elem = vlen->get_con();
2446 if (!arch_supports_vector(Op_VectorInsert, num_elem, elem_bt, VecMaskNotUsed)) {
2447 log_if_needed(" ** not supported: arity=1 op=insert vlen=%d etype=%s ismask=no",
2448 num_elem, type2name(elem_bt));
2449 return false; // not supported
2450 }
2451
2452 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2453 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2454
2455 Node* opd = unbox_vector(argument(3), vbox_type, elem_bt, num_elem);
2456 if (opd == nullptr) {
2457 return false;
2458 }
2459
2460 Node* insert_val = argument(5);
2461 assert(gvn().type(insert_val)->isa_long() != nullptr, "expected to be long");
2462
2463 // Convert insert value back to its appropriate type.
2464 switch (elem_bt) {
2465 case T_BYTE:
2466 insert_val = gvn().transform(new ConvL2INode(insert_val, TypeInt::BYTE));
2467 break;
2468 case T_SHORT:
2469 insert_val = gvn().transform(new ConvL2INode(insert_val, TypeInt::SHORT));
2470 break;
2471 case T_INT:
2472 insert_val = gvn().transform(new ConvL2INode(insert_val));
2473 break;
2474 case T_FLOAT:
2475 insert_val = gvn().transform(new ConvL2INode(insert_val));
2476 insert_val = gvn().transform(new MoveI2FNode(insert_val));
2477 break;
2478 case T_DOUBLE:
2479 insert_val = gvn().transform(new MoveL2DNode(insert_val));
2480 break;
2481 case T_LONG:
2482 // no conversion needed
2483 break;
2484 default: fatal("%s", type2name(elem_bt)); break;
2485 }
2486
2487 Node* operation = gvn().transform(VectorInsertNode::make(opd, insert_val, idx->get_con(), gvn()));
2488
2489 Node* vbox = box_vector(operation, vbox_type, elem_bt, num_elem);
2490 set_result(vbox);
2491 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2492 return true;
2493 }
2494
2495 // public static
2496 // <VM extends VectorPayload,
2497 // E>
2498 // long extract(Class<? extends VM> vClass, Class<E> eClass,
2499 // int length,
2500 // VM vm, int i,
2501 // VecExtractOp<VM> defaultImpl)
2502 bool LibraryCallKit::inline_vector_extract() {
2503 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
2504 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
2505 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
2506 const TypeInt* idx = gvn().type(argument(4))->isa_int();
2507
2508 if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr || idx == nullptr) {
2509 return false; // dead code
2510 }
2511 if (vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
2512 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s",
2513 NodeClassNames[argument(0)->Opcode()],
2514 NodeClassNames[argument(1)->Opcode()],
2515 NodeClassNames[argument(2)->Opcode()]);
2516 return false; // not enough info for intrinsification
2517 }
2518 if (!is_klass_initialized(vector_klass)) {
2519 log_if_needed(" ** klass argument not initialized");
2520 return false;
2521 }
2522 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2523 if (!elem_type->is_primitive_type()) {
2524 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2525 return false; // should be primitive type
2526 }
2527 BasicType elem_bt = elem_type->basic_type();
2528 int num_elem = vlen->get_con();
2529
2530 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2531 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2532
2533 Node* opd = nullptr;
2534
2535 if (is_vector_mask(vbox_klass)) {
2536 // vbox_klass is mask. This is used for VectorMask.laneIsSet(int).
2537
2538 Node* pos = argument(4); // can be variable
2539 if (arch_supports_vector(Op_ExtractUB, num_elem, elem_bt, VecMaskUseAll)) {
2540 // Transform mask to vector with type of boolean and utilize ExtractUB node.
2541 opd = unbox_vector(argument(3), vbox_type, elem_bt, num_elem);
2542 if (opd == nullptr) {
2543 return false;
2544 }
2545 opd = gvn().transform(VectorStoreMaskNode::make(gvn(), opd, elem_bt, num_elem));
2546 opd = gvn().transform(new ExtractUBNode(opd, pos));
2547 opd = gvn().transform(new ConvI2LNode(opd));
2548 } else if (arch_supports_vector(Op_VectorMaskToLong, num_elem, elem_bt, VecMaskUseLoad)) {
2549 opd = unbox_vector(argument(3), vbox_type, elem_bt, num_elem);
2550 if (opd == nullptr) {
2551 return false;
2552 }
2553 // VectorMaskToLongNode requires the input is either a mask or a vector with BOOLEAN type.
2554 if (opd->bottom_type()->isa_vectmask() == nullptr) {
2555 opd = gvn().transform(VectorStoreMaskNode::make(gvn(), opd, elem_bt, num_elem));
2556 }
2557 // ((toLong() >>> pos) & 1L
2558 opd = gvn().transform(new VectorMaskToLongNode(opd, TypeLong::LONG));
2559 opd = gvn().transform(new URShiftLNode(opd, pos));
2560 opd = gvn().transform(new AndLNode(opd, gvn().makecon(TypeLong::ONE)));
2561 } else {
2562 log_if_needed(" ** Rejected mask extraction because architecture does not support it");
2563 return false; // not supported
2564 }
2565 } else {
2566 // vbox_klass is vector. This is used for Vector.lane(int).
2567 if (!idx->is_con()) {
2568 log_if_needed(" ** missing constant: idx=%s", NodeClassNames[argument(4)->Opcode()]);
2569 return false; // not enough info for intrinsification
2570 }
2571
2572 int vopc = ExtractNode::opcode(elem_bt);
2573 if (!arch_supports_vector(vopc, num_elem, elem_bt, VecMaskNotUsed)) {
2574 log_if_needed(" ** not supported: arity=1 op=extract vlen=%d etype=%s ismask=no",
2575 num_elem, type2name(elem_bt));
2576 return false; // not supported
2577 }
2578
2579 opd = unbox_vector(argument(3), vbox_type, elem_bt, num_elem);
2580 if (opd == nullptr) {
2581 return false;
2582 }
2583 ConINode* idx_con = gvn().intcon(idx->get_con())->as_ConI();
2584
2585 opd = gvn().transform(ExtractNode::make(opd, idx_con, elem_bt));
2586 switch (elem_bt) {
2587 case T_BYTE:
2588 case T_SHORT:
2589 case T_INT: {
2590 opd = gvn().transform(new ConvI2LNode(opd));
2591 break;
2592 }
2593 case T_FLOAT: {
2594 opd = gvn().transform(new MoveF2INode(opd));
2595 opd = gvn().transform(new ConvI2LNode(opd));
2596 break;
2597 }
2598 case T_DOUBLE: {
2599 opd = gvn().transform(new MoveD2LNode(opd));
2600 break;
2601 }
2602 case T_LONG: {
2603 // no conversion needed
2604 break;
2605 }
2606 default: fatal("%s", type2name(elem_bt));
2607 }
2608 }
2609 set_result(opd);
2610 return true;
2611 }
2612
2613 static Node* LowerSelectFromTwoVectorOperation(PhaseGVN& phase, Node* index_vec, Node* src1, Node* src2, const TypeVect* vt) {
2614 int num_elem = vt->length();
2615 BasicType elem_bt = vt->element_basic_type();
2616
2617 // Lower selectFrom operation into its constituent operations.
2618 // SelectFromTwoVectorNode =
2619 // (VectorBlend
2620 // (VectorRearrange SRC1 (WRAPED_INDEX AND (VLEN-1))
2621 // (VectorRearrange SRC2 (WRAPED_INDEX AND (VLEN-1))
2622 // MASK)
2623 // Where
2624 // WRAPED_INDEX are computed by wrapping incoming indexes
2625 // to two vector index range [0, VLEN*2) and
2626 // MASK = WRAPED_INDEX < VLEN
2627 //
2628 // IR lowering prevents intrinsification failure and associated argument
2629 // boxing penalties.
2630 //
2631
2632 BasicType shuffle_bt = elem_bt;
2633 if (shuffle_bt == T_FLOAT) {
2634 shuffle_bt = T_INT;
2635 } else if (shuffle_bt == T_DOUBLE) {
2636 shuffle_bt = T_LONG;
2637 }
2638 const TypeVect* st = TypeVect::make(shuffle_bt, num_elem);
2639
2640 // Cast index vector to the corresponding bit type
2641 if (elem_bt != shuffle_bt) {
2642 int cast_vopc = VectorCastNode::opcode(0, elem_bt, true);
2643 index_vec = phase.transform(VectorCastNode::make(cast_vopc, index_vec, shuffle_bt, num_elem));
2644 }
2645
2646 // Wrap indexes into two vector index range [0, VLEN * 2)
2647 Node* two_vect_lane_cnt_m1 = phase.makecon(TypeInteger::make(2 * num_elem - 1, 2 * num_elem - 1, Type::WidenMin, shuffle_bt == T_LONG ? T_LONG : T_INT));
2648 Node* bcast_two_vect_lane_cnt_m1_vec = phase.transform(VectorNode::scalar2vector(two_vect_lane_cnt_m1, num_elem,
2649 shuffle_bt, false));
2650 index_vec = phase.transform(VectorNode::make(Op_AndV, index_vec, bcast_two_vect_lane_cnt_m1_vec, st));
2651
2652 // Compute the blend mask for merging two independently permitted vectors
2653 // using shuffle index in two vector index range [0, VLEN * 2).
2654 BoolTest::mask pred = BoolTest::le;
2655 ConINode* pred_node = phase.makecon(TypeInt::make(pred))->as_ConI();
2656 const TypeVect* vmask_type = TypeVect::makemask(shuffle_bt, num_elem);
2657 Node* lane_cnt_m1 = phase.makecon(TypeInteger::make(num_elem - 1, num_elem - 1, Type::WidenMin, shuffle_bt == T_LONG ? T_LONG : T_INT));
2658 Node* bcast_lane_cnt_m1_vec = phase.transform(VectorNode::scalar2vector(lane_cnt_m1, num_elem, shuffle_bt, false));
2659 Node* mask = phase.transform(new VectorMaskCmpNode(pred, index_vec, bcast_lane_cnt_m1_vec, pred_node, vmask_type));
2660
2661 // Rearrange expects the indexes to lie within single vector index range [0, VLEN).
2662 Node* wrapped_index_vec = phase.transform(VectorNode::make(Op_AndV, index_vec, bcast_lane_cnt_m1_vec, st));
2663
2664 // Load indexes from byte vector and appropriately transform them to target
2665 // specific permutation index format.
2666 if (Matcher::vector_rearrange_requires_load_shuffle(shuffle_bt, num_elem)) {
2667 wrapped_index_vec = phase.transform(new VectorLoadShuffleNode(wrapped_index_vec, st));
2668 }
2669
2670 vmask_type = TypeVect::makemask(elem_bt, num_elem);
2671 mask = phase.transform(new VectorMaskCastNode(mask, vmask_type));
2672
2673 Node* p1 = phase.transform(new VectorRearrangeNode(src1, wrapped_index_vec));
2674 Node* p2 = phase.transform(new VectorRearrangeNode(src2, wrapped_index_vec));
2675
2676 return new VectorBlendNode(p2, p1, mask);
2677 }
2678
2679 // public static
2680 // <V extends Vector<E>,
2681 // E>
2682 // V selectFromTwoVectorOp(Class<? extends V> vClass, Class<E> eClass, int length,
2683 // V v1, V v2, V v3,
2684 // SelectFromTwoVector<V> defaultImpl)
2685 bool LibraryCallKit::inline_vector_select_from_two_vectors() {
2686 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
2687 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
2688 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
2689
2690 if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr || vector_klass->const_oop() == nullptr ||
2691 elem_klass->const_oop() == nullptr ||!vlen->is_con()) {
2692 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s",
2693 NodeClassNames[argument(0)->Opcode()],
2694 NodeClassNames[argument(1)->Opcode()],
2695 NodeClassNames[argument(2)->Opcode()]);
2696 return false; // not enough info for intrinsification
2697 }
2698
2699 if (!is_klass_initialized(vector_klass)) {
2700 log_if_needed(" ** klass argument not initialized");
2701 return false;
2702 }
2703
2704 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2705 if (!elem_type->is_primitive_type()) {
2706 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2707 return false; // should be primitive type
2708 }
2709
2710 int num_elem = vlen->get_con();
2711 if (!is_power_of_2(num_elem)) {
2712 log_if_needed(" ** vlen is not power of two=%d", num_elem);
2713 return false;
2714 }
2715
2716 BasicType elem_bt = elem_type->basic_type();
2717 BasicType index_elem_bt = elem_bt;
2718 if (elem_bt == T_FLOAT) {
2719 index_elem_bt = T_INT;
2720 } else if (elem_bt == T_DOUBLE) {
2721 index_elem_bt = T_LONG;
2722 }
2723
2724 bool lowerSelectFromOp = false;
2725 if (!arch_supports_vector(Op_SelectFromTwoVector, num_elem, elem_bt, VecMaskNotUsed)) {
2726 int cast_vopc = VectorCastNode::opcode(-1, elem_bt, true);
2727 if ((elem_bt != index_elem_bt && !arch_supports_vector(cast_vopc, num_elem, index_elem_bt, VecMaskNotUsed)) ||
2728 !arch_supports_vector(Op_VectorMaskCmp, num_elem, index_elem_bt, VecMaskNotUsed) ||
2729 !arch_supports_vector(Op_AndV, num_elem, index_elem_bt, VecMaskNotUsed) ||
2730 !arch_supports_vector(Op_VectorMaskCast, num_elem, elem_bt, VecMaskNotUsed) ||
2731 !arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad) ||
2732 !arch_supports_vector(Op_VectorRearrange, num_elem, elem_bt, VecMaskNotUsed) ||
2733 !arch_supports_vector(Op_VectorLoadShuffle, num_elem, index_elem_bt, VecMaskNotUsed) ||
2734 !arch_supports_vector(Op_Replicate, num_elem, index_elem_bt, VecMaskNotUsed)) {
2735 log_if_needed(" ** not supported: opc=%d vlen=%d etype=%s ismask=useload",
2736 Op_SelectFromTwoVector, num_elem, type2name(elem_bt));
2737 return false; // not supported
2738 }
2739 lowerSelectFromOp = true;
2740 }
2741
2742 int cast_vopc = VectorCastNode::opcode(-1, elem_bt, true);
2743 if (!lowerSelectFromOp) {
2744 if (!arch_supports_vector(Op_AndV, num_elem, index_elem_bt, VecMaskNotUsed) ||
2745 !arch_supports_vector(Op_Replicate, num_elem, index_elem_bt, VecMaskNotUsed) ||
2746 (is_floating_point_type(elem_bt) &&
2747 !arch_supports_vector(cast_vopc, num_elem, index_elem_bt, VecMaskNotUsed))) {
2748 log_if_needed(" ** index wrapping not supported: vlen=%d etype=%s" ,
2749 num_elem, type2name(elem_bt));
2750 return false; // not supported
2751 }
2752 }
2753
2754 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2755 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2756
2757 Node* opd1 = unbox_vector(argument(3), vbox_type, elem_bt, num_elem);
2758 if (opd1 == nullptr) {
2759 log_if_needed(" ** unbox failed v1=%s",
2760 NodeClassNames[argument(3)->Opcode()]);
2761 return false;
2762 }
2763 Node* opd2 = unbox_vector(argument(4), vbox_type, elem_bt, num_elem);
2764 if (opd2 == nullptr) {
2765 log_if_needed(" ** unbox failed v2=%s",
2766 NodeClassNames[argument(4)->Opcode()]);
2767 return false;
2768 }
2769 Node* opd3 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
2770 if (opd3 == nullptr) {
2771 log_if_needed(" ** unbox failed v3=%s",
2772 NodeClassNames[argument(5)->Opcode()]);
2773 return false;
2774 }
2775
2776 const TypeVect* vt = TypeVect::make(elem_bt, num_elem);
2777
2778 Node* operation = nullptr;
2779 if (lowerSelectFromOp) {
2780 operation = gvn().transform(LowerSelectFromTwoVectorOperation(gvn(), opd1, opd2, opd3, vt));
2781 } else {
2782 if (index_elem_bt != elem_bt) {
2783 opd1 = gvn().transform(VectorCastNode::make(cast_vopc, opd1, index_elem_bt, num_elem));
2784 }
2785 int indexRangeMask = 2 * num_elem - 1;
2786 Node* wrap_mask = gvn().makecon(TypeInteger::make(indexRangeMask, indexRangeMask, Type::WidenMin, index_elem_bt != T_LONG ? T_INT : index_elem_bt));
2787 Node* wrap_mask_vec = gvn().transform(VectorNode::scalar2vector(wrap_mask, num_elem, index_elem_bt, false));
2788 opd1 = gvn().transform(VectorNode::make(Op_AndV, opd1, wrap_mask_vec, opd1->bottom_type()->is_vect()));
2789 operation = gvn().transform(VectorNode::make(Op_SelectFromTwoVector, opd1, opd2, opd3, vt));
2790 }
2791
2792 // Wrap it up in VectorBox to keep object type information.
2793 Node* vbox = box_vector(operation, vbox_type, elem_bt, num_elem);
2794 set_result(vbox);
2795 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2796 return true;
2797 }
2798
2799 // public static
2800 // <V extends Vector<E>,
2801 // M extends VectorMask<E>,
2802 // E>
2803 // V compressExpandOp(int opr,
2804 // Class<? extends V> vClass, Class<? extends M> mClass, Class<E> eClass,
2805 // int length, V v, M m,
2806 // CompressExpandOperation<V, M> defaultImpl)
2807 bool LibraryCallKit::inline_vector_compress_expand() {
2808 const TypeInt* opr = gvn().type(argument(0))->isa_int();
2809 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
2810 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
2811 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
2812 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
2813
2814 if (vector_klass == nullptr || elem_klass == nullptr || mask_klass == nullptr || vlen == nullptr ||
2815 vector_klass->const_oop() == nullptr || mask_klass->const_oop() == nullptr ||
2816 elem_klass->const_oop() == nullptr || !vlen->is_con() || !opr->is_con()) {
2817 log_if_needed(" ** missing constant: opr=%s vclass=%s mclass=%s etype=%s vlen=%s",
2818 NodeClassNames[argument(0)->Opcode()],
2819 NodeClassNames[argument(1)->Opcode()],
2820 NodeClassNames[argument(2)->Opcode()],
2821 NodeClassNames[argument(3)->Opcode()],
2822 NodeClassNames[argument(4)->Opcode()]);
2823 return false; // not enough info for intrinsification
2824 }
2825
2826 if (!is_klass_initialized(vector_klass) || !is_klass_initialized(mask_klass)) {
2827 log_if_needed(" ** klass argument not initialized");
2828 return false;
2829 }
2830
2831 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2832 if (!elem_type->is_primitive_type()) {
2833 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2834 return false; // should be primitive type
2835 }
2836
2837 int num_elem = vlen->get_con();
2838 BasicType elem_bt = elem_type->basic_type();
2839 int opc = VectorSupport::vop2ideal(opr->get_con(), elem_bt);
2840
2841 if (!arch_supports_vector(opc, num_elem, elem_bt, VecMaskUseLoad)) {
2842 log_if_needed(" ** not supported: opc=%d vlen=%d etype=%s ismask=useload",
2843 opc, num_elem, type2name(elem_bt));
2844 return false; // not supported
2845 }
2846
2847 Node* opd1 = nullptr;
2848 const TypeInstPtr* vbox_type = nullptr;
2849 if (opc != Op_CompressM) {
2850 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2851 vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2852 opd1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
2853 if (opd1 == nullptr) {
2854 log_if_needed(" ** unbox failed vector=%s",
2855 NodeClassNames[argument(5)->Opcode()]);
2856 return false;
2857 }
2858 }
2859
2860 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
2861 assert(is_vector_mask(mbox_klass), "argument(6) should be a mask class");
2862 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
2863
2864 Node* mask = unbox_vector(argument(6), mbox_type, elem_bt, num_elem);
2865 if (mask == nullptr) {
2866 log_if_needed(" ** unbox failed mask=%s",
2867 NodeClassNames[argument(6)->Opcode()]);
2868 return false;
2869 }
2870
2871 const TypeVect* vt = TypeVect::make(elem_bt, num_elem, opc == Op_CompressM);
2872 Node* operation = gvn().transform(VectorNode::make(opc, opd1, mask, vt));
2873
2874 // Wrap it up in VectorBox to keep object type information.
2875 const TypeInstPtr* box_type = opc == Op_CompressM ? mbox_type : vbox_type;
2876 Node* vbox = box_vector(operation, box_type, elem_bt, num_elem);
2877 set_result(vbox);
2878 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2879 return true;
2880 }
2881
2882 // public static
2883 // <V extends Vector<E>,
2884 // E,
2885 // S extends VectorSpecies<E>>
2886 // V indexVector(Class<? extends V> vClass, Class<E> eClass,
2887 // int length,
2888 // V v, int step, S s,
2889 // IndexOperation<V, S> defaultImpl)
2890 bool LibraryCallKit::inline_index_vector() {
2891 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
2892 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
2893 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
2894
2895 if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr ||
2896 vector_klass->const_oop() == nullptr || !vlen->is_con() ||
2897 elem_klass->const_oop() == nullptr) {
2898 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s",
2899 NodeClassNames[argument(0)->Opcode()],
2900 NodeClassNames[argument(1)->Opcode()],
2901 NodeClassNames[argument(2)->Opcode()]);
2902 return false; // not enough info for intrinsification
2903 }
2904
2905 if (!is_klass_initialized(vector_klass)) {
2906 log_if_needed(" ** klass argument not initialized");
2907 return false;
2908 }
2909
2910 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2911 if (!elem_type->is_primitive_type()) {
2912 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2913 return false; // should be primitive type
2914 }
2915
2916 int num_elem = vlen->get_con();
2917 BasicType elem_bt = elem_type->basic_type();
2918
2919 // Check whether the iota index generation op is supported by the current hardware
2920 if (!arch_supports_vector(Op_VectorLoadConst, num_elem, elem_bt, VecMaskNotUsed)) {
2921 log_if_needed(" ** not supported: vlen=%d etype=%s", num_elem, type2name(elem_bt));
2922 return false; // not supported
2923 }
2924
2925 int mul_op = VectorSupport::vop2ideal(VectorSupport::VECTOR_OP_MUL, elem_bt);
2926 int vmul_op = VectorNode::opcode(mul_op, elem_bt);
2927 bool needs_mul = true;
2928 Node* scale = argument(4);
2929 const TypeInt* scale_type = gvn().type(scale)->isa_int();
2930 // Multiply is not needed if the scale is a constant "1".
2931 if (scale_type && scale_type->is_con() && scale_type->get_con() == 1) {
2932 needs_mul = false;
2933 } else {
2934 // Check whether the vector multiply op is supported by the current hardware
2935 if (!arch_supports_vector(vmul_op, num_elem, elem_bt, VecMaskNotUsed)) {
2936 log_if_needed(" ** not supported: vlen=%d etype=%s", num_elem, type2name(elem_bt));
2937 return false; // not supported
2938 }
2939
2940 // Check whether the scalar cast op is supported by the current hardware
2941 if (is_floating_point_type(elem_bt) || elem_bt == T_LONG) {
2942 int cast_op = elem_bt == T_LONG ? Op_ConvI2L :
2943 elem_bt == T_FLOAT? Op_ConvI2F : Op_ConvI2D;
2944 if (!Matcher::match_rule_supported(cast_op)) {
2945 log_if_needed(" ** Rejected op (%s) because architecture does not support it",
2946 NodeClassNames[cast_op]);
2947 return false; // not supported
2948 }
2949 }
2950 }
2951
2952 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2953 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2954 Node* opd = unbox_vector(argument(3), vbox_type, elem_bt, num_elem);
2955 if (opd == nullptr) {
2956 log_if_needed(" ** unbox failed vector=%s",
2957 NodeClassNames[argument(3)->Opcode()]);
2958 return false;
2959 }
2960
2961 int add_op = VectorSupport::vop2ideal(VectorSupport::VECTOR_OP_ADD, elem_bt);
2962 int vadd_op = VectorNode::opcode(add_op, elem_bt);
2963 bool needs_add = true;
2964 // The addition is not needed if all the element values of "opd" are zero
2965 if (VectorNode::is_all_zeros_vector(opd)) {
2966 needs_add = false;
2967 } else {
2968 // Check whether the vector addition op is supported by the current hardware
2969 if (!arch_supports_vector(vadd_op, num_elem, elem_bt, VecMaskNotUsed)) {
2970 log_if_needed(" ** not supported: vlen=%d etype=%s", num_elem, type2name(elem_bt));
2971 return false; // not supported
2972 }
2973 }
2974
2975 // Compute the iota indice vector
2976 const TypeVect* vt = TypeVect::make(elem_bt, num_elem);
2977 Node* index = gvn().transform(new VectorLoadConstNode(gvn().makecon(TypeInt::ZERO), vt));
2978
2979 // Broadcast the "scale" to a vector, and multiply the "scale" with iota indice vector.
2980 if (needs_mul) {
2981 switch (elem_bt) {
2982 case T_BOOLEAN: // fall-through
2983 case T_BYTE: // fall-through
2984 case T_SHORT: // fall-through
2985 case T_CHAR: // fall-through
2986 case T_INT: {
2987 // no conversion needed
2988 break;
2989 }
2990 case T_LONG: {
2991 scale = gvn().transform(new ConvI2LNode(scale));
2992 break;
2993 }
2994 case T_FLOAT: {
2995 scale = gvn().transform(new ConvI2FNode(scale));
2996 break;
2997 }
2998 case T_DOUBLE: {
2999 scale = gvn().transform(new ConvI2DNode(scale));
3000 break;
3001 }
3002 default: fatal("%s", type2name(elem_bt));
3003 }
3004 scale = gvn().transform(VectorNode::scalar2vector(scale, num_elem, elem_bt));
3005 index = gvn().transform(VectorNode::make(vmul_op, index, scale, vt));
3006 }
3007
3008 // Add "opd" if addition is needed.
3009 if (needs_add) {
3010 index = gvn().transform(VectorNode::make(vadd_op, opd, index, vt));
3011 }
3012 Node* vbox = box_vector(index, vbox_type, elem_bt, num_elem);
3013 set_result(vbox);
3014 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
3015 return true;
3016 }
3017
3018 // public static
3019 // <E,
3020 // M extends VectorMask<E>>
3021 // M indexPartiallyInUpperRange(Class<? extends M> mClass, Class<E> eClass, int length,
3022 // long offset, long limit,
3023 // IndexPartiallyInUpperRangeOperation<E, M> defaultImpl)
3024 bool LibraryCallKit::inline_index_partially_in_upper_range() {
3025 const TypeInstPtr* mask_klass = gvn().type(argument(0))->isa_instptr();
3026 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
3027 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
3028
3029 if (mask_klass == nullptr || elem_klass == nullptr || vlen == nullptr ||
3030 mask_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
3031 log_if_needed(" ** missing constant: mclass=%s etype=%s vlen=%s",
3032 NodeClassNames[argument(0)->Opcode()],
3033 NodeClassNames[argument(1)->Opcode()],
3034 NodeClassNames[argument(2)->Opcode()]);
3035 return false; // not enough info for intrinsification
3036 }
3037
3038 if (!is_klass_initialized(mask_klass)) {
3039 log_if_needed(" ** klass argument not initialized");
3040 return false;
3041 }
3042
3043 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
3044 if (!elem_type->is_primitive_type()) {
3045 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
3046 return false; // should be primitive type
3047 }
3048
3049 int num_elem = vlen->get_con();
3050 BasicType elem_bt = elem_type->basic_type();
3051
3052 // Check whether the necessary ops are supported by current hardware.
3053 bool supports_mask_gen = arch_supports_vector(Op_VectorMaskGen, num_elem, elem_bt, VecMaskUseStore);
3054 if (!supports_mask_gen) {
3055 if (!arch_supports_vector(Op_VectorLoadConst, num_elem, elem_bt, VecMaskNotUsed) ||
3056 !arch_supports_vector(Op_Replicate, num_elem, elem_bt, VecMaskNotUsed) ||
3057 !arch_supports_vector(Op_VectorMaskCmp, num_elem, elem_bt, VecMaskUseStore)) {
3058 log_if_needed(" ** not supported: vlen=%d etype=%s", num_elem, type2name(elem_bt));
3059 return false; // not supported
3060 }
3061
3062 // Check whether the scalar cast operation is supported by current hardware.
3063 if (elem_bt != T_LONG) {
3064 int cast_op = is_integral_type(elem_bt) ? Op_ConvL2I
3065 : (elem_bt == T_FLOAT ? Op_ConvL2F : Op_ConvL2D);
3066 if (!Matcher::match_rule_supported(cast_op)) {
3067 log_if_needed(" ** Rejected op (%s) because architecture does not support it",
3068 NodeClassNames[cast_op]);
3069 return false; // not supported
3070 }
3071 }
3072 }
3073
3074 Node* offset = argument(3);
3075 Node* limit = argument(5);
3076 if (offset == nullptr || limit == nullptr) {
3077 log_if_needed(" ** offset or limit argument is null");
3078 return false; // not supported
3079 }
3080
3081 ciKlass* box_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
3082 assert(is_vector_mask(box_klass), "argument(0) should be a mask class");
3083 const TypeInstPtr* box_type = TypeInstPtr::make_exact(TypePtr::NotNull, box_klass);
3084
3085 // We assume "offset > 0 && limit >= offset && limit - offset < num_elem".
3086 // So directly get indexLimit with "indexLimit = limit - offset".
3087 Node* indexLimit = gvn().transform(new SubLNode(limit, offset));
3088 Node* mask = nullptr;
3089 if (supports_mask_gen) {
3090 mask = gvn().transform(VectorMaskGenNode::make(indexLimit, elem_bt, num_elem));
3091 } else {
3092 // Generate the vector mask based on "mask = iota < indexLimit".
3093 // Broadcast "indexLimit" to a vector.
3094 switch (elem_bt) {
3095 case T_BOOLEAN: // fall-through
3096 case T_BYTE: // fall-through
3097 case T_SHORT: // fall-through
3098 case T_CHAR: // fall-through
3099 case T_INT: {
3100 indexLimit = gvn().transform(new ConvL2INode(indexLimit));
3101 break;
3102 }
3103 case T_DOUBLE: {
3104 indexLimit = gvn().transform(new ConvL2DNode(indexLimit));
3105 break;
3106 }
3107 case T_FLOAT: {
3108 indexLimit = gvn().transform(new ConvL2FNode(indexLimit));
3109 break;
3110 }
3111 case T_LONG: {
3112 // no conversion needed
3113 break;
3114 }
3115 default: fatal("%s", type2name(elem_bt));
3116 }
3117 indexLimit = gvn().transform(VectorNode::scalar2vector(indexLimit, num_elem, elem_bt));
3118
3119 // Load the "iota" vector.
3120 const TypeVect* vt = TypeVect::make(elem_bt, num_elem);
3121 Node* iota = gvn().transform(new VectorLoadConstNode(gvn().makecon(TypeInt::ZERO), vt));
3122
3123 // Compute the vector mask with "mask = iota < indexLimit".
3124 ConINode* pred_node = (ConINode*)gvn().makecon(TypeInt::make(BoolTest::lt));
3125 const TypeVect* vmask_type = TypeVect::makemask(elem_bt, num_elem);
3126 mask = gvn().transform(new VectorMaskCmpNode(BoolTest::lt, iota, indexLimit, pred_node, vmask_type));
3127 }
3128 Node* vbox = box_vector(mask, box_type, elem_bt, num_elem);
3129 set_result(vbox);
3130 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
3131 return true;
3132 }
3133
3134 #undef non_product_log_if_needed
3135 #undef log_if_needed