1 /*
2 * Copyright (c) 2020, 2026, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "ci/ciSymbols.hpp"
26 #include "classfile/vmSymbols.hpp"
27 #include "opto/library_call.hpp"
28 #include "opto/rootnode.hpp"
29 #include "opto/runtime.hpp"
30 #include "opto/vectornode.hpp"
31 #include "prims/vectorSupport.hpp"
32 #include "runtime/stubRoutines.hpp"
33
34 #ifdef ASSERT
35 static bool is_vector(ciKlass* klass) {
36 return klass->is_subclass_of(ciEnv::current()->vector_VectorPayload_klass());
37 }
38
39 static bool check_vbox(const TypeInstPtr* vbox_type) {
40 assert(vbox_type->klass_is_exact(), "");
41
42 ciInstanceKlass* ik = vbox_type->instance_klass();
43 assert(is_vector(ik), "not a vector");
44
45 ciField* fd1 = ik->get_field_by_name(ciSymbols::ETYPE_name(), ciSymbols::class_signature(), /* is_static */ true);
46 assert(fd1 != nullptr, "element type info is missing");
47
48 ciConstant val1 = fd1->constant_value();
49 BasicType elem_bt = val1.as_object()->as_instance()->java_mirror_type()->basic_type();
50 assert(is_java_primitive(elem_bt), "element type info is missing");
51
52 ciField* fd2 = ik->get_field_by_name(ciSymbols::VLENGTH_name(), ciSymbols::int_signature(), /* is_static */ true);
53 assert(fd2 != nullptr, "vector length info is missing");
54
55 ciConstant val2 = fd2->constant_value();
56 assert(val2.as_int() > 0, "vector length info is missing");
57
58 return true;
59 }
60 #endif
61
62 #define log_if_needed(...) \
63 if (C->print_intrinsics()) { \
64 tty->print_cr(__VA_ARGS__); \
65 }
66
67 #ifndef PRODUCT
68 #define non_product_log_if_needed(...) log_if_needed(__VA_ARGS__)
69 #else
70 #define non_product_log_if_needed(...)
71 #endif
72
73 static bool is_vector_mask(ciKlass* klass) {
74 return klass->is_subclass_of(ciEnv::current()->vector_VectorMask_klass());
75 }
76
77 static Node* trace_vector(Node* operation) {
78 VectorNode::trace_new_vector(operation, "VectorAPI");
79 return operation;
80 }
81
82 bool LibraryCallKit::arch_supports_vector_rotate(int opc, int num_elem, BasicType elem_bt,
83 VectorMaskUseType mask_use_type, bool has_scalar_args) {
84 bool is_supported = true;
85
86 // has_scalar_args flag is true only for non-constant scalar shift count,
87 // since in this case shift needs to be broadcasted.
88 if (!Matcher::match_rule_supported_vector(opc, num_elem, elem_bt) ||
89 (has_scalar_args && !arch_supports_vector(Op_Replicate, num_elem, elem_bt, VecMaskNotUsed))) {
90 is_supported = false;
91 }
92
93 if (is_supported) {
94 // Check if mask unboxing is supported, this is a two step process which first loads the contents
95 // of boolean array into vector followed by either lane expansion to match the lane size of masked
96 // vector operation or populate the predicate register.
97 if ((mask_use_type & VecMaskUseLoad) != 0) {
98 if (!Matcher::match_rule_supported_vector(Op_VectorLoadMask, num_elem, elem_bt) ||
99 !Matcher::match_rule_supported_vector(Op_LoadVector, num_elem, T_BOOLEAN)) {
100 non_product_log_if_needed(" ** Rejected vector mask loading (%s,%s,%d) because architecture does not support it",
101 NodeClassNames[Op_VectorLoadMask], type2name(elem_bt), num_elem);
102 return false;
103 }
104 }
105
106 if ((mask_use_type & VecMaskUsePred) != 0) {
107 if (!Matcher::has_predicated_vectors() ||
108 !Matcher::match_rule_supported_vector_masked(opc, num_elem, elem_bt)) {
109 non_product_log_if_needed("Rejected vector mask predicate using (%s,%s,%d) because architecture does not support it",
110 NodeClassNames[opc], type2name(elem_bt), num_elem);
111 return false;
112 }
113 }
114 }
115
116 int lshiftopc, rshiftopc;
117 switch(elem_bt) {
118 case T_BYTE:
119 lshiftopc = Op_LShiftI;
120 rshiftopc = Op_URShiftB;
121 break;
122 case T_SHORT:
123 lshiftopc = Op_LShiftI;
124 rshiftopc = Op_URShiftS;
125 break;
126 case T_INT:
127 lshiftopc = Op_LShiftI;
128 rshiftopc = Op_URShiftI;
129 break;
130 case T_LONG:
131 lshiftopc = Op_LShiftL;
132 rshiftopc = Op_URShiftL;
133 break;
134 default: fatal("Unexpected type: %s", type2name(elem_bt));
135 }
136 int lshiftvopc = VectorNode::opcode(lshiftopc, elem_bt);
137 int rshiftvopc = VectorNode::opcode(rshiftopc, elem_bt);
138 if (!is_supported &&
139 arch_supports_vector(lshiftvopc, num_elem, elem_bt, VecMaskNotUsed, has_scalar_args) &&
140 arch_supports_vector(rshiftvopc, num_elem, elem_bt, VecMaskNotUsed, has_scalar_args) &&
141 arch_supports_vector(Op_OrV, num_elem, elem_bt, VecMaskNotUsed)) {
142 is_supported = true;
143 }
144 return is_supported;
145 }
146
147 Node* GraphKit::box_vector(Node* vector, const TypeInstPtr* vbox_type, BasicType elem_bt, int num_elem, bool deoptimize_on_exception) {
148 assert(EnableVectorSupport, "");
149
150 PreserveReexecuteState preexecs(this);
151 jvms()->set_should_reexecute(true);
152
153 VectorBoxAllocateNode* alloc = new VectorBoxAllocateNode(C, vbox_type);
154 set_edges_for_java_call(alloc, /*must_throw=*/false, /*separate_io_proj=*/true);
155 make_slow_call_ex(alloc, env()->Throwable_klass(), /*separate_io_proj=*/true, deoptimize_on_exception);
156 set_i_o(gvn().transform( new ProjNode(alloc, TypeFunc::I_O) ));
157 set_all_memory(gvn().transform( new ProjNode(alloc, TypeFunc::Memory) ));
158 Node* ret = gvn().transform(new ProjNode(alloc, TypeFunc::Parms));
159
160 assert(check_vbox(vbox_type), "");
161 const TypeVect* vt = TypeVect::make(elem_bt, num_elem, is_vector_mask(vbox_type->instance_klass()));
162 VectorBoxNode* vbox = new VectorBoxNode(C, ret, vector, vbox_type, vt);
163 return gvn().transform(vbox);
164 }
165
166 Node* GraphKit::unbox_vector(Node* v, const TypeInstPtr* vbox_type, BasicType elem_bt, int num_elem) {
167 assert(EnableVectorSupport, "");
168 const TypeInstPtr* vbox_type_v = gvn().type(v)->isa_instptr();
169 if (vbox_type_v == nullptr || vbox_type->instance_klass() != vbox_type_v->instance_klass()) {
170 return nullptr; // arguments don't agree on vector shapes
171 }
172 if (vbox_type_v->maybe_null()) {
173 return nullptr; // no nulls are allowed
174 }
175 assert(check_vbox(vbox_type), "");
176 const TypeVect* vt = TypeVect::make(elem_bt, num_elem, is_vector_mask(vbox_type->instance_klass()));
177 Node* unbox = gvn().transform(new VectorUnboxNode(C, vt, v, merged_memory()));
178 if (gvn().type(unbox)->isa_vect() == nullptr) {
179 assert(gvn().type(unbox) == Type::TOP, "sanity");
180 return nullptr; // not a vector
181 }
182 return unbox;
183 }
184
185 Node* GraphKit::vector_shift_count(Node* cnt, int shift_op, BasicType bt, int num_elem) {
186 assert(bt == T_INT || bt == T_LONG || bt == T_SHORT || bt == T_BYTE, "byte, short, long and int are supported");
187 juint mask = (type2aelembytes(bt) * BitsPerByte - 1);
188 Node* nmask = gvn().transform(ConNode::make(TypeInt::make(mask)));
189 Node* mcnt = gvn().transform(new AndINode(cnt, nmask));
190 return gvn().transform(VectorNode::shift_count(shift_op, mcnt, num_elem, bt));
191 }
192
193 bool LibraryCallKit::arch_supports_vector(int sopc, int num_elem, BasicType type, VectorMaskUseType mask_use_type, bool has_scalar_args) {
194 // Check that the operation is valid.
195 if (sopc <= 0) {
196 non_product_log_if_needed(" ** Rejected intrinsification because no valid vector op could be extracted");
197 return false;
198 }
199
200 if (VectorNode::is_vector_rotate(sopc)) {
201 if(!arch_supports_vector_rotate(sopc, num_elem, type, mask_use_type, has_scalar_args)) {
202 non_product_log_if_needed(" ** Rejected vector op (%s,%s,%d) because architecture does not support variable vector shifts",
203 NodeClassNames[sopc], type2name(type), num_elem);
204 return false;
205 }
206 } else if (VectorNode::is_vector_integral_negate(sopc)) {
207 if (!VectorNode::is_vector_integral_negate_supported(sopc, num_elem, type, false)) {
208 non_product_log_if_needed(" ** Rejected vector op (%s,%s,%d) because architecture does not support integral vector negate",
209 NodeClassNames[sopc], type2name(type), num_elem);
210 return false;
211 }
212 } else {
213 // Check that architecture supports this op-size-type combination.
214 if (!Matcher::match_rule_supported_vector(sopc, num_elem, type)) {
215 non_product_log_if_needed(" ** Rejected vector op (%s,%s,%d) because architecture does not support it",
216 NodeClassNames[sopc], type2name(type), num_elem);
217 return false;
218 } else {
219 assert(Matcher::match_rule_supported(sopc), "must be supported");
220 }
221 }
222
223 if (num_elem == 1) {
224 if (mask_use_type != VecMaskNotUsed) {
225 non_product_log_if_needed(" ** Rejected vector mask op (%s,%s,%d) because architecture does not support it",
226 NodeClassNames[sopc], type2name(type), num_elem);
227 return false;
228 }
229
230 if (sopc != 0) {
231 if (sopc != Op_LoadVector && sopc != Op_StoreVector) {
232 non_product_log_if_needed(" ** Not a svml call or load/store vector op (%s,%s,%d)",
233 NodeClassNames[sopc], type2name(type), num_elem);
234 return false;
235 }
236 }
237 }
238
239 if (!has_scalar_args && VectorNode::is_vector_shift(sopc) &&
240 Matcher::supports_vector_variable_shifts() == false) {
241 log_if_needed(" ** Rejected vector op (%s,%s,%d) because architecture does not support variable vector shifts",
242 NodeClassNames[sopc], type2name(type), num_elem);
243 return false;
244 }
245
246 // Check if mask unboxing is supported, this is a two step process which first loads the contents
247 // of boolean array into vector followed by either lane expansion to match the lane size of masked
248 // vector operation or populate the predicate register.
249 if ((mask_use_type & VecMaskUseLoad) != 0) {
250 if (!Matcher::match_rule_supported_vector(Op_VectorLoadMask, num_elem, type) ||
251 !Matcher::match_rule_supported_vector(Op_LoadVector, num_elem, T_BOOLEAN)) {
252 non_product_log_if_needed(" ** Rejected vector mask loading (%s,%s,%d) because architecture does not support it",
253 NodeClassNames[Op_VectorLoadMask], type2name(type), num_elem);
254 return false;
255 }
256 }
257
258 // Check if mask boxing is supported, this is a two step process which first stores the contents
259 // of mask vector / predicate register into a boolean vector followed by vector store operation to
260 // transfer the contents to underlined storage of mask boxes which is a boolean array.
261 if ((mask_use_type & VecMaskUseStore) != 0) {
262 if (!Matcher::match_rule_supported_vector(Op_VectorStoreMask, num_elem, type) ||
263 !Matcher::match_rule_supported_vector(Op_StoreVector, num_elem, T_BOOLEAN)) {
264 non_product_log_if_needed("Rejected vector mask storing (%s,%s,%d) because architecture does not support it",
265 NodeClassNames[Op_VectorStoreMask], type2name(type), num_elem);
266 return false;
267 }
268 }
269
270 if ((mask_use_type & VecMaskUsePred) != 0) {
271 bool is_supported = false;
272 if (Matcher::has_predicated_vectors()) {
273 if (VectorNode::is_vector_integral_negate(sopc)) {
274 is_supported = VectorNode::is_vector_integral_negate_supported(sopc, num_elem, type, true);
275 } else {
276 is_supported = Matcher::match_rule_supported_vector_masked(sopc, num_elem, type);
277 }
278 }
279 is_supported |= Matcher::supports_vector_predicate_op_emulation(sopc, num_elem, type);
280
281 if (!is_supported) {
282 non_product_log_if_needed("Rejected vector mask predicate using (%s,%s,%d) because architecture does not support it",
283 NodeClassNames[sopc], type2name(type), num_elem);
284 return false;
285 }
286 }
287
288 return true;
289 }
290
291 static bool is_klass_initialized(const TypeInstPtr* vec_klass) {
292 if (vec_klass->const_oop() == nullptr) {
293 return false; // uninitialized or some kind of unsafe access
294 }
295 assert(vec_klass->const_oop()->as_instance()->java_lang_Class_klass() != nullptr, "klass instance expected");
296 ciInstanceKlass* klass = vec_klass->const_oop()->as_instance()->java_lang_Class_klass()->as_instance_klass();
297 return klass->is_initialized();
298 }
299
300 // public static
301 // <V extends Vector<E>,
302 // M extends VectorMask<E>,
303 // E>
304 // V unaryOp(int oprId, Class<? extends V> vmClass, Class<? extends M> maskClass, Class<E> elementType,
305 // int length, V v, M m,
306 // UnaryOperation<V, M> defaultImpl)
307 //
308 // public static
309 // <V,
310 // M extends VectorMask<E>,
311 // E>
312 // V binaryOp(int oprId, Class<? extends V> vmClass, Class<? extends M> maskClass, Class<E> elementType,
313 // int length, V v1, V v2, M m,
314 // BinaryOperation<V, M> defaultImpl)
315 //
316 // public static
317 // <V extends Vector<E>,
318 // M extends VectorMask<E>,
319 // E>
320 // V ternaryOp(int oprId, Class<? extends V> vmClass, Class<? extends M> maskClass, Class<E> elementType,
321 // int length, V v1, V v2, V v3, M m,
322 // TernaryOperation<V, M> defaultImpl)
323 //
324 bool LibraryCallKit::inline_vector_nary_operation(int n) {
325 const TypeInt* opr = gvn().type(argument(0))->isa_int();
326 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
327 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
328 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
329 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
330
331 if (opr == nullptr || !opr->is_con() ||
332 vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
333 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
334 vlen == nullptr || !vlen->is_con()) {
335 log_if_needed(" ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
336 NodeClassNames[argument(0)->Opcode()],
337 NodeClassNames[argument(1)->Opcode()],
338 NodeClassNames[argument(3)->Opcode()],
339 NodeClassNames[argument(4)->Opcode()]);
340 return false; // not enough info for intrinsification
341 }
342
343 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
344 if (!elem_type->is_primitive_type()) {
345 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
346 return false; // should be primitive type
347 }
348 if (!is_klass_initialized(vector_klass)) {
349 log_if_needed(" ** klass argument not initialized");
350 return false;
351 }
352
353 // "argument(n + 5)" should be the mask object. We assume it is "null" when no mask
354 // is used to control this operation.
355 const Type* vmask_type = gvn().type(argument(n + 5));
356 bool is_masked_op = vmask_type != TypePtr::NULL_PTR;
357 if (is_masked_op) {
358 if (mask_klass == nullptr || mask_klass->const_oop() == nullptr) {
359 log_if_needed(" ** missing constant: maskclass=%s", NodeClassNames[argument(2)->Opcode()]);
360 return false; // not enough info for intrinsification
361 }
362
363 if (!is_klass_initialized(mask_klass)) {
364 log_if_needed(" ** mask klass argument not initialized");
365 return false;
366 }
367
368 if (vmask_type->maybe_null()) {
369 log_if_needed(" ** null mask values are not allowed for masked op");
370 return false;
371 }
372 }
373
374 BasicType elem_bt = elem_type->basic_type();
375 bool has_scalar_op = VectorSupport::has_scalar_op(opr->get_con());
376 bool is_unsigned = VectorSupport::is_unsigned_op(opr->get_con());
377
378 int num_elem = vlen->get_con();
379 int opc = VectorSupport::vop2ideal(opr->get_con(), elem_bt);
380 int sopc = has_scalar_op ? VectorNode::opcode(opc, elem_bt) : opc;
381 if (sopc == 0 || num_elem == 1) {
382 log_if_needed(" ** operation not supported: arity=%d opc=%s[%d] vlen=%d etype=%s",
383 n, NodeClassNames[opc], opc, num_elem, type2name(elem_bt));
384 return false; // operation not supported
385 }
386 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
387 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
388
389 if (is_vector_mask(vbox_klass)) {
390 assert(!is_masked_op, "mask operations do not need mask to control");
391 }
392
393 // When using mask, mask use type needs to be VecMaskUseLoad.
394 VectorMaskUseType mask_use_type = is_vector_mask(vbox_klass) ? VecMaskUseAll
395 : is_masked_op ? VecMaskUseLoad : VecMaskNotUsed;
396 if (!arch_supports_vector(sopc, num_elem, elem_bt, mask_use_type)) {
397 log_if_needed(" ** not supported: arity=%d opc=%d vlen=%d etype=%s ismask=%d is_masked_op=%d",
398 n, sopc, num_elem, type2name(elem_bt),
399 is_vector_mask(vbox_klass) ? 1 : 0, is_masked_op ? 1 : 0);
400 return false; // not supported
401 }
402
403 // Return true if current platform has implemented the masked operation with predicate feature.
404 bool use_predicate = is_masked_op && arch_supports_vector(sopc, num_elem, elem_bt, VecMaskUsePred);
405 if (is_masked_op && !use_predicate && !arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad)) {
406 log_if_needed(" ** not supported: arity=%d opc=%d vlen=%d etype=%s ismask=0 is_masked_op=1",
407 n, sopc, num_elem, type2name(elem_bt));
408 return false;
409 }
410
411 Node* opd1 = nullptr; Node* opd2 = nullptr; Node* opd3 = nullptr;
412 switch (n) {
413 case 3: {
414 opd3 = unbox_vector(argument(7), vbox_type, elem_bt, num_elem);
415 if (opd3 == nullptr) {
416 log_if_needed(" ** unbox failed v3=%s",
417 NodeClassNames[argument(7)->Opcode()]);
418 return false;
419 }
420 // fall-through
421 }
422 case 2: {
423 opd2 = unbox_vector(argument(6), vbox_type, elem_bt, num_elem);
424 if (opd2 == nullptr) {
425 log_if_needed(" ** unbox failed v2=%s",
426 NodeClassNames[argument(6)->Opcode()]);
427 return false;
428 }
429 // fall-through
430 }
431 case 1: {
432 opd1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
433 if (opd1 == nullptr) {
434 log_if_needed(" ** unbox failed v1=%s",
435 NodeClassNames[argument(5)->Opcode()]);
436 return false;
437 }
438 break;
439 }
440 default: fatal("unsupported arity: %d", n);
441 }
442
443 Node* mask = nullptr;
444 if (is_masked_op) {
445 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
446 assert(is_vector_mask(mbox_klass), "argument(2) should be a mask class");
447 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
448 mask = unbox_vector(argument(n + 5), mbox_type, elem_bt, num_elem);
449 if (mask == nullptr) {
450 log_if_needed(" ** unbox failed mask=%s",
451 NodeClassNames[argument(n + 5)->Opcode()]);
452 return false;
453 }
454 }
455
456 Node* operation = nullptr;
457 const TypeVect* vt = TypeVect::make(elem_bt, num_elem, is_vector_mask(vbox_klass));
458 switch (n) {
459 case 1:
460 case 2: {
461 operation = VectorNode::make(sopc, opd1, opd2, vt, is_vector_mask(vbox_klass), VectorNode::is_shift_opcode(opc), is_unsigned);
462 break;
463 }
464 case 3: {
465 operation = VectorNode::make(sopc, opd1, opd2, opd3, vt);
466 break;
467 }
468 default: fatal("unsupported arity: %d", n);
469 }
470 trace_vector(operation);
471 if (is_masked_op && mask != nullptr) {
472 if (use_predicate) {
473 operation->add_req(mask);
474 operation->add_flag(Node::Flag_is_predicated_vector);
475 } else {
476 operation->add_flag(Node::Flag_is_predicated_using_blend);
477 operation = gvn().transform(operation);
478 operation = trace_vector(new VectorBlendNode(opd1, operation, mask));
479 }
480 }
481 operation = gvn().transform(operation);
482
483 // Wrap it up in VectorBox to keep object type information.
484 Node* vbox = box_vector(operation, vbox_type, elem_bt, num_elem);
485 set_result(vbox);
486 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
487 return true;
488 }
489
490 // public static
491 // <V extends Vector<E>, E>
492 // V libraryUnaryOp(long address, Class<? extends V> vClass, Class<E> elementType, int length, String debugName,
493 // V v,
494 // UnaryOperation<V, ?> defaultImpl)
495 //
496 // public static
497 // <V extends VectorPayload, E>
498 // V libraryBinaryOp(long address, Class<? extends V> vClass, Class<E> elementType, int length, String debugName,
499 // V v1, V v2,
500 // BinaryOperation<V, ?> defaultImpl)
501 bool LibraryCallKit::inline_vector_call(int arity) {
502 assert(Matcher::supports_vector_calling_convention(), "required");
503
504 const TypeLong* entry = gvn().type(argument(0))->isa_long();
505 const TypeInstPtr* vector_klass = gvn().type(argument(2))->isa_instptr();
506 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
507 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
508 const TypeInstPtr* debug_name_oop = gvn().type(argument(5))->isa_instptr();
509
510 if (entry == nullptr || !entry->is_con() ||
511 vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
512 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
513 vlen == nullptr || !vlen->is_con() ||
514 debug_name_oop == nullptr || debug_name_oop->const_oop() == nullptr) {
515 log_if_needed(" ** missing constant: opr=%s vclass=%s etype=%s vlen=%s debug_name=%s",
516 NodeClassNames[argument(0)->Opcode()],
517 NodeClassNames[argument(2)->Opcode()],
518 NodeClassNames[argument(3)->Opcode()],
519 NodeClassNames[argument(4)->Opcode()],
520 NodeClassNames[argument(5)->Opcode()]);
521 return false; // not enough info for intrinsification
522 }
523
524 if (entry->get_con() == 0) {
525 log_if_needed(" ** missing entry point");
526 return false;
527 }
528
529 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
530 if (!elem_type->is_primitive_type()) {
531 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
532 return false; // should be primitive type
533 }
534 if (!is_klass_initialized(vector_klass)) {
535 log_if_needed(" ** klass argument not initialized");
536 return false;
537 }
538
539 BasicType elem_bt = elem_type->basic_type();
540 int num_elem = vlen->get_con();
541 if (!Matcher::vector_size_supported(elem_bt, num_elem)) {
542 log_if_needed(" ** vector size (vlen=%d, etype=%s) is not supported",
543 num_elem, type2name(elem_bt));
544 return false;
545 }
546
547 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
548 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
549
550 Node* opd1 = unbox_vector(argument(6), vbox_type, elem_bt, num_elem);
551 if (opd1 == nullptr) {
552 log_if_needed(" ** unbox failed v1=%s", NodeClassNames[argument(6)->Opcode()]);
553 return false;
554 }
555
556 Node* opd2 = nullptr;
557 if (arity > 1) {
558 opd2 = unbox_vector(argument(7), vbox_type, elem_bt, num_elem);
559 if (opd2 == nullptr) {
560 log_if_needed(" ** unbox failed v2=%s", NodeClassNames[argument(7)->Opcode()]);
561 return false;
562 }
563 }
564 assert(arity == 1 || arity == 2, "arity %d not supported", arity);
565 const TypeVect* vt = TypeVect::make(elem_bt, num_elem);
566 const TypeFunc* call_type = OptoRuntime::Math_Vector_Vector_Type(arity, vt, vt);
567 address entry_addr = (address)entry->get_con();
568
569 const char* debug_name = "<unknown>";
570 if (!debug_name_oop->const_oop()->is_null_object()) {
571 size_t buflen = 100;
572 char* buf = NEW_ARENA_ARRAY(C->comp_arena(), char, buflen);
573 debug_name = debug_name_oop->const_oop()->as_instance()->java_lang_String_str(buf, buflen);
574 }
575 Node* vcall = make_runtime_call(RC_VECTOR,
576 call_type,
577 entry_addr,
578 debug_name,
579 TypePtr::BOTTOM,
580 opd1,
581 opd2);
582
583 vcall = gvn().transform(new ProjNode(gvn().transform(vcall), TypeFunc::Parms));
584
585 // Wrap it up in VectorBox to keep object type information.
586 Node* vbox = box_vector(vcall, vbox_type, elem_bt, num_elem);
587 set_result(vbox);
588 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
589 return true;
590 }
591
592 // <E, M>
593 // long maskReductionCoerced(int oper, Class<? extends M> maskClass, Class<?> elemClass,
594 // int length, M m, VectorMaskOp<M> defaultImpl)
595 bool LibraryCallKit::inline_vector_mask_operation() {
596 const TypeInt* oper = gvn().type(argument(0))->isa_int();
597 const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
598 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
599 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
600 Node* mask = argument(4);
601
602 if (mask_klass == nullptr || mask_klass->const_oop() == nullptr ||
603 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
604 vlen == nullptr || !vlen->is_con() ||
605 oper == nullptr || !oper->is_con() ||
606 mask->is_top()) {
607 return false; // dead code
608 }
609
610 if (!is_klass_initialized(mask_klass)) {
611 log_if_needed(" ** klass argument not initialized");
612 return false;
613 }
614
615 int num_elem = vlen->get_con();
616 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
617 BasicType elem_bt = elem_type->basic_type();
618
619 int mopc = VectorSupport::vop2ideal(oper->get_con(), elem_bt);
620 if (!arch_supports_vector(mopc, num_elem, elem_bt, VecMaskUseLoad)) {
621 log_if_needed(" ** not supported: arity=1 op=cast#%d/3 vlen2=%d etype2=%s",
622 mopc, num_elem, type2name(elem_bt));
623 return false; // not supported
624 }
625
626 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
627 const TypeInstPtr* mask_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
628 Node* mask_vec = unbox_vector(mask, mask_box_type, elem_bt, num_elem);
629 if (mask_vec == nullptr) {
630 log_if_needed(" ** unbox failed mask=%s",
631 NodeClassNames[argument(4)->Opcode()]);
632 return false;
633 }
634
635 if (!Matcher::mask_op_prefers_predicate(mopc, mask_vec->bottom_type()->is_vect())) {
636 mask_vec = gvn().transform(VectorStoreMaskNode::make(gvn(), mask_vec, elem_bt, num_elem));
637 }
638 const Type* maskoper_ty = mopc == Op_VectorMaskToLong ? (const Type*)TypeLong::LONG : (const Type*)TypeInt::INT;
639 Node* maskoper = gvn().transform(trace_vector(VectorMaskOpNode::make(mask_vec, maskoper_ty, mopc)));
640 if (mopc != Op_VectorMaskToLong) {
641 maskoper = ConvI2L(maskoper);
642 }
643 set_result(maskoper);
644
645 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
646 return true;
647 }
648
649 // public static
650 // <M,
651 // S extends VectorSpecies<E>,
652 // E>
653 // M fromBitsCoerced(Class<? extends M> vmClass, Class<E> elementType, int length,
654 // long bits, int mode, S s,
655 // BroadcastOperation<M, E, S> defaultImpl)
656 bool LibraryCallKit::inline_vector_frombits_coerced() {
657 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
658 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
659 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
660 const TypeLong* bits_type = gvn().type(argument(3))->isa_long();
661 // Mode argument determines the mode of operation it can take following values:-
662 // MODE_BROADCAST for vector Vector.broadcast and VectorMask.maskAll operations.
663 // MODE_BITS_COERCED_LONG_TO_MASK for VectorMask.fromLong operation.
664 const TypeInt* mode = gvn().type(argument(5))->isa_int();
665
666 if (vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
667 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
668 vlen == nullptr || !vlen->is_con() ||
669 bits_type == nullptr ||
670 mode == nullptr || !mode->is_con()) {
671 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s bitwise=%s",
672 NodeClassNames[argument(0)->Opcode()],
673 NodeClassNames[argument(1)->Opcode()],
674 NodeClassNames[argument(2)->Opcode()],
675 NodeClassNames[argument(5)->Opcode()]);
676 return false; // not enough info for intrinsification
677 }
678
679 if (!is_klass_initialized(vector_klass)) {
680 log_if_needed(" ** klass argument not initialized");
681 return false;
682 }
683 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
684 if (!elem_type->is_primitive_type()) {
685 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
686 return false; // should be primitive type
687 }
688 BasicType elem_bt = elem_type->basic_type();
689 int num_elem = vlen->get_con();
690 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
691 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
692
693 bool is_mask = is_vector_mask(vbox_klass);
694 int bcast_mode = mode->get_con();
695 VectorMaskUseType checkFlags = (VectorMaskUseType)(is_mask ? VecMaskUseAll : VecMaskNotUsed);
696 int opc = bcast_mode == VectorSupport::MODE_BITS_COERCED_LONG_TO_MASK ? Op_VectorLongToMask : Op_Replicate;
697
698 if (!arch_supports_vector(opc, num_elem, elem_bt, checkFlags, true /*has_scalar_args*/)) {
699 // If the input long sets or unsets all lanes and Replicate is supported,
700 // generate a MaskAll or Replicate instead.
701
702 // The "maskAll" API uses the corresponding integer types for floating-point data.
703 BasicType maskall_bt = elem_bt == T_DOUBLE ? T_LONG : (elem_bt == T_FLOAT ? T_INT: elem_bt);
704 if (!(opc == Op_VectorLongToMask &&
705 VectorNode::is_maskall_type(bits_type, num_elem) &&
706 arch_supports_vector(Op_Replicate, num_elem, maskall_bt, checkFlags, true /*has_scalar_args*/))) {
707 log_if_needed(" ** not supported: arity=0 op=broadcast vlen=%d etype=%s ismask=%d bcast_mode=%d",
708 num_elem, type2name(elem_bt),
709 is_mask ? 1 : 0,
710 bcast_mode);
711 return false; // not supported
712 }
713 }
714
715 Node* broadcast = nullptr;
716 Node* bits = argument(3);
717 Node* elem = bits;
718
719 if (opc == Op_VectorLongToMask) {
720 const TypeVect* vt = TypeVect::makemask(elem_bt, num_elem);
721 if (Matcher::mask_op_prefers_predicate(opc, vt)) {
722 broadcast = gvn().transform(trace_vector(new VectorLongToMaskNode(elem, vt)));
723 } else {
724 const TypeVect* mvt = TypeVect::make(T_BOOLEAN, num_elem);
725 broadcast = gvn().transform(trace_vector(new VectorLongToMaskNode(elem, mvt)));
726 broadcast = gvn().transform(new VectorLoadMaskNode(broadcast, vt));
727 }
728 } else {
729 switch (elem_bt) {
730 case T_BOOLEAN: // fall-through
731 case T_BYTE: // fall-through
732 case T_SHORT: // fall-through
733 case T_CHAR: // fall-through
734 case T_INT: {
735 elem = gvn().transform(new ConvL2INode(bits));
736 break;
737 }
738 case T_DOUBLE: {
739 elem = gvn().transform(new MoveL2DNode(bits));
740 break;
741 }
742 case T_FLOAT: {
743 bits = gvn().transform(new ConvL2INode(bits));
744 elem = gvn().transform(new MoveI2FNode(bits));
745 break;
746 }
747 case T_LONG: {
748 // no conversion needed
749 break;
750 }
751 default: fatal("%s", type2name(elem_bt));
752 }
753 broadcast = trace_vector(VectorNode::scalar2vector(elem, num_elem, elem_bt, is_mask));
754 broadcast = gvn().transform(broadcast);
755 }
756
757 Node* box = box_vector(broadcast, vbox_type, elem_bt, num_elem);
758 set_result(box);
759 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
760 return true;
761 }
762
763 static bool elem_consistent_with_arr(BasicType elem_bt, const TypeAryPtr* arr_type, bool mismatched_ms) {
764 assert(arr_type != nullptr, "unexpected");
765 BasicType arr_elem_bt = arr_type->elem()->array_element_basic_type();
766 if (elem_bt == arr_elem_bt) {
767 return true;
768 } else if (elem_bt == T_SHORT && arr_elem_bt == T_CHAR) {
769 // Load/store of short vector from/to char[] is supported
770 return true;
771 } else if (elem_bt == T_BYTE && arr_elem_bt == T_BOOLEAN) {
772 // Load/store of byte vector from/to boolean[] is supported
773 return true;
774 } else {
775 return mismatched_ms;
776 }
777 }
778
779 // public static
780 // <C,
781 // VM extends VectorPayload,
782 // E,
783 // S extends VectorSpecies<E>>
784 // VM load(Class<? extends VM> vmClass, Class<E> eClass,
785 // int length,
786 // Object base, long offset, // Unsafe addressing
787 // boolean fromSegment,
788 // C container, long index, S s, // Arguments for default implementation
789 // LoadOperation<C, VM, S> defaultImpl) {
790 // public static
791 // <C,
792 // V extends VectorPayload>
793 // void store(Class<?> vClass, Class<?> eClass,
794 // int length,
795 // Object base, long offset, // Unsafe addressing
796 // boolean fromSegment,
797 // V v, C container, long index, // Arguments for default implementation
798 // StoreVectorOperation<C, V> defaultImpl) {
799 bool LibraryCallKit::inline_vector_mem_operation(bool is_store) {
800 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
801 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
802 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
803 const TypeInt* from_ms = gvn().type(argument(6))->isa_int();
804
805 if (vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
806 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
807 vlen == nullptr || !vlen->is_con() ||
808 from_ms == nullptr || !from_ms->is_con()) {
809 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s from_ms=%s",
810 NodeClassNames[argument(0)->Opcode()],
811 NodeClassNames[argument(1)->Opcode()],
812 NodeClassNames[argument(2)->Opcode()],
813 NodeClassNames[argument(6)->Opcode()]);
814 return false; // not enough info for intrinsification
815 }
816 if (!is_klass_initialized(vector_klass)) {
817 log_if_needed(" ** klass argument not initialized");
818 return false;
819 }
820
821 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
822 if (!elem_type->is_primitive_type()) {
823 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
824 return false; // should be primitive type
825 }
826 BasicType elem_bt = elem_type->basic_type();
827 int num_elem = vlen->get_con();
828
829 // TODO When mask usage is supported, VecMaskNotUsed needs to be VecMaskUseLoad.
830 if (!arch_supports_vector(is_store ? Op_StoreVector : Op_LoadVector, num_elem, elem_bt, VecMaskNotUsed)) {
831 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d etype=%s ismask=no",
832 is_store, is_store ? "store" : "load",
833 num_elem, type2name(elem_bt));
834 return false; // not supported
835 }
836
837 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
838 bool is_mask = is_vector_mask(vbox_klass);
839
840 Node* base = argument(3);
841 Node* offset = ConvL2X(argument(4));
842
843 // Save state and restore on bailout
844 SavedState old_state(this);
845
846 Node* addr = make_unsafe_address(base, offset, (is_mask ? T_BOOLEAN : elem_bt), true);
847
848 // The memory barrier checks are based on ones for unsafe access.
849 // This is not 1-1 implementation.
850 const Type *const base_type = gvn().type(base);
851
852 const TypePtr *addr_type = gvn().type(addr)->isa_ptr();
853 const TypeAryPtr* arr_type = addr_type->isa_aryptr();
854
855 const bool in_native = TypePtr::NULL_PTR == base_type; // base always null
856 const bool in_heap = !TypePtr::NULL_PTR->higher_equal(base_type); // base never null
857
858 const bool is_mixed_access = !in_heap && !in_native;
859
860 const bool is_mismatched_access = in_heap && (addr_type->isa_aryptr() == nullptr);
861
862 const bool needs_cpu_membar = is_mixed_access || is_mismatched_access;
863
864 // For non-masked mismatched memory segment vector read/write accesses, intrinsification can continue
865 // with unknown backing storage type and compiler can skip inserting explicit reinterpretation IR after
866 // loading from or before storing to backing storage which is mandatory for semantic correctness of
867 // big-endian memory layout.
868 bool mismatched_ms = LITTLE_ENDIAN_ONLY(false)
869 BIG_ENDIAN_ONLY(from_ms->get_con() && !is_mask && arr_type != nullptr &&
870 arr_type->elem()->array_element_basic_type() != elem_bt);
871 BasicType mem_elem_bt = mismatched_ms ? arr_type->elem()->array_element_basic_type() : elem_bt;
872 if (!is_java_primitive(mem_elem_bt)) {
873 log_if_needed(" ** non-primitive array element type");
874 return false;
875 }
876 int mem_num_elem = mismatched_ms ? (num_elem * type2aelembytes(elem_bt)) / type2aelembytes(mem_elem_bt) : num_elem;
877 if (arr_type != nullptr && !is_mask && !elem_consistent_with_arr(elem_bt, arr_type, mismatched_ms)) {
878 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d etype=%s atype=%s ismask=no",
879 is_store, is_store ? "store" : "load",
880 num_elem, type2name(elem_bt), type2name(arr_type->elem()->array_element_basic_type()));
881 return false;
882 }
883
884 // In case of mismatched memory segment accesses, we need to double check that the source type memory operations are supported by backend.
885 if (mismatched_ms) {
886 if (is_store) {
887 if (!arch_supports_vector(Op_StoreVector, num_elem, elem_bt, VecMaskNotUsed)
888 || !arch_supports_vector(Op_VectorReinterpret, mem_num_elem, mem_elem_bt, VecMaskNotUsed)) {
889 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d*8 etype=%s/8 ismask=no",
890 is_store, "store",
891 num_elem, type2name(elem_bt));
892 return false; // not supported
893 }
894 } else {
895 if (!arch_supports_vector(Op_LoadVector, mem_num_elem, mem_elem_bt, VecMaskNotUsed)
896 || !arch_supports_vector(Op_VectorReinterpret, num_elem, elem_bt, VecMaskNotUsed)) {
897 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d*8 etype=%s/8 ismask=no",
898 is_store, "load",
899 mem_num_elem, type2name(mem_elem_bt));
900 return false; // not supported
901 }
902 }
903 }
904 if (is_mask) {
905 if (!is_store) {
906 if (!arch_supports_vector(Op_LoadVector, num_elem, elem_bt, VecMaskUseLoad)) {
907 return false; // not supported
908 }
909 } else {
910 if (!arch_supports_vector(Op_StoreVector, num_elem, elem_bt, VecMaskUseStore)) {
911 return false; // not supported
912 }
913 }
914 }
915
916 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
917
918 if (needs_cpu_membar) {
919 insert_mem_bar(Op_MemBarCPUOrder);
920 }
921
922 if (is_store) {
923 Node* val = unbox_vector(argument(7), vbox_type, elem_bt, num_elem);
924 if (val == nullptr) {
925 return false; // operand unboxing failed
926 }
927 set_all_memory(reset_memory());
928
929 // In case the store needs to happen to byte array, reinterpret the incoming vector to byte vector.
930 int store_num_elem = num_elem;
931 if (mismatched_ms) {
932 store_num_elem = mem_num_elem;
933 const TypeVect* to_vect_type = TypeVect::make(mem_elem_bt, store_num_elem);
934 val = gvn().transform(new VectorReinterpretNode(val, val->bottom_type()->is_vect(), to_vect_type));
935 }
936 if (is_mask) {
937 val = gvn().transform(VectorStoreMaskNode::make(gvn(), val, elem_bt, num_elem));
938 }
939 Node* vstore = gvn().transform(trace_vector(StoreVectorNode::make(0, control(), memory(addr), addr, addr_type, val, store_num_elem)));
940 set_memory(vstore, addr_type);
941 } else {
942 // When using byte array, we need to load as byte then reinterpret the value. Otherwise, do a simple vector load.
943 Node* vload = nullptr;
944 if (mismatched_ms) {
945 vload = gvn().transform(trace_vector(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, mem_num_elem, mem_elem_bt)));
946 const TypeVect* to_vect_type = TypeVect::make(elem_bt, num_elem);
947 vload = gvn().transform(new VectorReinterpretNode(vload, vload->bottom_type()->is_vect(), to_vect_type));
948 } else {
949 // Special handle for masks
950 if (is_mask) {
951 vload = gvn().transform(trace_vector(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, num_elem, T_BOOLEAN)));
952 vload = gvn().transform(new VectorLoadMaskNode(vload, TypeVect::makemask(elem_bt, num_elem)));
953 } else {
954 vload = gvn().transform(trace_vector(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, num_elem, elem_bt)));
955 }
956 }
957 Node* box = box_vector(vload, vbox_type, elem_bt, num_elem);
958 set_result(box);
959 }
960
961 old_state.discard();
962
963 if (needs_cpu_membar) {
964 insert_mem_bar(Op_MemBarCPUOrder);
965 }
966
967 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
968 return true;
969 }
970
971 // public static
972 // <C,
973 // V extends Vector<?>,
974 // E,
975 // S extends VectorSpecies<E>,
976 // M extends VectorMask<E>>
977 // V loadMasked(Class<? extends V> vClass, Class<M> mClass, Class<E> eClass,
978 // int length, Object base, long offset, // Unsafe addressing
979 // boolean fromSegment,
980 // M m, int offsetInRange,
981 // C container, long index, S s, // Arguments for default implementation
982 // LoadVectorMaskedOperation<C, V, S, M> defaultImpl) {
983 // public static
984 // <C,
985 // V extends Vector<E>,
986 // M extends VectorMask<E>,
987 // E>
988 // void storeMasked(Class<? extends V> vClass, Class<M> mClass, Class<E> eClass,
989 // int length,
990 // Object base, long offset, // Unsafe addressing
991 // boolean fromSegment,
992 // V v, M m, C container, long index, // Arguments for default implementation
993 // StoreVectorMaskedOperation<C, V, M> defaultImpl) {
994
995 bool LibraryCallKit::inline_vector_mem_masked_operation(bool is_store) {
996 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
997 const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
998 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
999 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
1000 const TypeInt* from_ms = gvn().type(argument(7))->isa_int();
1001
1002 if (vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
1003 mask_klass == nullptr || mask_klass->const_oop() == nullptr ||
1004 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
1005 vlen == nullptr || !vlen->is_con() ||
1006 from_ms == nullptr || !from_ms->is_con()) {
1007 log_if_needed(" ** missing constant: vclass=%s mclass=%s etype=%s vlen=%s from_ms=%s",
1008 NodeClassNames[argument(0)->Opcode()],
1009 NodeClassNames[argument(1)->Opcode()],
1010 NodeClassNames[argument(2)->Opcode()],
1011 NodeClassNames[argument(3)->Opcode()],
1012 NodeClassNames[argument(7)->Opcode()]);
1013 return false; // not enough info for intrinsification
1014 }
1015 if (!is_klass_initialized(vector_klass)) {
1016 log_if_needed(" ** klass argument not initialized");
1017 return false;
1018 }
1019
1020 if (!is_klass_initialized(mask_klass)) {
1021 log_if_needed(" ** mask klass argument not initialized");
1022 return false;
1023 }
1024
1025 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1026 if (!elem_type->is_primitive_type()) {
1027 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1028 return false; // should be primitive type
1029 }
1030
1031 BasicType elem_bt = elem_type->basic_type();
1032 int num_elem = vlen->get_con();
1033
1034 Node* base = argument(4);
1035 Node* offset = ConvL2X(argument(5));
1036
1037 // Save state and restore on bailout
1038 SavedState old_state(this);
1039
1040 Node* addr = make_unsafe_address(base, offset, elem_bt, true);
1041 const TypePtr *addr_type = gvn().type(addr)->isa_ptr();
1042 const TypeAryPtr* arr_type = addr_type->isa_aryptr();
1043
1044 bool mismatched_ms = from_ms->get_con() && arr_type != nullptr && arr_type->elem()->array_element_basic_type() != elem_bt;
1045 BIG_ENDIAN_ONLY(if (mismatched_ms) return false;)
1046 // If there is no consistency between array and vector element types, it must be special byte array case
1047 if (arr_type != nullptr && !elem_consistent_with_arr(elem_bt, arr_type, mismatched_ms)) {
1048 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d etype=%s atype=%s",
1049 is_store, is_store ? "storeMasked" : "loadMasked",
1050 num_elem, type2name(elem_bt), type2name(arr_type->elem()->array_element_basic_type()));
1051 return false;
1052 }
1053
1054 int mem_num_elem = mismatched_ms ? num_elem * type2aelembytes(elem_bt) : num_elem;
1055 BasicType mem_elem_bt = mismatched_ms ? T_BYTE : elem_bt;
1056 bool supports_predicate = arch_supports_vector(is_store ? Op_StoreVectorMasked : Op_LoadVectorMasked,
1057 mem_num_elem, mem_elem_bt, VecMaskUseLoad);
1058
1059 // If current arch does not support the predicated operations, we have to bail
1060 // out when current case uses the predicate feature.
1061 if (!supports_predicate) {
1062 bool needs_predicate = false;
1063 if (is_store) {
1064 // Masked vector store always uses the predicated store.
1065 needs_predicate = true;
1066 } else {
1067 // Masked vector load with IOOBE always uses the predicated load.
1068 const TypeInt* offset_in_range = gvn().type(argument(9))->isa_int();
1069 if (!offset_in_range->is_con()) {
1070 log_if_needed(" ** missing constant: offsetInRange=%s",
1071 NodeClassNames[argument(8)->Opcode()]);
1072 return false;
1073 }
1074 needs_predicate = (offset_in_range->get_con() == 0);
1075 }
1076
1077 if (needs_predicate) {
1078 log_if_needed(" ** not supported: op=%s vlen=%d etype=%s mismatched_ms=%d",
1079 is_store ? "storeMasked" : "loadMasked",
1080 num_elem, type2name(elem_bt), mismatched_ms ? 1 : 0);
1081 return false;
1082 }
1083 }
1084
1085 // This only happens for masked vector load. If predicate is not supported, then check whether
1086 // the normal vector load and blend operations are supported by backend.
1087 if (!supports_predicate && (!arch_supports_vector(Op_LoadVector, mem_num_elem, mem_elem_bt, VecMaskNotUsed) ||
1088 !arch_supports_vector(Op_VectorBlend, mem_num_elem, mem_elem_bt, VecMaskUseLoad))) {
1089 log_if_needed(" ** not supported: op=loadMasked vlen=%d etype=%s mismatched_ms=%d",
1090 num_elem, type2name(elem_bt), mismatched_ms ? 1 : 0);
1091 return false;
1092 }
1093
1094 // Since we are using byte array, we need to double check that the vector reinterpret operation
1095 // with byte type is supported by backend.
1096 if (mismatched_ms) {
1097 if (!arch_supports_vector(Op_VectorReinterpret, mem_num_elem, T_BYTE, VecMaskNotUsed)) {
1098 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d etype=%s mismatched_ms=1",
1099 is_store, is_store ? "storeMasked" : "loadMasked",
1100 num_elem, type2name(elem_bt));
1101 return false;
1102 }
1103 }
1104
1105 // Since it needs to unbox the mask, we need to double check that the related load operations
1106 // for mask are supported by backend.
1107 if (!arch_supports_vector(Op_LoadVector, num_elem, elem_bt, VecMaskUseLoad)) {
1108 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d etype=%s",
1109 is_store, is_store ? "storeMasked" : "loadMasked",
1110 num_elem, type2name(elem_bt));
1111 return false;
1112 }
1113
1114 // Can base be null? Otherwise, always on-heap access.
1115 bool can_access_non_heap = TypePtr::NULL_PTR->higher_equal(gvn().type(base));
1116 if (can_access_non_heap) {
1117 insert_mem_bar(Op_MemBarCPUOrder);
1118 }
1119
1120 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1121 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1122 assert(!is_vector_mask(vbox_klass) && is_vector_mask(mbox_klass), "Invalid class type");
1123 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1124 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1125
1126 Node* mask = unbox_vector(is_store ? argument(9) : argument(8), mbox_type, elem_bt, num_elem);
1127 if (mask == nullptr) {
1128 log_if_needed(" ** unbox failed mask=%s",
1129 is_store ? NodeClassNames[argument(9)->Opcode()]
1130 : NodeClassNames[argument(8)->Opcode()]);
1131 return false;
1132 }
1133
1134 if (is_store) {
1135 Node* val = unbox_vector(argument(8), vbox_type, elem_bt, num_elem);
1136 if (val == nullptr) {
1137 log_if_needed(" ** unbox failed vector=%s",
1138 NodeClassNames[argument(8)->Opcode()]);
1139 return false; // operand unboxing failed
1140 }
1141 set_all_memory(reset_memory());
1142
1143 if (mismatched_ms) {
1144 // Reinterpret the incoming vector to byte vector.
1145 const TypeVect* to_vect_type = TypeVect::make(mem_elem_bt, mem_num_elem);
1146 val = gvn().transform(new VectorReinterpretNode(val, val->bottom_type()->is_vect(), to_vect_type));
1147 // Reinterpret the vector mask to byte type.
1148 const TypeVect* from_mask_type = TypeVect::makemask(elem_bt, num_elem);
1149 const TypeVect* to_mask_type = TypeVect::makemask(mem_elem_bt, mem_num_elem);
1150 mask = gvn().transform(new VectorReinterpretNode(mask, from_mask_type, to_mask_type));
1151 }
1152 Node* vstore = gvn().transform(trace_vector(new StoreVectorMaskedNode(control(), memory(addr), addr, val, addr_type, mask)));
1153 set_memory(vstore, addr_type);
1154 } else {
1155 Node* vload = nullptr;
1156
1157 if (mismatched_ms) {
1158 // Reinterpret the vector mask to byte type.
1159 const TypeVect* from_mask_type = TypeVect::makemask(elem_bt, num_elem);
1160 const TypeVect* to_mask_type = TypeVect::makemask(mem_elem_bt, mem_num_elem);
1161 mask = gvn().transform(new VectorReinterpretNode(mask, from_mask_type, to_mask_type));
1162 }
1163
1164 if (supports_predicate) {
1165 // Generate masked load vector node if predicate feature is supported.
1166 const TypeVect* vt = TypeVect::make(mem_elem_bt, mem_num_elem);
1167 vload = gvn().transform(trace_vector(new LoadVectorMaskedNode(control(), memory(addr), addr, addr_type, vt, mask)));
1168 } else {
1169 // Use the vector blend to implement the masked load vector. The biased elements are zeros.
1170 Node* zero = gvn().transform(gvn().zerocon(mem_elem_bt));
1171 zero = gvn().transform(VectorNode::scalar2vector(zero, mem_num_elem, mem_elem_bt));
1172 vload = gvn().transform(trace_vector(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, mem_num_elem, mem_elem_bt)));
1173 vload = gvn().transform(trace_vector(new VectorBlendNode(zero, vload, mask)));
1174 }
1175
1176 if (mismatched_ms) {
1177 const TypeVect* to_vect_type = TypeVect::make(elem_bt, num_elem);
1178 vload = gvn().transform(new VectorReinterpretNode(vload, vload->bottom_type()->is_vect(), to_vect_type));
1179 }
1180
1181 Node* box = box_vector(vload, vbox_type, elem_bt, num_elem);
1182 set_result(box);
1183 }
1184
1185 old_state.discard();
1186
1187 if (can_access_non_heap) {
1188 insert_mem_bar(Op_MemBarCPUOrder);
1189 }
1190
1191 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1192 return true;
1193 }
1194
1195 //
1196 // <C,
1197 // V extends Vector<?>,
1198 // W extends Vector<Integer>,
1199 // S extends VectorSpecies<E>,
1200 // M extends VectorMask<E>,
1201 // E>
1202 // V loadWithMap(Class<? extends V> vClass, Class<M> mClass, Class<E> eClass, int length,
1203 // Class<? extends Vector<Integer>> vectorIndexClass, int indexLength,
1204 // Object base, long offset,
1205 // W indexVector1, W indexVector2, W indexVector3, W indexVector4,
1206 // M m, C container, int index, int[] indexMap, int indexM, S s,
1207 // LoadVectorOperationWithMap<C, V, S, M> defaultImpl)
1208 //
1209 // <C,
1210 // V extends Vector<E>,
1211 // W extends Vector<Integer>,
1212 // M extends VectorMask<E>,
1213 // E>
1214 // void storeWithMap(Class<? extends V> vClass, Class<M> mClass, Class<E> eClass, int length,
1215 // Class<? extends Vector<Integer>> vectorIndexClass, int indexLength,
1216 // Object base, long offset, // Unsafe addressing
1217 // W indexVector, V v, M m,
1218 // C container, int index, int[] indexMap, int indexM, // Arguments for default implementation
1219 // StoreVectorOperationWithMap<C, V, M> defaultImpl)
1220 //
1221 bool LibraryCallKit::inline_vector_gather_scatter(bool is_scatter) {
1222 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
1223 const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
1224 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
1225 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
1226 const TypeInstPtr* vector_idx_klass = gvn().type(argument(4))->isa_instptr();
1227 const TypeInt* idx_vlen = gvn().type(argument(5))->isa_int();
1228
1229 if (vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
1230 // mask_klass == nullptr || mask_klass->const_oop() == nullptr ||
1231 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
1232 vlen == nullptr || !vlen->is_con() ||
1233 vector_idx_klass == nullptr || vector_idx_klass->const_oop() == nullptr ||
1234 idx_vlen == nullptr || !idx_vlen->is_con()) {
1235 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s viclass=%s idx_vlen=%s",
1236 NodeClassNames[argument(0)->Opcode()],
1237 NodeClassNames[argument(2)->Opcode()],
1238 NodeClassNames[argument(3)->Opcode()],
1239 NodeClassNames[argument(4)->Opcode()],
1240 NodeClassNames[argument(5)->Opcode()]);
1241 return false; // not enough info for intrinsification
1242 }
1243
1244 if (!is_klass_initialized(vector_klass) || !is_klass_initialized(vector_idx_klass)) {
1245 log_if_needed(" ** klass argument not initialized");
1246 return false;
1247 }
1248
1249 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1250 if (!elem_type->is_primitive_type()) {
1251 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1252 return false; // should be primitive type
1253 }
1254
1255 BasicType elem_bt = elem_type->basic_type();
1256 int num_elem = vlen->get_con();
1257 int idx_num_elem = idx_vlen->get_con();
1258
1259 Node* m = is_scatter ? argument(11) : argument(13);
1260 const Type* vmask_type = gvn().type(m);
1261 bool is_masked_op = vmask_type != TypePtr::NULL_PTR;
1262 if (is_masked_op) {
1263 if (mask_klass == nullptr || mask_klass->const_oop() == nullptr) {
1264 log_if_needed(" ** missing constant: maskclass=%s", NodeClassNames[argument(1)->Opcode()]);
1265 return false; // not enough info for intrinsification
1266 }
1267
1268 if (!is_klass_initialized(mask_klass)) {
1269 log_if_needed(" ** mask klass argument not initialized");
1270 return false;
1271 }
1272
1273 if (vmask_type->maybe_null()) {
1274 log_if_needed(" ** null mask values are not allowed for masked op");
1275 return false;
1276 }
1277
1278 // Check whether the predicated gather/scatter node is supported by architecture.
1279 VectorMaskUseType mask = (VectorMaskUseType) (VecMaskUseLoad | VecMaskUsePred);
1280 if (!arch_supports_vector(is_scatter ? Op_StoreVectorScatterMasked : Op_LoadVectorGatherMasked, num_elem, elem_bt, mask)) {
1281 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d etype=%s is_masked_op=1",
1282 is_scatter, is_scatter ? "scatterMasked" : "gatherMasked",
1283 num_elem, type2name(elem_bt));
1284 return false; // not supported
1285 }
1286 } else {
1287 // Check whether the normal gather/scatter node is supported for non-masked operation.
1288 if (!arch_supports_vector(is_scatter ? Op_StoreVectorScatter : Op_LoadVectorGather, num_elem, elem_bt, VecMaskNotUsed)) {
1289 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d etype=%s is_masked_op=0",
1290 is_scatter, is_scatter ? "scatter" : "gather",
1291 num_elem, type2name(elem_bt));
1292 return false; // not supported
1293 }
1294 }
1295
1296 // Check that the vector holding indices is supported by architecture
1297 // For sub-word gathers expander receive index array.
1298 if (!is_subword_type(elem_bt) && !arch_supports_vector(Op_LoadVector, idx_num_elem, T_INT, VecMaskNotUsed)) {
1299 log_if_needed(" ** not supported: arity=%d op=%s/loadindex vlen=%d etype=int is_masked_op=%d",
1300 is_scatter, is_scatter ? "scatter" : "gather",
1301 idx_num_elem, is_masked_op ? 1 : 0);
1302 return false; // not supported
1303 }
1304
1305 Node* base = argument(6);
1306 Node* offset = ConvL2X(argument(7));
1307
1308 // Save state and restore on bailout
1309 SavedState old_state(this);
1310
1311 Node* addr = nullptr;
1312 if (!is_subword_type(elem_bt)) {
1313 addr = make_unsafe_address(base, offset, elem_bt, true);
1314 } else {
1315 assert(!is_scatter, "Only supports gather operation for subword types now");
1316 uint header = arrayOopDesc::base_offset_in_bytes(elem_bt);
1317 assert(offset->is_Con() && offset->bottom_type()->is_long()->get_con() == header,
1318 "offset must be the array base offset");
1319 Node* index = argument(15);
1320 addr = array_element_address(base, index, elem_bt);
1321 }
1322
1323 const TypePtr* addr_type = gvn().type(addr)->isa_ptr();
1324 const TypeAryPtr* arr_type = addr_type->isa_aryptr();
1325
1326 // The array must be consistent with vector type
1327 if (arr_type == nullptr || (arr_type != nullptr && !elem_consistent_with_arr(elem_bt, arr_type, false))) {
1328 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d etype=%s atype=%s ismask=no",
1329 is_scatter, is_scatter ? "scatter" : "gather",
1330 num_elem, type2name(elem_bt), type2name(arr_type->elem()->array_element_basic_type()));
1331 return false;
1332 }
1333
1334 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1335 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1336 ciKlass* vbox_idx_klass = vector_idx_klass->const_oop()->as_instance()->java_lang_Class_klass();
1337 if (vbox_idx_klass == nullptr) {
1338 return false;
1339 }
1340
1341 // Get the indexes for gather/scatter.
1342 Node* indexes = nullptr;
1343 const TypeInstPtr* vbox_idx_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_idx_klass);
1344 if (is_subword_type(elem_bt)) {
1345 Node* indexMap = argument(16);
1346 Node* indexM = argument(17);
1347 indexes = array_element_address(indexMap, indexM, T_INT);
1348 } else {
1349 // Get the first index vector.
1350 indexes = unbox_vector(argument(9), vbox_idx_type, T_INT, idx_num_elem);
1351 if (indexes == nullptr) {
1352 return false;
1353 }
1354 }
1355
1356 // Get the vector mask value.
1357 Node* mask = nullptr;
1358 if (is_masked_op) {
1359 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1360 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1361 mask = unbox_vector(m, mbox_type, elem_bt, num_elem);
1362 if (mask == nullptr) {
1363 log_if_needed(" ** unbox failed mask=%s", NodeClassNames[m->Opcode()]);
1364 return false;
1365 }
1366 }
1367
1368 const TypeVect* vector_type = TypeVect::make(elem_bt, num_elem);
1369 if (is_scatter) {
1370 Node* val = unbox_vector(argument(10), vbox_type, elem_bt, num_elem);
1371 if (val == nullptr) {
1372 return false; // operand unboxing failed
1373 }
1374 set_all_memory(reset_memory());
1375
1376 Node* vstore = nullptr;
1377 if (mask != nullptr) {
1378 vstore = gvn().transform(trace_vector(new StoreVectorScatterMaskedNode(control(), memory(addr), addr, addr_type, val, indexes, mask)));
1379 } else {
1380 vstore = gvn().transform(trace_vector(new StoreVectorScatterNode(control(), memory(addr), addr, addr_type, val, indexes)));
1381 }
1382 set_memory(vstore, addr_type);
1383 } else {
1384 Node* vload = nullptr;
1385 if (mask != nullptr) {
1386 vload = gvn().transform(trace_vector(new LoadVectorGatherMaskedNode(control(), memory(addr), addr, addr_type, vector_type, indexes, mask)));
1387 } else {
1388 vload = gvn().transform(trace_vector(new LoadVectorGatherNode(control(), memory(addr), addr, addr_type, vector_type, indexes)));
1389 }
1390 Node* box = box_vector(vload, vbox_type, elem_bt, num_elem);
1391 set_result(box);
1392 }
1393
1394 old_state.discard();
1395 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1396 return true;
1397 }
1398
1399 // public static
1400 // <V extends Vector<E>,
1401 // M extends VectorMask<E>,
1402 // E>
1403 // long reductionCoerced(int oprId, Class<? extends V> vectorClass, Class<? extends M> maskClass,
1404 // Class<E> elementType, int length, V v, M m,
1405 // ReductionOperation<V, M> defaultImpl)
1406 bool LibraryCallKit::inline_vector_reduction() {
1407 const TypeInt* opr = gvn().type(argument(0))->isa_int();
1408 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
1409 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
1410 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
1411 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
1412
1413 if (opr == nullptr || !opr->is_con() ||
1414 vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
1415 // mask_klass == nullptr || mask_klass->const_oop() == nullptr ||
1416 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
1417 vlen == nullptr || !vlen->is_con()) {
1418 log_if_needed(" ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
1419 NodeClassNames[argument(0)->Opcode()],
1420 NodeClassNames[argument(1)->Opcode()],
1421 NodeClassNames[argument(3)->Opcode()],
1422 NodeClassNames[argument(4)->Opcode()]);
1423 return false; // not enough info for intrinsification
1424 }
1425 if (!is_klass_initialized(vector_klass)) {
1426 log_if_needed(" ** klass argument not initialized");
1427 return false;
1428 }
1429 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1430 if (!elem_type->is_primitive_type()) {
1431 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1432 return false; // should be primitive type
1433 }
1434
1435 const Type* vmask_type = gvn().type(argument(6));
1436 bool is_masked_op = vmask_type != TypePtr::NULL_PTR;
1437 if (is_masked_op) {
1438 if (mask_klass == nullptr || mask_klass->const_oop() == nullptr) {
1439 log_if_needed(" ** missing constant: maskclass=%s", NodeClassNames[argument(2)->Opcode()]);
1440 return false; // not enough info for intrinsification
1441 }
1442
1443 if (!is_klass_initialized(mask_klass)) {
1444 log_if_needed(" ** mask klass argument not initialized");
1445 return false;
1446 }
1447
1448 if (vmask_type->maybe_null()) {
1449 log_if_needed(" ** null mask values are not allowed for masked op");
1450 return false;
1451 }
1452 }
1453
1454 BasicType elem_bt = elem_type->basic_type();
1455 int num_elem = vlen->get_con();
1456 int opc = VectorSupport::vop2ideal(opr->get_con(), elem_bt);
1457 int sopc = ReductionNode::opcode(opc, elem_bt);
1458
1459 // Ensure reduction operation for lanewise operation
1460 // When using mask, mask use type needs to be VecMaskUseLoad.
1461 if (sopc == opc || !arch_supports_vector(sopc, num_elem, elem_bt, is_masked_op ? VecMaskUseLoad : VecMaskNotUsed)) {
1462 log_if_needed(" ** not supported: arity=1 op=%d/reduce vlen=%d etype=%s is_masked_op=%d",
1463 sopc, num_elem, type2name(elem_bt), is_masked_op ? 1 : 0);
1464 return false;
1465 }
1466
1467 // Return true if current platform has implemented the masked operation with predicate feature.
1468 bool use_predicate = is_masked_op && arch_supports_vector(sopc, num_elem, elem_bt, VecMaskUsePred);
1469 if (is_masked_op && !use_predicate && !arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad)) {
1470 log_if_needed(" ** not supported: arity=1 op=%d/reduce vlen=%d etype=%s is_masked_op=1",
1471 sopc, num_elem, type2name(elem_bt));
1472 return false;
1473 }
1474
1475 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1476 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1477
1478 Node* opd = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
1479 if (opd == nullptr) {
1480 return false; // operand unboxing failed
1481 }
1482
1483 Node* mask = nullptr;
1484 if (is_masked_op) {
1485 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1486 assert(is_vector_mask(mbox_klass), "argument(2) should be a mask class");
1487 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1488 mask = unbox_vector(argument(6), mbox_type, elem_bt, num_elem);
1489 if (mask == nullptr) {
1490 log_if_needed(" ** unbox failed mask=%s",
1491 NodeClassNames[argument(6)->Opcode()]);
1492 return false;
1493 }
1494 }
1495
1496 Node* init = ReductionNode::make_identity_con_scalar(gvn(), opc, elem_bt);
1497 Node* value = opd;
1498
1499 assert(mask != nullptr || !is_masked_op, "Masked op needs the mask value never null");
1500 if (mask != nullptr && !use_predicate) {
1501 Node* reduce_identity = gvn().transform(VectorNode::scalar2vector(init, num_elem, elem_bt));
1502 value = gvn().transform(new VectorBlendNode(reduce_identity, value, mask));
1503 }
1504
1505 // Make an unordered Reduction node. This affects only AddReductionVF/VD and MulReductionVF/VD,
1506 // as these operations are allowed to be associative (not requiring strict order) in VectorAPI.
1507 value = trace_vector(ReductionNode::make(opc, nullptr, init, value, elem_bt, /* requires_strict_order */ false));
1508
1509 if (mask != nullptr && use_predicate) {
1510 value->add_req(mask);
1511 value->add_flag(Node::Flag_is_predicated_vector);
1512 }
1513
1514 value = gvn().transform(value);
1515
1516 Node* bits = nullptr;
1517 switch (elem_bt) {
1518 case T_BYTE:
1519 case T_SHORT:
1520 case T_INT: {
1521 bits = gvn().transform(new ConvI2LNode(value));
1522 break;
1523 }
1524 case T_FLOAT: {
1525 value = gvn().transform(new MoveF2INode(value));
1526 bits = gvn().transform(new ConvI2LNode(value));
1527 break;
1528 }
1529 case T_DOUBLE: {
1530 bits = gvn().transform(new MoveD2LNode(value));
1531 break;
1532 }
1533 case T_LONG: {
1534 bits = value; // no conversion needed
1535 break;
1536 }
1537 default: fatal("%s", type2name(elem_bt));
1538 }
1539 set_result(bits);
1540 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1541 return true;
1542 }
1543
1544 // public static <V> boolean test(int cond, Class<?> vectorClass, Class<?> elementType, int vlen,
1545 // V v1, V v2,
1546 // BiFunction<V, V, Boolean> defaultImpl)
1547 //
1548 bool LibraryCallKit::inline_vector_test() {
1549 const TypeInt* cond = gvn().type(argument(0))->isa_int();
1550 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
1551 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
1552 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
1553
1554 if (cond == nullptr || !cond->is_con() ||
1555 vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
1556 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
1557 vlen == nullptr || !vlen->is_con()) {
1558 log_if_needed(" ** missing constant: cond=%s vclass=%s etype=%s vlen=%s",
1559 NodeClassNames[argument(0)->Opcode()],
1560 NodeClassNames[argument(1)->Opcode()],
1561 NodeClassNames[argument(2)->Opcode()],
1562 NodeClassNames[argument(3)->Opcode()]);
1563 return false; // not enough info for intrinsification
1564 }
1565 if (!is_klass_initialized(vector_klass)) {
1566 log_if_needed(" ** klass argument not initialized");
1567 return false;
1568 }
1569 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1570 if (!elem_type->is_primitive_type()) {
1571 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1572 return false; // should be primitive type
1573 }
1574 BasicType elem_bt = elem_type->basic_type();
1575 int num_elem = vlen->get_con();
1576 BoolTest::mask booltest = (BoolTest::mask)cond->get_con();
1577 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1578 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1579
1580 if (!arch_supports_vector(Op_VectorTest, num_elem, elem_bt, is_vector_mask(vbox_klass) ? VecMaskUseLoad : VecMaskNotUsed)) {
1581 log_if_needed(" ** not supported: arity=2 op=test/%d vlen=%d etype=%s ismask=%d",
1582 cond->get_con(), num_elem, type2name(elem_bt),
1583 is_vector_mask(vbox_klass));
1584 return false;
1585 }
1586
1587 Node* opd1 = unbox_vector(argument(4), vbox_type, elem_bt, num_elem);
1588 Node* opd2;
1589 if (Matcher::vectortest_needs_second_argument(booltest == BoolTest::overflow,
1590 opd1->bottom_type()->isa_vectmask())) {
1591 opd2 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
1592 } else {
1593 opd2 = opd1;
1594 }
1595 if (opd1 == nullptr || opd2 == nullptr) {
1596 return false; // operand unboxing failed
1597 }
1598
1599 Node* cmp = gvn().transform(trace_vector(new VectorTestNode(opd1, opd2, booltest)));
1600 BoolTest::mask test = Matcher::vectortest_mask(booltest == BoolTest::overflow,
1601 opd1->bottom_type()->isa_vectmask(), num_elem);
1602 Node* bol = gvn().transform(new BoolNode(cmp, test));
1603 Node* res = gvn().transform(new CMoveINode(bol, gvn().intcon(0), gvn().intcon(1), TypeInt::BOOL));
1604
1605 set_result(res);
1606 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1607 return true;
1608 }
1609
1610 // public static
1611 // <V extends Vector<E>,
1612 // M extends VectorMask<E>,
1613 // E>
1614 // V blend(Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType, int vlen,
1615 // V v1, V v2, M m,
1616 // VectorBlendOp<V, M, E> defaultImpl)
1617 bool LibraryCallKit::inline_vector_blend() {
1618 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
1619 const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
1620 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
1621 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
1622
1623 if (mask_klass == nullptr || vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr) {
1624 return false; // dead code
1625 }
1626 if (mask_klass->const_oop() == nullptr || vector_klass->const_oop() == nullptr ||
1627 elem_klass->const_oop() == nullptr || !vlen->is_con()) {
1628 log_if_needed(" ** missing constant: vclass=%s mclass=%s etype=%s vlen=%s",
1629 NodeClassNames[argument(0)->Opcode()],
1630 NodeClassNames[argument(1)->Opcode()],
1631 NodeClassNames[argument(2)->Opcode()],
1632 NodeClassNames[argument(3)->Opcode()]);
1633 return false; // not enough info for intrinsification
1634 }
1635 if (!is_klass_initialized(vector_klass) || !is_klass_initialized(mask_klass)) {
1636 log_if_needed(" ** klass argument not initialized");
1637 return false;
1638 }
1639 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1640 if (!elem_type->is_primitive_type()) {
1641 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1642 return false; // should be primitive type
1643 }
1644 BasicType elem_bt = elem_type->basic_type();
1645 BasicType mask_bt = elem_bt;
1646 int num_elem = vlen->get_con();
1647
1648 if (!arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad)) {
1649 log_if_needed(" ** not supported: arity=2 op=blend vlen=%d etype=%s ismask=useload",
1650 num_elem, type2name(elem_bt));
1651 return false; // not supported
1652 }
1653 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1654 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1655
1656 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1657 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1658
1659 Node* v1 = unbox_vector(argument(4), vbox_type, elem_bt, num_elem);
1660 Node* v2 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
1661 Node* mask = unbox_vector(argument(6), mbox_type, mask_bt, num_elem);
1662
1663 if (v1 == nullptr || v2 == nullptr || mask == nullptr) {
1664 return false; // operand unboxing failed
1665 }
1666
1667 Node* blend = gvn().transform(trace_vector(new VectorBlendNode(v1, v2, mask)));
1668
1669 Node* box = box_vector(blend, vbox_type, elem_bt, num_elem);
1670 set_result(box);
1671 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1672 return true;
1673 }
1674
1675 // public static
1676 // <V extends Vector<E>,
1677 // M extends VectorMask<E>,
1678 // E>
1679 // M compare(int cond, Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType, int vlen,
1680 // V v1, V v2, M m,
1681 // VectorCompareOp<V,M> defaultImpl)
1682 bool LibraryCallKit::inline_vector_compare() {
1683 const TypeInt* cond = gvn().type(argument(0))->isa_int();
1684 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
1685 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
1686 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
1687 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
1688
1689 if (cond == nullptr || vector_klass == nullptr || mask_klass == nullptr || elem_klass == nullptr || vlen == nullptr) {
1690 return false; // dead code
1691 }
1692 if (!cond->is_con() || vector_klass->const_oop() == nullptr || mask_klass->const_oop() == nullptr ||
1693 elem_klass->const_oop() == nullptr || !vlen->is_con()) {
1694 log_if_needed(" ** missing constant: cond=%s vclass=%s mclass=%s etype=%s vlen=%s",
1695 NodeClassNames[argument(0)->Opcode()],
1696 NodeClassNames[argument(1)->Opcode()],
1697 NodeClassNames[argument(2)->Opcode()],
1698 NodeClassNames[argument(3)->Opcode()],
1699 NodeClassNames[argument(4)->Opcode()]);
1700 return false; // not enough info for intrinsification
1701 }
1702 if (!is_klass_initialized(vector_klass) || !is_klass_initialized(mask_klass)) {
1703 log_if_needed(" ** klass argument not initialized");
1704 return false;
1705 }
1706 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1707 if (!elem_type->is_primitive_type()) {
1708 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1709 return false; // should be primitive type
1710 }
1711
1712 int num_elem = vlen->get_con();
1713 BasicType elem_bt = elem_type->basic_type();
1714 BasicType mask_bt = elem_bt;
1715
1716 if ((cond->get_con() & BoolTest::unsigned_compare) != 0) {
1717 if (!Matcher::supports_vector_comparison_unsigned(num_elem, elem_bt)) {
1718 log_if_needed(" ** not supported: unsigned comparison op=comp/%d vlen=%d etype=%s ismask=usestore",
1719 cond->get_con() & (BoolTest::unsigned_compare - 1), num_elem, type2name(elem_bt));
1720 return false;
1721 }
1722 }
1723
1724 if (!arch_supports_vector(Op_VectorMaskCmp, num_elem, elem_bt, VecMaskUseStore)) {
1725 log_if_needed(" ** not supported: arity=2 op=comp/%d vlen=%d etype=%s ismask=usestore",
1726 cond->get_con(), num_elem, type2name(elem_bt));
1727 return false;
1728 }
1729
1730 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1731 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1732
1733 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1734 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1735
1736 Node* v1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
1737 Node* v2 = unbox_vector(argument(6), vbox_type, elem_bt, num_elem);
1738
1739 bool is_masked_op = argument(7)->bottom_type() != TypePtr::NULL_PTR;
1740 Node* mask = is_masked_op ? unbox_vector(argument(7), mbox_type, elem_bt, num_elem) : nullptr;
1741 if (is_masked_op && mask == nullptr) {
1742 log_if_needed(" ** not supported: mask = null arity=2 op=comp/%d vlen=%d etype=%s ismask=usestore is_masked_op=1",
1743 cond->get_con(), num_elem, type2name(elem_bt));
1744 return false;
1745 }
1746
1747 bool use_predicate = is_masked_op && arch_supports_vector(Op_VectorMaskCmp, num_elem, elem_bt, VecMaskUsePred);
1748 if (is_masked_op && !use_predicate && !arch_supports_vector(Op_AndV, num_elem, elem_bt, VecMaskUseLoad)) {
1749 log_if_needed(" ** not supported: arity=2 op=comp/%d vlen=%d etype=%s ismask=usestore is_masked_op=1",
1750 cond->get_con(), num_elem, type2name(elem_bt));
1751 return false;
1752 }
1753
1754 if (v1 == nullptr || v2 == nullptr) {
1755 return false; // operand unboxing failed
1756 }
1757 BoolTest::mask pred = (BoolTest::mask)cond->get_con();
1758 ConINode* pred_node = (ConINode*)gvn().makecon(cond);
1759
1760 const TypeVect* vmask_type = TypeVect::makemask(mask_bt, num_elem);
1761 Node* operation = new VectorMaskCmpNode(pred, v1, v2, pred_node, vmask_type);
1762 trace_vector(operation);
1763
1764 if (is_masked_op) {
1765 if (use_predicate) {
1766 operation->add_req(mask);
1767 operation->add_flag(Node::Flag_is_predicated_vector);
1768 } else {
1769 operation = gvn().transform(operation);
1770 operation = VectorNode::make(Op_AndV, operation, mask, vmask_type);
1771 }
1772 }
1773
1774 operation = gvn().transform(operation);
1775
1776 Node* box = box_vector(operation, mbox_type, mask_bt, num_elem);
1777 set_result(box);
1778 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1779 return true;
1780 }
1781
1782 // public static
1783 // <V extends Vector<E>,
1784 // Sh extends VectorShuffle<E>,
1785 // M extends VectorMask<E>,
1786 // E>
1787 // V rearrangeOp(Class<? extends V> vectorClass, Class<Sh> shuffleClass, Class<M> maskClass, Class<E> elementType, int vlen,
1788 // V v1, Sh sh, M m,
1789 // VectorRearrangeOp<V, Sh, M, E> defaultImpl)
1790 bool LibraryCallKit::inline_vector_rearrange() {
1791 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
1792 const TypeInstPtr* shuffle_klass = gvn().type(argument(1))->isa_instptr();
1793 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
1794 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
1795 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
1796
1797 if (vector_klass == nullptr || shuffle_klass == nullptr || elem_klass == nullptr || vlen == nullptr) {
1798 return false; // dead code
1799 }
1800 if (shuffle_klass->const_oop() == nullptr ||
1801 vector_klass->const_oop() == nullptr ||
1802 elem_klass->const_oop() == nullptr ||
1803 !vlen->is_con()) {
1804 log_if_needed(" ** missing constant: vclass=%s sclass=%s etype=%s vlen=%s",
1805 NodeClassNames[argument(0)->Opcode()],
1806 NodeClassNames[argument(1)->Opcode()],
1807 NodeClassNames[argument(3)->Opcode()],
1808 NodeClassNames[argument(4)->Opcode()]);
1809 return false; // not enough info for intrinsification
1810 }
1811 if (!is_klass_initialized(vector_klass) ||
1812 !is_klass_initialized(shuffle_klass)) {
1813 log_if_needed(" ** klass argument not initialized");
1814 return false;
1815 }
1816 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1817 if (!elem_type->is_primitive_type()) {
1818 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1819 return false; // should be primitive type
1820 }
1821
1822 BasicType elem_bt = elem_type->basic_type();
1823 BasicType shuffle_bt = elem_bt;
1824 if (shuffle_bt == T_FLOAT) {
1825 shuffle_bt = T_INT;
1826 } else if (shuffle_bt == T_DOUBLE) {
1827 shuffle_bt = T_LONG;
1828 }
1829
1830 int num_elem = vlen->get_con();
1831 bool need_load_shuffle = Matcher::vector_rearrange_requires_load_shuffle(shuffle_bt, num_elem);
1832
1833 if (need_load_shuffle && !arch_supports_vector(Op_VectorLoadShuffle, num_elem, shuffle_bt, VecMaskNotUsed)) {
1834 if (C->print_intrinsics()) {
1835 tty->print_cr(" ** not supported: arity=0 op=load/shuffle vlen=%d etype=%s ismask=no",
1836 num_elem, type2name(shuffle_bt));
1837 }
1838 return false; // not supported
1839 }
1840
1841 bool is_masked_op = argument(7)->bottom_type() != TypePtr::NULL_PTR;
1842 bool use_predicate = is_masked_op;
1843 if (is_masked_op &&
1844 (mask_klass == nullptr ||
1845 mask_klass->const_oop() == nullptr ||
1846 !is_klass_initialized(mask_klass))) {
1847 log_if_needed(" ** mask_klass argument not initialized");
1848 }
1849 if (!arch_supports_vector(Op_AndV, num_elem, elem_bt, VecMaskNotUsed)) {
1850 log_if_needed(" ** not supported: arity=2 op=and vlen=%d etype=%s ismask=no",
1851 num_elem, type2name(elem_bt));
1852 return false;
1853 }
1854 VectorMaskUseType checkFlags = (VectorMaskUseType)(is_masked_op ? (VecMaskUseLoad | VecMaskUsePred) : VecMaskNotUsed);
1855 if (!arch_supports_vector(Op_VectorRearrange, num_elem, elem_bt, checkFlags)) {
1856 use_predicate = false;
1857 if(!is_masked_op ||
1858 (!arch_supports_vector(Op_VectorRearrange, num_elem, elem_bt, VecMaskNotUsed) ||
1859 !arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad) ||
1860 !arch_supports_vector(Op_Replicate, num_elem, elem_bt, VecMaskNotUsed))) {
1861 log_if_needed(" ** not supported: arity=2 op=shuffle/rearrange vlen=%d etype=%s ismask=no",
1862 num_elem, type2name(elem_bt));
1863 return false; // not supported
1864 }
1865 }
1866 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1867 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1868
1869 ciKlass* shbox_klass = shuffle_klass->const_oop()->as_instance()->java_lang_Class_klass();
1870 const TypeInstPtr* shbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, shbox_klass);
1871
1872 Node* v1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
1873 Node* shuffle = unbox_vector(argument(6), shbox_type, shuffle_bt, num_elem);
1874 const TypeVect* st = TypeVect::make(shuffle_bt, num_elem);
1875
1876 if (v1 == nullptr || shuffle == nullptr) {
1877 return false; // operand unboxing failed
1878 }
1879
1880 assert(is_power_of_2(num_elem), "wrapping invalid");
1881 Node* wrapping_mask_elem = gvn().makecon(TypeInteger::make(num_elem - 1, num_elem - 1, Type::WidenMin, shuffle_bt == T_LONG ? T_LONG : T_INT));
1882 Node* wrapping_mask = gvn().transform(VectorNode::scalar2vector(wrapping_mask_elem, num_elem, shuffle_bt));
1883 shuffle = gvn().transform(new AndVNode(shuffle, wrapping_mask, st));
1884
1885 Node* mask = nullptr;
1886 if (is_masked_op) {
1887 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1888 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1889 mask = unbox_vector(argument(7), mbox_type, elem_bt, num_elem);
1890 if (mask == nullptr) {
1891 log_if_needed(" ** not supported: arity=3 op=shuffle/rearrange vlen=%d etype=%s ismask=useload is_masked_op=1",
1892 num_elem, type2name(elem_bt));
1893 return false;
1894 }
1895 }
1896
1897 if (need_load_shuffle) {
1898 shuffle = gvn().transform(new VectorLoadShuffleNode(shuffle, st));
1899 }
1900
1901 Node* rearrange = new VectorRearrangeNode(v1, shuffle);
1902 trace_vector(rearrange);
1903 if (is_masked_op) {
1904 if (use_predicate) {
1905 rearrange->add_req(mask);
1906 rearrange->add_flag(Node::Flag_is_predicated_vector);
1907 } else {
1908 rearrange = gvn().transform(rearrange);
1909 Node* zero = gvn().makecon(Type::get_zero_type(elem_bt));
1910 Node* zerovec = gvn().transform(VectorNode::scalar2vector(zero, num_elem, elem_bt));
1911 rearrange = new VectorBlendNode(zerovec, rearrange, mask);
1912 }
1913 }
1914 rearrange = gvn().transform(rearrange);
1915
1916 Node* box = box_vector(rearrange, vbox_type, elem_bt, num_elem);
1917 set_result(box);
1918 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1919 return true;
1920 }
1921
1922 // public static
1923 // <V extends Vector<E>,
1924 // M extends VectorMask<E>,
1925 // E>
1926 // V selectFromOp(Class<? extends V> vClass, Class<M> mClass, Class<E> eClass,
1927 // int length, V v1, V v2, M m,
1928 // VectorSelectFromOp<V, M> defaultImpl)
1929 bool LibraryCallKit::inline_vector_select_from() {
1930 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
1931 const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
1932 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
1933 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
1934
1935 if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr ||
1936 vector_klass->const_oop() == nullptr ||
1937 elem_klass->const_oop() == nullptr ||
1938 !vlen->is_con()) {
1939 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s",
1940 NodeClassNames[argument(0)->Opcode()],
1941 NodeClassNames[argument(2)->Opcode()],
1942 NodeClassNames[argument(3)->Opcode()]);
1943 return false; // not enough info for intrinsification
1944 }
1945 if (!is_klass_initialized(vector_klass)) {
1946 log_if_needed(" ** klass argument not initialized");
1947 return false;
1948 }
1949 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1950 if (!elem_type->is_primitive_type()) {
1951 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1952 return false; // should be primitive type
1953 }
1954 BasicType elem_bt = elem_type->basic_type();
1955 int num_elem = vlen->get_con();
1956 if (!is_power_of_2(num_elem)) {
1957 log_if_needed(" ** vlen not power of two=%d", num_elem);
1958 return false;
1959 }
1960
1961 BasicType shuffle_bt = elem_bt;
1962 if (shuffle_bt == T_FLOAT) {
1963 shuffle_bt = T_INT;
1964 } else if (shuffle_bt == T_DOUBLE) {
1965 shuffle_bt = T_LONG;
1966 }
1967 bool need_load_shuffle = Matcher::vector_rearrange_requires_load_shuffle(shuffle_bt, num_elem);
1968
1969 int cast_vopc = VectorCastNode::opcode(-1, elem_bt); // from vector of type elem_bt
1970 if ((need_load_shuffle && !arch_supports_vector(Op_VectorLoadShuffle, num_elem, elem_bt, VecMaskNotUsed)) ||
1971 (elem_bt != shuffle_bt && !arch_supports_vector(cast_vopc, num_elem, shuffle_bt, VecMaskNotUsed)) ||
1972 !arch_supports_vector(Op_AndV, num_elem, shuffle_bt, VecMaskNotUsed) ||
1973 !arch_supports_vector(Op_Replicate, num_elem, shuffle_bt, VecMaskNotUsed)) {
1974 log_if_needed(" ** not supported: arity=0 op=selectFrom vlen=%d etype=%s ismask=no",
1975 num_elem, type2name(elem_bt));
1976 return false; // not supported
1977 }
1978
1979 bool is_masked_op = argument(6)->bottom_type() != TypePtr::NULL_PTR;
1980 bool use_predicate = is_masked_op;
1981 if (is_masked_op &&
1982 (mask_klass == nullptr ||
1983 mask_klass->const_oop() == nullptr ||
1984 !is_klass_initialized(mask_klass))) {
1985 log_if_needed(" ** mask_klass argument not initialized");
1986 return false; // not supported
1987 }
1988 VectorMaskUseType checkFlags = (VectorMaskUseType)(is_masked_op ? (VecMaskUseLoad | VecMaskUsePred) : VecMaskNotUsed);
1989 if (!arch_supports_vector(Op_VectorRearrange, num_elem, elem_bt, checkFlags)) {
1990 use_predicate = false;
1991 if(!is_masked_op ||
1992 (!arch_supports_vector(Op_VectorRearrange, num_elem, elem_bt, VecMaskNotUsed) ||
1993 !arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad) ||
1994 !arch_supports_vector(Op_Replicate, num_elem, elem_bt, VecMaskNotUsed))) {
1995 log_if_needed(" ** not supported: op=selectFrom vlen=%d etype=%s is_masked_op=%d",
1996 num_elem, type2name(elem_bt), is_masked_op);
1997 return false; // not supported
1998 }
1999 }
2000 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2001 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2002
2003 // v1 is the index vector
2004 Node* v1 = unbox_vector(argument(4), vbox_type, elem_bt, num_elem);
2005 // v2 is the vector being rearranged
2006 Node* v2 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
2007
2008 if (v1 == nullptr) {
2009 log_if_needed(" ** unbox failed v1=%s", NodeClassNames[argument(4)->Opcode()]);
2010 return false; // operand unboxing failed
2011 }
2012
2013 if (v2 == nullptr) {
2014 log_if_needed(" ** unbox failed v2=%s", NodeClassNames[argument(5)->Opcode()]);
2015 return false; // operand unboxing failed
2016 }
2017
2018 Node* mask = nullptr;
2019 if (is_masked_op) {
2020 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
2021 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
2022 mask = unbox_vector(argument(6), mbox_type, elem_bt, num_elem);
2023 if (mask == nullptr) {
2024 log_if_needed(" ** unbox failed mask=%s", NodeClassNames[argument(6)->Opcode()]);
2025 return false;
2026 }
2027 }
2028
2029 // cast index vector from elem_bt vector to byte vector
2030 const TypeVect* shuffle_vt = TypeVect::make(shuffle_bt, num_elem);
2031 Node* shuffle = v1;
2032
2033 if (shuffle_bt != elem_bt) {
2034 shuffle = gvn().transform(VectorCastNode::make(cast_vopc, v1, shuffle_bt, num_elem));
2035 }
2036
2037 // wrap the byte vector lanes to (num_elem - 1) to form the shuffle vector where num_elem is vector length
2038 // this is a simple AND operation as we come here only for power of two vector length
2039 Node* mod_val = gvn().makecon(TypeInteger::make(num_elem - 1, num_elem - 1, Type::WidenMin, shuffle_bt == T_LONG ? T_LONG : T_INT));
2040 Node* bcast_mod = gvn().transform(VectorNode::scalar2vector(mod_val, num_elem, shuffle_bt));
2041 shuffle = gvn().transform(VectorNode::make(Op_AndV, shuffle, bcast_mod, shuffle_vt));
2042
2043 // load the shuffle to use in rearrange
2044 if (need_load_shuffle) {
2045 shuffle = gvn().transform(new VectorLoadShuffleNode(shuffle, shuffle_vt));
2046 }
2047
2048 // and finally rearrange
2049 Node* rearrange = new VectorRearrangeNode(v2, shuffle);
2050 trace_vector(rearrange);
2051 if (is_masked_op) {
2052 if (use_predicate) {
2053 // masked rearrange is supported so use that directly
2054 rearrange->add_req(mask);
2055 rearrange->add_flag(Node::Flag_is_predicated_vector);
2056 } else {
2057 // masked rearrange is not supported so emulate usig blend
2058 const TypeVect* vt = v1->bottom_type()->is_vect();
2059 rearrange = gvn().transform(rearrange);
2060
2061 // create a zero vector with each lane element set as zero
2062 Node* zero = gvn().makecon(Type::get_zero_type(elem_bt));
2063 Node* zerovec = gvn().transform(VectorNode::scalar2vector(zero, num_elem, elem_bt));
2064
2065 // For each lane for which mask is set, blend in the rearranged lane into zero vector
2066 rearrange = new VectorBlendNode(zerovec, rearrange, mask);
2067 }
2068 }
2069 rearrange = gvn().transform(rearrange);
2070
2071 // box the result
2072 Node* box = box_vector(rearrange, vbox_type, elem_bt, num_elem);
2073 set_result(box);
2074
2075 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2076 return true;
2077 }
2078
2079 // public static
2080 // <V extends Vector<E>,
2081 // M extends VectorMask<E>,
2082 // E>
2083 // V broadcastInt(int opr, Class<? extends V> vectorClass, Class<? extends M> maskClass,
2084 // Class<E> elementType, int length,
2085 // V v, int n, M m,
2086 // VectorBroadcastIntOp<V, M> defaultImpl)
2087 bool LibraryCallKit::inline_vector_broadcast_int() {
2088 const TypeInt* opr = gvn().type(argument(0))->isa_int();
2089 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
2090 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
2091 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
2092 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
2093
2094 if (opr == nullptr || vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr) {
2095 return false; // dead code
2096 }
2097 if (!opr->is_con() || vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
2098 log_if_needed(" ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
2099 NodeClassNames[argument(0)->Opcode()],
2100 NodeClassNames[argument(1)->Opcode()],
2101 NodeClassNames[argument(3)->Opcode()],
2102 NodeClassNames[argument(4)->Opcode()]);
2103 return false; // not enough info for intrinsification
2104 }
2105 if (!is_klass_initialized(vector_klass)) {
2106 log_if_needed(" ** klass argument not initialized");
2107 return false;
2108 }
2109
2110 const Type* vmask_type = gvn().type(argument(7));
2111 bool is_masked_op = vmask_type != TypePtr::NULL_PTR;
2112 if (is_masked_op) {
2113 if (mask_klass == nullptr || mask_klass->const_oop() == nullptr) {
2114 log_if_needed(" ** missing constant: maskclass=%s", NodeClassNames[argument(2)->Opcode()]);
2115 return false; // not enough info for intrinsification
2116 }
2117
2118 if (!is_klass_initialized(mask_klass)) {
2119 log_if_needed(" ** mask klass argument not initialized");
2120 return false;
2121 }
2122
2123 if (vmask_type->maybe_null()) {
2124 log_if_needed(" ** null mask values are not allowed for masked op");
2125 return false;
2126 }
2127 }
2128
2129 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2130 if (!elem_type->is_primitive_type()) {
2131 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2132 return false; // should be primitive type
2133 }
2134
2135 int num_elem = vlen->get_con();
2136 BasicType elem_bt = elem_type->basic_type();
2137 int opc = VectorSupport::vop2ideal(opr->get_con(), elem_bt);
2138
2139 bool is_shift = VectorNode::is_shift_opcode(opc);
2140 bool is_rotate = VectorNode::is_rotate_opcode(opc);
2141
2142 if (opc == 0 || (!is_shift && !is_rotate)) {
2143 log_if_needed(" ** operation not supported: op=%d bt=%s", opr->get_con(), type2name(elem_bt));
2144 return false; // operation not supported
2145 }
2146
2147 int sopc = VectorNode::opcode(opc, elem_bt);
2148 if (sopc == 0) {
2149 log_if_needed(" ** operation not supported: opc=%s bt=%s", NodeClassNames[opc], type2name(elem_bt));
2150 return false; // operation not supported
2151 }
2152
2153 Node* cnt = argument(6);
2154 const TypeInt* cnt_type = cnt->bottom_type()->isa_int();
2155
2156 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2157 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2158
2159 // If CPU supports vector constant rotate instructions pass it directly
2160 bool is_const_rotate = is_rotate && cnt_type && cnt_type->is_con() &&
2161 Matcher::supports_vector_constant_rotates(cnt_type->get_con());
2162 bool has_scalar_args = is_rotate ? !is_const_rotate : true;
2163
2164 VectorMaskUseType checkFlags = (VectorMaskUseType)(is_masked_op ? (VecMaskUseLoad | VecMaskUsePred) : VecMaskNotUsed);
2165 bool use_predicate = is_masked_op;
2166
2167 if (!arch_supports_vector(sopc, num_elem, elem_bt, checkFlags, has_scalar_args)) {
2168 use_predicate = false;
2169 if (!is_masked_op ||
2170 (!arch_supports_vector(sopc, num_elem, elem_bt, VecMaskNotUsed, has_scalar_args) ||
2171 !arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad))) {
2172
2173 log_if_needed(" ** not supported: arity=0 op=int/%d vlen=%d etype=%s is_masked_op=%d",
2174 sopc, num_elem, type2name(elem_bt), is_masked_op ? 1 : 0);
2175 return false; // not supported
2176 }
2177 }
2178
2179 Node* opd1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
2180 Node* opd2 = nullptr;
2181 if (is_shift) {
2182 opd2 = vector_shift_count(cnt, opc, elem_bt, num_elem);
2183 } else {
2184 assert(is_rotate, "unexpected operation");
2185 if (!is_const_rotate) {
2186 cnt = elem_bt == T_LONG ? gvn().transform(new ConvI2LNode(cnt)) : cnt;
2187 opd2 = gvn().transform(VectorNode::scalar2vector(cnt, num_elem, elem_bt));
2188 } else {
2189 // Constant shift value.
2190 opd2 = cnt;
2191 }
2192 }
2193
2194 if (opd1 == nullptr || opd2 == nullptr) {
2195 return false;
2196 }
2197
2198 Node* mask = nullptr;
2199 if (is_masked_op) {
2200 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
2201 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
2202 mask = unbox_vector(argument(7), mbox_type, elem_bt, num_elem);
2203 if (mask == nullptr) {
2204 log_if_needed(" ** unbox failed mask=%s", NodeClassNames[argument(7)->Opcode()]);
2205 return false;
2206 }
2207 }
2208
2209 Node* operation = VectorNode::make(opc, opd1, opd2, num_elem, elem_bt);
2210 trace_vector(operation);
2211 if (is_masked_op && mask != nullptr) {
2212 if (use_predicate) {
2213 operation->add_req(mask);
2214 operation->add_flag(Node::Flag_is_predicated_vector);
2215 } else {
2216 operation = gvn().transform(operation);
2217 operation = new VectorBlendNode(opd1, operation, mask);
2218 }
2219 }
2220 operation = gvn().transform(operation);
2221 Node* vbox = box_vector(operation, vbox_type, elem_bt, num_elem);
2222 set_result(vbox);
2223 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2224 return true;
2225 }
2226
2227 // public static <VOUT extends VectorPayload,
2228 // VIN extends VectorPayload,
2229 // S extends VectorSpecies>
2230 // VOUT convert(int oprId,
2231 // Class<?> fromVectorClass, Class<?> fromElementType, int fromVLen,
2232 // Class<?> toVectorClass, Class<?> toElementType, int toVLen,
2233 // VIN v, S s,
2234 // VectorConvertOp<VOUT, VIN, S> defaultImpl)
2235 //
2236 bool LibraryCallKit::inline_vector_convert() {
2237 const TypeInt* opr = gvn().type(argument(0))->isa_int();
2238
2239 const TypeInstPtr* vector_klass_from = gvn().type(argument(1))->isa_instptr();
2240 const TypeInstPtr* elem_klass_from = gvn().type(argument(2))->isa_instptr();
2241 const TypeInt* vlen_from = gvn().type(argument(3))->isa_int();
2242
2243 const TypeInstPtr* vector_klass_to = gvn().type(argument(4))->isa_instptr();
2244 const TypeInstPtr* elem_klass_to = gvn().type(argument(5))->isa_instptr();
2245 const TypeInt* vlen_to = gvn().type(argument(6))->isa_int();
2246
2247 if (opr == nullptr ||
2248 vector_klass_from == nullptr || elem_klass_from == nullptr || vlen_from == nullptr ||
2249 vector_klass_to == nullptr || elem_klass_to == nullptr || vlen_to == nullptr) {
2250 return false; // dead code
2251 }
2252 if (!opr->is_con() ||
2253 vector_klass_from->const_oop() == nullptr || elem_klass_from->const_oop() == nullptr || !vlen_from->is_con() ||
2254 vector_klass_to->const_oop() == nullptr || elem_klass_to->const_oop() == nullptr || !vlen_to->is_con()) {
2255 log_if_needed(" ** missing constant: opr=%s vclass_from=%s etype_from=%s vlen_from=%s vclass_to=%s etype_to=%s vlen_to=%s",
2256 NodeClassNames[argument(0)->Opcode()],
2257 NodeClassNames[argument(1)->Opcode()],
2258 NodeClassNames[argument(2)->Opcode()],
2259 NodeClassNames[argument(3)->Opcode()],
2260 NodeClassNames[argument(4)->Opcode()],
2261 NodeClassNames[argument(5)->Opcode()],
2262 NodeClassNames[argument(6)->Opcode()]);
2263 return false; // not enough info for intrinsification
2264 }
2265 if (!is_klass_initialized(vector_klass_from) || !is_klass_initialized(vector_klass_to)) {
2266 log_if_needed(" ** klass argument not initialized");
2267 return false;
2268 }
2269
2270 assert(opr->get_con() == VectorSupport::VECTOR_OP_CAST ||
2271 opr->get_con() == VectorSupport::VECTOR_OP_UCAST ||
2272 opr->get_con() == VectorSupport::VECTOR_OP_REINTERPRET, "wrong opcode");
2273 bool is_cast = (opr->get_con() == VectorSupport::VECTOR_OP_CAST || opr->get_con() == VectorSupport::VECTOR_OP_UCAST);
2274 bool is_ucast = (opr->get_con() == VectorSupport::VECTOR_OP_UCAST);
2275
2276 ciKlass* vbox_klass_from = vector_klass_from->const_oop()->as_instance()->java_lang_Class_klass();
2277 ciKlass* vbox_klass_to = vector_klass_to->const_oop()->as_instance()->java_lang_Class_klass();
2278
2279 bool is_mask = is_vector_mask(vbox_klass_from);
2280
2281 ciType* elem_type_from = elem_klass_from->const_oop()->as_instance()->java_mirror_type();
2282 if (!elem_type_from->is_primitive_type()) {
2283 return false; // should be primitive type
2284 }
2285 BasicType elem_bt_from = elem_type_from->basic_type();
2286 ciType* elem_type_to = elem_klass_to->const_oop()->as_instance()->java_mirror_type();
2287 if (!elem_type_to->is_primitive_type()) {
2288 return false; // should be primitive type
2289 }
2290 BasicType elem_bt_to = elem_type_to->basic_type();
2291
2292 int num_elem_from = vlen_from->get_con();
2293 int num_elem_to = vlen_to->get_con();
2294
2295 // Check whether we can unbox to appropriate size. Even with casting, checking for reinterpret is needed
2296 // since we may need to change size.
2297 if (!arch_supports_vector(Op_VectorReinterpret,
2298 num_elem_from,
2299 elem_bt_from,
2300 is_mask ? VecMaskUseAll : VecMaskNotUsed)) {
2301 log_if_needed(" ** not supported: arity=1 op=%s/1 vlen1=%d etype1=%s ismask=%d",
2302 is_cast ? "cast" : "reinterpret",
2303 num_elem_from, type2name(elem_bt_from), is_mask);
2304 return false;
2305 }
2306
2307 // Check whether we can support resizing/reinterpreting to the new size.
2308 if (!arch_supports_vector(Op_VectorReinterpret,
2309 num_elem_to,
2310 elem_bt_to,
2311 is_mask ? VecMaskUseAll : VecMaskNotUsed)) {
2312 log_if_needed(" ** not supported: arity=1 op=%s/2 vlen2=%d etype2=%s ismask=%d",
2313 is_cast ? "cast" : "reinterpret",
2314 num_elem_to, type2name(elem_bt_to), is_mask);
2315 return false;
2316 }
2317
2318 // At this point, we know that both input and output vector registers are supported
2319 // by the architecture. Next check if the casted type is simply to same type - which means
2320 // that it is actually a resize and not a cast.
2321 if (is_cast && elem_bt_from == elem_bt_to) {
2322 is_cast = false;
2323 }
2324
2325 const TypeInstPtr* vbox_type_from = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass_from);
2326
2327 Node* opd1 = unbox_vector(argument(7), vbox_type_from, elem_bt_from, num_elem_from);
2328 if (opd1 == nullptr) {
2329 return false;
2330 }
2331
2332 const TypeVect* src_type = TypeVect::make(elem_bt_from, num_elem_from, is_mask);
2333 const TypeVect* dst_type = TypeVect::make(elem_bt_to, num_elem_to, is_mask);
2334
2335 // Safety check to prevent casting if source mask is of type vector
2336 // and destination mask of type predicate vector and vice-versa.
2337 // From X86 standpoint, this case will only arise over KNL target,
2338 // where certain masks (depending on the species) are either propagated
2339 // through a vector or predicate register.
2340 if (is_mask &&
2341 ((src_type->isa_vectmask() == nullptr && dst_type->isa_vectmask()) ||
2342 (dst_type->isa_vectmask() == nullptr && src_type->isa_vectmask()))) {
2343 return false;
2344 }
2345
2346 Node* op = opd1;
2347 if (is_cast) {
2348 assert(!is_mask || num_elem_from == num_elem_to, "vector mask cast needs the same elem num");
2349
2350 // Make sure the precondition of VectorCastNode::opcode holds: we can only have
2351 // unsigned casts for integral types (excluding long). VectorAPI code is not
2352 // expected to violate this at runtime, but we may compile unreachable code
2353 // where such impossible combinations arise.
2354 if (is_ucast && (!is_integral_type(elem_bt_from) || elem_bt_from == T_LONG)) {
2355 // Halt-and-catch fire here. This condition should never happen at runtime.
2356 stringStream ss;
2357 ss.print("impossible combination: unsigned vector cast from %s", type2name(elem_bt_from));
2358 halt(control(), frameptr(), ss.as_string(C->comp_arena()));
2359 stop_and_kill_map();
2360 log_if_needed(" ** impossible combination: unsigned cast from %s", type2name(elem_bt_from));
2361 return true;
2362 }
2363
2364 int cast_vopc = VectorCastNode::opcode(-1, elem_bt_from, !is_ucast);
2365
2366 // Make sure that vector cast is implemented to particular type/size combination if it is
2367 // not a mask casting.
2368 if (!is_mask && !arch_supports_vector(cast_vopc, num_elem_to, elem_bt_to, VecMaskNotUsed)) {
2369 log_if_needed(" ** not supported: arity=1 op=cast#%d/3 vlen2=%d etype2=%s ismask=%d",
2370 cast_vopc, num_elem_to, type2name(elem_bt_to), is_mask);
2371 return false;
2372 }
2373
2374 if (num_elem_from < num_elem_to) {
2375 // Since input and output number of elements are not consistent, we need to make sure we
2376 // properly size. Thus, first make a cast that retains the number of elements from source.
2377 int num_elem_for_cast = num_elem_from;
2378
2379 // It is possible that arch does not support this intermediate vector size
2380 // TODO More complex logic required here to handle this corner case for the sizes.
2381 if (!arch_supports_vector(cast_vopc, num_elem_for_cast, elem_bt_to, VecMaskNotUsed)) {
2382 log_if_needed(" ** not supported: arity=1 op=cast#%d/4 vlen1=%d etype2=%s ismask=%d",
2383 cast_vopc,
2384 num_elem_for_cast, type2name(elem_bt_to), is_mask);
2385 return false;
2386 }
2387
2388 op = gvn().transform(trace_vector(VectorCastNode::make(cast_vopc, op, elem_bt_to, num_elem_for_cast)));
2389 // Now ensure that the destination gets properly resized to needed size.
2390 op = gvn().transform(new VectorReinterpretNode(op, op->bottom_type()->is_vect(), dst_type));
2391 } else if (num_elem_from > num_elem_to) {
2392 // Since number of elements from input is larger than output, simply reduce size of input
2393 // (we are supposed to drop top elements anyway).
2394 int num_elem_for_resize = num_elem_to;
2395
2396 // It is possible that arch does not support this intermediate vector size
2397 // TODO More complex logic required here to handle this corner case for the sizes.
2398 if (!arch_supports_vector(Op_VectorReinterpret,
2399 num_elem_for_resize,
2400 elem_bt_from,
2401 VecMaskNotUsed)) {
2402 log_if_needed(" ** not supported: arity=1 op=cast/5 vlen2=%d etype1=%s ismask=%d",
2403 num_elem_for_resize, type2name(elem_bt_from), is_mask);
2404 return false;
2405 }
2406
2407 const TypeVect* resize_type = TypeVect::make(elem_bt_from, num_elem_for_resize);
2408 op = gvn().transform(new VectorReinterpretNode(op, src_type, resize_type));
2409 op = gvn().transform(trace_vector(VectorCastNode::make(cast_vopc, op, elem_bt_to, num_elem_to)));
2410 } else { // num_elem_from == num_elem_to
2411 if (is_mask) {
2412 // Make sure that cast for vector mask is implemented to particular type/size combination.
2413 if (!arch_supports_vector(Op_VectorMaskCast, num_elem_to, elem_bt_to, VecMaskNotUsed)) {
2414 log_if_needed(" ** not supported: arity=1 op=maskcast vlen2=%d etype2=%s ismask=%d",
2415 num_elem_to, type2name(elem_bt_to), is_mask);
2416 return false;
2417 }
2418 op = gvn().transform(trace_vector(new VectorMaskCastNode(op, dst_type)));
2419 } else {
2420 // Since input and output number of elements match, and since we know this vector size is
2421 // supported, simply do a cast with no resize needed.
2422 op = gvn().transform(trace_vector(VectorCastNode::make(cast_vopc, op, elem_bt_to, num_elem_to)));
2423 }
2424 }
2425 } else if (!Type::equals(src_type, dst_type)) {
2426 assert(!is_cast, "must be reinterpret");
2427 op = gvn().transform(trace_vector(new VectorReinterpretNode(op, src_type, dst_type)));
2428 }
2429
2430 const TypeInstPtr* vbox_type_to = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass_to);
2431 Node* vbox = box_vector(op, vbox_type_to, elem_bt_to, num_elem_to);
2432 set_result(vbox);
2433 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem_to * type2aelembytes(elem_bt_to))));
2434 return true;
2435 }
2436
2437 // public static
2438 // <V extends Vector<E>,
2439 // E>
2440 // V insert(Class<? extends V> vectorClass, Class<E> elementType, int vlen,
2441 // V vec, int ix, long val,
2442 // VecInsertOp<V> defaultImpl)
2443 bool LibraryCallKit::inline_vector_insert() {
2444 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
2445 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
2446 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
2447 const TypeInt* idx = gvn().type(argument(4))->isa_int();
2448
2449 if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr || idx == nullptr) {
2450 return false; // dead code
2451 }
2452 if (vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con() || !idx->is_con()) {
2453 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s idx=%s",
2454 NodeClassNames[argument(0)->Opcode()],
2455 NodeClassNames[argument(1)->Opcode()],
2456 NodeClassNames[argument(2)->Opcode()],
2457 NodeClassNames[argument(4)->Opcode()]);
2458 return false; // not enough info for intrinsification
2459 }
2460 if (!is_klass_initialized(vector_klass)) {
2461 log_if_needed(" ** klass argument not initialized");
2462 return false;
2463 }
2464 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2465 if (!elem_type->is_primitive_type()) {
2466 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2467 return false; // should be primitive type
2468 }
2469 BasicType elem_bt = elem_type->basic_type();
2470 int num_elem = vlen->get_con();
2471 if (!arch_supports_vector(Op_VectorInsert, num_elem, elem_bt, VecMaskNotUsed)) {
2472 log_if_needed(" ** not supported: arity=1 op=insert vlen=%d etype=%s ismask=no",
2473 num_elem, type2name(elem_bt));
2474 return false; // not supported
2475 }
2476
2477 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2478 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2479
2480 Node* opd = unbox_vector(argument(3), vbox_type, elem_bt, num_elem);
2481 if (opd == nullptr) {
2482 return false;
2483 }
2484
2485 Node* insert_val = argument(5);
2486 assert(gvn().type(insert_val)->isa_long() != nullptr, "expected to be long");
2487
2488 // Convert insert value back to its appropriate type.
2489 switch (elem_bt) {
2490 case T_BYTE:
2491 insert_val = gvn().transform(new ConvL2INode(insert_val, TypeInt::BYTE));
2492 break;
2493 case T_SHORT:
2494 insert_val = gvn().transform(new ConvL2INode(insert_val, TypeInt::SHORT));
2495 break;
2496 case T_INT:
2497 insert_val = gvn().transform(new ConvL2INode(insert_val));
2498 break;
2499 case T_FLOAT:
2500 insert_val = gvn().transform(new ConvL2INode(insert_val));
2501 insert_val = gvn().transform(new MoveI2FNode(insert_val));
2502 break;
2503 case T_DOUBLE:
2504 insert_val = gvn().transform(new MoveL2DNode(insert_val));
2505 break;
2506 case T_LONG:
2507 // no conversion needed
2508 break;
2509 default: fatal("%s", type2name(elem_bt)); break;
2510 }
2511
2512 Node* operation = gvn().transform(trace_vector(VectorInsertNode::make(opd, insert_val, idx->get_con(), gvn())));
2513
2514 Node* vbox = box_vector(operation, vbox_type, elem_bt, num_elem);
2515 set_result(vbox);
2516 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2517 return true;
2518 }
2519
2520 // public static
2521 // <VM extends VectorPayload,
2522 // E>
2523 // long extract(Class<? extends VM> vClass, Class<E> eClass,
2524 // int length,
2525 // VM vm, int i,
2526 // VecExtractOp<VM> defaultImpl)
2527 bool LibraryCallKit::inline_vector_extract() {
2528 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
2529 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
2530 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
2531 const TypeInt* idx = gvn().type(argument(4))->isa_int();
2532
2533 if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr || idx == nullptr) {
2534 return false; // dead code
2535 }
2536 if (vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
2537 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s",
2538 NodeClassNames[argument(0)->Opcode()],
2539 NodeClassNames[argument(1)->Opcode()],
2540 NodeClassNames[argument(2)->Opcode()]);
2541 return false; // not enough info for intrinsification
2542 }
2543 if (!is_klass_initialized(vector_klass)) {
2544 log_if_needed(" ** klass argument not initialized");
2545 return false;
2546 }
2547 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2548 if (!elem_type->is_primitive_type()) {
2549 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2550 return false; // should be primitive type
2551 }
2552 BasicType elem_bt = elem_type->basic_type();
2553 int num_elem = vlen->get_con();
2554
2555 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2556 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2557
2558 Node* opd = nullptr;
2559
2560 if (is_vector_mask(vbox_klass)) {
2561 // vbox_klass is mask. This is used for VectorMask.laneIsSet(int).
2562
2563 Node* pos = argument(4); // can be variable
2564 if (arch_supports_vector(Op_ExtractUB, num_elem, elem_bt, VecMaskUseAll)) {
2565 // Transform mask to vector with type of boolean and utilize ExtractUB node.
2566 opd = unbox_vector(argument(3), vbox_type, elem_bt, num_elem);
2567 if (opd == nullptr) {
2568 return false;
2569 }
2570 opd = gvn().transform(trace_vector(VectorStoreMaskNode::make(gvn(), opd, elem_bt, num_elem)));
2571 opd = gvn().transform(new ExtractUBNode(opd, pos));
2572 opd = gvn().transform(new ConvI2LNode(opd));
2573 } else if (arch_supports_vector(Op_VectorMaskToLong, num_elem, elem_bt, VecMaskUseLoad)) {
2574 opd = unbox_vector(argument(3), vbox_type, elem_bt, num_elem);
2575 if (opd == nullptr) {
2576 return false;
2577 }
2578 // VectorMaskToLongNode requires the input is either a mask or a vector with BOOLEAN type.
2579 if (!Matcher::mask_op_prefers_predicate(Op_VectorMaskToLong, opd->bottom_type()->is_vect())) {
2580 opd = gvn().transform(trace_vector(VectorStoreMaskNode::make(gvn(), opd, elem_bt, num_elem)));
2581 }
2582 // ((toLong() >>> pos) & 1L
2583 opd = gvn().transform(new VectorMaskToLongNode(opd, TypeLong::LONG));
2584 opd = gvn().transform(new URShiftLNode(opd, pos));
2585 opd = gvn().transform(new AndLNode(opd, gvn().makecon(TypeLong::ONE)));
2586 } else {
2587 log_if_needed(" ** Rejected mask extraction because architecture does not support it");
2588 return false; // not supported
2589 }
2590 } else {
2591 // vbox_klass is vector. This is used for Vector.lane(int).
2592 if (!idx->is_con()) {
2593 log_if_needed(" ** missing constant: idx=%s", NodeClassNames[argument(4)->Opcode()]);
2594 return false; // not enough info for intrinsification
2595 }
2596
2597 int vopc = ExtractNode::opcode(elem_bt);
2598 if (!arch_supports_vector(vopc, num_elem, elem_bt, VecMaskNotUsed)) {
2599 log_if_needed(" ** not supported: arity=1 op=extract vlen=%d etype=%s ismask=no",
2600 num_elem, type2name(elem_bt));
2601 return false; // not supported
2602 }
2603
2604 opd = unbox_vector(argument(3), vbox_type, elem_bt, num_elem);
2605 if (opd == nullptr) {
2606 return false;
2607 }
2608 ConINode* idx_con = gvn().intcon(idx->get_con())->as_ConI();
2609
2610 opd = gvn().transform(ExtractNode::make(opd, idx_con, elem_bt));
2611 switch (elem_bt) {
2612 case T_BYTE:
2613 case T_SHORT:
2614 case T_INT: {
2615 opd = gvn().transform(new ConvI2LNode(opd));
2616 break;
2617 }
2618 case T_FLOAT: {
2619 opd = gvn().transform(new MoveF2INode(opd));
2620 opd = gvn().transform(new ConvI2LNode(opd));
2621 break;
2622 }
2623 case T_DOUBLE: {
2624 opd = gvn().transform(new MoveD2LNode(opd));
2625 break;
2626 }
2627 case T_LONG: {
2628 // no conversion needed
2629 break;
2630 }
2631 default: fatal("%s", type2name(elem_bt));
2632 }
2633 }
2634 set_result(opd);
2635 return true;
2636 }
2637
2638 static Node* LowerSelectFromTwoVectorOperation(PhaseGVN& phase, Node* index_vec, Node* src1, Node* src2, const TypeVect* vt) {
2639 int num_elem = vt->length();
2640 BasicType elem_bt = vt->element_basic_type();
2641
2642 // Lower selectFrom operation into its constituent operations.
2643 // SelectFromTwoVectorNode =
2644 // (VectorBlend
2645 // (VectorRearrange SRC1 (WRAPED_INDEX AND (VLEN-1))
2646 // (VectorRearrange SRC2 (WRAPED_INDEX AND (VLEN-1))
2647 // MASK)
2648 // Where
2649 // WRAPED_INDEX are computed by wrapping incoming indexes
2650 // to two vector index range [0, VLEN*2) and
2651 // MASK = WRAPED_INDEX < VLEN
2652 //
2653 // IR lowering prevents intrinsification failure and associated argument
2654 // boxing penalties.
2655 //
2656
2657 BasicType shuffle_bt = elem_bt;
2658 if (shuffle_bt == T_FLOAT) {
2659 shuffle_bt = T_INT;
2660 } else if (shuffle_bt == T_DOUBLE) {
2661 shuffle_bt = T_LONG;
2662 }
2663 const TypeVect* st = TypeVect::make(shuffle_bt, num_elem);
2664
2665 // Cast index vector to the corresponding bit type
2666 if (elem_bt != shuffle_bt) {
2667 int cast_vopc = VectorCastNode::opcode(0, elem_bt, true);
2668 index_vec = phase.transform(VectorCastNode::make(cast_vopc, index_vec, shuffle_bt, num_elem));
2669 }
2670
2671 // Wrap indexes into two vector index range [0, VLEN * 2)
2672 Node* two_vect_lane_cnt_m1 = phase.makecon(TypeInteger::make(2 * num_elem - 1, 2 * num_elem - 1, Type::WidenMin, shuffle_bt == T_LONG ? T_LONG : T_INT));
2673 Node* bcast_two_vect_lane_cnt_m1_vec = phase.transform(VectorNode::scalar2vector(two_vect_lane_cnt_m1, num_elem,
2674 shuffle_bt, false));
2675 index_vec = phase.transform(VectorNode::make(Op_AndV, index_vec, bcast_two_vect_lane_cnt_m1_vec, st));
2676
2677 // Compute the blend mask for merging two independently permitted vectors
2678 // using shuffle index in two vector index range [0, VLEN * 2).
2679 BoolTest::mask pred = BoolTest::le;
2680 ConINode* pred_node = phase.makecon(TypeInt::make(pred))->as_ConI();
2681 const TypeVect* vmask_type = TypeVect::makemask(shuffle_bt, num_elem);
2682 Node* lane_cnt_m1 = phase.makecon(TypeInteger::make(num_elem - 1, num_elem - 1, Type::WidenMin, shuffle_bt == T_LONG ? T_LONG : T_INT));
2683 Node* bcast_lane_cnt_m1_vec = phase.transform(VectorNode::scalar2vector(lane_cnt_m1, num_elem, shuffle_bt, false));
2684 Node* mask = phase.transform(new VectorMaskCmpNode(pred, index_vec, bcast_lane_cnt_m1_vec, pred_node, vmask_type));
2685
2686 // Rearrange expects the indexes to lie within single vector index range [0, VLEN).
2687 Node* wrapped_index_vec = phase.transform(VectorNode::make(Op_AndV, index_vec, bcast_lane_cnt_m1_vec, st));
2688
2689 // Load indexes from byte vector and appropriately transform them to target
2690 // specific permutation index format.
2691 if (Matcher::vector_rearrange_requires_load_shuffle(shuffle_bt, num_elem)) {
2692 wrapped_index_vec = phase.transform(new VectorLoadShuffleNode(wrapped_index_vec, st));
2693 }
2694
2695 vmask_type = TypeVect::makemask(elem_bt, num_elem);
2696 mask = phase.transform(new VectorMaskCastNode(mask, vmask_type));
2697
2698 Node* p1 = phase.transform(trace_vector(new VectorRearrangeNode(src1, wrapped_index_vec)));
2699 Node* p2 = phase.transform(trace_vector(new VectorRearrangeNode(src2, wrapped_index_vec)));
2700
2701 return new VectorBlendNode(p2, p1, mask);
2702 }
2703
2704 // public static
2705 // <V extends Vector<E>,
2706 // E>
2707 // V selectFromTwoVectorOp(Class<? extends V> vClass, Class<E> eClass, int length,
2708 // V v1, V v2, V v3,
2709 // SelectFromTwoVector<V> defaultImpl)
2710 bool LibraryCallKit::inline_vector_select_from_two_vectors() {
2711 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
2712 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
2713 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
2714
2715 if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr || vector_klass->const_oop() == nullptr ||
2716 elem_klass->const_oop() == nullptr ||!vlen->is_con()) {
2717 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s",
2718 NodeClassNames[argument(0)->Opcode()],
2719 NodeClassNames[argument(1)->Opcode()],
2720 NodeClassNames[argument(2)->Opcode()]);
2721 return false; // not enough info for intrinsification
2722 }
2723
2724 if (!is_klass_initialized(vector_klass)) {
2725 log_if_needed(" ** klass argument not initialized");
2726 return false;
2727 }
2728
2729 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2730 if (!elem_type->is_primitive_type()) {
2731 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2732 return false; // should be primitive type
2733 }
2734
2735 int num_elem = vlen->get_con();
2736 if (!is_power_of_2(num_elem)) {
2737 log_if_needed(" ** vlen is not power of two=%d", num_elem);
2738 return false;
2739 }
2740
2741 BasicType elem_bt = elem_type->basic_type();
2742 BasicType index_elem_bt = elem_bt;
2743 if (elem_bt == T_FLOAT) {
2744 index_elem_bt = T_INT;
2745 } else if (elem_bt == T_DOUBLE) {
2746 index_elem_bt = T_LONG;
2747 }
2748
2749 // Check if the platform requires a VectorLoadShuffle node to be generated
2750 bool need_load_shuffle = Matcher::vector_rearrange_requires_load_shuffle(index_elem_bt, num_elem);
2751
2752 bool lowerSelectFromOp = false;
2753 if (!arch_supports_vector(Op_SelectFromTwoVector, num_elem, elem_bt, VecMaskNotUsed)) {
2754 int cast_vopc = VectorCastNode::opcode(-1, elem_bt, true);
2755 if ((elem_bt != index_elem_bt && !arch_supports_vector(cast_vopc, num_elem, index_elem_bt, VecMaskNotUsed)) ||
2756 !arch_supports_vector(Op_VectorMaskCmp, num_elem, index_elem_bt, VecMaskNotUsed) ||
2757 !arch_supports_vector(Op_AndV, num_elem, index_elem_bt, VecMaskNotUsed) ||
2758 !arch_supports_vector(Op_VectorMaskCast, num_elem, elem_bt, VecMaskNotUsed) ||
2759 !arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad) ||
2760 !arch_supports_vector(Op_VectorRearrange, num_elem, elem_bt, VecMaskNotUsed) ||
2761 (need_load_shuffle && !arch_supports_vector(Op_VectorLoadShuffle, num_elem, index_elem_bt, VecMaskNotUsed)) ||
2762 !arch_supports_vector(Op_Replicate, num_elem, index_elem_bt, VecMaskNotUsed)) {
2763 log_if_needed(" ** not supported: opc=%d vlen=%d etype=%s ismask=useload",
2764 Op_SelectFromTwoVector, num_elem, type2name(elem_bt));
2765 return false; // not supported
2766 }
2767 lowerSelectFromOp = true;
2768 }
2769
2770 int cast_vopc = VectorCastNode::opcode(-1, elem_bt, true);
2771 if (!lowerSelectFromOp) {
2772 if (!arch_supports_vector(Op_AndV, num_elem, index_elem_bt, VecMaskNotUsed) ||
2773 !arch_supports_vector(Op_Replicate, num_elem, index_elem_bt, VecMaskNotUsed) ||
2774 (is_floating_point_type(elem_bt) &&
2775 !arch_supports_vector(cast_vopc, num_elem, index_elem_bt, VecMaskNotUsed))) {
2776 log_if_needed(" ** index wrapping not supported: vlen=%d etype=%s" ,
2777 num_elem, type2name(elem_bt));
2778 return false; // not supported
2779 }
2780 }
2781
2782 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2783 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2784
2785 Node* opd1 = unbox_vector(argument(3), vbox_type, elem_bt, num_elem);
2786 if (opd1 == nullptr) {
2787 log_if_needed(" ** unbox failed v1=%s",
2788 NodeClassNames[argument(3)->Opcode()]);
2789 return false;
2790 }
2791 Node* opd2 = unbox_vector(argument(4), vbox_type, elem_bt, num_elem);
2792 if (opd2 == nullptr) {
2793 log_if_needed(" ** unbox failed v2=%s",
2794 NodeClassNames[argument(4)->Opcode()]);
2795 return false;
2796 }
2797 Node* opd3 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
2798 if (opd3 == nullptr) {
2799 log_if_needed(" ** unbox failed v3=%s",
2800 NodeClassNames[argument(5)->Opcode()]);
2801 return false;
2802 }
2803
2804 const TypeVect* vt = TypeVect::make(elem_bt, num_elem);
2805
2806 Node* operation = nullptr;
2807 if (lowerSelectFromOp) {
2808 operation = gvn().transform(LowerSelectFromTwoVectorOperation(gvn(), opd1, opd2, opd3, vt));
2809 } else {
2810 if (index_elem_bt != elem_bt) {
2811 opd1 = gvn().transform(VectorCastNode::make(cast_vopc, opd1, index_elem_bt, num_elem));
2812 }
2813 int indexRangeMask = 2 * num_elem - 1;
2814 Node* wrap_mask = gvn().makecon(TypeInteger::make(indexRangeMask, indexRangeMask, Type::WidenMin, index_elem_bt != T_LONG ? T_INT : index_elem_bt));
2815 Node* wrap_mask_vec = gvn().transform(VectorNode::scalar2vector(wrap_mask, num_elem, index_elem_bt, false));
2816 opd1 = gvn().transform(VectorNode::make(Op_AndV, opd1, wrap_mask_vec, opd1->bottom_type()->is_vect()));
2817 operation = gvn().transform(trace_vector(VectorNode::make(Op_SelectFromTwoVector, opd1, opd2, opd3, vt)));
2818 }
2819
2820 // Wrap it up in VectorBox to keep object type information.
2821 Node* vbox = box_vector(operation, vbox_type, elem_bt, num_elem);
2822 set_result(vbox);
2823 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2824 return true;
2825 }
2826
2827 // public static
2828 // <V extends Vector<E>,
2829 // M extends VectorMask<E>,
2830 // E>
2831 // V compressExpandOp(int opr,
2832 // Class<? extends V> vClass, Class<? extends M> mClass, Class<E> eClass,
2833 // int length, V v, M m,
2834 // CompressExpandOperation<V, M> defaultImpl)
2835 bool LibraryCallKit::inline_vector_compress_expand() {
2836 const TypeInt* opr = gvn().type(argument(0))->isa_int();
2837 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
2838 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
2839 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
2840 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
2841
2842 if (opr == nullptr || !opr->is_con() ||
2843 vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
2844 mask_klass == nullptr || mask_klass->const_oop() == nullptr ||
2845 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
2846 vlen == nullptr || !vlen->is_con()) {
2847 log_if_needed(" ** missing constant: opr=%s vclass=%s mclass=%s etype=%s vlen=%s",
2848 NodeClassNames[argument(0)->Opcode()],
2849 NodeClassNames[argument(1)->Opcode()],
2850 NodeClassNames[argument(2)->Opcode()],
2851 NodeClassNames[argument(3)->Opcode()],
2852 NodeClassNames[argument(4)->Opcode()]);
2853 return false; // not enough info for intrinsification
2854 }
2855
2856 if (!is_klass_initialized(vector_klass) || !is_klass_initialized(mask_klass)) {
2857 log_if_needed(" ** klass argument not initialized");
2858 return false;
2859 }
2860
2861 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2862 if (!elem_type->is_primitive_type()) {
2863 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2864 return false; // should be primitive type
2865 }
2866
2867 int num_elem = vlen->get_con();
2868 BasicType elem_bt = elem_type->basic_type();
2869 int opc = VectorSupport::vop2ideal(opr->get_con(), elem_bt);
2870
2871 if (!arch_supports_vector(opc, num_elem, elem_bt, VecMaskUseLoad)) {
2872 log_if_needed(" ** not supported: opc=%d vlen=%d etype=%s ismask=useload",
2873 opc, num_elem, type2name(elem_bt));
2874 return false; // not supported
2875 }
2876
2877 Node* opd1 = nullptr;
2878 const TypeInstPtr* vbox_type = nullptr;
2879 if (opc != Op_CompressM) {
2880 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2881 vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2882 opd1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
2883 if (opd1 == nullptr) {
2884 log_if_needed(" ** unbox failed vector=%s",
2885 NodeClassNames[argument(5)->Opcode()]);
2886 return false;
2887 }
2888 }
2889
2890 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
2891 assert(is_vector_mask(mbox_klass), "argument(6) should be a mask class");
2892 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
2893
2894 Node* mask = unbox_vector(argument(6), mbox_type, elem_bt, num_elem);
2895 if (mask == nullptr) {
2896 log_if_needed(" ** unbox failed mask=%s",
2897 NodeClassNames[argument(6)->Opcode()]);
2898 return false;
2899 }
2900
2901 const TypeVect* vt = TypeVect::make(elem_bt, num_elem, opc == Op_CompressM);
2902 Node* operation = gvn().transform(trace_vector(VectorNode::make(opc, opd1, mask, vt)));
2903
2904 // Wrap it up in VectorBox to keep object type information.
2905 const TypeInstPtr* box_type = opc == Op_CompressM ? mbox_type : vbox_type;
2906 Node* vbox = box_vector(operation, box_type, elem_bt, num_elem);
2907 set_result(vbox);
2908 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2909 return true;
2910 }
2911
2912 // public static
2913 // <V extends Vector<E>,
2914 // E,
2915 // S extends VectorSpecies<E>>
2916 // V indexVector(Class<? extends V> vClass, Class<E> eClass,
2917 // int length,
2918 // V v, int step, S s,
2919 // IndexOperation<V, S> defaultImpl)
2920 bool LibraryCallKit::inline_index_vector() {
2921 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
2922 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
2923 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
2924
2925 if (vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
2926 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
2927 vlen == nullptr || !vlen->is_con() ) {
2928 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s",
2929 NodeClassNames[argument(0)->Opcode()],
2930 NodeClassNames[argument(1)->Opcode()],
2931 NodeClassNames[argument(2)->Opcode()]);
2932 return false; // not enough info for intrinsification
2933 }
2934
2935 if (!is_klass_initialized(vector_klass)) {
2936 log_if_needed(" ** klass argument not initialized");
2937 return false;
2938 }
2939
2940 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2941 if (!elem_type->is_primitive_type()) {
2942 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2943 return false; // should be primitive type
2944 }
2945
2946 int num_elem = vlen->get_con();
2947 BasicType elem_bt = elem_type->basic_type();
2948
2949 // Check whether the iota index generation op is supported by the current hardware
2950 if (!arch_supports_vector(Op_VectorLoadConst, num_elem, elem_bt, VecMaskNotUsed)) {
2951 log_if_needed(" ** not supported: vlen=%d etype=%s", num_elem, type2name(elem_bt));
2952 return false; // not supported
2953 }
2954
2955 int mul_op = VectorSupport::vop2ideal(VectorSupport::VECTOR_OP_MUL, elem_bt);
2956 int vmul_op = VectorNode::opcode(mul_op, elem_bt);
2957 bool needs_mul = true;
2958 Node* scale = argument(4);
2959 const TypeInt* scale_type = gvn().type(scale)->isa_int();
2960 // Multiply is not needed if the scale is a constant "1".
2961 if (scale_type && scale_type->is_con() && scale_type->get_con() == 1) {
2962 needs_mul = false;
2963 } else {
2964 // Check whether the vector multiply op is supported by the current hardware
2965 if (!arch_supports_vector(vmul_op, num_elem, elem_bt, VecMaskNotUsed)) {
2966 log_if_needed(" ** not supported: vlen=%d etype=%s", num_elem, type2name(elem_bt));
2967 return false; // not supported
2968 }
2969
2970 // Check whether the scalar cast op is supported by the current hardware
2971 if (is_floating_point_type(elem_bt) || elem_bt == T_LONG) {
2972 int cast_op = elem_bt == T_LONG ? Op_ConvI2L :
2973 elem_bt == T_FLOAT? Op_ConvI2F : Op_ConvI2D;
2974 if (!Matcher::match_rule_supported(cast_op)) {
2975 log_if_needed(" ** Rejected op (%s) because architecture does not support it",
2976 NodeClassNames[cast_op]);
2977 return false; // not supported
2978 }
2979 }
2980 }
2981
2982 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2983 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2984 Node* opd = unbox_vector(argument(3), vbox_type, elem_bt, num_elem);
2985 if (opd == nullptr) {
2986 log_if_needed(" ** unbox failed vector=%s",
2987 NodeClassNames[argument(3)->Opcode()]);
2988 return false;
2989 }
2990
2991 int add_op = VectorSupport::vop2ideal(VectorSupport::VECTOR_OP_ADD, elem_bt);
2992 int vadd_op = VectorNode::opcode(add_op, elem_bt);
2993 bool needs_add = true;
2994 // The addition is not needed if all the element values of "opd" are zero
2995 if (VectorNode::is_all_zeros_vector(opd)) {
2996 needs_add = false;
2997 } else {
2998 // Check whether the vector addition op is supported by the current hardware
2999 if (!arch_supports_vector(vadd_op, num_elem, elem_bt, VecMaskNotUsed)) {
3000 log_if_needed(" ** not supported: vlen=%d etype=%s", num_elem, type2name(elem_bt));
3001 return false; // not supported
3002 }
3003 }
3004
3005 // Compute the iota indice vector
3006 const TypeVect* vt = TypeVect::make(elem_bt, num_elem);
3007 Node* index = gvn().transform(new VectorLoadConstNode(gvn().makecon(TypeInt::ZERO), vt));
3008
3009 // Broadcast the "scale" to a vector, and multiply the "scale" with iota indice vector.
3010 if (needs_mul) {
3011 switch (elem_bt) {
3012 case T_BOOLEAN: // fall-through
3013 case T_BYTE: // fall-through
3014 case T_SHORT: // fall-through
3015 case T_CHAR: // fall-through
3016 case T_INT: {
3017 // no conversion needed
3018 break;
3019 }
3020 case T_LONG: {
3021 scale = gvn().transform(new ConvI2LNode(scale));
3022 break;
3023 }
3024 case T_FLOAT: {
3025 scale = gvn().transform(new ConvI2FNode(scale));
3026 break;
3027 }
3028 case T_DOUBLE: {
3029 scale = gvn().transform(new ConvI2DNode(scale));
3030 break;
3031 }
3032 default: fatal("%s", type2name(elem_bt));
3033 }
3034 scale = gvn().transform(VectorNode::scalar2vector(scale, num_elem, elem_bt));
3035 index = gvn().transform(trace_vector(VectorNode::make(vmul_op, index, scale, vt)));
3036 }
3037
3038 // Add "opd" if addition is needed.
3039 if (needs_add) {
3040 index = gvn().transform(trace_vector(VectorNode::make(vadd_op, opd, index, vt)));
3041 }
3042 Node* vbox = box_vector(index, vbox_type, elem_bt, num_elem);
3043 set_result(vbox);
3044 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
3045 return true;
3046 }
3047
3048 // public static
3049 // <E,
3050 // M extends VectorMask<E>>
3051 // M indexPartiallyInUpperRange(Class<? extends M> mClass, Class<E> eClass, int length,
3052 // long offset, long limit,
3053 // IndexPartiallyInUpperRangeOperation<E, M> defaultImpl)
3054 bool LibraryCallKit::inline_index_partially_in_upper_range() {
3055 const TypeInstPtr* mask_klass = gvn().type(argument(0))->isa_instptr();
3056 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
3057 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
3058
3059 if (mask_klass == nullptr || mask_klass->const_oop() == nullptr ||
3060 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
3061 vlen == nullptr || !vlen->is_con()) {
3062 log_if_needed(" ** missing constant: mclass=%s etype=%s vlen=%s",
3063 NodeClassNames[argument(0)->Opcode()],
3064 NodeClassNames[argument(1)->Opcode()],
3065 NodeClassNames[argument(2)->Opcode()]);
3066 return false; // not enough info for intrinsification
3067 }
3068
3069 if (!is_klass_initialized(mask_klass)) {
3070 log_if_needed(" ** klass argument not initialized");
3071 return false;
3072 }
3073
3074 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
3075 if (!elem_type->is_primitive_type()) {
3076 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
3077 return false; // should be primitive type
3078 }
3079
3080 int num_elem = vlen->get_con();
3081 BasicType elem_bt = elem_type->basic_type();
3082
3083 // Check whether the necessary ops are supported by current hardware.
3084 bool supports_mask_gen = arch_supports_vector(Op_VectorMaskGen, num_elem, elem_bt, VecMaskUseStore);
3085 if (!supports_mask_gen) {
3086 if (!arch_supports_vector(Op_VectorLoadConst, num_elem, elem_bt, VecMaskNotUsed) ||
3087 !arch_supports_vector(Op_Replicate, num_elem, elem_bt, VecMaskNotUsed) ||
3088 !arch_supports_vector(Op_VectorMaskCmp, num_elem, elem_bt, VecMaskUseStore)) {
3089 log_if_needed(" ** not supported: vlen=%d etype=%s", num_elem, type2name(elem_bt));
3090 return false; // not supported
3091 }
3092
3093 // Check whether the scalar cast operation is supported by current hardware.
3094 if (elem_bt != T_LONG) {
3095 int cast_op = is_integral_type(elem_bt) ? Op_ConvL2I
3096 : (elem_bt == T_FLOAT ? Op_ConvL2F : Op_ConvL2D);
3097 if (!Matcher::match_rule_supported(cast_op)) {
3098 log_if_needed(" ** Rejected op (%s) because architecture does not support it",
3099 NodeClassNames[cast_op]);
3100 return false; // not supported
3101 }
3102 }
3103 }
3104
3105 Node* offset = argument(3);
3106 Node* limit = argument(5);
3107 if (offset == nullptr || limit == nullptr) {
3108 log_if_needed(" ** offset or limit argument is null");
3109 return false; // not supported
3110 }
3111
3112 ciKlass* box_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
3113 assert(is_vector_mask(box_klass), "argument(0) should be a mask class");
3114 const TypeInstPtr* box_type = TypeInstPtr::make_exact(TypePtr::NotNull, box_klass);
3115
3116 // We assume "offset > 0 && limit >= offset && limit - offset < num_elem".
3117 // So directly get indexLimit with "indexLimit = limit - offset".
3118 Node* indexLimit = gvn().transform(new SubLNode(limit, offset));
3119 Node* mask = nullptr;
3120 if (supports_mask_gen) {
3121 mask = gvn().transform(VectorMaskGenNode::make(indexLimit, elem_bt, num_elem));
3122 } else {
3123 // Generate the vector mask based on "mask = iota < indexLimit".
3124 // Broadcast "indexLimit" to a vector.
3125 switch (elem_bt) {
3126 case T_BOOLEAN: // fall-through
3127 case T_BYTE: // fall-through
3128 case T_SHORT: // fall-through
3129 case T_CHAR: // fall-through
3130 case T_INT: {
3131 indexLimit = gvn().transform(new ConvL2INode(indexLimit));
3132 break;
3133 }
3134 case T_DOUBLE: {
3135 indexLimit = gvn().transform(new ConvL2DNode(indexLimit));
3136 break;
3137 }
3138 case T_FLOAT: {
3139 indexLimit = gvn().transform(new ConvL2FNode(indexLimit));
3140 break;
3141 }
3142 case T_LONG: {
3143 // no conversion needed
3144 break;
3145 }
3146 default: fatal("%s", type2name(elem_bt));
3147 }
3148 indexLimit = gvn().transform(VectorNode::scalar2vector(indexLimit, num_elem, elem_bt));
3149
3150 // Load the "iota" vector.
3151 const TypeVect* vt = TypeVect::make(elem_bt, num_elem);
3152 Node* iota = gvn().transform(new VectorLoadConstNode(gvn().makecon(TypeInt::ZERO), vt));
3153
3154 // Compute the vector mask with "mask = iota < indexLimit".
3155 ConINode* pred_node = (ConINode*)gvn().makecon(TypeInt::make(BoolTest::lt));
3156 const TypeVect* vmask_type = TypeVect::makemask(elem_bt, num_elem);
3157 mask = gvn().transform(trace_vector(new VectorMaskCmpNode(BoolTest::lt, iota, indexLimit, pred_node, vmask_type)));
3158 }
3159 Node* vbox = box_vector(mask, box_type, elem_bt, num_elem);
3160 set_result(vbox);
3161 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
3162 return true;
3163 }
3164
3165 #undef non_product_log_if_needed
3166 #undef log_if_needed