1 /*
2 * Copyright (c) 2020, 2026, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "ci/ciSymbols.hpp"
26 #include "classfile/vmSymbols.hpp"
27 #include "opto/library_call.hpp"
28 #include "opto/rootnode.hpp"
29 #include "opto/runtime.hpp"
30 #include "opto/vectornode.hpp"
31 #include "prims/vectorSupport.hpp"
32 #include "runtime/stubRoutines.hpp"
33
34 #ifdef ASSERT
35 static bool is_vector(ciKlass* klass) {
36 return klass->is_subclass_of(ciEnv::current()->vector_VectorPayload_klass());
37 }
38
39 static bool check_vbox(const TypeInstPtr* vbox_type) {
40 assert(vbox_type->klass_is_exact(), "");
41
42 ciInstanceKlass* ik = vbox_type->instance_klass();
43 assert(is_vector(ik), "not a vector");
44
45 ciField* fd1 = ik->get_field_by_name(ciSymbols::ETYPE_name(), ciSymbols::class_signature(), /* is_static */ true);
46 assert(fd1 != nullptr, "element type info is missing");
47
48 ciConstant val1 = fd1->constant_value();
49 BasicType elem_bt = val1.as_object()->as_instance()->java_mirror_type()->basic_type();
50 assert(is_java_primitive(elem_bt), "element type info is missing");
51
52 ciField* fd2 = ik->get_field_by_name(ciSymbols::VLENGTH_name(), ciSymbols::int_signature(), /* is_static */ true);
53 assert(fd2 != nullptr, "vector length info is missing");
54
55 ciConstant val2 = fd2->constant_value();
56 assert(val2.as_int() > 0, "vector length info is missing");
57
58 return true;
59 }
60 #endif
61
62 #define log_if_needed(...) \
63 if (C->print_intrinsics()) { \
64 tty->print_cr(__VA_ARGS__); \
65 }
66
67 #ifndef PRODUCT
68 #define non_product_log_if_needed(...) log_if_needed(__VA_ARGS__)
69 #else
70 #define non_product_log_if_needed(...)
71 #endif
72
73 static bool is_vector_mask(ciKlass* klass) {
74 return klass->is_subclass_of(ciEnv::current()->vector_VectorMask_klass());
75 }
76
77 bool LibraryCallKit::arch_supports_vector_rotate(int opc, int num_elem, BasicType elem_bt,
78 VectorMaskUseType mask_use_type, bool has_scalar_args) {
79 bool is_supported = true;
80
81 // has_scalar_args flag is true only for non-constant scalar shift count,
82 // since in this case shift needs to be broadcasted.
83 if (!Matcher::match_rule_supported_vector(opc, num_elem, elem_bt) ||
84 (has_scalar_args && !arch_supports_vector(Op_Replicate, num_elem, elem_bt, VecMaskNotUsed))) {
85 is_supported = false;
86 }
87
88 if (is_supported) {
89 // Check if mask unboxing is supported, this is a two step process which first loads the contents
90 // of boolean array into vector followed by either lane expansion to match the lane size of masked
91 // vector operation or populate the predicate register.
92 if ((mask_use_type & VecMaskUseLoad) != 0) {
93 if (!Matcher::match_rule_supported_vector(Op_VectorLoadMask, num_elem, elem_bt) ||
94 !Matcher::match_rule_supported_vector(Op_LoadVector, num_elem, T_BOOLEAN)) {
95 non_product_log_if_needed(" ** Rejected vector mask loading (%s,%s,%d) because architecture does not support it",
96 NodeClassNames[Op_VectorLoadMask], type2name(elem_bt), num_elem);
97 return false;
98 }
99 }
100
101 if ((mask_use_type & VecMaskUsePred) != 0) {
102 if (!Matcher::has_predicated_vectors() ||
103 !Matcher::match_rule_supported_vector_masked(opc, num_elem, elem_bt)) {
104 non_product_log_if_needed("Rejected vector mask predicate using (%s,%s,%d) because architecture does not support it",
105 NodeClassNames[opc], type2name(elem_bt), num_elem);
106 return false;
107 }
108 }
109 }
110
111 int lshiftopc, rshiftopc;
112 switch(elem_bt) {
113 case T_BYTE:
114 lshiftopc = Op_LShiftI;
115 rshiftopc = Op_URShiftB;
116 break;
117 case T_SHORT:
118 lshiftopc = Op_LShiftI;
119 rshiftopc = Op_URShiftS;
120 break;
121 case T_INT:
122 lshiftopc = Op_LShiftI;
123 rshiftopc = Op_URShiftI;
124 break;
125 case T_LONG:
126 lshiftopc = Op_LShiftL;
127 rshiftopc = Op_URShiftL;
128 break;
129 default: fatal("Unexpected type: %s", type2name(elem_bt));
130 }
131 int lshiftvopc = VectorNode::opcode(lshiftopc, elem_bt);
132 int rshiftvopc = VectorNode::opcode(rshiftopc, elem_bt);
133 if (!is_supported &&
134 arch_supports_vector(lshiftvopc, num_elem, elem_bt, VecMaskNotUsed, has_scalar_args) &&
135 arch_supports_vector(rshiftvopc, num_elem, elem_bt, VecMaskNotUsed, has_scalar_args) &&
136 arch_supports_vector(Op_OrV, num_elem, elem_bt, VecMaskNotUsed)) {
137 is_supported = true;
138 }
139 return is_supported;
140 }
141
142 Node* GraphKit::box_vector(Node* vector, const TypeInstPtr* vbox_type, BasicType elem_bt, int num_elem, bool deoptimize_on_exception) {
143 assert(EnableVectorSupport, "");
144
145 PreserveReexecuteState preexecs(this);
146 jvms()->set_should_reexecute(true);
147
148 VectorBoxAllocateNode* alloc = new VectorBoxAllocateNode(C, vbox_type);
149 set_edges_for_java_call(alloc, /*must_throw=*/false, /*separate_io_proj=*/true);
150 make_slow_call_ex(alloc, env()->Throwable_klass(), /*separate_io_proj=*/true, deoptimize_on_exception);
151 set_i_o(gvn().transform( new ProjNode(alloc, TypeFunc::I_O) ));
152 set_all_memory(gvn().transform( new ProjNode(alloc, TypeFunc::Memory) ));
153 Node* ret = gvn().transform(new ProjNode(alloc, TypeFunc::Parms));
154
155 assert(check_vbox(vbox_type), "");
156 const TypeVect* vt = TypeVect::make(elem_bt, num_elem, is_vector_mask(vbox_type->instance_klass()));
157 VectorBoxNode* vbox = new VectorBoxNode(C, ret, vector, vbox_type, vt);
158 return gvn().transform(vbox);
159 }
160
161 Node* GraphKit::unbox_vector(Node* v, const TypeInstPtr* vbox_type, BasicType elem_bt, int num_elem) {
162 assert(EnableVectorSupport, "");
163 const TypeInstPtr* vbox_type_v = gvn().type(v)->isa_instptr();
164 if (vbox_type_v == nullptr || vbox_type->instance_klass() != vbox_type_v->instance_klass()) {
165 return nullptr; // arguments don't agree on vector shapes
166 }
167 if (vbox_type_v->maybe_null()) {
168 return nullptr; // no nulls are allowed
169 }
170 assert(check_vbox(vbox_type), "");
171 const TypeVect* vt = TypeVect::make(elem_bt, num_elem, is_vector_mask(vbox_type->instance_klass()));
172 Node* unbox = gvn().transform(new VectorUnboxNode(C, vt, v, merged_memory()));
173 return unbox;
174 }
175
176 Node* GraphKit::vector_shift_count(Node* cnt, int shift_op, BasicType bt, int num_elem) {
177 assert(bt == T_INT || bt == T_LONG || bt == T_SHORT || bt == T_BYTE, "byte, short, long and int are supported");
178 juint mask = (type2aelembytes(bt) * BitsPerByte - 1);
179 Node* nmask = gvn().transform(ConNode::make(TypeInt::make(mask)));
180 Node* mcnt = gvn().transform(new AndINode(cnt, nmask));
181 return gvn().transform(VectorNode::shift_count(shift_op, mcnt, num_elem, bt));
182 }
183
184 bool LibraryCallKit::arch_supports_vector(int sopc, int num_elem, BasicType type, VectorMaskUseType mask_use_type, bool has_scalar_args) {
185 // Check that the operation is valid.
186 if (sopc <= 0) {
187 non_product_log_if_needed(" ** Rejected intrinsification because no valid vector op could be extracted");
188 return false;
189 }
190
191 if (VectorNode::is_vector_rotate(sopc)) {
192 if(!arch_supports_vector_rotate(sopc, num_elem, type, mask_use_type, has_scalar_args)) {
193 non_product_log_if_needed(" ** Rejected vector op (%s,%s,%d) because architecture does not support variable vector shifts",
194 NodeClassNames[sopc], type2name(type), num_elem);
195 return false;
196 }
197 } else if (VectorNode::is_vector_integral_negate(sopc)) {
198 if (!VectorNode::is_vector_integral_negate_supported(sopc, num_elem, type, false)) {
199 non_product_log_if_needed(" ** Rejected vector op (%s,%s,%d) because architecture does not support integral vector negate",
200 NodeClassNames[sopc], type2name(type), num_elem);
201 return false;
202 }
203 } else {
204 // Check that architecture supports this op-size-type combination.
205 if (!Matcher::match_rule_supported_vector(sopc, num_elem, type)) {
206 non_product_log_if_needed(" ** Rejected vector op (%s,%s,%d) because architecture does not support it",
207 NodeClassNames[sopc], type2name(type), num_elem);
208 return false;
209 } else {
210 assert(Matcher::match_rule_supported(sopc), "must be supported");
211 }
212 }
213
214 if (num_elem == 1) {
215 if (mask_use_type != VecMaskNotUsed) {
216 non_product_log_if_needed(" ** Rejected vector mask op (%s,%s,%d) because architecture does not support it",
217 NodeClassNames[sopc], type2name(type), num_elem);
218 return false;
219 }
220
221 if (sopc != 0) {
222 if (sopc != Op_LoadVector && sopc != Op_StoreVector) {
223 non_product_log_if_needed(" ** Not a svml call or load/store vector op (%s,%s,%d)",
224 NodeClassNames[sopc], type2name(type), num_elem);
225 return false;
226 }
227 }
228 }
229
230 if (!has_scalar_args && VectorNode::is_vector_shift(sopc) &&
231 Matcher::supports_vector_variable_shifts() == false) {
232 log_if_needed(" ** Rejected vector op (%s,%s,%d) because architecture does not support variable vector shifts",
233 NodeClassNames[sopc], type2name(type), num_elem);
234 return false;
235 }
236
237 // Check if mask unboxing is supported, this is a two step process which first loads the contents
238 // of boolean array into vector followed by either lane expansion to match the lane size of masked
239 // vector operation or populate the predicate register.
240 if ((mask_use_type & VecMaskUseLoad) != 0) {
241 if (!Matcher::match_rule_supported_vector(Op_VectorLoadMask, num_elem, type) ||
242 !Matcher::match_rule_supported_vector(Op_LoadVector, num_elem, T_BOOLEAN)) {
243 non_product_log_if_needed(" ** Rejected vector mask loading (%s,%s,%d) because architecture does not support it",
244 NodeClassNames[Op_VectorLoadMask], type2name(type), num_elem);
245 return false;
246 }
247 }
248
249 // Check if mask boxing is supported, this is a two step process which first stores the contents
250 // of mask vector / predicate register into a boolean vector followed by vector store operation to
251 // transfer the contents to underlined storage of mask boxes which is a boolean array.
252 if ((mask_use_type & VecMaskUseStore) != 0) {
253 if (!Matcher::match_rule_supported_vector(Op_VectorStoreMask, num_elem, type) ||
254 !Matcher::match_rule_supported_vector(Op_StoreVector, num_elem, T_BOOLEAN)) {
255 non_product_log_if_needed("Rejected vector mask storing (%s,%s,%d) because architecture does not support it",
256 NodeClassNames[Op_VectorStoreMask], type2name(type), num_elem);
257 return false;
258 }
259 }
260
261 if ((mask_use_type & VecMaskUsePred) != 0) {
262 bool is_supported = false;
263 if (Matcher::has_predicated_vectors()) {
264 if (VectorNode::is_vector_integral_negate(sopc)) {
265 is_supported = VectorNode::is_vector_integral_negate_supported(sopc, num_elem, type, true);
266 } else {
267 is_supported = Matcher::match_rule_supported_vector_masked(sopc, num_elem, type);
268 }
269 }
270 is_supported |= Matcher::supports_vector_predicate_op_emulation(sopc, num_elem, type);
271
272 if (!is_supported) {
273 non_product_log_if_needed("Rejected vector mask predicate using (%s,%s,%d) because architecture does not support it",
274 NodeClassNames[sopc], type2name(type), num_elem);
275 return false;
276 }
277 }
278
279 return true;
280 }
281
282 static bool is_klass_initialized(const TypeInstPtr* vec_klass) {
283 if (vec_klass->const_oop() == nullptr) {
284 return false; // uninitialized or some kind of unsafe access
285 }
286 assert(vec_klass->const_oop()->as_instance()->java_lang_Class_klass() != nullptr, "klass instance expected");
287 ciInstanceKlass* klass = vec_klass->const_oop()->as_instance()->java_lang_Class_klass()->as_instance_klass();
288 return klass->is_initialized();
289 }
290
291 // public static
292 // <V extends Vector<E>,
293 // M extends VectorMask<E>,
294 // E>
295 // V unaryOp(int oprId, Class<? extends V> vmClass, Class<? extends M> maskClass, Class<E> elementType,
296 // int length, V v, M m,
297 // UnaryOperation<V, M> defaultImpl)
298 //
299 // public static
300 // <V,
301 // M extends VectorMask<E>,
302 // E>
303 // V binaryOp(int oprId, Class<? extends V> vmClass, Class<? extends M> maskClass, Class<E> elementType,
304 // int length, V v1, V v2, M m,
305 // BinaryOperation<V, M> defaultImpl)
306 //
307 // public static
308 // <V extends Vector<E>,
309 // M extends VectorMask<E>,
310 // E>
311 // V ternaryOp(int oprId, Class<? extends V> vmClass, Class<? extends M> maskClass, Class<E> elementType,
312 // int length, V v1, V v2, V v3, M m,
313 // TernaryOperation<V, M> defaultImpl)
314 //
315 bool LibraryCallKit::inline_vector_nary_operation(int n) {
316 const TypeInt* opr = gvn().type(argument(0))->isa_int();
317 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
318 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
319 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
320 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
321
322 if (opr == nullptr || !opr->is_con() ||
323 vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
324 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
325 vlen == nullptr || !vlen->is_con()) {
326 log_if_needed(" ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
327 NodeClassNames[argument(0)->Opcode()],
328 NodeClassNames[argument(1)->Opcode()],
329 NodeClassNames[argument(3)->Opcode()],
330 NodeClassNames[argument(4)->Opcode()]);
331 return false; // not enough info for intrinsification
332 }
333
334 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
335 if (!elem_type->is_primitive_type()) {
336 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
337 return false; // should be primitive type
338 }
339 if (!is_klass_initialized(vector_klass)) {
340 log_if_needed(" ** klass argument not initialized");
341 return false;
342 }
343
344 // "argument(n + 5)" should be the mask object. We assume it is "null" when no mask
345 // is used to control this operation.
346 const Type* vmask_type = gvn().type(argument(n + 5));
347 bool is_masked_op = vmask_type != TypePtr::NULL_PTR;
348 if (is_masked_op) {
349 if (mask_klass == nullptr || mask_klass->const_oop() == nullptr) {
350 log_if_needed(" ** missing constant: maskclass=%s", NodeClassNames[argument(2)->Opcode()]);
351 return false; // not enough info for intrinsification
352 }
353
354 if (!is_klass_initialized(mask_klass)) {
355 log_if_needed(" ** mask klass argument not initialized");
356 return false;
357 }
358
359 if (vmask_type->maybe_null()) {
360 log_if_needed(" ** null mask values are not allowed for masked op");
361 return false;
362 }
363 }
364
365 BasicType elem_bt = elem_type->basic_type();
366 bool has_scalar_op = VectorSupport::has_scalar_op(opr->get_con());
367 bool is_unsigned = VectorSupport::is_unsigned_op(opr->get_con());
368
369 int num_elem = vlen->get_con();
370 int opc = VectorSupport::vop2ideal(opr->get_con(), elem_bt);
371 int sopc = has_scalar_op ? VectorNode::opcode(opc, elem_bt) : opc;
372 if (sopc == 0 || num_elem == 1) {
373 log_if_needed(" ** operation not supported: arity=%d opc=%s[%d] vlen=%d etype=%s",
374 n, NodeClassNames[opc], opc, num_elem, type2name(elem_bt));
375 return false; // operation not supported
376 }
377 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
378 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
379
380 if (is_vector_mask(vbox_klass)) {
381 assert(!is_masked_op, "mask operations do not need mask to control");
382 }
383
384 // When using mask, mask use type needs to be VecMaskUseLoad.
385 VectorMaskUseType mask_use_type = is_vector_mask(vbox_klass) ? VecMaskUseAll
386 : is_masked_op ? VecMaskUseLoad : VecMaskNotUsed;
387 if (!arch_supports_vector(sopc, num_elem, elem_bt, mask_use_type)) {
388 log_if_needed(" ** not supported: arity=%d opc=%d vlen=%d etype=%s ismask=%d is_masked_op=%d",
389 n, sopc, num_elem, type2name(elem_bt),
390 is_vector_mask(vbox_klass) ? 1 : 0, is_masked_op ? 1 : 0);
391 return false; // not supported
392 }
393
394 // Return true if current platform has implemented the masked operation with predicate feature.
395 bool use_predicate = is_masked_op && arch_supports_vector(sopc, num_elem, elem_bt, VecMaskUsePred);
396 if (is_masked_op && !use_predicate && !arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad)) {
397 log_if_needed(" ** not supported: arity=%d opc=%d vlen=%d etype=%s ismask=0 is_masked_op=1",
398 n, sopc, num_elem, type2name(elem_bt));
399 return false;
400 }
401
402 Node* opd1 = nullptr; Node* opd2 = nullptr; Node* opd3 = nullptr;
403 switch (n) {
404 case 3: {
405 opd3 = unbox_vector(argument(7), vbox_type, elem_bt, num_elem);
406 if (opd3 == nullptr) {
407 log_if_needed(" ** unbox failed v3=%s",
408 NodeClassNames[argument(7)->Opcode()]);
409 return false;
410 }
411 // fall-through
412 }
413 case 2: {
414 opd2 = unbox_vector(argument(6), vbox_type, elem_bt, num_elem);
415 if (opd2 == nullptr) {
416 log_if_needed(" ** unbox failed v2=%s",
417 NodeClassNames[argument(6)->Opcode()]);
418 return false;
419 }
420 // fall-through
421 }
422 case 1: {
423 opd1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
424 if (opd1 == nullptr) {
425 log_if_needed(" ** unbox failed v1=%s",
426 NodeClassNames[argument(5)->Opcode()]);
427 return false;
428 }
429 break;
430 }
431 default: fatal("unsupported arity: %d", n);
432 }
433
434 Node* mask = nullptr;
435 if (is_masked_op) {
436 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
437 assert(is_vector_mask(mbox_klass), "argument(2) should be a mask class");
438 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
439 mask = unbox_vector(argument(n + 5), mbox_type, elem_bt, num_elem);
440 if (mask == nullptr) {
441 log_if_needed(" ** unbox failed mask=%s",
442 NodeClassNames[argument(n + 5)->Opcode()]);
443 return false;
444 }
445 }
446
447 Node* operation = nullptr;
448 const TypeVect* vt = TypeVect::make(elem_bt, num_elem, is_vector_mask(vbox_klass));
449 switch (n) {
450 case 1:
451 case 2: {
452 operation = VectorNode::make(sopc, opd1, opd2, vt, is_vector_mask(vbox_klass), VectorNode::is_shift_opcode(opc), is_unsigned);
453 break;
454 }
455 case 3: {
456 operation = VectorNode::make(sopc, opd1, opd2, opd3, vt);
457 break;
458 }
459 default: fatal("unsupported arity: %d", n);
460 }
461
462 if (is_masked_op && mask != nullptr) {
463 if (use_predicate) {
464 operation->add_req(mask);
465 operation->add_flag(Node::Flag_is_predicated_vector);
466 } else {
467 operation->add_flag(Node::Flag_is_predicated_using_blend);
468 operation = gvn().transform(operation);
469 operation = new VectorBlendNode(opd1, operation, mask);
470 }
471 }
472 operation = gvn().transform(operation);
473
474 // Wrap it up in VectorBox to keep object type information.
475 Node* vbox = box_vector(operation, vbox_type, elem_bt, num_elem);
476 set_result(vbox);
477 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
478 return true;
479 }
480
481 // public static
482 // <V extends Vector<E>, E>
483 // V libraryUnaryOp(long address, Class<? extends V> vClass, Class<E> elementType, int length, String debugName,
484 // V v,
485 // UnaryOperation<V, ?> defaultImpl)
486 //
487 // public static
488 // <V extends VectorPayload, E>
489 // V libraryBinaryOp(long address, Class<? extends V> vClass, Class<E> elementType, int length, String debugName,
490 // V v1, V v2,
491 // BinaryOperation<V, ?> defaultImpl)
492 bool LibraryCallKit::inline_vector_call(int arity) {
493 assert(Matcher::supports_vector_calling_convention(), "required");
494
495 const TypeLong* entry = gvn().type(argument(0))->isa_long();
496 const TypeInstPtr* vector_klass = gvn().type(argument(2))->isa_instptr();
497 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
498 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
499 const TypeInstPtr* debug_name_oop = gvn().type(argument(5))->isa_instptr();
500
501 if (entry == nullptr || !entry->is_con() ||
502 vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
503 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
504 vlen == nullptr || !vlen->is_con() ||
505 debug_name_oop == nullptr || debug_name_oop->const_oop() == nullptr) {
506 log_if_needed(" ** missing constant: opr=%s vclass=%s etype=%s vlen=%s debug_name=%s",
507 NodeClassNames[argument(0)->Opcode()],
508 NodeClassNames[argument(2)->Opcode()],
509 NodeClassNames[argument(3)->Opcode()],
510 NodeClassNames[argument(4)->Opcode()],
511 NodeClassNames[argument(5)->Opcode()]);
512 return false; // not enough info for intrinsification
513 }
514
515 if (entry->get_con() == 0) {
516 log_if_needed(" ** missing entry point");
517 return false;
518 }
519
520 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
521 if (!elem_type->is_primitive_type()) {
522 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
523 return false; // should be primitive type
524 }
525 if (!is_klass_initialized(vector_klass)) {
526 log_if_needed(" ** klass argument not initialized");
527 return false;
528 }
529
530 BasicType elem_bt = elem_type->basic_type();
531 int num_elem = vlen->get_con();
532 if (!Matcher::vector_size_supported(elem_bt, num_elem)) {
533 log_if_needed(" ** vector size (vlen=%d, etype=%s) is not supported",
534 num_elem, type2name(elem_bt));
535 return false;
536 }
537
538 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
539 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
540
541 Node* opd1 = unbox_vector(argument(6), vbox_type, elem_bt, num_elem);
542 if (opd1 == nullptr) {
543 log_if_needed(" ** unbox failed v1=%s", NodeClassNames[argument(6)->Opcode()]);
544 return false;
545 }
546
547 Node* opd2 = nullptr;
548 if (arity > 1) {
549 opd2 = unbox_vector(argument(7), vbox_type, elem_bt, num_elem);
550 if (opd2 == nullptr) {
551 log_if_needed(" ** unbox failed v2=%s", NodeClassNames[argument(7)->Opcode()]);
552 return false;
553 }
554 }
555 assert(arity == 1 || arity == 2, "arity %d not supported", arity);
556 const TypeVect* vt = TypeVect::make(elem_bt, num_elem);
557 const TypeFunc* call_type = OptoRuntime::Math_Vector_Vector_Type(arity, vt, vt);
558 address entry_addr = (address)entry->get_con();
559
560 const char* debug_name = "<unknown>";
561 if (!debug_name_oop->const_oop()->is_null_object()) {
562 size_t buflen = 100;
563 char* buf = NEW_ARENA_ARRAY(C->comp_arena(), char, buflen);
564 debug_name = debug_name_oop->const_oop()->as_instance()->java_lang_String_str(buf, buflen);
565 }
566 Node* vcall = make_runtime_call(RC_VECTOR,
567 call_type,
568 entry_addr,
569 debug_name,
570 TypePtr::BOTTOM,
571 opd1,
572 opd2);
573
574 vcall = gvn().transform(new ProjNode(gvn().transform(vcall), TypeFunc::Parms));
575
576 // Wrap it up in VectorBox to keep object type information.
577 Node* vbox = box_vector(vcall, vbox_type, elem_bt, num_elem);
578 set_result(vbox);
579 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
580 return true;
581 }
582
583 // <E, M>
584 // long maskReductionCoerced(int oper, Class<? extends M> maskClass, Class<?> elemClass,
585 // int length, M m, VectorMaskOp<M> defaultImpl)
586 bool LibraryCallKit::inline_vector_mask_operation() {
587 const TypeInt* oper = gvn().type(argument(0))->isa_int();
588 const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
589 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
590 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
591 Node* mask = argument(4);
592
593 if (mask_klass == nullptr || mask_klass->const_oop() == nullptr ||
594 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
595 vlen == nullptr || !vlen->is_con() ||
596 oper == nullptr || !oper->is_con() ||
597 mask->is_top()) {
598 return false; // dead code
599 }
600
601 if (!is_klass_initialized(mask_klass)) {
602 log_if_needed(" ** klass argument not initialized");
603 return false;
604 }
605
606 int num_elem = vlen->get_con();
607 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
608 BasicType elem_bt = elem_type->basic_type();
609
610 int mopc = VectorSupport::vop2ideal(oper->get_con(), elem_bt);
611 if (!arch_supports_vector(mopc, num_elem, elem_bt, VecMaskUseLoad)) {
612 log_if_needed(" ** not supported: arity=1 op=cast#%d/3 vlen2=%d etype2=%s",
613 mopc, num_elem, type2name(elem_bt));
614 return false; // not supported
615 }
616
617 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
618 const TypeInstPtr* mask_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
619 Node* mask_vec = unbox_vector(mask, mask_box_type, elem_bt, num_elem);
620 if (mask_vec == nullptr) {
621 log_if_needed(" ** unbox failed mask=%s",
622 NodeClassNames[argument(4)->Opcode()]);
623 return false;
624 }
625
626 if (!Matcher::mask_op_prefers_predicate(mopc, mask_vec->bottom_type()->is_vect())) {
627 mask_vec = gvn().transform(VectorStoreMaskNode::make(gvn(), mask_vec, elem_bt, num_elem));
628 }
629 const Type* maskoper_ty = mopc == Op_VectorMaskToLong ? (const Type*)TypeLong::LONG : (const Type*)TypeInt::INT;
630 Node* maskoper = gvn().transform(VectorMaskOpNode::make(mask_vec, maskoper_ty, mopc));
631 if (mopc != Op_VectorMaskToLong) {
632 maskoper = ConvI2L(maskoper);
633 }
634 set_result(maskoper);
635
636 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
637 return true;
638 }
639
640 // public static
641 // <M,
642 // S extends VectorSpecies<E>,
643 // E>
644 // M fromBitsCoerced(Class<? extends M> vmClass, Class<E> elementType, int length,
645 // long bits, int mode, S s,
646 // BroadcastOperation<M, E, S> defaultImpl)
647 bool LibraryCallKit::inline_vector_frombits_coerced() {
648 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
649 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
650 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
651 const TypeLong* bits_type = gvn().type(argument(3))->isa_long();
652 // Mode argument determines the mode of operation it can take following values:-
653 // MODE_BROADCAST for vector Vector.broadcast and VectorMask.maskAll operations.
654 // MODE_BITS_COERCED_LONG_TO_MASK for VectorMask.fromLong operation.
655 const TypeInt* mode = gvn().type(argument(5))->isa_int();
656
657 if (vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
658 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
659 vlen == nullptr || !vlen->is_con() ||
660 bits_type == nullptr ||
661 mode == nullptr || !mode->is_con()) {
662 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s bitwise=%s",
663 NodeClassNames[argument(0)->Opcode()],
664 NodeClassNames[argument(1)->Opcode()],
665 NodeClassNames[argument(2)->Opcode()],
666 NodeClassNames[argument(5)->Opcode()]);
667 return false; // not enough info for intrinsification
668 }
669
670 if (!is_klass_initialized(vector_klass)) {
671 log_if_needed(" ** klass argument not initialized");
672 return false;
673 }
674 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
675 if (!elem_type->is_primitive_type()) {
676 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
677 return false; // should be primitive type
678 }
679 BasicType elem_bt = elem_type->basic_type();
680 int num_elem = vlen->get_con();
681 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
682 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
683
684 bool is_mask = is_vector_mask(vbox_klass);
685 int bcast_mode = mode->get_con();
686 VectorMaskUseType checkFlags = (VectorMaskUseType)(is_mask ? VecMaskUseAll : VecMaskNotUsed);
687 int opc = bcast_mode == VectorSupport::MODE_BITS_COERCED_LONG_TO_MASK ? Op_VectorLongToMask : Op_Replicate;
688
689 if (!arch_supports_vector(opc, num_elem, elem_bt, checkFlags, true /*has_scalar_args*/)) {
690 // If the input long sets or unsets all lanes and Replicate is supported,
691 // generate a MaskAll or Replicate instead.
692
693 // The "maskAll" API uses the corresponding integer types for floating-point data.
694 BasicType maskall_bt = elem_bt == T_DOUBLE ? T_LONG : (elem_bt == T_FLOAT ? T_INT: elem_bt);
695 if (!(opc == Op_VectorLongToMask &&
696 VectorNode::is_maskall_type(bits_type, num_elem) &&
697 arch_supports_vector(Op_Replicate, num_elem, maskall_bt, checkFlags, true /*has_scalar_args*/))) {
698 log_if_needed(" ** not supported: arity=0 op=broadcast vlen=%d etype=%s ismask=%d bcast_mode=%d",
699 num_elem, type2name(elem_bt),
700 is_mask ? 1 : 0,
701 bcast_mode);
702 return false; // not supported
703 }
704 }
705
706 Node* broadcast = nullptr;
707 Node* bits = argument(3);
708 Node* elem = bits;
709
710 if (opc == Op_VectorLongToMask) {
711 const TypeVect* vt = TypeVect::makemask(elem_bt, num_elem);
712 if (Matcher::mask_op_prefers_predicate(opc, vt)) {
713 broadcast = gvn().transform(new VectorLongToMaskNode(elem, vt));
714 } else {
715 const TypeVect* mvt = TypeVect::make(T_BOOLEAN, num_elem);
716 broadcast = gvn().transform(new VectorLongToMaskNode(elem, mvt));
717 broadcast = gvn().transform(new VectorLoadMaskNode(broadcast, vt));
718 }
719 } else {
720 switch (elem_bt) {
721 case T_BOOLEAN: // fall-through
722 case T_BYTE: // fall-through
723 case T_SHORT: // fall-through
724 case T_CHAR: // fall-through
725 case T_INT: {
726 elem = gvn().transform(new ConvL2INode(bits));
727 break;
728 }
729 case T_DOUBLE: {
730 elem = gvn().transform(new MoveL2DNode(bits));
731 break;
732 }
733 case T_FLOAT: {
734 bits = gvn().transform(new ConvL2INode(bits));
735 elem = gvn().transform(new MoveI2FNode(bits));
736 break;
737 }
738 case T_LONG: {
739 // no conversion needed
740 break;
741 }
742 default: fatal("%s", type2name(elem_bt));
743 }
744 broadcast = VectorNode::scalar2vector(elem, num_elem, elem_bt, is_mask);
745 broadcast = gvn().transform(broadcast);
746 }
747
748 Node* box = box_vector(broadcast, vbox_type, elem_bt, num_elem);
749 set_result(box);
750 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
751 return true;
752 }
753
754 static bool elem_consistent_with_arr(BasicType elem_bt, const TypeAryPtr* arr_type, bool mismatched_ms) {
755 assert(arr_type != nullptr, "unexpected");
756 BasicType arr_elem_bt = arr_type->elem()->array_element_basic_type();
757 if (elem_bt == arr_elem_bt) {
758 return true;
759 } else if (elem_bt == T_SHORT && arr_elem_bt == T_CHAR) {
760 // Load/store of short vector from/to char[] is supported
761 return true;
762 } else if (elem_bt == T_BYTE && arr_elem_bt == T_BOOLEAN) {
763 // Load/store of byte vector from/to boolean[] is supported
764 return true;
765 } else {
766 return mismatched_ms;
767 }
768 }
769
770 // public static
771 // <C,
772 // VM extends VectorPayload,
773 // E,
774 // S extends VectorSpecies<E>>
775 // VM load(Class<? extends VM> vmClass, Class<E> eClass,
776 // int length,
777 // Object base, long offset, // Unsafe addressing
778 // boolean fromSegment,
779 // C container, long index, S s, // Arguments for default implementation
780 // LoadOperation<C, VM, S> defaultImpl) {
781 // public static
782 // <C,
783 // V extends VectorPayload>
784 // void store(Class<?> vClass, Class<?> eClass,
785 // int length,
786 // Object base, long offset, // Unsafe addressing
787 // boolean fromSegment,
788 // V v, C container, long index, // Arguments for default implementation
789 // StoreVectorOperation<C, V> defaultImpl) {
790 bool LibraryCallKit::inline_vector_mem_operation(bool is_store) {
791 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
792 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
793 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
794 const TypeInt* from_ms = gvn().type(argument(6))->isa_int();
795
796 if (vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
797 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
798 vlen == nullptr || !vlen->is_con() ||
799 from_ms == nullptr || !from_ms->is_con()) {
800 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s from_ms=%s",
801 NodeClassNames[argument(0)->Opcode()],
802 NodeClassNames[argument(1)->Opcode()],
803 NodeClassNames[argument(2)->Opcode()],
804 NodeClassNames[argument(6)->Opcode()]);
805 return false; // not enough info for intrinsification
806 }
807 if (!is_klass_initialized(vector_klass)) {
808 log_if_needed(" ** klass argument not initialized");
809 return false;
810 }
811
812 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
813 if (!elem_type->is_primitive_type()) {
814 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
815 return false; // should be primitive type
816 }
817 BasicType elem_bt = elem_type->basic_type();
818 int num_elem = vlen->get_con();
819
820 // TODO When mask usage is supported, VecMaskNotUsed needs to be VecMaskUseLoad.
821 if (!arch_supports_vector(is_store ? Op_StoreVector : Op_LoadVector, num_elem, elem_bt, VecMaskNotUsed)) {
822 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d etype=%s ismask=no",
823 is_store, is_store ? "store" : "load",
824 num_elem, type2name(elem_bt));
825 return false; // not supported
826 }
827
828 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
829 bool is_mask = is_vector_mask(vbox_klass);
830
831 Node* base = argument(3);
832 Node* offset = ConvL2X(argument(4));
833
834 // Save state and restore on bailout
835 SavedState old_state(this);
836
837 Node* addr = make_unsafe_address(base, offset, (is_mask ? T_BOOLEAN : elem_bt), true);
838
839 // The memory barrier checks are based on ones for unsafe access.
840 // This is not 1-1 implementation.
841 const Type *const base_type = gvn().type(base);
842
843 const TypePtr *addr_type = gvn().type(addr)->isa_ptr();
844 const TypeAryPtr* arr_type = addr_type->isa_aryptr();
845
846 const bool in_native = TypePtr::NULL_PTR == base_type; // base always null
847 const bool in_heap = !TypePtr::NULL_PTR->higher_equal(base_type); // base never null
848
849 const bool is_mixed_access = !in_heap && !in_native;
850
851 const bool is_mismatched_access = in_heap && (addr_type->isa_aryptr() == nullptr);
852
853 const bool needs_cpu_membar = is_mixed_access || is_mismatched_access;
854
855 // For non-masked mismatched memory segment vector read/write accesses, intrinsification can continue
856 // with unknown backing storage type and compiler can skip inserting explicit reinterpretation IR after
857 // loading from or before storing to backing storage which is mandatory for semantic correctness of
858 // big-endian memory layout.
859 bool mismatched_ms = LITTLE_ENDIAN_ONLY(false)
860 BIG_ENDIAN_ONLY(from_ms->get_con() && !is_mask && arr_type != nullptr &&
861 arr_type->elem()->array_element_basic_type() != elem_bt);
862 BasicType mem_elem_bt = mismatched_ms ? arr_type->elem()->array_element_basic_type() : elem_bt;
863 if (!is_java_primitive(mem_elem_bt)) {
864 log_if_needed(" ** non-primitive array element type");
865 return false;
866 }
867 int mem_num_elem = mismatched_ms ? (num_elem * type2aelembytes(elem_bt)) / type2aelembytes(mem_elem_bt) : num_elem;
868 if (arr_type != nullptr && !is_mask && !elem_consistent_with_arr(elem_bt, arr_type, mismatched_ms)) {
869 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d etype=%s atype=%s ismask=no",
870 is_store, is_store ? "store" : "load",
871 num_elem, type2name(elem_bt), type2name(arr_type->elem()->array_element_basic_type()));
872 return false;
873 }
874
875 // In case of mismatched memory segment accesses, we need to double check that the source type memory operations are supported by backend.
876 if (mismatched_ms) {
877 if (is_store) {
878 if (!arch_supports_vector(Op_StoreVector, num_elem, elem_bt, VecMaskNotUsed)
879 || !arch_supports_vector(Op_VectorReinterpret, mem_num_elem, mem_elem_bt, VecMaskNotUsed)) {
880 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d*8 etype=%s/8 ismask=no",
881 is_store, "store",
882 num_elem, type2name(elem_bt));
883 return false; // not supported
884 }
885 } else {
886 if (!arch_supports_vector(Op_LoadVector, mem_num_elem, mem_elem_bt, VecMaskNotUsed)
887 || !arch_supports_vector(Op_VectorReinterpret, num_elem, elem_bt, VecMaskNotUsed)) {
888 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d*8 etype=%s/8 ismask=no",
889 is_store, "load",
890 mem_num_elem, type2name(mem_elem_bt));
891 return false; // not supported
892 }
893 }
894 }
895 if (is_mask) {
896 if (!is_store) {
897 if (!arch_supports_vector(Op_LoadVector, num_elem, elem_bt, VecMaskUseLoad)) {
898 return false; // not supported
899 }
900 } else {
901 if (!arch_supports_vector(Op_StoreVector, num_elem, elem_bt, VecMaskUseStore)) {
902 return false; // not supported
903 }
904 }
905 }
906
907 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
908
909 if (needs_cpu_membar) {
910 insert_mem_bar(Op_MemBarCPUOrder);
911 }
912
913 if (is_store) {
914 Node* val = unbox_vector(argument(7), vbox_type, elem_bt, num_elem);
915 if (val == nullptr) {
916 return false; // operand unboxing failed
917 }
918 set_all_memory(reset_memory());
919
920 // In case the store needs to happen to byte array, reinterpret the incoming vector to byte vector.
921 int store_num_elem = num_elem;
922 if (mismatched_ms) {
923 store_num_elem = mem_num_elem;
924 const TypeVect* to_vect_type = TypeVect::make(mem_elem_bt, store_num_elem);
925 val = gvn().transform(new VectorReinterpretNode(val, val->bottom_type()->is_vect(), to_vect_type));
926 }
927 if (is_mask) {
928 val = gvn().transform(VectorStoreMaskNode::make(gvn(), val, elem_bt, num_elem));
929 }
930 Node* vstore = gvn().transform(StoreVectorNode::make(0, control(), memory(addr), addr, addr_type, val, store_num_elem));
931 set_memory(vstore, addr_type);
932 } else {
933 // When using byte array, we need to load as byte then reinterpret the value. Otherwise, do a simple vector load.
934 Node* vload = nullptr;
935 if (mismatched_ms) {
936 vload = gvn().transform(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, mem_num_elem, mem_elem_bt));
937 const TypeVect* to_vect_type = TypeVect::make(elem_bt, num_elem);
938 vload = gvn().transform(new VectorReinterpretNode(vload, vload->bottom_type()->is_vect(), to_vect_type));
939 } else {
940 // Special handle for masks
941 if (is_mask) {
942 vload = gvn().transform(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, num_elem, T_BOOLEAN));
943 vload = gvn().transform(new VectorLoadMaskNode(vload, TypeVect::makemask(elem_bt, num_elem)));
944 } else {
945 vload = gvn().transform(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, num_elem, elem_bt));
946 }
947 }
948 Node* box = box_vector(vload, vbox_type, elem_bt, num_elem);
949 set_result(box);
950 }
951
952 old_state.discard();
953
954 if (needs_cpu_membar) {
955 insert_mem_bar(Op_MemBarCPUOrder);
956 }
957
958 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
959 return true;
960 }
961
962 // public static
963 // <C,
964 // V extends Vector<?>,
965 // E,
966 // S extends VectorSpecies<E>,
967 // M extends VectorMask<E>>
968 // V loadMasked(Class<? extends V> vClass, Class<M> mClass, Class<E> eClass,
969 // int length, Object base, long offset, // Unsafe addressing
970 // boolean fromSegment,
971 // M m, int offsetInRange,
972 // C container, long index, S s, // Arguments for default implementation
973 // LoadVectorMaskedOperation<C, V, S, M> defaultImpl) {
974 // public static
975 // <C,
976 // V extends Vector<E>,
977 // M extends VectorMask<E>,
978 // E>
979 // void storeMasked(Class<? extends V> vClass, Class<M> mClass, Class<E> eClass,
980 // int length,
981 // Object base, long offset, // Unsafe addressing
982 // boolean fromSegment,
983 // V v, M m, C container, long index, // Arguments for default implementation
984 // StoreVectorMaskedOperation<C, V, M> defaultImpl) {
985
986 bool LibraryCallKit::inline_vector_mem_masked_operation(bool is_store) {
987 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
988 const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
989 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
990 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
991 const TypeInt* from_ms = gvn().type(argument(7))->isa_int();
992
993 if (vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
994 mask_klass == nullptr || mask_klass->const_oop() == nullptr ||
995 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
996 vlen == nullptr || !vlen->is_con() ||
997 from_ms == nullptr || !from_ms->is_con()) {
998 log_if_needed(" ** missing constant: vclass=%s mclass=%s etype=%s vlen=%s from_ms=%s",
999 NodeClassNames[argument(0)->Opcode()],
1000 NodeClassNames[argument(1)->Opcode()],
1001 NodeClassNames[argument(2)->Opcode()],
1002 NodeClassNames[argument(3)->Opcode()],
1003 NodeClassNames[argument(7)->Opcode()]);
1004 return false; // not enough info for intrinsification
1005 }
1006 if (!is_klass_initialized(vector_klass)) {
1007 log_if_needed(" ** klass argument not initialized");
1008 return false;
1009 }
1010
1011 if (!is_klass_initialized(mask_klass)) {
1012 log_if_needed(" ** mask klass argument not initialized");
1013 return false;
1014 }
1015
1016 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1017 if (!elem_type->is_primitive_type()) {
1018 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1019 return false; // should be primitive type
1020 }
1021
1022 BasicType elem_bt = elem_type->basic_type();
1023 int num_elem = vlen->get_con();
1024
1025 Node* base = argument(4);
1026 Node* offset = ConvL2X(argument(5));
1027
1028 // Save state and restore on bailout
1029 SavedState old_state(this);
1030
1031 Node* addr = make_unsafe_address(base, offset, elem_bt, true);
1032 const TypePtr *addr_type = gvn().type(addr)->isa_ptr();
1033 const TypeAryPtr* arr_type = addr_type->isa_aryptr();
1034
1035 bool mismatched_ms = from_ms->get_con() && arr_type != nullptr && arr_type->elem()->array_element_basic_type() != elem_bt;
1036 BIG_ENDIAN_ONLY(if (mismatched_ms) return false;)
1037 // If there is no consistency between array and vector element types, it must be special byte array case
1038 if (arr_type != nullptr && !elem_consistent_with_arr(elem_bt, arr_type, mismatched_ms)) {
1039 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d etype=%s atype=%s",
1040 is_store, is_store ? "storeMasked" : "loadMasked",
1041 num_elem, type2name(elem_bt), type2name(arr_type->elem()->array_element_basic_type()));
1042 return false;
1043 }
1044
1045 int mem_num_elem = mismatched_ms ? num_elem * type2aelembytes(elem_bt) : num_elem;
1046 BasicType mem_elem_bt = mismatched_ms ? T_BYTE : elem_bt;
1047 bool supports_predicate = arch_supports_vector(is_store ? Op_StoreVectorMasked : Op_LoadVectorMasked,
1048 mem_num_elem, mem_elem_bt, VecMaskUseLoad);
1049
1050 // If current arch does not support the predicated operations, we have to bail
1051 // out when current case uses the predicate feature.
1052 if (!supports_predicate) {
1053 bool needs_predicate = false;
1054 if (is_store) {
1055 // Masked vector store always uses the predicated store.
1056 needs_predicate = true;
1057 } else {
1058 // Masked vector load with IOOBE always uses the predicated load.
1059 const TypeInt* offset_in_range = gvn().type(argument(9))->isa_int();
1060 if (!offset_in_range->is_con()) {
1061 log_if_needed(" ** missing constant: offsetInRange=%s",
1062 NodeClassNames[argument(8)->Opcode()]);
1063 return false;
1064 }
1065 needs_predicate = (offset_in_range->get_con() == 0);
1066 }
1067
1068 if (needs_predicate) {
1069 log_if_needed(" ** not supported: op=%s vlen=%d etype=%s mismatched_ms=%d",
1070 is_store ? "storeMasked" : "loadMasked",
1071 num_elem, type2name(elem_bt), mismatched_ms ? 1 : 0);
1072 return false;
1073 }
1074 }
1075
1076 // This only happens for masked vector load. If predicate is not supported, then check whether
1077 // the normal vector load and blend operations are supported by backend.
1078 if (!supports_predicate && (!arch_supports_vector(Op_LoadVector, mem_num_elem, mem_elem_bt, VecMaskNotUsed) ||
1079 !arch_supports_vector(Op_VectorBlend, mem_num_elem, mem_elem_bt, VecMaskUseLoad))) {
1080 log_if_needed(" ** not supported: op=loadMasked vlen=%d etype=%s mismatched_ms=%d",
1081 num_elem, type2name(elem_bt), mismatched_ms ? 1 : 0);
1082 return false;
1083 }
1084
1085 // Since we are using byte array, we need to double check that the vector reinterpret operation
1086 // with byte type is supported by backend.
1087 if (mismatched_ms) {
1088 if (!arch_supports_vector(Op_VectorReinterpret, mem_num_elem, T_BYTE, VecMaskNotUsed)) {
1089 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d etype=%s mismatched_ms=1",
1090 is_store, is_store ? "storeMasked" : "loadMasked",
1091 num_elem, type2name(elem_bt));
1092 return false;
1093 }
1094 }
1095
1096 // Since it needs to unbox the mask, we need to double check that the related load operations
1097 // for mask are supported by backend.
1098 if (!arch_supports_vector(Op_LoadVector, num_elem, elem_bt, VecMaskUseLoad)) {
1099 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d etype=%s",
1100 is_store, is_store ? "storeMasked" : "loadMasked",
1101 num_elem, type2name(elem_bt));
1102 return false;
1103 }
1104
1105 // Can base be null? Otherwise, always on-heap access.
1106 bool can_access_non_heap = TypePtr::NULL_PTR->higher_equal(gvn().type(base));
1107 if (can_access_non_heap) {
1108 insert_mem_bar(Op_MemBarCPUOrder);
1109 }
1110
1111 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1112 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1113 assert(!is_vector_mask(vbox_klass) && is_vector_mask(mbox_klass), "Invalid class type");
1114 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1115 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1116
1117 Node* mask = unbox_vector(is_store ? argument(9) : argument(8), mbox_type, elem_bt, num_elem);
1118 if (mask == nullptr) {
1119 log_if_needed(" ** unbox failed mask=%s",
1120 is_store ? NodeClassNames[argument(9)->Opcode()]
1121 : NodeClassNames[argument(8)->Opcode()]);
1122 return false;
1123 }
1124
1125 if (is_store) {
1126 Node* val = unbox_vector(argument(8), vbox_type, elem_bt, num_elem);
1127 if (val == nullptr) {
1128 log_if_needed(" ** unbox failed vector=%s",
1129 NodeClassNames[argument(8)->Opcode()]);
1130 return false; // operand unboxing failed
1131 }
1132 set_all_memory(reset_memory());
1133
1134 if (mismatched_ms) {
1135 // Reinterpret the incoming vector to byte vector.
1136 const TypeVect* to_vect_type = TypeVect::make(mem_elem_bt, mem_num_elem);
1137 val = gvn().transform(new VectorReinterpretNode(val, val->bottom_type()->is_vect(), to_vect_type));
1138 // Reinterpret the vector mask to byte type.
1139 const TypeVect* from_mask_type = TypeVect::makemask(elem_bt, num_elem);
1140 const TypeVect* to_mask_type = TypeVect::makemask(mem_elem_bt, mem_num_elem);
1141 mask = gvn().transform(new VectorReinterpretNode(mask, from_mask_type, to_mask_type));
1142 }
1143 Node* vstore = gvn().transform(new StoreVectorMaskedNode(control(), memory(addr), addr, val, addr_type, mask));
1144 set_memory(vstore, addr_type);
1145 } else {
1146 Node* vload = nullptr;
1147
1148 if (mismatched_ms) {
1149 // Reinterpret the vector mask to byte type.
1150 const TypeVect* from_mask_type = TypeVect::makemask(elem_bt, num_elem);
1151 const TypeVect* to_mask_type = TypeVect::makemask(mem_elem_bt, mem_num_elem);
1152 mask = gvn().transform(new VectorReinterpretNode(mask, from_mask_type, to_mask_type));
1153 }
1154
1155 if (supports_predicate) {
1156 // Generate masked load vector node if predicate feature is supported.
1157 const TypeVect* vt = TypeVect::make(mem_elem_bt, mem_num_elem);
1158 vload = gvn().transform(new LoadVectorMaskedNode(control(), memory(addr), addr, addr_type, vt, mask));
1159 } else {
1160 // Use the vector blend to implement the masked load vector. The biased elements are zeros.
1161 Node* zero = gvn().transform(gvn().zerocon(mem_elem_bt));
1162 zero = gvn().transform(VectorNode::scalar2vector(zero, mem_num_elem, mem_elem_bt));
1163 vload = gvn().transform(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, mem_num_elem, mem_elem_bt));
1164 vload = gvn().transform(new VectorBlendNode(zero, vload, mask));
1165 }
1166
1167 if (mismatched_ms) {
1168 const TypeVect* to_vect_type = TypeVect::make(elem_bt, num_elem);
1169 vload = gvn().transform(new VectorReinterpretNode(vload, vload->bottom_type()->is_vect(), to_vect_type));
1170 }
1171
1172 Node* box = box_vector(vload, vbox_type, elem_bt, num_elem);
1173 set_result(box);
1174 }
1175
1176 old_state.discard();
1177
1178 if (can_access_non_heap) {
1179 insert_mem_bar(Op_MemBarCPUOrder);
1180 }
1181
1182 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1183 return true;
1184 }
1185
1186 //
1187 // <C,
1188 // V extends Vector<?>,
1189 // W extends Vector<Integer>,
1190 // S extends VectorSpecies<E>,
1191 // M extends VectorMask<E>,
1192 // E>
1193 // V loadWithMap(Class<? extends V> vClass, Class<M> mClass, Class<E> eClass, int length,
1194 // Class<? extends Vector<Integer>> vectorIndexClass, int indexLength,
1195 // Object base, long offset,
1196 // W indexVector1, W indexVector2, W indexVector3, W indexVector4,
1197 // M m, C container, int index, int[] indexMap, int indexM, S s,
1198 // LoadVectorOperationWithMap<C, V, S, M> defaultImpl)
1199 //
1200 // <C,
1201 // V extends Vector<E>,
1202 // W extends Vector<Integer>,
1203 // M extends VectorMask<E>,
1204 // E>
1205 // void storeWithMap(Class<? extends V> vClass, Class<M> mClass, Class<E> eClass, int length,
1206 // Class<? extends Vector<Integer>> vectorIndexClass, int indexLength,
1207 // Object base, long offset, // Unsafe addressing
1208 // W indexVector, V v, M m,
1209 // C container, int index, int[] indexMap, int indexM, // Arguments for default implementation
1210 // StoreVectorOperationWithMap<C, V, M> defaultImpl)
1211 //
1212 bool LibraryCallKit::inline_vector_gather_scatter(bool is_scatter) {
1213 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
1214 const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
1215 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
1216 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
1217 const TypeInstPtr* vector_idx_klass = gvn().type(argument(4))->isa_instptr();
1218 const TypeInt* idx_vlen = gvn().type(argument(5))->isa_int();
1219
1220 if (vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
1221 // mask_klass == nullptr || mask_klass->const_oop() == nullptr ||
1222 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
1223 vlen == nullptr || !vlen->is_con() ||
1224 vector_idx_klass == nullptr || vector_idx_klass->const_oop() == nullptr ||
1225 idx_vlen == nullptr || !idx_vlen->is_con()) {
1226 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s viclass=%s idx_vlen=%s",
1227 NodeClassNames[argument(0)->Opcode()],
1228 NodeClassNames[argument(2)->Opcode()],
1229 NodeClassNames[argument(3)->Opcode()],
1230 NodeClassNames[argument(4)->Opcode()],
1231 NodeClassNames[argument(5)->Opcode()]);
1232 return false; // not enough info for intrinsification
1233 }
1234
1235 if (!is_klass_initialized(vector_klass) || !is_klass_initialized(vector_idx_klass)) {
1236 log_if_needed(" ** klass argument not initialized");
1237 return false;
1238 }
1239
1240 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1241 if (!elem_type->is_primitive_type()) {
1242 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1243 return false; // should be primitive type
1244 }
1245
1246 BasicType elem_bt = elem_type->basic_type();
1247 int num_elem = vlen->get_con();
1248 int idx_num_elem = idx_vlen->get_con();
1249
1250 Node* m = is_scatter ? argument(11) : argument(13);
1251 const Type* vmask_type = gvn().type(m);
1252 bool is_masked_op = vmask_type != TypePtr::NULL_PTR;
1253 if (is_masked_op) {
1254 if (mask_klass == nullptr || mask_klass->const_oop() == nullptr) {
1255 log_if_needed(" ** missing constant: maskclass=%s", NodeClassNames[argument(1)->Opcode()]);
1256 return false; // not enough info for intrinsification
1257 }
1258
1259 if (!is_klass_initialized(mask_klass)) {
1260 log_if_needed(" ** mask klass argument not initialized");
1261 return false;
1262 }
1263
1264 if (vmask_type->maybe_null()) {
1265 log_if_needed(" ** null mask values are not allowed for masked op");
1266 return false;
1267 }
1268
1269 // Check whether the predicated gather/scatter node is supported by architecture.
1270 VectorMaskUseType mask = (VectorMaskUseType) (VecMaskUseLoad | VecMaskUsePred);
1271 if (!arch_supports_vector(is_scatter ? Op_StoreVectorScatterMasked : Op_LoadVectorGatherMasked, num_elem, elem_bt, mask)) {
1272 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d etype=%s is_masked_op=1",
1273 is_scatter, is_scatter ? "scatterMasked" : "gatherMasked",
1274 num_elem, type2name(elem_bt));
1275 return false; // not supported
1276 }
1277 } else {
1278 // Check whether the normal gather/scatter node is supported for non-masked operation.
1279 if (!arch_supports_vector(is_scatter ? Op_StoreVectorScatter : Op_LoadVectorGather, num_elem, elem_bt, VecMaskNotUsed)) {
1280 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d etype=%s is_masked_op=0",
1281 is_scatter, is_scatter ? "scatter" : "gather",
1282 num_elem, type2name(elem_bt));
1283 return false; // not supported
1284 }
1285 }
1286
1287 // Check that the vector holding indices is supported by architecture
1288 // For sub-word gathers expander receive index array.
1289 if (!is_subword_type(elem_bt) && !arch_supports_vector(Op_LoadVector, idx_num_elem, T_INT, VecMaskNotUsed)) {
1290 log_if_needed(" ** not supported: arity=%d op=%s/loadindex vlen=%d etype=int is_masked_op=%d",
1291 is_scatter, is_scatter ? "scatter" : "gather",
1292 idx_num_elem, is_masked_op ? 1 : 0);
1293 return false; // not supported
1294 }
1295
1296 Node* base = argument(6);
1297 Node* offset = ConvL2X(argument(7));
1298
1299 // Save state and restore on bailout
1300 SavedState old_state(this);
1301
1302 Node* addr = nullptr;
1303 if (!is_subword_type(elem_bt)) {
1304 addr = make_unsafe_address(base, offset, elem_bt, true);
1305 } else {
1306 assert(!is_scatter, "Only supports gather operation for subword types now");
1307 uint header = arrayOopDesc::base_offset_in_bytes(elem_bt);
1308 assert(offset->is_Con() && offset->bottom_type()->is_long()->get_con() == header,
1309 "offset must be the array base offset");
1310 Node* index = argument(15);
1311 addr = array_element_address(base, index, elem_bt);
1312 }
1313
1314 const TypePtr* addr_type = gvn().type(addr)->isa_ptr();
1315 const TypeAryPtr* arr_type = addr_type->isa_aryptr();
1316
1317 // The array must be consistent with vector type
1318 if (arr_type == nullptr || (arr_type != nullptr && !elem_consistent_with_arr(elem_bt, arr_type, false))) {
1319 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d etype=%s atype=%s ismask=no",
1320 is_scatter, is_scatter ? "scatter" : "gather",
1321 num_elem, type2name(elem_bt), type2name(arr_type->elem()->array_element_basic_type()));
1322 return false;
1323 }
1324
1325 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1326 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1327 ciKlass* vbox_idx_klass = vector_idx_klass->const_oop()->as_instance()->java_lang_Class_klass();
1328 if (vbox_idx_klass == nullptr) {
1329 return false;
1330 }
1331
1332 // Get the indexes for gather/scatter.
1333 Node* indexes = nullptr;
1334 const TypeInstPtr* vbox_idx_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_idx_klass);
1335 if (is_subword_type(elem_bt)) {
1336 Node* indexMap = argument(16);
1337 Node* indexM = argument(17);
1338 indexes = array_element_address(indexMap, indexM, T_INT);
1339 } else {
1340 // Get the first index vector.
1341 indexes = unbox_vector(argument(9), vbox_idx_type, T_INT, idx_num_elem);
1342 if (indexes == nullptr) {
1343 return false;
1344 }
1345 }
1346
1347 // Get the vector mask value.
1348 Node* mask = nullptr;
1349 if (is_masked_op) {
1350 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1351 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1352 mask = unbox_vector(m, mbox_type, elem_bt, num_elem);
1353 if (mask == nullptr) {
1354 log_if_needed(" ** unbox failed mask=%s", NodeClassNames[m->Opcode()]);
1355 return false;
1356 }
1357 }
1358
1359 const TypeVect* vector_type = TypeVect::make(elem_bt, num_elem);
1360 if (is_scatter) {
1361 Node* val = unbox_vector(argument(10), vbox_type, elem_bt, num_elem);
1362 if (val == nullptr) {
1363 return false; // operand unboxing failed
1364 }
1365 set_all_memory(reset_memory());
1366
1367 Node* vstore = nullptr;
1368 if (mask != nullptr) {
1369 vstore = gvn().transform(new StoreVectorScatterMaskedNode(control(), memory(addr), addr, addr_type, val, indexes, mask));
1370 } else {
1371 vstore = gvn().transform(new StoreVectorScatterNode(control(), memory(addr), addr, addr_type, val, indexes));
1372 }
1373 set_memory(vstore, addr_type);
1374 } else {
1375 Node* vload = nullptr;
1376 if (mask != nullptr) {
1377 vload = gvn().transform(new LoadVectorGatherMaskedNode(control(), memory(addr), addr, addr_type, vector_type, indexes, mask));
1378 } else {
1379 vload = gvn().transform(new LoadVectorGatherNode(control(), memory(addr), addr, addr_type, vector_type, indexes));
1380 }
1381 Node* box = box_vector(vload, vbox_type, elem_bt, num_elem);
1382 set_result(box);
1383 }
1384
1385 old_state.discard();
1386 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1387 return true;
1388 }
1389
1390 // public static
1391 // <V extends Vector<E>,
1392 // M extends VectorMask<E>,
1393 // E>
1394 // long reductionCoerced(int oprId, Class<? extends V> vectorClass, Class<? extends M> maskClass,
1395 // Class<E> elementType, int length, V v, M m,
1396 // ReductionOperation<V, M> defaultImpl)
1397 bool LibraryCallKit::inline_vector_reduction() {
1398 const TypeInt* opr = gvn().type(argument(0))->isa_int();
1399 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
1400 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
1401 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
1402 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
1403
1404 if (opr == nullptr || !opr->is_con() ||
1405 vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
1406 // mask_klass == nullptr || mask_klass->const_oop() == nullptr ||
1407 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
1408 vlen == nullptr || !vlen->is_con()) {
1409 log_if_needed(" ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
1410 NodeClassNames[argument(0)->Opcode()],
1411 NodeClassNames[argument(1)->Opcode()],
1412 NodeClassNames[argument(3)->Opcode()],
1413 NodeClassNames[argument(4)->Opcode()]);
1414 return false; // not enough info for intrinsification
1415 }
1416 if (!is_klass_initialized(vector_klass)) {
1417 log_if_needed(" ** klass argument not initialized");
1418 return false;
1419 }
1420 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1421 if (!elem_type->is_primitive_type()) {
1422 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1423 return false; // should be primitive type
1424 }
1425
1426 const Type* vmask_type = gvn().type(argument(6));
1427 bool is_masked_op = vmask_type != TypePtr::NULL_PTR;
1428 if (is_masked_op) {
1429 if (mask_klass == nullptr || mask_klass->const_oop() == nullptr) {
1430 log_if_needed(" ** missing constant: maskclass=%s", NodeClassNames[argument(2)->Opcode()]);
1431 return false; // not enough info for intrinsification
1432 }
1433
1434 if (!is_klass_initialized(mask_klass)) {
1435 log_if_needed(" ** mask klass argument not initialized");
1436 return false;
1437 }
1438
1439 if (vmask_type->maybe_null()) {
1440 log_if_needed(" ** null mask values are not allowed for masked op");
1441 return false;
1442 }
1443 }
1444
1445 BasicType elem_bt = elem_type->basic_type();
1446 int num_elem = vlen->get_con();
1447 int opc = VectorSupport::vop2ideal(opr->get_con(), elem_bt);
1448 int sopc = ReductionNode::opcode(opc, elem_bt);
1449
1450 // Ensure reduction operation for lanewise operation
1451 // When using mask, mask use type needs to be VecMaskUseLoad.
1452 if (sopc == opc || !arch_supports_vector(sopc, num_elem, elem_bt, is_masked_op ? VecMaskUseLoad : VecMaskNotUsed)) {
1453 log_if_needed(" ** not supported: arity=1 op=%d/reduce vlen=%d etype=%s is_masked_op=%d",
1454 sopc, num_elem, type2name(elem_bt), is_masked_op ? 1 : 0);
1455 return false;
1456 }
1457
1458 // Return true if current platform has implemented the masked operation with predicate feature.
1459 bool use_predicate = is_masked_op && arch_supports_vector(sopc, num_elem, elem_bt, VecMaskUsePred);
1460 if (is_masked_op && !use_predicate && !arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad)) {
1461 log_if_needed(" ** not supported: arity=1 op=%d/reduce vlen=%d etype=%s is_masked_op=1",
1462 sopc, num_elem, type2name(elem_bt));
1463 return false;
1464 }
1465
1466 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1467 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1468
1469 Node* opd = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
1470 if (opd == nullptr) {
1471 return false; // operand unboxing failed
1472 }
1473
1474 Node* mask = nullptr;
1475 if (is_masked_op) {
1476 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1477 assert(is_vector_mask(mbox_klass), "argument(2) should be a mask class");
1478 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1479 mask = unbox_vector(argument(6), mbox_type, elem_bt, num_elem);
1480 if (mask == nullptr) {
1481 log_if_needed(" ** unbox failed mask=%s",
1482 NodeClassNames[argument(6)->Opcode()]);
1483 return false;
1484 }
1485 }
1486
1487 Node* init = ReductionNode::make_identity_con_scalar(gvn(), opc, elem_bt);
1488 Node* value = opd;
1489
1490 assert(mask != nullptr || !is_masked_op, "Masked op needs the mask value never null");
1491 if (mask != nullptr && !use_predicate) {
1492 Node* reduce_identity = gvn().transform(VectorNode::scalar2vector(init, num_elem, elem_bt));
1493 value = gvn().transform(new VectorBlendNode(reduce_identity, value, mask));
1494 }
1495
1496 // Make an unordered Reduction node. This affects only AddReductionVF/VD and MulReductionVF/VD,
1497 // as these operations are allowed to be associative (not requiring strict order) in VectorAPI.
1498 value = ReductionNode::make(opc, nullptr, init, value, elem_bt, /* requires_strict_order */ false);
1499
1500 if (mask != nullptr && use_predicate) {
1501 value->add_req(mask);
1502 value->add_flag(Node::Flag_is_predicated_vector);
1503 }
1504
1505 value = gvn().transform(value);
1506
1507 Node* bits = nullptr;
1508 switch (elem_bt) {
1509 case T_BYTE:
1510 case T_SHORT:
1511 case T_INT: {
1512 bits = gvn().transform(new ConvI2LNode(value));
1513 break;
1514 }
1515 case T_FLOAT: {
1516 value = gvn().transform(new MoveF2INode(value));
1517 bits = gvn().transform(new ConvI2LNode(value));
1518 break;
1519 }
1520 case T_DOUBLE: {
1521 bits = gvn().transform(new MoveD2LNode(value));
1522 break;
1523 }
1524 case T_LONG: {
1525 bits = value; // no conversion needed
1526 break;
1527 }
1528 default: fatal("%s", type2name(elem_bt));
1529 }
1530 set_result(bits);
1531 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1532 return true;
1533 }
1534
1535 // public static <V> boolean test(int cond, Class<?> vectorClass, Class<?> elementType, int vlen,
1536 // V v1, V v2,
1537 // BiFunction<V, V, Boolean> defaultImpl)
1538 //
1539 bool LibraryCallKit::inline_vector_test() {
1540 const TypeInt* cond = gvn().type(argument(0))->isa_int();
1541 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
1542 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
1543 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
1544
1545 if (cond == nullptr || !cond->is_con() ||
1546 vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
1547 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
1548 vlen == nullptr || !vlen->is_con()) {
1549 log_if_needed(" ** missing constant: cond=%s vclass=%s etype=%s vlen=%s",
1550 NodeClassNames[argument(0)->Opcode()],
1551 NodeClassNames[argument(1)->Opcode()],
1552 NodeClassNames[argument(2)->Opcode()],
1553 NodeClassNames[argument(3)->Opcode()]);
1554 return false; // not enough info for intrinsification
1555 }
1556 if (!is_klass_initialized(vector_klass)) {
1557 log_if_needed(" ** klass argument not initialized");
1558 return false;
1559 }
1560 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1561 if (!elem_type->is_primitive_type()) {
1562 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1563 return false; // should be primitive type
1564 }
1565 BasicType elem_bt = elem_type->basic_type();
1566 int num_elem = vlen->get_con();
1567 BoolTest::mask booltest = (BoolTest::mask)cond->get_con();
1568 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1569 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1570
1571 if (!arch_supports_vector(Op_VectorTest, num_elem, elem_bt, is_vector_mask(vbox_klass) ? VecMaskUseLoad : VecMaskNotUsed)) {
1572 log_if_needed(" ** not supported: arity=2 op=test/%d vlen=%d etype=%s ismask=%d",
1573 cond->get_con(), num_elem, type2name(elem_bt),
1574 is_vector_mask(vbox_klass));
1575 return false;
1576 }
1577
1578 Node* opd1 = unbox_vector(argument(4), vbox_type, elem_bt, num_elem);
1579 Node* opd2;
1580 if (Matcher::vectortest_needs_second_argument(booltest == BoolTest::overflow,
1581 opd1->bottom_type()->isa_vectmask())) {
1582 opd2 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
1583 } else {
1584 opd2 = opd1;
1585 }
1586 if (opd1 == nullptr || opd2 == nullptr) {
1587 return false; // operand unboxing failed
1588 }
1589
1590 Node* cmp = gvn().transform(new VectorTestNode(opd1, opd2, booltest));
1591 BoolTest::mask test = Matcher::vectortest_mask(booltest == BoolTest::overflow,
1592 opd1->bottom_type()->isa_vectmask(), num_elem);
1593 Node* bol = gvn().transform(new BoolNode(cmp, test));
1594 Node* res = gvn().transform(new CMoveINode(bol, gvn().intcon(0), gvn().intcon(1), TypeInt::BOOL));
1595
1596 set_result(res);
1597 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1598 return true;
1599 }
1600
1601 // public static
1602 // <V extends Vector<E>,
1603 // M extends VectorMask<E>,
1604 // E>
1605 // V blend(Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType, int vlen,
1606 // V v1, V v2, M m,
1607 // VectorBlendOp<V, M, E> defaultImpl)
1608 bool LibraryCallKit::inline_vector_blend() {
1609 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
1610 const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
1611 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
1612 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
1613
1614 if (mask_klass == nullptr || vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr) {
1615 return false; // dead code
1616 }
1617 if (mask_klass->const_oop() == nullptr || vector_klass->const_oop() == nullptr ||
1618 elem_klass->const_oop() == nullptr || !vlen->is_con()) {
1619 log_if_needed(" ** missing constant: vclass=%s mclass=%s etype=%s vlen=%s",
1620 NodeClassNames[argument(0)->Opcode()],
1621 NodeClassNames[argument(1)->Opcode()],
1622 NodeClassNames[argument(2)->Opcode()],
1623 NodeClassNames[argument(3)->Opcode()]);
1624 return false; // not enough info for intrinsification
1625 }
1626 if (!is_klass_initialized(vector_klass) || !is_klass_initialized(mask_klass)) {
1627 log_if_needed(" ** klass argument not initialized");
1628 return false;
1629 }
1630 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1631 if (!elem_type->is_primitive_type()) {
1632 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1633 return false; // should be primitive type
1634 }
1635 BasicType elem_bt = elem_type->basic_type();
1636 BasicType mask_bt = elem_bt;
1637 int num_elem = vlen->get_con();
1638
1639 if (!arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad)) {
1640 log_if_needed(" ** not supported: arity=2 op=blend vlen=%d etype=%s ismask=useload",
1641 num_elem, type2name(elem_bt));
1642 return false; // not supported
1643 }
1644 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1645 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1646
1647 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1648 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1649
1650 Node* v1 = unbox_vector(argument(4), vbox_type, elem_bt, num_elem);
1651 Node* v2 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
1652 Node* mask = unbox_vector(argument(6), mbox_type, mask_bt, num_elem);
1653
1654 if (v1 == nullptr || v2 == nullptr || mask == nullptr) {
1655 return false; // operand unboxing failed
1656 }
1657
1658 Node* blend = gvn().transform(new VectorBlendNode(v1, v2, mask));
1659
1660 Node* box = box_vector(blend, vbox_type, elem_bt, num_elem);
1661 set_result(box);
1662 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1663 return true;
1664 }
1665
1666 // public static
1667 // <V extends Vector<E>,
1668 // M extends VectorMask<E>,
1669 // E>
1670 // M compare(int cond, Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType, int vlen,
1671 // V v1, V v2, M m,
1672 // VectorCompareOp<V,M> defaultImpl)
1673 bool LibraryCallKit::inline_vector_compare() {
1674 const TypeInt* cond = gvn().type(argument(0))->isa_int();
1675 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
1676 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
1677 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
1678 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
1679
1680 if (cond == nullptr || vector_klass == nullptr || mask_klass == nullptr || elem_klass == nullptr || vlen == nullptr) {
1681 return false; // dead code
1682 }
1683 if (!cond->is_con() || vector_klass->const_oop() == nullptr || mask_klass->const_oop() == nullptr ||
1684 elem_klass->const_oop() == nullptr || !vlen->is_con()) {
1685 log_if_needed(" ** missing constant: cond=%s vclass=%s mclass=%s etype=%s vlen=%s",
1686 NodeClassNames[argument(0)->Opcode()],
1687 NodeClassNames[argument(1)->Opcode()],
1688 NodeClassNames[argument(2)->Opcode()],
1689 NodeClassNames[argument(3)->Opcode()],
1690 NodeClassNames[argument(4)->Opcode()]);
1691 return false; // not enough info for intrinsification
1692 }
1693 if (!is_klass_initialized(vector_klass) || !is_klass_initialized(mask_klass)) {
1694 log_if_needed(" ** klass argument not initialized");
1695 return false;
1696 }
1697 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1698 if (!elem_type->is_primitive_type()) {
1699 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1700 return false; // should be primitive type
1701 }
1702
1703 int num_elem = vlen->get_con();
1704 BasicType elem_bt = elem_type->basic_type();
1705 BasicType mask_bt = elem_bt;
1706
1707 if ((cond->get_con() & BoolTest::unsigned_compare) != 0) {
1708 if (!Matcher::supports_vector_comparison_unsigned(num_elem, elem_bt)) {
1709 log_if_needed(" ** not supported: unsigned comparison op=comp/%d vlen=%d etype=%s ismask=usestore",
1710 cond->get_con() & (BoolTest::unsigned_compare - 1), num_elem, type2name(elem_bt));
1711 return false;
1712 }
1713 }
1714
1715 if (!arch_supports_vector(Op_VectorMaskCmp, num_elem, elem_bt, VecMaskUseStore)) {
1716 log_if_needed(" ** not supported: arity=2 op=comp/%d vlen=%d etype=%s ismask=usestore",
1717 cond->get_con(), num_elem, type2name(elem_bt));
1718 return false;
1719 }
1720
1721 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1722 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1723
1724 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1725 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1726
1727 Node* v1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
1728 Node* v2 = unbox_vector(argument(6), vbox_type, elem_bt, num_elem);
1729
1730 bool is_masked_op = argument(7)->bottom_type() != TypePtr::NULL_PTR;
1731 Node* mask = is_masked_op ? unbox_vector(argument(7), mbox_type, elem_bt, num_elem) : nullptr;
1732 if (is_masked_op && mask == nullptr) {
1733 log_if_needed(" ** not supported: mask = null arity=2 op=comp/%d vlen=%d etype=%s ismask=usestore is_masked_op=1",
1734 cond->get_con(), num_elem, type2name(elem_bt));
1735 return false;
1736 }
1737
1738 bool use_predicate = is_masked_op && arch_supports_vector(Op_VectorMaskCmp, num_elem, elem_bt, VecMaskUsePred);
1739 if (is_masked_op && !use_predicate && !arch_supports_vector(Op_AndV, num_elem, elem_bt, VecMaskUseLoad)) {
1740 log_if_needed(" ** not supported: arity=2 op=comp/%d vlen=%d etype=%s ismask=usestore is_masked_op=1",
1741 cond->get_con(), num_elem, type2name(elem_bt));
1742 return false;
1743 }
1744
1745 if (v1 == nullptr || v2 == nullptr) {
1746 return false; // operand unboxing failed
1747 }
1748 BoolTest::mask pred = (BoolTest::mask)cond->get_con();
1749 ConINode* pred_node = (ConINode*)gvn().makecon(cond);
1750
1751 const TypeVect* vmask_type = TypeVect::makemask(mask_bt, num_elem);
1752 Node* operation = new VectorMaskCmpNode(pred, v1, v2, pred_node, vmask_type);
1753
1754 if (is_masked_op) {
1755 if (use_predicate) {
1756 operation->add_req(mask);
1757 operation->add_flag(Node::Flag_is_predicated_vector);
1758 } else {
1759 operation = gvn().transform(operation);
1760 operation = VectorNode::make(Op_AndV, operation, mask, vmask_type);
1761 }
1762 }
1763
1764 operation = gvn().transform(operation);
1765
1766 Node* box = box_vector(operation, mbox_type, mask_bt, num_elem);
1767 set_result(box);
1768 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1769 return true;
1770 }
1771
1772 // public static
1773 // <V extends Vector<E>,
1774 // Sh extends VectorShuffle<E>,
1775 // M extends VectorMask<E>,
1776 // E>
1777 // V rearrangeOp(Class<? extends V> vectorClass, Class<Sh> shuffleClass, Class<M> maskClass, Class<E> elementType, int vlen,
1778 // V v1, Sh sh, M m,
1779 // VectorRearrangeOp<V, Sh, M, E> defaultImpl)
1780 bool LibraryCallKit::inline_vector_rearrange() {
1781 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
1782 const TypeInstPtr* shuffle_klass = gvn().type(argument(1))->isa_instptr();
1783 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
1784 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
1785 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
1786
1787 if (vector_klass == nullptr || shuffle_klass == nullptr || elem_klass == nullptr || vlen == nullptr) {
1788 return false; // dead code
1789 }
1790 if (shuffle_klass->const_oop() == nullptr ||
1791 vector_klass->const_oop() == nullptr ||
1792 elem_klass->const_oop() == nullptr ||
1793 !vlen->is_con()) {
1794 log_if_needed(" ** missing constant: vclass=%s sclass=%s etype=%s vlen=%s",
1795 NodeClassNames[argument(0)->Opcode()],
1796 NodeClassNames[argument(1)->Opcode()],
1797 NodeClassNames[argument(3)->Opcode()],
1798 NodeClassNames[argument(4)->Opcode()]);
1799 return false; // not enough info for intrinsification
1800 }
1801 if (!is_klass_initialized(vector_klass) ||
1802 !is_klass_initialized(shuffle_klass)) {
1803 log_if_needed(" ** klass argument not initialized");
1804 return false;
1805 }
1806 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1807 if (!elem_type->is_primitive_type()) {
1808 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1809 return false; // should be primitive type
1810 }
1811
1812 BasicType elem_bt = elem_type->basic_type();
1813 BasicType shuffle_bt = elem_bt;
1814 if (shuffle_bt == T_FLOAT) {
1815 shuffle_bt = T_INT;
1816 } else if (shuffle_bt == T_DOUBLE) {
1817 shuffle_bt = T_LONG;
1818 }
1819
1820 int num_elem = vlen->get_con();
1821 bool need_load_shuffle = Matcher::vector_rearrange_requires_load_shuffle(shuffle_bt, num_elem);
1822
1823 if (need_load_shuffle && !arch_supports_vector(Op_VectorLoadShuffle, num_elem, shuffle_bt, VecMaskNotUsed)) {
1824 if (C->print_intrinsics()) {
1825 tty->print_cr(" ** not supported: arity=0 op=load/shuffle vlen=%d etype=%s ismask=no",
1826 num_elem, type2name(shuffle_bt));
1827 }
1828 return false; // not supported
1829 }
1830
1831 bool is_masked_op = argument(7)->bottom_type() != TypePtr::NULL_PTR;
1832 bool use_predicate = is_masked_op;
1833 if (is_masked_op &&
1834 (mask_klass == nullptr ||
1835 mask_klass->const_oop() == nullptr ||
1836 !is_klass_initialized(mask_klass))) {
1837 log_if_needed(" ** mask_klass argument not initialized");
1838 }
1839 if (!arch_supports_vector(Op_AndV, num_elem, elem_bt, VecMaskNotUsed)) {
1840 log_if_needed(" ** not supported: arity=2 op=and vlen=%d etype=%s ismask=no",
1841 num_elem, type2name(elem_bt));
1842 return false;
1843 }
1844 VectorMaskUseType checkFlags = (VectorMaskUseType)(is_masked_op ? (VecMaskUseLoad | VecMaskUsePred) : VecMaskNotUsed);
1845 if (!arch_supports_vector(Op_VectorRearrange, num_elem, elem_bt, checkFlags)) {
1846 use_predicate = false;
1847 if(!is_masked_op ||
1848 (!arch_supports_vector(Op_VectorRearrange, num_elem, elem_bt, VecMaskNotUsed) ||
1849 !arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad) ||
1850 !arch_supports_vector(Op_Replicate, num_elem, elem_bt, VecMaskNotUsed))) {
1851 log_if_needed(" ** not supported: arity=2 op=shuffle/rearrange vlen=%d etype=%s ismask=no",
1852 num_elem, type2name(elem_bt));
1853 return false; // not supported
1854 }
1855 }
1856 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1857 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1858
1859 ciKlass* shbox_klass = shuffle_klass->const_oop()->as_instance()->java_lang_Class_klass();
1860 const TypeInstPtr* shbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, shbox_klass);
1861
1862 Node* v1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
1863 Node* shuffle = unbox_vector(argument(6), shbox_type, shuffle_bt, num_elem);
1864 const TypeVect* st = TypeVect::make(shuffle_bt, num_elem);
1865
1866 if (v1 == nullptr || shuffle == nullptr) {
1867 return false; // operand unboxing failed
1868 }
1869
1870 assert(is_power_of_2(num_elem), "wrapping invalid");
1871 Node* wrapping_mask_elem = gvn().makecon(TypeInteger::make(num_elem - 1, num_elem - 1, Type::WidenMin, shuffle_bt == T_LONG ? T_LONG : T_INT));
1872 Node* wrapping_mask = gvn().transform(VectorNode::scalar2vector(wrapping_mask_elem, num_elem, shuffle_bt));
1873 shuffle = gvn().transform(new AndVNode(shuffle, wrapping_mask, st));
1874
1875 Node* mask = nullptr;
1876 if (is_masked_op) {
1877 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1878 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1879 mask = unbox_vector(argument(7), mbox_type, elem_bt, num_elem);
1880 if (mask == nullptr) {
1881 log_if_needed(" ** not supported: arity=3 op=shuffle/rearrange vlen=%d etype=%s ismask=useload is_masked_op=1",
1882 num_elem, type2name(elem_bt));
1883 return false;
1884 }
1885 }
1886
1887 if (need_load_shuffle) {
1888 shuffle = gvn().transform(new VectorLoadShuffleNode(shuffle, st));
1889 }
1890
1891 Node* rearrange = new VectorRearrangeNode(v1, shuffle);
1892 if (is_masked_op) {
1893 if (use_predicate) {
1894 rearrange->add_req(mask);
1895 rearrange->add_flag(Node::Flag_is_predicated_vector);
1896 } else {
1897 rearrange = gvn().transform(rearrange);
1898 Node* zero = gvn().makecon(Type::get_zero_type(elem_bt));
1899 Node* zerovec = gvn().transform(VectorNode::scalar2vector(zero, num_elem, elem_bt));
1900 rearrange = new VectorBlendNode(zerovec, rearrange, mask);
1901 }
1902 }
1903 rearrange = gvn().transform(rearrange);
1904
1905 Node* box = box_vector(rearrange, vbox_type, elem_bt, num_elem);
1906 set_result(box);
1907 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1908 return true;
1909 }
1910
1911 // public static
1912 // <V extends Vector<E>,
1913 // M extends VectorMask<E>,
1914 // E>
1915 // V selectFromOp(Class<? extends V> vClass, Class<M> mClass, Class<E> eClass,
1916 // int length, V v1, V v2, M m,
1917 // VectorSelectFromOp<V, M> defaultImpl)
1918 bool LibraryCallKit::inline_vector_select_from() {
1919 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
1920 const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
1921 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
1922 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
1923
1924 if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr ||
1925 vector_klass->const_oop() == nullptr ||
1926 elem_klass->const_oop() == nullptr ||
1927 !vlen->is_con()) {
1928 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s",
1929 NodeClassNames[argument(0)->Opcode()],
1930 NodeClassNames[argument(2)->Opcode()],
1931 NodeClassNames[argument(3)->Opcode()]);
1932 return false; // not enough info for intrinsification
1933 }
1934 if (!is_klass_initialized(vector_klass)) {
1935 log_if_needed(" ** klass argument not initialized");
1936 return false;
1937 }
1938 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1939 if (!elem_type->is_primitive_type()) {
1940 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1941 return false; // should be primitive type
1942 }
1943 BasicType elem_bt = elem_type->basic_type();
1944 int num_elem = vlen->get_con();
1945 if (!is_power_of_2(num_elem)) {
1946 log_if_needed(" ** vlen not power of two=%d", num_elem);
1947 return false;
1948 }
1949
1950 BasicType shuffle_bt = elem_bt;
1951 if (shuffle_bt == T_FLOAT) {
1952 shuffle_bt = T_INT;
1953 } else if (shuffle_bt == T_DOUBLE) {
1954 shuffle_bt = T_LONG;
1955 }
1956 bool need_load_shuffle = Matcher::vector_rearrange_requires_load_shuffle(shuffle_bt, num_elem);
1957
1958 int cast_vopc = VectorCastNode::opcode(-1, elem_bt); // from vector of type elem_bt
1959 if ((need_load_shuffle && !arch_supports_vector(Op_VectorLoadShuffle, num_elem, elem_bt, VecMaskNotUsed)) ||
1960 (elem_bt != shuffle_bt && !arch_supports_vector(cast_vopc, num_elem, shuffle_bt, VecMaskNotUsed)) ||
1961 !arch_supports_vector(Op_AndV, num_elem, shuffle_bt, VecMaskNotUsed) ||
1962 !arch_supports_vector(Op_Replicate, num_elem, shuffle_bt, VecMaskNotUsed)) {
1963 log_if_needed(" ** not supported: arity=0 op=selectFrom vlen=%d etype=%s ismask=no",
1964 num_elem, type2name(elem_bt));
1965 return false; // not supported
1966 }
1967
1968 bool is_masked_op = argument(6)->bottom_type() != TypePtr::NULL_PTR;
1969 bool use_predicate = is_masked_op;
1970 if (is_masked_op &&
1971 (mask_klass == nullptr ||
1972 mask_klass->const_oop() == nullptr ||
1973 !is_klass_initialized(mask_klass))) {
1974 log_if_needed(" ** mask_klass argument not initialized");
1975 return false; // not supported
1976 }
1977 VectorMaskUseType checkFlags = (VectorMaskUseType)(is_masked_op ? (VecMaskUseLoad | VecMaskUsePred) : VecMaskNotUsed);
1978 if (!arch_supports_vector(Op_VectorRearrange, num_elem, elem_bt, checkFlags)) {
1979 use_predicate = false;
1980 if(!is_masked_op ||
1981 (!arch_supports_vector(Op_VectorRearrange, num_elem, elem_bt, VecMaskNotUsed) ||
1982 !arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad) ||
1983 !arch_supports_vector(Op_Replicate, num_elem, elem_bt, VecMaskNotUsed))) {
1984 log_if_needed(" ** not supported: op=selectFrom vlen=%d etype=%s is_masked_op=%d",
1985 num_elem, type2name(elem_bt), is_masked_op);
1986 return false; // not supported
1987 }
1988 }
1989 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1990 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1991
1992 // v1 is the index vector
1993 Node* v1 = unbox_vector(argument(4), vbox_type, elem_bt, num_elem);
1994 // v2 is the vector being rearranged
1995 Node* v2 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
1996
1997 if (v1 == nullptr) {
1998 log_if_needed(" ** unbox failed v1=%s", NodeClassNames[argument(4)->Opcode()]);
1999 return false; // operand unboxing failed
2000 }
2001
2002 if (v2 == nullptr) {
2003 log_if_needed(" ** unbox failed v2=%s", NodeClassNames[argument(5)->Opcode()]);
2004 return false; // operand unboxing failed
2005 }
2006
2007 Node* mask = nullptr;
2008 if (is_masked_op) {
2009 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
2010 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
2011 mask = unbox_vector(argument(6), mbox_type, elem_bt, num_elem);
2012 if (mask == nullptr) {
2013 log_if_needed(" ** unbox failed mask=%s", NodeClassNames[argument(6)->Opcode()]);
2014 return false;
2015 }
2016 }
2017
2018 // cast index vector from elem_bt vector to byte vector
2019 const TypeVect* shuffle_vt = TypeVect::make(shuffle_bt, num_elem);
2020 Node* shuffle = v1;
2021
2022 if (shuffle_bt != elem_bt) {
2023 shuffle = gvn().transform(VectorCastNode::make(cast_vopc, v1, shuffle_bt, num_elem));
2024 }
2025
2026 // wrap the byte vector lanes to (num_elem - 1) to form the shuffle vector where num_elem is vector length
2027 // this is a simple AND operation as we come here only for power of two vector length
2028 Node* mod_val = gvn().makecon(TypeInteger::make(num_elem - 1, num_elem - 1, Type::WidenMin, shuffle_bt == T_LONG ? T_LONG : T_INT));
2029 Node* bcast_mod = gvn().transform(VectorNode::scalar2vector(mod_val, num_elem, shuffle_bt));
2030 shuffle = gvn().transform(VectorNode::make(Op_AndV, shuffle, bcast_mod, shuffle_vt));
2031
2032 // load the shuffle to use in rearrange
2033 if (need_load_shuffle) {
2034 shuffle = gvn().transform(new VectorLoadShuffleNode(shuffle, shuffle_vt));
2035 }
2036
2037 // and finally rearrange
2038 Node* rearrange = new VectorRearrangeNode(v2, shuffle);
2039 if (is_masked_op) {
2040 if (use_predicate) {
2041 // masked rearrange is supported so use that directly
2042 rearrange->add_req(mask);
2043 rearrange->add_flag(Node::Flag_is_predicated_vector);
2044 } else {
2045 // masked rearrange is not supported so emulate usig blend
2046 const TypeVect* vt = v1->bottom_type()->is_vect();
2047 rearrange = gvn().transform(rearrange);
2048
2049 // create a zero vector with each lane element set as zero
2050 Node* zero = gvn().makecon(Type::get_zero_type(elem_bt));
2051 Node* zerovec = gvn().transform(VectorNode::scalar2vector(zero, num_elem, elem_bt));
2052
2053 // For each lane for which mask is set, blend in the rearranged lane into zero vector
2054 rearrange = new VectorBlendNode(zerovec, rearrange, mask);
2055 }
2056 }
2057 rearrange = gvn().transform(rearrange);
2058
2059 // box the result
2060 Node* box = box_vector(rearrange, vbox_type, elem_bt, num_elem);
2061 set_result(box);
2062
2063 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2064 return true;
2065 }
2066
2067 // public static
2068 // <V extends Vector<E>,
2069 // M extends VectorMask<E>,
2070 // E>
2071 // V broadcastInt(int opr, Class<? extends V> vectorClass, Class<? extends M> maskClass,
2072 // Class<E> elementType, int length,
2073 // V v, int n, M m,
2074 // VectorBroadcastIntOp<V, M> defaultImpl)
2075 bool LibraryCallKit::inline_vector_broadcast_int() {
2076 const TypeInt* opr = gvn().type(argument(0))->isa_int();
2077 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
2078 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
2079 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
2080 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
2081
2082 if (opr == nullptr || vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr) {
2083 return false; // dead code
2084 }
2085 if (!opr->is_con() || vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
2086 log_if_needed(" ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
2087 NodeClassNames[argument(0)->Opcode()],
2088 NodeClassNames[argument(1)->Opcode()],
2089 NodeClassNames[argument(3)->Opcode()],
2090 NodeClassNames[argument(4)->Opcode()]);
2091 return false; // not enough info for intrinsification
2092 }
2093 if (!is_klass_initialized(vector_klass)) {
2094 log_if_needed(" ** klass argument not initialized");
2095 return false;
2096 }
2097
2098 const Type* vmask_type = gvn().type(argument(7));
2099 bool is_masked_op = vmask_type != TypePtr::NULL_PTR;
2100 if (is_masked_op) {
2101 if (mask_klass == nullptr || mask_klass->const_oop() == nullptr) {
2102 log_if_needed(" ** missing constant: maskclass=%s", NodeClassNames[argument(2)->Opcode()]);
2103 return false; // not enough info for intrinsification
2104 }
2105
2106 if (!is_klass_initialized(mask_klass)) {
2107 log_if_needed(" ** mask klass argument not initialized");
2108 return false;
2109 }
2110
2111 if (vmask_type->maybe_null()) {
2112 log_if_needed(" ** null mask values are not allowed for masked op");
2113 return false;
2114 }
2115 }
2116
2117 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2118 if (!elem_type->is_primitive_type()) {
2119 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2120 return false; // should be primitive type
2121 }
2122
2123 int num_elem = vlen->get_con();
2124 BasicType elem_bt = elem_type->basic_type();
2125 int opc = VectorSupport::vop2ideal(opr->get_con(), elem_bt);
2126
2127 bool is_shift = VectorNode::is_shift_opcode(opc);
2128 bool is_rotate = VectorNode::is_rotate_opcode(opc);
2129
2130 if (opc == 0 || (!is_shift && !is_rotate)) {
2131 log_if_needed(" ** operation not supported: op=%d bt=%s", opr->get_con(), type2name(elem_bt));
2132 return false; // operation not supported
2133 }
2134
2135 int sopc = VectorNode::opcode(opc, elem_bt);
2136 if (sopc == 0) {
2137 log_if_needed(" ** operation not supported: opc=%s bt=%s", NodeClassNames[opc], type2name(elem_bt));
2138 return false; // operation not supported
2139 }
2140
2141 Node* cnt = argument(6);
2142 const TypeInt* cnt_type = cnt->bottom_type()->isa_int();
2143
2144 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2145 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2146
2147 // If CPU supports vector constant rotate instructions pass it directly
2148 bool is_const_rotate = is_rotate && cnt_type && cnt_type->is_con() &&
2149 Matcher::supports_vector_constant_rotates(cnt_type->get_con());
2150 bool has_scalar_args = is_rotate ? !is_const_rotate : true;
2151
2152 VectorMaskUseType checkFlags = (VectorMaskUseType)(is_masked_op ? (VecMaskUseLoad | VecMaskUsePred) : VecMaskNotUsed);
2153 bool use_predicate = is_masked_op;
2154
2155 if (!arch_supports_vector(sopc, num_elem, elem_bt, checkFlags, has_scalar_args)) {
2156 use_predicate = false;
2157 if (!is_masked_op ||
2158 (!arch_supports_vector(sopc, num_elem, elem_bt, VecMaskNotUsed, has_scalar_args) ||
2159 !arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad))) {
2160
2161 log_if_needed(" ** not supported: arity=0 op=int/%d vlen=%d etype=%s is_masked_op=%d",
2162 sopc, num_elem, type2name(elem_bt), is_masked_op ? 1 : 0);
2163 return false; // not supported
2164 }
2165 }
2166
2167 Node* opd1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
2168 Node* opd2 = nullptr;
2169 if (is_shift) {
2170 opd2 = vector_shift_count(cnt, opc, elem_bt, num_elem);
2171 } else {
2172 assert(is_rotate, "unexpected operation");
2173 if (!is_const_rotate) {
2174 cnt = elem_bt == T_LONG ? gvn().transform(new ConvI2LNode(cnt)) : cnt;
2175 opd2 = gvn().transform(VectorNode::scalar2vector(cnt, num_elem, elem_bt));
2176 } else {
2177 // Constant shift value.
2178 opd2 = cnt;
2179 }
2180 }
2181
2182 if (opd1 == nullptr || opd2 == nullptr) {
2183 return false;
2184 }
2185
2186 Node* mask = nullptr;
2187 if (is_masked_op) {
2188 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
2189 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
2190 mask = unbox_vector(argument(7), mbox_type, elem_bt, num_elem);
2191 if (mask == nullptr) {
2192 log_if_needed(" ** unbox failed mask=%s", NodeClassNames[argument(7)->Opcode()]);
2193 return false;
2194 }
2195 }
2196
2197 Node* operation = VectorNode::make(opc, opd1, opd2, num_elem, elem_bt);
2198 if (is_masked_op && mask != nullptr) {
2199 if (use_predicate) {
2200 operation->add_req(mask);
2201 operation->add_flag(Node::Flag_is_predicated_vector);
2202 } else {
2203 operation = gvn().transform(operation);
2204 operation = new VectorBlendNode(opd1, operation, mask);
2205 }
2206 }
2207 operation = gvn().transform(operation);
2208 Node* vbox = box_vector(operation, vbox_type, elem_bt, num_elem);
2209 set_result(vbox);
2210 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2211 return true;
2212 }
2213
2214 // public static <VOUT extends VectorPayload,
2215 // VIN extends VectorPayload,
2216 // S extends VectorSpecies>
2217 // VOUT convert(int oprId,
2218 // Class<?> fromVectorClass, Class<?> fromElementType, int fromVLen,
2219 // Class<?> toVectorClass, Class<?> toElementType, int toVLen,
2220 // VIN v, S s,
2221 // VectorConvertOp<VOUT, VIN, S> defaultImpl)
2222 //
2223 bool LibraryCallKit::inline_vector_convert() {
2224 const TypeInt* opr = gvn().type(argument(0))->isa_int();
2225
2226 const TypeInstPtr* vector_klass_from = gvn().type(argument(1))->isa_instptr();
2227 const TypeInstPtr* elem_klass_from = gvn().type(argument(2))->isa_instptr();
2228 const TypeInt* vlen_from = gvn().type(argument(3))->isa_int();
2229
2230 const TypeInstPtr* vector_klass_to = gvn().type(argument(4))->isa_instptr();
2231 const TypeInstPtr* elem_klass_to = gvn().type(argument(5))->isa_instptr();
2232 const TypeInt* vlen_to = gvn().type(argument(6))->isa_int();
2233
2234 if (opr == nullptr ||
2235 vector_klass_from == nullptr || elem_klass_from == nullptr || vlen_from == nullptr ||
2236 vector_klass_to == nullptr || elem_klass_to == nullptr || vlen_to == nullptr) {
2237 return false; // dead code
2238 }
2239 if (!opr->is_con() ||
2240 vector_klass_from->const_oop() == nullptr || elem_klass_from->const_oop() == nullptr || !vlen_from->is_con() ||
2241 vector_klass_to->const_oop() == nullptr || elem_klass_to->const_oop() == nullptr || !vlen_to->is_con()) {
2242 log_if_needed(" ** missing constant: opr=%s vclass_from=%s etype_from=%s vlen_from=%s vclass_to=%s etype_to=%s vlen_to=%s",
2243 NodeClassNames[argument(0)->Opcode()],
2244 NodeClassNames[argument(1)->Opcode()],
2245 NodeClassNames[argument(2)->Opcode()],
2246 NodeClassNames[argument(3)->Opcode()],
2247 NodeClassNames[argument(4)->Opcode()],
2248 NodeClassNames[argument(5)->Opcode()],
2249 NodeClassNames[argument(6)->Opcode()]);
2250 return false; // not enough info for intrinsification
2251 }
2252 if (!is_klass_initialized(vector_klass_from) || !is_klass_initialized(vector_klass_to)) {
2253 log_if_needed(" ** klass argument not initialized");
2254 return false;
2255 }
2256
2257 assert(opr->get_con() == VectorSupport::VECTOR_OP_CAST ||
2258 opr->get_con() == VectorSupport::VECTOR_OP_UCAST ||
2259 opr->get_con() == VectorSupport::VECTOR_OP_REINTERPRET, "wrong opcode");
2260 bool is_cast = (opr->get_con() == VectorSupport::VECTOR_OP_CAST || opr->get_con() == VectorSupport::VECTOR_OP_UCAST);
2261 bool is_ucast = (opr->get_con() == VectorSupport::VECTOR_OP_UCAST);
2262
2263 ciKlass* vbox_klass_from = vector_klass_from->const_oop()->as_instance()->java_lang_Class_klass();
2264 ciKlass* vbox_klass_to = vector_klass_to->const_oop()->as_instance()->java_lang_Class_klass();
2265
2266 bool is_mask = is_vector_mask(vbox_klass_from);
2267
2268 ciType* elem_type_from = elem_klass_from->const_oop()->as_instance()->java_mirror_type();
2269 if (!elem_type_from->is_primitive_type()) {
2270 return false; // should be primitive type
2271 }
2272 BasicType elem_bt_from = elem_type_from->basic_type();
2273 ciType* elem_type_to = elem_klass_to->const_oop()->as_instance()->java_mirror_type();
2274 if (!elem_type_to->is_primitive_type()) {
2275 return false; // should be primitive type
2276 }
2277 BasicType elem_bt_to = elem_type_to->basic_type();
2278
2279 int num_elem_from = vlen_from->get_con();
2280 int num_elem_to = vlen_to->get_con();
2281
2282 // Check whether we can unbox to appropriate size. Even with casting, checking for reinterpret is needed
2283 // since we may need to change size.
2284 if (!arch_supports_vector(Op_VectorReinterpret,
2285 num_elem_from,
2286 elem_bt_from,
2287 is_mask ? VecMaskUseAll : VecMaskNotUsed)) {
2288 log_if_needed(" ** not supported: arity=1 op=%s/1 vlen1=%d etype1=%s ismask=%d",
2289 is_cast ? "cast" : "reinterpret",
2290 num_elem_from, type2name(elem_bt_from), is_mask);
2291 return false;
2292 }
2293
2294 // Check whether we can support resizing/reinterpreting to the new size.
2295 if (!arch_supports_vector(Op_VectorReinterpret,
2296 num_elem_to,
2297 elem_bt_to,
2298 is_mask ? VecMaskUseAll : VecMaskNotUsed)) {
2299 log_if_needed(" ** not supported: arity=1 op=%s/2 vlen2=%d etype2=%s ismask=%d",
2300 is_cast ? "cast" : "reinterpret",
2301 num_elem_to, type2name(elem_bt_to), is_mask);
2302 return false;
2303 }
2304
2305 // At this point, we know that both input and output vector registers are supported
2306 // by the architecture. Next check if the casted type is simply to same type - which means
2307 // that it is actually a resize and not a cast.
2308 if (is_cast && elem_bt_from == elem_bt_to) {
2309 is_cast = false;
2310 }
2311
2312 const TypeInstPtr* vbox_type_from = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass_from);
2313
2314 Node* opd1 = unbox_vector(argument(7), vbox_type_from, elem_bt_from, num_elem_from);
2315 if (opd1 == nullptr) {
2316 return false;
2317 }
2318
2319 const TypeVect* src_type = TypeVect::make(elem_bt_from, num_elem_from, is_mask);
2320 const TypeVect* dst_type = TypeVect::make(elem_bt_to, num_elem_to, is_mask);
2321
2322 // Safety check to prevent casting if source mask is of type vector
2323 // and destination mask of type predicate vector and vice-versa.
2324 // From X86 standpoint, this case will only arise over KNL target,
2325 // where certain masks (depending on the species) are either propagated
2326 // through a vector or predicate register.
2327 if (is_mask &&
2328 ((src_type->isa_vectmask() == nullptr && dst_type->isa_vectmask()) ||
2329 (dst_type->isa_vectmask() == nullptr && src_type->isa_vectmask()))) {
2330 return false;
2331 }
2332
2333 Node* op = opd1;
2334 if (is_cast) {
2335 assert(!is_mask || num_elem_from == num_elem_to, "vector mask cast needs the same elem num");
2336
2337 // Make sure the precondition of VectorCastNode::opcode holds: we can only have
2338 // unsigned casts for integral types (excluding long). VectorAPI code is not
2339 // expected to violate this at runtime, but we may compile unreachable code
2340 // where such impossible combinations arise.
2341 if (is_ucast && (!is_integral_type(elem_bt_from) || elem_bt_from == T_LONG)) {
2342 // Halt-and-catch fire here. This condition should never happen at runtime.
2343 stringStream ss;
2344 ss.print("impossible combination: unsigned vector cast from %s", type2name(elem_bt_from));
2345 halt(control(), frameptr(), ss.as_string(C->comp_arena()));
2346 stop_and_kill_map();
2347 log_if_needed(" ** impossible combination: unsigned cast from %s", type2name(elem_bt_from));
2348 return true;
2349 }
2350
2351 int cast_vopc = VectorCastNode::opcode(-1, elem_bt_from, !is_ucast);
2352
2353 // Make sure that vector cast is implemented to particular type/size combination if it is
2354 // not a mask casting.
2355 if (!is_mask && !arch_supports_vector(cast_vopc, num_elem_to, elem_bt_to, VecMaskNotUsed)) {
2356 log_if_needed(" ** not supported: arity=1 op=cast#%d/3 vlen2=%d etype2=%s ismask=%d",
2357 cast_vopc, num_elem_to, type2name(elem_bt_to), is_mask);
2358 return false;
2359 }
2360
2361 if (num_elem_from < num_elem_to) {
2362 // Since input and output number of elements are not consistent, we need to make sure we
2363 // properly size. Thus, first make a cast that retains the number of elements from source.
2364 int num_elem_for_cast = num_elem_from;
2365
2366 // It is possible that arch does not support this intermediate vector size
2367 // TODO More complex logic required here to handle this corner case for the sizes.
2368 if (!arch_supports_vector(cast_vopc, num_elem_for_cast, elem_bt_to, VecMaskNotUsed)) {
2369 log_if_needed(" ** not supported: arity=1 op=cast#%d/4 vlen1=%d etype2=%s ismask=%d",
2370 cast_vopc,
2371 num_elem_for_cast, type2name(elem_bt_to), is_mask);
2372 return false;
2373 }
2374
2375 op = gvn().transform(VectorCastNode::make(cast_vopc, op, elem_bt_to, num_elem_for_cast));
2376 // Now ensure that the destination gets properly resized to needed size.
2377 op = gvn().transform(new VectorReinterpretNode(op, op->bottom_type()->is_vect(), dst_type));
2378 } else if (num_elem_from > num_elem_to) {
2379 // Since number of elements from input is larger than output, simply reduce size of input
2380 // (we are supposed to drop top elements anyway).
2381 int num_elem_for_resize = num_elem_to;
2382
2383 // It is possible that arch does not support this intermediate vector size
2384 // TODO More complex logic required here to handle this corner case for the sizes.
2385 if (!arch_supports_vector(Op_VectorReinterpret,
2386 num_elem_for_resize,
2387 elem_bt_from,
2388 VecMaskNotUsed)) {
2389 log_if_needed(" ** not supported: arity=1 op=cast/5 vlen2=%d etype1=%s ismask=%d",
2390 num_elem_for_resize, type2name(elem_bt_from), is_mask);
2391 return false;
2392 }
2393
2394 const TypeVect* resize_type = TypeVect::make(elem_bt_from, num_elem_for_resize);
2395 op = gvn().transform(new VectorReinterpretNode(op, src_type, resize_type));
2396 op = gvn().transform(VectorCastNode::make(cast_vopc, op, elem_bt_to, num_elem_to));
2397 } else { // num_elem_from == num_elem_to
2398 if (is_mask) {
2399 // Make sure that cast for vector mask is implemented to particular type/size combination.
2400 if (!arch_supports_vector(Op_VectorMaskCast, num_elem_to, elem_bt_to, VecMaskNotUsed)) {
2401 log_if_needed(" ** not supported: arity=1 op=maskcast vlen2=%d etype2=%s ismask=%d",
2402 num_elem_to, type2name(elem_bt_to), is_mask);
2403 return false;
2404 }
2405 op = gvn().transform(new VectorMaskCastNode(op, dst_type));
2406 } else {
2407 // Since input and output number of elements match, and since we know this vector size is
2408 // supported, simply do a cast with no resize needed.
2409 op = gvn().transform(VectorCastNode::make(cast_vopc, op, elem_bt_to, num_elem_to));
2410 }
2411 }
2412 } else if (!Type::equals(src_type, dst_type)) {
2413 assert(!is_cast, "must be reinterpret");
2414 op = gvn().transform(new VectorReinterpretNode(op, src_type, dst_type));
2415 }
2416
2417 const TypeInstPtr* vbox_type_to = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass_to);
2418 Node* vbox = box_vector(op, vbox_type_to, elem_bt_to, num_elem_to);
2419 set_result(vbox);
2420 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem_to * type2aelembytes(elem_bt_to))));
2421 return true;
2422 }
2423
2424 // public static
2425 // <V extends Vector<E>,
2426 // E>
2427 // V insert(Class<? extends V> vectorClass, Class<E> elementType, int vlen,
2428 // V vec, int ix, long val,
2429 // VecInsertOp<V> defaultImpl)
2430 bool LibraryCallKit::inline_vector_insert() {
2431 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
2432 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
2433 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
2434 const TypeInt* idx = gvn().type(argument(4))->isa_int();
2435
2436 if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr || idx == nullptr) {
2437 return false; // dead code
2438 }
2439 if (vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con() || !idx->is_con()) {
2440 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s idx=%s",
2441 NodeClassNames[argument(0)->Opcode()],
2442 NodeClassNames[argument(1)->Opcode()],
2443 NodeClassNames[argument(2)->Opcode()],
2444 NodeClassNames[argument(4)->Opcode()]);
2445 return false; // not enough info for intrinsification
2446 }
2447 if (!is_klass_initialized(vector_klass)) {
2448 log_if_needed(" ** klass argument not initialized");
2449 return false;
2450 }
2451 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2452 if (!elem_type->is_primitive_type()) {
2453 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2454 return false; // should be primitive type
2455 }
2456 BasicType elem_bt = elem_type->basic_type();
2457 int num_elem = vlen->get_con();
2458 if (!arch_supports_vector(Op_VectorInsert, num_elem, elem_bt, VecMaskNotUsed)) {
2459 log_if_needed(" ** not supported: arity=1 op=insert vlen=%d etype=%s ismask=no",
2460 num_elem, type2name(elem_bt));
2461 return false; // not supported
2462 }
2463
2464 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2465 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2466
2467 Node* opd = unbox_vector(argument(3), vbox_type, elem_bt, num_elem);
2468 if (opd == nullptr) {
2469 return false;
2470 }
2471
2472 Node* insert_val = argument(5);
2473 assert(gvn().type(insert_val)->isa_long() != nullptr, "expected to be long");
2474
2475 // Convert insert value back to its appropriate type.
2476 switch (elem_bt) {
2477 case T_BYTE:
2478 insert_val = gvn().transform(new ConvL2INode(insert_val, TypeInt::BYTE));
2479 break;
2480 case T_SHORT:
2481 insert_val = gvn().transform(new ConvL2INode(insert_val, TypeInt::SHORT));
2482 break;
2483 case T_INT:
2484 insert_val = gvn().transform(new ConvL2INode(insert_val));
2485 break;
2486 case T_FLOAT:
2487 insert_val = gvn().transform(new ConvL2INode(insert_val));
2488 insert_val = gvn().transform(new MoveI2FNode(insert_val));
2489 break;
2490 case T_DOUBLE:
2491 insert_val = gvn().transform(new MoveL2DNode(insert_val));
2492 break;
2493 case T_LONG:
2494 // no conversion needed
2495 break;
2496 default: fatal("%s", type2name(elem_bt)); break;
2497 }
2498
2499 Node* operation = gvn().transform(VectorInsertNode::make(opd, insert_val, idx->get_con(), gvn()));
2500
2501 Node* vbox = box_vector(operation, vbox_type, elem_bt, num_elem);
2502 set_result(vbox);
2503 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2504 return true;
2505 }
2506
2507 // public static
2508 // <VM extends VectorPayload,
2509 // E>
2510 // long extract(Class<? extends VM> vClass, Class<E> eClass,
2511 // int length,
2512 // VM vm, int i,
2513 // VecExtractOp<VM> defaultImpl)
2514 bool LibraryCallKit::inline_vector_extract() {
2515 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
2516 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
2517 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
2518 const TypeInt* idx = gvn().type(argument(4))->isa_int();
2519
2520 if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr || idx == nullptr) {
2521 return false; // dead code
2522 }
2523 if (vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
2524 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s",
2525 NodeClassNames[argument(0)->Opcode()],
2526 NodeClassNames[argument(1)->Opcode()],
2527 NodeClassNames[argument(2)->Opcode()]);
2528 return false; // not enough info for intrinsification
2529 }
2530 if (!is_klass_initialized(vector_klass)) {
2531 log_if_needed(" ** klass argument not initialized");
2532 return false;
2533 }
2534 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2535 if (!elem_type->is_primitive_type()) {
2536 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2537 return false; // should be primitive type
2538 }
2539 BasicType elem_bt = elem_type->basic_type();
2540 int num_elem = vlen->get_con();
2541
2542 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2543 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2544
2545 Node* opd = nullptr;
2546
2547 if (is_vector_mask(vbox_klass)) {
2548 // vbox_klass is mask. This is used for VectorMask.laneIsSet(int).
2549
2550 Node* pos = argument(4); // can be variable
2551 if (arch_supports_vector(Op_ExtractUB, num_elem, elem_bt, VecMaskUseAll)) {
2552 // Transform mask to vector with type of boolean and utilize ExtractUB node.
2553 opd = unbox_vector(argument(3), vbox_type, elem_bt, num_elem);
2554 if (opd == nullptr) {
2555 return false;
2556 }
2557 opd = gvn().transform(VectorStoreMaskNode::make(gvn(), opd, elem_bt, num_elem));
2558 opd = gvn().transform(new ExtractUBNode(opd, pos));
2559 opd = gvn().transform(new ConvI2LNode(opd));
2560 } else if (arch_supports_vector(Op_VectorMaskToLong, num_elem, elem_bt, VecMaskUseLoad)) {
2561 opd = unbox_vector(argument(3), vbox_type, elem_bt, num_elem);
2562 if (opd == nullptr) {
2563 return false;
2564 }
2565 // VectorMaskToLongNode requires the input is either a mask or a vector with BOOLEAN type.
2566 if (!Matcher::mask_op_prefers_predicate(Op_VectorMaskToLong, opd->bottom_type()->is_vect())) {
2567 opd = gvn().transform(VectorStoreMaskNode::make(gvn(), opd, elem_bt, num_elem));
2568 }
2569 // ((toLong() >>> pos) & 1L
2570 opd = gvn().transform(new VectorMaskToLongNode(opd, TypeLong::LONG));
2571 opd = gvn().transform(new URShiftLNode(opd, pos));
2572 opd = gvn().transform(new AndLNode(opd, gvn().makecon(TypeLong::ONE)));
2573 } else {
2574 log_if_needed(" ** Rejected mask extraction because architecture does not support it");
2575 return false; // not supported
2576 }
2577 } else {
2578 // vbox_klass is vector. This is used for Vector.lane(int).
2579 if (!idx->is_con()) {
2580 log_if_needed(" ** missing constant: idx=%s", NodeClassNames[argument(4)->Opcode()]);
2581 return false; // not enough info for intrinsification
2582 }
2583
2584 int vopc = ExtractNode::opcode(elem_bt);
2585 if (!arch_supports_vector(vopc, num_elem, elem_bt, VecMaskNotUsed)) {
2586 log_if_needed(" ** not supported: arity=1 op=extract vlen=%d etype=%s ismask=no",
2587 num_elem, type2name(elem_bt));
2588 return false; // not supported
2589 }
2590
2591 opd = unbox_vector(argument(3), vbox_type, elem_bt, num_elem);
2592 if (opd == nullptr) {
2593 return false;
2594 }
2595 ConINode* idx_con = gvn().intcon(idx->get_con())->as_ConI();
2596
2597 opd = gvn().transform(ExtractNode::make(opd, idx_con, elem_bt));
2598 switch (elem_bt) {
2599 case T_BYTE:
2600 case T_SHORT:
2601 case T_INT: {
2602 opd = gvn().transform(new ConvI2LNode(opd));
2603 break;
2604 }
2605 case T_FLOAT: {
2606 opd = gvn().transform(new MoveF2INode(opd));
2607 opd = gvn().transform(new ConvI2LNode(opd));
2608 break;
2609 }
2610 case T_DOUBLE: {
2611 opd = gvn().transform(new MoveD2LNode(opd));
2612 break;
2613 }
2614 case T_LONG: {
2615 // no conversion needed
2616 break;
2617 }
2618 default: fatal("%s", type2name(elem_bt));
2619 }
2620 }
2621 set_result(opd);
2622 return true;
2623 }
2624
2625 static Node* LowerSelectFromTwoVectorOperation(PhaseGVN& phase, Node* index_vec, Node* src1, Node* src2, const TypeVect* vt) {
2626 int num_elem = vt->length();
2627 BasicType elem_bt = vt->element_basic_type();
2628
2629 // Lower selectFrom operation into its constituent operations.
2630 // SelectFromTwoVectorNode =
2631 // (VectorBlend
2632 // (VectorRearrange SRC1 (WRAPED_INDEX AND (VLEN-1))
2633 // (VectorRearrange SRC2 (WRAPED_INDEX AND (VLEN-1))
2634 // MASK)
2635 // Where
2636 // WRAPED_INDEX are computed by wrapping incoming indexes
2637 // to two vector index range [0, VLEN*2) and
2638 // MASK = WRAPED_INDEX < VLEN
2639 //
2640 // IR lowering prevents intrinsification failure and associated argument
2641 // boxing penalties.
2642 //
2643
2644 BasicType shuffle_bt = elem_bt;
2645 if (shuffle_bt == T_FLOAT) {
2646 shuffle_bt = T_INT;
2647 } else if (shuffle_bt == T_DOUBLE) {
2648 shuffle_bt = T_LONG;
2649 }
2650 const TypeVect* st = TypeVect::make(shuffle_bt, num_elem);
2651
2652 // Cast index vector to the corresponding bit type
2653 if (elem_bt != shuffle_bt) {
2654 int cast_vopc = VectorCastNode::opcode(0, elem_bt, true);
2655 index_vec = phase.transform(VectorCastNode::make(cast_vopc, index_vec, shuffle_bt, num_elem));
2656 }
2657
2658 // Wrap indexes into two vector index range [0, VLEN * 2)
2659 Node* two_vect_lane_cnt_m1 = phase.makecon(TypeInteger::make(2 * num_elem - 1, 2 * num_elem - 1, Type::WidenMin, shuffle_bt == T_LONG ? T_LONG : T_INT));
2660 Node* bcast_two_vect_lane_cnt_m1_vec = phase.transform(VectorNode::scalar2vector(two_vect_lane_cnt_m1, num_elem,
2661 shuffle_bt, false));
2662 index_vec = phase.transform(VectorNode::make(Op_AndV, index_vec, bcast_two_vect_lane_cnt_m1_vec, st));
2663
2664 // Compute the blend mask for merging two independently permitted vectors
2665 // using shuffle index in two vector index range [0, VLEN * 2).
2666 BoolTest::mask pred = BoolTest::le;
2667 ConINode* pred_node = phase.makecon(TypeInt::make(pred))->as_ConI();
2668 const TypeVect* vmask_type = TypeVect::makemask(shuffle_bt, num_elem);
2669 Node* lane_cnt_m1 = phase.makecon(TypeInteger::make(num_elem - 1, num_elem - 1, Type::WidenMin, shuffle_bt == T_LONG ? T_LONG : T_INT));
2670 Node* bcast_lane_cnt_m1_vec = phase.transform(VectorNode::scalar2vector(lane_cnt_m1, num_elem, shuffle_bt, false));
2671 Node* mask = phase.transform(new VectorMaskCmpNode(pred, index_vec, bcast_lane_cnt_m1_vec, pred_node, vmask_type));
2672
2673 // Rearrange expects the indexes to lie within single vector index range [0, VLEN).
2674 Node* wrapped_index_vec = phase.transform(VectorNode::make(Op_AndV, index_vec, bcast_lane_cnt_m1_vec, st));
2675
2676 // Load indexes from byte vector and appropriately transform them to target
2677 // specific permutation index format.
2678 if (Matcher::vector_rearrange_requires_load_shuffle(shuffle_bt, num_elem)) {
2679 wrapped_index_vec = phase.transform(new VectorLoadShuffleNode(wrapped_index_vec, st));
2680 }
2681
2682 vmask_type = TypeVect::makemask(elem_bt, num_elem);
2683 mask = phase.transform(new VectorMaskCastNode(mask, vmask_type));
2684
2685 Node* p1 = phase.transform(new VectorRearrangeNode(src1, wrapped_index_vec));
2686 Node* p2 = phase.transform(new VectorRearrangeNode(src2, wrapped_index_vec));
2687
2688 return new VectorBlendNode(p2, p1, mask);
2689 }
2690
2691 // public static
2692 // <V extends Vector<E>,
2693 // E>
2694 // V selectFromTwoVectorOp(Class<? extends V> vClass, Class<E> eClass, int length,
2695 // V v1, V v2, V v3,
2696 // SelectFromTwoVector<V> defaultImpl)
2697 bool LibraryCallKit::inline_vector_select_from_two_vectors() {
2698 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
2699 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
2700 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
2701
2702 if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr || vector_klass->const_oop() == nullptr ||
2703 elem_klass->const_oop() == nullptr ||!vlen->is_con()) {
2704 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s",
2705 NodeClassNames[argument(0)->Opcode()],
2706 NodeClassNames[argument(1)->Opcode()],
2707 NodeClassNames[argument(2)->Opcode()]);
2708 return false; // not enough info for intrinsification
2709 }
2710
2711 if (!is_klass_initialized(vector_klass)) {
2712 log_if_needed(" ** klass argument not initialized");
2713 return false;
2714 }
2715
2716 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2717 if (!elem_type->is_primitive_type()) {
2718 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2719 return false; // should be primitive type
2720 }
2721
2722 int num_elem = vlen->get_con();
2723 if (!is_power_of_2(num_elem)) {
2724 log_if_needed(" ** vlen is not power of two=%d", num_elem);
2725 return false;
2726 }
2727
2728 BasicType elem_bt = elem_type->basic_type();
2729 BasicType index_elem_bt = elem_bt;
2730 if (elem_bt == T_FLOAT) {
2731 index_elem_bt = T_INT;
2732 } else if (elem_bt == T_DOUBLE) {
2733 index_elem_bt = T_LONG;
2734 }
2735
2736 // Check if the platform requires a VectorLoadShuffle node to be generated
2737 bool need_load_shuffle = Matcher::vector_rearrange_requires_load_shuffle(index_elem_bt, num_elem);
2738
2739 bool lowerSelectFromOp = false;
2740 if (!arch_supports_vector(Op_SelectFromTwoVector, num_elem, elem_bt, VecMaskNotUsed)) {
2741 int cast_vopc = VectorCastNode::opcode(-1, elem_bt, true);
2742 if ((elem_bt != index_elem_bt && !arch_supports_vector(cast_vopc, num_elem, index_elem_bt, VecMaskNotUsed)) ||
2743 !arch_supports_vector(Op_VectorMaskCmp, num_elem, index_elem_bt, VecMaskNotUsed) ||
2744 !arch_supports_vector(Op_AndV, num_elem, index_elem_bt, VecMaskNotUsed) ||
2745 !arch_supports_vector(Op_VectorMaskCast, num_elem, elem_bt, VecMaskNotUsed) ||
2746 !arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad) ||
2747 !arch_supports_vector(Op_VectorRearrange, num_elem, elem_bt, VecMaskNotUsed) ||
2748 (need_load_shuffle && !arch_supports_vector(Op_VectorLoadShuffle, num_elem, index_elem_bt, VecMaskNotUsed)) ||
2749 !arch_supports_vector(Op_Replicate, num_elem, index_elem_bt, VecMaskNotUsed)) {
2750 log_if_needed(" ** not supported: opc=%d vlen=%d etype=%s ismask=useload",
2751 Op_SelectFromTwoVector, num_elem, type2name(elem_bt));
2752 return false; // not supported
2753 }
2754 lowerSelectFromOp = true;
2755 }
2756
2757 int cast_vopc = VectorCastNode::opcode(-1, elem_bt, true);
2758 if (!lowerSelectFromOp) {
2759 if (!arch_supports_vector(Op_AndV, num_elem, index_elem_bt, VecMaskNotUsed) ||
2760 !arch_supports_vector(Op_Replicate, num_elem, index_elem_bt, VecMaskNotUsed) ||
2761 (is_floating_point_type(elem_bt) &&
2762 !arch_supports_vector(cast_vopc, num_elem, index_elem_bt, VecMaskNotUsed))) {
2763 log_if_needed(" ** index wrapping not supported: vlen=%d etype=%s" ,
2764 num_elem, type2name(elem_bt));
2765 return false; // not supported
2766 }
2767 }
2768
2769 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2770 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2771
2772 Node* opd1 = unbox_vector(argument(3), vbox_type, elem_bt, num_elem);
2773 if (opd1 == nullptr) {
2774 log_if_needed(" ** unbox failed v1=%s",
2775 NodeClassNames[argument(3)->Opcode()]);
2776 return false;
2777 }
2778 Node* opd2 = unbox_vector(argument(4), vbox_type, elem_bt, num_elem);
2779 if (opd2 == nullptr) {
2780 log_if_needed(" ** unbox failed v2=%s",
2781 NodeClassNames[argument(4)->Opcode()]);
2782 return false;
2783 }
2784 Node* opd3 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
2785 if (opd3 == nullptr) {
2786 log_if_needed(" ** unbox failed v3=%s",
2787 NodeClassNames[argument(5)->Opcode()]);
2788 return false;
2789 }
2790
2791 const TypeVect* vt = TypeVect::make(elem_bt, num_elem);
2792
2793 Node* operation = nullptr;
2794 if (lowerSelectFromOp) {
2795 operation = gvn().transform(LowerSelectFromTwoVectorOperation(gvn(), opd1, opd2, opd3, vt));
2796 } else {
2797 if (index_elem_bt != elem_bt) {
2798 opd1 = gvn().transform(VectorCastNode::make(cast_vopc, opd1, index_elem_bt, num_elem));
2799 }
2800 int indexRangeMask = 2 * num_elem - 1;
2801 Node* wrap_mask = gvn().makecon(TypeInteger::make(indexRangeMask, indexRangeMask, Type::WidenMin, index_elem_bt != T_LONG ? T_INT : index_elem_bt));
2802 Node* wrap_mask_vec = gvn().transform(VectorNode::scalar2vector(wrap_mask, num_elem, index_elem_bt, false));
2803 opd1 = gvn().transform(VectorNode::make(Op_AndV, opd1, wrap_mask_vec, opd1->bottom_type()->is_vect()));
2804 operation = gvn().transform(VectorNode::make(Op_SelectFromTwoVector, opd1, opd2, opd3, vt));
2805 }
2806
2807 // Wrap it up in VectorBox to keep object type information.
2808 Node* vbox = box_vector(operation, vbox_type, elem_bt, num_elem);
2809 set_result(vbox);
2810 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2811 return true;
2812 }
2813
2814 // public static
2815 // <V extends Vector<E>,
2816 // M extends VectorMask<E>,
2817 // E>
2818 // V compressExpandOp(int opr,
2819 // Class<? extends V> vClass, Class<? extends M> mClass, Class<E> eClass,
2820 // int length, V v, M m,
2821 // CompressExpandOperation<V, M> defaultImpl)
2822 bool LibraryCallKit::inline_vector_compress_expand() {
2823 const TypeInt* opr = gvn().type(argument(0))->isa_int();
2824 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
2825 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
2826 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
2827 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
2828
2829 if (opr == nullptr || !opr->is_con() ||
2830 vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
2831 mask_klass == nullptr || mask_klass->const_oop() == nullptr ||
2832 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
2833 vlen == nullptr || !vlen->is_con()) {
2834 log_if_needed(" ** missing constant: opr=%s vclass=%s mclass=%s etype=%s vlen=%s",
2835 NodeClassNames[argument(0)->Opcode()],
2836 NodeClassNames[argument(1)->Opcode()],
2837 NodeClassNames[argument(2)->Opcode()],
2838 NodeClassNames[argument(3)->Opcode()],
2839 NodeClassNames[argument(4)->Opcode()]);
2840 return false; // not enough info for intrinsification
2841 }
2842
2843 if (!is_klass_initialized(vector_klass) || !is_klass_initialized(mask_klass)) {
2844 log_if_needed(" ** klass argument not initialized");
2845 return false;
2846 }
2847
2848 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2849 if (!elem_type->is_primitive_type()) {
2850 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2851 return false; // should be primitive type
2852 }
2853
2854 int num_elem = vlen->get_con();
2855 BasicType elem_bt = elem_type->basic_type();
2856 int opc = VectorSupport::vop2ideal(opr->get_con(), elem_bt);
2857
2858 if (!arch_supports_vector(opc, num_elem, elem_bt, VecMaskUseLoad)) {
2859 log_if_needed(" ** not supported: opc=%d vlen=%d etype=%s ismask=useload",
2860 opc, num_elem, type2name(elem_bt));
2861 return false; // not supported
2862 }
2863
2864 Node* opd1 = nullptr;
2865 const TypeInstPtr* vbox_type = nullptr;
2866 if (opc != Op_CompressM) {
2867 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2868 vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2869 opd1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
2870 if (opd1 == nullptr) {
2871 log_if_needed(" ** unbox failed vector=%s",
2872 NodeClassNames[argument(5)->Opcode()]);
2873 return false;
2874 }
2875 }
2876
2877 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
2878 assert(is_vector_mask(mbox_klass), "argument(6) should be a mask class");
2879 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
2880
2881 Node* mask = unbox_vector(argument(6), mbox_type, elem_bt, num_elem);
2882 if (mask == nullptr) {
2883 log_if_needed(" ** unbox failed mask=%s",
2884 NodeClassNames[argument(6)->Opcode()]);
2885 return false;
2886 }
2887
2888 const TypeVect* vt = TypeVect::make(elem_bt, num_elem, opc == Op_CompressM);
2889 Node* operation = gvn().transform(VectorNode::make(opc, opd1, mask, vt));
2890
2891 // Wrap it up in VectorBox to keep object type information.
2892 const TypeInstPtr* box_type = opc == Op_CompressM ? mbox_type : vbox_type;
2893 Node* vbox = box_vector(operation, box_type, elem_bt, num_elem);
2894 set_result(vbox);
2895 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2896 return true;
2897 }
2898
2899 // public static
2900 // <V extends Vector<E>,
2901 // E,
2902 // S extends VectorSpecies<E>>
2903 // V indexVector(Class<? extends V> vClass, Class<E> eClass,
2904 // int length,
2905 // V v, int step, S s,
2906 // IndexOperation<V, S> defaultImpl)
2907 bool LibraryCallKit::inline_index_vector() {
2908 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
2909 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
2910 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
2911
2912 if (vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
2913 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
2914 vlen == nullptr || !vlen->is_con() ) {
2915 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s",
2916 NodeClassNames[argument(0)->Opcode()],
2917 NodeClassNames[argument(1)->Opcode()],
2918 NodeClassNames[argument(2)->Opcode()]);
2919 return false; // not enough info for intrinsification
2920 }
2921
2922 if (!is_klass_initialized(vector_klass)) {
2923 log_if_needed(" ** klass argument not initialized");
2924 return false;
2925 }
2926
2927 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2928 if (!elem_type->is_primitive_type()) {
2929 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2930 return false; // should be primitive type
2931 }
2932
2933 int num_elem = vlen->get_con();
2934 BasicType elem_bt = elem_type->basic_type();
2935
2936 // Check whether the iota index generation op is supported by the current hardware
2937 if (!arch_supports_vector(Op_VectorLoadConst, num_elem, elem_bt, VecMaskNotUsed)) {
2938 log_if_needed(" ** not supported: vlen=%d etype=%s", num_elem, type2name(elem_bt));
2939 return false; // not supported
2940 }
2941
2942 int mul_op = VectorSupport::vop2ideal(VectorSupport::VECTOR_OP_MUL, elem_bt);
2943 int vmul_op = VectorNode::opcode(mul_op, elem_bt);
2944 bool needs_mul = true;
2945 Node* scale = argument(4);
2946 const TypeInt* scale_type = gvn().type(scale)->isa_int();
2947 // Multiply is not needed if the scale is a constant "1".
2948 if (scale_type && scale_type->is_con() && scale_type->get_con() == 1) {
2949 needs_mul = false;
2950 } else {
2951 // Check whether the vector multiply op is supported by the current hardware
2952 if (!arch_supports_vector(vmul_op, num_elem, elem_bt, VecMaskNotUsed)) {
2953 log_if_needed(" ** not supported: vlen=%d etype=%s", num_elem, type2name(elem_bt));
2954 return false; // not supported
2955 }
2956
2957 // Check whether the scalar cast op is supported by the current hardware
2958 if (is_floating_point_type(elem_bt) || elem_bt == T_LONG) {
2959 int cast_op = elem_bt == T_LONG ? Op_ConvI2L :
2960 elem_bt == T_FLOAT? Op_ConvI2F : Op_ConvI2D;
2961 if (!Matcher::match_rule_supported(cast_op)) {
2962 log_if_needed(" ** Rejected op (%s) because architecture does not support it",
2963 NodeClassNames[cast_op]);
2964 return false; // not supported
2965 }
2966 }
2967 }
2968
2969 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2970 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2971 Node* opd = unbox_vector(argument(3), vbox_type, elem_bt, num_elem);
2972 if (opd == nullptr) {
2973 log_if_needed(" ** unbox failed vector=%s",
2974 NodeClassNames[argument(3)->Opcode()]);
2975 return false;
2976 }
2977
2978 int add_op = VectorSupport::vop2ideal(VectorSupport::VECTOR_OP_ADD, elem_bt);
2979 int vadd_op = VectorNode::opcode(add_op, elem_bt);
2980 bool needs_add = true;
2981 // The addition is not needed if all the element values of "opd" are zero
2982 if (VectorNode::is_all_zeros_vector(opd)) {
2983 needs_add = false;
2984 } else {
2985 // Check whether the vector addition op is supported by the current hardware
2986 if (!arch_supports_vector(vadd_op, num_elem, elem_bt, VecMaskNotUsed)) {
2987 log_if_needed(" ** not supported: vlen=%d etype=%s", num_elem, type2name(elem_bt));
2988 return false; // not supported
2989 }
2990 }
2991
2992 // Compute the iota indice vector
2993 const TypeVect* vt = TypeVect::make(elem_bt, num_elem);
2994 Node* index = gvn().transform(new VectorLoadConstNode(gvn().makecon(TypeInt::ZERO), vt));
2995
2996 // Broadcast the "scale" to a vector, and multiply the "scale" with iota indice vector.
2997 if (needs_mul) {
2998 switch (elem_bt) {
2999 case T_BOOLEAN: // fall-through
3000 case T_BYTE: // fall-through
3001 case T_SHORT: // fall-through
3002 case T_CHAR: // fall-through
3003 case T_INT: {
3004 // no conversion needed
3005 break;
3006 }
3007 case T_LONG: {
3008 scale = gvn().transform(new ConvI2LNode(scale));
3009 break;
3010 }
3011 case T_FLOAT: {
3012 scale = gvn().transform(new ConvI2FNode(scale));
3013 break;
3014 }
3015 case T_DOUBLE: {
3016 scale = gvn().transform(new ConvI2DNode(scale));
3017 break;
3018 }
3019 default: fatal("%s", type2name(elem_bt));
3020 }
3021 scale = gvn().transform(VectorNode::scalar2vector(scale, num_elem, elem_bt));
3022 index = gvn().transform(VectorNode::make(vmul_op, index, scale, vt));
3023 }
3024
3025 // Add "opd" if addition is needed.
3026 if (needs_add) {
3027 index = gvn().transform(VectorNode::make(vadd_op, opd, index, vt));
3028 }
3029 Node* vbox = box_vector(index, vbox_type, elem_bt, num_elem);
3030 set_result(vbox);
3031 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
3032 return true;
3033 }
3034
3035 // public static
3036 // <E,
3037 // M extends VectorMask<E>>
3038 // M indexPartiallyInUpperRange(Class<? extends M> mClass, Class<E> eClass, int length,
3039 // long offset, long limit,
3040 // IndexPartiallyInUpperRangeOperation<E, M> defaultImpl)
3041 bool LibraryCallKit::inline_index_partially_in_upper_range() {
3042 const TypeInstPtr* mask_klass = gvn().type(argument(0))->isa_instptr();
3043 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
3044 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
3045
3046 if (mask_klass == nullptr || mask_klass->const_oop() == nullptr ||
3047 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
3048 vlen == nullptr || !vlen->is_con()) {
3049 log_if_needed(" ** missing constant: mclass=%s etype=%s vlen=%s",
3050 NodeClassNames[argument(0)->Opcode()],
3051 NodeClassNames[argument(1)->Opcode()],
3052 NodeClassNames[argument(2)->Opcode()]);
3053 return false; // not enough info for intrinsification
3054 }
3055
3056 if (!is_klass_initialized(mask_klass)) {
3057 log_if_needed(" ** klass argument not initialized");
3058 return false;
3059 }
3060
3061 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
3062 if (!elem_type->is_primitive_type()) {
3063 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
3064 return false; // should be primitive type
3065 }
3066
3067 int num_elem = vlen->get_con();
3068 BasicType elem_bt = elem_type->basic_type();
3069
3070 // Check whether the necessary ops are supported by current hardware.
3071 bool supports_mask_gen = arch_supports_vector(Op_VectorMaskGen, num_elem, elem_bt, VecMaskUseStore);
3072 if (!supports_mask_gen) {
3073 if (!arch_supports_vector(Op_VectorLoadConst, num_elem, elem_bt, VecMaskNotUsed) ||
3074 !arch_supports_vector(Op_Replicate, num_elem, elem_bt, VecMaskNotUsed) ||
3075 !arch_supports_vector(Op_VectorMaskCmp, num_elem, elem_bt, VecMaskUseStore)) {
3076 log_if_needed(" ** not supported: vlen=%d etype=%s", num_elem, type2name(elem_bt));
3077 return false; // not supported
3078 }
3079
3080 // Check whether the scalar cast operation is supported by current hardware.
3081 if (elem_bt != T_LONG) {
3082 int cast_op = is_integral_type(elem_bt) ? Op_ConvL2I
3083 : (elem_bt == T_FLOAT ? Op_ConvL2F : Op_ConvL2D);
3084 if (!Matcher::match_rule_supported(cast_op)) {
3085 log_if_needed(" ** Rejected op (%s) because architecture does not support it",
3086 NodeClassNames[cast_op]);
3087 return false; // not supported
3088 }
3089 }
3090 }
3091
3092 Node* offset = argument(3);
3093 Node* limit = argument(5);
3094 if (offset == nullptr || limit == nullptr) {
3095 log_if_needed(" ** offset or limit argument is null");
3096 return false; // not supported
3097 }
3098
3099 ciKlass* box_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
3100 assert(is_vector_mask(box_klass), "argument(0) should be a mask class");
3101 const TypeInstPtr* box_type = TypeInstPtr::make_exact(TypePtr::NotNull, box_klass);
3102
3103 // We assume "offset > 0 && limit >= offset && limit - offset < num_elem".
3104 // So directly get indexLimit with "indexLimit = limit - offset".
3105 Node* indexLimit = gvn().transform(new SubLNode(limit, offset));
3106 Node* mask = nullptr;
3107 if (supports_mask_gen) {
3108 mask = gvn().transform(VectorMaskGenNode::make(indexLimit, elem_bt, num_elem));
3109 } else {
3110 // Generate the vector mask based on "mask = iota < indexLimit".
3111 // Broadcast "indexLimit" to a vector.
3112 switch (elem_bt) {
3113 case T_BOOLEAN: // fall-through
3114 case T_BYTE: // fall-through
3115 case T_SHORT: // fall-through
3116 case T_CHAR: // fall-through
3117 case T_INT: {
3118 indexLimit = gvn().transform(new ConvL2INode(indexLimit));
3119 break;
3120 }
3121 case T_DOUBLE: {
3122 indexLimit = gvn().transform(new ConvL2DNode(indexLimit));
3123 break;
3124 }
3125 case T_FLOAT: {
3126 indexLimit = gvn().transform(new ConvL2FNode(indexLimit));
3127 break;
3128 }
3129 case T_LONG: {
3130 // no conversion needed
3131 break;
3132 }
3133 default: fatal("%s", type2name(elem_bt));
3134 }
3135 indexLimit = gvn().transform(VectorNode::scalar2vector(indexLimit, num_elem, elem_bt));
3136
3137 // Load the "iota" vector.
3138 const TypeVect* vt = TypeVect::make(elem_bt, num_elem);
3139 Node* iota = gvn().transform(new VectorLoadConstNode(gvn().makecon(TypeInt::ZERO), vt));
3140
3141 // Compute the vector mask with "mask = iota < indexLimit".
3142 ConINode* pred_node = (ConINode*)gvn().makecon(TypeInt::make(BoolTest::lt));
3143 const TypeVect* vmask_type = TypeVect::makemask(elem_bt, num_elem);
3144 mask = gvn().transform(new VectorMaskCmpNode(BoolTest::lt, iota, indexLimit, pred_node, vmask_type));
3145 }
3146 Node* vbox = box_vector(mask, box_type, elem_bt, num_elem);
3147 set_result(vbox);
3148 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
3149 return true;
3150 }
3151
3152 #undef non_product_log_if_needed
3153 #undef log_if_needed