1 /*
2 * Copyright (c) 2020, 2025, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "ci/ciSymbols.hpp"
26 #include "classfile/vmSymbols.hpp"
27 #include "opto/library_call.hpp"
28 #include "opto/runtime.hpp"
29 #include "opto/vectornode.hpp"
30 #include "prims/vectorSupport.hpp"
31 #include "runtime/stubRoutines.hpp"
32
33 #ifdef ASSERT
34 static bool is_vector(ciKlass* klass) {
35 return klass->is_subclass_of(ciEnv::current()->vector_VectorPayload_klass());
36 }
37
38 static bool check_vbox(const TypeInstPtr* vbox_type) {
39 assert(vbox_type->klass_is_exact(), "");
40
41 ciInstanceKlass* ik = vbox_type->instance_klass();
42 assert(is_vector(ik), "not a vector");
43
44 ciField* fd1 = ik->get_field_by_name(ciSymbols::ETYPE_name(), ciSymbols::class_signature(), /* is_static */ true);
45 assert(fd1 != nullptr, "element type info is missing");
46
47 ciConstant val1 = fd1->constant_value();
48 BasicType elem_bt = val1.as_object()->as_instance()->java_mirror_type()->basic_type();
49 assert(is_java_primitive(elem_bt), "element type info is missing");
50
51 ciField* fd2 = ik->get_field_by_name(ciSymbols::VLENGTH_name(), ciSymbols::int_signature(), /* is_static */ true);
52 assert(fd2 != nullptr, "vector length info is missing");
53
54 ciConstant val2 = fd2->constant_value();
55 assert(val2.as_int() > 0, "vector length info is missing");
56
57 return true;
58 }
59 #endif
60
61 #define log_if_needed(...) \
62 if (C->print_intrinsics()) { \
63 tty->print_cr(__VA_ARGS__); \
64 }
65
66 #ifndef PRODUCT
67 #define non_product_log_if_needed(...) log_if_needed(__VA_ARGS__)
68 #else
69 #define non_product_log_if_needed(...)
70 #endif
71
72 static bool is_vector_mask(ciKlass* klass) {
73 return klass->is_subclass_of(ciEnv::current()->vector_VectorMask_klass());
74 }
75
76 bool LibraryCallKit::arch_supports_vector_rotate(int opc, int num_elem, BasicType elem_bt,
77 VectorMaskUseType mask_use_type, bool has_scalar_args) {
78 bool is_supported = true;
79
80 // has_scalar_args flag is true only for non-constant scalar shift count,
81 // since in this case shift needs to be broadcasted.
82 if (!Matcher::match_rule_supported_vector(opc, num_elem, elem_bt) ||
83 (has_scalar_args && !arch_supports_vector(Op_Replicate, num_elem, elem_bt, VecMaskNotUsed))) {
84 is_supported = false;
85 }
86
87 if (is_supported) {
88 // Check if mask unboxing is supported, this is a two step process which first loads the contents
89 // of boolean array into vector followed by either lane expansion to match the lane size of masked
90 // vector operation or populate the predicate register.
91 if ((mask_use_type & VecMaskUseLoad) != 0) {
92 if (!Matcher::match_rule_supported_vector(Op_VectorLoadMask, num_elem, elem_bt) ||
93 !Matcher::match_rule_supported_vector(Op_LoadVector, num_elem, T_BOOLEAN)) {
94 non_product_log_if_needed(" ** Rejected vector mask loading (%s,%s,%d) because architecture does not support it",
95 NodeClassNames[Op_VectorLoadMask], type2name(elem_bt), num_elem);
96 return false;
97 }
98 }
99
100 if ((mask_use_type & VecMaskUsePred) != 0) {
101 if (!Matcher::has_predicated_vectors() ||
102 !Matcher::match_rule_supported_vector_masked(opc, num_elem, elem_bt)) {
103 non_product_log_if_needed("Rejected vector mask predicate using (%s,%s,%d) because architecture does not support it",
104 NodeClassNames[opc], type2name(elem_bt), num_elem);
105 return false;
106 }
107 }
108 }
109
110 int lshiftopc, rshiftopc;
111 switch(elem_bt) {
112 case T_BYTE:
113 lshiftopc = Op_LShiftI;
114 rshiftopc = Op_URShiftB;
115 break;
116 case T_SHORT:
117 lshiftopc = Op_LShiftI;
118 rshiftopc = Op_URShiftS;
119 break;
120 case T_INT:
121 lshiftopc = Op_LShiftI;
122 rshiftopc = Op_URShiftI;
123 break;
124 case T_LONG:
125 lshiftopc = Op_LShiftL;
126 rshiftopc = Op_URShiftL;
127 break;
128 default: fatal("Unexpected type: %s", type2name(elem_bt));
129 }
130 int lshiftvopc = VectorNode::opcode(lshiftopc, elem_bt);
131 int rshiftvopc = VectorNode::opcode(rshiftopc, elem_bt);
132 if (!is_supported &&
133 arch_supports_vector(lshiftvopc, num_elem, elem_bt, VecMaskNotUsed, has_scalar_args) &&
134 arch_supports_vector(rshiftvopc, num_elem, elem_bt, VecMaskNotUsed, has_scalar_args) &&
135 arch_supports_vector(Op_OrV, num_elem, elem_bt, VecMaskNotUsed)) {
136 is_supported = true;
137 }
138 return is_supported;
139 }
140
141 Node* GraphKit::box_vector(Node* vector, const TypeInstPtr* vbox_type, BasicType elem_bt, int num_elem, bool deoptimize_on_exception) {
142 assert(EnableVectorSupport, "");
143
144 PreserveReexecuteState preexecs(this);
145 jvms()->set_should_reexecute(true);
146
147 VectorBoxAllocateNode* alloc = new VectorBoxAllocateNode(C, vbox_type);
148 set_edges_for_java_call(alloc, /*must_throw=*/false, /*separate_io_proj=*/true);
149 make_slow_call_ex(alloc, env()->Throwable_klass(), /*separate_io_proj=*/true, deoptimize_on_exception);
150 set_i_o(gvn().transform( new ProjNode(alloc, TypeFunc::I_O) ));
151 set_all_memory(gvn().transform( new ProjNode(alloc, TypeFunc::Memory) ));
152 Node* ret = gvn().transform(new ProjNode(alloc, TypeFunc::Parms));
153
154 assert(check_vbox(vbox_type), "");
155 const TypeVect* vt = TypeVect::make(elem_bt, num_elem, is_vector_mask(vbox_type->instance_klass()));
156 VectorBoxNode* vbox = new VectorBoxNode(C, ret, vector, vbox_type, vt);
157 return gvn().transform(vbox);
158 }
159
160 Node* GraphKit::unbox_vector(Node* v, const TypeInstPtr* vbox_type, BasicType elem_bt, int num_elem) {
161 assert(EnableVectorSupport, "");
162 const TypeInstPtr* vbox_type_v = gvn().type(v)->isa_instptr();
163 if (vbox_type_v == nullptr || vbox_type->instance_klass() != vbox_type_v->instance_klass()) {
164 return nullptr; // arguments don't agree on vector shapes
165 }
166 if (vbox_type_v->maybe_null()) {
167 return nullptr; // no nulls are allowed
168 }
169 assert(check_vbox(vbox_type), "");
170 const TypeVect* vt = TypeVect::make(elem_bt, num_elem, is_vector_mask(vbox_type->instance_klass()));
171 Node* unbox = gvn().transform(new VectorUnboxNode(C, vt, v, merged_memory()));
172 return unbox;
173 }
174
175 Node* GraphKit::vector_shift_count(Node* cnt, int shift_op, BasicType bt, int num_elem) {
176 assert(bt == T_INT || bt == T_LONG || bt == T_SHORT || bt == T_BYTE, "byte, short, long and int are supported");
177 juint mask = (type2aelembytes(bt) * BitsPerByte - 1);
178 Node* nmask = gvn().transform(ConNode::make(TypeInt::make(mask)));
179 Node* mcnt = gvn().transform(new AndINode(cnt, nmask));
180 return gvn().transform(VectorNode::shift_count(shift_op, mcnt, num_elem, bt));
181 }
182
183 bool LibraryCallKit::arch_supports_vector(int sopc, int num_elem, BasicType type, VectorMaskUseType mask_use_type, bool has_scalar_args) {
184 // Check that the operation is valid.
185 if (sopc <= 0) {
186 non_product_log_if_needed(" ** Rejected intrinsification because no valid vector op could be extracted");
187 return false;
188 }
189
190 if (VectorNode::is_vector_rotate(sopc)) {
191 if(!arch_supports_vector_rotate(sopc, num_elem, type, mask_use_type, has_scalar_args)) {
192 non_product_log_if_needed(" ** Rejected vector op (%s,%s,%d) because architecture does not support variable vector shifts",
193 NodeClassNames[sopc], type2name(type), num_elem);
194 return false;
195 }
196 } else if (VectorNode::is_vector_integral_negate(sopc)) {
197 if (!VectorNode::is_vector_integral_negate_supported(sopc, num_elem, type, false)) {
198 non_product_log_if_needed(" ** Rejected vector op (%s,%s,%d) because architecture does not support integral vector negate",
199 NodeClassNames[sopc], type2name(type), num_elem);
200 return false;
201 }
202 } else {
203 // Check that architecture supports this op-size-type combination.
204 if (!Matcher::match_rule_supported_vector(sopc, num_elem, type)) {
205 non_product_log_if_needed(" ** Rejected vector op (%s,%s,%d) because architecture does not support it",
206 NodeClassNames[sopc], type2name(type), num_elem);
207 return false;
208 } else {
209 assert(Matcher::match_rule_supported(sopc), "must be supported");
210 }
211 }
212
213 if (num_elem == 1) {
214 if (mask_use_type != VecMaskNotUsed) {
215 non_product_log_if_needed(" ** Rejected vector mask op (%s,%s,%d) because architecture does not support it",
216 NodeClassNames[sopc], type2name(type), num_elem);
217 return false;
218 }
219
220 if (sopc != 0) {
221 if (sopc != Op_LoadVector && sopc != Op_StoreVector) {
222 non_product_log_if_needed(" ** Not a svml call or load/store vector op (%s,%s,%d)",
223 NodeClassNames[sopc], type2name(type), num_elem);
224 return false;
225 }
226 }
227 }
228
229 if (!has_scalar_args && VectorNode::is_vector_shift(sopc) &&
230 Matcher::supports_vector_variable_shifts() == false) {
231 log_if_needed(" ** Rejected vector op (%s,%s,%d) because architecture does not support variable vector shifts",
232 NodeClassNames[sopc], type2name(type), num_elem);
233 return false;
234 }
235
236 // Check if mask unboxing is supported, this is a two step process which first loads the contents
237 // of boolean array into vector followed by either lane expansion to match the lane size of masked
238 // vector operation or populate the predicate register.
239 if ((mask_use_type & VecMaskUseLoad) != 0) {
240 if (!Matcher::match_rule_supported_vector(Op_VectorLoadMask, num_elem, type) ||
241 !Matcher::match_rule_supported_vector(Op_LoadVector, num_elem, T_BOOLEAN)) {
242 non_product_log_if_needed(" ** Rejected vector mask loading (%s,%s,%d) because architecture does not support it",
243 NodeClassNames[Op_VectorLoadMask], type2name(type), num_elem);
244 return false;
245 }
246 }
247
248 // Check if mask boxing is supported, this is a two step process which first stores the contents
249 // of mask vector / predicate register into a boolean vector followed by vector store operation to
250 // transfer the contents to underlined storage of mask boxes which is a boolean array.
251 if ((mask_use_type & VecMaskUseStore) != 0) {
252 if (!Matcher::match_rule_supported_vector(Op_VectorStoreMask, num_elem, type) ||
253 !Matcher::match_rule_supported_vector(Op_StoreVector, num_elem, T_BOOLEAN)) {
254 non_product_log_if_needed("Rejected vector mask storing (%s,%s,%d) because architecture does not support it",
255 NodeClassNames[Op_VectorStoreMask], type2name(type), num_elem);
256 return false;
257 }
258 }
259
260 if ((mask_use_type & VecMaskUsePred) != 0) {
261 bool is_supported = false;
262 if (Matcher::has_predicated_vectors()) {
263 if (VectorNode::is_vector_integral_negate(sopc)) {
264 is_supported = VectorNode::is_vector_integral_negate_supported(sopc, num_elem, type, true);
265 } else {
266 is_supported = Matcher::match_rule_supported_vector_masked(sopc, num_elem, type);
267 }
268 }
269 is_supported |= Matcher::supports_vector_predicate_op_emulation(sopc, num_elem, type);
270
271 if (!is_supported) {
272 non_product_log_if_needed("Rejected vector mask predicate using (%s,%s,%d) because architecture does not support it",
273 NodeClassNames[sopc], type2name(type), num_elem);
274 return false;
275 }
276 }
277
278 return true;
279 }
280
281 static bool is_klass_initialized(const TypeInstPtr* vec_klass) {
282 if (vec_klass->const_oop() == nullptr) {
283 return false; // uninitialized or some kind of unsafe access
284 }
285 assert(vec_klass->const_oop()->as_instance()->java_lang_Class_klass() != nullptr, "klass instance expected");
286 ciInstanceKlass* klass = vec_klass->const_oop()->as_instance()->java_lang_Class_klass()->as_instance_klass();
287 return klass->is_initialized();
288 }
289
290 // public static
291 // <V extends Vector<E>,
292 // M extends VectorMask<E>,
293 // E>
294 // V unaryOp(int oprId, Class<? extends V> vmClass, Class<? extends M> maskClass, Class<E> elementType,
295 // int length, V v, M m,
296 // UnaryOperation<V, M> defaultImpl)
297 //
298 // public static
299 // <V,
300 // M extends VectorMask<E>,
301 // E>
302 // V binaryOp(int oprId, Class<? extends V> vmClass, Class<? extends M> maskClass, Class<E> elementType,
303 // int length, V v1, V v2, M m,
304 // BinaryOperation<V, M> defaultImpl)
305 //
306 // public static
307 // <V extends Vector<E>,
308 // M extends VectorMask<E>,
309 // E>
310 // V ternaryOp(int oprId, Class<? extends V> vmClass, Class<? extends M> maskClass, Class<E> elementType,
311 // int length, V v1, V v2, V v3, M m,
312 // TernaryOperation<V, M> defaultImpl)
313 //
314 bool LibraryCallKit::inline_vector_nary_operation(int n) {
315 const TypeInt* opr = gvn().type(argument(0))->isa_int();
316 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
317 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
318 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
319 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
320
321 if (opr == nullptr || !opr->is_con() ||
322 vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
323 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
324 vlen == nullptr || !vlen->is_con()) {
325 log_if_needed(" ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
326 NodeClassNames[argument(0)->Opcode()],
327 NodeClassNames[argument(1)->Opcode()],
328 NodeClassNames[argument(3)->Opcode()],
329 NodeClassNames[argument(4)->Opcode()]);
330 return false; // not enough info for intrinsification
331 }
332
333 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
334 if (!elem_type->is_primitive_type()) {
335 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
336 return false; // should be primitive type
337 }
338 if (!is_klass_initialized(vector_klass)) {
339 log_if_needed(" ** klass argument not initialized");
340 return false;
341 }
342
343 // "argument(n + 5)" should be the mask object. We assume it is "null" when no mask
344 // is used to control this operation.
345 const Type* vmask_type = gvn().type(argument(n + 5));
346 bool is_masked_op = vmask_type != TypePtr::NULL_PTR;
347 if (is_masked_op) {
348 if (mask_klass == nullptr || mask_klass->const_oop() == nullptr) {
349 log_if_needed(" ** missing constant: maskclass=%s", NodeClassNames[argument(2)->Opcode()]);
350 return false; // not enough info for intrinsification
351 }
352
353 if (!is_klass_initialized(mask_klass)) {
354 log_if_needed(" ** mask klass argument not initialized");
355 return false;
356 }
357
358 if (vmask_type->maybe_null()) {
359 log_if_needed(" ** null mask values are not allowed for masked op");
360 return false;
361 }
362 }
363
364 BasicType elem_bt = elem_type->basic_type();
365 bool has_scalar_op = VectorSupport::has_scalar_op(opr->get_con());
366 bool is_unsigned = VectorSupport::is_unsigned_op(opr->get_con());
367
368 int num_elem = vlen->get_con();
369 int opc = VectorSupport::vop2ideal(opr->get_con(), elem_bt);
370 int sopc = has_scalar_op ? VectorNode::opcode(opc, elem_bt) : opc;
371 if (sopc == 0 || num_elem == 1) {
372 log_if_needed(" ** operation not supported: arity=%d opc=%s[%d] vlen=%d etype=%s",
373 n, NodeClassNames[opc], opc, num_elem, type2name(elem_bt));
374 return false; // operation not supported
375 }
376 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
377 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
378
379 if (is_vector_mask(vbox_klass)) {
380 assert(!is_masked_op, "mask operations do not need mask to control");
381 }
382
383 // When using mask, mask use type needs to be VecMaskUseLoad.
384 VectorMaskUseType mask_use_type = is_vector_mask(vbox_klass) ? VecMaskUseAll
385 : is_masked_op ? VecMaskUseLoad : VecMaskNotUsed;
386 if (!arch_supports_vector(sopc, num_elem, elem_bt, mask_use_type)) {
387 log_if_needed(" ** not supported: arity=%d opc=%d vlen=%d etype=%s ismask=%d is_masked_op=%d",
388 n, sopc, num_elem, type2name(elem_bt),
389 is_vector_mask(vbox_klass) ? 1 : 0, is_masked_op ? 1 : 0);
390 return false; // not supported
391 }
392
393 // Return true if current platform has implemented the masked operation with predicate feature.
394 bool use_predicate = is_masked_op && arch_supports_vector(sopc, num_elem, elem_bt, VecMaskUsePred);
395 if (is_masked_op && !use_predicate && !arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad)) {
396 log_if_needed(" ** not supported: arity=%d opc=%d vlen=%d etype=%s ismask=0 is_masked_op=1",
397 n, sopc, num_elem, type2name(elem_bt));
398 return false;
399 }
400
401 Node* opd1 = nullptr; Node* opd2 = nullptr; Node* opd3 = nullptr;
402 switch (n) {
403 case 3: {
404 opd3 = unbox_vector(argument(7), vbox_type, elem_bt, num_elem);
405 if (opd3 == nullptr) {
406 log_if_needed(" ** unbox failed v3=%s",
407 NodeClassNames[argument(7)->Opcode()]);
408 return false;
409 }
410 // fall-through
411 }
412 case 2: {
413 opd2 = unbox_vector(argument(6), vbox_type, elem_bt, num_elem);
414 if (opd2 == nullptr) {
415 log_if_needed(" ** unbox failed v2=%s",
416 NodeClassNames[argument(6)->Opcode()]);
417 return false;
418 }
419 // fall-through
420 }
421 case 1: {
422 opd1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
423 if (opd1 == nullptr) {
424 log_if_needed(" ** unbox failed v1=%s",
425 NodeClassNames[argument(5)->Opcode()]);
426 return false;
427 }
428 break;
429 }
430 default: fatal("unsupported arity: %d", n);
431 }
432
433 Node* mask = nullptr;
434 if (is_masked_op) {
435 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
436 assert(is_vector_mask(mbox_klass), "argument(2) should be a mask class");
437 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
438 mask = unbox_vector(argument(n + 5), mbox_type, elem_bt, num_elem);
439 if (mask == nullptr) {
440 log_if_needed(" ** unbox failed mask=%s",
441 NodeClassNames[argument(n + 5)->Opcode()]);
442 return false;
443 }
444 }
445
446 Node* operation = nullptr;
447 const TypeVect* vt = TypeVect::make(elem_bt, num_elem, is_vector_mask(vbox_klass));
448 switch (n) {
449 case 1:
450 case 2: {
451 operation = VectorNode::make(sopc, opd1, opd2, vt, is_vector_mask(vbox_klass), VectorNode::is_shift_opcode(opc), is_unsigned);
452 break;
453 }
454 case 3: {
455 operation = VectorNode::make(sopc, opd1, opd2, opd3, vt);
456 break;
457 }
458 default: fatal("unsupported arity: %d", n);
459 }
460
461 if (is_masked_op && mask != nullptr) {
462 if (use_predicate) {
463 operation->add_req(mask);
464 operation->add_flag(Node::Flag_is_predicated_vector);
465 } else {
466 operation->add_flag(Node::Flag_is_predicated_using_blend);
467 operation = gvn().transform(operation);
468 operation = new VectorBlendNode(opd1, operation, mask);
469 }
470 }
471 operation = gvn().transform(operation);
472
473 // Wrap it up in VectorBox to keep object type information.
474 Node* vbox = box_vector(operation, vbox_type, elem_bt, num_elem);
475 set_result(vbox);
476 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
477 return true;
478 }
479
480 // public static
481 // <V extends Vector<E>, E>
482 // V libraryUnaryOp(long address, Class<? extends V> vClass, Class<E> elementType, int length, String debugName,
483 // V v,
484 // UnaryOperation<V, ?> defaultImpl)
485 //
486 // public static
487 // <V extends VectorPayload, E>
488 // V libraryBinaryOp(long address, Class<? extends V> vClass, Class<E> elementType, int length, String debugName,
489 // V v1, V v2,
490 // BinaryOperation<V, ?> defaultImpl)
491 bool LibraryCallKit::inline_vector_call(int arity) {
492 assert(Matcher::supports_vector_calling_convention(), "required");
493
494 const TypeLong* entry = gvn().type(argument(0))->isa_long();
495 const TypeInstPtr* vector_klass = gvn().type(argument(2))->isa_instptr();
496 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
497 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
498 const TypeInstPtr* debug_name_oop = gvn().type(argument(5))->isa_instptr();
499
500 if (entry == nullptr || !entry->is_con() ||
501 vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
502 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
503 vlen == nullptr || !vlen->is_con() ||
504 debug_name_oop == nullptr || debug_name_oop->const_oop() == nullptr) {
505 log_if_needed(" ** missing constant: opr=%s vclass=%s etype=%s vlen=%s debug_name=%s",
506 NodeClassNames[argument(0)->Opcode()],
507 NodeClassNames[argument(2)->Opcode()],
508 NodeClassNames[argument(3)->Opcode()],
509 NodeClassNames[argument(4)->Opcode()],
510 NodeClassNames[argument(5)->Opcode()]);
511 return false; // not enough info for intrinsification
512 }
513
514 if (entry->get_con() == 0) {
515 log_if_needed(" ** missing entry point");
516 return false;
517 }
518
519 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
520 if (!elem_type->is_primitive_type()) {
521 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
522 return false; // should be primitive type
523 }
524 if (!is_klass_initialized(vector_klass)) {
525 log_if_needed(" ** klass argument not initialized");
526 return false;
527 }
528
529 BasicType elem_bt = elem_type->basic_type();
530 int num_elem = vlen->get_con();
531 if (!Matcher::vector_size_supported(elem_bt, num_elem)) {
532 log_if_needed(" ** vector size (vlen=%d, etype=%s) is not supported",
533 num_elem, type2name(elem_bt));
534 return false;
535 }
536
537 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
538 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
539
540 Node* opd1 = unbox_vector(argument(6), vbox_type, elem_bt, num_elem);
541 if (opd1 == nullptr) {
542 log_if_needed(" ** unbox failed v1=%s", NodeClassNames[argument(6)->Opcode()]);
543 return false;
544 }
545
546 Node* opd2 = nullptr;
547 if (arity > 1) {
548 opd2 = unbox_vector(argument(7), vbox_type, elem_bt, num_elem);
549 if (opd2 == nullptr) {
550 log_if_needed(" ** unbox failed v2=%s", NodeClassNames[argument(7)->Opcode()]);
551 return false;
552 }
553 }
554 assert(arity == 1 || arity == 2, "arity %d not supported", arity);
555 const TypeVect* vt = TypeVect::make(elem_bt, num_elem);
556 const TypeFunc* call_type = OptoRuntime::Math_Vector_Vector_Type(arity, vt, vt);
557 address entry_addr = (address)entry->get_con();
558
559 const char* debug_name = "<unknown>";
560 if (!debug_name_oop->const_oop()->is_null_object()) {
561 size_t buflen = 100;
562 char* buf = NEW_ARENA_ARRAY(C->comp_arena(), char, buflen);
563 debug_name = debug_name_oop->const_oop()->as_instance()->java_lang_String_str(buf, buflen);
564 }
565 Node* vcall = make_runtime_call(RC_VECTOR,
566 call_type,
567 entry_addr,
568 debug_name,
569 TypePtr::BOTTOM,
570 opd1,
571 opd2);
572
573 vcall = gvn().transform(new ProjNode(gvn().transform(vcall), TypeFunc::Parms));
574
575 // Wrap it up in VectorBox to keep object type information.
576 Node* vbox = box_vector(vcall, vbox_type, elem_bt, num_elem);
577 set_result(vbox);
578 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
579 return true;
580 }
581
582 // <E, M>
583 // long maskReductionCoerced(int oper, Class<? extends M> maskClass, Class<?> elemClass,
584 // int length, M m, VectorMaskOp<M> defaultImpl)
585 bool LibraryCallKit::inline_vector_mask_operation() {
586 const TypeInt* oper = gvn().type(argument(0))->isa_int();
587 const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
588 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
589 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
590 Node* mask = argument(4);
591
592 if (mask_klass == nullptr || mask_klass->const_oop() == nullptr ||
593 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
594 vlen == nullptr || !vlen->is_con() ||
595 oper == nullptr || !oper->is_con() ||
596 mask->is_top()) {
597 return false; // dead code
598 }
599
600 if (!is_klass_initialized(mask_klass)) {
601 log_if_needed(" ** klass argument not initialized");
602 return false;
603 }
604
605 int num_elem = vlen->get_con();
606 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
607 BasicType elem_bt = elem_type->basic_type();
608
609 int mopc = VectorSupport::vop2ideal(oper->get_con(), elem_bt);
610 if (!arch_supports_vector(mopc, num_elem, elem_bt, VecMaskUseLoad)) {
611 log_if_needed(" ** not supported: arity=1 op=cast#%d/3 vlen2=%d etype2=%s",
612 mopc, num_elem, type2name(elem_bt));
613 return false; // not supported
614 }
615
616 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
617 const TypeInstPtr* mask_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
618 Node* mask_vec = unbox_vector(mask, mask_box_type, elem_bt, num_elem);
619 if (mask_vec == nullptr) {
620 log_if_needed(" ** unbox failed mask=%s",
621 NodeClassNames[argument(4)->Opcode()]);
622 return false;
623 }
624
625 if (mask_vec->bottom_type()->isa_vectmask() == nullptr) {
626 mask_vec = gvn().transform(VectorStoreMaskNode::make(gvn(), mask_vec, elem_bt, num_elem));
627 }
628 const Type* maskoper_ty = mopc == Op_VectorMaskToLong ? (const Type*)TypeLong::LONG : (const Type*)TypeInt::INT;
629 Node* maskoper = gvn().transform(VectorMaskOpNode::make(mask_vec, maskoper_ty, mopc));
630 if (mopc != Op_VectorMaskToLong) {
631 maskoper = ConvI2L(maskoper);
632 }
633 set_result(maskoper);
634
635 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
636 return true;
637 }
638
639 // public static
640 // <M,
641 // S extends VectorSpecies<E>,
642 // E>
643 // M fromBitsCoerced(Class<? extends M> vmClass, Class<E> elementType, int length,
644 // long bits, int mode, S s,
645 // BroadcastOperation<M, E, S> defaultImpl)
646 bool LibraryCallKit::inline_vector_frombits_coerced() {
647 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
648 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
649 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
650 const TypeLong* bits_type = gvn().type(argument(3))->isa_long();
651 // Mode argument determines the mode of operation it can take following values:-
652 // MODE_BROADCAST for vector Vector.broadcast and VectorMask.maskAll operations.
653 // MODE_BITS_COERCED_LONG_TO_MASK for VectorMask.fromLong operation.
654 const TypeInt* mode = gvn().type(argument(5))->isa_int();
655
656 if (vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
657 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
658 vlen == nullptr || !vlen->is_con() ||
659 bits_type == nullptr ||
660 mode == nullptr || !mode->is_con()) {
661 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s bitwise=%s",
662 NodeClassNames[argument(0)->Opcode()],
663 NodeClassNames[argument(1)->Opcode()],
664 NodeClassNames[argument(2)->Opcode()],
665 NodeClassNames[argument(5)->Opcode()]);
666 return false; // not enough info for intrinsification
667 }
668
669 if (!is_klass_initialized(vector_klass)) {
670 log_if_needed(" ** klass argument not initialized");
671 return false;
672 }
673 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
674 if (!elem_type->is_primitive_type()) {
675 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
676 return false; // should be primitive type
677 }
678 BasicType elem_bt = elem_type->basic_type();
679 int num_elem = vlen->get_con();
680 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
681 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
682
683 bool is_mask = is_vector_mask(vbox_klass);
684 int bcast_mode = mode->get_con();
685 VectorMaskUseType checkFlags = (VectorMaskUseType)(is_mask ? VecMaskUseAll : VecMaskNotUsed);
686 int opc = bcast_mode == VectorSupport::MODE_BITS_COERCED_LONG_TO_MASK ? Op_VectorLongToMask : Op_Replicate;
687
688 if (!arch_supports_vector(opc, num_elem, elem_bt, checkFlags, true /*has_scalar_args*/)) {
689 // If the input long sets or unsets all lanes and Replicate is supported,
690 // generate a MaskAll or Replicate instead.
691
692 // The "maskAll" API uses the corresponding integer types for floating-point data.
693 BasicType maskall_bt = elem_bt == T_DOUBLE ? T_LONG : (elem_bt == T_FLOAT ? T_INT: elem_bt);
694 if (!(opc == Op_VectorLongToMask &&
695 VectorNode::is_maskall_type(bits_type, num_elem) &&
696 arch_supports_vector(Op_Replicate, num_elem, maskall_bt, checkFlags, true /*has_scalar_args*/))) {
697 log_if_needed(" ** not supported: arity=0 op=broadcast vlen=%d etype=%s ismask=%d bcast_mode=%d",
698 num_elem, type2name(elem_bt),
699 is_mask ? 1 : 0,
700 bcast_mode);
701 return false; // not supported
702 }
703 }
704
705 Node* broadcast = nullptr;
706 Node* bits = argument(3);
707 Node* elem = bits;
708
709 if (opc == Op_VectorLongToMask) {
710 const TypeVect* vt = TypeVect::makemask(elem_bt, num_elem);
711 if (vt->isa_vectmask()) {
712 broadcast = gvn().transform(new VectorLongToMaskNode(elem, vt));
713 } else {
714 const TypeVect* mvt = TypeVect::make(T_BOOLEAN, num_elem);
715 broadcast = gvn().transform(new VectorLongToMaskNode(elem, mvt));
716 broadcast = gvn().transform(new VectorLoadMaskNode(broadcast, vt));
717 }
718 } else {
719 switch (elem_bt) {
720 case T_BOOLEAN: // fall-through
721 case T_BYTE: // fall-through
722 case T_SHORT: // fall-through
723 case T_CHAR: // fall-through
724 case T_INT: {
725 elem = gvn().transform(new ConvL2INode(bits));
726 break;
727 }
728 case T_DOUBLE: {
729 elem = gvn().transform(new MoveL2DNode(bits));
730 break;
731 }
732 case T_FLOAT: {
733 bits = gvn().transform(new ConvL2INode(bits));
734 elem = gvn().transform(new MoveI2FNode(bits));
735 break;
736 }
737 case T_LONG: {
738 // no conversion needed
739 break;
740 }
741 default: fatal("%s", type2name(elem_bt));
742 }
743 broadcast = VectorNode::scalar2vector(elem, num_elem, elem_bt, is_mask);
744 broadcast = gvn().transform(broadcast);
745 }
746
747 Node* box = box_vector(broadcast, vbox_type, elem_bt, num_elem);
748 set_result(box);
749 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
750 return true;
751 }
752
753 static bool elem_consistent_with_arr(BasicType elem_bt, const TypeAryPtr* arr_type, bool mismatched_ms) {
754 assert(arr_type != nullptr, "unexpected");
755 BasicType arr_elem_bt = arr_type->elem()->array_element_basic_type();
756 if (elem_bt == arr_elem_bt) {
757 return true;
758 } else if (elem_bt == T_SHORT && arr_elem_bt == T_CHAR) {
759 // Load/store of short vector from/to char[] is supported
760 return true;
761 } else if (elem_bt == T_BYTE && arr_elem_bt == T_BOOLEAN) {
762 // Load/store of byte vector from/to boolean[] is supported
763 return true;
764 } else {
765 return mismatched_ms;
766 }
767 }
768
769 // public static
770 // <C,
771 // VM extends VectorPayload,
772 // E,
773 // S extends VectorSpecies<E>>
774 // VM load(Class<? extends VM> vmClass, Class<E> eClass,
775 // int length,
776 // Object base, long offset, // Unsafe addressing
777 // boolean fromSegment,
778 // C container, long index, S s, // Arguments for default implementation
779 // LoadOperation<C, VM, S> defaultImpl) {
780 // public static
781 // <C,
782 // V extends VectorPayload>
783 // void store(Class<?> vClass, Class<?> eClass,
784 // int length,
785 // Object base, long offset, // Unsafe addressing
786 // boolean fromSegment,
787 // V v, C container, long index, // Arguments for default implementation
788 // StoreVectorOperation<C, V> defaultImpl) {
789 bool LibraryCallKit::inline_vector_mem_operation(bool is_store) {
790 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
791 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
792 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
793 const TypeInt* from_ms = gvn().type(argument(6))->isa_int();
794
795 if (vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
796 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
797 vlen == nullptr || !vlen->is_con() ||
798 from_ms == nullptr || !from_ms->is_con()) {
799 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s from_ms=%s",
800 NodeClassNames[argument(0)->Opcode()],
801 NodeClassNames[argument(1)->Opcode()],
802 NodeClassNames[argument(2)->Opcode()],
803 NodeClassNames[argument(6)->Opcode()]);
804 return false; // not enough info for intrinsification
805 }
806 if (!is_klass_initialized(vector_klass)) {
807 log_if_needed(" ** klass argument not initialized");
808 return false;
809 }
810
811 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
812 if (!elem_type->is_primitive_type()) {
813 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
814 return false; // should be primitive type
815 }
816 BasicType elem_bt = elem_type->basic_type();
817 int num_elem = vlen->get_con();
818
819 // TODO When mask usage is supported, VecMaskNotUsed needs to be VecMaskUseLoad.
820 if (!arch_supports_vector(is_store ? Op_StoreVector : Op_LoadVector, num_elem, elem_bt, VecMaskNotUsed)) {
821 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d etype=%s ismask=no",
822 is_store, is_store ? "store" : "load",
823 num_elem, type2name(elem_bt));
824 return false; // not supported
825 }
826
827 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
828 bool is_mask = is_vector_mask(vbox_klass);
829
830 Node* base = argument(3);
831 Node* offset = ConvL2X(argument(4));
832
833 // Save state and restore on bailout
834 SavedState old_state(this);
835
836 Node* addr = make_unsafe_address(base, offset, (is_mask ? T_BOOLEAN : elem_bt), true);
837
838 // The memory barrier checks are based on ones for unsafe access.
839 // This is not 1-1 implementation.
840 const Type *const base_type = gvn().type(base);
841
842 const TypePtr *addr_type = gvn().type(addr)->isa_ptr();
843 const TypeAryPtr* arr_type = addr_type->isa_aryptr();
844
845 const bool in_native = TypePtr::NULL_PTR == base_type; // base always null
846 const bool in_heap = !TypePtr::NULL_PTR->higher_equal(base_type); // base never null
847
848 const bool is_mixed_access = !in_heap && !in_native;
849
850 const bool is_mismatched_access = in_heap && (addr_type->isa_aryptr() == nullptr);
851
852 const bool needs_cpu_membar = is_mixed_access || is_mismatched_access;
853
854 // For non-masked mismatched memory segment vector read/write accesses, intrinsification can continue
855 // with unknown backing storage type and compiler can skip inserting explicit reinterpretation IR after
856 // loading from or before storing to backing storage which is mandatory for semantic correctness of
857 // big-endian memory layout.
858 bool mismatched_ms = LITTLE_ENDIAN_ONLY(false)
859 BIG_ENDIAN_ONLY(from_ms->get_con() && !is_mask && arr_type != nullptr &&
860 arr_type->elem()->array_element_basic_type() != elem_bt);
861 BasicType mem_elem_bt = mismatched_ms ? arr_type->elem()->array_element_basic_type() : elem_bt;
862 if (!is_java_primitive(mem_elem_bt)) {
863 log_if_needed(" ** non-primitive array element type");
864 return false;
865 }
866 int mem_num_elem = mismatched_ms ? (num_elem * type2aelembytes(elem_bt)) / type2aelembytes(mem_elem_bt) : num_elem;
867 if (arr_type != nullptr && !is_mask && !elem_consistent_with_arr(elem_bt, arr_type, mismatched_ms)) {
868 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d etype=%s atype=%s ismask=no",
869 is_store, is_store ? "store" : "load",
870 num_elem, type2name(elem_bt), type2name(arr_type->elem()->array_element_basic_type()));
871 return false;
872 }
873
874 // In case of mismatched memory segment accesses, we need to double check that the source type memory operations are supported by backend.
875 if (mismatched_ms) {
876 if (is_store) {
877 if (!arch_supports_vector(Op_StoreVector, num_elem, elem_bt, VecMaskNotUsed)
878 || !arch_supports_vector(Op_VectorReinterpret, mem_num_elem, mem_elem_bt, VecMaskNotUsed)) {
879 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d*8 etype=%s/8 ismask=no",
880 is_store, "store",
881 num_elem, type2name(elem_bt));
882 return false; // not supported
883 }
884 } else {
885 if (!arch_supports_vector(Op_LoadVector, mem_num_elem, mem_elem_bt, VecMaskNotUsed)
886 || !arch_supports_vector(Op_VectorReinterpret, num_elem, elem_bt, VecMaskNotUsed)) {
887 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d*8 etype=%s/8 ismask=no",
888 is_store, "load",
889 mem_num_elem, type2name(mem_elem_bt));
890 return false; // not supported
891 }
892 }
893 }
894 if (is_mask) {
895 if (!is_store) {
896 if (!arch_supports_vector(Op_LoadVector, num_elem, elem_bt, VecMaskUseLoad)) {
897 return false; // not supported
898 }
899 } else {
900 if (!arch_supports_vector(Op_StoreVector, num_elem, elem_bt, VecMaskUseStore)) {
901 return false; // not supported
902 }
903 }
904 }
905
906 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
907
908 if (needs_cpu_membar) {
909 insert_mem_bar(Op_MemBarCPUOrder);
910 }
911
912 if (is_store) {
913 Node* val = unbox_vector(argument(7), vbox_type, elem_bt, num_elem);
914 if (val == nullptr) {
915 return false; // operand unboxing failed
916 }
917 set_all_memory(reset_memory());
918
919 // In case the store needs to happen to byte array, reinterpret the incoming vector to byte vector.
920 int store_num_elem = num_elem;
921 if (mismatched_ms) {
922 store_num_elem = mem_num_elem;
923 const TypeVect* to_vect_type = TypeVect::make(mem_elem_bt, store_num_elem);
924 val = gvn().transform(new VectorReinterpretNode(val, val->bottom_type()->is_vect(), to_vect_type));
925 }
926 if (is_mask) {
927 val = gvn().transform(VectorStoreMaskNode::make(gvn(), val, elem_bt, num_elem));
928 }
929 Node* vstore = gvn().transform(StoreVectorNode::make(0, control(), memory(addr), addr, addr_type, val, store_num_elem));
930 set_memory(vstore, addr_type);
931 } else {
932 // When using byte array, we need to load as byte then reinterpret the value. Otherwise, do a simple vector load.
933 Node* vload = nullptr;
934 if (mismatched_ms) {
935 vload = gvn().transform(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, mem_num_elem, mem_elem_bt));
936 const TypeVect* to_vect_type = TypeVect::make(elem_bt, num_elem);
937 vload = gvn().transform(new VectorReinterpretNode(vload, vload->bottom_type()->is_vect(), to_vect_type));
938 } else {
939 // Special handle for masks
940 if (is_mask) {
941 vload = gvn().transform(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, num_elem, T_BOOLEAN));
942 vload = gvn().transform(new VectorLoadMaskNode(vload, TypeVect::makemask(elem_bt, num_elem)));
943 } else {
944 vload = gvn().transform(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, num_elem, elem_bt));
945 }
946 }
947 Node* box = box_vector(vload, vbox_type, elem_bt, num_elem);
948 set_result(box);
949 }
950
951 old_state.discard();
952
953 if (needs_cpu_membar) {
954 insert_mem_bar(Op_MemBarCPUOrder);
955 }
956
957 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
958 return true;
959 }
960
961 // public static
962 // <C,
963 // V extends Vector<?>,
964 // E,
965 // S extends VectorSpecies<E>,
966 // M extends VectorMask<E>>
967 // V loadMasked(Class<? extends V> vClass, Class<M> mClass, Class<E> eClass,
968 // int length, Object base, long offset, // Unsafe addressing
969 // boolean fromSegment,
970 // M m, int offsetInRange,
971 // C container, long index, S s, // Arguments for default implementation
972 // LoadVectorMaskedOperation<C, V, S, M> defaultImpl) {
973 // public static
974 // <C,
975 // V extends Vector<E>,
976 // M extends VectorMask<E>,
977 // E>
978 // void storeMasked(Class<? extends V> vClass, Class<M> mClass, Class<E> eClass,
979 // int length,
980 // Object base, long offset, // Unsafe addressing
981 // boolean fromSegment,
982 // V v, M m, C container, long index, // Arguments for default implementation
983 // StoreVectorMaskedOperation<C, V, M> defaultImpl) {
984
985 bool LibraryCallKit::inline_vector_mem_masked_operation(bool is_store) {
986 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
987 const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
988 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
989 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
990 const TypeInt* from_ms = gvn().type(argument(7))->isa_int();
991
992 if (vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
993 mask_klass == nullptr || mask_klass->const_oop() == nullptr ||
994 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
995 vlen == nullptr || !vlen->is_con() ||
996 from_ms == nullptr || !from_ms->is_con()) {
997 log_if_needed(" ** missing constant: vclass=%s mclass=%s etype=%s vlen=%s from_ms=%s",
998 NodeClassNames[argument(0)->Opcode()],
999 NodeClassNames[argument(1)->Opcode()],
1000 NodeClassNames[argument(2)->Opcode()],
1001 NodeClassNames[argument(3)->Opcode()],
1002 NodeClassNames[argument(7)->Opcode()]);
1003 return false; // not enough info for intrinsification
1004 }
1005 if (!is_klass_initialized(vector_klass)) {
1006 log_if_needed(" ** klass argument not initialized");
1007 return false;
1008 }
1009
1010 if (!is_klass_initialized(mask_klass)) {
1011 log_if_needed(" ** mask klass argument not initialized");
1012 return false;
1013 }
1014
1015 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1016 if (!elem_type->is_primitive_type()) {
1017 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1018 return false; // should be primitive type
1019 }
1020
1021 BasicType elem_bt = elem_type->basic_type();
1022 int num_elem = vlen->get_con();
1023
1024 Node* base = argument(4);
1025 Node* offset = ConvL2X(argument(5));
1026
1027 // Save state and restore on bailout
1028 SavedState old_state(this);
1029
1030 Node* addr = make_unsafe_address(base, offset, elem_bt, true);
1031 const TypePtr *addr_type = gvn().type(addr)->isa_ptr();
1032 const TypeAryPtr* arr_type = addr_type->isa_aryptr();
1033
1034 bool mismatched_ms = from_ms->get_con() && arr_type != nullptr && arr_type->elem()->array_element_basic_type() != elem_bt;
1035 BIG_ENDIAN_ONLY(if (mismatched_ms) return false;)
1036 // If there is no consistency between array and vector element types, it must be special byte array case
1037 if (arr_type != nullptr && !elem_consistent_with_arr(elem_bt, arr_type, mismatched_ms)) {
1038 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d etype=%s atype=%s",
1039 is_store, is_store ? "storeMasked" : "loadMasked",
1040 num_elem, type2name(elem_bt), type2name(arr_type->elem()->array_element_basic_type()));
1041 return false;
1042 }
1043
1044 int mem_num_elem = mismatched_ms ? num_elem * type2aelembytes(elem_bt) : num_elem;
1045 BasicType mem_elem_bt = mismatched_ms ? T_BYTE : elem_bt;
1046 bool supports_predicate = arch_supports_vector(is_store ? Op_StoreVectorMasked : Op_LoadVectorMasked,
1047 mem_num_elem, mem_elem_bt, VecMaskUseLoad);
1048
1049 // If current arch does not support the predicated operations, we have to bail
1050 // out when current case uses the predicate feature.
1051 if (!supports_predicate) {
1052 bool needs_predicate = false;
1053 if (is_store) {
1054 // Masked vector store always uses the predicated store.
1055 needs_predicate = true;
1056 } else {
1057 // Masked vector load with IOOBE always uses the predicated load.
1058 const TypeInt* offset_in_range = gvn().type(argument(9))->isa_int();
1059 if (!offset_in_range->is_con()) {
1060 log_if_needed(" ** missing constant: offsetInRange=%s",
1061 NodeClassNames[argument(8)->Opcode()]);
1062 return false;
1063 }
1064 needs_predicate = (offset_in_range->get_con() == 0);
1065 }
1066
1067 if (needs_predicate) {
1068 log_if_needed(" ** not supported: op=%s vlen=%d etype=%s mismatched_ms=%d",
1069 is_store ? "storeMasked" : "loadMasked",
1070 num_elem, type2name(elem_bt), mismatched_ms ? 1 : 0);
1071 return false;
1072 }
1073 }
1074
1075 // This only happens for masked vector load. If predicate is not supported, then check whether
1076 // the normal vector load and blend operations are supported by backend.
1077 if (!supports_predicate && (!arch_supports_vector(Op_LoadVector, mem_num_elem, mem_elem_bt, VecMaskNotUsed) ||
1078 !arch_supports_vector(Op_VectorBlend, mem_num_elem, mem_elem_bt, VecMaskUseLoad))) {
1079 log_if_needed(" ** not supported: op=loadMasked vlen=%d etype=%s mismatched_ms=%d",
1080 num_elem, type2name(elem_bt), mismatched_ms ? 1 : 0);
1081 return false;
1082 }
1083
1084 // Since we are using byte array, we need to double check that the vector reinterpret operation
1085 // with byte type is supported by backend.
1086 if (mismatched_ms) {
1087 if (!arch_supports_vector(Op_VectorReinterpret, mem_num_elem, T_BYTE, VecMaskNotUsed)) {
1088 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d etype=%s mismatched_ms=1",
1089 is_store, is_store ? "storeMasked" : "loadMasked",
1090 num_elem, type2name(elem_bt));
1091 return false;
1092 }
1093 }
1094
1095 // Since it needs to unbox the mask, we need to double check that the related load operations
1096 // for mask are supported by backend.
1097 if (!arch_supports_vector(Op_LoadVector, num_elem, elem_bt, VecMaskUseLoad)) {
1098 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d etype=%s",
1099 is_store, is_store ? "storeMasked" : "loadMasked",
1100 num_elem, type2name(elem_bt));
1101 return false;
1102 }
1103
1104 // Can base be null? Otherwise, always on-heap access.
1105 bool can_access_non_heap = TypePtr::NULL_PTR->higher_equal(gvn().type(base));
1106 if (can_access_non_heap) {
1107 insert_mem_bar(Op_MemBarCPUOrder);
1108 }
1109
1110 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1111 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1112 assert(!is_vector_mask(vbox_klass) && is_vector_mask(mbox_klass), "Invalid class type");
1113 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1114 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1115
1116 Node* mask = unbox_vector(is_store ? argument(9) : argument(8), mbox_type, elem_bt, num_elem);
1117 if (mask == nullptr) {
1118 log_if_needed(" ** unbox failed mask=%s",
1119 is_store ? NodeClassNames[argument(9)->Opcode()]
1120 : NodeClassNames[argument(8)->Opcode()]);
1121 return false;
1122 }
1123
1124 if (is_store) {
1125 Node* val = unbox_vector(argument(8), vbox_type, elem_bt, num_elem);
1126 if (val == nullptr) {
1127 log_if_needed(" ** unbox failed vector=%s",
1128 NodeClassNames[argument(8)->Opcode()]);
1129 return false; // operand unboxing failed
1130 }
1131 set_all_memory(reset_memory());
1132
1133 if (mismatched_ms) {
1134 // Reinterpret the incoming vector to byte vector.
1135 const TypeVect* to_vect_type = TypeVect::make(mem_elem_bt, mem_num_elem);
1136 val = gvn().transform(new VectorReinterpretNode(val, val->bottom_type()->is_vect(), to_vect_type));
1137 // Reinterpret the vector mask to byte type.
1138 const TypeVect* from_mask_type = TypeVect::makemask(elem_bt, num_elem);
1139 const TypeVect* to_mask_type = TypeVect::makemask(mem_elem_bt, mem_num_elem);
1140 mask = gvn().transform(new VectorReinterpretNode(mask, from_mask_type, to_mask_type));
1141 }
1142 Node* vstore = gvn().transform(new StoreVectorMaskedNode(control(), memory(addr), addr, val, addr_type, mask));
1143 set_memory(vstore, addr_type);
1144 } else {
1145 Node* vload = nullptr;
1146
1147 if (mismatched_ms) {
1148 // Reinterpret the vector mask to byte type.
1149 const TypeVect* from_mask_type = TypeVect::makemask(elem_bt, num_elem);
1150 const TypeVect* to_mask_type = TypeVect::makemask(mem_elem_bt, mem_num_elem);
1151 mask = gvn().transform(new VectorReinterpretNode(mask, from_mask_type, to_mask_type));
1152 }
1153
1154 if (supports_predicate) {
1155 // Generate masked load vector node if predicate feature is supported.
1156 const TypeVect* vt = TypeVect::make(mem_elem_bt, mem_num_elem);
1157 vload = gvn().transform(new LoadVectorMaskedNode(control(), memory(addr), addr, addr_type, vt, mask));
1158 } else {
1159 // Use the vector blend to implement the masked load vector. The biased elements are zeros.
1160 Node* zero = gvn().transform(gvn().zerocon(mem_elem_bt));
1161 zero = gvn().transform(VectorNode::scalar2vector(zero, mem_num_elem, mem_elem_bt));
1162 vload = gvn().transform(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, mem_num_elem, mem_elem_bt));
1163 vload = gvn().transform(new VectorBlendNode(zero, vload, mask));
1164 }
1165
1166 if (mismatched_ms) {
1167 const TypeVect* to_vect_type = TypeVect::make(elem_bt, num_elem);
1168 vload = gvn().transform(new VectorReinterpretNode(vload, vload->bottom_type()->is_vect(), to_vect_type));
1169 }
1170
1171 Node* box = box_vector(vload, vbox_type, elem_bt, num_elem);
1172 set_result(box);
1173 }
1174
1175 old_state.discard();
1176
1177 if (can_access_non_heap) {
1178 insert_mem_bar(Op_MemBarCPUOrder);
1179 }
1180
1181 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1182 return true;
1183 }
1184
1185 //
1186 // <C,
1187 // V extends Vector<?>,
1188 // W extends Vector<Integer>,
1189 // S extends VectorSpecies<E>,
1190 // M extends VectorMask<E>,
1191 // E>
1192 // V loadWithMap(Class<? extends V> vClass, Class<M> mClass, Class<E> eClass, int length,
1193 // Class<? extends Vector<Integer>> vectorIndexClass, int indexLength,
1194 // Object base, long offset,
1195 // W indexVector1, W indexVector2, W indexVector3, W indexVector4,
1196 // M m, C container, int index, int[] indexMap, int indexM, S s,
1197 // LoadVectorOperationWithMap<C, V, S, M> defaultImpl)
1198 //
1199 // <C,
1200 // V extends Vector<E>,
1201 // W extends Vector<Integer>,
1202 // M extends VectorMask<E>,
1203 // E>
1204 // void storeWithMap(Class<? extends V> vClass, Class<M> mClass, Class<E> eClass, int length,
1205 // Class<? extends Vector<Integer>> vectorIndexClass, int indexLength,
1206 // Object base, long offset, // Unsafe addressing
1207 // W indexVector, V v, M m,
1208 // C container, int index, int[] indexMap, int indexM, // Arguments for default implementation
1209 // StoreVectorOperationWithMap<C, V, M> defaultImpl)
1210 //
1211 bool LibraryCallKit::inline_vector_gather_scatter(bool is_scatter) {
1212 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
1213 const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
1214 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
1215 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
1216 const TypeInstPtr* vector_idx_klass = gvn().type(argument(4))->isa_instptr();
1217 const TypeInt* idx_vlen = gvn().type(argument(5))->isa_int();
1218
1219 if (vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
1220 // mask_klass == nullptr || mask_klass->const_oop() == nullptr ||
1221 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
1222 vlen == nullptr || !vlen->is_con() ||
1223 vector_idx_klass == nullptr || vector_idx_klass->const_oop() == nullptr ||
1224 idx_vlen == nullptr || !idx_vlen->is_con()) {
1225 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s viclass=%s idx_vlen=%s",
1226 NodeClassNames[argument(0)->Opcode()],
1227 NodeClassNames[argument(2)->Opcode()],
1228 NodeClassNames[argument(3)->Opcode()],
1229 NodeClassNames[argument(4)->Opcode()],
1230 NodeClassNames[argument(5)->Opcode()]);
1231 return false; // not enough info for intrinsification
1232 }
1233
1234 if (!is_klass_initialized(vector_klass) || !is_klass_initialized(vector_idx_klass)) {
1235 log_if_needed(" ** klass argument not initialized");
1236 return false;
1237 }
1238
1239 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1240 if (!elem_type->is_primitive_type()) {
1241 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1242 return false; // should be primitive type
1243 }
1244
1245 BasicType elem_bt = elem_type->basic_type();
1246 int num_elem = vlen->get_con();
1247 int idx_num_elem = idx_vlen->get_con();
1248
1249 Node* m = is_scatter ? argument(11) : argument(13);
1250 const Type* vmask_type = gvn().type(m);
1251 bool is_masked_op = vmask_type != TypePtr::NULL_PTR;
1252 if (is_masked_op) {
1253 if (mask_klass == nullptr || mask_klass->const_oop() == nullptr) {
1254 log_if_needed(" ** missing constant: maskclass=%s", NodeClassNames[argument(1)->Opcode()]);
1255 return false; // not enough info for intrinsification
1256 }
1257
1258 if (!is_klass_initialized(mask_klass)) {
1259 log_if_needed(" ** mask klass argument not initialized");
1260 return false;
1261 }
1262
1263 if (vmask_type->maybe_null()) {
1264 log_if_needed(" ** null mask values are not allowed for masked op");
1265 return false;
1266 }
1267
1268 // Check whether the predicated gather/scatter node is supported by architecture.
1269 VectorMaskUseType mask = (VectorMaskUseType) (VecMaskUseLoad | VecMaskUsePred);
1270 if (!arch_supports_vector(is_scatter ? Op_StoreVectorScatterMasked : Op_LoadVectorGatherMasked, num_elem, elem_bt, mask)) {
1271 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d etype=%s is_masked_op=1",
1272 is_scatter, is_scatter ? "scatterMasked" : "gatherMasked",
1273 num_elem, type2name(elem_bt));
1274 return false; // not supported
1275 }
1276 } else {
1277 // Check whether the normal gather/scatter node is supported for non-masked operation.
1278 if (!arch_supports_vector(is_scatter ? Op_StoreVectorScatter : Op_LoadVectorGather, num_elem, elem_bt, VecMaskNotUsed)) {
1279 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d etype=%s is_masked_op=0",
1280 is_scatter, is_scatter ? "scatter" : "gather",
1281 num_elem, type2name(elem_bt));
1282 return false; // not supported
1283 }
1284 }
1285
1286 // Check that the vector holding indices is supported by architecture
1287 // For sub-word gathers expander receive index array.
1288 if (!is_subword_type(elem_bt) && !arch_supports_vector(Op_LoadVector, idx_num_elem, T_INT, VecMaskNotUsed)) {
1289 log_if_needed(" ** not supported: arity=%d op=%s/loadindex vlen=%d etype=int is_masked_op=%d",
1290 is_scatter, is_scatter ? "scatter" : "gather",
1291 idx_num_elem, is_masked_op ? 1 : 0);
1292 return false; // not supported
1293 }
1294
1295 Node* base = argument(6);
1296 Node* offset = ConvL2X(argument(7));
1297
1298 // Save state and restore on bailout
1299 SavedState old_state(this);
1300
1301 Node* addr = nullptr;
1302 if (!is_subword_type(elem_bt)) {
1303 addr = make_unsafe_address(base, offset, elem_bt, true);
1304 } else {
1305 assert(!is_scatter, "Only supports gather operation for subword types now");
1306 uint header = arrayOopDesc::base_offset_in_bytes(elem_bt);
1307 assert(offset->is_Con() && offset->bottom_type()->is_long()->get_con() == header,
1308 "offset must be the array base offset");
1309 Node* index = argument(15);
1310 addr = array_element_address(base, index, elem_bt);
1311 }
1312
1313 const TypePtr* addr_type = gvn().type(addr)->isa_ptr();
1314 const TypeAryPtr* arr_type = addr_type->isa_aryptr();
1315
1316 // The array must be consistent with vector type
1317 if (arr_type == nullptr || (arr_type != nullptr && !elem_consistent_with_arr(elem_bt, arr_type, false))) {
1318 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d etype=%s atype=%s ismask=no",
1319 is_scatter, is_scatter ? "scatter" : "gather",
1320 num_elem, type2name(elem_bt), type2name(arr_type->elem()->array_element_basic_type()));
1321 return false;
1322 }
1323
1324 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1325 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1326 ciKlass* vbox_idx_klass = vector_idx_klass->const_oop()->as_instance()->java_lang_Class_klass();
1327 if (vbox_idx_klass == nullptr) {
1328 return false;
1329 }
1330
1331 // Get the indexes for gather/scatter.
1332 Node* indexes = nullptr;
1333 const TypeInstPtr* vbox_idx_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_idx_klass);
1334 if (is_subword_type(elem_bt)) {
1335 Node* indexMap = argument(16);
1336 Node* indexM = argument(17);
1337 indexes = array_element_address(indexMap, indexM, T_INT);
1338 } else {
1339 // Get the first index vector.
1340 indexes = unbox_vector(argument(9), vbox_idx_type, T_INT, idx_num_elem);
1341 if (indexes == nullptr) {
1342 return false;
1343 }
1344 }
1345
1346 // Get the vector mask value.
1347 Node* mask = nullptr;
1348 if (is_masked_op) {
1349 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1350 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1351 mask = unbox_vector(m, mbox_type, elem_bt, num_elem);
1352 if (mask == nullptr) {
1353 log_if_needed(" ** unbox failed mask=%s", NodeClassNames[m->Opcode()]);
1354 return false;
1355 }
1356 }
1357
1358 const TypeVect* vector_type = TypeVect::make(elem_bt, num_elem);
1359 if (is_scatter) {
1360 Node* val = unbox_vector(argument(10), vbox_type, elem_bt, num_elem);
1361 if (val == nullptr) {
1362 return false; // operand unboxing failed
1363 }
1364 set_all_memory(reset_memory());
1365
1366 Node* vstore = nullptr;
1367 if (mask != nullptr) {
1368 vstore = gvn().transform(new StoreVectorScatterMaskedNode(control(), memory(addr), addr, addr_type, val, indexes, mask));
1369 } else {
1370 vstore = gvn().transform(new StoreVectorScatterNode(control(), memory(addr), addr, addr_type, val, indexes));
1371 }
1372 set_memory(vstore, addr_type);
1373 } else {
1374 Node* vload = nullptr;
1375 if (mask != nullptr) {
1376 vload = gvn().transform(new LoadVectorGatherMaskedNode(control(), memory(addr), addr, addr_type, vector_type, indexes, mask));
1377 } else {
1378 vload = gvn().transform(new LoadVectorGatherNode(control(), memory(addr), addr, addr_type, vector_type, indexes));
1379 }
1380 Node* box = box_vector(vload, vbox_type, elem_bt, num_elem);
1381 set_result(box);
1382 }
1383
1384 old_state.discard();
1385 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1386 return true;
1387 }
1388
1389 // public static
1390 // <V extends Vector<E>,
1391 // M extends VectorMask<E>,
1392 // E>
1393 // long reductionCoerced(int oprId, Class<? extends V> vectorClass, Class<? extends M> maskClass,
1394 // Class<E> elementType, int length, V v, M m,
1395 // ReductionOperation<V, M> defaultImpl)
1396 bool LibraryCallKit::inline_vector_reduction() {
1397 const TypeInt* opr = gvn().type(argument(0))->isa_int();
1398 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
1399 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
1400 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
1401 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
1402
1403 if (opr == nullptr || !opr->is_con() ||
1404 vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
1405 // mask_klass == nullptr || mask_klass->const_oop() == nullptr ||
1406 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
1407 vlen == nullptr || !vlen->is_con()) {
1408 log_if_needed(" ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
1409 NodeClassNames[argument(0)->Opcode()],
1410 NodeClassNames[argument(1)->Opcode()],
1411 NodeClassNames[argument(3)->Opcode()],
1412 NodeClassNames[argument(4)->Opcode()]);
1413 return false; // not enough info for intrinsification
1414 }
1415 if (!is_klass_initialized(vector_klass)) {
1416 log_if_needed(" ** klass argument not initialized");
1417 return false;
1418 }
1419 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1420 if (!elem_type->is_primitive_type()) {
1421 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1422 return false; // should be primitive type
1423 }
1424
1425 const Type* vmask_type = gvn().type(argument(6));
1426 bool is_masked_op = vmask_type != TypePtr::NULL_PTR;
1427 if (is_masked_op) {
1428 if (mask_klass == nullptr || mask_klass->const_oop() == nullptr) {
1429 log_if_needed(" ** missing constant: maskclass=%s", NodeClassNames[argument(2)->Opcode()]);
1430 return false; // not enough info for intrinsification
1431 }
1432
1433 if (!is_klass_initialized(mask_klass)) {
1434 log_if_needed(" ** mask klass argument not initialized");
1435 return false;
1436 }
1437
1438 if (vmask_type->maybe_null()) {
1439 log_if_needed(" ** null mask values are not allowed for masked op");
1440 return false;
1441 }
1442 }
1443
1444 BasicType elem_bt = elem_type->basic_type();
1445 int num_elem = vlen->get_con();
1446 int opc = VectorSupport::vop2ideal(opr->get_con(), elem_bt);
1447 int sopc = ReductionNode::opcode(opc, elem_bt);
1448
1449 // Ensure reduction operation for lanewise operation
1450 // When using mask, mask use type needs to be VecMaskUseLoad.
1451 if (sopc == opc || !arch_supports_vector(sopc, num_elem, elem_bt, is_masked_op ? VecMaskUseLoad : VecMaskNotUsed)) {
1452 log_if_needed(" ** not supported: arity=1 op=%d/reduce vlen=%d etype=%s is_masked_op=%d",
1453 sopc, num_elem, type2name(elem_bt), is_masked_op ? 1 : 0);
1454 return false;
1455 }
1456
1457 // Return true if current platform has implemented the masked operation with predicate feature.
1458 bool use_predicate = is_masked_op && arch_supports_vector(sopc, num_elem, elem_bt, VecMaskUsePred);
1459 if (is_masked_op && !use_predicate && !arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad)) {
1460 log_if_needed(" ** not supported: arity=1 op=%d/reduce vlen=%d etype=%s is_masked_op=1",
1461 sopc, num_elem, type2name(elem_bt));
1462 return false;
1463 }
1464
1465 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1466 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1467
1468 Node* opd = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
1469 if (opd == nullptr) {
1470 return false; // operand unboxing failed
1471 }
1472
1473 Node* mask = nullptr;
1474 if (is_masked_op) {
1475 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1476 assert(is_vector_mask(mbox_klass), "argument(2) should be a mask class");
1477 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1478 mask = unbox_vector(argument(6), mbox_type, elem_bt, num_elem);
1479 if (mask == nullptr) {
1480 log_if_needed(" ** unbox failed mask=%s",
1481 NodeClassNames[argument(6)->Opcode()]);
1482 return false;
1483 }
1484 }
1485
1486 Node* init = ReductionNode::make_identity_con_scalar(gvn(), opc, elem_bt);
1487 Node* value = opd;
1488
1489 assert(mask != nullptr || !is_masked_op, "Masked op needs the mask value never null");
1490 if (mask != nullptr && !use_predicate) {
1491 Node* reduce_identity = gvn().transform(VectorNode::scalar2vector(init, num_elem, elem_bt));
1492 value = gvn().transform(new VectorBlendNode(reduce_identity, value, mask));
1493 }
1494
1495 // Make an unordered Reduction node. This affects only AddReductionVF/VD and MulReductionVF/VD,
1496 // as these operations are allowed to be associative (not requiring strict order) in VectorAPI.
1497 value = ReductionNode::make(opc, nullptr, init, value, elem_bt, /* requires_strict_order */ false);
1498
1499 if (mask != nullptr && use_predicate) {
1500 value->add_req(mask);
1501 value->add_flag(Node::Flag_is_predicated_vector);
1502 }
1503
1504 value = gvn().transform(value);
1505
1506 Node* bits = nullptr;
1507 switch (elem_bt) {
1508 case T_BYTE:
1509 case T_SHORT:
1510 case T_INT: {
1511 bits = gvn().transform(new ConvI2LNode(value));
1512 break;
1513 }
1514 case T_FLOAT: {
1515 value = gvn().transform(new MoveF2INode(value));
1516 bits = gvn().transform(new ConvI2LNode(value));
1517 break;
1518 }
1519 case T_DOUBLE: {
1520 bits = gvn().transform(new MoveD2LNode(value));
1521 break;
1522 }
1523 case T_LONG: {
1524 bits = value; // no conversion needed
1525 break;
1526 }
1527 default: fatal("%s", type2name(elem_bt));
1528 }
1529 set_result(bits);
1530 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1531 return true;
1532 }
1533
1534 // public static <V> boolean test(int cond, Class<?> vectorClass, Class<?> elementType, int vlen,
1535 // V v1, V v2,
1536 // BiFunction<V, V, Boolean> defaultImpl)
1537 //
1538 bool LibraryCallKit::inline_vector_test() {
1539 const TypeInt* cond = gvn().type(argument(0))->isa_int();
1540 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
1541 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
1542 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
1543
1544 if (cond == nullptr || !cond->is_con() ||
1545 vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
1546 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
1547 vlen == nullptr || !vlen->is_con()) {
1548 log_if_needed(" ** missing constant: cond=%s vclass=%s etype=%s vlen=%s",
1549 NodeClassNames[argument(0)->Opcode()],
1550 NodeClassNames[argument(1)->Opcode()],
1551 NodeClassNames[argument(2)->Opcode()],
1552 NodeClassNames[argument(3)->Opcode()]);
1553 return false; // not enough info for intrinsification
1554 }
1555 if (!is_klass_initialized(vector_klass)) {
1556 log_if_needed(" ** klass argument not initialized");
1557 return false;
1558 }
1559 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1560 if (!elem_type->is_primitive_type()) {
1561 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1562 return false; // should be primitive type
1563 }
1564 BasicType elem_bt = elem_type->basic_type();
1565 int num_elem = vlen->get_con();
1566 BoolTest::mask booltest = (BoolTest::mask)cond->get_con();
1567 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1568 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1569
1570 if (!arch_supports_vector(Op_VectorTest, num_elem, elem_bt, is_vector_mask(vbox_klass) ? VecMaskUseLoad : VecMaskNotUsed)) {
1571 log_if_needed(" ** not supported: arity=2 op=test/%d vlen=%d etype=%s ismask=%d",
1572 cond->get_con(), num_elem, type2name(elem_bt),
1573 is_vector_mask(vbox_klass));
1574 return false;
1575 }
1576
1577 Node* opd1 = unbox_vector(argument(4), vbox_type, elem_bt, num_elem);
1578 Node* opd2;
1579 if (Matcher::vectortest_needs_second_argument(booltest == BoolTest::overflow,
1580 opd1->bottom_type()->isa_vectmask())) {
1581 opd2 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
1582 } else {
1583 opd2 = opd1;
1584 }
1585 if (opd1 == nullptr || opd2 == nullptr) {
1586 return false; // operand unboxing failed
1587 }
1588
1589 Node* cmp = gvn().transform(new VectorTestNode(opd1, opd2, booltest));
1590 BoolTest::mask test = Matcher::vectortest_mask(booltest == BoolTest::overflow,
1591 opd1->bottom_type()->isa_vectmask(), num_elem);
1592 Node* bol = gvn().transform(new BoolNode(cmp, test));
1593 Node* res = gvn().transform(new CMoveINode(bol, gvn().intcon(0), gvn().intcon(1), TypeInt::BOOL));
1594
1595 set_result(res);
1596 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1597 return true;
1598 }
1599
1600 // public static
1601 // <V extends Vector<E>,
1602 // M extends VectorMask<E>,
1603 // E>
1604 // V blend(Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType, int vlen,
1605 // V v1, V v2, M m,
1606 // VectorBlendOp<V, M, E> defaultImpl)
1607 bool LibraryCallKit::inline_vector_blend() {
1608 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
1609 const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
1610 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
1611 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
1612
1613 if (mask_klass == nullptr || vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr) {
1614 return false; // dead code
1615 }
1616 if (mask_klass->const_oop() == nullptr || vector_klass->const_oop() == nullptr ||
1617 elem_klass->const_oop() == nullptr || !vlen->is_con()) {
1618 log_if_needed(" ** missing constant: vclass=%s mclass=%s etype=%s vlen=%s",
1619 NodeClassNames[argument(0)->Opcode()],
1620 NodeClassNames[argument(1)->Opcode()],
1621 NodeClassNames[argument(2)->Opcode()],
1622 NodeClassNames[argument(3)->Opcode()]);
1623 return false; // not enough info for intrinsification
1624 }
1625 if (!is_klass_initialized(vector_klass) || !is_klass_initialized(mask_klass)) {
1626 log_if_needed(" ** klass argument not initialized");
1627 return false;
1628 }
1629 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1630 if (!elem_type->is_primitive_type()) {
1631 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1632 return false; // should be primitive type
1633 }
1634 BasicType elem_bt = elem_type->basic_type();
1635 BasicType mask_bt = elem_bt;
1636 int num_elem = vlen->get_con();
1637
1638 if (!arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad)) {
1639 log_if_needed(" ** not supported: arity=2 op=blend vlen=%d etype=%s ismask=useload",
1640 num_elem, type2name(elem_bt));
1641 return false; // not supported
1642 }
1643 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1644 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1645
1646 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1647 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1648
1649 Node* v1 = unbox_vector(argument(4), vbox_type, elem_bt, num_elem);
1650 Node* v2 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
1651 Node* mask = unbox_vector(argument(6), mbox_type, mask_bt, num_elem);
1652
1653 if (v1 == nullptr || v2 == nullptr || mask == nullptr) {
1654 return false; // operand unboxing failed
1655 }
1656
1657 Node* blend = gvn().transform(new VectorBlendNode(v1, v2, mask));
1658
1659 Node* box = box_vector(blend, vbox_type, elem_bt, num_elem);
1660 set_result(box);
1661 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1662 return true;
1663 }
1664
1665 // public static
1666 // <V extends Vector<E>,
1667 // M extends VectorMask<E>,
1668 // E>
1669 // M compare(int cond, Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType, int vlen,
1670 // V v1, V v2, M m,
1671 // VectorCompareOp<V,M> defaultImpl)
1672 bool LibraryCallKit::inline_vector_compare() {
1673 const TypeInt* cond = gvn().type(argument(0))->isa_int();
1674 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
1675 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
1676 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
1677 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
1678
1679 if (cond == nullptr || vector_klass == nullptr || mask_klass == nullptr || elem_klass == nullptr || vlen == nullptr) {
1680 return false; // dead code
1681 }
1682 if (!cond->is_con() || vector_klass->const_oop() == nullptr || mask_klass->const_oop() == nullptr ||
1683 elem_klass->const_oop() == nullptr || !vlen->is_con()) {
1684 log_if_needed(" ** missing constant: cond=%s vclass=%s mclass=%s etype=%s vlen=%s",
1685 NodeClassNames[argument(0)->Opcode()],
1686 NodeClassNames[argument(1)->Opcode()],
1687 NodeClassNames[argument(2)->Opcode()],
1688 NodeClassNames[argument(3)->Opcode()],
1689 NodeClassNames[argument(4)->Opcode()]);
1690 return false; // not enough info for intrinsification
1691 }
1692 if (!is_klass_initialized(vector_klass) || !is_klass_initialized(mask_klass)) {
1693 log_if_needed(" ** klass argument not initialized");
1694 return false;
1695 }
1696 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1697 if (!elem_type->is_primitive_type()) {
1698 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1699 return false; // should be primitive type
1700 }
1701
1702 int num_elem = vlen->get_con();
1703 BasicType elem_bt = elem_type->basic_type();
1704 BasicType mask_bt = elem_bt;
1705
1706 if ((cond->get_con() & BoolTest::unsigned_compare) != 0) {
1707 if (!Matcher::supports_vector_comparison_unsigned(num_elem, elem_bt)) {
1708 log_if_needed(" ** not supported: unsigned comparison op=comp/%d vlen=%d etype=%s ismask=usestore",
1709 cond->get_con() & (BoolTest::unsigned_compare - 1), num_elem, type2name(elem_bt));
1710 return false;
1711 }
1712 }
1713
1714 if (!arch_supports_vector(Op_VectorMaskCmp, num_elem, elem_bt, VecMaskUseStore)) {
1715 log_if_needed(" ** not supported: arity=2 op=comp/%d vlen=%d etype=%s ismask=usestore",
1716 cond->get_con(), num_elem, type2name(elem_bt));
1717 return false;
1718 }
1719
1720 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1721 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1722
1723 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1724 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1725
1726 Node* v1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
1727 Node* v2 = unbox_vector(argument(6), vbox_type, elem_bt, num_elem);
1728
1729 bool is_masked_op = argument(7)->bottom_type() != TypePtr::NULL_PTR;
1730 Node* mask = is_masked_op ? unbox_vector(argument(7), mbox_type, elem_bt, num_elem) : nullptr;
1731 if (is_masked_op && mask == nullptr) {
1732 log_if_needed(" ** not supported: mask = null arity=2 op=comp/%d vlen=%d etype=%s ismask=usestore is_masked_op=1",
1733 cond->get_con(), num_elem, type2name(elem_bt));
1734 return false;
1735 }
1736
1737 bool use_predicate = is_masked_op && arch_supports_vector(Op_VectorMaskCmp, num_elem, elem_bt, VecMaskUsePred);
1738 if (is_masked_op && !use_predicate && !arch_supports_vector(Op_AndV, num_elem, elem_bt, VecMaskUseLoad)) {
1739 log_if_needed(" ** not supported: arity=2 op=comp/%d vlen=%d etype=%s ismask=usestore is_masked_op=1",
1740 cond->get_con(), num_elem, type2name(elem_bt));
1741 return false;
1742 }
1743
1744 if (v1 == nullptr || v2 == nullptr) {
1745 return false; // operand unboxing failed
1746 }
1747 BoolTest::mask pred = (BoolTest::mask)cond->get_con();
1748 ConINode* pred_node = (ConINode*)gvn().makecon(cond);
1749
1750 const TypeVect* vmask_type = TypeVect::makemask(mask_bt, num_elem);
1751 Node* operation = new VectorMaskCmpNode(pred, v1, v2, pred_node, vmask_type);
1752
1753 if (is_masked_op) {
1754 if (use_predicate) {
1755 operation->add_req(mask);
1756 operation->add_flag(Node::Flag_is_predicated_vector);
1757 } else {
1758 operation = gvn().transform(operation);
1759 operation = VectorNode::make(Op_AndV, operation, mask, vmask_type);
1760 }
1761 }
1762
1763 operation = gvn().transform(operation);
1764
1765 Node* box = box_vector(operation, mbox_type, mask_bt, num_elem);
1766 set_result(box);
1767 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1768 return true;
1769 }
1770
1771 // public static
1772 // <V extends Vector<E>,
1773 // Sh extends VectorShuffle<E>,
1774 // M extends VectorMask<E>,
1775 // E>
1776 // V rearrangeOp(Class<? extends V> vectorClass, Class<Sh> shuffleClass, Class<M> maskClass, Class<E> elementType, int vlen,
1777 // V v1, Sh sh, M m,
1778 // VectorRearrangeOp<V, Sh, M, E> defaultImpl)
1779 bool LibraryCallKit::inline_vector_rearrange() {
1780 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
1781 const TypeInstPtr* shuffle_klass = gvn().type(argument(1))->isa_instptr();
1782 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
1783 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
1784 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
1785
1786 if (vector_klass == nullptr || shuffle_klass == nullptr || elem_klass == nullptr || vlen == nullptr) {
1787 return false; // dead code
1788 }
1789 if (shuffle_klass->const_oop() == nullptr ||
1790 vector_klass->const_oop() == nullptr ||
1791 elem_klass->const_oop() == nullptr ||
1792 !vlen->is_con()) {
1793 log_if_needed(" ** missing constant: vclass=%s sclass=%s etype=%s vlen=%s",
1794 NodeClassNames[argument(0)->Opcode()],
1795 NodeClassNames[argument(1)->Opcode()],
1796 NodeClassNames[argument(3)->Opcode()],
1797 NodeClassNames[argument(4)->Opcode()]);
1798 return false; // not enough info for intrinsification
1799 }
1800 if (!is_klass_initialized(vector_klass) ||
1801 !is_klass_initialized(shuffle_klass)) {
1802 log_if_needed(" ** klass argument not initialized");
1803 return false;
1804 }
1805 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1806 if (!elem_type->is_primitive_type()) {
1807 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1808 return false; // should be primitive type
1809 }
1810
1811 BasicType elem_bt = elem_type->basic_type();
1812 BasicType shuffle_bt = elem_bt;
1813 if (shuffle_bt == T_FLOAT) {
1814 shuffle_bt = T_INT;
1815 } else if (shuffle_bt == T_DOUBLE) {
1816 shuffle_bt = T_LONG;
1817 }
1818
1819 int num_elem = vlen->get_con();
1820 bool need_load_shuffle = Matcher::vector_rearrange_requires_load_shuffle(shuffle_bt, num_elem);
1821
1822 if (need_load_shuffle && !arch_supports_vector(Op_VectorLoadShuffle, num_elem, shuffle_bt, VecMaskNotUsed)) {
1823 if (C->print_intrinsics()) {
1824 tty->print_cr(" ** not supported: arity=0 op=load/shuffle vlen=%d etype=%s ismask=no",
1825 num_elem, type2name(shuffle_bt));
1826 }
1827 return false; // not supported
1828 }
1829
1830 bool is_masked_op = argument(7)->bottom_type() != TypePtr::NULL_PTR;
1831 bool use_predicate = is_masked_op;
1832 if (is_masked_op &&
1833 (mask_klass == nullptr ||
1834 mask_klass->const_oop() == nullptr ||
1835 !is_klass_initialized(mask_klass))) {
1836 log_if_needed(" ** mask_klass argument not initialized");
1837 }
1838 if (!arch_supports_vector(Op_AndV, num_elem, elem_bt, VecMaskNotUsed)) {
1839 log_if_needed(" ** not supported: arity=2 op=and vlen=%d etype=%s ismask=no",
1840 num_elem, type2name(elem_bt));
1841 return false;
1842 }
1843 VectorMaskUseType checkFlags = (VectorMaskUseType)(is_masked_op ? (VecMaskUseLoad | VecMaskUsePred) : VecMaskNotUsed);
1844 if (!arch_supports_vector(Op_VectorRearrange, num_elem, elem_bt, checkFlags)) {
1845 use_predicate = false;
1846 if(!is_masked_op ||
1847 (!arch_supports_vector(Op_VectorRearrange, num_elem, elem_bt, VecMaskNotUsed) ||
1848 !arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad) ||
1849 !arch_supports_vector(Op_Replicate, num_elem, elem_bt, VecMaskNotUsed))) {
1850 log_if_needed(" ** not supported: arity=2 op=shuffle/rearrange vlen=%d etype=%s ismask=no",
1851 num_elem, type2name(elem_bt));
1852 return false; // not supported
1853 }
1854 }
1855 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1856 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1857
1858 ciKlass* shbox_klass = shuffle_klass->const_oop()->as_instance()->java_lang_Class_klass();
1859 const TypeInstPtr* shbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, shbox_klass);
1860
1861 Node* v1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
1862 Node* shuffle = unbox_vector(argument(6), shbox_type, shuffle_bt, num_elem);
1863 const TypeVect* st = TypeVect::make(shuffle_bt, num_elem);
1864
1865 if (v1 == nullptr || shuffle == nullptr) {
1866 return false; // operand unboxing failed
1867 }
1868
1869 assert(is_power_of_2(num_elem), "wrapping invalid");
1870 Node* wrapping_mask_elem = gvn().makecon(TypeInteger::make(num_elem - 1, num_elem - 1, Type::WidenMin, shuffle_bt == T_LONG ? T_LONG : T_INT));
1871 Node* wrapping_mask = gvn().transform(VectorNode::scalar2vector(wrapping_mask_elem, num_elem, shuffle_bt));
1872 shuffle = gvn().transform(new AndVNode(shuffle, wrapping_mask, st));
1873
1874 Node* mask = nullptr;
1875 if (is_masked_op) {
1876 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1877 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1878 mask = unbox_vector(argument(7), mbox_type, elem_bt, num_elem);
1879 if (mask == nullptr) {
1880 log_if_needed(" ** not supported: arity=3 op=shuffle/rearrange vlen=%d etype=%s ismask=useload is_masked_op=1",
1881 num_elem, type2name(elem_bt));
1882 return false;
1883 }
1884 }
1885
1886 if (need_load_shuffle) {
1887 shuffle = gvn().transform(new VectorLoadShuffleNode(shuffle, st));
1888 }
1889
1890 Node* rearrange = new VectorRearrangeNode(v1, shuffle);
1891 if (is_masked_op) {
1892 if (use_predicate) {
1893 rearrange->add_req(mask);
1894 rearrange->add_flag(Node::Flag_is_predicated_vector);
1895 } else {
1896 rearrange = gvn().transform(rearrange);
1897 Node* zero = gvn().makecon(Type::get_zero_type(elem_bt));
1898 Node* zerovec = gvn().transform(VectorNode::scalar2vector(zero, num_elem, elem_bt));
1899 rearrange = new VectorBlendNode(zerovec, rearrange, mask);
1900 }
1901 }
1902 rearrange = gvn().transform(rearrange);
1903
1904 Node* box = box_vector(rearrange, vbox_type, elem_bt, num_elem);
1905 set_result(box);
1906 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1907 return true;
1908 }
1909
1910 // public static
1911 // <V extends Vector<E>,
1912 // M extends VectorMask<E>,
1913 // E>
1914 // V selectFromOp(Class<? extends V> vClass, Class<M> mClass, Class<E> eClass,
1915 // int length, V v1, V v2, M m,
1916 // VectorSelectFromOp<V, M> defaultImpl)
1917 bool LibraryCallKit::inline_vector_select_from() {
1918 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
1919 const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
1920 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
1921 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
1922
1923 if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr ||
1924 vector_klass->const_oop() == nullptr ||
1925 elem_klass->const_oop() == nullptr ||
1926 !vlen->is_con()) {
1927 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s",
1928 NodeClassNames[argument(0)->Opcode()],
1929 NodeClassNames[argument(2)->Opcode()],
1930 NodeClassNames[argument(3)->Opcode()]);
1931 return false; // not enough info for intrinsification
1932 }
1933 if (!is_klass_initialized(vector_klass)) {
1934 log_if_needed(" ** klass argument not initialized");
1935 return false;
1936 }
1937 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1938 if (!elem_type->is_primitive_type()) {
1939 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1940 return false; // should be primitive type
1941 }
1942 BasicType elem_bt = elem_type->basic_type();
1943 int num_elem = vlen->get_con();
1944 if (!is_power_of_2(num_elem)) {
1945 log_if_needed(" ** vlen not power of two=%d", num_elem);
1946 return false;
1947 }
1948
1949 BasicType shuffle_bt = elem_bt;
1950 if (shuffle_bt == T_FLOAT) {
1951 shuffle_bt = T_INT;
1952 } else if (shuffle_bt == T_DOUBLE) {
1953 shuffle_bt = T_LONG;
1954 }
1955 bool need_load_shuffle = Matcher::vector_rearrange_requires_load_shuffle(shuffle_bt, num_elem);
1956
1957 int cast_vopc = VectorCastNode::opcode(-1, elem_bt); // from vector of type elem_bt
1958 if ((need_load_shuffle && !arch_supports_vector(Op_VectorLoadShuffle, num_elem, elem_bt, VecMaskNotUsed)) ||
1959 (elem_bt != shuffle_bt && !arch_supports_vector(cast_vopc, num_elem, shuffle_bt, VecMaskNotUsed)) ||
1960 !arch_supports_vector(Op_AndV, num_elem, shuffle_bt, VecMaskNotUsed) ||
1961 !arch_supports_vector(Op_Replicate, num_elem, shuffle_bt, VecMaskNotUsed)) {
1962 log_if_needed(" ** not supported: arity=0 op=selectFrom vlen=%d etype=%s ismask=no",
1963 num_elem, type2name(elem_bt));
1964 return false; // not supported
1965 }
1966
1967 bool is_masked_op = argument(6)->bottom_type() != TypePtr::NULL_PTR;
1968 bool use_predicate = is_masked_op;
1969 if (is_masked_op &&
1970 (mask_klass == nullptr ||
1971 mask_klass->const_oop() == nullptr ||
1972 !is_klass_initialized(mask_klass))) {
1973 log_if_needed(" ** mask_klass argument not initialized");
1974 return false; // not supported
1975 }
1976 VectorMaskUseType checkFlags = (VectorMaskUseType)(is_masked_op ? (VecMaskUseLoad | VecMaskUsePred) : VecMaskNotUsed);
1977 if (!arch_supports_vector(Op_VectorRearrange, num_elem, elem_bt, checkFlags)) {
1978 use_predicate = false;
1979 if(!is_masked_op ||
1980 (!arch_supports_vector(Op_VectorRearrange, num_elem, elem_bt, VecMaskNotUsed) ||
1981 !arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad) ||
1982 !arch_supports_vector(Op_Replicate, num_elem, elem_bt, VecMaskNotUsed))) {
1983 log_if_needed(" ** not supported: op=selectFrom vlen=%d etype=%s is_masked_op=%d",
1984 num_elem, type2name(elem_bt), is_masked_op);
1985 return false; // not supported
1986 }
1987 }
1988 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1989 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1990
1991 // v1 is the index vector
1992 Node* v1 = unbox_vector(argument(4), vbox_type, elem_bt, num_elem);
1993 // v2 is the vector being rearranged
1994 Node* v2 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
1995
1996 if (v1 == nullptr) {
1997 log_if_needed(" ** unbox failed v1=%s", NodeClassNames[argument(4)->Opcode()]);
1998 return false; // operand unboxing failed
1999 }
2000
2001 if (v2 == nullptr) {
2002 log_if_needed(" ** unbox failed v2=%s", NodeClassNames[argument(5)->Opcode()]);
2003 return false; // operand unboxing failed
2004 }
2005
2006 Node* mask = nullptr;
2007 if (is_masked_op) {
2008 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
2009 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
2010 mask = unbox_vector(argument(6), mbox_type, elem_bt, num_elem);
2011 if (mask == nullptr) {
2012 log_if_needed(" ** unbox failed mask=%s", NodeClassNames[argument(6)->Opcode()]);
2013 return false;
2014 }
2015 }
2016
2017 // cast index vector from elem_bt vector to byte vector
2018 const TypeVect* shuffle_vt = TypeVect::make(shuffle_bt, num_elem);
2019 Node* shuffle = v1;
2020
2021 if (shuffle_bt != elem_bt) {
2022 shuffle = gvn().transform(VectorCastNode::make(cast_vopc, v1, shuffle_bt, num_elem));
2023 }
2024
2025 // wrap the byte vector lanes to (num_elem - 1) to form the shuffle vector where num_elem is vector length
2026 // this is a simple AND operation as we come here only for power of two vector length
2027 Node* mod_val = gvn().makecon(TypeInteger::make(num_elem - 1, num_elem - 1, Type::WidenMin, shuffle_bt == T_LONG ? T_LONG : T_INT));
2028 Node* bcast_mod = gvn().transform(VectorNode::scalar2vector(mod_val, num_elem, shuffle_bt));
2029 shuffle = gvn().transform(VectorNode::make(Op_AndV, shuffle, bcast_mod, shuffle_vt));
2030
2031 // load the shuffle to use in rearrange
2032 if (need_load_shuffle) {
2033 shuffle = gvn().transform(new VectorLoadShuffleNode(shuffle, shuffle_vt));
2034 }
2035
2036 // and finally rearrange
2037 Node* rearrange = new VectorRearrangeNode(v2, shuffle);
2038 if (is_masked_op) {
2039 if (use_predicate) {
2040 // masked rearrange is supported so use that directly
2041 rearrange->add_req(mask);
2042 rearrange->add_flag(Node::Flag_is_predicated_vector);
2043 } else {
2044 // masked rearrange is not supported so emulate usig blend
2045 const TypeVect* vt = v1->bottom_type()->is_vect();
2046 rearrange = gvn().transform(rearrange);
2047
2048 // create a zero vector with each lane element set as zero
2049 Node* zero = gvn().makecon(Type::get_zero_type(elem_bt));
2050 Node* zerovec = gvn().transform(VectorNode::scalar2vector(zero, num_elem, elem_bt));
2051
2052 // For each lane for which mask is set, blend in the rearranged lane into zero vector
2053 rearrange = new VectorBlendNode(zerovec, rearrange, mask);
2054 }
2055 }
2056 rearrange = gvn().transform(rearrange);
2057
2058 // box the result
2059 Node* box = box_vector(rearrange, vbox_type, elem_bt, num_elem);
2060 set_result(box);
2061
2062 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2063 return true;
2064 }
2065
2066 // public static
2067 // <V extends Vector<E>,
2068 // M extends VectorMask<E>,
2069 // E>
2070 // V broadcastInt(int opr, Class<? extends V> vectorClass, Class<? extends M> maskClass,
2071 // Class<E> elementType, int length,
2072 // V v, int n, M m,
2073 // VectorBroadcastIntOp<V, M> defaultImpl)
2074 bool LibraryCallKit::inline_vector_broadcast_int() {
2075 const TypeInt* opr = gvn().type(argument(0))->isa_int();
2076 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
2077 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
2078 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
2079 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
2080
2081 if (opr == nullptr || vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr) {
2082 return false; // dead code
2083 }
2084 if (!opr->is_con() || vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
2085 log_if_needed(" ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
2086 NodeClassNames[argument(0)->Opcode()],
2087 NodeClassNames[argument(1)->Opcode()],
2088 NodeClassNames[argument(3)->Opcode()],
2089 NodeClassNames[argument(4)->Opcode()]);
2090 return false; // not enough info for intrinsification
2091 }
2092 if (!is_klass_initialized(vector_klass)) {
2093 log_if_needed(" ** klass argument not initialized");
2094 return false;
2095 }
2096
2097 const Type* vmask_type = gvn().type(argument(7));
2098 bool is_masked_op = vmask_type != TypePtr::NULL_PTR;
2099 if (is_masked_op) {
2100 if (mask_klass == nullptr || mask_klass->const_oop() == nullptr) {
2101 log_if_needed(" ** missing constant: maskclass=%s", NodeClassNames[argument(2)->Opcode()]);
2102 return false; // not enough info for intrinsification
2103 }
2104
2105 if (!is_klass_initialized(mask_klass)) {
2106 log_if_needed(" ** mask klass argument not initialized");
2107 return false;
2108 }
2109
2110 if (vmask_type->maybe_null()) {
2111 log_if_needed(" ** null mask values are not allowed for masked op");
2112 return false;
2113 }
2114 }
2115
2116 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2117 if (!elem_type->is_primitive_type()) {
2118 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2119 return false; // should be primitive type
2120 }
2121
2122 int num_elem = vlen->get_con();
2123 BasicType elem_bt = elem_type->basic_type();
2124 int opc = VectorSupport::vop2ideal(opr->get_con(), elem_bt);
2125
2126 bool is_shift = VectorNode::is_shift_opcode(opc);
2127 bool is_rotate = VectorNode::is_rotate_opcode(opc);
2128
2129 if (opc == 0 || (!is_shift && !is_rotate)) {
2130 log_if_needed(" ** operation not supported: op=%d bt=%s", opr->get_con(), type2name(elem_bt));
2131 return false; // operation not supported
2132 }
2133
2134 int sopc = VectorNode::opcode(opc, elem_bt);
2135 if (sopc == 0) {
2136 log_if_needed(" ** operation not supported: opc=%s bt=%s", NodeClassNames[opc], type2name(elem_bt));
2137 return false; // operation not supported
2138 }
2139
2140 Node* cnt = argument(6);
2141 const TypeInt* cnt_type = cnt->bottom_type()->isa_int();
2142
2143 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2144 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2145
2146 // If CPU supports vector constant rotate instructions pass it directly
2147 bool is_const_rotate = is_rotate && cnt_type && cnt_type->is_con() &&
2148 Matcher::supports_vector_constant_rotates(cnt_type->get_con());
2149 bool has_scalar_args = is_rotate ? !is_const_rotate : true;
2150
2151 VectorMaskUseType checkFlags = (VectorMaskUseType)(is_masked_op ? (VecMaskUseLoad | VecMaskUsePred) : VecMaskNotUsed);
2152 bool use_predicate = is_masked_op;
2153
2154 if (!arch_supports_vector(sopc, num_elem, elem_bt, checkFlags, has_scalar_args)) {
2155 use_predicate = false;
2156 if (!is_masked_op ||
2157 (!arch_supports_vector(sopc, num_elem, elem_bt, VecMaskNotUsed, has_scalar_args) ||
2158 !arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad))) {
2159
2160 log_if_needed(" ** not supported: arity=0 op=int/%d vlen=%d etype=%s is_masked_op=%d",
2161 sopc, num_elem, type2name(elem_bt), is_masked_op ? 1 : 0);
2162 return false; // not supported
2163 }
2164 }
2165
2166 Node* opd1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
2167 Node* opd2 = nullptr;
2168 if (is_shift) {
2169 opd2 = vector_shift_count(cnt, opc, elem_bt, num_elem);
2170 } else {
2171 assert(is_rotate, "unexpected operation");
2172 if (!is_const_rotate) {
2173 cnt = elem_bt == T_LONG ? gvn().transform(new ConvI2LNode(cnt)) : cnt;
2174 opd2 = gvn().transform(VectorNode::scalar2vector(cnt, num_elem, elem_bt));
2175 } else {
2176 // Constant shift value.
2177 opd2 = cnt;
2178 }
2179 }
2180
2181 if (opd1 == nullptr || opd2 == nullptr) {
2182 return false;
2183 }
2184
2185 Node* mask = nullptr;
2186 if (is_masked_op) {
2187 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
2188 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
2189 mask = unbox_vector(argument(7), mbox_type, elem_bt, num_elem);
2190 if (mask == nullptr) {
2191 log_if_needed(" ** unbox failed mask=%s", NodeClassNames[argument(7)->Opcode()]);
2192 return false;
2193 }
2194 }
2195
2196 Node* operation = VectorNode::make(opc, opd1, opd2, num_elem, elem_bt);
2197 if (is_masked_op && mask != nullptr) {
2198 if (use_predicate) {
2199 operation->add_req(mask);
2200 operation->add_flag(Node::Flag_is_predicated_vector);
2201 } else {
2202 operation = gvn().transform(operation);
2203 operation = new VectorBlendNode(opd1, operation, mask);
2204 }
2205 }
2206 operation = gvn().transform(operation);
2207 Node* vbox = box_vector(operation, vbox_type, elem_bt, num_elem);
2208 set_result(vbox);
2209 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2210 return true;
2211 }
2212
2213 // public static <VOUT extends VectorPayload,
2214 // VIN extends VectorPayload,
2215 // S extends VectorSpecies>
2216 // VOUT convert(int oprId,
2217 // Class<?> fromVectorClass, Class<?> fromElementType, int fromVLen,
2218 // Class<?> toVectorClass, Class<?> toElementType, int toVLen,
2219 // VIN v, S s,
2220 // VectorConvertOp<VOUT, VIN, S> defaultImpl)
2221 //
2222 bool LibraryCallKit::inline_vector_convert() {
2223 const TypeInt* opr = gvn().type(argument(0))->isa_int();
2224
2225 const TypeInstPtr* vector_klass_from = gvn().type(argument(1))->isa_instptr();
2226 const TypeInstPtr* elem_klass_from = gvn().type(argument(2))->isa_instptr();
2227 const TypeInt* vlen_from = gvn().type(argument(3))->isa_int();
2228
2229 const TypeInstPtr* vector_klass_to = gvn().type(argument(4))->isa_instptr();
2230 const TypeInstPtr* elem_klass_to = gvn().type(argument(5))->isa_instptr();
2231 const TypeInt* vlen_to = gvn().type(argument(6))->isa_int();
2232
2233 if (opr == nullptr ||
2234 vector_klass_from == nullptr || elem_klass_from == nullptr || vlen_from == nullptr ||
2235 vector_klass_to == nullptr || elem_klass_to == nullptr || vlen_to == nullptr) {
2236 return false; // dead code
2237 }
2238 if (!opr->is_con() ||
2239 vector_klass_from->const_oop() == nullptr || elem_klass_from->const_oop() == nullptr || !vlen_from->is_con() ||
2240 vector_klass_to->const_oop() == nullptr || elem_klass_to->const_oop() == nullptr || !vlen_to->is_con()) {
2241 log_if_needed(" ** missing constant: opr=%s vclass_from=%s etype_from=%s vlen_from=%s vclass_to=%s etype_to=%s vlen_to=%s",
2242 NodeClassNames[argument(0)->Opcode()],
2243 NodeClassNames[argument(1)->Opcode()],
2244 NodeClassNames[argument(2)->Opcode()],
2245 NodeClassNames[argument(3)->Opcode()],
2246 NodeClassNames[argument(4)->Opcode()],
2247 NodeClassNames[argument(5)->Opcode()],
2248 NodeClassNames[argument(6)->Opcode()]);
2249 return false; // not enough info for intrinsification
2250 }
2251 if (!is_klass_initialized(vector_klass_from) || !is_klass_initialized(vector_klass_to)) {
2252 log_if_needed(" ** klass argument not initialized");
2253 return false;
2254 }
2255
2256 assert(opr->get_con() == VectorSupport::VECTOR_OP_CAST ||
2257 opr->get_con() == VectorSupport::VECTOR_OP_UCAST ||
2258 opr->get_con() == VectorSupport::VECTOR_OP_REINTERPRET, "wrong opcode");
2259 bool is_cast = (opr->get_con() == VectorSupport::VECTOR_OP_CAST || opr->get_con() == VectorSupport::VECTOR_OP_UCAST);
2260 bool is_ucast = (opr->get_con() == VectorSupport::VECTOR_OP_UCAST);
2261
2262 ciKlass* vbox_klass_from = vector_klass_from->const_oop()->as_instance()->java_lang_Class_klass();
2263 ciKlass* vbox_klass_to = vector_klass_to->const_oop()->as_instance()->java_lang_Class_klass();
2264
2265 bool is_mask = is_vector_mask(vbox_klass_from);
2266
2267 ciType* elem_type_from = elem_klass_from->const_oop()->as_instance()->java_mirror_type();
2268 if (!elem_type_from->is_primitive_type()) {
2269 return false; // should be primitive type
2270 }
2271 BasicType elem_bt_from = elem_type_from->basic_type();
2272 ciType* elem_type_to = elem_klass_to->const_oop()->as_instance()->java_mirror_type();
2273 if (!elem_type_to->is_primitive_type()) {
2274 return false; // should be primitive type
2275 }
2276 BasicType elem_bt_to = elem_type_to->basic_type();
2277
2278 int num_elem_from = vlen_from->get_con();
2279 int num_elem_to = vlen_to->get_con();
2280
2281 // Check whether we can unbox to appropriate size. Even with casting, checking for reinterpret is needed
2282 // since we may need to change size.
2283 if (!arch_supports_vector(Op_VectorReinterpret,
2284 num_elem_from,
2285 elem_bt_from,
2286 is_mask ? VecMaskUseAll : VecMaskNotUsed)) {
2287 log_if_needed(" ** not supported: arity=1 op=%s/1 vlen1=%d etype1=%s ismask=%d",
2288 is_cast ? "cast" : "reinterpret",
2289 num_elem_from, type2name(elem_bt_from), is_mask);
2290 return false;
2291 }
2292
2293 // Check whether we can support resizing/reinterpreting to the new size.
2294 if (!arch_supports_vector(Op_VectorReinterpret,
2295 num_elem_to,
2296 elem_bt_to,
2297 is_mask ? VecMaskUseAll : VecMaskNotUsed)) {
2298 log_if_needed(" ** not supported: arity=1 op=%s/2 vlen2=%d etype2=%s ismask=%d",
2299 is_cast ? "cast" : "reinterpret",
2300 num_elem_to, type2name(elem_bt_to), is_mask);
2301 return false;
2302 }
2303
2304 // At this point, we know that both input and output vector registers are supported
2305 // by the architecture. Next check if the casted type is simply to same type - which means
2306 // that it is actually a resize and not a cast.
2307 if (is_cast && elem_bt_from == elem_bt_to) {
2308 is_cast = false;
2309 }
2310
2311 const TypeInstPtr* vbox_type_from = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass_from);
2312
2313 Node* opd1 = unbox_vector(argument(7), vbox_type_from, elem_bt_from, num_elem_from);
2314 if (opd1 == nullptr) {
2315 return false;
2316 }
2317
2318 const TypeVect* src_type = TypeVect::make(elem_bt_from, num_elem_from, is_mask);
2319 const TypeVect* dst_type = TypeVect::make(elem_bt_to, num_elem_to, is_mask);
2320
2321 // Safety check to prevent casting if source mask is of type vector
2322 // and destination mask of type predicate vector and vice-versa.
2323 // From X86 standpoint, this case will only arise over KNL target,
2324 // where certain masks (depending on the species) are either propagated
2325 // through a vector or predicate register.
2326 if (is_mask &&
2327 ((src_type->isa_vectmask() == nullptr && dst_type->isa_vectmask()) ||
2328 (dst_type->isa_vectmask() == nullptr && src_type->isa_vectmask()))) {
2329 return false;
2330 }
2331
2332 Node* op = opd1;
2333 if (is_cast) {
2334 assert(!is_mask || num_elem_from == num_elem_to, "vector mask cast needs the same elem num");
2335 int cast_vopc = VectorCastNode::opcode(-1, elem_bt_from, !is_ucast);
2336
2337 // Make sure that vector cast is implemented to particular type/size combination if it is
2338 // not a mask casting.
2339 if (!is_mask && !arch_supports_vector(cast_vopc, num_elem_to, elem_bt_to, VecMaskNotUsed)) {
2340 log_if_needed(" ** not supported: arity=1 op=cast#%d/3 vlen2=%d etype2=%s ismask=%d",
2341 cast_vopc, num_elem_to, type2name(elem_bt_to), is_mask);
2342 return false;
2343 }
2344
2345 if (num_elem_from < num_elem_to) {
2346 // Since input and output number of elements are not consistent, we need to make sure we
2347 // properly size. Thus, first make a cast that retains the number of elements from source.
2348 int num_elem_for_cast = num_elem_from;
2349
2350 // It is possible that arch does not support this intermediate vector size
2351 // TODO More complex logic required here to handle this corner case for the sizes.
2352 if (!arch_supports_vector(cast_vopc, num_elem_for_cast, elem_bt_to, VecMaskNotUsed)) {
2353 log_if_needed(" ** not supported: arity=1 op=cast#%d/4 vlen1=%d etype2=%s ismask=%d",
2354 cast_vopc,
2355 num_elem_for_cast, type2name(elem_bt_to), is_mask);
2356 return false;
2357 }
2358
2359 op = gvn().transform(VectorCastNode::make(cast_vopc, op, elem_bt_to, num_elem_for_cast));
2360 // Now ensure that the destination gets properly resized to needed size.
2361 op = gvn().transform(new VectorReinterpretNode(op, op->bottom_type()->is_vect(), dst_type));
2362 } else if (num_elem_from > num_elem_to) {
2363 // Since number of elements from input is larger than output, simply reduce size of input
2364 // (we are supposed to drop top elements anyway).
2365 int num_elem_for_resize = num_elem_to;
2366
2367 // It is possible that arch does not support this intermediate vector size
2368 // TODO More complex logic required here to handle this corner case for the sizes.
2369 if (!arch_supports_vector(Op_VectorReinterpret,
2370 num_elem_for_resize,
2371 elem_bt_from,
2372 VecMaskNotUsed)) {
2373 log_if_needed(" ** not supported: arity=1 op=cast/5 vlen2=%d etype1=%s ismask=%d",
2374 num_elem_for_resize, type2name(elem_bt_from), is_mask);
2375 return false;
2376 }
2377
2378 const TypeVect* resize_type = TypeVect::make(elem_bt_from, num_elem_for_resize);
2379 op = gvn().transform(new VectorReinterpretNode(op, src_type, resize_type));
2380 op = gvn().transform(VectorCastNode::make(cast_vopc, op, elem_bt_to, num_elem_to));
2381 } else { // num_elem_from == num_elem_to
2382 if (is_mask) {
2383 // Make sure that cast for vector mask is implemented to particular type/size combination.
2384 if (!arch_supports_vector(Op_VectorMaskCast, num_elem_to, elem_bt_to, VecMaskNotUsed)) {
2385 log_if_needed(" ** not supported: arity=1 op=maskcast vlen2=%d etype2=%s ismask=%d",
2386 num_elem_to, type2name(elem_bt_to), is_mask);
2387 return false;
2388 }
2389 op = gvn().transform(new VectorMaskCastNode(op, dst_type));
2390 } else {
2391 // Since input and output number of elements match, and since we know this vector size is
2392 // supported, simply do a cast with no resize needed.
2393 op = gvn().transform(VectorCastNode::make(cast_vopc, op, elem_bt_to, num_elem_to));
2394 }
2395 }
2396 } else if (!Type::equals(src_type, dst_type)) {
2397 assert(!is_cast, "must be reinterpret");
2398 op = gvn().transform(new VectorReinterpretNode(op, src_type, dst_type));
2399 }
2400
2401 const TypeInstPtr* vbox_type_to = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass_to);
2402 Node* vbox = box_vector(op, vbox_type_to, elem_bt_to, num_elem_to);
2403 set_result(vbox);
2404 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem_to * type2aelembytes(elem_bt_to))));
2405 return true;
2406 }
2407
2408 // public static
2409 // <V extends Vector<E>,
2410 // E>
2411 // V insert(Class<? extends V> vectorClass, Class<E> elementType, int vlen,
2412 // V vec, int ix, long val,
2413 // VecInsertOp<V> defaultImpl)
2414 bool LibraryCallKit::inline_vector_insert() {
2415 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
2416 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
2417 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
2418 const TypeInt* idx = gvn().type(argument(4))->isa_int();
2419
2420 if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr || idx == nullptr) {
2421 return false; // dead code
2422 }
2423 if (vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con() || !idx->is_con()) {
2424 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s idx=%s",
2425 NodeClassNames[argument(0)->Opcode()],
2426 NodeClassNames[argument(1)->Opcode()],
2427 NodeClassNames[argument(2)->Opcode()],
2428 NodeClassNames[argument(4)->Opcode()]);
2429 return false; // not enough info for intrinsification
2430 }
2431 if (!is_klass_initialized(vector_klass)) {
2432 log_if_needed(" ** klass argument not initialized");
2433 return false;
2434 }
2435 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2436 if (!elem_type->is_primitive_type()) {
2437 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2438 return false; // should be primitive type
2439 }
2440 BasicType elem_bt = elem_type->basic_type();
2441 int num_elem = vlen->get_con();
2442 if (!arch_supports_vector(Op_VectorInsert, num_elem, elem_bt, VecMaskNotUsed)) {
2443 log_if_needed(" ** not supported: arity=1 op=insert vlen=%d etype=%s ismask=no",
2444 num_elem, type2name(elem_bt));
2445 return false; // not supported
2446 }
2447
2448 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2449 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2450
2451 Node* opd = unbox_vector(argument(3), vbox_type, elem_bt, num_elem);
2452 if (opd == nullptr) {
2453 return false;
2454 }
2455
2456 Node* insert_val = argument(5);
2457 assert(gvn().type(insert_val)->isa_long() != nullptr, "expected to be long");
2458
2459 // Convert insert value back to its appropriate type.
2460 switch (elem_bt) {
2461 case T_BYTE:
2462 insert_val = gvn().transform(new ConvL2INode(insert_val, TypeInt::BYTE));
2463 break;
2464 case T_SHORT:
2465 insert_val = gvn().transform(new ConvL2INode(insert_val, TypeInt::SHORT));
2466 break;
2467 case T_INT:
2468 insert_val = gvn().transform(new ConvL2INode(insert_val));
2469 break;
2470 case T_FLOAT:
2471 insert_val = gvn().transform(new ConvL2INode(insert_val));
2472 insert_val = gvn().transform(new MoveI2FNode(insert_val));
2473 break;
2474 case T_DOUBLE:
2475 insert_val = gvn().transform(new MoveL2DNode(insert_val));
2476 break;
2477 case T_LONG:
2478 // no conversion needed
2479 break;
2480 default: fatal("%s", type2name(elem_bt)); break;
2481 }
2482
2483 Node* operation = gvn().transform(VectorInsertNode::make(opd, insert_val, idx->get_con(), gvn()));
2484
2485 Node* vbox = box_vector(operation, vbox_type, elem_bt, num_elem);
2486 set_result(vbox);
2487 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2488 return true;
2489 }
2490
2491 // public static
2492 // <VM extends VectorPayload,
2493 // E>
2494 // long extract(Class<? extends VM> vClass, Class<E> eClass,
2495 // int length,
2496 // VM vm, int i,
2497 // VecExtractOp<VM> defaultImpl)
2498 bool LibraryCallKit::inline_vector_extract() {
2499 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
2500 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
2501 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
2502 const TypeInt* idx = gvn().type(argument(4))->isa_int();
2503
2504 if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr || idx == nullptr) {
2505 return false; // dead code
2506 }
2507 if (vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con() || !idx->is_con()) {
2508 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s",
2509 NodeClassNames[argument(0)->Opcode()],
2510 NodeClassNames[argument(1)->Opcode()],
2511 NodeClassNames[argument(2)->Opcode()]);
2512 return false; // not enough info for intrinsification
2513 }
2514 if (!is_klass_initialized(vector_klass)) {
2515 log_if_needed(" ** klass argument not initialized");
2516 return false;
2517 }
2518 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2519 if (!elem_type->is_primitive_type()) {
2520 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2521 return false; // should be primitive type
2522 }
2523 BasicType elem_bt = elem_type->basic_type();
2524 int num_elem = vlen->get_con();
2525
2526 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2527 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2528
2529 Node* opd = nullptr;
2530
2531 if (is_vector_mask(vbox_klass)) {
2532 // vbox_klass is mask. This is used for VectorMask.laneIsSet(int).
2533
2534 Node* pos = argument(4); // can be variable
2535 if (arch_supports_vector(Op_ExtractUB, num_elem, elem_bt, VecMaskUseAll)) {
2536 // Transform mask to vector with type of boolean and utilize ExtractUB node.
2537 opd = unbox_vector(argument(3), vbox_type, elem_bt, num_elem);
2538 if (opd == nullptr) {
2539 return false;
2540 }
2541 opd = gvn().transform(VectorStoreMaskNode::make(gvn(), opd, elem_bt, num_elem));
2542 opd = gvn().transform(new ExtractUBNode(opd, pos));
2543 opd = gvn().transform(new ConvI2LNode(opd));
2544 } else if (arch_supports_vector(Op_VectorMaskToLong, num_elem, elem_bt, VecMaskUseLoad)) {
2545 opd = unbox_vector(argument(3), vbox_type, elem_bt, num_elem);
2546 if (opd == nullptr) {
2547 return false;
2548 }
2549 // VectorMaskToLongNode requires the input is either a mask or a vector with BOOLEAN type.
2550 if (opd->bottom_type()->isa_vectmask() == nullptr) {
2551 opd = gvn().transform(VectorStoreMaskNode::make(gvn(), opd, elem_bt, num_elem));
2552 }
2553 // ((toLong() >>> pos) & 1L
2554 opd = gvn().transform(new VectorMaskToLongNode(opd, TypeLong::LONG));
2555 opd = gvn().transform(new URShiftLNode(opd, pos));
2556 opd = gvn().transform(new AndLNode(opd, gvn().makecon(TypeLong::ONE)));
2557 } else {
2558 log_if_needed(" ** Rejected mask extraction because architecture does not support it");
2559 return false; // not supported
2560 }
2561 } else {
2562 // vbox_klass is vector. This is used for Vector.lane(int).
2563 if (!idx->is_con()) {
2564 log_if_needed(" ** missing constant: idx=%s", NodeClassNames[argument(4)->Opcode()]);
2565 return false; // not enough info for intrinsification
2566 }
2567
2568 int vopc = ExtractNode::opcode(elem_bt);
2569 if (!arch_supports_vector(vopc, num_elem, elem_bt, VecMaskNotUsed)) {
2570 log_if_needed(" ** not supported: arity=1 op=extract vlen=%d etype=%s ismask=no",
2571 num_elem, type2name(elem_bt));
2572 return false; // not supported
2573 }
2574
2575 opd = unbox_vector(argument(3), vbox_type, elem_bt, num_elem);
2576 if (opd == nullptr) {
2577 return false;
2578 }
2579 ConINode* idx_con = gvn().intcon(idx->get_con())->as_ConI();
2580
2581 opd = gvn().transform(ExtractNode::make(opd, idx_con, elem_bt));
2582 switch (elem_bt) {
2583 case T_BYTE:
2584 case T_SHORT:
2585 case T_INT: {
2586 opd = gvn().transform(new ConvI2LNode(opd));
2587 break;
2588 }
2589 case T_FLOAT: {
2590 opd = gvn().transform(new MoveF2INode(opd));
2591 opd = gvn().transform(new ConvI2LNode(opd));
2592 break;
2593 }
2594 case T_DOUBLE: {
2595 opd = gvn().transform(new MoveD2LNode(opd));
2596 break;
2597 }
2598 case T_LONG: {
2599 // no conversion needed
2600 break;
2601 }
2602 default: fatal("%s", type2name(elem_bt));
2603 }
2604 }
2605 set_result(opd);
2606 return true;
2607 }
2608
2609 static Node* LowerSelectFromTwoVectorOperation(PhaseGVN& phase, Node* index_vec, Node* src1, Node* src2, const TypeVect* vt) {
2610 int num_elem = vt->length();
2611 BasicType elem_bt = vt->element_basic_type();
2612
2613 // Lower selectFrom operation into its constituent operations.
2614 // SelectFromTwoVectorNode =
2615 // (VectorBlend
2616 // (VectorRearrange SRC1 (WRAPED_INDEX AND (VLEN-1))
2617 // (VectorRearrange SRC2 (WRAPED_INDEX AND (VLEN-1))
2618 // MASK)
2619 // Where
2620 // WRAPED_INDEX are computed by wrapping incoming indexes
2621 // to two vector index range [0, VLEN*2) and
2622 // MASK = WRAPED_INDEX < VLEN
2623 //
2624 // IR lowering prevents intrinsification failure and associated argument
2625 // boxing penalties.
2626 //
2627
2628 BasicType shuffle_bt = elem_bt;
2629 if (shuffle_bt == T_FLOAT) {
2630 shuffle_bt = T_INT;
2631 } else if (shuffle_bt == T_DOUBLE) {
2632 shuffle_bt = T_LONG;
2633 }
2634 const TypeVect* st = TypeVect::make(shuffle_bt, num_elem);
2635
2636 // Cast index vector to the corresponding bit type
2637 if (elem_bt != shuffle_bt) {
2638 int cast_vopc = VectorCastNode::opcode(0, elem_bt, true);
2639 index_vec = phase.transform(VectorCastNode::make(cast_vopc, index_vec, shuffle_bt, num_elem));
2640 }
2641
2642 // Wrap indexes into two vector index range [0, VLEN * 2)
2643 Node* two_vect_lane_cnt_m1 = phase.makecon(TypeInteger::make(2 * num_elem - 1, 2 * num_elem - 1, Type::WidenMin, shuffle_bt == T_LONG ? T_LONG : T_INT));
2644 Node* bcast_two_vect_lane_cnt_m1_vec = phase.transform(VectorNode::scalar2vector(two_vect_lane_cnt_m1, num_elem,
2645 shuffle_bt, false));
2646 index_vec = phase.transform(VectorNode::make(Op_AndV, index_vec, bcast_two_vect_lane_cnt_m1_vec, st));
2647
2648 // Compute the blend mask for merging two independently permitted vectors
2649 // using shuffle index in two vector index range [0, VLEN * 2).
2650 BoolTest::mask pred = BoolTest::le;
2651 ConINode* pred_node = phase.makecon(TypeInt::make(pred))->as_ConI();
2652 const TypeVect* vmask_type = TypeVect::makemask(shuffle_bt, num_elem);
2653 Node* lane_cnt_m1 = phase.makecon(TypeInteger::make(num_elem - 1, num_elem - 1, Type::WidenMin, shuffle_bt == T_LONG ? T_LONG : T_INT));
2654 Node* bcast_lane_cnt_m1_vec = phase.transform(VectorNode::scalar2vector(lane_cnt_m1, num_elem, shuffle_bt, false));
2655 Node* mask = phase.transform(new VectorMaskCmpNode(pred, index_vec, bcast_lane_cnt_m1_vec, pred_node, vmask_type));
2656
2657 // Rearrange expects the indexes to lie within single vector index range [0, VLEN).
2658 Node* wrapped_index_vec = phase.transform(VectorNode::make(Op_AndV, index_vec, bcast_lane_cnt_m1_vec, st));
2659
2660 // Load indexes from byte vector and appropriately transform them to target
2661 // specific permutation index format.
2662 if (Matcher::vector_rearrange_requires_load_shuffle(shuffle_bt, num_elem)) {
2663 wrapped_index_vec = phase.transform(new VectorLoadShuffleNode(wrapped_index_vec, st));
2664 }
2665
2666 vmask_type = TypeVect::makemask(elem_bt, num_elem);
2667 mask = phase.transform(new VectorMaskCastNode(mask, vmask_type));
2668
2669 Node* p1 = phase.transform(new VectorRearrangeNode(src1, wrapped_index_vec));
2670 Node* p2 = phase.transform(new VectorRearrangeNode(src2, wrapped_index_vec));
2671
2672 return new VectorBlendNode(p2, p1, mask);
2673 }
2674
2675 // public static
2676 // <V extends Vector<E>,
2677 // E>
2678 // V selectFromTwoVectorOp(Class<? extends V> vClass, Class<E> eClass, int length,
2679 // V v1, V v2, V v3,
2680 // SelectFromTwoVector<V> defaultImpl)
2681 bool LibraryCallKit::inline_vector_select_from_two_vectors() {
2682 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
2683 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
2684 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
2685
2686 if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr || vector_klass->const_oop() == nullptr ||
2687 elem_klass->const_oop() == nullptr ||!vlen->is_con()) {
2688 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s",
2689 NodeClassNames[argument(0)->Opcode()],
2690 NodeClassNames[argument(1)->Opcode()],
2691 NodeClassNames[argument(2)->Opcode()]);
2692 return false; // not enough info for intrinsification
2693 }
2694
2695 if (!is_klass_initialized(vector_klass)) {
2696 log_if_needed(" ** klass argument not initialized");
2697 return false;
2698 }
2699
2700 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2701 if (!elem_type->is_primitive_type()) {
2702 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2703 return false; // should be primitive type
2704 }
2705
2706 int num_elem = vlen->get_con();
2707 if (!is_power_of_2(num_elem)) {
2708 log_if_needed(" ** vlen is not power of two=%d", num_elem);
2709 return false;
2710 }
2711
2712 BasicType elem_bt = elem_type->basic_type();
2713 BasicType index_elem_bt = elem_bt;
2714 if (elem_bt == T_FLOAT) {
2715 index_elem_bt = T_INT;
2716 } else if (elem_bt == T_DOUBLE) {
2717 index_elem_bt = T_LONG;
2718 }
2719
2720 // Check if the platform requires a VectorLoadShuffle node to be generated
2721 bool need_load_shuffle = Matcher::vector_rearrange_requires_load_shuffle(index_elem_bt, num_elem);
2722
2723 bool lowerSelectFromOp = false;
2724 if (!arch_supports_vector(Op_SelectFromTwoVector, num_elem, elem_bt, VecMaskNotUsed)) {
2725 int cast_vopc = VectorCastNode::opcode(-1, elem_bt, true);
2726 if ((elem_bt != index_elem_bt && !arch_supports_vector(cast_vopc, num_elem, index_elem_bt, VecMaskNotUsed)) ||
2727 !arch_supports_vector(Op_VectorMaskCmp, num_elem, index_elem_bt, VecMaskNotUsed) ||
2728 !arch_supports_vector(Op_AndV, num_elem, index_elem_bt, VecMaskNotUsed) ||
2729 !arch_supports_vector(Op_VectorMaskCast, num_elem, elem_bt, VecMaskNotUsed) ||
2730 !arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad) ||
2731 !arch_supports_vector(Op_VectorRearrange, num_elem, elem_bt, VecMaskNotUsed) ||
2732 (need_load_shuffle && !arch_supports_vector(Op_VectorLoadShuffle, num_elem, index_elem_bt, VecMaskNotUsed)) ||
2733 !arch_supports_vector(Op_Replicate, num_elem, index_elem_bt, VecMaskNotUsed)) {
2734 log_if_needed(" ** not supported: opc=%d vlen=%d etype=%s ismask=useload",
2735 Op_SelectFromTwoVector, num_elem, type2name(elem_bt));
2736 return false; // not supported
2737 }
2738 lowerSelectFromOp = true;
2739 }
2740
2741 int cast_vopc = VectorCastNode::opcode(-1, elem_bt, true);
2742 if (!lowerSelectFromOp) {
2743 if (!arch_supports_vector(Op_AndV, num_elem, index_elem_bt, VecMaskNotUsed) ||
2744 !arch_supports_vector(Op_Replicate, num_elem, index_elem_bt, VecMaskNotUsed) ||
2745 (is_floating_point_type(elem_bt) &&
2746 !arch_supports_vector(cast_vopc, num_elem, index_elem_bt, VecMaskNotUsed))) {
2747 log_if_needed(" ** index wrapping not supported: vlen=%d etype=%s" ,
2748 num_elem, type2name(elem_bt));
2749 return false; // not supported
2750 }
2751 }
2752
2753 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2754 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2755
2756 Node* opd1 = unbox_vector(argument(3), vbox_type, elem_bt, num_elem);
2757 if (opd1 == nullptr) {
2758 log_if_needed(" ** unbox failed v1=%s",
2759 NodeClassNames[argument(3)->Opcode()]);
2760 return false;
2761 }
2762 Node* opd2 = unbox_vector(argument(4), vbox_type, elem_bt, num_elem);
2763 if (opd2 == nullptr) {
2764 log_if_needed(" ** unbox failed v2=%s",
2765 NodeClassNames[argument(4)->Opcode()]);
2766 return false;
2767 }
2768 Node* opd3 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
2769 if (opd3 == nullptr) {
2770 log_if_needed(" ** unbox failed v3=%s",
2771 NodeClassNames[argument(5)->Opcode()]);
2772 return false;
2773 }
2774
2775 const TypeVect* vt = TypeVect::make(elem_bt, num_elem);
2776
2777 Node* operation = nullptr;
2778 if (lowerSelectFromOp) {
2779 operation = gvn().transform(LowerSelectFromTwoVectorOperation(gvn(), opd1, opd2, opd3, vt));
2780 } else {
2781 if (index_elem_bt != elem_bt) {
2782 opd1 = gvn().transform(VectorCastNode::make(cast_vopc, opd1, index_elem_bt, num_elem));
2783 }
2784 int indexRangeMask = 2 * num_elem - 1;
2785 Node* wrap_mask = gvn().makecon(TypeInteger::make(indexRangeMask, indexRangeMask, Type::WidenMin, index_elem_bt != T_LONG ? T_INT : index_elem_bt));
2786 Node* wrap_mask_vec = gvn().transform(VectorNode::scalar2vector(wrap_mask, num_elem, index_elem_bt, false));
2787 opd1 = gvn().transform(VectorNode::make(Op_AndV, opd1, wrap_mask_vec, opd1->bottom_type()->is_vect()));
2788 operation = gvn().transform(VectorNode::make(Op_SelectFromTwoVector, opd1, opd2, opd3, vt));
2789 }
2790
2791 // Wrap it up in VectorBox to keep object type information.
2792 Node* vbox = box_vector(operation, vbox_type, elem_bt, num_elem);
2793 set_result(vbox);
2794 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2795 return true;
2796 }
2797
2798 // public static
2799 // <V extends Vector<E>,
2800 // M extends VectorMask<E>,
2801 // E>
2802 // V compressExpandOp(int opr,
2803 // Class<? extends V> vClass, Class<? extends M> mClass, Class<E> eClass,
2804 // int length, V v, M m,
2805 // CompressExpandOperation<V, M> defaultImpl)
2806 bool LibraryCallKit::inline_vector_compress_expand() {
2807 const TypeInt* opr = gvn().type(argument(0))->isa_int();
2808 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
2809 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
2810 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
2811 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
2812
2813 if (opr == nullptr || !opr->is_con() ||
2814 vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
2815 mask_klass == nullptr || mask_klass->const_oop() == nullptr ||
2816 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
2817 vlen == nullptr || !vlen->is_con()) {
2818 log_if_needed(" ** missing constant: opr=%s vclass=%s mclass=%s etype=%s vlen=%s",
2819 NodeClassNames[argument(0)->Opcode()],
2820 NodeClassNames[argument(1)->Opcode()],
2821 NodeClassNames[argument(2)->Opcode()],
2822 NodeClassNames[argument(3)->Opcode()],
2823 NodeClassNames[argument(4)->Opcode()]);
2824 return false; // not enough info for intrinsification
2825 }
2826
2827 if (!is_klass_initialized(vector_klass) || !is_klass_initialized(mask_klass)) {
2828 log_if_needed(" ** klass argument not initialized");
2829 return false;
2830 }
2831
2832 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2833 if (!elem_type->is_primitive_type()) {
2834 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2835 return false; // should be primitive type
2836 }
2837
2838 int num_elem = vlen->get_con();
2839 BasicType elem_bt = elem_type->basic_type();
2840 int opc = VectorSupport::vop2ideal(opr->get_con(), elem_bt);
2841
2842 if (!arch_supports_vector(opc, num_elem, elem_bt, VecMaskUseLoad)) {
2843 log_if_needed(" ** not supported: opc=%d vlen=%d etype=%s ismask=useload",
2844 opc, num_elem, type2name(elem_bt));
2845 return false; // not supported
2846 }
2847
2848 Node* opd1 = nullptr;
2849 const TypeInstPtr* vbox_type = nullptr;
2850 if (opc != Op_CompressM) {
2851 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2852 vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2853 opd1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
2854 if (opd1 == nullptr) {
2855 log_if_needed(" ** unbox failed vector=%s",
2856 NodeClassNames[argument(5)->Opcode()]);
2857 return false;
2858 }
2859 }
2860
2861 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
2862 assert(is_vector_mask(mbox_klass), "argument(6) should be a mask class");
2863 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
2864
2865 Node* mask = unbox_vector(argument(6), mbox_type, elem_bt, num_elem);
2866 if (mask == nullptr) {
2867 log_if_needed(" ** unbox failed mask=%s",
2868 NodeClassNames[argument(6)->Opcode()]);
2869 return false;
2870 }
2871
2872 const TypeVect* vt = TypeVect::make(elem_bt, num_elem, opc == Op_CompressM);
2873 Node* operation = gvn().transform(VectorNode::make(opc, opd1, mask, vt));
2874
2875 // Wrap it up in VectorBox to keep object type information.
2876 const TypeInstPtr* box_type = opc == Op_CompressM ? mbox_type : vbox_type;
2877 Node* vbox = box_vector(operation, box_type, elem_bt, num_elem);
2878 set_result(vbox);
2879 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2880 return true;
2881 }
2882
2883 // public static
2884 // <V extends Vector<E>,
2885 // E,
2886 // S extends VectorSpecies<E>>
2887 // V indexVector(Class<? extends V> vClass, Class<E> eClass,
2888 // int length,
2889 // V v, int step, S s,
2890 // IndexOperation<V, S> defaultImpl)
2891 bool LibraryCallKit::inline_index_vector() {
2892 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
2893 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
2894 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
2895
2896 if (vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
2897 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
2898 vlen == nullptr || !vlen->is_con() ) {
2899 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s",
2900 NodeClassNames[argument(0)->Opcode()],
2901 NodeClassNames[argument(1)->Opcode()],
2902 NodeClassNames[argument(2)->Opcode()]);
2903 return false; // not enough info for intrinsification
2904 }
2905
2906 if (!is_klass_initialized(vector_klass)) {
2907 log_if_needed(" ** klass argument not initialized");
2908 return false;
2909 }
2910
2911 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2912 if (!elem_type->is_primitive_type()) {
2913 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2914 return false; // should be primitive type
2915 }
2916
2917 int num_elem = vlen->get_con();
2918 BasicType elem_bt = elem_type->basic_type();
2919
2920 // Check whether the iota index generation op is supported by the current hardware
2921 if (!arch_supports_vector(Op_VectorLoadConst, num_elem, elem_bt, VecMaskNotUsed)) {
2922 log_if_needed(" ** not supported: vlen=%d etype=%s", num_elem, type2name(elem_bt));
2923 return false; // not supported
2924 }
2925
2926 int mul_op = VectorSupport::vop2ideal(VectorSupport::VECTOR_OP_MUL, elem_bt);
2927 int vmul_op = VectorNode::opcode(mul_op, elem_bt);
2928 bool needs_mul = true;
2929 Node* scale = argument(4);
2930 const TypeInt* scale_type = gvn().type(scale)->isa_int();
2931 // Multiply is not needed if the scale is a constant "1".
2932 if (scale_type && scale_type->is_con() && scale_type->get_con() == 1) {
2933 needs_mul = false;
2934 } else {
2935 // Check whether the vector multiply op is supported by the current hardware
2936 if (!arch_supports_vector(vmul_op, num_elem, elem_bt, VecMaskNotUsed)) {
2937 log_if_needed(" ** not supported: vlen=%d etype=%s", num_elem, type2name(elem_bt));
2938 return false; // not supported
2939 }
2940
2941 // Check whether the scalar cast op is supported by the current hardware
2942 if (is_floating_point_type(elem_bt) || elem_bt == T_LONG) {
2943 int cast_op = elem_bt == T_LONG ? Op_ConvI2L :
2944 elem_bt == T_FLOAT? Op_ConvI2F : Op_ConvI2D;
2945 if (!Matcher::match_rule_supported(cast_op)) {
2946 log_if_needed(" ** Rejected op (%s) because architecture does not support it",
2947 NodeClassNames[cast_op]);
2948 return false; // not supported
2949 }
2950 }
2951 }
2952
2953 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2954 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2955 Node* opd = unbox_vector(argument(3), vbox_type, elem_bt, num_elem);
2956 if (opd == nullptr) {
2957 log_if_needed(" ** unbox failed vector=%s",
2958 NodeClassNames[argument(3)->Opcode()]);
2959 return false;
2960 }
2961
2962 int add_op = VectorSupport::vop2ideal(VectorSupport::VECTOR_OP_ADD, elem_bt);
2963 int vadd_op = VectorNode::opcode(add_op, elem_bt);
2964 bool needs_add = true;
2965 // The addition is not needed if all the element values of "opd" are zero
2966 if (VectorNode::is_all_zeros_vector(opd)) {
2967 needs_add = false;
2968 } else {
2969 // Check whether the vector addition op is supported by the current hardware
2970 if (!arch_supports_vector(vadd_op, num_elem, elem_bt, VecMaskNotUsed)) {
2971 log_if_needed(" ** not supported: vlen=%d etype=%s", num_elem, type2name(elem_bt));
2972 return false; // not supported
2973 }
2974 }
2975
2976 // Compute the iota indice vector
2977 const TypeVect* vt = TypeVect::make(elem_bt, num_elem);
2978 Node* index = gvn().transform(new VectorLoadConstNode(gvn().makecon(TypeInt::ZERO), vt));
2979
2980 // Broadcast the "scale" to a vector, and multiply the "scale" with iota indice vector.
2981 if (needs_mul) {
2982 switch (elem_bt) {
2983 case T_BOOLEAN: // fall-through
2984 case T_BYTE: // fall-through
2985 case T_SHORT: // fall-through
2986 case T_CHAR: // fall-through
2987 case T_INT: {
2988 // no conversion needed
2989 break;
2990 }
2991 case T_LONG: {
2992 scale = gvn().transform(new ConvI2LNode(scale));
2993 break;
2994 }
2995 case T_FLOAT: {
2996 scale = gvn().transform(new ConvI2FNode(scale));
2997 break;
2998 }
2999 case T_DOUBLE: {
3000 scale = gvn().transform(new ConvI2DNode(scale));
3001 break;
3002 }
3003 default: fatal("%s", type2name(elem_bt));
3004 }
3005 scale = gvn().transform(VectorNode::scalar2vector(scale, num_elem, elem_bt));
3006 index = gvn().transform(VectorNode::make(vmul_op, index, scale, vt));
3007 }
3008
3009 // Add "opd" if addition is needed.
3010 if (needs_add) {
3011 index = gvn().transform(VectorNode::make(vadd_op, opd, index, vt));
3012 }
3013 Node* vbox = box_vector(index, vbox_type, elem_bt, num_elem);
3014 set_result(vbox);
3015 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
3016 return true;
3017 }
3018
3019 // public static
3020 // <E,
3021 // M extends VectorMask<E>>
3022 // M indexPartiallyInUpperRange(Class<? extends M> mClass, Class<E> eClass, int length,
3023 // long offset, long limit,
3024 // IndexPartiallyInUpperRangeOperation<E, M> defaultImpl)
3025 bool LibraryCallKit::inline_index_partially_in_upper_range() {
3026 const TypeInstPtr* mask_klass = gvn().type(argument(0))->isa_instptr();
3027 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
3028 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
3029
3030 if (mask_klass == nullptr || mask_klass->const_oop() == nullptr ||
3031 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
3032 vlen == nullptr || !vlen->is_con()) {
3033 log_if_needed(" ** missing constant: mclass=%s etype=%s vlen=%s",
3034 NodeClassNames[argument(0)->Opcode()],
3035 NodeClassNames[argument(1)->Opcode()],
3036 NodeClassNames[argument(2)->Opcode()]);
3037 return false; // not enough info for intrinsification
3038 }
3039
3040 if (!is_klass_initialized(mask_klass)) {
3041 log_if_needed(" ** klass argument not initialized");
3042 return false;
3043 }
3044
3045 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
3046 if (!elem_type->is_primitive_type()) {
3047 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
3048 return false; // should be primitive type
3049 }
3050
3051 int num_elem = vlen->get_con();
3052 BasicType elem_bt = elem_type->basic_type();
3053
3054 // Check whether the necessary ops are supported by current hardware.
3055 bool supports_mask_gen = arch_supports_vector(Op_VectorMaskGen, num_elem, elem_bt, VecMaskUseStore);
3056 if (!supports_mask_gen) {
3057 if (!arch_supports_vector(Op_VectorLoadConst, num_elem, elem_bt, VecMaskNotUsed) ||
3058 !arch_supports_vector(Op_Replicate, num_elem, elem_bt, VecMaskNotUsed) ||
3059 !arch_supports_vector(Op_VectorMaskCmp, num_elem, elem_bt, VecMaskUseStore)) {
3060 log_if_needed(" ** not supported: vlen=%d etype=%s", num_elem, type2name(elem_bt));
3061 return false; // not supported
3062 }
3063
3064 // Check whether the scalar cast operation is supported by current hardware.
3065 if (elem_bt != T_LONG) {
3066 int cast_op = is_integral_type(elem_bt) ? Op_ConvL2I
3067 : (elem_bt == T_FLOAT ? Op_ConvL2F : Op_ConvL2D);
3068 if (!Matcher::match_rule_supported(cast_op)) {
3069 log_if_needed(" ** Rejected op (%s) because architecture does not support it",
3070 NodeClassNames[cast_op]);
3071 return false; // not supported
3072 }
3073 }
3074 }
3075
3076 Node* offset = argument(3);
3077 Node* limit = argument(5);
3078 if (offset == nullptr || limit == nullptr) {
3079 log_if_needed(" ** offset or limit argument is null");
3080 return false; // not supported
3081 }
3082
3083 ciKlass* box_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
3084 assert(is_vector_mask(box_klass), "argument(0) should be a mask class");
3085 const TypeInstPtr* box_type = TypeInstPtr::make_exact(TypePtr::NotNull, box_klass);
3086
3087 // We assume "offset > 0 && limit >= offset && limit - offset < num_elem".
3088 // So directly get indexLimit with "indexLimit = limit - offset".
3089 Node* indexLimit = gvn().transform(new SubLNode(limit, offset));
3090 Node* mask = nullptr;
3091 if (supports_mask_gen) {
3092 mask = gvn().transform(VectorMaskGenNode::make(indexLimit, elem_bt, num_elem));
3093 } else {
3094 // Generate the vector mask based on "mask = iota < indexLimit".
3095 // Broadcast "indexLimit" to a vector.
3096 switch (elem_bt) {
3097 case T_BOOLEAN: // fall-through
3098 case T_BYTE: // fall-through
3099 case T_SHORT: // fall-through
3100 case T_CHAR: // fall-through
3101 case T_INT: {
3102 indexLimit = gvn().transform(new ConvL2INode(indexLimit));
3103 break;
3104 }
3105 case T_DOUBLE: {
3106 indexLimit = gvn().transform(new ConvL2DNode(indexLimit));
3107 break;
3108 }
3109 case T_FLOAT: {
3110 indexLimit = gvn().transform(new ConvL2FNode(indexLimit));
3111 break;
3112 }
3113 case T_LONG: {
3114 // no conversion needed
3115 break;
3116 }
3117 default: fatal("%s", type2name(elem_bt));
3118 }
3119 indexLimit = gvn().transform(VectorNode::scalar2vector(indexLimit, num_elem, elem_bt));
3120
3121 // Load the "iota" vector.
3122 const TypeVect* vt = TypeVect::make(elem_bt, num_elem);
3123 Node* iota = gvn().transform(new VectorLoadConstNode(gvn().makecon(TypeInt::ZERO), vt));
3124
3125 // Compute the vector mask with "mask = iota < indexLimit".
3126 ConINode* pred_node = (ConINode*)gvn().makecon(TypeInt::make(BoolTest::lt));
3127 const TypeVect* vmask_type = TypeVect::makemask(elem_bt, num_elem);
3128 mask = gvn().transform(new VectorMaskCmpNode(BoolTest::lt, iota, indexLimit, pred_node, vmask_type));
3129 }
3130 Node* vbox = box_vector(mask, box_type, elem_bt, num_elem);
3131 set_result(vbox);
3132 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
3133 return true;
3134 }
3135
3136 #undef non_product_log_if_needed
3137 #undef log_if_needed