1 /*
2 * Copyright (c) 2020, 2024, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "ci/ciSymbols.hpp"
27 #include "classfile/vmSymbols.hpp"
28 #include "opto/library_call.hpp"
29 #include "opto/runtime.hpp"
30 #include "opto/vectornode.hpp"
31 #include "prims/vectorSupport.hpp"
32 #include "runtime/stubRoutines.hpp"
33
34 #ifdef ASSERT
35 static bool is_vector(ciKlass* klass) {
36 return klass->is_subclass_of(ciEnv::current()->vector_VectorPayload_klass());
37 }
38
39 static bool check_vbox(const TypeInstPtr* vbox_type) {
40 assert(vbox_type->klass_is_exact(), "");
41
42 ciInstanceKlass* ik = vbox_type->instance_klass();
43 assert(is_vector(ik), "not a vector");
44
45 ciField* fd1 = ik->get_field_by_name(ciSymbols::ETYPE_name(), ciSymbols::class_signature(), /* is_static */ true);
46 assert(fd1 != nullptr, "element type info is missing");
47
48 ciConstant val1 = fd1->constant_value();
49 BasicType elem_bt = val1.as_object()->as_instance()->java_mirror_type()->basic_type();
50 assert(is_java_primitive(elem_bt), "element type info is missing");
51
52 ciField* fd2 = ik->get_field_by_name(ciSymbols::VLENGTH_name(), ciSymbols::int_signature(), /* is_static */ true);
53 assert(fd2 != nullptr, "vector length info is missing");
54
55 ciConstant val2 = fd2->constant_value();
56 assert(val2.as_int() > 0, "vector length info is missing");
57
58 return true;
59 }
60 #endif
61
62 #define log_if_needed(...) \
63 if (C->print_intrinsics()) { \
64 tty->print_cr(__VA_ARGS__); \
65 }
66
67 #ifndef PRODUCT
68 #define non_product_log_if_needed(...) log_if_needed(__VA_ARGS__)
69 #else
70 #define non_product_log_if_needed(...)
71 #endif
72
73 static bool is_vector_mask(ciKlass* klass) {
74 return klass->is_subclass_of(ciEnv::current()->vector_VectorMask_klass());
75 }
76
77 static bool is_vector_shuffle(ciKlass* klass) {
78 return klass->is_subclass_of(ciEnv::current()->vector_VectorShuffle_klass());
79 }
80
81 bool LibraryCallKit::arch_supports_vector_rotate(int opc, int num_elem, BasicType elem_bt,
82 VectorMaskUseType mask_use_type, bool has_scalar_args) {
83 bool is_supported = true;
84
85 // has_scalar_args flag is true only for non-constant scalar shift count,
86 // since in this case shift needs to be broadcasted.
87 if (!Matcher::match_rule_supported_vector(opc, num_elem, elem_bt) ||
88 (has_scalar_args && !arch_supports_vector(Op_Replicate, num_elem, elem_bt, VecMaskNotUsed))) {
89 is_supported = false;
90 }
91
92 if (is_supported) {
93 // Check if mask unboxing is supported, this is a two step process which first loads the contents
94 // of boolean array into vector followed by either lane expansion to match the lane size of masked
95 // vector operation or populate the predicate register.
96 if ((mask_use_type & VecMaskUseLoad) != 0) {
97 if (!Matcher::match_rule_supported_vector(Op_VectorLoadMask, num_elem, elem_bt) ||
98 !Matcher::match_rule_supported_vector(Op_LoadVector, num_elem, T_BOOLEAN)) {
99 non_product_log_if_needed(" ** Rejected vector mask loading (%s,%s,%d) because architecture does not support it",
100 NodeClassNames[Op_VectorLoadMask], type2name(elem_bt), num_elem);
101 return false;
102 }
103 }
104
105 if ((mask_use_type & VecMaskUsePred) != 0) {
106 if (!Matcher::has_predicated_vectors() ||
107 !Matcher::match_rule_supported_vector_masked(opc, num_elem, elem_bt)) {
108 non_product_log_if_needed("Rejected vector mask predicate using (%s,%s,%d) because architecture does not support it",
109 NodeClassNames[opc], type2name(elem_bt), num_elem);
110 return false;
111 }
112 }
113 }
114
115 int lshiftopc, rshiftopc;
116 switch(elem_bt) {
117 case T_BYTE:
118 lshiftopc = Op_LShiftI;
119 rshiftopc = Op_URShiftB;
120 break;
121 case T_SHORT:
122 lshiftopc = Op_LShiftI;
123 rshiftopc = Op_URShiftS;
124 break;
125 case T_INT:
126 lshiftopc = Op_LShiftI;
127 rshiftopc = Op_URShiftI;
128 break;
129 case T_LONG:
130 lshiftopc = Op_LShiftL;
131 rshiftopc = Op_URShiftL;
132 break;
133 default: fatal("Unexpected type: %s", type2name(elem_bt));
134 }
135 int lshiftvopc = VectorNode::opcode(lshiftopc, elem_bt);
136 int rshiftvopc = VectorNode::opcode(rshiftopc, elem_bt);
137 if (!is_supported &&
138 arch_supports_vector(lshiftvopc, num_elem, elem_bt, VecMaskNotUsed, has_scalar_args) &&
139 arch_supports_vector(rshiftvopc, num_elem, elem_bt, VecMaskNotUsed, has_scalar_args) &&
140 arch_supports_vector(Op_OrV, num_elem, elem_bt, VecMaskNotUsed)) {
141 is_supported = true;
142 }
143 return is_supported;
144 }
145
146 Node* GraphKit::box_vector(Node* vector, const TypeInstPtr* vbox_type, BasicType elem_bt, int num_elem, bool deoptimize_on_exception) {
147 assert(EnableVectorSupport, "");
148
149 PreserveReexecuteState preexecs(this);
150 jvms()->set_should_reexecute(true);
151
152 VectorBoxAllocateNode* alloc = new VectorBoxAllocateNode(C, vbox_type);
153 set_edges_for_java_call(alloc, /*must_throw=*/false, /*separate_io_proj=*/true);
154 make_slow_call_ex(alloc, env()->Throwable_klass(), /*separate_io_proj=*/true, deoptimize_on_exception);
155 set_i_o(gvn().transform( new ProjNode(alloc, TypeFunc::I_O) ));
156 set_all_memory(gvn().transform( new ProjNode(alloc, TypeFunc::Memory) ));
157 Node* ret = gvn().transform(new ProjNode(alloc, TypeFunc::Parms));
158
159 assert(check_vbox(vbox_type), "");
160 const TypeVect* vt = TypeVect::make(elem_bt, num_elem, is_vector_mask(vbox_type->instance_klass()));
161 VectorBoxNode* vbox = new VectorBoxNode(C, ret, vector, vbox_type, vt);
162 return gvn().transform(vbox);
163 }
164
165 Node* GraphKit::unbox_vector(Node* v, const TypeInstPtr* vbox_type, BasicType elem_bt, int num_elem, bool shuffle_to_vector) {
166 assert(EnableVectorSupport, "");
167 const TypeInstPtr* vbox_type_v = gvn().type(v)->isa_instptr();
168 if (vbox_type_v == nullptr || vbox_type->instance_klass() != vbox_type_v->instance_klass()) {
169 return nullptr; // arguments don't agree on vector shapes
170 }
171 if (vbox_type_v->maybe_null()) {
172 return nullptr; // no nulls are allowed
173 }
174 assert(check_vbox(vbox_type), "");
175 const TypeVect* vt = TypeVect::make(elem_bt, num_elem, is_vector_mask(vbox_type->instance_klass()));
176 Node* unbox = gvn().transform(new VectorUnboxNode(C, vt, v, merged_memory(), shuffle_to_vector));
177 return unbox;
178 }
179
180 Node* GraphKit::vector_shift_count(Node* cnt, int shift_op, BasicType bt, int num_elem) {
181 assert(bt == T_INT || bt == T_LONG || bt == T_SHORT || bt == T_BYTE, "byte, short, long and int are supported");
182 juint mask = (type2aelembytes(bt) * BitsPerByte - 1);
183 Node* nmask = gvn().transform(ConNode::make(TypeInt::make(mask)));
184 Node* mcnt = gvn().transform(new AndINode(cnt, nmask));
185 return gvn().transform(VectorNode::shift_count(shift_op, mcnt, num_elem, bt));
186 }
187
188 bool LibraryCallKit::arch_supports_vector(int sopc, int num_elem, BasicType type, VectorMaskUseType mask_use_type, bool has_scalar_args) {
189 // Check that the operation is valid.
190 if (sopc <= 0) {
191 non_product_log_if_needed(" ** Rejected intrinsification because no valid vector op could be extracted");
192 return false;
193 }
194
195 if (VectorNode::is_vector_rotate(sopc)) {
196 if(!arch_supports_vector_rotate(sopc, num_elem, type, mask_use_type, has_scalar_args)) {
197 non_product_log_if_needed(" ** Rejected vector op (%s,%s,%d) because architecture does not support variable vector shifts",
198 NodeClassNames[sopc], type2name(type), num_elem);
199 return false;
200 }
201 } else if (VectorNode::is_vector_integral_negate(sopc)) {
202 if (!VectorNode::is_vector_integral_negate_supported(sopc, num_elem, type, false)) {
203 non_product_log_if_needed(" ** Rejected vector op (%s,%s,%d) because architecture does not support integral vector negate",
204 NodeClassNames[sopc], type2name(type), num_elem);
205 return false;
206 }
207 } else {
208 // Check that architecture supports this op-size-type combination.
209 if (!Matcher::match_rule_supported_vector(sopc, num_elem, type)) {
210 non_product_log_if_needed(" ** Rejected vector op (%s,%s,%d) because architecture does not support it",
211 NodeClassNames[sopc], type2name(type), num_elem);
212 return false;
213 } else {
214 assert(Matcher::match_rule_supported(sopc), "must be supported");
215 }
216 }
217
218 if (num_elem == 1) {
219 if (mask_use_type != VecMaskNotUsed) {
220 non_product_log_if_needed(" ** Rejected vector mask op (%s,%s,%d) because architecture does not support it",
221 NodeClassNames[sopc], type2name(type), num_elem);
222 return false;
223 }
224
225 if (sopc != 0) {
226 if (sopc != Op_LoadVector && sopc != Op_StoreVector) {
227 non_product_log_if_needed(" ** Not a svml call or load/store vector op (%s,%s,%d)",
228 NodeClassNames[sopc], type2name(type), num_elem);
229 return false;
230 }
231 }
232 }
233
234 if (!has_scalar_args && VectorNode::is_vector_shift(sopc) &&
235 Matcher::supports_vector_variable_shifts() == false) {
236 log_if_needed(" ** Rejected vector op (%s,%s,%d) because architecture does not support variable vector shifts",
237 NodeClassNames[sopc], type2name(type), num_elem);
238 return false;
239 }
240
241 // Check if mask unboxing is supported, this is a two step process which first loads the contents
242 // of boolean array into vector followed by either lane expansion to match the lane size of masked
243 // vector operation or populate the predicate register.
244 if ((mask_use_type & VecMaskUseLoad) != 0) {
245 if (!Matcher::match_rule_supported_vector(Op_VectorLoadMask, num_elem, type) ||
246 !Matcher::match_rule_supported_vector(Op_LoadVector, num_elem, T_BOOLEAN)) {
247 non_product_log_if_needed(" ** Rejected vector mask loading (%s,%s,%d) because architecture does not support it",
248 NodeClassNames[Op_VectorLoadMask], type2name(type), num_elem);
249 return false;
250 }
251 }
252
253 // Check if mask boxing is supported, this is a two step process which first stores the contents
254 // of mask vector / predicate register into a boolean vector followed by vector store operation to
255 // transfer the contents to underlined storage of mask boxes which is a boolean array.
256 if ((mask_use_type & VecMaskUseStore) != 0) {
257 if (!Matcher::match_rule_supported_vector(Op_VectorStoreMask, num_elem, type) ||
258 !Matcher::match_rule_supported_vector(Op_StoreVector, num_elem, T_BOOLEAN)) {
259 non_product_log_if_needed("Rejected vector mask storing (%s,%s,%d) because architecture does not support it",
260 NodeClassNames[Op_VectorStoreMask], type2name(type), num_elem);
261 return false;
262 }
263 }
264
265 if ((mask_use_type & VecMaskUsePred) != 0) {
266 bool is_supported = false;
267 if (Matcher::has_predicated_vectors()) {
268 if (VectorNode::is_vector_integral_negate(sopc)) {
269 is_supported = VectorNode::is_vector_integral_negate_supported(sopc, num_elem, type, true);
270 } else {
271 is_supported = Matcher::match_rule_supported_vector_masked(sopc, num_elem, type);
272 }
273 }
274 is_supported |= Matcher::supports_vector_predicate_op_emulation(sopc, num_elem, type);
275
276 if (!is_supported) {
277 non_product_log_if_needed("Rejected vector mask predicate using (%s,%s,%d) because architecture does not support it",
278 NodeClassNames[sopc], type2name(type), num_elem);
279 return false;
280 }
281 }
282
283 return true;
284 }
285
286 static bool is_klass_initialized(const TypeInstPtr* vec_klass) {
287 if (vec_klass->const_oop() == nullptr) {
288 return false; // uninitialized or some kind of unsafe access
289 }
290 assert(vec_klass->const_oop()->as_instance()->java_lang_Class_klass() != nullptr, "klass instance expected");
291 ciInstanceKlass* klass = vec_klass->const_oop()->as_instance()->java_lang_Class_klass()->as_instance_klass();
292 return klass->is_initialized();
293 }
294
295 // public static
296 // <V extends Vector<E>,
297 // M extends VectorMask<E>,
298 // E>
299 // V unaryOp(int oprId, Class<? extends V> vmClass, Class<? extends M> maskClass, Class<E> elementType,
300 // int length, V v, M m,
301 // UnaryOperation<V, M> defaultImpl)
302 //
303 // public static
304 // <V,
305 // M extends VectorMask<E>,
306 // E>
307 // V binaryOp(int oprId, Class<? extends V> vmClass, Class<? extends M> maskClass, Class<E> elementType,
308 // int length, V v1, V v2, M m,
309 // BinaryOperation<V, M> defaultImpl)
310 //
311 // public static
312 // <V extends Vector<E>,
313 // M extends VectorMask<E>,
314 // E>
315 // V ternaryOp(int oprId, Class<? extends V> vmClass, Class<? extends M> maskClass, Class<E> elementType,
316 // int length, V v1, V v2, V v3, M m,
317 // TernaryOperation<V, M> defaultImpl)
318 //
319 bool LibraryCallKit::inline_vector_nary_operation(int n) {
320 const TypeInt* opr = gvn().type(argument(0))->isa_int();
321 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
322 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
323 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
324 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
325
326 if (opr == nullptr || !opr->is_con() ||
327 vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
328 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
329 vlen == nullptr || !vlen->is_con()) {
330 log_if_needed(" ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
331 NodeClassNames[argument(0)->Opcode()],
332 NodeClassNames[argument(1)->Opcode()],
333 NodeClassNames[argument(3)->Opcode()],
334 NodeClassNames[argument(4)->Opcode()]);
335 return false; // not enough info for intrinsification
336 }
337
338 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
339 if (!elem_type->is_primitive_type()) {
340 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
341 return false; // should be primitive type
342 }
343 if (!is_klass_initialized(vector_klass)) {
344 log_if_needed(" ** klass argument not initialized");
345 return false;
346 }
347
348 // "argument(n + 5)" should be the mask object. We assume it is "null" when no mask
349 // is used to control this operation.
350 const Type* vmask_type = gvn().type(argument(n + 5));
351 bool is_masked_op = vmask_type != TypePtr::NULL_PTR;
352 if (is_masked_op) {
353 if (mask_klass == nullptr || mask_klass->const_oop() == nullptr) {
354 log_if_needed(" ** missing constant: maskclass=%s", NodeClassNames[argument(2)->Opcode()]);
355 return false; // not enough info for intrinsification
356 }
357
358 if (!is_klass_initialized(mask_klass)) {
359 log_if_needed(" ** mask klass argument not initialized");
360 return false;
361 }
362
363 if (vmask_type->maybe_null()) {
364 log_if_needed(" ** null mask values are not allowed for masked op");
365 return false;
366 }
367 }
368
369 BasicType elem_bt = elem_type->basic_type();
370 int num_elem = vlen->get_con();
371 int opc = VectorSupport::vop2ideal(opr->get_con(), elem_bt);
372 int sopc = VectorNode::opcode(opc, elem_bt);
373 if ((opc != Op_CallLeafVector) && (sopc == 0)) {
374 log_if_needed(" ** operation not supported: opc=%s bt=%s", NodeClassNames[opc], type2name(elem_bt));
375 return false; // operation not supported
376 }
377 if (num_elem == 1) {
378 if (opc != Op_CallLeafVector || elem_bt != T_DOUBLE) {
379 log_if_needed(" ** not a svml call: arity=%d opc=%d vlen=%d etype=%s",
380 n, opc, num_elem, type2name(elem_bt));
381 return false;
382 }
383 }
384 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
385 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
386
387 if (is_vector_mask(vbox_klass)) {
388 assert(!is_masked_op, "mask operations do not need mask to control");
389 }
390
391 if (opc == Op_CallLeafVector) {
392 if (!UseVectorStubs) {
393 log_if_needed(" ** vector stubs support is disabled");
394 return false;
395 }
396 if (!Matcher::supports_vector_calling_convention()) {
397 log_if_needed(" ** no vector calling conventions supported");
398 return false;
399 }
400 if (!Matcher::vector_size_supported(elem_bt, num_elem)) {
401 log_if_needed(" ** vector size (vlen=%d, etype=%s) is not supported",
402 num_elem, type2name(elem_bt));
403 return false;
404 }
405 }
406
407 // When using mask, mask use type needs to be VecMaskUseLoad.
408 VectorMaskUseType mask_use_type = is_vector_mask(vbox_klass) ? VecMaskUseAll
409 : is_masked_op ? VecMaskUseLoad : VecMaskNotUsed;
410 if ((sopc != 0) && !arch_supports_vector(sopc, num_elem, elem_bt, mask_use_type)) {
411 log_if_needed(" ** not supported: arity=%d opc=%d vlen=%d etype=%s ismask=%d is_masked_op=%d",
412 n, sopc, num_elem, type2name(elem_bt),
413 is_vector_mask(vbox_klass) ? 1 : 0, is_masked_op ? 1 : 0);
414 return false; // not supported
415 }
416
417 // Return true if current platform has implemented the masked operation with predicate feature.
418 bool use_predicate = is_masked_op && sopc != 0 && arch_supports_vector(sopc, num_elem, elem_bt, VecMaskUsePred);
419 if (is_masked_op && !use_predicate && !arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad)) {
420 log_if_needed(" ** not supported: arity=%d opc=%d vlen=%d etype=%s ismask=0 is_masked_op=1",
421 n, sopc, num_elem, type2name(elem_bt));
422 return false;
423 }
424
425 Node* opd1 = nullptr; Node* opd2 = nullptr; Node* opd3 = nullptr;
426 switch (n) {
427 case 3: {
428 opd3 = unbox_vector(argument(7), vbox_type, elem_bt, num_elem);
429 if (opd3 == nullptr) {
430 log_if_needed(" ** unbox failed v3=%s",
431 NodeClassNames[argument(7)->Opcode()]);
432 return false;
433 }
434 // fall-through
435 }
436 case 2: {
437 opd2 = unbox_vector(argument(6), vbox_type, elem_bt, num_elem);
438 if (opd2 == nullptr) {
439 log_if_needed(" ** unbox failed v2=%s",
440 NodeClassNames[argument(6)->Opcode()]);
441 return false;
442 }
443 // fall-through
444 }
445 case 1: {
446 opd1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
447 if (opd1 == nullptr) {
448 log_if_needed(" ** unbox failed v1=%s",
449 NodeClassNames[argument(5)->Opcode()]);
450 return false;
451 }
452 break;
453 }
454 default: fatal("unsupported arity: %d", n);
455 }
456
457 Node* mask = nullptr;
458 if (is_masked_op) {
459 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
460 assert(is_vector_mask(mbox_klass), "argument(2) should be a mask class");
461 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
462 mask = unbox_vector(argument(n + 5), mbox_type, elem_bt, num_elem);
463 if (mask == nullptr) {
464 log_if_needed(" ** unbox failed mask=%s",
465 NodeClassNames[argument(n + 5)->Opcode()]);
466 return false;
467 }
468 }
469
470 Node* operation = nullptr;
471 if (opc == Op_CallLeafVector) {
472 assert(UseVectorStubs, "sanity");
473 operation = gen_call_to_svml(opr->get_con(), elem_bt, num_elem, opd1, opd2);
474 if (operation == nullptr) {
475 log_if_needed(" ** svml call failed for %s_%s_%d",
476 (elem_bt == T_FLOAT)?"float":"double",
477 VectorSupport::svmlname[opr->get_con() - VectorSupport::VECTOR_OP_SVML_START],
478 num_elem * type2aelembytes(elem_bt));
479 return false;
480 }
481 } else {
482 const TypeVect* vt = TypeVect::make(elem_bt, num_elem, is_vector_mask(vbox_klass));
483 switch (n) {
484 case 1:
485 case 2: {
486 operation = VectorNode::make(sopc, opd1, opd2, vt, is_vector_mask(vbox_klass), VectorNode::is_shift_opcode(opc));
487 break;
488 }
489 case 3: {
490 operation = VectorNode::make(sopc, opd1, opd2, opd3, vt);
491 break;
492 }
493 default: fatal("unsupported arity: %d", n);
494 }
495 }
496
497 if (is_masked_op && mask != nullptr) {
498 if (use_predicate) {
499 operation->add_req(mask);
500 operation->add_flag(Node::Flag_is_predicated_vector);
501 } else {
502 operation->add_flag(Node::Flag_is_predicated_using_blend);
503 operation = gvn().transform(operation);
504 operation = new VectorBlendNode(opd1, operation, mask);
505 }
506 }
507 operation = gvn().transform(operation);
508
509 // Wrap it up in VectorBox to keep object type information.
510 Node* vbox = box_vector(operation, vbox_type, elem_bt, num_elem);
511 set_result(vbox);
512 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
513 return true;
514 }
515
516 // <Sh extends VectorShuffle<E>, E>
517 // Sh ShuffleIota(Class<?> E, Class<?> shuffleClass, Vector.Species<E> s, int length,
518 // int start, int step, int wrap, ShuffleIotaOperation<Sh, E> defaultImpl)
519 bool LibraryCallKit::inline_vector_shuffle_iota() {
520 const TypeInstPtr* shuffle_klass = gvn().type(argument(1))->isa_instptr();
521 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
522 const TypeInt* start_val = gvn().type(argument(4))->isa_int();
523 const TypeInt* step_val = gvn().type(argument(5))->isa_int();
524 const TypeInt* wrap = gvn().type(argument(6))->isa_int();
525
526 if (shuffle_klass == nullptr || shuffle_klass->const_oop() == nullptr ||
527 vlen == nullptr || !vlen->is_con() ||
528 start_val == nullptr ||
529 step_val == nullptr ||
530 wrap == nullptr || !wrap->is_con()) {
531 return false; // not enough info for intrinsification
532 }
533
534 if (!is_klass_initialized(shuffle_klass)) {
535 log_if_needed(" ** klass argument not initialized");
536 return false;
537 }
538
539 int do_wrap = wrap->get_con();
540 int num_elem = vlen->get_con();
541 BasicType elem_bt = T_BYTE;
542
543 bool effective_indices_in_range = false;
544 if (start_val->is_con() && step_val->is_con()) {
545 int effective_min_index = start_val->get_con();
546 int effective_max_index = start_val->get_con() + step_val->get_con() * (num_elem - 1);
547 effective_indices_in_range = effective_max_index >= effective_min_index && effective_min_index >= -128 && effective_max_index <= 127;
548 }
549
550 if (!do_wrap && !effective_indices_in_range) {
551 // Disable instrinsification for unwrapped shuffle iota if start/step
552 // values are non-constant OR if intermediate result overflows byte value range.
553 return false;
554 }
555
556 if (!arch_supports_vector(Op_AddVB, num_elem, elem_bt, VecMaskNotUsed) ||
557 !arch_supports_vector(Op_AndV, num_elem, elem_bt, VecMaskNotUsed) ||
558 !arch_supports_vector(Op_VectorLoadConst, num_elem, elem_bt, VecMaskNotUsed) ||
559 !arch_supports_vector(Op_Replicate, num_elem, elem_bt, VecMaskNotUsed)) {
560 return false;
561 }
562
563 if (!do_wrap &&
564 (!arch_supports_vector(Op_SubVB, num_elem, elem_bt, VecMaskNotUsed) ||
565 !arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskNotUsed) ||
566 !arch_supports_vector(Op_VectorMaskCmp, num_elem, elem_bt, VecMaskNotUsed))) {
567 return false;
568 }
569
570 bool step_multiply = !step_val->is_con() || !is_power_of_2(step_val->get_con());
571 if ((step_multiply && !arch_supports_vector(Op_MulVB, num_elem, elem_bt, VecMaskNotUsed)) ||
572 (!step_multiply && !arch_supports_vector(Op_LShiftVB, num_elem, elem_bt, VecMaskNotUsed))) {
573 return false;
574 }
575
576 const Type * type_bt = Type::get_const_basic_type(elem_bt);
577 const TypeVect * vt = TypeVect::make(type_bt, num_elem);
578
579 Node* res = gvn().transform(new VectorLoadConstNode(gvn().makecon(TypeInt::ZERO), vt));
580
581 Node* start = argument(4);
582 Node* step = argument(5);
583
584 if (step_multiply) {
585 Node* bcast_step = gvn().transform(VectorNode::scalar2vector(step, num_elem, type_bt));
586 res = gvn().transform(VectorNode::make(Op_MulVB, res, bcast_step, vt));
587 } else if (step_val->get_con() > 1) {
588 Node* cnt = gvn().makecon(TypeInt::make(log2i_exact(step_val->get_con())));
589 Node* shift_cnt = vector_shift_count(cnt, Op_LShiftI, elem_bt, num_elem);
590 res = gvn().transform(VectorNode::make(Op_LShiftVB, res, shift_cnt, vt));
591 }
592
593 if (!start_val->is_con() || start_val->get_con() != 0) {
594 Node* bcast_start = gvn().transform(VectorNode::scalar2vector(start, num_elem, type_bt));
595 res = gvn().transform(VectorNode::make(Op_AddVB, res, bcast_start, vt));
596 }
597
598 Node * mod_val = gvn().makecon(TypeInt::make(num_elem-1));
599 Node * bcast_mod = gvn().transform(VectorNode::scalar2vector(mod_val, num_elem, type_bt));
600
601 if (do_wrap) {
602 // Wrap the indices greater than lane count.
603 res = gvn().transform(VectorNode::make(Op_AndV, res, bcast_mod, vt));
604 } else {
605 ConINode* pred_node = (ConINode*)gvn().makecon(TypeInt::make(BoolTest::ugt));
606 Node * lane_cnt = gvn().makecon(TypeInt::make(num_elem));
607 Node * bcast_lane_cnt = gvn().transform(VectorNode::scalar2vector(lane_cnt, num_elem, type_bt));
608 const TypeVect* vmask_type = TypeVect::makemask(elem_bt, num_elem);
609 Node* mask = gvn().transform(new VectorMaskCmpNode(BoolTest::ugt, bcast_lane_cnt, res, pred_node, vmask_type));
610
611 // Make the indices greater than lane count as -ve values to match the java side implementation.
612 res = gvn().transform(VectorNode::make(Op_AndV, res, bcast_mod, vt));
613 Node * biased_val = gvn().transform(VectorNode::make(Op_SubVB, res, bcast_lane_cnt, vt));
614 res = gvn().transform(new VectorBlendNode(biased_val, res, mask));
615 }
616
617 ciKlass* sbox_klass = shuffle_klass->const_oop()->as_instance()->java_lang_Class_klass();
618 const TypeInstPtr* shuffle_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, sbox_klass);
619
620 // Wrap it up in VectorBox to keep object type information.
621 res = box_vector(res, shuffle_box_type, elem_bt, num_elem);
622 set_result(res);
623 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
624 return true;
625 }
626
627 // <E, M>
628 // long maskReductionCoerced(int oper, Class<? extends M> maskClass, Class<?> elemClass,
629 // int length, M m, VectorMaskOp<M> defaultImpl)
630 bool LibraryCallKit::inline_vector_mask_operation() {
631 const TypeInt* oper = gvn().type(argument(0))->isa_int();
632 const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
633 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
634 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
635 Node* mask = argument(4);
636
637 if (mask_klass == nullptr || mask_klass->const_oop() == nullptr ||
638 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
639 vlen == nullptr || !vlen->is_con() ||
640 oper == nullptr || !oper->is_con() ||
641 mask->is_top()) {
642 return false; // dead code
643 }
644
645 if (!is_klass_initialized(mask_klass)) {
646 log_if_needed(" ** klass argument not initialized");
647 return false;
648 }
649
650 int num_elem = vlen->get_con();
651 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
652 BasicType elem_bt = elem_type->basic_type();
653
654 int mopc = VectorSupport::vop2ideal(oper->get_con(), elem_bt);
655 if (!arch_supports_vector(mopc, num_elem, elem_bt, VecMaskUseLoad)) {
656 log_if_needed(" ** not supported: arity=1 op=cast#%d/3 vlen2=%d etype2=%s",
657 mopc, num_elem, type2name(elem_bt));
658 return false; // not supported
659 }
660
661 const Type* elem_ty = Type::get_const_basic_type(elem_bt);
662 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
663 const TypeInstPtr* mask_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
664 Node* mask_vec = unbox_vector(mask, mask_box_type, elem_bt, num_elem, true);
665 if (mask_vec == nullptr) {
666 log_if_needed(" ** unbox failed mask=%s",
667 NodeClassNames[argument(4)->Opcode()]);
668 return false;
669 }
670
671 if (mask_vec->bottom_type()->isa_vectmask() == nullptr) {
672 mask_vec = gvn().transform(VectorStoreMaskNode::make(gvn(), mask_vec, elem_bt, num_elem));
673 }
674 const Type* maskoper_ty = mopc == Op_VectorMaskToLong ? (const Type*)TypeLong::LONG : (const Type*)TypeInt::INT;
675 Node* maskoper = gvn().transform(VectorMaskOpNode::make(mask_vec, maskoper_ty, mopc));
676 if (mopc != Op_VectorMaskToLong) {
677 maskoper = ConvI2L(maskoper);
678 }
679 set_result(maskoper);
680
681 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
682 return true;
683 }
684
685 // public static
686 // <V,
687 // Sh extends VectorShuffle<E>,
688 // E>
689 // V shuffleToVector(Class<? extends Vector<E>> vclass, Class<E> elementType,
690 // Class<? extends Sh> shuffleClass, Sh s, int length,
691 // ShuffleToVectorOperation<V, Sh, E> defaultImpl)
692 bool LibraryCallKit::inline_vector_shuffle_to_vector() {
693 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
694 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
695 const TypeInstPtr* shuffle_klass = gvn().type(argument(2))->isa_instptr();
696 Node* shuffle = argument(3);
697 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
698
699 if (vector_klass == nullptr || elem_klass == nullptr || shuffle_klass == nullptr || shuffle->is_top() || vlen == nullptr) {
700 return false; // dead code
701 }
702 if (vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || shuffle_klass->const_oop() == nullptr || !vlen->is_con()) {
703 return false; // not enough info for intrinsification
704 }
705 if (!is_klass_initialized(shuffle_klass) || !is_klass_initialized(vector_klass) ) {
706 log_if_needed(" ** klass argument not initialized");
707 return false;
708 }
709
710 int num_elem = vlen->get_con();
711 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
712 BasicType elem_bt = elem_type->basic_type();
713
714 if (num_elem < 4) {
715 return false;
716 }
717
718 int cast_vopc = VectorCastNode::opcode(-1, T_BYTE); // from shuffle of type T_BYTE
719 // Make sure that cast is implemented to particular type/size combination.
720 if (!arch_supports_vector(cast_vopc, num_elem, elem_bt, VecMaskNotUsed)) {
721 log_if_needed(" ** not supported: arity=1 op=cast#%d/3 vlen2=%d etype2=%s",
722 cast_vopc, num_elem, type2name(elem_bt));
723 return false;
724 }
725
726 ciKlass* sbox_klass = shuffle_klass->const_oop()->as_instance()->java_lang_Class_klass();
727 const TypeInstPtr* shuffle_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, sbox_klass);
728
729 // Unbox shuffle with true flag to indicate its load shuffle to vector
730 // shuffle is a byte array
731 Node* shuffle_vec = unbox_vector(shuffle, shuffle_box_type, T_BYTE, num_elem, true);
732
733 // cast byte to target element type
734 shuffle_vec = gvn().transform(VectorCastNode::make(cast_vopc, shuffle_vec, elem_bt, num_elem));
735
736 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
737 const TypeInstPtr* vec_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
738
739 // Box vector
740 Node* res = box_vector(shuffle_vec, vec_box_type, elem_bt, num_elem);
741 set_result(res);
742 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
743 return true;
744 }
745
746 // public static
747 // <M,
748 // S extends VectorSpecies<E>,
749 // E>
750 // M fromBitsCoerced(Class<? extends M> vmClass, Class<E> elementType, int length,
751 // long bits, int mode, S s,
752 // BroadcastOperation<M, E, S> defaultImpl)
753 bool LibraryCallKit::inline_vector_frombits_coerced() {
754 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
755 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
756 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
757 const TypeLong* bits_type = gvn().type(argument(3))->isa_long();
758 // Mode argument determines the mode of operation it can take following values:-
759 // MODE_BROADCAST for vector Vector.broadcast and VectorMask.maskAll operations.
760 // MODE_BITS_COERCED_LONG_TO_MASK for VectorMask.fromLong operation.
761 const TypeInt* mode = gvn().type(argument(5))->isa_int();
762
763 if (vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
764 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
765 vlen == nullptr || !vlen->is_con() ||
766 bits_type == nullptr ||
767 mode == nullptr || !mode->is_con()) {
768 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s bitwise=%s",
769 NodeClassNames[argument(0)->Opcode()],
770 NodeClassNames[argument(1)->Opcode()],
771 NodeClassNames[argument(2)->Opcode()],
772 NodeClassNames[argument(5)->Opcode()]);
773 return false; // not enough info for intrinsification
774 }
775
776 if (!is_klass_initialized(vector_klass)) {
777 log_if_needed(" ** klass argument not initialized");
778 return false;
779 }
780 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
781 if (!elem_type->is_primitive_type()) {
782 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
783 return false; // should be primitive type
784 }
785 BasicType elem_bt = elem_type->basic_type();
786 int num_elem = vlen->get_con();
787 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
788 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
789
790 bool is_mask = is_vector_mask(vbox_klass);
791 int bcast_mode = mode->get_con();
792 VectorMaskUseType checkFlags = (VectorMaskUseType)(is_mask ? VecMaskUseAll : VecMaskNotUsed);
793 int opc = bcast_mode == VectorSupport::MODE_BITS_COERCED_LONG_TO_MASK ? Op_VectorLongToMask : Op_Replicate;
794
795 if (!arch_supports_vector(opc, num_elem, elem_bt, checkFlags, true /*has_scalar_args*/)) {
796 log_if_needed(" ** not supported: arity=0 op=broadcast vlen=%d etype=%s ismask=%d bcast_mode=%d",
797 num_elem, type2name(elem_bt),
798 is_mask ? 1 : 0,
799 bcast_mode);
800 return false; // not supported
801 }
802
803 Node* broadcast = nullptr;
804 Node* bits = argument(3);
805 Node* elem = bits;
806
807 if (opc == Op_VectorLongToMask) {
808 const TypeVect* vt = TypeVect::makemask(elem_bt, num_elem);
809 if (vt->isa_vectmask()) {
810 broadcast = gvn().transform(new VectorLongToMaskNode(elem, vt));
811 } else {
812 const TypeVect* mvt = TypeVect::make(T_BOOLEAN, num_elem);
813 broadcast = gvn().transform(new VectorLongToMaskNode(elem, mvt));
814 broadcast = gvn().transform(new VectorLoadMaskNode(broadcast, vt));
815 }
816 } else {
817 switch (elem_bt) {
818 case T_BOOLEAN: // fall-through
819 case T_BYTE: // fall-through
820 case T_SHORT: // fall-through
821 case T_CHAR: // fall-through
822 case T_INT: {
823 elem = gvn().transform(new ConvL2INode(bits));
824 break;
825 }
826 case T_DOUBLE: {
827 elem = gvn().transform(new MoveL2DNode(bits));
828 break;
829 }
830 case T_FLOAT: {
831 bits = gvn().transform(new ConvL2INode(bits));
832 elem = gvn().transform(new MoveI2FNode(bits));
833 break;
834 }
835 case T_LONG: {
836 // no conversion needed
837 break;
838 }
839 default: fatal("%s", type2name(elem_bt));
840 }
841 broadcast = VectorNode::scalar2vector(elem, num_elem, Type::get_const_basic_type(elem_bt), is_mask);
842 broadcast = gvn().transform(broadcast);
843 }
844
845 Node* box = box_vector(broadcast, vbox_type, elem_bt, num_elem);
846 set_result(box);
847 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
848 return true;
849 }
850
851 static bool elem_consistent_with_arr(BasicType elem_bt, const TypeAryPtr* arr_type, bool mismatched_ms) {
852 assert(arr_type != nullptr, "unexpected");
853 BasicType arr_elem_bt = arr_type->elem()->array_element_basic_type();
854 if (elem_bt == arr_elem_bt) {
855 return true;
856 } else if (elem_bt == T_SHORT && arr_elem_bt == T_CHAR) {
857 // Load/store of short vector from/to char[] is supported
858 return true;
859 } else if (elem_bt == T_BYTE && arr_elem_bt == T_BOOLEAN) {
860 // Load/store of byte vector from/to boolean[] is supported
861 return true;
862 } else {
863 return mismatched_ms;
864 }
865 }
866
867 // public static
868 // <C,
869 // VM extends VectorPayload,
870 // E,
871 // S extends VectorSpecies<E>>
872 // VM load(Class<? extends VM> vmClass, Class<E> eClass,
873 // int length,
874 // Object base, long offset, // Unsafe addressing
875 // boolean fromSegment,
876 // C container, long index, S s, // Arguments for default implementation
877 // LoadOperation<C, VM, S> defaultImpl) {
878 // public static
879 // <C,
880 // V extends VectorPayload>
881 // void store(Class<?> vClass, Class<?> eClass,
882 // int length,
883 // Object base, long offset, // Unsafe addressing
884 // boolean fromSegment,
885 // V v, C container, long index, // Arguments for default implementation
886 // StoreVectorOperation<C, V> defaultImpl) {
887 bool LibraryCallKit::inline_vector_mem_operation(bool is_store) {
888 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
889 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
890 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
891 const TypeInt* from_ms = gvn().type(argument(6))->isa_int();
892
893 if (vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
894 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
895 vlen == nullptr || !vlen->is_con() ||
896 from_ms == nullptr || !from_ms->is_con()) {
897 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s from_ms=%s",
898 NodeClassNames[argument(0)->Opcode()],
899 NodeClassNames[argument(1)->Opcode()],
900 NodeClassNames[argument(2)->Opcode()],
901 NodeClassNames[argument(6)->Opcode()]);
902 return false; // not enough info for intrinsification
903 }
904 if (!is_klass_initialized(vector_klass)) {
905 log_if_needed(" ** klass argument not initialized");
906 return false;
907 }
908
909 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
910 if (!elem_type->is_primitive_type()) {
911 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
912 return false; // should be primitive type
913 }
914 BasicType elem_bt = elem_type->basic_type();
915 int num_elem = vlen->get_con();
916
917 // TODO When mask usage is supported, VecMaskNotUsed needs to be VecMaskUseLoad.
918 if (!arch_supports_vector(is_store ? Op_StoreVector : Op_LoadVector, num_elem, elem_bt, VecMaskNotUsed)) {
919 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d etype=%s ismask=no",
920 is_store, is_store ? "store" : "load",
921 num_elem, type2name(elem_bt));
922 return false; // not supported
923 }
924
925 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
926 bool is_mask = is_vector_mask(vbox_klass);
927
928 Node* base = argument(3);
929 Node* offset = ConvL2X(argument(4));
930
931 // Save state and restore on bailout
932 uint old_sp = sp();
933 SafePointNode* old_map = clone_map();
934
935 Node* addr = make_unsafe_address(base, offset, (is_mask ? T_BOOLEAN : elem_bt), true);
936
937 // The memory barrier checks are based on ones for unsafe access.
938 // This is not 1-1 implementation.
939 const Type *const base_type = gvn().type(base);
940
941 const TypePtr *addr_type = gvn().type(addr)->isa_ptr();
942 const TypeAryPtr* arr_type = addr_type->isa_aryptr();
943
944 const bool in_native = TypePtr::NULL_PTR == base_type; // base always null
945 const bool in_heap = !TypePtr::NULL_PTR->higher_equal(base_type); // base never null
946
947 const bool is_mixed_access = !in_heap && !in_native;
948
949 const bool is_mismatched_access = in_heap && (addr_type->isa_aryptr() == nullptr);
950
951 const bool needs_cpu_membar = is_mixed_access || is_mismatched_access;
952
953 // For non-masked mismatched memory segment vector read/write accesses, intrinsification can continue
954 // with unknown backing storage type and compiler can skip inserting explicit reinterpretation IR after
955 // loading from or before storing to backing storage which is mandatory for semantic correctness of
956 // big-endian memory layout.
957 bool mismatched_ms = LITTLE_ENDIAN_ONLY(false)
958 BIG_ENDIAN_ONLY(from_ms->get_con() && !is_mask && arr_type != nullptr &&
959 arr_type->elem()->array_element_basic_type() != elem_bt);
960 BasicType mem_elem_bt = mismatched_ms ? arr_type->elem()->array_element_basic_type() : elem_bt;
961 if (!is_java_primitive(mem_elem_bt)) {
962 log_if_needed(" ** non-primitive array element type");
963 return false;
964 }
965 int mem_num_elem = mismatched_ms ? (num_elem * type2aelembytes(elem_bt)) / type2aelembytes(mem_elem_bt) : num_elem;
966 if (arr_type != nullptr && !is_mask && !elem_consistent_with_arr(elem_bt, arr_type, mismatched_ms)) {
967 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d etype=%s atype=%s ismask=no",
968 is_store, is_store ? "store" : "load",
969 num_elem, type2name(elem_bt), type2name(arr_type->elem()->array_element_basic_type()));
970 set_map(old_map);
971 set_sp(old_sp);
972 return false;
973 }
974
975 // In case of mismatched memory segment accesses, we need to double check that the source type memory operations are supported by backend.
976 if (mismatched_ms) {
977 if (is_store) {
978 if (!arch_supports_vector(Op_StoreVector, num_elem, elem_bt, VecMaskNotUsed)
979 || !arch_supports_vector(Op_VectorReinterpret, mem_num_elem, mem_elem_bt, VecMaskNotUsed)) {
980 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d*8 etype=%s/8 ismask=no",
981 is_store, "store",
982 num_elem, type2name(elem_bt));
983 set_map(old_map);
984 set_sp(old_sp);
985 return false; // not supported
986 }
987 } else {
988 if (!arch_supports_vector(Op_LoadVector, mem_num_elem, mem_elem_bt, VecMaskNotUsed)
989 || !arch_supports_vector(Op_VectorReinterpret, num_elem, elem_bt, VecMaskNotUsed)) {
990 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d*8 etype=%s/8 ismask=no",
991 is_store, "load",
992 mem_num_elem, type2name(mem_elem_bt));
993 set_map(old_map);
994 set_sp(old_sp);
995 return false; // not supported
996 }
997 }
998 }
999 if (is_mask) {
1000 if (!is_store) {
1001 if (!arch_supports_vector(Op_LoadVector, num_elem, elem_bt, VecMaskUseLoad)) {
1002 set_map(old_map);
1003 set_sp(old_sp);
1004 return false; // not supported
1005 }
1006 } else {
1007 if (!arch_supports_vector(Op_StoreVector, num_elem, elem_bt, VecMaskUseStore)) {
1008 set_map(old_map);
1009 set_sp(old_sp);
1010 return false; // not supported
1011 }
1012 }
1013 }
1014
1015 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1016
1017 if (needs_cpu_membar) {
1018 insert_mem_bar(Op_MemBarCPUOrder);
1019 }
1020
1021 if (is_store) {
1022 Node* val = unbox_vector(argument(7), vbox_type, elem_bt, num_elem);
1023 if (val == nullptr) {
1024 set_map(old_map);
1025 set_sp(old_sp);
1026 return false; // operand unboxing failed
1027 }
1028 set_all_memory(reset_memory());
1029
1030 // In case the store needs to happen to byte array, reinterpret the incoming vector to byte vector.
1031 int store_num_elem = num_elem;
1032 if (mismatched_ms) {
1033 store_num_elem = mem_num_elem;
1034 const TypeVect* to_vect_type = TypeVect::make(mem_elem_bt, store_num_elem);
1035 val = gvn().transform(new VectorReinterpretNode(val, val->bottom_type()->is_vect(), to_vect_type));
1036 }
1037 if (is_mask) {
1038 val = gvn().transform(VectorStoreMaskNode::make(gvn(), val, elem_bt, num_elem));
1039 }
1040 Node* vstore = gvn().transform(StoreVectorNode::make(0, control(), memory(addr), addr, addr_type, val, store_num_elem));
1041 set_memory(vstore, addr_type);
1042 } else {
1043 // When using byte array, we need to load as byte then reinterpret the value. Otherwise, do a simple vector load.
1044 Node* vload = nullptr;
1045 if (mismatched_ms) {
1046 vload = gvn().transform(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, mem_num_elem, mem_elem_bt));
1047 const TypeVect* to_vect_type = TypeVect::make(elem_bt, num_elem);
1048 vload = gvn().transform(new VectorReinterpretNode(vload, vload->bottom_type()->is_vect(), to_vect_type));
1049 } else {
1050 // Special handle for masks
1051 if (is_mask) {
1052 vload = gvn().transform(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, num_elem, T_BOOLEAN));
1053 vload = gvn().transform(new VectorLoadMaskNode(vload, TypeVect::makemask(elem_bt, num_elem)));
1054 } else {
1055 vload = gvn().transform(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, num_elem, elem_bt));
1056 }
1057 }
1058 Node* box = box_vector(vload, vbox_type, elem_bt, num_elem);
1059 set_result(box);
1060 }
1061
1062 destruct_map_clone(old_map);
1063
1064 if (needs_cpu_membar) {
1065 insert_mem_bar(Op_MemBarCPUOrder);
1066 }
1067
1068 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1069 return true;
1070 }
1071
1072 // public static
1073 // <C,
1074 // V extends Vector<?>,
1075 // E,
1076 // S extends VectorSpecies<E>,
1077 // M extends VectorMask<E>>
1078 // V loadMasked(Class<? extends V> vClass, Class<M> mClass, Class<E> eClass,
1079 // int length, Object base, long offset, // Unsafe addressing
1080 // boolean fromSegment,
1081 // M m, int offsetInRange,
1082 // C container, long index, S s, // Arguments for default implementation
1083 // LoadVectorMaskedOperation<C, V, S, M> defaultImpl) {
1084 // public static
1085 // <C,
1086 // V extends Vector<E>,
1087 // M extends VectorMask<E>,
1088 // E>
1089 // void storeMasked(Class<? extends V> vClass, Class<M> mClass, Class<E> eClass,
1090 // int length,
1091 // Object base, long offset, // Unsafe addressing
1092 // boolean fromSegment,
1093 // V v, M m, C container, long index, // Arguments for default implementation
1094 // StoreVectorMaskedOperation<C, V, M> defaultImpl) {
1095
1096 bool LibraryCallKit::inline_vector_mem_masked_operation(bool is_store) {
1097 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
1098 const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
1099 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
1100 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
1101 const TypeInt* from_ms = gvn().type(argument(7))->isa_int();
1102
1103 if (vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
1104 mask_klass == nullptr || mask_klass->const_oop() == nullptr ||
1105 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
1106 vlen == nullptr || !vlen->is_con() ||
1107 from_ms == nullptr || !from_ms->is_con()) {
1108 log_if_needed(" ** missing constant: vclass=%s mclass=%s etype=%s vlen=%s from_ms=%s",
1109 NodeClassNames[argument(0)->Opcode()],
1110 NodeClassNames[argument(1)->Opcode()],
1111 NodeClassNames[argument(2)->Opcode()],
1112 NodeClassNames[argument(3)->Opcode()],
1113 NodeClassNames[argument(7)->Opcode()]);
1114 return false; // not enough info for intrinsification
1115 }
1116 if (!is_klass_initialized(vector_klass)) {
1117 log_if_needed(" ** klass argument not initialized");
1118 return false;
1119 }
1120
1121 if (!is_klass_initialized(mask_klass)) {
1122 log_if_needed(" ** mask klass argument not initialized");
1123 return false;
1124 }
1125
1126 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1127 if (!elem_type->is_primitive_type()) {
1128 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1129 return false; // should be primitive type
1130 }
1131
1132 BasicType elem_bt = elem_type->basic_type();
1133 int num_elem = vlen->get_con();
1134
1135 Node* base = argument(4);
1136 Node* offset = ConvL2X(argument(5));
1137
1138 // Save state and restore on bailout
1139 uint old_sp = sp();
1140 SafePointNode* old_map = clone_map();
1141
1142 Node* addr = make_unsafe_address(base, offset, elem_bt, true);
1143 const TypePtr *addr_type = gvn().type(addr)->isa_ptr();
1144 const TypeAryPtr* arr_type = addr_type->isa_aryptr();
1145
1146 bool mismatched_ms = from_ms->get_con() && arr_type != nullptr && arr_type->elem()->array_element_basic_type() != elem_bt;
1147 BIG_ENDIAN_ONLY(if (mismatched_ms) return false;)
1148 // If there is no consistency between array and vector element types, it must be special byte array case
1149 if (arr_type != nullptr && !elem_consistent_with_arr(elem_bt, arr_type, mismatched_ms)) {
1150 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d etype=%s atype=%s",
1151 is_store, is_store ? "storeMasked" : "loadMasked",
1152 num_elem, type2name(elem_bt), type2name(arr_type->elem()->array_element_basic_type()));
1153 set_map(old_map);
1154 set_sp(old_sp);
1155 return false;
1156 }
1157
1158 int mem_num_elem = mismatched_ms ? num_elem * type2aelembytes(elem_bt) : num_elem;
1159 BasicType mem_elem_bt = mismatched_ms ? T_BYTE : elem_bt;
1160 bool supports_predicate = arch_supports_vector(is_store ? Op_StoreVectorMasked : Op_LoadVectorMasked,
1161 mem_num_elem, mem_elem_bt, VecMaskUseLoad);
1162
1163 // If current arch does not support the predicated operations, we have to bail
1164 // out when current case uses the predicate feature.
1165 if (!supports_predicate) {
1166 bool needs_predicate = false;
1167 if (is_store) {
1168 // Masked vector store always uses the predicated store.
1169 needs_predicate = true;
1170 } else {
1171 // Masked vector load with IOOBE always uses the predicated load.
1172 const TypeInt* offset_in_range = gvn().type(argument(9))->isa_int();
1173 if (!offset_in_range->is_con()) {
1174 log_if_needed(" ** missing constant: offsetInRange=%s",
1175 NodeClassNames[argument(8)->Opcode()]);
1176 set_map(old_map);
1177 set_sp(old_sp);
1178 return false;
1179 }
1180 needs_predicate = (offset_in_range->get_con() == 0);
1181 }
1182
1183 if (needs_predicate) {
1184 log_if_needed(" ** not supported: op=%s vlen=%d etype=%s mismatched_ms=%d",
1185 is_store ? "storeMasked" : "loadMasked",
1186 num_elem, type2name(elem_bt), mismatched_ms ? 1 : 0);
1187 set_map(old_map);
1188 set_sp(old_sp);
1189 return false;
1190 }
1191 }
1192
1193 // This only happens for masked vector load. If predicate is not supported, then check whether
1194 // the normal vector load and blend operations are supported by backend.
1195 if (!supports_predicate && (!arch_supports_vector(Op_LoadVector, mem_num_elem, mem_elem_bt, VecMaskNotUsed) ||
1196 !arch_supports_vector(Op_VectorBlend, mem_num_elem, mem_elem_bt, VecMaskUseLoad))) {
1197 log_if_needed(" ** not supported: op=loadMasked vlen=%d etype=%s mismatched_ms=%d",
1198 num_elem, type2name(elem_bt), mismatched_ms ? 1 : 0);
1199 set_map(old_map);
1200 set_sp(old_sp);
1201 return false;
1202 }
1203
1204 // Since we are using byte array, we need to double check that the vector reinterpret operation
1205 // with byte type is supported by backend.
1206 if (mismatched_ms) {
1207 if (!arch_supports_vector(Op_VectorReinterpret, mem_num_elem, T_BYTE, VecMaskNotUsed)) {
1208 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d etype=%s mismatched_ms=1",
1209 is_store, is_store ? "storeMasked" : "loadMasked",
1210 num_elem, type2name(elem_bt));
1211 set_map(old_map);
1212 set_sp(old_sp);
1213 return false;
1214 }
1215 }
1216
1217 // Since it needs to unbox the mask, we need to double check that the related load operations
1218 // for mask are supported by backend.
1219 if (!arch_supports_vector(Op_LoadVector, num_elem, elem_bt, VecMaskUseLoad)) {
1220 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d etype=%s",
1221 is_store, is_store ? "storeMasked" : "loadMasked",
1222 num_elem, type2name(elem_bt));
1223 set_map(old_map);
1224 set_sp(old_sp);
1225 return false;
1226 }
1227
1228 // Can base be null? Otherwise, always on-heap access.
1229 bool can_access_non_heap = TypePtr::NULL_PTR->higher_equal(gvn().type(base));
1230 if (can_access_non_heap) {
1231 insert_mem_bar(Op_MemBarCPUOrder);
1232 }
1233
1234 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1235 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1236 assert(!is_vector_mask(vbox_klass) && is_vector_mask(mbox_klass), "Invalid class type");
1237 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1238 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1239
1240 Node* mask = unbox_vector(is_store ? argument(9) : argument(8), mbox_type, elem_bt, num_elem);
1241 if (mask == nullptr) {
1242 log_if_needed(" ** unbox failed mask=%s",
1243 is_store ? NodeClassNames[argument(9)->Opcode()]
1244 : NodeClassNames[argument(8)->Opcode()]);
1245 set_map(old_map);
1246 set_sp(old_sp);
1247 return false;
1248 }
1249
1250 if (is_store) {
1251 Node* val = unbox_vector(argument(8), vbox_type, elem_bt, num_elem);
1252 if (val == nullptr) {
1253 log_if_needed(" ** unbox failed vector=%s",
1254 NodeClassNames[argument(8)->Opcode()]);
1255 set_map(old_map);
1256 set_sp(old_sp);
1257 return false; // operand unboxing failed
1258 }
1259 set_all_memory(reset_memory());
1260
1261 if (mismatched_ms) {
1262 // Reinterpret the incoming vector to byte vector.
1263 const TypeVect* to_vect_type = TypeVect::make(mem_elem_bt, mem_num_elem);
1264 val = gvn().transform(new VectorReinterpretNode(val, val->bottom_type()->is_vect(), to_vect_type));
1265 // Reinterpret the vector mask to byte type.
1266 const TypeVect* from_mask_type = TypeVect::makemask(elem_bt, num_elem);
1267 const TypeVect* to_mask_type = TypeVect::makemask(mem_elem_bt, mem_num_elem);
1268 mask = gvn().transform(new VectorReinterpretNode(mask, from_mask_type, to_mask_type));
1269 }
1270 Node* vstore = gvn().transform(new StoreVectorMaskedNode(control(), memory(addr), addr, val, addr_type, mask));
1271 set_memory(vstore, addr_type);
1272 } else {
1273 Node* vload = nullptr;
1274
1275 if (mismatched_ms) {
1276 // Reinterpret the vector mask to byte type.
1277 const TypeVect* from_mask_type = TypeVect::makemask(elem_bt, num_elem);
1278 const TypeVect* to_mask_type = TypeVect::makemask(mem_elem_bt, mem_num_elem);
1279 mask = gvn().transform(new VectorReinterpretNode(mask, from_mask_type, to_mask_type));
1280 }
1281
1282 if (supports_predicate) {
1283 // Generate masked load vector node if predicate feature is supported.
1284 const TypeVect* vt = TypeVect::make(mem_elem_bt, mem_num_elem);
1285 vload = gvn().transform(new LoadVectorMaskedNode(control(), memory(addr), addr, addr_type, vt, mask));
1286 } else {
1287 // Use the vector blend to implement the masked load vector. The biased elements are zeros.
1288 Node* zero = gvn().transform(gvn().zerocon(mem_elem_bt));
1289 zero = gvn().transform(VectorNode::scalar2vector(zero, mem_num_elem, Type::get_const_basic_type(mem_elem_bt)));
1290 vload = gvn().transform(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, mem_num_elem, mem_elem_bt));
1291 vload = gvn().transform(new VectorBlendNode(zero, vload, mask));
1292 }
1293
1294 if (mismatched_ms) {
1295 const TypeVect* to_vect_type = TypeVect::make(elem_bt, num_elem);
1296 vload = gvn().transform(new VectorReinterpretNode(vload, vload->bottom_type()->is_vect(), to_vect_type));
1297 }
1298
1299 Node* box = box_vector(vload, vbox_type, elem_bt, num_elem);
1300 set_result(box);
1301 }
1302
1303 destruct_map_clone(old_map);
1304
1305 if (can_access_non_heap) {
1306 insert_mem_bar(Op_MemBarCPUOrder);
1307 }
1308
1309 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1310 return true;
1311 }
1312
1313 // <C,
1314 // V extends Vector<?>,
1315 // W extends Vector<Integer>,
1316 // S extends VectorSpecies<E>,
1317 // M extends VectorMask<E>,
1318 // E>
1319 // V loadWithMap(Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType, int length,
1320 // Class<? extends Vector<Integer>> vectorIndexClass,
1321 // Object base, long offset, // Unsafe addressing
1322 // W index_vector, M m,
1323 // C container, int index, int[] indexMap, int indexM, S s, // Arguments for default implementation
1324 // LoadVectorOperationWithMap<C, V, E, S, M> defaultImpl)
1325 //
1326 // <C,
1327 // V extends Vector<E>,
1328 // W extends Vector<Integer>,
1329 // M extends VectorMask<E>,
1330 // E>
1331 // void storeWithMap(Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType,
1332 // int length, Class<? extends Vector<Integer>> vectorIndexClass, Object base, long offset, // Unsafe addressing
1333 // W index_vector, V v, M m,
1334 // C container, int index, int[] indexMap, int indexM, // Arguments for default implementation
1335 // StoreVectorOperationWithMap<C, V, M, E> defaultImpl)
1336 //
1337 bool LibraryCallKit::inline_vector_gather_scatter(bool is_scatter) {
1338 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
1339 const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
1340 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
1341 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
1342 const TypeInstPtr* vector_idx_klass = gvn().type(argument(4))->isa_instptr();
1343
1344 if (vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
1345 // mask_klass == nullptr || mask_klass->const_oop() == nullptr ||
1346 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
1347 vlen == nullptr || !vlen->is_con() ||
1348 vector_idx_klass == nullptr || vector_idx_klass->const_oop() == nullptr) {
1349 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s viclass=%s",
1350 NodeClassNames[argument(0)->Opcode()],
1351 NodeClassNames[argument(2)->Opcode()],
1352 NodeClassNames[argument(3)->Opcode()],
1353 NodeClassNames[argument(4)->Opcode()]);
1354 return false; // not enough info for intrinsification
1355 }
1356
1357 if (!is_klass_initialized(vector_klass) || !is_klass_initialized(vector_idx_klass)) {
1358 log_if_needed(" ** klass argument not initialized");
1359 return false;
1360 }
1361
1362 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1363 if (!elem_type->is_primitive_type()) {
1364 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1365 return false; // should be primitive type
1366 }
1367
1368 BasicType elem_bt = elem_type->basic_type();
1369 int num_elem = vlen->get_con();
1370
1371 const Type* vmask_type = gvn().type(is_scatter ? argument(10) : argument(9));
1372 bool is_masked_op = vmask_type != TypePtr::NULL_PTR;
1373 if (is_masked_op) {
1374 if (mask_klass == nullptr || mask_klass->const_oop() == nullptr) {
1375 log_if_needed(" ** missing constant: maskclass=%s", NodeClassNames[argument(1)->Opcode()]);
1376 return false; // not enough info for intrinsification
1377 }
1378
1379 if (!is_klass_initialized(mask_klass)) {
1380 log_if_needed(" ** mask klass argument not initialized");
1381 return false;
1382 }
1383
1384 if (vmask_type->maybe_null()) {
1385 log_if_needed(" ** null mask values are not allowed for masked op");
1386 return false;
1387 }
1388
1389 // Check whether the predicated gather/scatter node is supported by architecture.
1390 VectorMaskUseType mask = (VectorMaskUseType) (VecMaskUseLoad | VecMaskUsePred);
1391 if (!arch_supports_vector(is_scatter ? Op_StoreVectorScatterMasked : Op_LoadVectorGatherMasked, num_elem, elem_bt, mask)) {
1392 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d etype=%s is_masked_op=1",
1393 is_scatter, is_scatter ? "scatterMasked" : "gatherMasked",
1394 num_elem, type2name(elem_bt));
1395 return false; // not supported
1396 }
1397 } else {
1398 // Check whether the normal gather/scatter node is supported for non-masked operation.
1399 if (!arch_supports_vector(is_scatter ? Op_StoreVectorScatter : Op_LoadVectorGather, num_elem, elem_bt, VecMaskNotUsed)) {
1400 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d etype=%s is_masked_op=0",
1401 is_scatter, is_scatter ? "scatter" : "gather",
1402 num_elem, type2name(elem_bt));
1403 return false; // not supported
1404 }
1405 }
1406
1407 // Check that the vector holding indices is supported by architecture
1408 // For sub-word gathers expander receive index array.
1409 if (!is_subword_type(elem_bt) && !arch_supports_vector(Op_LoadVector, num_elem, T_INT, VecMaskNotUsed)) {
1410 log_if_needed(" ** not supported: arity=%d op=%s/loadindex vlen=%d etype=int is_masked_op=%d",
1411 is_scatter, is_scatter ? "scatter" : "gather",
1412 num_elem, is_masked_op ? 1 : 0);
1413 return false; // not supported
1414 }
1415
1416 Node* base = argument(5);
1417 Node* offset = ConvL2X(argument(6));
1418
1419 // Save state and restore on bailout
1420 uint old_sp = sp();
1421 SafePointNode* old_map = clone_map();
1422
1423 Node* addr = make_unsafe_address(base, offset, elem_bt, true);
1424
1425 const TypePtr *addr_type = gvn().type(addr)->isa_ptr();
1426 const TypeAryPtr* arr_type = addr_type->isa_aryptr();
1427
1428 // The array must be consistent with vector type
1429 if (arr_type == nullptr || (arr_type != nullptr && !elem_consistent_with_arr(elem_bt, arr_type, false))) {
1430 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d etype=%s atype=%s ismask=no",
1431 is_scatter, is_scatter ? "scatter" : "gather",
1432 num_elem, type2name(elem_bt), type2name(arr_type->elem()->array_element_basic_type()));
1433 set_map(old_map);
1434 set_sp(old_sp);
1435 return false;
1436 }
1437
1438 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1439 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1440 ciKlass* vbox_idx_klass = vector_idx_klass->const_oop()->as_instance()->java_lang_Class_klass();
1441 if (vbox_idx_klass == nullptr) {
1442 set_map(old_map);
1443 set_sp(old_sp);
1444 return false;
1445 }
1446
1447 Node* index_vect = nullptr;
1448 const TypeInstPtr* vbox_idx_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_idx_klass);
1449 if (!is_subword_type(elem_bt)) {
1450 index_vect = unbox_vector(argument(8), vbox_idx_type, T_INT, num_elem);
1451 if (index_vect == nullptr) {
1452 set_map(old_map);
1453 set_sp(old_sp);
1454 return false;
1455 }
1456 }
1457
1458 Node* mask = nullptr;
1459 if (is_masked_op) {
1460 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1461 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1462 mask = unbox_vector(is_scatter ? argument(10) : argument(9), mbox_type, elem_bt, num_elem);
1463 if (mask == nullptr) {
1464 log_if_needed(" ** unbox failed mask=%s",
1465 is_scatter ? NodeClassNames[argument(10)->Opcode()]
1466 : NodeClassNames[argument(9)->Opcode()]);
1467 set_map(old_map);
1468 set_sp(old_sp);
1469 return false;
1470 }
1471 }
1472
1473 const TypeVect* vector_type = TypeVect::make(elem_bt, num_elem);
1474 if (is_scatter) {
1475 Node* val = unbox_vector(argument(9), vbox_type, elem_bt, num_elem);
1476 if (val == nullptr) {
1477 set_map(old_map);
1478 set_sp(old_sp);
1479 return false; // operand unboxing failed
1480 }
1481 set_all_memory(reset_memory());
1482
1483 Node* vstore = nullptr;
1484 if (mask != nullptr) {
1485 vstore = gvn().transform(new StoreVectorScatterMaskedNode(control(), memory(addr), addr, addr_type, val, index_vect, mask));
1486 } else {
1487 vstore = gvn().transform(new StoreVectorScatterNode(control(), memory(addr), addr, addr_type, val, index_vect));
1488 }
1489 set_memory(vstore, addr_type);
1490 } else {
1491 Node* vload = nullptr;
1492 Node* index = argument(11);
1493 Node* indexMap = argument(12);
1494 Node* indexM = argument(13);
1495 if (mask != nullptr) {
1496 if (is_subword_type(elem_bt)) {
1497 Node* index_arr_base = array_element_address(indexMap, indexM, T_INT);
1498 vload = gvn().transform(new LoadVectorGatherMaskedNode(control(), memory(addr), addr, addr_type, vector_type, index_arr_base, mask, index));
1499 } else {
1500 vload = gvn().transform(new LoadVectorGatherMaskedNode(control(), memory(addr), addr, addr_type, vector_type, index_vect, mask));
1501 }
1502 } else {
1503 if (is_subword_type(elem_bt)) {
1504 Node* index_arr_base = array_element_address(indexMap, indexM, T_INT);
1505 vload = gvn().transform(new LoadVectorGatherNode(control(), memory(addr), addr, addr_type, vector_type, index_arr_base, index));
1506 } else {
1507 vload = gvn().transform(new LoadVectorGatherNode(control(), memory(addr), addr, addr_type, vector_type, index_vect));
1508 }
1509 }
1510 Node* box = box_vector(vload, vbox_type, elem_bt, num_elem);
1511 set_result(box);
1512 }
1513
1514 destruct_map_clone(old_map);
1515
1516 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1517 return true;
1518 }
1519
1520 // public static
1521 // <V extends Vector<E>,
1522 // M extends VectorMask<E>,
1523 // E>
1524 // long reductionCoerced(int oprId, Class<? extends V> vectorClass, Class<? extends M> maskClass,
1525 // Class<E> elementType, int length, V v, M m,
1526 // ReductionOperation<V, M> defaultImpl)
1527 bool LibraryCallKit::inline_vector_reduction() {
1528 const TypeInt* opr = gvn().type(argument(0))->isa_int();
1529 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
1530 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
1531 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
1532 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
1533
1534 if (opr == nullptr || !opr->is_con() ||
1535 vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
1536 // mask_klass == nullptr || mask_klass->const_oop() == nullptr ||
1537 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
1538 vlen == nullptr || !vlen->is_con()) {
1539 log_if_needed(" ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
1540 NodeClassNames[argument(0)->Opcode()],
1541 NodeClassNames[argument(1)->Opcode()],
1542 NodeClassNames[argument(3)->Opcode()],
1543 NodeClassNames[argument(4)->Opcode()]);
1544 return false; // not enough info for intrinsification
1545 }
1546 if (!is_klass_initialized(vector_klass)) {
1547 log_if_needed(" ** klass argument not initialized");
1548 return false;
1549 }
1550 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1551 if (!elem_type->is_primitive_type()) {
1552 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1553 return false; // should be primitive type
1554 }
1555
1556 const Type* vmask_type = gvn().type(argument(6));
1557 bool is_masked_op = vmask_type != TypePtr::NULL_PTR;
1558 if (is_masked_op) {
1559 if (mask_klass == nullptr || mask_klass->const_oop() == nullptr) {
1560 log_if_needed(" ** missing constant: maskclass=%s", NodeClassNames[argument(2)->Opcode()]);
1561 return false; // not enough info for intrinsification
1562 }
1563
1564 if (!is_klass_initialized(mask_klass)) {
1565 log_if_needed(" ** mask klass argument not initialized");
1566 return false;
1567 }
1568
1569 if (vmask_type->maybe_null()) {
1570 log_if_needed(" ** null mask values are not allowed for masked op");
1571 return false;
1572 }
1573 }
1574
1575 BasicType elem_bt = elem_type->basic_type();
1576 int num_elem = vlen->get_con();
1577 int opc = VectorSupport::vop2ideal(opr->get_con(), elem_bt);
1578 int sopc = ReductionNode::opcode(opc, elem_bt);
1579
1580 // When using mask, mask use type needs to be VecMaskUseLoad.
1581 if (!arch_supports_vector(sopc, num_elem, elem_bt, is_masked_op ? VecMaskUseLoad : VecMaskNotUsed)) {
1582 log_if_needed(" ** not supported: arity=1 op=%d/reduce vlen=%d etype=%s is_masked_op=%d",
1583 sopc, num_elem, type2name(elem_bt), is_masked_op ? 1 : 0);
1584 return false;
1585 }
1586
1587 // Return true if current platform has implemented the masked operation with predicate feature.
1588 bool use_predicate = is_masked_op && arch_supports_vector(sopc, num_elem, elem_bt, VecMaskUsePred);
1589 if (is_masked_op && !use_predicate && !arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad)) {
1590 log_if_needed(" ** not supported: arity=1 op=%d/reduce vlen=%d etype=%s is_masked_op=1",
1591 sopc, num_elem, type2name(elem_bt));
1592 return false;
1593 }
1594
1595 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1596 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1597
1598 Node* opd = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
1599 if (opd == nullptr) {
1600 return false; // operand unboxing failed
1601 }
1602
1603 Node* mask = nullptr;
1604 if (is_masked_op) {
1605 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1606 assert(is_vector_mask(mbox_klass), "argument(2) should be a mask class");
1607 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1608 mask = unbox_vector(argument(6), mbox_type, elem_bt, num_elem);
1609 if (mask == nullptr) {
1610 log_if_needed(" ** unbox failed mask=%s",
1611 NodeClassNames[argument(6)->Opcode()]);
1612 return false;
1613 }
1614 }
1615
1616 Node* init = ReductionNode::make_identity_con_scalar(gvn(), opc, elem_bt);
1617 Node* value = nullptr;
1618 if (mask == nullptr) {
1619 assert(!is_masked_op, "Masked op needs the mask value never null");
1620 value = ReductionNode::make(opc, nullptr, init, opd, elem_bt);
1621 } else {
1622 if (use_predicate) {
1623 value = ReductionNode::make(opc, nullptr, init, opd, elem_bt);
1624 value->add_req(mask);
1625 value->add_flag(Node::Flag_is_predicated_vector);
1626 } else {
1627 Node* reduce_identity = gvn().transform(VectorNode::scalar2vector(init, num_elem, Type::get_const_basic_type(elem_bt)));
1628 value = gvn().transform(new VectorBlendNode(reduce_identity, opd, mask));
1629 value = ReductionNode::make(opc, nullptr, init, value, elem_bt);
1630 }
1631 }
1632 value = gvn().transform(value);
1633
1634 Node* bits = nullptr;
1635 switch (elem_bt) {
1636 case T_BYTE:
1637 case T_SHORT:
1638 case T_INT: {
1639 bits = gvn().transform(new ConvI2LNode(value));
1640 break;
1641 }
1642 case T_FLOAT: {
1643 value = gvn().transform(new MoveF2INode(value));
1644 bits = gvn().transform(new ConvI2LNode(value));
1645 break;
1646 }
1647 case T_DOUBLE: {
1648 bits = gvn().transform(new MoveD2LNode(value));
1649 break;
1650 }
1651 case T_LONG: {
1652 bits = value; // no conversion needed
1653 break;
1654 }
1655 default: fatal("%s", type2name(elem_bt));
1656 }
1657 set_result(bits);
1658 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1659 return true;
1660 }
1661
1662 // public static <V> boolean test(int cond, Class<?> vectorClass, Class<?> elementType, int vlen,
1663 // V v1, V v2,
1664 // BiFunction<V, V, Boolean> defaultImpl)
1665 //
1666 bool LibraryCallKit::inline_vector_test() {
1667 const TypeInt* cond = gvn().type(argument(0))->isa_int();
1668 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
1669 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
1670 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
1671
1672 if (cond == nullptr || !cond->is_con() ||
1673 vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
1674 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
1675 vlen == nullptr || !vlen->is_con()) {
1676 log_if_needed(" ** missing constant: cond=%s vclass=%s etype=%s vlen=%s",
1677 NodeClassNames[argument(0)->Opcode()],
1678 NodeClassNames[argument(1)->Opcode()],
1679 NodeClassNames[argument(2)->Opcode()],
1680 NodeClassNames[argument(3)->Opcode()]);
1681 return false; // not enough info for intrinsification
1682 }
1683 if (!is_klass_initialized(vector_klass)) {
1684 log_if_needed(" ** klass argument not initialized");
1685 return false;
1686 }
1687 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1688 if (!elem_type->is_primitive_type()) {
1689 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1690 return false; // should be primitive type
1691 }
1692 BasicType elem_bt = elem_type->basic_type();
1693 int num_elem = vlen->get_con();
1694 BoolTest::mask booltest = (BoolTest::mask)cond->get_con();
1695 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1696 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1697
1698 if (!arch_supports_vector(Op_VectorTest, num_elem, elem_bt, is_vector_mask(vbox_klass) ? VecMaskUseLoad : VecMaskNotUsed)) {
1699 log_if_needed(" ** not supported: arity=2 op=test/%d vlen=%d etype=%s ismask=%d",
1700 cond->get_con(), num_elem, type2name(elem_bt),
1701 is_vector_mask(vbox_klass));
1702 return false;
1703 }
1704
1705 Node* opd1 = unbox_vector(argument(4), vbox_type, elem_bt, num_elem);
1706 Node* opd2;
1707 if (Matcher::vectortest_needs_second_argument(booltest == BoolTest::overflow,
1708 opd1->bottom_type()->isa_vectmask())) {
1709 opd2 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
1710 } else {
1711 opd2 = opd1;
1712 }
1713 if (opd1 == nullptr || opd2 == nullptr) {
1714 return false; // operand unboxing failed
1715 }
1716
1717 Node* cmp = gvn().transform(new VectorTestNode(opd1, opd2, booltest));
1718 BoolTest::mask test = Matcher::vectortest_mask(booltest == BoolTest::overflow,
1719 opd1->bottom_type()->isa_vectmask(), num_elem);
1720 Node* bol = gvn().transform(new BoolNode(cmp, test));
1721 Node* res = gvn().transform(new CMoveINode(bol, gvn().intcon(0), gvn().intcon(1), TypeInt::BOOL));
1722
1723 set_result(res);
1724 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1725 return true;
1726 }
1727
1728 // public static
1729 // <V extends Vector<E>,
1730 // M extends VectorMask<E>,
1731 // E>
1732 // V blend(Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType, int vlen,
1733 // V v1, V v2, M m,
1734 // VectorBlendOp<V, M, E> defaultImpl)
1735 bool LibraryCallKit::inline_vector_blend() {
1736 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
1737 const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
1738 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
1739 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
1740
1741 if (mask_klass == nullptr || vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr) {
1742 return false; // dead code
1743 }
1744 if (mask_klass->const_oop() == nullptr || vector_klass->const_oop() == nullptr ||
1745 elem_klass->const_oop() == nullptr || !vlen->is_con()) {
1746 log_if_needed(" ** missing constant: vclass=%s mclass=%s etype=%s vlen=%s",
1747 NodeClassNames[argument(0)->Opcode()],
1748 NodeClassNames[argument(1)->Opcode()],
1749 NodeClassNames[argument(2)->Opcode()],
1750 NodeClassNames[argument(3)->Opcode()]);
1751 return false; // not enough info for intrinsification
1752 }
1753 if (!is_klass_initialized(vector_klass) || !is_klass_initialized(mask_klass)) {
1754 log_if_needed(" ** klass argument not initialized");
1755 return false;
1756 }
1757 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1758 if (!elem_type->is_primitive_type()) {
1759 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1760 return false; // should be primitive type
1761 }
1762 BasicType elem_bt = elem_type->basic_type();
1763 BasicType mask_bt = elem_bt;
1764 int num_elem = vlen->get_con();
1765
1766 if (!arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad)) {
1767 log_if_needed(" ** not supported: arity=2 op=blend vlen=%d etype=%s ismask=useload",
1768 num_elem, type2name(elem_bt));
1769 return false; // not supported
1770 }
1771 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1772 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1773
1774 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1775 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1776
1777 Node* v1 = unbox_vector(argument(4), vbox_type, elem_bt, num_elem);
1778 Node* v2 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
1779 Node* mask = unbox_vector(argument(6), mbox_type, mask_bt, num_elem);
1780
1781 if (v1 == nullptr || v2 == nullptr || mask == nullptr) {
1782 return false; // operand unboxing failed
1783 }
1784
1785 Node* blend = gvn().transform(new VectorBlendNode(v1, v2, mask));
1786
1787 Node* box = box_vector(blend, vbox_type, elem_bt, num_elem);
1788 set_result(box);
1789 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1790 return true;
1791 }
1792
1793 // public static
1794 // <V extends Vector<E>,
1795 // M extends VectorMask<E>,
1796 // E>
1797 // M compare(int cond, Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType, int vlen,
1798 // V v1, V v2, M m,
1799 // VectorCompareOp<V,M> defaultImpl)
1800 bool LibraryCallKit::inline_vector_compare() {
1801 const TypeInt* cond = gvn().type(argument(0))->isa_int();
1802 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
1803 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
1804 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
1805 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
1806
1807 if (cond == nullptr || vector_klass == nullptr || mask_klass == nullptr || elem_klass == nullptr || vlen == nullptr) {
1808 return false; // dead code
1809 }
1810 if (!cond->is_con() || vector_klass->const_oop() == nullptr || mask_klass->const_oop() == nullptr ||
1811 elem_klass->const_oop() == nullptr || !vlen->is_con()) {
1812 log_if_needed(" ** missing constant: cond=%s vclass=%s mclass=%s etype=%s vlen=%s",
1813 NodeClassNames[argument(0)->Opcode()],
1814 NodeClassNames[argument(1)->Opcode()],
1815 NodeClassNames[argument(2)->Opcode()],
1816 NodeClassNames[argument(3)->Opcode()],
1817 NodeClassNames[argument(4)->Opcode()]);
1818 return false; // not enough info for intrinsification
1819 }
1820 if (!is_klass_initialized(vector_klass) || !is_klass_initialized(mask_klass)) {
1821 log_if_needed(" ** klass argument not initialized");
1822 return false;
1823 }
1824 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1825 if (!elem_type->is_primitive_type()) {
1826 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1827 return false; // should be primitive type
1828 }
1829
1830 int num_elem = vlen->get_con();
1831 BasicType elem_bt = elem_type->basic_type();
1832 BasicType mask_bt = elem_bt;
1833
1834 if ((cond->get_con() & BoolTest::unsigned_compare) != 0) {
1835 if (!Matcher::supports_vector_comparison_unsigned(num_elem, elem_bt)) {
1836 log_if_needed(" ** not supported: unsigned comparison op=comp/%d vlen=%d etype=%s ismask=usestore",
1837 cond->get_con() & (BoolTest::unsigned_compare - 1), num_elem, type2name(elem_bt));
1838 return false;
1839 }
1840 }
1841
1842 if (!arch_supports_vector(Op_VectorMaskCmp, num_elem, elem_bt, VecMaskUseStore)) {
1843 log_if_needed(" ** not supported: arity=2 op=comp/%d vlen=%d etype=%s ismask=usestore",
1844 cond->get_con(), num_elem, type2name(elem_bt));
1845 return false;
1846 }
1847
1848 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1849 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1850
1851 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1852 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1853
1854 Node* v1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
1855 Node* v2 = unbox_vector(argument(6), vbox_type, elem_bt, num_elem);
1856
1857 bool is_masked_op = argument(7)->bottom_type() != TypePtr::NULL_PTR;
1858 Node* mask = is_masked_op ? unbox_vector(argument(7), mbox_type, elem_bt, num_elem) : nullptr;
1859 if (is_masked_op && mask == nullptr) {
1860 log_if_needed(" ** not supported: mask = null arity=2 op=comp/%d vlen=%d etype=%s ismask=usestore is_masked_op=1",
1861 cond->get_con(), num_elem, type2name(elem_bt));
1862 return false;
1863 }
1864
1865 bool use_predicate = is_masked_op && arch_supports_vector(Op_VectorMaskCmp, num_elem, elem_bt, VecMaskUsePred);
1866 if (is_masked_op && !use_predicate && !arch_supports_vector(Op_AndV, num_elem, elem_bt, VecMaskUseLoad)) {
1867 log_if_needed(" ** not supported: arity=2 op=comp/%d vlen=%d etype=%s ismask=usestore is_masked_op=1",
1868 cond->get_con(), num_elem, type2name(elem_bt));
1869 return false;
1870 }
1871
1872 if (v1 == nullptr || v2 == nullptr) {
1873 return false; // operand unboxing failed
1874 }
1875 BoolTest::mask pred = (BoolTest::mask)cond->get_con();
1876 ConINode* pred_node = (ConINode*)gvn().makecon(cond);
1877
1878 const TypeVect* vmask_type = TypeVect::makemask(mask_bt, num_elem);
1879 Node* operation = new VectorMaskCmpNode(pred, v1, v2, pred_node, vmask_type);
1880
1881 if (is_masked_op) {
1882 if (use_predicate) {
1883 operation->add_req(mask);
1884 operation->add_flag(Node::Flag_is_predicated_vector);
1885 } else {
1886 operation = gvn().transform(operation);
1887 operation = VectorNode::make(Op_AndV, operation, mask, vmask_type);
1888 }
1889 }
1890
1891 operation = gvn().transform(operation);
1892
1893 Node* box = box_vector(operation, mbox_type, mask_bt, num_elem);
1894 set_result(box);
1895 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1896 return true;
1897 }
1898
1899 // public static
1900 // <V extends Vector<E>,
1901 // Sh extends VectorShuffle<E>,
1902 // M extends VectorMask<E>,
1903 // E>
1904 // V rearrangeOp(Class<? extends V> vectorClass, Class<Sh> shuffleClass, Class<M> maskClass, Class<E> elementType, int vlen,
1905 // V v1, Sh sh, M m,
1906 // VectorRearrangeOp<V, Sh, M, E> defaultImpl)
1907 bool LibraryCallKit::inline_vector_rearrange() {
1908 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
1909 const TypeInstPtr* shuffle_klass = gvn().type(argument(1))->isa_instptr();
1910 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
1911 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
1912 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
1913
1914 if (vector_klass == nullptr || shuffle_klass == nullptr || elem_klass == nullptr || vlen == nullptr) {
1915 return false; // dead code
1916 }
1917 if (shuffle_klass->const_oop() == nullptr ||
1918 vector_klass->const_oop() == nullptr ||
1919 elem_klass->const_oop() == nullptr ||
1920 !vlen->is_con()) {
1921 log_if_needed(" ** missing constant: vclass=%s sclass=%s etype=%s vlen=%s",
1922 NodeClassNames[argument(0)->Opcode()],
1923 NodeClassNames[argument(1)->Opcode()],
1924 NodeClassNames[argument(3)->Opcode()],
1925 NodeClassNames[argument(4)->Opcode()]);
1926 return false; // not enough info for intrinsification
1927 }
1928 if (!is_klass_initialized(vector_klass) ||
1929 !is_klass_initialized(shuffle_klass)) {
1930 log_if_needed(" ** klass argument not initialized");
1931 return false;
1932 }
1933 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1934 if (!elem_type->is_primitive_type()) {
1935 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1936 return false; // should be primitive type
1937 }
1938 BasicType elem_bt = elem_type->basic_type();
1939 BasicType shuffle_bt = elem_bt;
1940 int num_elem = vlen->get_con();
1941
1942 if (!arch_supports_vector(Op_VectorLoadShuffle, num_elem, elem_bt, VecMaskNotUsed)) {
1943 log_if_needed(" ** not supported: arity=0 op=load/shuffle vlen=%d etype=%s ismask=no",
1944 num_elem, type2name(elem_bt));
1945 return false; // not supported
1946 }
1947
1948 bool is_masked_op = argument(7)->bottom_type() != TypePtr::NULL_PTR;
1949 bool use_predicate = is_masked_op;
1950 if (is_masked_op &&
1951 (mask_klass == nullptr ||
1952 mask_klass->const_oop() == nullptr ||
1953 !is_klass_initialized(mask_klass))) {
1954 log_if_needed(" ** mask_klass argument not initialized");
1955 }
1956 VectorMaskUseType checkFlags = (VectorMaskUseType)(is_masked_op ? (VecMaskUseLoad | VecMaskUsePred) : VecMaskNotUsed);
1957 if (!arch_supports_vector(Op_VectorRearrange, num_elem, elem_bt, checkFlags)) {
1958 use_predicate = false;
1959 if(!is_masked_op ||
1960 (!arch_supports_vector(Op_VectorRearrange, num_elem, elem_bt, VecMaskNotUsed) ||
1961 !arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad) ||
1962 !arch_supports_vector(Op_Replicate, num_elem, elem_bt, VecMaskNotUsed))) {
1963 log_if_needed(" ** not supported: arity=2 op=shuffle/rearrange vlen=%d etype=%s ismask=no",
1964 num_elem, type2name(elem_bt));
1965 return false; // not supported
1966 }
1967 }
1968 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1969 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1970
1971 ciKlass* shbox_klass = shuffle_klass->const_oop()->as_instance()->java_lang_Class_klass();
1972 const TypeInstPtr* shbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, shbox_klass);
1973
1974 Node* v1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
1975 Node* shuffle = unbox_vector(argument(6), shbox_type, shuffle_bt, num_elem);
1976
1977 if (v1 == nullptr || shuffle == nullptr) {
1978 return false; // operand unboxing failed
1979 }
1980
1981 Node* mask = nullptr;
1982 if (is_masked_op) {
1983 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1984 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1985 mask = unbox_vector(argument(7), mbox_type, elem_bt, num_elem);
1986 if (mask == nullptr) {
1987 log_if_needed(" ** not supported: arity=3 op=shuffle/rearrange vlen=%d etype=%s ismask=useload is_masked_op=1",
1988 num_elem, type2name(elem_bt));
1989 return false;
1990 }
1991 }
1992
1993 Node* rearrange = new VectorRearrangeNode(v1, shuffle);
1994 if (is_masked_op) {
1995 if (use_predicate) {
1996 rearrange->add_req(mask);
1997 rearrange->add_flag(Node::Flag_is_predicated_vector);
1998 } else {
1999 const TypeVect* vt = v1->bottom_type()->is_vect();
2000 rearrange = gvn().transform(rearrange);
2001 Node* zero = gvn().makecon(Type::get_zero_type(elem_bt));
2002 Node* zerovec = gvn().transform(VectorNode::scalar2vector(zero, num_elem, Type::get_const_basic_type(elem_bt)));
2003 rearrange = new VectorBlendNode(zerovec, rearrange, mask);
2004 }
2005 }
2006 rearrange = gvn().transform(rearrange);
2007
2008 Node* box = box_vector(rearrange, vbox_type, elem_bt, num_elem);
2009 set_result(box);
2010 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2011 return true;
2012 }
2013
2014 static address get_svml_address(int vop, int bits, BasicType bt, char* name_ptr, int name_len) {
2015 address addr = nullptr;
2016 assert(UseVectorStubs, "sanity");
2017 assert(name_ptr != nullptr, "unexpected");
2018 assert((vop >= VectorSupport::VECTOR_OP_SVML_START) && (vop <= VectorSupport::VECTOR_OP_SVML_END), "unexpected");
2019 int op = vop - VectorSupport::VECTOR_OP_SVML_START;
2020
2021 switch(bits) {
2022 case 64: //fallthough
2023 case 128: //fallthough
2024 case 256: //fallthough
2025 case 512:
2026 if (bt == T_FLOAT) {
2027 snprintf(name_ptr, name_len, "vector_%s_float%d", VectorSupport::svmlname[op], bits);
2028 addr = StubRoutines::_vector_f_math[exact_log2(bits/64)][op];
2029 } else {
2030 assert(bt == T_DOUBLE, "must be FP type only");
2031 snprintf(name_ptr, name_len, "vector_%s_double%d", VectorSupport::svmlname[op], bits);
2032 addr = StubRoutines::_vector_d_math[exact_log2(bits/64)][op];
2033 }
2034 break;
2035 default:
2036 snprintf(name_ptr, name_len, "invalid");
2037 addr = nullptr;
2038 Unimplemented();
2039 break;
2040 }
2041
2042 return addr;
2043 }
2044
2045 Node* LibraryCallKit::gen_call_to_svml(int vector_api_op_id, BasicType bt, int num_elem, Node* opd1, Node* opd2) {
2046 assert(UseVectorStubs, "sanity");
2047 assert(vector_api_op_id >= VectorSupport::VECTOR_OP_SVML_START && vector_api_op_id <= VectorSupport::VECTOR_OP_SVML_END, "need valid op id");
2048 assert(opd1 != nullptr, "must not be null");
2049 const TypeVect* vt = TypeVect::make(bt, num_elem);
2050 const TypeFunc* call_type = OptoRuntime::Math_Vector_Vector_Type(opd2 != nullptr ? 2 : 1, vt, vt);
2051 char name[100] = "";
2052
2053 // Get address for svml method.
2054 address addr = get_svml_address(vector_api_op_id, vt->length_in_bytes() * BitsPerByte, bt, name, 100);
2055
2056 if (addr == nullptr) {
2057 return nullptr;
2058 }
2059
2060 assert(name[0] != '\0', "name must not be null");
2061 Node* operation = make_runtime_call(RC_VECTOR,
2062 call_type,
2063 addr,
2064 name,
2065 TypePtr::BOTTOM,
2066 opd1,
2067 opd2);
2068 return gvn().transform(new ProjNode(gvn().transform(operation), TypeFunc::Parms));
2069 }
2070
2071 // public static
2072 // <V extends Vector<E>,
2073 // M extends VectorMask<E>,
2074 // E>
2075 // V broadcastInt(int opr, Class<? extends V> vectorClass, Class<? extends M> maskClass,
2076 // Class<E> elementType, int length,
2077 // V v, int n, M m,
2078 // VectorBroadcastIntOp<V, M> defaultImpl)
2079 bool LibraryCallKit::inline_vector_broadcast_int() {
2080 const TypeInt* opr = gvn().type(argument(0))->isa_int();
2081 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
2082 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
2083 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
2084 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
2085
2086 if (opr == nullptr || vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr) {
2087 return false; // dead code
2088 }
2089 if (!opr->is_con() || vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
2090 log_if_needed(" ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
2091 NodeClassNames[argument(0)->Opcode()],
2092 NodeClassNames[argument(1)->Opcode()],
2093 NodeClassNames[argument(3)->Opcode()],
2094 NodeClassNames[argument(4)->Opcode()]);
2095 return false; // not enough info for intrinsification
2096 }
2097 if (!is_klass_initialized(vector_klass)) {
2098 log_if_needed(" ** klass argument not initialized");
2099 return false;
2100 }
2101
2102 const Type* vmask_type = gvn().type(argument(7));
2103 bool is_masked_op = vmask_type != TypePtr::NULL_PTR;
2104 if (is_masked_op) {
2105 if (mask_klass == nullptr || mask_klass->const_oop() == nullptr) {
2106 log_if_needed(" ** missing constant: maskclass=%s", NodeClassNames[argument(2)->Opcode()]);
2107 return false; // not enough info for intrinsification
2108 }
2109
2110 if (!is_klass_initialized(mask_klass)) {
2111 log_if_needed(" ** mask klass argument not initialized");
2112 return false;
2113 }
2114
2115 if (vmask_type->maybe_null()) {
2116 log_if_needed(" ** null mask values are not allowed for masked op");
2117 return false;
2118 }
2119 }
2120
2121 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2122 if (!elem_type->is_primitive_type()) {
2123 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2124 return false; // should be primitive type
2125 }
2126
2127 int num_elem = vlen->get_con();
2128 BasicType elem_bt = elem_type->basic_type();
2129 int opc = VectorSupport::vop2ideal(opr->get_con(), elem_bt);
2130
2131 bool is_shift = VectorNode::is_shift_opcode(opc);
2132 bool is_rotate = VectorNode::is_rotate_opcode(opc);
2133
2134 if (opc == 0 || (!is_shift && !is_rotate)) {
2135 log_if_needed(" ** operation not supported: op=%d bt=%s", opr->get_con(), type2name(elem_bt));
2136 return false; // operation not supported
2137 }
2138
2139 int sopc = VectorNode::opcode(opc, elem_bt);
2140 if (sopc == 0) {
2141 log_if_needed(" ** operation not supported: opc=%s bt=%s", NodeClassNames[opc], type2name(elem_bt));
2142 return false; // operation not supported
2143 }
2144
2145 Node* cnt = argument(6);
2146 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2147 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2148 const TypeInt* cnt_type = cnt->bottom_type()->isa_int();
2149
2150 // If CPU supports vector constant rotate instructions pass it directly
2151 bool is_const_rotate = is_rotate && cnt_type && cnt_type->is_con() &&
2152 Matcher::supports_vector_constant_rotates(cnt_type->get_con());
2153 bool has_scalar_args = is_rotate ? !is_const_rotate : true;
2154
2155 VectorMaskUseType checkFlags = (VectorMaskUseType)(is_masked_op ? (VecMaskUseLoad | VecMaskUsePred) : VecMaskNotUsed);
2156 bool use_predicate = is_masked_op;
2157
2158 if (!arch_supports_vector(sopc, num_elem, elem_bt, checkFlags, has_scalar_args)) {
2159 use_predicate = false;
2160 if (!is_masked_op ||
2161 (!arch_supports_vector(sopc, num_elem, elem_bt, VecMaskNotUsed, has_scalar_args) ||
2162 !arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad))) {
2163
2164 log_if_needed(" ** not supported: arity=0 op=int/%d vlen=%d etype=%s is_masked_op=%d",
2165 sopc, num_elem, type2name(elem_bt), is_masked_op ? 1 : 0);
2166 return false; // not supported
2167 }
2168 }
2169
2170 Node* opd1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
2171 Node* opd2 = nullptr;
2172 if (is_shift) {
2173 opd2 = vector_shift_count(cnt, opc, elem_bt, num_elem);
2174 } else {
2175 assert(is_rotate, "unexpected operation");
2176 if (!is_const_rotate) {
2177 const Type * type_bt = Type::get_const_basic_type(elem_bt);
2178 cnt = elem_bt == T_LONG ? gvn().transform(new ConvI2LNode(cnt)) : cnt;
2179 opd2 = gvn().transform(VectorNode::scalar2vector(cnt, num_elem, type_bt));
2180 } else {
2181 // Constant shift value.
2182 opd2 = cnt;
2183 }
2184 }
2185
2186 if (opd1 == nullptr || opd2 == nullptr) {
2187 return false;
2188 }
2189
2190 Node* mask = nullptr;
2191 if (is_masked_op) {
2192 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
2193 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
2194 mask = unbox_vector(argument(7), mbox_type, elem_bt, num_elem);
2195 if (mask == nullptr) {
2196 log_if_needed(" ** unbox failed mask=%s", NodeClassNames[argument(7)->Opcode()]);
2197 return false;
2198 }
2199 }
2200
2201 Node* operation = VectorNode::make(opc, opd1, opd2, num_elem, elem_bt);
2202 if (is_masked_op && mask != nullptr) {
2203 if (use_predicate) {
2204 operation->add_req(mask);
2205 operation->add_flag(Node::Flag_is_predicated_vector);
2206 } else {
2207 operation = gvn().transform(operation);
2208 operation = new VectorBlendNode(opd1, operation, mask);
2209 }
2210 }
2211 operation = gvn().transform(operation);
2212 Node* vbox = box_vector(operation, vbox_type, elem_bt, num_elem);
2213 set_result(vbox);
2214 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2215 return true;
2216 }
2217
2218 // public static <VOUT extends VectorPayload,
2219 // VIN extends VectorPayload,
2220 // S extends VectorSpecies>
2221 // VOUT convert(int oprId,
2222 // Class<?> fromVectorClass, Class<?> fromElementType, int fromVLen,
2223 // Class<?> toVectorClass, Class<?> toElementType, int toVLen,
2224 // VIN v, S s,
2225 // VectorConvertOp<VOUT, VIN, S> defaultImpl)
2226 //
2227 bool LibraryCallKit::inline_vector_convert() {
2228 const TypeInt* opr = gvn().type(argument(0))->isa_int();
2229
2230 const TypeInstPtr* vector_klass_from = gvn().type(argument(1))->isa_instptr();
2231 const TypeInstPtr* elem_klass_from = gvn().type(argument(2))->isa_instptr();
2232 const TypeInt* vlen_from = gvn().type(argument(3))->isa_int();
2233
2234 const TypeInstPtr* vector_klass_to = gvn().type(argument(4))->isa_instptr();
2235 const TypeInstPtr* elem_klass_to = gvn().type(argument(5))->isa_instptr();
2236 const TypeInt* vlen_to = gvn().type(argument(6))->isa_int();
2237
2238 if (opr == nullptr ||
2239 vector_klass_from == nullptr || elem_klass_from == nullptr || vlen_from == nullptr ||
2240 vector_klass_to == nullptr || elem_klass_to == nullptr || vlen_to == nullptr) {
2241 return false; // dead code
2242 }
2243 if (!opr->is_con() ||
2244 vector_klass_from->const_oop() == nullptr || elem_klass_from->const_oop() == nullptr || !vlen_from->is_con() ||
2245 vector_klass_to->const_oop() == nullptr || elem_klass_to->const_oop() == nullptr || !vlen_to->is_con()) {
2246 log_if_needed(" ** missing constant: opr=%s vclass_from=%s etype_from=%s vlen_from=%s vclass_to=%s etype_to=%s vlen_to=%s",
2247 NodeClassNames[argument(0)->Opcode()],
2248 NodeClassNames[argument(1)->Opcode()],
2249 NodeClassNames[argument(2)->Opcode()],
2250 NodeClassNames[argument(3)->Opcode()],
2251 NodeClassNames[argument(4)->Opcode()],
2252 NodeClassNames[argument(5)->Opcode()],
2253 NodeClassNames[argument(6)->Opcode()]);
2254 return false; // not enough info for intrinsification
2255 }
2256 if (!is_klass_initialized(vector_klass_from) || !is_klass_initialized(vector_klass_to)) {
2257 log_if_needed(" ** klass argument not initialized");
2258 return false;
2259 }
2260
2261 assert(opr->get_con() == VectorSupport::VECTOR_OP_CAST ||
2262 opr->get_con() == VectorSupport::VECTOR_OP_UCAST ||
2263 opr->get_con() == VectorSupport::VECTOR_OP_REINTERPRET, "wrong opcode");
2264 bool is_cast = (opr->get_con() == VectorSupport::VECTOR_OP_CAST || opr->get_con() == VectorSupport::VECTOR_OP_UCAST);
2265 bool is_ucast = (opr->get_con() == VectorSupport::VECTOR_OP_UCAST);
2266
2267 ciKlass* vbox_klass_from = vector_klass_from->const_oop()->as_instance()->java_lang_Class_klass();
2268 ciKlass* vbox_klass_to = vector_klass_to->const_oop()->as_instance()->java_lang_Class_klass();
2269 if (is_vector_shuffle(vbox_klass_from)) {
2270 return false; // vector shuffles aren't supported
2271 }
2272 bool is_mask = is_vector_mask(vbox_klass_from);
2273
2274 ciType* elem_type_from = elem_klass_from->const_oop()->as_instance()->java_mirror_type();
2275 if (!elem_type_from->is_primitive_type()) {
2276 return false; // should be primitive type
2277 }
2278 BasicType elem_bt_from = elem_type_from->basic_type();
2279 ciType* elem_type_to = elem_klass_to->const_oop()->as_instance()->java_mirror_type();
2280 if (!elem_type_to->is_primitive_type()) {
2281 return false; // should be primitive type
2282 }
2283 BasicType elem_bt_to = elem_type_to->basic_type();
2284
2285 int num_elem_from = vlen_from->get_con();
2286 int num_elem_to = vlen_to->get_con();
2287
2288 // Check whether we can unbox to appropriate size. Even with casting, checking for reinterpret is needed
2289 // since we may need to change size.
2290 if (!arch_supports_vector(Op_VectorReinterpret,
2291 num_elem_from,
2292 elem_bt_from,
2293 is_mask ? VecMaskUseAll : VecMaskNotUsed)) {
2294 log_if_needed(" ** not supported: arity=1 op=%s/1 vlen1=%d etype1=%s ismask=%d",
2295 is_cast ? "cast" : "reinterpret",
2296 num_elem_from, type2name(elem_bt_from), is_mask);
2297 return false;
2298 }
2299
2300 // Check whether we can support resizing/reinterpreting to the new size.
2301 if (!arch_supports_vector(Op_VectorReinterpret,
2302 num_elem_to,
2303 elem_bt_to,
2304 is_mask ? VecMaskUseAll : VecMaskNotUsed)) {
2305 log_if_needed(" ** not supported: arity=1 op=%s/2 vlen2=%d etype2=%s ismask=%d",
2306 is_cast ? "cast" : "reinterpret",
2307 num_elem_to, type2name(elem_bt_to), is_mask);
2308 return false;
2309 }
2310
2311 // At this point, we know that both input and output vector registers are supported
2312 // by the architecture. Next check if the casted type is simply to same type - which means
2313 // that it is actually a resize and not a cast.
2314 if (is_cast && elem_bt_from == elem_bt_to) {
2315 is_cast = false;
2316 }
2317
2318 const TypeInstPtr* vbox_type_from = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass_from);
2319
2320 Node* opd1 = unbox_vector(argument(7), vbox_type_from, elem_bt_from, num_elem_from);
2321 if (opd1 == nullptr) {
2322 return false;
2323 }
2324
2325 const TypeVect* src_type = TypeVect::make(elem_bt_from, num_elem_from, is_mask);
2326 const TypeVect* dst_type = TypeVect::make(elem_bt_to, num_elem_to, is_mask);
2327
2328 // Safety check to prevent casting if source mask is of type vector
2329 // and destination mask of type predicate vector and vice-versa.
2330 // From X86 standpoint, this case will only arise over KNL target,
2331 // where certain masks (depending on the species) are either propagated
2332 // through a vector or predicate register.
2333 if (is_mask &&
2334 ((src_type->isa_vectmask() == nullptr && dst_type->isa_vectmask()) ||
2335 (dst_type->isa_vectmask() == nullptr && src_type->isa_vectmask()))) {
2336 return false;
2337 }
2338
2339 Node* op = opd1;
2340 if (is_cast) {
2341 assert(!is_mask || num_elem_from == num_elem_to, "vector mask cast needs the same elem num");
2342 int cast_vopc = VectorCastNode::opcode(-1, elem_bt_from, !is_ucast);
2343
2344 // Make sure that vector cast is implemented to particular type/size combination if it is
2345 // not a mask casting.
2346 if (!is_mask && !arch_supports_vector(cast_vopc, num_elem_to, elem_bt_to, VecMaskNotUsed)) {
2347 log_if_needed(" ** not supported: arity=1 op=cast#%d/3 vlen2=%d etype2=%s ismask=%d",
2348 cast_vopc, num_elem_to, type2name(elem_bt_to), is_mask);
2349 return false;
2350 }
2351
2352 if (num_elem_from < num_elem_to) {
2353 // Since input and output number of elements are not consistent, we need to make sure we
2354 // properly size. Thus, first make a cast that retains the number of elements from source.
2355 int num_elem_for_cast = num_elem_from;
2356
2357 // It is possible that arch does not support this intermediate vector size
2358 // TODO More complex logic required here to handle this corner case for the sizes.
2359 if (!arch_supports_vector(cast_vopc, num_elem_for_cast, elem_bt_to, VecMaskNotUsed)) {
2360 log_if_needed(" ** not supported: arity=1 op=cast#%d/4 vlen1=%d etype2=%s ismask=%d",
2361 cast_vopc,
2362 num_elem_for_cast, type2name(elem_bt_to), is_mask);
2363 return false;
2364 }
2365
2366 op = gvn().transform(VectorCastNode::make(cast_vopc, op, elem_bt_to, num_elem_for_cast));
2367 // Now ensure that the destination gets properly resized to needed size.
2368 op = gvn().transform(new VectorReinterpretNode(op, op->bottom_type()->is_vect(), dst_type));
2369 } else if (num_elem_from > num_elem_to) {
2370 // Since number of elements from input is larger than output, simply reduce size of input
2371 // (we are supposed to drop top elements anyway).
2372 int num_elem_for_resize = num_elem_to;
2373
2374 // It is possible that arch does not support this intermediate vector size
2375 // TODO More complex logic required here to handle this corner case for the sizes.
2376 if (!arch_supports_vector(Op_VectorReinterpret,
2377 num_elem_for_resize,
2378 elem_bt_from,
2379 VecMaskNotUsed)) {
2380 log_if_needed(" ** not supported: arity=1 op=cast/5 vlen2=%d etype1=%s ismask=%d",
2381 num_elem_for_resize, type2name(elem_bt_from), is_mask);
2382 return false;
2383 }
2384
2385 const TypeVect* resize_type = TypeVect::make(elem_bt_from, num_elem_for_resize);
2386 op = gvn().transform(new VectorReinterpretNode(op, src_type, resize_type));
2387 op = gvn().transform(VectorCastNode::make(cast_vopc, op, elem_bt_to, num_elem_to));
2388 } else { // num_elem_from == num_elem_to
2389 if (is_mask) {
2390 // Make sure that cast for vector mask is implemented to particular type/size combination.
2391 if (!arch_supports_vector(Op_VectorMaskCast, num_elem_to, elem_bt_to, VecMaskNotUsed)) {
2392 log_if_needed(" ** not supported: arity=1 op=maskcast vlen2=%d etype2=%s ismask=%d",
2393 num_elem_to, type2name(elem_bt_to), is_mask);
2394 return false;
2395 }
2396 op = gvn().transform(new VectorMaskCastNode(op, dst_type));
2397 } else {
2398 // Since input and output number of elements match, and since we know this vector size is
2399 // supported, simply do a cast with no resize needed.
2400 op = gvn().transform(VectorCastNode::make(cast_vopc, op, elem_bt_to, num_elem_to));
2401 }
2402 }
2403 } else if (!Type::equals(src_type, dst_type)) {
2404 assert(!is_cast, "must be reinterpret");
2405 op = gvn().transform(new VectorReinterpretNode(op, src_type, dst_type));
2406 }
2407
2408 const TypeInstPtr* vbox_type_to = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass_to);
2409 Node* vbox = box_vector(op, vbox_type_to, elem_bt_to, num_elem_to);
2410 set_result(vbox);
2411 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem_to * type2aelembytes(elem_bt_to))));
2412 return true;
2413 }
2414
2415 // public static
2416 // <V extends Vector<E>,
2417 // E>
2418 // V insert(Class<? extends V> vectorClass, Class<E> elementType, int vlen,
2419 // V vec, int ix, long val,
2420 // VecInsertOp<V> defaultImpl)
2421 bool LibraryCallKit::inline_vector_insert() {
2422 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
2423 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
2424 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
2425 const TypeInt* idx = gvn().type(argument(4))->isa_int();
2426
2427 if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr || idx == nullptr) {
2428 return false; // dead code
2429 }
2430 if (vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con() || !idx->is_con()) {
2431 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s idx=%s",
2432 NodeClassNames[argument(0)->Opcode()],
2433 NodeClassNames[argument(1)->Opcode()],
2434 NodeClassNames[argument(2)->Opcode()],
2435 NodeClassNames[argument(4)->Opcode()]);
2436 return false; // not enough info for intrinsification
2437 }
2438 if (!is_klass_initialized(vector_klass)) {
2439 log_if_needed(" ** klass argument not initialized");
2440 return false;
2441 }
2442 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2443 if (!elem_type->is_primitive_type()) {
2444 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2445 return false; // should be primitive type
2446 }
2447 BasicType elem_bt = elem_type->basic_type();
2448 int num_elem = vlen->get_con();
2449 if (!arch_supports_vector(Op_VectorInsert, num_elem, elem_bt, VecMaskNotUsed)) {
2450 log_if_needed(" ** not supported: arity=1 op=insert vlen=%d etype=%s ismask=no",
2451 num_elem, type2name(elem_bt));
2452 return false; // not supported
2453 }
2454
2455 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2456 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2457
2458 Node* opd = unbox_vector(argument(3), vbox_type, elem_bt, num_elem);
2459 if (opd == nullptr) {
2460 return false;
2461 }
2462
2463 Node* insert_val = argument(5);
2464 assert(gvn().type(insert_val)->isa_long() != nullptr, "expected to be long");
2465
2466 // Convert insert value back to its appropriate type.
2467 switch (elem_bt) {
2468 case T_BYTE:
2469 insert_val = gvn().transform(new ConvL2INode(insert_val));
2470 insert_val = gvn().transform(new CastIINode(insert_val, TypeInt::BYTE));
2471 break;
2472 case T_SHORT:
2473 insert_val = gvn().transform(new ConvL2INode(insert_val));
2474 insert_val = gvn().transform(new CastIINode(insert_val, TypeInt::SHORT));
2475 break;
2476 case T_INT:
2477 insert_val = gvn().transform(new ConvL2INode(insert_val));
2478 break;
2479 case T_FLOAT:
2480 insert_val = gvn().transform(new ConvL2INode(insert_val));
2481 insert_val = gvn().transform(new MoveI2FNode(insert_val));
2482 break;
2483 case T_DOUBLE:
2484 insert_val = gvn().transform(new MoveL2DNode(insert_val));
2485 break;
2486 case T_LONG:
2487 // no conversion needed
2488 break;
2489 default: fatal("%s", type2name(elem_bt)); break;
2490 }
2491
2492 Node* operation = gvn().transform(VectorInsertNode::make(opd, insert_val, idx->get_con(), gvn()));
2493
2494 Node* vbox = box_vector(operation, vbox_type, elem_bt, num_elem);
2495 set_result(vbox);
2496 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2497 return true;
2498 }
2499
2500 // public static
2501 // <VM extends VectorPayload,
2502 // E>
2503 // long extract(Class<? extends VM> vClass, Class<E> eClass,
2504 // int length,
2505 // VM vm, int i,
2506 // VecExtractOp<VM> defaultImpl)
2507 bool LibraryCallKit::inline_vector_extract() {
2508 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
2509 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
2510 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
2511 const TypeInt* idx = gvn().type(argument(4))->isa_int();
2512
2513 if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr || idx == nullptr) {
2514 return false; // dead code
2515 }
2516 if (vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con() || !idx->is_con()) {
2517 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s",
2518 NodeClassNames[argument(0)->Opcode()],
2519 NodeClassNames[argument(1)->Opcode()],
2520 NodeClassNames[argument(2)->Opcode()]);
2521 return false; // not enough info for intrinsification
2522 }
2523 if (!is_klass_initialized(vector_klass)) {
2524 log_if_needed(" ** klass argument not initialized");
2525 return false;
2526 }
2527 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2528 if (!elem_type->is_primitive_type()) {
2529 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2530 return false; // should be primitive type
2531 }
2532 BasicType elem_bt = elem_type->basic_type();
2533 int num_elem = vlen->get_con();
2534
2535 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2536 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2537
2538 Node* opd = nullptr;
2539
2540 if (is_vector_mask(vbox_klass)) {
2541 // vbox_klass is mask. This is used for VectorMask.laneIsSet(int).
2542
2543 Node* pos = argument(4); // can be variable
2544 if (arch_supports_vector(Op_ExtractUB, num_elem, elem_bt, VecMaskUseAll)) {
2545 // Transform mask to vector with type of boolean and utilize ExtractUB node.
2546 opd = unbox_vector(argument(3), vbox_type, elem_bt, num_elem);
2547 if (opd == nullptr) {
2548 return false;
2549 }
2550 opd = gvn().transform(VectorStoreMaskNode::make(gvn(), opd, elem_bt, num_elem));
2551 opd = gvn().transform(new ExtractUBNode(opd, pos));
2552 opd = gvn().transform(new ConvI2LNode(opd));
2553 } else if (arch_supports_vector(Op_VectorMaskToLong, num_elem, elem_bt, VecMaskUseLoad)) {
2554 opd = unbox_vector(argument(3), vbox_type, elem_bt, num_elem);
2555 if (opd == nullptr) {
2556 return false;
2557 }
2558 // VectorMaskToLongNode requires the input is either a mask or a vector with BOOLEAN type.
2559 if (opd->bottom_type()->isa_vectmask() == nullptr) {
2560 opd = gvn().transform(VectorStoreMaskNode::make(gvn(), opd, elem_bt, num_elem));
2561 }
2562 // ((toLong() >>> pos) & 1L
2563 opd = gvn().transform(new VectorMaskToLongNode(opd, TypeLong::LONG));
2564 opd = gvn().transform(new URShiftLNode(opd, pos));
2565 opd = gvn().transform(new AndLNode(opd, gvn().makecon(TypeLong::ONE)));
2566 } else {
2567 log_if_needed(" ** Rejected mask extraction because architecture does not support it");
2568 return false; // not supported
2569 }
2570 } else {
2571 // vbox_klass is vector. This is used for Vector.lane(int).
2572 if (!idx->is_con()) {
2573 log_if_needed(" ** missing constant: idx=%s", NodeClassNames[argument(4)->Opcode()]);
2574 return false; // not enough info for intrinsification
2575 }
2576
2577 int vopc = ExtractNode::opcode(elem_bt);
2578 if (!arch_supports_vector(vopc, num_elem, elem_bt, VecMaskNotUsed)) {
2579 log_if_needed(" ** not supported: arity=1 op=extract vlen=%d etype=%s ismask=no",
2580 num_elem, type2name(elem_bt));
2581 return false; // not supported
2582 }
2583
2584 opd = unbox_vector(argument(3), vbox_type, elem_bt, num_elem);
2585 if (opd == nullptr) {
2586 return false;
2587 }
2588 ConINode* idx_con = gvn().intcon(idx->get_con())->as_ConI();
2589
2590 opd = gvn().transform(ExtractNode::make(opd, idx_con, elem_bt));
2591 switch (elem_bt) {
2592 case T_BYTE:
2593 case T_SHORT:
2594 case T_INT: {
2595 opd = gvn().transform(new ConvI2LNode(opd));
2596 break;
2597 }
2598 case T_FLOAT: {
2599 opd = gvn().transform(new MoveF2INode(opd));
2600 opd = gvn().transform(new ConvI2LNode(opd));
2601 break;
2602 }
2603 case T_DOUBLE: {
2604 opd = gvn().transform(new MoveD2LNode(opd));
2605 break;
2606 }
2607 case T_LONG: {
2608 // no conversion needed
2609 break;
2610 }
2611 default: fatal("%s", type2name(elem_bt));
2612 }
2613 }
2614 set_result(opd);
2615 return true;
2616 }
2617
2618 // public static
2619 // <V extends Vector<E>,
2620 // M extends VectorMask<E>,
2621 // E>
2622 // V compressExpandOp(int opr,
2623 // Class<? extends V> vClass, Class<? extends M> mClass, Class<E> eClass,
2624 // int length, V v, M m,
2625 // CompressExpandOperation<V, M> defaultImpl)
2626 bool LibraryCallKit::inline_vector_compress_expand() {
2627 const TypeInt* opr = gvn().type(argument(0))->isa_int();
2628 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
2629 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
2630 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
2631 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
2632
2633 if (opr == nullptr || !opr->is_con() ||
2634 vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
2635 mask_klass == nullptr || mask_klass->const_oop() == nullptr ||
2636 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
2637 vlen == nullptr || !vlen->is_con()) {
2638 log_if_needed(" ** missing constant: opr=%s vclass=%s mclass=%s etype=%s vlen=%s",
2639 NodeClassNames[argument(0)->Opcode()],
2640 NodeClassNames[argument(1)->Opcode()],
2641 NodeClassNames[argument(2)->Opcode()],
2642 NodeClassNames[argument(3)->Opcode()],
2643 NodeClassNames[argument(4)->Opcode()]);
2644 return false; // not enough info for intrinsification
2645 }
2646
2647 if (!is_klass_initialized(vector_klass) || !is_klass_initialized(mask_klass)) {
2648 log_if_needed(" ** klass argument not initialized");
2649 return false;
2650 }
2651
2652 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2653 if (!elem_type->is_primitive_type()) {
2654 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2655 return false; // should be primitive type
2656 }
2657
2658 int num_elem = vlen->get_con();
2659 BasicType elem_bt = elem_type->basic_type();
2660 int opc = VectorSupport::vop2ideal(opr->get_con(), elem_bt);
2661
2662 if (!arch_supports_vector(opc, num_elem, elem_bt, VecMaskUseLoad)) {
2663 log_if_needed(" ** not supported: opc=%d vlen=%d etype=%s ismask=useload",
2664 opc, num_elem, type2name(elem_bt));
2665 return false; // not supported
2666 }
2667
2668 Node* opd1 = nullptr;
2669 const TypeInstPtr* vbox_type = nullptr;
2670 if (opc != Op_CompressM) {
2671 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2672 vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2673 opd1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
2674 if (opd1 == nullptr) {
2675 log_if_needed(" ** unbox failed vector=%s",
2676 NodeClassNames[argument(5)->Opcode()]);
2677 return false;
2678 }
2679 }
2680
2681 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
2682 assert(is_vector_mask(mbox_klass), "argument(6) should be a mask class");
2683 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
2684
2685 Node* mask = unbox_vector(argument(6), mbox_type, elem_bt, num_elem);
2686 if (mask == nullptr) {
2687 log_if_needed(" ** unbox failed mask=%s",
2688 NodeClassNames[argument(6)->Opcode()]);
2689 return false;
2690 }
2691
2692 const TypeVect* vt = TypeVect::make(elem_bt, num_elem, opc == Op_CompressM);
2693 Node* operation = gvn().transform(VectorNode::make(opc, opd1, mask, vt));
2694
2695 // Wrap it up in VectorBox to keep object type information.
2696 const TypeInstPtr* box_type = opc == Op_CompressM ? mbox_type : vbox_type;
2697 Node* vbox = box_vector(operation, box_type, elem_bt, num_elem);
2698 set_result(vbox);
2699 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2700 return true;
2701 }
2702
2703 // public static
2704 // <V extends Vector<E>,
2705 // E,
2706 // S extends VectorSpecies<E>>
2707 // V indexVector(Class<? extends V> vClass, Class<E> eClass,
2708 // int length,
2709 // V v, int step, S s,
2710 // IndexOperation<V, S> defaultImpl)
2711 bool LibraryCallKit::inline_index_vector() {
2712 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
2713 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
2714 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
2715
2716 if (vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
2717 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
2718 vlen == nullptr || !vlen->is_con() ) {
2719 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s",
2720 NodeClassNames[argument(0)->Opcode()],
2721 NodeClassNames[argument(1)->Opcode()],
2722 NodeClassNames[argument(2)->Opcode()]);
2723 return false; // not enough info for intrinsification
2724 }
2725
2726 if (!is_klass_initialized(vector_klass)) {
2727 log_if_needed(" ** klass argument not initialized");
2728 return false;
2729 }
2730
2731 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2732 if (!elem_type->is_primitive_type()) {
2733 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2734 return false; // should be primitive type
2735 }
2736
2737 int num_elem = vlen->get_con();
2738 BasicType elem_bt = elem_type->basic_type();
2739
2740 // Check whether the iota index generation op is supported by the current hardware
2741 if (!arch_supports_vector(Op_VectorLoadConst, num_elem, elem_bt, VecMaskNotUsed)) {
2742 log_if_needed(" ** not supported: vlen=%d etype=%s", num_elem, type2name(elem_bt));
2743 return false; // not supported
2744 }
2745
2746 int mul_op = VectorSupport::vop2ideal(VectorSupport::VECTOR_OP_MUL, elem_bt);
2747 int vmul_op = VectorNode::opcode(mul_op, elem_bt);
2748 bool needs_mul = true;
2749 Node* scale = argument(4);
2750 const TypeInt* scale_type = gvn().type(scale)->isa_int();
2751 // Multiply is not needed if the scale is a constant "1".
2752 if (scale_type && scale_type->is_con() && scale_type->get_con() == 1) {
2753 needs_mul = false;
2754 } else {
2755 // Check whether the vector multiply op is supported by the current hardware
2756 if (!arch_supports_vector(vmul_op, num_elem, elem_bt, VecMaskNotUsed)) {
2757 log_if_needed(" ** not supported: vlen=%d etype=%s", num_elem, type2name(elem_bt));
2758 return false; // not supported
2759 }
2760
2761 // Check whether the scalar cast op is supported by the current hardware
2762 if (is_floating_point_type(elem_bt) || elem_bt == T_LONG) {
2763 int cast_op = elem_bt == T_LONG ? Op_ConvI2L :
2764 elem_bt == T_FLOAT? Op_ConvI2F : Op_ConvI2D;
2765 if (!Matcher::match_rule_supported(cast_op)) {
2766 log_if_needed(" ** Rejected op (%s) because architecture does not support it",
2767 NodeClassNames[cast_op]);
2768 return false; // not supported
2769 }
2770 }
2771 }
2772
2773 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2774 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2775 Node* opd = unbox_vector(argument(3), vbox_type, elem_bt, num_elem);
2776 if (opd == nullptr) {
2777 log_if_needed(" ** unbox failed vector=%s",
2778 NodeClassNames[argument(3)->Opcode()]);
2779 return false;
2780 }
2781
2782 int add_op = VectorSupport::vop2ideal(VectorSupport::VECTOR_OP_ADD, elem_bt);
2783 int vadd_op = VectorNode::opcode(add_op, elem_bt);
2784 bool needs_add = true;
2785 // The addition is not needed if all the element values of "opd" are zero
2786 if (VectorNode::is_all_zeros_vector(opd)) {
2787 needs_add = false;
2788 } else {
2789 // Check whether the vector addition op is supported by the current hardware
2790 if (!arch_supports_vector(vadd_op, num_elem, elem_bt, VecMaskNotUsed)) {
2791 log_if_needed(" ** not supported: vlen=%d etype=%s", num_elem, type2name(elem_bt));
2792 return false; // not supported
2793 }
2794 }
2795
2796 // Compute the iota indice vector
2797 const TypeVect* vt = TypeVect::make(elem_bt, num_elem);
2798 Node* index = gvn().transform(new VectorLoadConstNode(gvn().makecon(TypeInt::ZERO), vt));
2799
2800 // Broadcast the "scale" to a vector, and multiply the "scale" with iota indice vector.
2801 if (needs_mul) {
2802 switch (elem_bt) {
2803 case T_BOOLEAN: // fall-through
2804 case T_BYTE: // fall-through
2805 case T_SHORT: // fall-through
2806 case T_CHAR: // fall-through
2807 case T_INT: {
2808 // no conversion needed
2809 break;
2810 }
2811 case T_LONG: {
2812 scale = gvn().transform(new ConvI2LNode(scale));
2813 break;
2814 }
2815 case T_FLOAT: {
2816 scale = gvn().transform(new ConvI2FNode(scale));
2817 break;
2818 }
2819 case T_DOUBLE: {
2820 scale = gvn().transform(new ConvI2DNode(scale));
2821 break;
2822 }
2823 default: fatal("%s", type2name(elem_bt));
2824 }
2825 scale = gvn().transform(VectorNode::scalar2vector(scale, num_elem, Type::get_const_basic_type(elem_bt)));
2826 index = gvn().transform(VectorNode::make(vmul_op, index, scale, vt));
2827 }
2828
2829 // Add "opd" if addition is needed.
2830 if (needs_add) {
2831 index = gvn().transform(VectorNode::make(vadd_op, opd, index, vt));
2832 }
2833 Node* vbox = box_vector(index, vbox_type, elem_bt, num_elem);
2834 set_result(vbox);
2835 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2836 return true;
2837 }
2838
2839 // public static
2840 // <E,
2841 // M extends VectorMask<E>>
2842 // M indexPartiallyInUpperRange(Class<? extends M> mClass, Class<E> eClass, int length,
2843 // long offset, long limit,
2844 // IndexPartiallyInUpperRangeOperation<E, M> defaultImpl)
2845 bool LibraryCallKit::inline_index_partially_in_upper_range() {
2846 const TypeInstPtr* mask_klass = gvn().type(argument(0))->isa_instptr();
2847 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
2848 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
2849
2850 if (mask_klass == nullptr || mask_klass->const_oop() == nullptr ||
2851 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
2852 vlen == nullptr || !vlen->is_con()) {
2853 log_if_needed(" ** missing constant: mclass=%s etype=%s vlen=%s",
2854 NodeClassNames[argument(0)->Opcode()],
2855 NodeClassNames[argument(1)->Opcode()],
2856 NodeClassNames[argument(2)->Opcode()]);
2857 return false; // not enough info for intrinsification
2858 }
2859
2860 if (!is_klass_initialized(mask_klass)) {
2861 log_if_needed(" ** klass argument not initialized");
2862 return false;
2863 }
2864
2865 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2866 if (!elem_type->is_primitive_type()) {
2867 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2868 return false; // should be primitive type
2869 }
2870
2871 int num_elem = vlen->get_con();
2872 BasicType elem_bt = elem_type->basic_type();
2873
2874 // Check whether the necessary ops are supported by current hardware.
2875 bool supports_mask_gen = arch_supports_vector(Op_VectorMaskGen, num_elem, elem_bt, VecMaskUseStore);
2876 if (!supports_mask_gen) {
2877 if (!arch_supports_vector(Op_VectorLoadConst, num_elem, elem_bt, VecMaskNotUsed) ||
2878 !arch_supports_vector(Op_Replicate, num_elem, elem_bt, VecMaskNotUsed) ||
2879 !arch_supports_vector(Op_VectorMaskCmp, num_elem, elem_bt, VecMaskUseStore)) {
2880 log_if_needed(" ** not supported: vlen=%d etype=%s", num_elem, type2name(elem_bt));
2881 return false; // not supported
2882 }
2883
2884 // Check whether the scalar cast operation is supported by current hardware.
2885 if (elem_bt != T_LONG) {
2886 int cast_op = is_integral_type(elem_bt) ? Op_ConvL2I
2887 : (elem_bt == T_FLOAT ? Op_ConvL2F : Op_ConvL2D);
2888 if (!Matcher::match_rule_supported(cast_op)) {
2889 log_if_needed(" ** Rejected op (%s) because architecture does not support it",
2890 NodeClassNames[cast_op]);
2891 return false; // not supported
2892 }
2893 }
2894 }
2895
2896 Node* offset = argument(3);
2897 Node* limit = argument(5);
2898 if (offset == nullptr || limit == nullptr) {
2899 log_if_needed(" ** offset or limit argument is null");
2900 return false; // not supported
2901 }
2902
2903 ciKlass* box_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
2904 assert(is_vector_mask(box_klass), "argument(0) should be a mask class");
2905 const TypeInstPtr* box_type = TypeInstPtr::make_exact(TypePtr::NotNull, box_klass);
2906
2907 // We assume "offset > 0 && limit >= offset && limit - offset < num_elem".
2908 // So directly get indexLimit with "indexLimit = limit - offset".
2909 Node* indexLimit = gvn().transform(new SubLNode(limit, offset));
2910 Node* mask = nullptr;
2911 if (supports_mask_gen) {
2912 mask = gvn().transform(VectorMaskGenNode::make(indexLimit, elem_bt, num_elem));
2913 } else {
2914 // Generate the vector mask based on "mask = iota < indexLimit".
2915 // Broadcast "indexLimit" to a vector.
2916 switch (elem_bt) {
2917 case T_BOOLEAN: // fall-through
2918 case T_BYTE: // fall-through
2919 case T_SHORT: // fall-through
2920 case T_CHAR: // fall-through
2921 case T_INT: {
2922 indexLimit = gvn().transform(new ConvL2INode(indexLimit));
2923 break;
2924 }
2925 case T_DOUBLE: {
2926 indexLimit = gvn().transform(new ConvL2DNode(indexLimit));
2927 break;
2928 }
2929 case T_FLOAT: {
2930 indexLimit = gvn().transform(new ConvL2FNode(indexLimit));
2931 break;
2932 }
2933 case T_LONG: {
2934 // no conversion needed
2935 break;
2936 }
2937 default: fatal("%s", type2name(elem_bt));
2938 }
2939 indexLimit = gvn().transform(VectorNode::scalar2vector(indexLimit, num_elem, Type::get_const_basic_type(elem_bt)));
2940
2941 // Load the "iota" vector.
2942 const TypeVect* vt = TypeVect::make(elem_bt, num_elem);
2943 Node* iota = gvn().transform(new VectorLoadConstNode(gvn().makecon(TypeInt::ZERO), vt));
2944
2945 // Compute the vector mask with "mask = iota < indexLimit".
2946 ConINode* pred_node = (ConINode*)gvn().makecon(TypeInt::make(BoolTest::lt));
2947 const TypeVect* vmask_type = TypeVect::makemask(elem_bt, num_elem);
2948 mask = gvn().transform(new VectorMaskCmpNode(BoolTest::lt, iota, indexLimit, pred_node, vmask_type));
2949 }
2950 Node* vbox = box_vector(mask, box_type, elem_bt, num_elem);
2951 set_result(vbox);
2952 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2953 return true;
2954 }
2955
2956 #undef non_product_log_if_needed
2957 #undef log_if_needed