1 /*
2 * Copyright (c) 2020, 2024, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "ci/ciSymbols.hpp"
27 #include "classfile/vmSymbols.hpp"
28 #include "opto/library_call.hpp"
29 #include "opto/runtime.hpp"
30 #include "opto/vectornode.hpp"
31 #include "prims/vectorSupport.hpp"
32 #include "runtime/stubRoutines.hpp"
33
34 #ifdef ASSERT
35 static bool is_vector(ciKlass* klass) {
36 return klass->is_subclass_of(ciEnv::current()->vector_VectorPayload_klass());
37 }
38
39 static bool check_vbox(const TypeInstPtr* vbox_type) {
40 assert(vbox_type->klass_is_exact(), "");
41
42 ciInstanceKlass* ik = vbox_type->instance_klass();
43 assert(is_vector(ik), "not a vector");
44
45 ciField* fd1 = ik->get_field_by_name(ciSymbols::ETYPE_name(), ciSymbols::class_signature(), /* is_static */ true);
46 assert(fd1 != nullptr, "element type info is missing");
47
48 ciConstant val1 = fd1->constant_value();
49 BasicType elem_bt = val1.as_object()->as_instance()->java_mirror_type()->basic_type();
50 assert(is_java_primitive(elem_bt), "element type info is missing");
51
52 ciField* fd2 = ik->get_field_by_name(ciSymbols::VLENGTH_name(), ciSymbols::int_signature(), /* is_static */ true);
53 assert(fd2 != nullptr, "vector length info is missing");
54
55 ciConstant val2 = fd2->constant_value();
56 assert(val2.as_int() > 0, "vector length info is missing");
57
58 return true;
59 }
60 #endif
61
62 #define log_if_needed(...) \
63 if (C->print_intrinsics()) { \
64 tty->print_cr(__VA_ARGS__); \
65 }
66
67 #ifndef PRODUCT
68 #define non_product_log_if_needed(...) log_if_needed(__VA_ARGS__)
69 #else
70 #define non_product_log_if_needed(...)
71 #endif
72
73 static bool is_vector_mask(ciKlass* klass) {
74 return klass->is_subclass_of(ciEnv::current()->vector_VectorMask_klass());
75 }
76
77 static bool is_vector_shuffle(ciKlass* klass) {
78 return klass->is_subclass_of(ciEnv::current()->vector_VectorShuffle_klass());
79 }
80
81 bool LibraryCallKit::arch_supports_vector_rotate(int opc, int num_elem, BasicType elem_bt,
82 VectorMaskUseType mask_use_type, bool has_scalar_args) {
83 bool is_supported = true;
84
85 // has_scalar_args flag is true only for non-constant scalar shift count,
86 // since in this case shift needs to be broadcasted.
87 if (!Matcher::match_rule_supported_vector(opc, num_elem, elem_bt) ||
88 (has_scalar_args && !arch_supports_vector(Op_Replicate, num_elem, elem_bt, VecMaskNotUsed))) {
89 is_supported = false;
90 }
91
92 if (is_supported) {
93 // Check if mask unboxing is supported, this is a two step process which first loads the contents
94 // of boolean array into vector followed by either lane expansion to match the lane size of masked
95 // vector operation or populate the predicate register.
96 if ((mask_use_type & VecMaskUseLoad) != 0) {
97 if (!Matcher::match_rule_supported_vector(Op_VectorLoadMask, num_elem, elem_bt) ||
98 !Matcher::match_rule_supported_vector(Op_LoadVector, num_elem, T_BOOLEAN)) {
99 non_product_log_if_needed(" ** Rejected vector mask loading (%s,%s,%d) because architecture does not support it",
100 NodeClassNames[Op_VectorLoadMask], type2name(elem_bt), num_elem);
101 return false;
102 }
103 }
104
105 if ((mask_use_type & VecMaskUsePred) != 0) {
106 if (!Matcher::has_predicated_vectors() ||
107 !Matcher::match_rule_supported_vector_masked(opc, num_elem, elem_bt)) {
108 non_product_log_if_needed("Rejected vector mask predicate using (%s,%s,%d) because architecture does not support it",
109 NodeClassNames[opc], type2name(elem_bt), num_elem);
110 return false;
111 }
112 }
113 }
114
115 int lshiftopc, rshiftopc;
116 switch(elem_bt) {
117 case T_BYTE:
118 lshiftopc = Op_LShiftI;
119 rshiftopc = Op_URShiftB;
120 break;
121 case T_SHORT:
122 lshiftopc = Op_LShiftI;
123 rshiftopc = Op_URShiftS;
124 break;
125 case T_INT:
126 lshiftopc = Op_LShiftI;
127 rshiftopc = Op_URShiftI;
128 break;
129 case T_LONG:
130 lshiftopc = Op_LShiftL;
131 rshiftopc = Op_URShiftL;
132 break;
133 default: fatal("Unexpected type: %s", type2name(elem_bt));
134 }
135 int lshiftvopc = VectorNode::opcode(lshiftopc, elem_bt);
136 int rshiftvopc = VectorNode::opcode(rshiftopc, elem_bt);
137 if (!is_supported &&
138 arch_supports_vector(lshiftvopc, num_elem, elem_bt, VecMaskNotUsed, has_scalar_args) &&
139 arch_supports_vector(rshiftvopc, num_elem, elem_bt, VecMaskNotUsed, has_scalar_args) &&
140 arch_supports_vector(Op_OrV, num_elem, elem_bt, VecMaskNotUsed)) {
141 is_supported = true;
142 }
143 return is_supported;
144 }
145
146 Node* GraphKit::box_vector(Node* vector, const TypeInstPtr* vbox_type, BasicType elem_bt, int num_elem, bool deoptimize_on_exception) {
147 assert(EnableVectorSupport, "");
148
149 PreserveReexecuteState preexecs(this);
150 jvms()->set_should_reexecute(true);
151
152 VectorBoxAllocateNode* alloc = new VectorBoxAllocateNode(C, vbox_type);
153 set_edges_for_java_call(alloc, /*must_throw=*/false, /*separate_io_proj=*/true);
154 make_slow_call_ex(alloc, env()->Throwable_klass(), /*separate_io_proj=*/true, deoptimize_on_exception);
155 set_i_o(gvn().transform( new ProjNode(alloc, TypeFunc::I_O) ));
156 set_all_memory(gvn().transform( new ProjNode(alloc, TypeFunc::Memory) ));
157 Node* ret = gvn().transform(new ProjNode(alloc, TypeFunc::Parms));
158
159 assert(check_vbox(vbox_type), "");
160 const TypeVect* vt = TypeVect::make(elem_bt, num_elem, is_vector_mask(vbox_type->instance_klass()));
161 VectorBoxNode* vbox = new VectorBoxNode(C, ret, vector, vbox_type, vt);
162 return gvn().transform(vbox);
163 }
164
165 Node* GraphKit::unbox_vector(Node* v, const TypeInstPtr* vbox_type, BasicType elem_bt, int num_elem, bool shuffle_to_vector) {
166 assert(EnableVectorSupport, "");
167 const TypeInstPtr* vbox_type_v = gvn().type(v)->isa_instptr();
168 if (vbox_type_v == nullptr || vbox_type->instance_klass() != vbox_type_v->instance_klass()) {
169 return nullptr; // arguments don't agree on vector shapes
170 }
171 if (vbox_type_v->maybe_null()) {
172 return nullptr; // no nulls are allowed
173 }
174 assert(check_vbox(vbox_type), "");
175 const TypeVect* vt = TypeVect::make(elem_bt, num_elem, is_vector_mask(vbox_type->instance_klass()));
176 Node* unbox = gvn().transform(new VectorUnboxNode(C, vt, v, merged_memory(), shuffle_to_vector));
177 return unbox;
178 }
179
180 Node* GraphKit::vector_shift_count(Node* cnt, int shift_op, BasicType bt, int num_elem) {
181 assert(bt == T_INT || bt == T_LONG || bt == T_SHORT || bt == T_BYTE, "byte, short, long and int are supported");
182 juint mask = (type2aelembytes(bt) * BitsPerByte - 1);
183 Node* nmask = gvn().transform(ConNode::make(TypeInt::make(mask)));
184 Node* mcnt = gvn().transform(new AndINode(cnt, nmask));
185 return gvn().transform(VectorNode::shift_count(shift_op, mcnt, num_elem, bt));
186 }
187
188 bool LibraryCallKit::arch_supports_vector(int sopc, int num_elem, BasicType type, VectorMaskUseType mask_use_type, bool has_scalar_args) {
189 // Check that the operation is valid.
190 if (sopc <= 0) {
191 non_product_log_if_needed(" ** Rejected intrinsification because no valid vector op could be extracted");
192 return false;
193 }
194
195 if (VectorNode::is_vector_rotate(sopc)) {
196 if(!arch_supports_vector_rotate(sopc, num_elem, type, mask_use_type, has_scalar_args)) {
197 non_product_log_if_needed(" ** Rejected vector op (%s,%s,%d) because architecture does not support variable vector shifts",
198 NodeClassNames[sopc], type2name(type), num_elem);
199 return false;
200 }
201 } else if (VectorNode::is_vector_integral_negate(sopc)) {
202 if (!VectorNode::is_vector_integral_negate_supported(sopc, num_elem, type, false)) {
203 non_product_log_if_needed(" ** Rejected vector op (%s,%s,%d) because architecture does not support integral vector negate",
204 NodeClassNames[sopc], type2name(type), num_elem);
205 return false;
206 }
207 } else {
208 // Check that architecture supports this op-size-type combination.
209 if (!Matcher::match_rule_supported_vector(sopc, num_elem, type)) {
210 non_product_log_if_needed(" ** Rejected vector op (%s,%s,%d) because architecture does not support it",
211 NodeClassNames[sopc], type2name(type), num_elem);
212 return false;
213 } else {
214 assert(Matcher::match_rule_supported(sopc), "must be supported");
215 }
216 }
217
218 if (num_elem == 1) {
219 if (mask_use_type != VecMaskNotUsed) {
220 non_product_log_if_needed(" ** Rejected vector mask op (%s,%s,%d) because architecture does not support it",
221 NodeClassNames[sopc], type2name(type), num_elem);
222 return false;
223 }
224
225 if (sopc != 0) {
226 if (sopc != Op_LoadVector && sopc != Op_StoreVector) {
227 non_product_log_if_needed(" ** Not a svml call or load/store vector op (%s,%s,%d)",
228 NodeClassNames[sopc], type2name(type), num_elem);
229 return false;
230 }
231 }
232 }
233
234 if (!has_scalar_args && VectorNode::is_vector_shift(sopc) &&
235 Matcher::supports_vector_variable_shifts() == false) {
236 log_if_needed(" ** Rejected vector op (%s,%s,%d) because architecture does not support variable vector shifts",
237 NodeClassNames[sopc], type2name(type), num_elem);
238 return false;
239 }
240
241 // Check if mask unboxing is supported, this is a two step process which first loads the contents
242 // of boolean array into vector followed by either lane expansion to match the lane size of masked
243 // vector operation or populate the predicate register.
244 if ((mask_use_type & VecMaskUseLoad) != 0) {
245 if (!Matcher::match_rule_supported_vector(Op_VectorLoadMask, num_elem, type) ||
246 !Matcher::match_rule_supported_vector(Op_LoadVector, num_elem, T_BOOLEAN)) {
247 non_product_log_if_needed(" ** Rejected vector mask loading (%s,%s,%d) because architecture does not support it",
248 NodeClassNames[Op_VectorLoadMask], type2name(type), num_elem);
249 return false;
250 }
251 }
252
253 // Check if mask boxing is supported, this is a two step process which first stores the contents
254 // of mask vector / predicate register into a boolean vector followed by vector store operation to
255 // transfer the contents to underlined storage of mask boxes which is a boolean array.
256 if ((mask_use_type & VecMaskUseStore) != 0) {
257 if (!Matcher::match_rule_supported_vector(Op_VectorStoreMask, num_elem, type) ||
258 !Matcher::match_rule_supported_vector(Op_StoreVector, num_elem, T_BOOLEAN)) {
259 non_product_log_if_needed("Rejected vector mask storing (%s,%s,%d) because architecture does not support it",
260 NodeClassNames[Op_VectorStoreMask], type2name(type), num_elem);
261 return false;
262 }
263 }
264
265 if ((mask_use_type & VecMaskUsePred) != 0) {
266 bool is_supported = false;
267 if (Matcher::has_predicated_vectors()) {
268 if (VectorNode::is_vector_integral_negate(sopc)) {
269 is_supported = VectorNode::is_vector_integral_negate_supported(sopc, num_elem, type, true);
270 } else {
271 is_supported = Matcher::match_rule_supported_vector_masked(sopc, num_elem, type);
272 }
273 }
274 is_supported |= Matcher::supports_vector_predicate_op_emulation(sopc, num_elem, type);
275
276 if (!is_supported) {
277 non_product_log_if_needed("Rejected vector mask predicate using (%s,%s,%d) because architecture does not support it",
278 NodeClassNames[sopc], type2name(type), num_elem);
279 return false;
280 }
281 }
282
283 return true;
284 }
285
286 static bool is_klass_initialized(const TypeInstPtr* vec_klass) {
287 if (vec_klass->const_oop() == nullptr) {
288 return false; // uninitialized or some kind of unsafe access
289 }
290 assert(vec_klass->const_oop()->as_instance()->java_lang_Class_klass() != nullptr, "klass instance expected");
291 ciInstanceKlass* klass = vec_klass->const_oop()->as_instance()->java_lang_Class_klass()->as_instance_klass();
292 return klass->is_initialized();
293 }
294
295 // public static
296 // <V extends Vector<E>,
297 // M extends VectorMask<E>,
298 // E>
299 // V unaryOp(int oprId, Class<? extends V> vmClass, Class<? extends M> maskClass, Class<E> elementType,
300 // int length, V v, M m,
301 // UnaryOperation<V, M> defaultImpl)
302 //
303 // public static
304 // <V,
305 // M extends VectorMask<E>,
306 // E>
307 // V binaryOp(int oprId, Class<? extends V> vmClass, Class<? extends M> maskClass, Class<E> elementType,
308 // int length, V v1, V v2, M m,
309 // BinaryOperation<V, M> defaultImpl)
310 //
311 // public static
312 // <V extends Vector<E>,
313 // M extends VectorMask<E>,
314 // E>
315 // V ternaryOp(int oprId, Class<? extends V> vmClass, Class<? extends M> maskClass, Class<E> elementType,
316 // int length, V v1, V v2, V v3, M m,
317 // TernaryOperation<V, M> defaultImpl)
318 //
319 bool LibraryCallKit::inline_vector_nary_operation(int n) {
320 const TypeInt* opr = gvn().type(argument(0))->isa_int();
321 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
322 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
323 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
324 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
325
326 if (opr == nullptr || !opr->is_con() ||
327 vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
328 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
329 vlen == nullptr || !vlen->is_con()) {
330 log_if_needed(" ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
331 NodeClassNames[argument(0)->Opcode()],
332 NodeClassNames[argument(1)->Opcode()],
333 NodeClassNames[argument(3)->Opcode()],
334 NodeClassNames[argument(4)->Opcode()]);
335 return false; // not enough info for intrinsification
336 }
337
338 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
339 if (!elem_type->is_primitive_type()) {
340 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
341 return false; // should be primitive type
342 }
343 if (!is_klass_initialized(vector_klass)) {
344 log_if_needed(" ** klass argument not initialized");
345 return false;
346 }
347
348 // "argument(n + 5)" should be the mask object. We assume it is "null" when no mask
349 // is used to control this operation.
350 const Type* vmask_type = gvn().type(argument(n + 5));
351 bool is_masked_op = vmask_type != TypePtr::NULL_PTR;
352 if (is_masked_op) {
353 if (mask_klass == nullptr || mask_klass->const_oop() == nullptr) {
354 log_if_needed(" ** missing constant: maskclass=%s", NodeClassNames[argument(2)->Opcode()]);
355 return false; // not enough info for intrinsification
356 }
357
358 if (!is_klass_initialized(mask_klass)) {
359 log_if_needed(" ** mask klass argument not initialized");
360 return false;
361 }
362
363 if (vmask_type->maybe_null()) {
364 log_if_needed(" ** null mask values are not allowed for masked op");
365 return false;
366 }
367 }
368
369 BasicType elem_bt = elem_type->basic_type();
370 int num_elem = vlen->get_con();
371 int opc = VectorSupport::vop2ideal(opr->get_con(), elem_bt);
372 int sopc = VectorNode::opcode(opc, elem_bt);
373 if ((opc != Op_CallLeafVector) && (sopc == 0)) {
374 log_if_needed(" ** operation not supported: opc=%s bt=%s", NodeClassNames[opc], type2name(elem_bt));
375 return false; // operation not supported
376 }
377 if (num_elem == 1) {
378 if (opc != Op_CallLeafVector || elem_bt != T_DOUBLE) {
379 log_if_needed(" ** not a svml call: arity=%d opc=%d vlen=%d etype=%s",
380 n, opc, num_elem, type2name(elem_bt));
381 return false;
382 }
383 }
384 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
385 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
386
387 if (is_vector_mask(vbox_klass)) {
388 assert(!is_masked_op, "mask operations do not need mask to control");
389 }
390
391 if (opc == Op_CallLeafVector) {
392 if (!UseVectorStubs) {
393 log_if_needed(" ** vector stubs support is disabled");
394 return false;
395 }
396 if (!Matcher::supports_vector_calling_convention()) {
397 log_if_needed(" ** no vector calling conventions supported");
398 return false;
399 }
400 if (!Matcher::vector_size_supported(elem_bt, num_elem)) {
401 log_if_needed(" ** vector size (vlen=%d, etype=%s) is not supported",
402 num_elem, type2name(elem_bt));
403 return false;
404 }
405 }
406
407 // When using mask, mask use type needs to be VecMaskUseLoad.
408 VectorMaskUseType mask_use_type = is_vector_mask(vbox_klass) ? VecMaskUseAll
409 : is_masked_op ? VecMaskUseLoad : VecMaskNotUsed;
410 if ((sopc != 0) && !arch_supports_vector(sopc, num_elem, elem_bt, mask_use_type)) {
411 log_if_needed(" ** not supported: arity=%d opc=%d vlen=%d etype=%s ismask=%d is_masked_op=%d",
412 n, sopc, num_elem, type2name(elem_bt),
413 is_vector_mask(vbox_klass) ? 1 : 0, is_masked_op ? 1 : 0);
414 return false; // not supported
415 }
416
417 // Return true if current platform has implemented the masked operation with predicate feature.
418 bool use_predicate = is_masked_op && sopc != 0 && arch_supports_vector(sopc, num_elem, elem_bt, VecMaskUsePred);
419 if (is_masked_op && !use_predicate && !arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad)) {
420 log_if_needed(" ** not supported: arity=%d opc=%d vlen=%d etype=%s ismask=0 is_masked_op=1",
421 n, sopc, num_elem, type2name(elem_bt));
422 return false;
423 }
424
425 Node* opd1 = nullptr; Node* opd2 = nullptr; Node* opd3 = nullptr;
426 switch (n) {
427 case 3: {
428 opd3 = unbox_vector(argument(7), vbox_type, elem_bt, num_elem);
429 if (opd3 == nullptr) {
430 log_if_needed(" ** unbox failed v3=%s",
431 NodeClassNames[argument(7)->Opcode()]);
432 return false;
433 }
434 // fall-through
435 }
436 case 2: {
437 opd2 = unbox_vector(argument(6), vbox_type, elem_bt, num_elem);
438 if (opd2 == nullptr) {
439 log_if_needed(" ** unbox failed v2=%s",
440 NodeClassNames[argument(6)->Opcode()]);
441 return false;
442 }
443 // fall-through
444 }
445 case 1: {
446 opd1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
447 if (opd1 == nullptr) {
448 log_if_needed(" ** unbox failed v1=%s",
449 NodeClassNames[argument(5)->Opcode()]);
450 return false;
451 }
452 break;
453 }
454 default: fatal("unsupported arity: %d", n);
455 }
456
457 Node* mask = nullptr;
458 if (is_masked_op) {
459 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
460 assert(is_vector_mask(mbox_klass), "argument(2) should be a mask class");
461 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
462 mask = unbox_vector(argument(n + 5), mbox_type, elem_bt, num_elem);
463 if (mask == nullptr) {
464 log_if_needed(" ** unbox failed mask=%s",
465 NodeClassNames[argument(n + 5)->Opcode()]);
466 return false;
467 }
468 }
469
470 Node* operation = nullptr;
471 if (opc == Op_CallLeafVector) {
472 assert(UseVectorStubs, "sanity");
473 operation = gen_call_to_svml(opr->get_con(), elem_bt, num_elem, opd1, opd2);
474 if (operation == nullptr) {
475 log_if_needed(" ** svml call failed for %s_%s_%d",
476 (elem_bt == T_FLOAT)?"float":"double",
477 VectorSupport::svmlname[opr->get_con() - VectorSupport::VECTOR_OP_SVML_START],
478 num_elem * type2aelembytes(elem_bt));
479 return false;
480 }
481 } else {
482 const TypeVect* vt = TypeVect::make(elem_bt, num_elem, is_vector_mask(vbox_klass));
483 switch (n) {
484 case 1:
485 case 2: {
486 operation = VectorNode::make(sopc, opd1, opd2, vt, is_vector_mask(vbox_klass), VectorNode::is_shift_opcode(opc));
487 break;
488 }
489 case 3: {
490 operation = VectorNode::make(sopc, opd1, opd2, opd3, vt);
491 break;
492 }
493 default: fatal("unsupported arity: %d", n);
494 }
495 }
496
497 if (is_masked_op && mask != nullptr) {
498 if (use_predicate) {
499 operation->add_req(mask);
500 operation->add_flag(Node::Flag_is_predicated_vector);
501 } else {
502 operation->add_flag(Node::Flag_is_predicated_using_blend);
503 operation = gvn().transform(operation);
504 operation = new VectorBlendNode(opd1, operation, mask);
505 }
506 }
507 operation = gvn().transform(operation);
508
509 // Wrap it up in VectorBox to keep object type information.
510 Node* vbox = box_vector(operation, vbox_type, elem_bt, num_elem);
511 set_result(vbox);
512 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
513 return true;
514 }
515
516 // Following routine generates IR corresponding to AbstractShuffle::partiallyWrapIndex method,
517 // which partially wraps index by modulo VEC_LENGTH and generates a negative index value if original
518 // index is out of valid index range [0, VEC_LENGTH)
519 //
520 // wrapped_index = (VEC_LENGTH - 1) & index
521 // if (index u> VEC_LENGTH) {
522 // wrapped_index -= VEC_LENGTH;
523 //
524 // Note: Unsigned greater than comparison treat both <0 and >VEC_LENGTH indices as out-of-bound
525 // indexes.
526 Node* LibraryCallKit::partially_wrap_indexes(Node* index_vec, int num_elem, BasicType elem_bt) {
527 assert(elem_bt == T_BYTE, "Shuffles use byte array based backing storage.");
528 const TypeVect* vt = TypeVect::make(elem_bt, num_elem);
529 const Type* type_bt = Type::get_const_basic_type(elem_bt);
530
531 Node* mod_mask = gvn().makecon(TypeInt::make(num_elem-1));
532 Node* bcast_mod_mask = gvn().transform(VectorNode::scalar2vector(mod_mask, num_elem, type_bt));
533
534 BoolTest::mask pred = BoolTest::ugt;
535 ConINode* pred_node = (ConINode*)gvn().makecon(TypeInt::make(pred));
536 Node* lane_cnt = gvn().makecon(TypeInt::make(num_elem));
537 Node* bcast_lane_cnt = gvn().transform(VectorNode::scalar2vector(lane_cnt, num_elem, type_bt));
538 const TypeVect* vmask_type = TypeVect::makemask(type_bt, num_elem);
539 Node* mask = gvn().transform(new VectorMaskCmpNode(pred, bcast_lane_cnt, index_vec, pred_node, vmask_type));
540
541 // Make the indices greater than lane count as -ve values to match the java side implementation.
542 index_vec = gvn().transform(VectorNode::make(Op_AndV, index_vec, bcast_mod_mask, vt));
543 Node* biased_val = gvn().transform(VectorNode::make(Op_SubVB, index_vec, bcast_lane_cnt, vt));
544 return gvn().transform(new VectorBlendNode(biased_val, index_vec, mask));
545 }
546
547 // <Sh extends VectorShuffle<E>, E>
548 // Sh ShuffleIota(Class<?> E, Class<?> shuffleClass, Vector.Species<E> s, int length,
549 // int start, int step, int wrap, ShuffleIotaOperation<Sh, E> defaultImpl)
550 bool LibraryCallKit::inline_vector_shuffle_iota() {
551 const TypeInstPtr* shuffle_klass = gvn().type(argument(1))->isa_instptr();
552 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
553 const TypeInt* start_val = gvn().type(argument(4))->isa_int();
554 const TypeInt* step_val = gvn().type(argument(5))->isa_int();
555 const TypeInt* wrap = gvn().type(argument(6))->isa_int();
556
557 if (shuffle_klass == nullptr || shuffle_klass->const_oop() == nullptr ||
558 vlen == nullptr || !vlen->is_con() ||
559 start_val == nullptr ||
560 step_val == nullptr ||
561 wrap == nullptr || !wrap->is_con()) {
562 return false; // not enough info for intrinsification
563 }
564
565 if (!is_klass_initialized(shuffle_klass)) {
566 log_if_needed(" ** klass argument not initialized");
567 return false;
568 }
569
570 int do_wrap = wrap->get_con();
571 int num_elem = vlen->get_con();
572 BasicType elem_bt = T_BYTE;
573
574 bool effective_indices_in_range = false;
575 if (start_val->is_con() && step_val->is_con()) {
576 int effective_min_index = start_val->get_con();
577 int effective_max_index = start_val->get_con() + step_val->get_con() * (num_elem - 1);
578 effective_indices_in_range = effective_max_index >= effective_min_index && effective_min_index >= -128 && effective_max_index <= 127;
579 }
580
581 if (!do_wrap && !effective_indices_in_range) {
582 // Disable instrinsification for unwrapped shuffle iota if start/step
583 // values are non-constant OR if intermediate result overflows byte value range.
584 return false;
585 }
586
587 if (!arch_supports_vector(Op_AddVB, num_elem, elem_bt, VecMaskNotUsed) ||
588 !arch_supports_vector(Op_AndV, num_elem, elem_bt, VecMaskNotUsed) ||
589 !arch_supports_vector(Op_VectorLoadConst, num_elem, elem_bt, VecMaskNotUsed) ||
590 !arch_supports_vector(Op_Replicate, num_elem, elem_bt, VecMaskNotUsed)) {
591 return false;
592 }
593
594 if (!do_wrap &&
595 (!arch_supports_vector(Op_SubVB, num_elem, elem_bt, VecMaskNotUsed) ||
596 !arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskNotUsed) ||
597 !arch_supports_vector(Op_VectorMaskCmp, num_elem, elem_bt, VecMaskNotUsed))) {
598 return false;
599 }
600
601 bool step_multiply = !step_val->is_con() || !is_power_of_2(step_val->get_con());
602 if ((step_multiply && !arch_supports_vector(Op_MulVB, num_elem, elem_bt, VecMaskNotUsed)) ||
603 (!step_multiply && !arch_supports_vector(Op_LShiftVB, num_elem, elem_bt, VecMaskNotUsed))) {
604 return false;
605 }
606
607 const Type * type_bt = Type::get_const_basic_type(elem_bt);
608 const TypeVect * vt = TypeVect::make(type_bt, num_elem);
609
610 Node* res = gvn().transform(new VectorLoadConstNode(gvn().makecon(TypeInt::ZERO), vt));
611
612 Node* start = argument(4);
613 Node* step = argument(5);
614
615 if (step_multiply) {
616 Node* bcast_step = gvn().transform(VectorNode::scalar2vector(step, num_elem, type_bt));
617 res = gvn().transform(VectorNode::make(Op_MulVB, res, bcast_step, vt));
618 } else if (step_val->get_con() > 1) {
619 Node* cnt = gvn().makecon(TypeInt::make(log2i_exact(step_val->get_con())));
620 Node* shift_cnt = vector_shift_count(cnt, Op_LShiftI, elem_bt, num_elem);
621 res = gvn().transform(VectorNode::make(Op_LShiftVB, res, shift_cnt, vt));
622 }
623
624 if (!start_val->is_con() || start_val->get_con() != 0) {
625 Node* bcast_start = gvn().transform(VectorNode::scalar2vector(start, num_elem, type_bt));
626 res = gvn().transform(VectorNode::make(Op_AddVB, res, bcast_start, vt));
627 }
628
629 Node * mod_val = gvn().makecon(TypeInt::make(num_elem-1));
630 Node * bcast_mod = gvn().transform(VectorNode::scalar2vector(mod_val, num_elem, type_bt));
631
632 if (do_wrap) {
633 // Wrap the indices greater than lane count.
634 res = gvn().transform(VectorNode::make(Op_AndV, res, bcast_mod, vt));
635 } else {
636 res = partially_wrap_indexes(res, num_elem, elem_bt);
637 }
638
639 ciKlass* sbox_klass = shuffle_klass->const_oop()->as_instance()->java_lang_Class_klass();
640 const TypeInstPtr* shuffle_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, sbox_klass);
641
642 // Wrap it up in VectorBox to keep object type information.
643 res = box_vector(res, shuffle_box_type, elem_bt, num_elem);
644 set_result(res);
645 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
646 return true;
647 }
648
649 // <E, M>
650 // long maskReductionCoerced(int oper, Class<? extends M> maskClass, Class<?> elemClass,
651 // int length, M m, VectorMaskOp<M> defaultImpl)
652 bool LibraryCallKit::inline_vector_mask_operation() {
653 const TypeInt* oper = gvn().type(argument(0))->isa_int();
654 const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
655 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
656 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
657 Node* mask = argument(4);
658
659 if (mask_klass == nullptr || mask_klass->const_oop() == nullptr ||
660 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
661 vlen == nullptr || !vlen->is_con() ||
662 oper == nullptr || !oper->is_con() ||
663 mask->is_top()) {
664 return false; // dead code
665 }
666
667 if (!is_klass_initialized(mask_klass)) {
668 log_if_needed(" ** klass argument not initialized");
669 return false;
670 }
671
672 int num_elem = vlen->get_con();
673 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
674 BasicType elem_bt = elem_type->basic_type();
675
676 int mopc = VectorSupport::vop2ideal(oper->get_con(), elem_bt);
677 if (!arch_supports_vector(mopc, num_elem, elem_bt, VecMaskUseLoad)) {
678 log_if_needed(" ** not supported: arity=1 op=cast#%d/3 vlen2=%d etype2=%s",
679 mopc, num_elem, type2name(elem_bt));
680 return false; // not supported
681 }
682
683 const Type* elem_ty = Type::get_const_basic_type(elem_bt);
684 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
685 const TypeInstPtr* mask_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
686 Node* mask_vec = unbox_vector(mask, mask_box_type, elem_bt, num_elem, true);
687 if (mask_vec == nullptr) {
688 log_if_needed(" ** unbox failed mask=%s",
689 NodeClassNames[argument(4)->Opcode()]);
690 return false;
691 }
692
693 if (mask_vec->bottom_type()->isa_vectmask() == nullptr) {
694 mask_vec = gvn().transform(VectorStoreMaskNode::make(gvn(), mask_vec, elem_bt, num_elem));
695 }
696 const Type* maskoper_ty = mopc == Op_VectorMaskToLong ? (const Type*)TypeLong::LONG : (const Type*)TypeInt::INT;
697 Node* maskoper = gvn().transform(VectorMaskOpNode::make(mask_vec, maskoper_ty, mopc));
698 if (mopc != Op_VectorMaskToLong) {
699 maskoper = ConvI2L(maskoper);
700 }
701 set_result(maskoper);
702
703 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
704 return true;
705 }
706
707 // public static
708 // <V,
709 // Sh extends VectorShuffle<E>,
710 // E>
711 // V shuffleToVector(Class<? extends Vector<E>> vclass, Class<E> elementType,
712 // Class<? extends Sh> shuffleClass, Sh s, int length,
713 // ShuffleToVectorOperation<V, Sh, E> defaultImpl)
714 bool LibraryCallKit::inline_vector_shuffle_to_vector() {
715 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
716 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
717 const TypeInstPtr* shuffle_klass = gvn().type(argument(2))->isa_instptr();
718 Node* shuffle = argument(3);
719 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
720
721 if (vector_klass == nullptr || elem_klass == nullptr || shuffle_klass == nullptr || shuffle->is_top() || vlen == nullptr) {
722 return false; // dead code
723 }
724 if (vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || shuffle_klass->const_oop() == nullptr || !vlen->is_con()) {
725 return false; // not enough info for intrinsification
726 }
727 if (!is_klass_initialized(shuffle_klass) || !is_klass_initialized(vector_klass) ) {
728 log_if_needed(" ** klass argument not initialized");
729 return false;
730 }
731
732 int num_elem = vlen->get_con();
733 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
734 BasicType elem_bt = elem_type->basic_type();
735
736 if (num_elem < 4) {
737 return false;
738 }
739
740 int cast_vopc = VectorCastNode::opcode(-1, T_BYTE); // from shuffle of type T_BYTE
741 // Make sure that cast is implemented to particular type/size combination.
742 if (!arch_supports_vector(cast_vopc, num_elem, elem_bt, VecMaskNotUsed)) {
743 log_if_needed(" ** not supported: arity=1 op=cast#%d/3 vlen2=%d etype2=%s",
744 cast_vopc, num_elem, type2name(elem_bt));
745 return false;
746 }
747
748 ciKlass* sbox_klass = shuffle_klass->const_oop()->as_instance()->java_lang_Class_klass();
749 const TypeInstPtr* shuffle_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, sbox_klass);
750
751 // Unbox shuffle with true flag to indicate its load shuffle to vector
752 // shuffle is a byte array
753 Node* shuffle_vec = unbox_vector(shuffle, shuffle_box_type, T_BYTE, num_elem, true);
754
755 // cast byte to target element type
756 shuffle_vec = gvn().transform(VectorCastNode::make(cast_vopc, shuffle_vec, elem_bt, num_elem));
757
758 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
759 const TypeInstPtr* vec_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
760
761 // Box vector
762 Node* res = box_vector(shuffle_vec, vec_box_type, elem_bt, num_elem);
763 set_result(res);
764 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
765 return true;
766 }
767
768 // public static
769 // <M,
770 // S extends VectorSpecies<E>,
771 // E>
772 // M fromBitsCoerced(Class<? extends M> vmClass, Class<E> elementType, int length,
773 // long bits, int mode, S s,
774 // BroadcastOperation<M, E, S> defaultImpl)
775 bool LibraryCallKit::inline_vector_frombits_coerced() {
776 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
777 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
778 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
779 const TypeLong* bits_type = gvn().type(argument(3))->isa_long();
780 // Mode argument determines the mode of operation it can take following values:-
781 // MODE_BROADCAST for vector Vector.broadcast and VectorMask.maskAll operations.
782 // MODE_BITS_COERCED_LONG_TO_MASK for VectorMask.fromLong operation.
783 const TypeInt* mode = gvn().type(argument(5))->isa_int();
784
785 if (vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
786 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
787 vlen == nullptr || !vlen->is_con() ||
788 bits_type == nullptr ||
789 mode == nullptr || !mode->is_con()) {
790 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s bitwise=%s",
791 NodeClassNames[argument(0)->Opcode()],
792 NodeClassNames[argument(1)->Opcode()],
793 NodeClassNames[argument(2)->Opcode()],
794 NodeClassNames[argument(5)->Opcode()]);
795 return false; // not enough info for intrinsification
796 }
797
798 if (!is_klass_initialized(vector_klass)) {
799 log_if_needed(" ** klass argument not initialized");
800 return false;
801 }
802 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
803 if (!elem_type->is_primitive_type()) {
804 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
805 return false; // should be primitive type
806 }
807 BasicType elem_bt = elem_type->basic_type();
808 int num_elem = vlen->get_con();
809 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
810 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
811
812 bool is_mask = is_vector_mask(vbox_klass);
813 int bcast_mode = mode->get_con();
814 VectorMaskUseType checkFlags = (VectorMaskUseType)(is_mask ? VecMaskUseAll : VecMaskNotUsed);
815 int opc = bcast_mode == VectorSupport::MODE_BITS_COERCED_LONG_TO_MASK ? Op_VectorLongToMask : Op_Replicate;
816
817 if (!arch_supports_vector(opc, num_elem, elem_bt, checkFlags, true /*has_scalar_args*/)) {
818 log_if_needed(" ** not supported: arity=0 op=broadcast vlen=%d etype=%s ismask=%d bcast_mode=%d",
819 num_elem, type2name(elem_bt),
820 is_mask ? 1 : 0,
821 bcast_mode);
822 return false; // not supported
823 }
824
825 Node* broadcast = nullptr;
826 Node* bits = argument(3);
827 Node* elem = bits;
828
829 if (opc == Op_VectorLongToMask) {
830 const TypeVect* vt = TypeVect::makemask(elem_bt, num_elem);
831 if (vt->isa_vectmask()) {
832 broadcast = gvn().transform(new VectorLongToMaskNode(elem, vt));
833 } else {
834 const TypeVect* mvt = TypeVect::make(T_BOOLEAN, num_elem);
835 broadcast = gvn().transform(new VectorLongToMaskNode(elem, mvt));
836 broadcast = gvn().transform(new VectorLoadMaskNode(broadcast, vt));
837 }
838 } else {
839 switch (elem_bt) {
840 case T_BOOLEAN: // fall-through
841 case T_BYTE: // fall-through
842 case T_SHORT: // fall-through
843 case T_CHAR: // fall-through
844 case T_INT: {
845 elem = gvn().transform(new ConvL2INode(bits));
846 break;
847 }
848 case T_DOUBLE: {
849 elem = gvn().transform(new MoveL2DNode(bits));
850 break;
851 }
852 case T_FLOAT: {
853 bits = gvn().transform(new ConvL2INode(bits));
854 elem = gvn().transform(new MoveI2FNode(bits));
855 break;
856 }
857 case T_LONG: {
858 // no conversion needed
859 break;
860 }
861 default: fatal("%s", type2name(elem_bt));
862 }
863 broadcast = VectorNode::scalar2vector(elem, num_elem, Type::get_const_basic_type(elem_bt), is_mask);
864 broadcast = gvn().transform(broadcast);
865 }
866
867 Node* box = box_vector(broadcast, vbox_type, elem_bt, num_elem);
868 set_result(box);
869 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
870 return true;
871 }
872
873 static bool elem_consistent_with_arr(BasicType elem_bt, const TypeAryPtr* arr_type, bool mismatched_ms) {
874 assert(arr_type != nullptr, "unexpected");
875 BasicType arr_elem_bt = arr_type->elem()->array_element_basic_type();
876 if (elem_bt == arr_elem_bt) {
877 return true;
878 } else if (elem_bt == T_SHORT && arr_elem_bt == T_CHAR) {
879 // Load/store of short vector from/to char[] is supported
880 return true;
881 } else if (elem_bt == T_BYTE && arr_elem_bt == T_BOOLEAN) {
882 // Load/store of byte vector from/to boolean[] is supported
883 return true;
884 } else {
885 return mismatched_ms;
886 }
887 }
888
889 // public static
890 // <C,
891 // VM extends VectorPayload,
892 // E,
893 // S extends VectorSpecies<E>>
894 // VM load(Class<? extends VM> vmClass, Class<E> eClass,
895 // int length,
896 // Object base, long offset, // Unsafe addressing
897 // boolean fromSegment,
898 // C container, long index, S s, // Arguments for default implementation
899 // LoadOperation<C, VM, S> defaultImpl) {
900 // public static
901 // <C,
902 // V extends VectorPayload>
903 // void store(Class<?> vClass, Class<?> eClass,
904 // int length,
905 // Object base, long offset, // Unsafe addressing
906 // boolean fromSegment,
907 // V v, C container, long index, // Arguments for default implementation
908 // StoreVectorOperation<C, V> defaultImpl) {
909 bool LibraryCallKit::inline_vector_mem_operation(bool is_store) {
910 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
911 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
912 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
913 const TypeInt* from_ms = gvn().type(argument(6))->isa_int();
914
915 if (vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
916 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
917 vlen == nullptr || !vlen->is_con() ||
918 from_ms == nullptr || !from_ms->is_con()) {
919 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s from_ms=%s",
920 NodeClassNames[argument(0)->Opcode()],
921 NodeClassNames[argument(1)->Opcode()],
922 NodeClassNames[argument(2)->Opcode()],
923 NodeClassNames[argument(6)->Opcode()]);
924 return false; // not enough info for intrinsification
925 }
926 if (!is_klass_initialized(vector_klass)) {
927 log_if_needed(" ** klass argument not initialized");
928 return false;
929 }
930
931 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
932 if (!elem_type->is_primitive_type()) {
933 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
934 return false; // should be primitive type
935 }
936 BasicType elem_bt = elem_type->basic_type();
937 int num_elem = vlen->get_con();
938
939 // TODO When mask usage is supported, VecMaskNotUsed needs to be VecMaskUseLoad.
940 if (!arch_supports_vector(is_store ? Op_StoreVector : Op_LoadVector, num_elem, elem_bt, VecMaskNotUsed)) {
941 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d etype=%s ismask=no",
942 is_store, is_store ? "store" : "load",
943 num_elem, type2name(elem_bt));
944 return false; // not supported
945 }
946
947 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
948 bool is_mask = is_vector_mask(vbox_klass);
949
950 Node* base = argument(3);
951 Node* offset = ConvL2X(argument(4));
952
953 // Save state and restore on bailout
954 uint old_sp = sp();
955 SafePointNode* old_map = clone_map();
956
957 Node* addr = make_unsafe_address(base, offset, (is_mask ? T_BOOLEAN : elem_bt), true);
958
959 // The memory barrier checks are based on ones for unsafe access.
960 // This is not 1-1 implementation.
961 const Type *const base_type = gvn().type(base);
962
963 const TypePtr *addr_type = gvn().type(addr)->isa_ptr();
964 const TypeAryPtr* arr_type = addr_type->isa_aryptr();
965
966 const bool in_native = TypePtr::NULL_PTR == base_type; // base always null
967 const bool in_heap = !TypePtr::NULL_PTR->higher_equal(base_type); // base never null
968
969 const bool is_mixed_access = !in_heap && !in_native;
970
971 const bool is_mismatched_access = in_heap && (addr_type->isa_aryptr() == nullptr);
972
973 const bool needs_cpu_membar = is_mixed_access || is_mismatched_access;
974
975 // For non-masked mismatched memory segment vector read/write accesses, intrinsification can continue
976 // with unknown backing storage type and compiler can skip inserting explicit reinterpretation IR after
977 // loading from or before storing to backing storage which is mandatory for semantic correctness of
978 // big-endian memory layout.
979 bool mismatched_ms = LITTLE_ENDIAN_ONLY(false)
980 BIG_ENDIAN_ONLY(from_ms->get_con() && !is_mask && arr_type != nullptr &&
981 arr_type->elem()->array_element_basic_type() != elem_bt);
982 BasicType mem_elem_bt = mismatched_ms ? arr_type->elem()->array_element_basic_type() : elem_bt;
983 if (!is_java_primitive(mem_elem_bt)) {
984 log_if_needed(" ** non-primitive array element type");
985 return false;
986 }
987 int mem_num_elem = mismatched_ms ? (num_elem * type2aelembytes(elem_bt)) / type2aelembytes(mem_elem_bt) : num_elem;
988 if (arr_type != nullptr && !is_mask && !elem_consistent_with_arr(elem_bt, arr_type, mismatched_ms)) {
989 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d etype=%s atype=%s ismask=no",
990 is_store, is_store ? "store" : "load",
991 num_elem, type2name(elem_bt), type2name(arr_type->elem()->array_element_basic_type()));
992 set_map(old_map);
993 set_sp(old_sp);
994 return false;
995 }
996
997 // In case of mismatched memory segment accesses, we need to double check that the source type memory operations are supported by backend.
998 if (mismatched_ms) {
999 if (is_store) {
1000 if (!arch_supports_vector(Op_StoreVector, num_elem, elem_bt, VecMaskNotUsed)
1001 || !arch_supports_vector(Op_VectorReinterpret, mem_num_elem, mem_elem_bt, VecMaskNotUsed)) {
1002 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d*8 etype=%s/8 ismask=no",
1003 is_store, "store",
1004 num_elem, type2name(elem_bt));
1005 set_map(old_map);
1006 set_sp(old_sp);
1007 return false; // not supported
1008 }
1009 } else {
1010 if (!arch_supports_vector(Op_LoadVector, mem_num_elem, mem_elem_bt, VecMaskNotUsed)
1011 || !arch_supports_vector(Op_VectorReinterpret, num_elem, elem_bt, VecMaskNotUsed)) {
1012 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d*8 etype=%s/8 ismask=no",
1013 is_store, "load",
1014 mem_num_elem, type2name(mem_elem_bt));
1015 set_map(old_map);
1016 set_sp(old_sp);
1017 return false; // not supported
1018 }
1019 }
1020 }
1021 if (is_mask) {
1022 if (!is_store) {
1023 if (!arch_supports_vector(Op_LoadVector, num_elem, elem_bt, VecMaskUseLoad)) {
1024 set_map(old_map);
1025 set_sp(old_sp);
1026 return false; // not supported
1027 }
1028 } else {
1029 if (!arch_supports_vector(Op_StoreVector, num_elem, elem_bt, VecMaskUseStore)) {
1030 set_map(old_map);
1031 set_sp(old_sp);
1032 return false; // not supported
1033 }
1034 }
1035 }
1036
1037 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1038
1039 if (needs_cpu_membar) {
1040 insert_mem_bar(Op_MemBarCPUOrder);
1041 }
1042
1043 if (is_store) {
1044 Node* val = unbox_vector(argument(7), vbox_type, elem_bt, num_elem);
1045 if (val == nullptr) {
1046 set_map(old_map);
1047 set_sp(old_sp);
1048 return false; // operand unboxing failed
1049 }
1050 set_all_memory(reset_memory());
1051
1052 // In case the store needs to happen to byte array, reinterpret the incoming vector to byte vector.
1053 int store_num_elem = num_elem;
1054 if (mismatched_ms) {
1055 store_num_elem = mem_num_elem;
1056 const TypeVect* to_vect_type = TypeVect::make(mem_elem_bt, store_num_elem);
1057 val = gvn().transform(new VectorReinterpretNode(val, val->bottom_type()->is_vect(), to_vect_type));
1058 }
1059 if (is_mask) {
1060 val = gvn().transform(VectorStoreMaskNode::make(gvn(), val, elem_bt, num_elem));
1061 }
1062 Node* vstore = gvn().transform(StoreVectorNode::make(0, control(), memory(addr), addr, addr_type, val, store_num_elem));
1063 set_memory(vstore, addr_type);
1064 } else {
1065 // When using byte array, we need to load as byte then reinterpret the value. Otherwise, do a simple vector load.
1066 Node* vload = nullptr;
1067 if (mismatched_ms) {
1068 vload = gvn().transform(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, mem_num_elem, mem_elem_bt));
1069 const TypeVect* to_vect_type = TypeVect::make(elem_bt, num_elem);
1070 vload = gvn().transform(new VectorReinterpretNode(vload, vload->bottom_type()->is_vect(), to_vect_type));
1071 } else {
1072 // Special handle for masks
1073 if (is_mask) {
1074 vload = gvn().transform(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, num_elem, T_BOOLEAN));
1075 vload = gvn().transform(new VectorLoadMaskNode(vload, TypeVect::makemask(elem_bt, num_elem)));
1076 } else {
1077 vload = gvn().transform(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, num_elem, elem_bt));
1078 }
1079 }
1080 Node* box = box_vector(vload, vbox_type, elem_bt, num_elem);
1081 set_result(box);
1082 }
1083
1084 destruct_map_clone(old_map);
1085
1086 if (needs_cpu_membar) {
1087 insert_mem_bar(Op_MemBarCPUOrder);
1088 }
1089
1090 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1091 return true;
1092 }
1093
1094 // public static
1095 // <C,
1096 // V extends Vector<?>,
1097 // E,
1098 // S extends VectorSpecies<E>,
1099 // M extends VectorMask<E>>
1100 // V loadMasked(Class<? extends V> vClass, Class<M> mClass, Class<E> eClass,
1101 // int length, Object base, long offset, // Unsafe addressing
1102 // boolean fromSegment,
1103 // M m, int offsetInRange,
1104 // C container, long index, S s, // Arguments for default implementation
1105 // LoadVectorMaskedOperation<C, V, S, M> defaultImpl) {
1106 // public static
1107 // <C,
1108 // V extends Vector<E>,
1109 // M extends VectorMask<E>,
1110 // E>
1111 // void storeMasked(Class<? extends V> vClass, Class<M> mClass, Class<E> eClass,
1112 // int length,
1113 // Object base, long offset, // Unsafe addressing
1114 // boolean fromSegment,
1115 // V v, M m, C container, long index, // Arguments for default implementation
1116 // StoreVectorMaskedOperation<C, V, M> defaultImpl) {
1117
1118 bool LibraryCallKit::inline_vector_mem_masked_operation(bool is_store) {
1119 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
1120 const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
1121 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
1122 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
1123 const TypeInt* from_ms = gvn().type(argument(7))->isa_int();
1124
1125 if (vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
1126 mask_klass == nullptr || mask_klass->const_oop() == nullptr ||
1127 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
1128 vlen == nullptr || !vlen->is_con() ||
1129 from_ms == nullptr || !from_ms->is_con()) {
1130 log_if_needed(" ** missing constant: vclass=%s mclass=%s etype=%s vlen=%s from_ms=%s",
1131 NodeClassNames[argument(0)->Opcode()],
1132 NodeClassNames[argument(1)->Opcode()],
1133 NodeClassNames[argument(2)->Opcode()],
1134 NodeClassNames[argument(3)->Opcode()],
1135 NodeClassNames[argument(7)->Opcode()]);
1136 return false; // not enough info for intrinsification
1137 }
1138 if (!is_klass_initialized(vector_klass)) {
1139 log_if_needed(" ** klass argument not initialized");
1140 return false;
1141 }
1142
1143 if (!is_klass_initialized(mask_klass)) {
1144 log_if_needed(" ** mask klass argument not initialized");
1145 return false;
1146 }
1147
1148 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1149 if (!elem_type->is_primitive_type()) {
1150 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1151 return false; // should be primitive type
1152 }
1153
1154 BasicType elem_bt = elem_type->basic_type();
1155 int num_elem = vlen->get_con();
1156
1157 Node* base = argument(4);
1158 Node* offset = ConvL2X(argument(5));
1159
1160 // Save state and restore on bailout
1161 uint old_sp = sp();
1162 SafePointNode* old_map = clone_map();
1163
1164 Node* addr = make_unsafe_address(base, offset, elem_bt, true);
1165 const TypePtr *addr_type = gvn().type(addr)->isa_ptr();
1166 const TypeAryPtr* arr_type = addr_type->isa_aryptr();
1167
1168 bool mismatched_ms = from_ms->get_con() && arr_type != nullptr && arr_type->elem()->array_element_basic_type() != elem_bt;
1169 BIG_ENDIAN_ONLY(if (mismatched_ms) return false;)
1170 // If there is no consistency between array and vector element types, it must be special byte array case
1171 if (arr_type != nullptr && !elem_consistent_with_arr(elem_bt, arr_type, mismatched_ms)) {
1172 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d etype=%s atype=%s",
1173 is_store, is_store ? "storeMasked" : "loadMasked",
1174 num_elem, type2name(elem_bt), type2name(arr_type->elem()->array_element_basic_type()));
1175 set_map(old_map);
1176 set_sp(old_sp);
1177 return false;
1178 }
1179
1180 int mem_num_elem = mismatched_ms ? num_elem * type2aelembytes(elem_bt) : num_elem;
1181 BasicType mem_elem_bt = mismatched_ms ? T_BYTE : elem_bt;
1182 bool supports_predicate = arch_supports_vector(is_store ? Op_StoreVectorMasked : Op_LoadVectorMasked,
1183 mem_num_elem, mem_elem_bt, VecMaskUseLoad);
1184
1185 // If current arch does not support the predicated operations, we have to bail
1186 // out when current case uses the predicate feature.
1187 if (!supports_predicate) {
1188 bool needs_predicate = false;
1189 if (is_store) {
1190 // Masked vector store always uses the predicated store.
1191 needs_predicate = true;
1192 } else {
1193 // Masked vector load with IOOBE always uses the predicated load.
1194 const TypeInt* offset_in_range = gvn().type(argument(9))->isa_int();
1195 if (!offset_in_range->is_con()) {
1196 log_if_needed(" ** missing constant: offsetInRange=%s",
1197 NodeClassNames[argument(8)->Opcode()]);
1198 set_map(old_map);
1199 set_sp(old_sp);
1200 return false;
1201 }
1202 needs_predicate = (offset_in_range->get_con() == 0);
1203 }
1204
1205 if (needs_predicate) {
1206 log_if_needed(" ** not supported: op=%s vlen=%d etype=%s mismatched_ms=%d",
1207 is_store ? "storeMasked" : "loadMasked",
1208 num_elem, type2name(elem_bt), mismatched_ms ? 1 : 0);
1209 set_map(old_map);
1210 set_sp(old_sp);
1211 return false;
1212 }
1213 }
1214
1215 // This only happens for masked vector load. If predicate is not supported, then check whether
1216 // the normal vector load and blend operations are supported by backend.
1217 if (!supports_predicate && (!arch_supports_vector(Op_LoadVector, mem_num_elem, mem_elem_bt, VecMaskNotUsed) ||
1218 !arch_supports_vector(Op_VectorBlend, mem_num_elem, mem_elem_bt, VecMaskUseLoad))) {
1219 log_if_needed(" ** not supported: op=loadMasked vlen=%d etype=%s mismatched_ms=%d",
1220 num_elem, type2name(elem_bt), mismatched_ms ? 1 : 0);
1221 set_map(old_map);
1222 set_sp(old_sp);
1223 return false;
1224 }
1225
1226 // Since we are using byte array, we need to double check that the vector reinterpret operation
1227 // with byte type is supported by backend.
1228 if (mismatched_ms) {
1229 if (!arch_supports_vector(Op_VectorReinterpret, mem_num_elem, T_BYTE, VecMaskNotUsed)) {
1230 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d etype=%s mismatched_ms=1",
1231 is_store, is_store ? "storeMasked" : "loadMasked",
1232 num_elem, type2name(elem_bt));
1233 set_map(old_map);
1234 set_sp(old_sp);
1235 return false;
1236 }
1237 }
1238
1239 // Since it needs to unbox the mask, we need to double check that the related load operations
1240 // for mask are supported by backend.
1241 if (!arch_supports_vector(Op_LoadVector, num_elem, elem_bt, VecMaskUseLoad)) {
1242 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d etype=%s",
1243 is_store, is_store ? "storeMasked" : "loadMasked",
1244 num_elem, type2name(elem_bt));
1245 set_map(old_map);
1246 set_sp(old_sp);
1247 return false;
1248 }
1249
1250 // Can base be null? Otherwise, always on-heap access.
1251 bool can_access_non_heap = TypePtr::NULL_PTR->higher_equal(gvn().type(base));
1252 if (can_access_non_heap) {
1253 insert_mem_bar(Op_MemBarCPUOrder);
1254 }
1255
1256 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1257 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1258 assert(!is_vector_mask(vbox_klass) && is_vector_mask(mbox_klass), "Invalid class type");
1259 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1260 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1261
1262 Node* mask = unbox_vector(is_store ? argument(9) : argument(8), mbox_type, elem_bt, num_elem);
1263 if (mask == nullptr) {
1264 log_if_needed(" ** unbox failed mask=%s",
1265 is_store ? NodeClassNames[argument(9)->Opcode()]
1266 : NodeClassNames[argument(8)->Opcode()]);
1267 set_map(old_map);
1268 set_sp(old_sp);
1269 return false;
1270 }
1271
1272 if (is_store) {
1273 Node* val = unbox_vector(argument(8), vbox_type, elem_bt, num_elem);
1274 if (val == nullptr) {
1275 log_if_needed(" ** unbox failed vector=%s",
1276 NodeClassNames[argument(8)->Opcode()]);
1277 set_map(old_map);
1278 set_sp(old_sp);
1279 return false; // operand unboxing failed
1280 }
1281 set_all_memory(reset_memory());
1282
1283 if (mismatched_ms) {
1284 // Reinterpret the incoming vector to byte vector.
1285 const TypeVect* to_vect_type = TypeVect::make(mem_elem_bt, mem_num_elem);
1286 val = gvn().transform(new VectorReinterpretNode(val, val->bottom_type()->is_vect(), to_vect_type));
1287 // Reinterpret the vector mask to byte type.
1288 const TypeVect* from_mask_type = TypeVect::makemask(elem_bt, num_elem);
1289 const TypeVect* to_mask_type = TypeVect::makemask(mem_elem_bt, mem_num_elem);
1290 mask = gvn().transform(new VectorReinterpretNode(mask, from_mask_type, to_mask_type));
1291 }
1292 Node* vstore = gvn().transform(new StoreVectorMaskedNode(control(), memory(addr), addr, val, addr_type, mask));
1293 set_memory(vstore, addr_type);
1294 } else {
1295 Node* vload = nullptr;
1296
1297 if (mismatched_ms) {
1298 // Reinterpret the vector mask to byte type.
1299 const TypeVect* from_mask_type = TypeVect::makemask(elem_bt, num_elem);
1300 const TypeVect* to_mask_type = TypeVect::makemask(mem_elem_bt, mem_num_elem);
1301 mask = gvn().transform(new VectorReinterpretNode(mask, from_mask_type, to_mask_type));
1302 }
1303
1304 if (supports_predicate) {
1305 // Generate masked load vector node if predicate feature is supported.
1306 const TypeVect* vt = TypeVect::make(mem_elem_bt, mem_num_elem);
1307 vload = gvn().transform(new LoadVectorMaskedNode(control(), memory(addr), addr, addr_type, vt, mask));
1308 } else {
1309 // Use the vector blend to implement the masked load vector. The biased elements are zeros.
1310 Node* zero = gvn().transform(gvn().zerocon(mem_elem_bt));
1311 zero = gvn().transform(VectorNode::scalar2vector(zero, mem_num_elem, Type::get_const_basic_type(mem_elem_bt)));
1312 vload = gvn().transform(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, mem_num_elem, mem_elem_bt));
1313 vload = gvn().transform(new VectorBlendNode(zero, vload, mask));
1314 }
1315
1316 if (mismatched_ms) {
1317 const TypeVect* to_vect_type = TypeVect::make(elem_bt, num_elem);
1318 vload = gvn().transform(new VectorReinterpretNode(vload, vload->bottom_type()->is_vect(), to_vect_type));
1319 }
1320
1321 Node* box = box_vector(vload, vbox_type, elem_bt, num_elem);
1322 set_result(box);
1323 }
1324
1325 destruct_map_clone(old_map);
1326
1327 if (can_access_non_heap) {
1328 insert_mem_bar(Op_MemBarCPUOrder);
1329 }
1330
1331 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1332 return true;
1333 }
1334
1335 // <C,
1336 // V extends Vector<?>,
1337 // W extends Vector<Integer>,
1338 // S extends VectorSpecies<E>,
1339 // M extends VectorMask<E>,
1340 // E>
1341 // V loadWithMap(Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType, int length,
1342 // Class<? extends Vector<Integer>> vectorIndexClass,
1343 // Object base, long offset, // Unsafe addressing
1344 // W index_vector, M m,
1345 // C container, int index, int[] indexMap, int indexM, S s, // Arguments for default implementation
1346 // LoadVectorOperationWithMap<C, V, E, S, M> defaultImpl)
1347 //
1348 // <C,
1349 // V extends Vector<E>,
1350 // W extends Vector<Integer>,
1351 // M extends VectorMask<E>,
1352 // E>
1353 // void storeWithMap(Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType,
1354 // int length, Class<? extends Vector<Integer>> vectorIndexClass, Object base, long offset, // Unsafe addressing
1355 // W index_vector, V v, M m,
1356 // C container, int index, int[] indexMap, int indexM, // Arguments for default implementation
1357 // StoreVectorOperationWithMap<C, V, M, E> defaultImpl)
1358 //
1359 bool LibraryCallKit::inline_vector_gather_scatter(bool is_scatter) {
1360 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
1361 const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
1362 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
1363 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
1364 const TypeInstPtr* vector_idx_klass = gvn().type(argument(4))->isa_instptr();
1365
1366 if (vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
1367 // mask_klass == nullptr || mask_klass->const_oop() == nullptr ||
1368 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
1369 vlen == nullptr || !vlen->is_con() ||
1370 vector_idx_klass == nullptr || vector_idx_klass->const_oop() == nullptr) {
1371 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s viclass=%s",
1372 NodeClassNames[argument(0)->Opcode()],
1373 NodeClassNames[argument(2)->Opcode()],
1374 NodeClassNames[argument(3)->Opcode()],
1375 NodeClassNames[argument(4)->Opcode()]);
1376 return false; // not enough info for intrinsification
1377 }
1378
1379 if (!is_klass_initialized(vector_klass) || !is_klass_initialized(vector_idx_klass)) {
1380 log_if_needed(" ** klass argument not initialized");
1381 return false;
1382 }
1383
1384 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1385 if (!elem_type->is_primitive_type()) {
1386 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1387 return false; // should be primitive type
1388 }
1389
1390 BasicType elem_bt = elem_type->basic_type();
1391 int num_elem = vlen->get_con();
1392
1393 const Type* vmask_type = gvn().type(is_scatter ? argument(10) : argument(9));
1394 bool is_masked_op = vmask_type != TypePtr::NULL_PTR;
1395 if (is_masked_op) {
1396 if (mask_klass == nullptr || mask_klass->const_oop() == nullptr) {
1397 log_if_needed(" ** missing constant: maskclass=%s", NodeClassNames[argument(1)->Opcode()]);
1398 return false; // not enough info for intrinsification
1399 }
1400
1401 if (!is_klass_initialized(mask_klass)) {
1402 log_if_needed(" ** mask klass argument not initialized");
1403 return false;
1404 }
1405
1406 if (vmask_type->maybe_null()) {
1407 log_if_needed(" ** null mask values are not allowed for masked op");
1408 return false;
1409 }
1410
1411 // Check whether the predicated gather/scatter node is supported by architecture.
1412 VectorMaskUseType mask = (VectorMaskUseType) (VecMaskUseLoad | VecMaskUsePred);
1413 if (!arch_supports_vector(is_scatter ? Op_StoreVectorScatterMasked : Op_LoadVectorGatherMasked, num_elem, elem_bt, mask)) {
1414 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d etype=%s is_masked_op=1",
1415 is_scatter, is_scatter ? "scatterMasked" : "gatherMasked",
1416 num_elem, type2name(elem_bt));
1417 return false; // not supported
1418 }
1419 } else {
1420 // Check whether the normal gather/scatter node is supported for non-masked operation.
1421 if (!arch_supports_vector(is_scatter ? Op_StoreVectorScatter : Op_LoadVectorGather, num_elem, elem_bt, VecMaskNotUsed)) {
1422 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d etype=%s is_masked_op=0",
1423 is_scatter, is_scatter ? "scatter" : "gather",
1424 num_elem, type2name(elem_bt));
1425 return false; // not supported
1426 }
1427 }
1428
1429 // Check that the vector holding indices is supported by architecture
1430 // For sub-word gathers expander receive index array.
1431 if (!is_subword_type(elem_bt) && !arch_supports_vector(Op_LoadVector, num_elem, T_INT, VecMaskNotUsed)) {
1432 log_if_needed(" ** not supported: arity=%d op=%s/loadindex vlen=%d etype=int is_masked_op=%d",
1433 is_scatter, is_scatter ? "scatter" : "gather",
1434 num_elem, is_masked_op ? 1 : 0);
1435 return false; // not supported
1436 }
1437
1438 Node* base = argument(5);
1439 Node* offset = ConvL2X(argument(6));
1440
1441 // Save state and restore on bailout
1442 uint old_sp = sp();
1443 SafePointNode* old_map = clone_map();
1444
1445 Node* addr = make_unsafe_address(base, offset, elem_bt, true);
1446
1447 const TypePtr *addr_type = gvn().type(addr)->isa_ptr();
1448 const TypeAryPtr* arr_type = addr_type->isa_aryptr();
1449
1450 // The array must be consistent with vector type
1451 if (arr_type == nullptr || (arr_type != nullptr && !elem_consistent_with_arr(elem_bt, arr_type, false))) {
1452 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d etype=%s atype=%s ismask=no",
1453 is_scatter, is_scatter ? "scatter" : "gather",
1454 num_elem, type2name(elem_bt), type2name(arr_type->elem()->array_element_basic_type()));
1455 set_map(old_map);
1456 set_sp(old_sp);
1457 return false;
1458 }
1459
1460 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1461 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1462 ciKlass* vbox_idx_klass = vector_idx_klass->const_oop()->as_instance()->java_lang_Class_klass();
1463 if (vbox_idx_klass == nullptr) {
1464 set_map(old_map);
1465 set_sp(old_sp);
1466 return false;
1467 }
1468
1469 Node* index_vect = nullptr;
1470 const TypeInstPtr* vbox_idx_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_idx_klass);
1471 if (!is_subword_type(elem_bt)) {
1472 index_vect = unbox_vector(argument(8), vbox_idx_type, T_INT, num_elem);
1473 if (index_vect == nullptr) {
1474 set_map(old_map);
1475 set_sp(old_sp);
1476 return false;
1477 }
1478 }
1479
1480 Node* mask = nullptr;
1481 if (is_masked_op) {
1482 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1483 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1484 mask = unbox_vector(is_scatter ? argument(10) : argument(9), mbox_type, elem_bt, num_elem);
1485 if (mask == nullptr) {
1486 log_if_needed(" ** unbox failed mask=%s",
1487 is_scatter ? NodeClassNames[argument(10)->Opcode()]
1488 : NodeClassNames[argument(9)->Opcode()]);
1489 set_map(old_map);
1490 set_sp(old_sp);
1491 return false;
1492 }
1493 }
1494
1495 const TypeVect* vector_type = TypeVect::make(elem_bt, num_elem);
1496 if (is_scatter) {
1497 Node* val = unbox_vector(argument(9), vbox_type, elem_bt, num_elem);
1498 if (val == nullptr) {
1499 set_map(old_map);
1500 set_sp(old_sp);
1501 return false; // operand unboxing failed
1502 }
1503 set_all_memory(reset_memory());
1504
1505 Node* vstore = nullptr;
1506 if (mask != nullptr) {
1507 vstore = gvn().transform(new StoreVectorScatterMaskedNode(control(), memory(addr), addr, addr_type, val, index_vect, mask));
1508 } else {
1509 vstore = gvn().transform(new StoreVectorScatterNode(control(), memory(addr), addr, addr_type, val, index_vect));
1510 }
1511 set_memory(vstore, addr_type);
1512 } else {
1513 Node* vload = nullptr;
1514 Node* index = argument(11);
1515 Node* indexMap = argument(12);
1516 Node* indexM = argument(13);
1517 if (mask != nullptr) {
1518 if (is_subword_type(elem_bt)) {
1519 Node* index_arr_base = array_element_address(indexMap, indexM, T_INT);
1520 vload = gvn().transform(new LoadVectorGatherMaskedNode(control(), memory(addr), addr, addr_type, vector_type, index_arr_base, mask, index));
1521 } else {
1522 vload = gvn().transform(new LoadVectorGatherMaskedNode(control(), memory(addr), addr, addr_type, vector_type, index_vect, mask));
1523 }
1524 } else {
1525 if (is_subword_type(elem_bt)) {
1526 Node* index_arr_base = array_element_address(indexMap, indexM, T_INT);
1527 vload = gvn().transform(new LoadVectorGatherNode(control(), memory(addr), addr, addr_type, vector_type, index_arr_base, index));
1528 } else {
1529 vload = gvn().transform(new LoadVectorGatherNode(control(), memory(addr), addr, addr_type, vector_type, index_vect));
1530 }
1531 }
1532 Node* box = box_vector(vload, vbox_type, elem_bt, num_elem);
1533 set_result(box);
1534 }
1535
1536 destruct_map_clone(old_map);
1537
1538 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1539 return true;
1540 }
1541
1542 // public static
1543 // <V extends Vector<E>,
1544 // M extends VectorMask<E>,
1545 // E>
1546 // long reductionCoerced(int oprId, Class<? extends V> vectorClass, Class<? extends M> maskClass,
1547 // Class<E> elementType, int length, V v, M m,
1548 // ReductionOperation<V, M> defaultImpl)
1549 bool LibraryCallKit::inline_vector_reduction() {
1550 const TypeInt* opr = gvn().type(argument(0))->isa_int();
1551 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
1552 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
1553 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
1554 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
1555
1556 if (opr == nullptr || !opr->is_con() ||
1557 vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
1558 // mask_klass == nullptr || mask_klass->const_oop() == nullptr ||
1559 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
1560 vlen == nullptr || !vlen->is_con()) {
1561 log_if_needed(" ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
1562 NodeClassNames[argument(0)->Opcode()],
1563 NodeClassNames[argument(1)->Opcode()],
1564 NodeClassNames[argument(3)->Opcode()],
1565 NodeClassNames[argument(4)->Opcode()]);
1566 return false; // not enough info for intrinsification
1567 }
1568 if (!is_klass_initialized(vector_klass)) {
1569 log_if_needed(" ** klass argument not initialized");
1570 return false;
1571 }
1572 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1573 if (!elem_type->is_primitive_type()) {
1574 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1575 return false; // should be primitive type
1576 }
1577
1578 const Type* vmask_type = gvn().type(argument(6));
1579 bool is_masked_op = vmask_type != TypePtr::NULL_PTR;
1580 if (is_masked_op) {
1581 if (mask_klass == nullptr || mask_klass->const_oop() == nullptr) {
1582 log_if_needed(" ** missing constant: maskclass=%s", NodeClassNames[argument(2)->Opcode()]);
1583 return false; // not enough info for intrinsification
1584 }
1585
1586 if (!is_klass_initialized(mask_klass)) {
1587 log_if_needed(" ** mask klass argument not initialized");
1588 return false;
1589 }
1590
1591 if (vmask_type->maybe_null()) {
1592 log_if_needed(" ** null mask values are not allowed for masked op");
1593 return false;
1594 }
1595 }
1596
1597 BasicType elem_bt = elem_type->basic_type();
1598 int num_elem = vlen->get_con();
1599 int opc = VectorSupport::vop2ideal(opr->get_con(), elem_bt);
1600 int sopc = ReductionNode::opcode(opc, elem_bt);
1601
1602 // When using mask, mask use type needs to be VecMaskUseLoad.
1603 if (!arch_supports_vector(sopc, num_elem, elem_bt, is_masked_op ? VecMaskUseLoad : VecMaskNotUsed)) {
1604 log_if_needed(" ** not supported: arity=1 op=%d/reduce vlen=%d etype=%s is_masked_op=%d",
1605 sopc, num_elem, type2name(elem_bt), is_masked_op ? 1 : 0);
1606 return false;
1607 }
1608
1609 // Return true if current platform has implemented the masked operation with predicate feature.
1610 bool use_predicate = is_masked_op && arch_supports_vector(sopc, num_elem, elem_bt, VecMaskUsePred);
1611 if (is_masked_op && !use_predicate && !arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad)) {
1612 log_if_needed(" ** not supported: arity=1 op=%d/reduce vlen=%d etype=%s is_masked_op=1",
1613 sopc, num_elem, type2name(elem_bt));
1614 return false;
1615 }
1616
1617 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1618 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1619
1620 Node* opd = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
1621 if (opd == nullptr) {
1622 return false; // operand unboxing failed
1623 }
1624
1625 Node* mask = nullptr;
1626 if (is_masked_op) {
1627 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1628 assert(is_vector_mask(mbox_klass), "argument(2) should be a mask class");
1629 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1630 mask = unbox_vector(argument(6), mbox_type, elem_bt, num_elem);
1631 if (mask == nullptr) {
1632 log_if_needed(" ** unbox failed mask=%s",
1633 NodeClassNames[argument(6)->Opcode()]);
1634 return false;
1635 }
1636 }
1637
1638 Node* init = ReductionNode::make_identity_con_scalar(gvn(), opc, elem_bt);
1639 Node* value = opd;
1640
1641 assert(mask != nullptr || !is_masked_op, "Masked op needs the mask value never null");
1642 if (mask != nullptr && !use_predicate) {
1643 Node* reduce_identity = gvn().transform(VectorNode::scalar2vector(init, num_elem, Type::get_const_basic_type(elem_bt)));
1644 value = gvn().transform(new VectorBlendNode(reduce_identity, value, mask));
1645 }
1646
1647 // Make an unordered Reduction node. This affects only AddReductionVF/VD and MulReductionVF/VD,
1648 // as these operations are allowed to be associative (not requiring strict order) in VectorAPI.
1649 value = ReductionNode::make(opc, nullptr, init, value, elem_bt, /* requires_strict_order */ false);
1650
1651 if (mask != nullptr && use_predicate) {
1652 value->add_req(mask);
1653 value->add_flag(Node::Flag_is_predicated_vector);
1654 }
1655
1656 value = gvn().transform(value);
1657
1658 Node* bits = nullptr;
1659 switch (elem_bt) {
1660 case T_BYTE:
1661 case T_SHORT:
1662 case T_INT: {
1663 bits = gvn().transform(new ConvI2LNode(value));
1664 break;
1665 }
1666 case T_FLOAT: {
1667 value = gvn().transform(new MoveF2INode(value));
1668 bits = gvn().transform(new ConvI2LNode(value));
1669 break;
1670 }
1671 case T_DOUBLE: {
1672 bits = gvn().transform(new MoveD2LNode(value));
1673 break;
1674 }
1675 case T_LONG: {
1676 bits = value; // no conversion needed
1677 break;
1678 }
1679 default: fatal("%s", type2name(elem_bt));
1680 }
1681 set_result(bits);
1682 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1683 return true;
1684 }
1685
1686 // public static <V> boolean test(int cond, Class<?> vectorClass, Class<?> elementType, int vlen,
1687 // V v1, V v2,
1688 // BiFunction<V, V, Boolean> defaultImpl)
1689 //
1690 bool LibraryCallKit::inline_vector_test() {
1691 const TypeInt* cond = gvn().type(argument(0))->isa_int();
1692 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
1693 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
1694 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
1695
1696 if (cond == nullptr || !cond->is_con() ||
1697 vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
1698 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
1699 vlen == nullptr || !vlen->is_con()) {
1700 log_if_needed(" ** missing constant: cond=%s vclass=%s etype=%s vlen=%s",
1701 NodeClassNames[argument(0)->Opcode()],
1702 NodeClassNames[argument(1)->Opcode()],
1703 NodeClassNames[argument(2)->Opcode()],
1704 NodeClassNames[argument(3)->Opcode()]);
1705 return false; // not enough info for intrinsification
1706 }
1707 if (!is_klass_initialized(vector_klass)) {
1708 log_if_needed(" ** klass argument not initialized");
1709 return false;
1710 }
1711 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1712 if (!elem_type->is_primitive_type()) {
1713 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1714 return false; // should be primitive type
1715 }
1716 BasicType elem_bt = elem_type->basic_type();
1717 int num_elem = vlen->get_con();
1718 BoolTest::mask booltest = (BoolTest::mask)cond->get_con();
1719 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1720 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1721
1722 if (!arch_supports_vector(Op_VectorTest, num_elem, elem_bt, is_vector_mask(vbox_klass) ? VecMaskUseLoad : VecMaskNotUsed)) {
1723 log_if_needed(" ** not supported: arity=2 op=test/%d vlen=%d etype=%s ismask=%d",
1724 cond->get_con(), num_elem, type2name(elem_bt),
1725 is_vector_mask(vbox_klass));
1726 return false;
1727 }
1728
1729 Node* opd1 = unbox_vector(argument(4), vbox_type, elem_bt, num_elem);
1730 Node* opd2;
1731 if (Matcher::vectortest_needs_second_argument(booltest == BoolTest::overflow,
1732 opd1->bottom_type()->isa_vectmask())) {
1733 opd2 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
1734 } else {
1735 opd2 = opd1;
1736 }
1737 if (opd1 == nullptr || opd2 == nullptr) {
1738 return false; // operand unboxing failed
1739 }
1740
1741 Node* cmp = gvn().transform(new VectorTestNode(opd1, opd2, booltest));
1742 BoolTest::mask test = Matcher::vectortest_mask(booltest == BoolTest::overflow,
1743 opd1->bottom_type()->isa_vectmask(), num_elem);
1744 Node* bol = gvn().transform(new BoolNode(cmp, test));
1745 Node* res = gvn().transform(new CMoveINode(bol, gvn().intcon(0), gvn().intcon(1), TypeInt::BOOL));
1746
1747 set_result(res);
1748 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1749 return true;
1750 }
1751
1752 // public static
1753 // <V extends Vector<E>,
1754 // M extends VectorMask<E>,
1755 // E>
1756 // V blend(Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType, int vlen,
1757 // V v1, V v2, M m,
1758 // VectorBlendOp<V, M, E> defaultImpl)
1759 bool LibraryCallKit::inline_vector_blend() {
1760 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
1761 const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
1762 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
1763 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
1764
1765 if (mask_klass == nullptr || vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr) {
1766 return false; // dead code
1767 }
1768 if (mask_klass->const_oop() == nullptr || vector_klass->const_oop() == nullptr ||
1769 elem_klass->const_oop() == nullptr || !vlen->is_con()) {
1770 log_if_needed(" ** missing constant: vclass=%s mclass=%s etype=%s vlen=%s",
1771 NodeClassNames[argument(0)->Opcode()],
1772 NodeClassNames[argument(1)->Opcode()],
1773 NodeClassNames[argument(2)->Opcode()],
1774 NodeClassNames[argument(3)->Opcode()]);
1775 return false; // not enough info for intrinsification
1776 }
1777 if (!is_klass_initialized(vector_klass) || !is_klass_initialized(mask_klass)) {
1778 log_if_needed(" ** klass argument not initialized");
1779 return false;
1780 }
1781 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1782 if (!elem_type->is_primitive_type()) {
1783 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1784 return false; // should be primitive type
1785 }
1786 BasicType elem_bt = elem_type->basic_type();
1787 BasicType mask_bt = elem_bt;
1788 int num_elem = vlen->get_con();
1789
1790 if (!arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad)) {
1791 log_if_needed(" ** not supported: arity=2 op=blend vlen=%d etype=%s ismask=useload",
1792 num_elem, type2name(elem_bt));
1793 return false; // not supported
1794 }
1795 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1796 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1797
1798 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1799 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1800
1801 Node* v1 = unbox_vector(argument(4), vbox_type, elem_bt, num_elem);
1802 Node* v2 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
1803 Node* mask = unbox_vector(argument(6), mbox_type, mask_bt, num_elem);
1804
1805 if (v1 == nullptr || v2 == nullptr || mask == nullptr) {
1806 return false; // operand unboxing failed
1807 }
1808
1809 Node* blend = gvn().transform(new VectorBlendNode(v1, v2, mask));
1810
1811 Node* box = box_vector(blend, vbox_type, elem_bt, num_elem);
1812 set_result(box);
1813 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1814 return true;
1815 }
1816
1817 // public static
1818 // <V extends Vector<E>,
1819 // M extends VectorMask<E>,
1820 // E>
1821 // M compare(int cond, Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType, int vlen,
1822 // V v1, V v2, M m,
1823 // VectorCompareOp<V,M> defaultImpl)
1824 bool LibraryCallKit::inline_vector_compare() {
1825 const TypeInt* cond = gvn().type(argument(0))->isa_int();
1826 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
1827 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
1828 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
1829 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
1830
1831 if (cond == nullptr || vector_klass == nullptr || mask_klass == nullptr || elem_klass == nullptr || vlen == nullptr) {
1832 return false; // dead code
1833 }
1834 if (!cond->is_con() || vector_klass->const_oop() == nullptr || mask_klass->const_oop() == nullptr ||
1835 elem_klass->const_oop() == nullptr || !vlen->is_con()) {
1836 log_if_needed(" ** missing constant: cond=%s vclass=%s mclass=%s etype=%s vlen=%s",
1837 NodeClassNames[argument(0)->Opcode()],
1838 NodeClassNames[argument(1)->Opcode()],
1839 NodeClassNames[argument(2)->Opcode()],
1840 NodeClassNames[argument(3)->Opcode()],
1841 NodeClassNames[argument(4)->Opcode()]);
1842 return false; // not enough info for intrinsification
1843 }
1844 if (!is_klass_initialized(vector_klass) || !is_klass_initialized(mask_klass)) {
1845 log_if_needed(" ** klass argument not initialized");
1846 return false;
1847 }
1848 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1849 if (!elem_type->is_primitive_type()) {
1850 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1851 return false; // should be primitive type
1852 }
1853
1854 int num_elem = vlen->get_con();
1855 BasicType elem_bt = elem_type->basic_type();
1856 BasicType mask_bt = elem_bt;
1857
1858 if ((cond->get_con() & BoolTest::unsigned_compare) != 0) {
1859 if (!Matcher::supports_vector_comparison_unsigned(num_elem, elem_bt)) {
1860 log_if_needed(" ** not supported: unsigned comparison op=comp/%d vlen=%d etype=%s ismask=usestore",
1861 cond->get_con() & (BoolTest::unsigned_compare - 1), num_elem, type2name(elem_bt));
1862 return false;
1863 }
1864 }
1865
1866 if (!arch_supports_vector(Op_VectorMaskCmp, num_elem, elem_bt, VecMaskUseStore)) {
1867 log_if_needed(" ** not supported: arity=2 op=comp/%d vlen=%d etype=%s ismask=usestore",
1868 cond->get_con(), num_elem, type2name(elem_bt));
1869 return false;
1870 }
1871
1872 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1873 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1874
1875 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1876 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1877
1878 Node* v1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
1879 Node* v2 = unbox_vector(argument(6), vbox_type, elem_bt, num_elem);
1880
1881 bool is_masked_op = argument(7)->bottom_type() != TypePtr::NULL_PTR;
1882 Node* mask = is_masked_op ? unbox_vector(argument(7), mbox_type, elem_bt, num_elem) : nullptr;
1883 if (is_masked_op && mask == nullptr) {
1884 log_if_needed(" ** not supported: mask = null arity=2 op=comp/%d vlen=%d etype=%s ismask=usestore is_masked_op=1",
1885 cond->get_con(), num_elem, type2name(elem_bt));
1886 return false;
1887 }
1888
1889 bool use_predicate = is_masked_op && arch_supports_vector(Op_VectorMaskCmp, num_elem, elem_bt, VecMaskUsePred);
1890 if (is_masked_op && !use_predicate && !arch_supports_vector(Op_AndV, num_elem, elem_bt, VecMaskUseLoad)) {
1891 log_if_needed(" ** not supported: arity=2 op=comp/%d vlen=%d etype=%s ismask=usestore is_masked_op=1",
1892 cond->get_con(), num_elem, type2name(elem_bt));
1893 return false;
1894 }
1895
1896 if (v1 == nullptr || v2 == nullptr) {
1897 return false; // operand unboxing failed
1898 }
1899 BoolTest::mask pred = (BoolTest::mask)cond->get_con();
1900 ConINode* pred_node = (ConINode*)gvn().makecon(cond);
1901
1902 const TypeVect* vmask_type = TypeVect::makemask(mask_bt, num_elem);
1903 Node* operation = new VectorMaskCmpNode(pred, v1, v2, pred_node, vmask_type);
1904
1905 if (is_masked_op) {
1906 if (use_predicate) {
1907 operation->add_req(mask);
1908 operation->add_flag(Node::Flag_is_predicated_vector);
1909 } else {
1910 operation = gvn().transform(operation);
1911 operation = VectorNode::make(Op_AndV, operation, mask, vmask_type);
1912 }
1913 }
1914
1915 operation = gvn().transform(operation);
1916
1917 Node* box = box_vector(operation, mbox_type, mask_bt, num_elem);
1918 set_result(box);
1919 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1920 return true;
1921 }
1922
1923 // public static
1924 // <V extends Vector<E>,
1925 // Sh extends VectorShuffle<E>,
1926 // M extends VectorMask<E>,
1927 // E>
1928 // V rearrangeOp(Class<? extends V> vectorClass, Class<Sh> shuffleClass, Class<M> maskClass, Class<E> elementType, int vlen,
1929 // V v1, Sh sh, M m,
1930 // VectorRearrangeOp<V, Sh, M, E> defaultImpl)
1931 bool LibraryCallKit::inline_vector_rearrange() {
1932 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
1933 const TypeInstPtr* shuffle_klass = gvn().type(argument(1))->isa_instptr();
1934 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
1935 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
1936 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
1937
1938 if (vector_klass == nullptr || shuffle_klass == nullptr || elem_klass == nullptr || vlen == nullptr) {
1939 return false; // dead code
1940 }
1941 if (shuffle_klass->const_oop() == nullptr ||
1942 vector_klass->const_oop() == nullptr ||
1943 elem_klass->const_oop() == nullptr ||
1944 !vlen->is_con()) {
1945 log_if_needed(" ** missing constant: vclass=%s sclass=%s etype=%s vlen=%s",
1946 NodeClassNames[argument(0)->Opcode()],
1947 NodeClassNames[argument(1)->Opcode()],
1948 NodeClassNames[argument(3)->Opcode()],
1949 NodeClassNames[argument(4)->Opcode()]);
1950 return false; // not enough info for intrinsification
1951 }
1952 if (!is_klass_initialized(vector_klass) ||
1953 !is_klass_initialized(shuffle_klass)) {
1954 log_if_needed(" ** klass argument not initialized");
1955 return false;
1956 }
1957 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1958 if (!elem_type->is_primitive_type()) {
1959 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1960 return false; // should be primitive type
1961 }
1962 BasicType elem_bt = elem_type->basic_type();
1963 BasicType shuffle_bt = elem_bt;
1964 int num_elem = vlen->get_con();
1965
1966 if (!arch_supports_vector(Op_VectorLoadShuffle, num_elem, elem_bt, VecMaskNotUsed)) {
1967 log_if_needed(" ** not supported: arity=0 op=load/shuffle vlen=%d etype=%s ismask=no",
1968 num_elem, type2name(elem_bt));
1969 return false; // not supported
1970 }
1971
1972 bool is_masked_op = argument(7)->bottom_type() != TypePtr::NULL_PTR;
1973 bool use_predicate = is_masked_op;
1974 if (is_masked_op &&
1975 (mask_klass == nullptr ||
1976 mask_klass->const_oop() == nullptr ||
1977 !is_klass_initialized(mask_klass))) {
1978 log_if_needed(" ** mask_klass argument not initialized");
1979 }
1980 VectorMaskUseType checkFlags = (VectorMaskUseType)(is_masked_op ? (VecMaskUseLoad | VecMaskUsePred) : VecMaskNotUsed);
1981 if (!arch_supports_vector(Op_VectorRearrange, num_elem, elem_bt, checkFlags)) {
1982 use_predicate = false;
1983 if(!is_masked_op ||
1984 (!arch_supports_vector(Op_VectorRearrange, num_elem, elem_bt, VecMaskNotUsed) ||
1985 !arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad) ||
1986 !arch_supports_vector(Op_Replicate, num_elem, elem_bt, VecMaskNotUsed))) {
1987 log_if_needed(" ** not supported: arity=2 op=shuffle/rearrange vlen=%d etype=%s ismask=no",
1988 num_elem, type2name(elem_bt));
1989 return false; // not supported
1990 }
1991 }
1992 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1993 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1994
1995 ciKlass* shbox_klass = shuffle_klass->const_oop()->as_instance()->java_lang_Class_klass();
1996 const TypeInstPtr* shbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, shbox_klass);
1997
1998 Node* v1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
1999 Node* shuffle = unbox_vector(argument(6), shbox_type, shuffle_bt, num_elem);
2000
2001 if (v1 == nullptr || shuffle == nullptr) {
2002 return false; // operand unboxing failed
2003 }
2004
2005 Node* mask = nullptr;
2006 if (is_masked_op) {
2007 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
2008 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
2009 mask = unbox_vector(argument(7), mbox_type, elem_bt, num_elem);
2010 if (mask == nullptr) {
2011 log_if_needed(" ** not supported: arity=3 op=shuffle/rearrange vlen=%d etype=%s ismask=useload is_masked_op=1",
2012 num_elem, type2name(elem_bt));
2013 return false;
2014 }
2015 }
2016
2017 Node* rearrange = new VectorRearrangeNode(v1, shuffle);
2018 if (is_masked_op) {
2019 if (use_predicate) {
2020 rearrange->add_req(mask);
2021 rearrange->add_flag(Node::Flag_is_predicated_vector);
2022 } else {
2023 const TypeVect* vt = v1->bottom_type()->is_vect();
2024 rearrange = gvn().transform(rearrange);
2025 Node* zero = gvn().makecon(Type::get_zero_type(elem_bt));
2026 Node* zerovec = gvn().transform(VectorNode::scalar2vector(zero, num_elem, Type::get_const_basic_type(elem_bt)));
2027 rearrange = new VectorBlendNode(zerovec, rearrange, mask);
2028 }
2029 }
2030 rearrange = gvn().transform(rearrange);
2031
2032 Node* box = box_vector(rearrange, vbox_type, elem_bt, num_elem);
2033 set_result(box);
2034 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2035 return true;
2036 }
2037
2038 static address get_svml_address(int vop, int bits, BasicType bt, char* name_ptr, int name_len) {
2039 address addr = nullptr;
2040 assert(UseVectorStubs, "sanity");
2041 assert(name_ptr != nullptr, "unexpected");
2042 assert((vop >= VectorSupport::VECTOR_OP_SVML_START) && (vop <= VectorSupport::VECTOR_OP_SVML_END), "unexpected");
2043 int op = vop - VectorSupport::VECTOR_OP_SVML_START;
2044
2045 switch(bits) {
2046 case 64: //fallthough
2047 case 128: //fallthough
2048 case 256: //fallthough
2049 case 512:
2050 if (bt == T_FLOAT) {
2051 snprintf(name_ptr, name_len, "vector_%s_float%d", VectorSupport::svmlname[op], bits);
2052 addr = StubRoutines::_vector_f_math[exact_log2(bits/64)][op];
2053 } else {
2054 assert(bt == T_DOUBLE, "must be FP type only");
2055 snprintf(name_ptr, name_len, "vector_%s_double%d", VectorSupport::svmlname[op], bits);
2056 addr = StubRoutines::_vector_d_math[exact_log2(bits/64)][op];
2057 }
2058 break;
2059 default:
2060 snprintf(name_ptr, name_len, "invalid");
2061 addr = nullptr;
2062 Unimplemented();
2063 break;
2064 }
2065
2066 return addr;
2067 }
2068
2069 Node* LibraryCallKit::gen_call_to_svml(int vector_api_op_id, BasicType bt, int num_elem, Node* opd1, Node* opd2) {
2070 assert(UseVectorStubs, "sanity");
2071 assert(vector_api_op_id >= VectorSupport::VECTOR_OP_SVML_START && vector_api_op_id <= VectorSupport::VECTOR_OP_SVML_END, "need valid op id");
2072 assert(opd1 != nullptr, "must not be null");
2073 const TypeVect* vt = TypeVect::make(bt, num_elem);
2074 const TypeFunc* call_type = OptoRuntime::Math_Vector_Vector_Type(opd2 != nullptr ? 2 : 1, vt, vt);
2075 char name[100] = "";
2076
2077 // Get address for svml method.
2078 address addr = get_svml_address(vector_api_op_id, vt->length_in_bytes() * BitsPerByte, bt, name, 100);
2079
2080 if (addr == nullptr) {
2081 return nullptr;
2082 }
2083
2084 assert(name[0] != '\0', "name must not be null");
2085 Node* operation = make_runtime_call(RC_VECTOR,
2086 call_type,
2087 addr,
2088 name,
2089 TypePtr::BOTTOM,
2090 opd1,
2091 opd2);
2092 return gvn().transform(new ProjNode(gvn().transform(operation), TypeFunc::Parms));
2093 }
2094
2095 // public static
2096 // <V extends Vector<E>,
2097 // M extends VectorMask<E>,
2098 // E>
2099 // V broadcastInt(int opr, Class<? extends V> vectorClass, Class<? extends M> maskClass,
2100 // Class<E> elementType, int length,
2101 // V v, int n, M m,
2102 // VectorBroadcastIntOp<V, M> defaultImpl)
2103 bool LibraryCallKit::inline_vector_broadcast_int() {
2104 const TypeInt* opr = gvn().type(argument(0))->isa_int();
2105 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
2106 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
2107 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
2108 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
2109
2110 if (opr == nullptr || vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr) {
2111 return false; // dead code
2112 }
2113 if (!opr->is_con() || vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
2114 log_if_needed(" ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
2115 NodeClassNames[argument(0)->Opcode()],
2116 NodeClassNames[argument(1)->Opcode()],
2117 NodeClassNames[argument(3)->Opcode()],
2118 NodeClassNames[argument(4)->Opcode()]);
2119 return false; // not enough info for intrinsification
2120 }
2121 if (!is_klass_initialized(vector_klass)) {
2122 log_if_needed(" ** klass argument not initialized");
2123 return false;
2124 }
2125
2126 const Type* vmask_type = gvn().type(argument(7));
2127 bool is_masked_op = vmask_type != TypePtr::NULL_PTR;
2128 if (is_masked_op) {
2129 if (mask_klass == nullptr || mask_klass->const_oop() == nullptr) {
2130 log_if_needed(" ** missing constant: maskclass=%s", NodeClassNames[argument(2)->Opcode()]);
2131 return false; // not enough info for intrinsification
2132 }
2133
2134 if (!is_klass_initialized(mask_klass)) {
2135 log_if_needed(" ** mask klass argument not initialized");
2136 return false;
2137 }
2138
2139 if (vmask_type->maybe_null()) {
2140 log_if_needed(" ** null mask values are not allowed for masked op");
2141 return false;
2142 }
2143 }
2144
2145 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2146 if (!elem_type->is_primitive_type()) {
2147 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2148 return false; // should be primitive type
2149 }
2150
2151 int num_elem = vlen->get_con();
2152 BasicType elem_bt = elem_type->basic_type();
2153 int opc = VectorSupport::vop2ideal(opr->get_con(), elem_bt);
2154
2155 bool is_shift = VectorNode::is_shift_opcode(opc);
2156 bool is_rotate = VectorNode::is_rotate_opcode(opc);
2157
2158 if (opc == 0 || (!is_shift && !is_rotate)) {
2159 log_if_needed(" ** operation not supported: op=%d bt=%s", opr->get_con(), type2name(elem_bt));
2160 return false; // operation not supported
2161 }
2162
2163 int sopc = VectorNode::opcode(opc, elem_bt);
2164 if (sopc == 0) {
2165 log_if_needed(" ** operation not supported: opc=%s bt=%s", NodeClassNames[opc], type2name(elem_bt));
2166 return false; // operation not supported
2167 }
2168
2169 Node* cnt = argument(6);
2170 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2171 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2172 const TypeInt* cnt_type = cnt->bottom_type()->isa_int();
2173
2174 // If CPU supports vector constant rotate instructions pass it directly
2175 bool is_const_rotate = is_rotate && cnt_type && cnt_type->is_con() &&
2176 Matcher::supports_vector_constant_rotates(cnt_type->get_con());
2177 bool has_scalar_args = is_rotate ? !is_const_rotate : true;
2178
2179 VectorMaskUseType checkFlags = (VectorMaskUseType)(is_masked_op ? (VecMaskUseLoad | VecMaskUsePred) : VecMaskNotUsed);
2180 bool use_predicate = is_masked_op;
2181
2182 if (!arch_supports_vector(sopc, num_elem, elem_bt, checkFlags, has_scalar_args)) {
2183 use_predicate = false;
2184 if (!is_masked_op ||
2185 (!arch_supports_vector(sopc, num_elem, elem_bt, VecMaskNotUsed, has_scalar_args) ||
2186 !arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad))) {
2187
2188 log_if_needed(" ** not supported: arity=0 op=int/%d vlen=%d etype=%s is_masked_op=%d",
2189 sopc, num_elem, type2name(elem_bt), is_masked_op ? 1 : 0);
2190 return false; // not supported
2191 }
2192 }
2193
2194 Node* opd1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
2195 Node* opd2 = nullptr;
2196 if (is_shift) {
2197 opd2 = vector_shift_count(cnt, opc, elem_bt, num_elem);
2198 } else {
2199 assert(is_rotate, "unexpected operation");
2200 if (!is_const_rotate) {
2201 const Type * type_bt = Type::get_const_basic_type(elem_bt);
2202 cnt = elem_bt == T_LONG ? gvn().transform(new ConvI2LNode(cnt)) : cnt;
2203 opd2 = gvn().transform(VectorNode::scalar2vector(cnt, num_elem, type_bt));
2204 } else {
2205 // Constant shift value.
2206 opd2 = cnt;
2207 }
2208 }
2209
2210 if (opd1 == nullptr || opd2 == nullptr) {
2211 return false;
2212 }
2213
2214 Node* mask = nullptr;
2215 if (is_masked_op) {
2216 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
2217 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
2218 mask = unbox_vector(argument(7), mbox_type, elem_bt, num_elem);
2219 if (mask == nullptr) {
2220 log_if_needed(" ** unbox failed mask=%s", NodeClassNames[argument(7)->Opcode()]);
2221 return false;
2222 }
2223 }
2224
2225 Node* operation = VectorNode::make(opc, opd1, opd2, num_elem, elem_bt);
2226 if (is_masked_op && mask != nullptr) {
2227 if (use_predicate) {
2228 operation->add_req(mask);
2229 operation->add_flag(Node::Flag_is_predicated_vector);
2230 } else {
2231 operation = gvn().transform(operation);
2232 operation = new VectorBlendNode(opd1, operation, mask);
2233 }
2234 }
2235 operation = gvn().transform(operation);
2236 Node* vbox = box_vector(operation, vbox_type, elem_bt, num_elem);
2237 set_result(vbox);
2238 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2239 return true;
2240 }
2241
2242 // public static <VOUT extends VectorPayload,
2243 // VIN extends VectorPayload,
2244 // S extends VectorSpecies>
2245 // VOUT convert(int oprId,
2246 // Class<?> fromVectorClass, Class<?> fromElementType, int fromVLen,
2247 // Class<?> toVectorClass, Class<?> toElementType, int toVLen,
2248 // VIN v, S s,
2249 // VectorConvertOp<VOUT, VIN, S> defaultImpl)
2250 //
2251 bool LibraryCallKit::inline_vector_convert() {
2252 const TypeInt* opr = gvn().type(argument(0))->isa_int();
2253
2254 const TypeInstPtr* vector_klass_from = gvn().type(argument(1))->isa_instptr();
2255 const TypeInstPtr* elem_klass_from = gvn().type(argument(2))->isa_instptr();
2256 const TypeInt* vlen_from = gvn().type(argument(3))->isa_int();
2257
2258 const TypeInstPtr* vector_klass_to = gvn().type(argument(4))->isa_instptr();
2259 const TypeInstPtr* elem_klass_to = gvn().type(argument(5))->isa_instptr();
2260 const TypeInt* vlen_to = gvn().type(argument(6))->isa_int();
2261
2262 if (opr == nullptr ||
2263 vector_klass_from == nullptr || elem_klass_from == nullptr || vlen_from == nullptr ||
2264 vector_klass_to == nullptr || elem_klass_to == nullptr || vlen_to == nullptr) {
2265 return false; // dead code
2266 }
2267 if (!opr->is_con() ||
2268 vector_klass_from->const_oop() == nullptr || elem_klass_from->const_oop() == nullptr || !vlen_from->is_con() ||
2269 vector_klass_to->const_oop() == nullptr || elem_klass_to->const_oop() == nullptr || !vlen_to->is_con()) {
2270 log_if_needed(" ** missing constant: opr=%s vclass_from=%s etype_from=%s vlen_from=%s vclass_to=%s etype_to=%s vlen_to=%s",
2271 NodeClassNames[argument(0)->Opcode()],
2272 NodeClassNames[argument(1)->Opcode()],
2273 NodeClassNames[argument(2)->Opcode()],
2274 NodeClassNames[argument(3)->Opcode()],
2275 NodeClassNames[argument(4)->Opcode()],
2276 NodeClassNames[argument(5)->Opcode()],
2277 NodeClassNames[argument(6)->Opcode()]);
2278 return false; // not enough info for intrinsification
2279 }
2280 if (!is_klass_initialized(vector_klass_from) || !is_klass_initialized(vector_klass_to)) {
2281 log_if_needed(" ** klass argument not initialized");
2282 return false;
2283 }
2284
2285 assert(opr->get_con() == VectorSupport::VECTOR_OP_CAST ||
2286 opr->get_con() == VectorSupport::VECTOR_OP_UCAST ||
2287 opr->get_con() == VectorSupport::VECTOR_OP_REINTERPRET, "wrong opcode");
2288 bool is_cast = (opr->get_con() == VectorSupport::VECTOR_OP_CAST || opr->get_con() == VectorSupport::VECTOR_OP_UCAST);
2289 bool is_ucast = (opr->get_con() == VectorSupport::VECTOR_OP_UCAST);
2290
2291 ciKlass* vbox_klass_from = vector_klass_from->const_oop()->as_instance()->java_lang_Class_klass();
2292 ciKlass* vbox_klass_to = vector_klass_to->const_oop()->as_instance()->java_lang_Class_klass();
2293 if (is_vector_shuffle(vbox_klass_from)) {
2294 return false; // vector shuffles aren't supported
2295 }
2296 bool is_mask = is_vector_mask(vbox_klass_from);
2297
2298 ciType* elem_type_from = elem_klass_from->const_oop()->as_instance()->java_mirror_type();
2299 if (!elem_type_from->is_primitive_type()) {
2300 return false; // should be primitive type
2301 }
2302 BasicType elem_bt_from = elem_type_from->basic_type();
2303 ciType* elem_type_to = elem_klass_to->const_oop()->as_instance()->java_mirror_type();
2304 if (!elem_type_to->is_primitive_type()) {
2305 return false; // should be primitive type
2306 }
2307 BasicType elem_bt_to = elem_type_to->basic_type();
2308
2309 int num_elem_from = vlen_from->get_con();
2310 int num_elem_to = vlen_to->get_con();
2311
2312 // Check whether we can unbox to appropriate size. Even with casting, checking for reinterpret is needed
2313 // since we may need to change size.
2314 if (!arch_supports_vector(Op_VectorReinterpret,
2315 num_elem_from,
2316 elem_bt_from,
2317 is_mask ? VecMaskUseAll : VecMaskNotUsed)) {
2318 log_if_needed(" ** not supported: arity=1 op=%s/1 vlen1=%d etype1=%s ismask=%d",
2319 is_cast ? "cast" : "reinterpret",
2320 num_elem_from, type2name(elem_bt_from), is_mask);
2321 return false;
2322 }
2323
2324 // Check whether we can support resizing/reinterpreting to the new size.
2325 if (!arch_supports_vector(Op_VectorReinterpret,
2326 num_elem_to,
2327 elem_bt_to,
2328 is_mask ? VecMaskUseAll : VecMaskNotUsed)) {
2329 log_if_needed(" ** not supported: arity=1 op=%s/2 vlen2=%d etype2=%s ismask=%d",
2330 is_cast ? "cast" : "reinterpret",
2331 num_elem_to, type2name(elem_bt_to), is_mask);
2332 return false;
2333 }
2334
2335
2336 if (is_vector_shuffle(vbox_klass_to) &&
2337 (!arch_supports_vector(Op_SubVB, num_elem_to, elem_bt_to, VecMaskNotUsed) ||
2338 !arch_supports_vector(Op_VectorBlend, num_elem_to, elem_bt_to, VecMaskNotUsed) ||
2339 !arch_supports_vector(Op_VectorMaskCmp, num_elem_to, elem_bt_to, VecMaskNotUsed) ||
2340 !arch_supports_vector(Op_AndV, num_elem_to, elem_bt_to, VecMaskNotUsed) ||
2341 !arch_supports_vector(Op_Replicate, num_elem_to, elem_bt_to, VecMaskNotUsed))) {
2342 log_if_needed(" ** not supported: arity=1 op=shuffle_index_wrap vlen2=%d etype2=%s",
2343 num_elem_to, type2name(elem_bt_to));
2344 return false;
2345 }
2346
2347 // At this point, we know that both input and output vector registers are supported
2348 // by the architecture. Next check if the casted type is simply to same type - which means
2349 // that it is actually a resize and not a cast.
2350 if (is_cast && elem_bt_from == elem_bt_to) {
2351 is_cast = false;
2352 }
2353
2354 const TypeInstPtr* vbox_type_from = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass_from);
2355
2356 Node* opd1 = unbox_vector(argument(7), vbox_type_from, elem_bt_from, num_elem_from);
2357 if (opd1 == nullptr) {
2358 return false;
2359 }
2360
2361 const TypeVect* src_type = TypeVect::make(elem_bt_from, num_elem_from, is_mask);
2362 const TypeVect* dst_type = TypeVect::make(elem_bt_to, num_elem_to, is_mask);
2363
2364 // Safety check to prevent casting if source mask is of type vector
2365 // and destination mask of type predicate vector and vice-versa.
2366 // From X86 standpoint, this case will only arise over KNL target,
2367 // where certain masks (depending on the species) are either propagated
2368 // through a vector or predicate register.
2369 if (is_mask &&
2370 ((src_type->isa_vectmask() == nullptr && dst_type->isa_vectmask()) ||
2371 (dst_type->isa_vectmask() == nullptr && src_type->isa_vectmask()))) {
2372 return false;
2373 }
2374
2375 Node* op = opd1;
2376 if (is_cast) {
2377 assert(!is_mask || num_elem_from == num_elem_to, "vector mask cast needs the same elem num");
2378 int cast_vopc = VectorCastNode::opcode(-1, elem_bt_from, !is_ucast);
2379
2380 // Make sure that vector cast is implemented to particular type/size combination if it is
2381 // not a mask casting.
2382 if (!is_mask && !arch_supports_vector(cast_vopc, num_elem_to, elem_bt_to, VecMaskNotUsed)) {
2383 log_if_needed(" ** not supported: arity=1 op=cast#%d/3 vlen2=%d etype2=%s ismask=%d",
2384 cast_vopc, num_elem_to, type2name(elem_bt_to), is_mask);
2385 return false;
2386 }
2387
2388 if (num_elem_from < num_elem_to) {
2389 // Since input and output number of elements are not consistent, we need to make sure we
2390 // properly size. Thus, first make a cast that retains the number of elements from source.
2391 int num_elem_for_cast = num_elem_from;
2392
2393 // It is possible that arch does not support this intermediate vector size
2394 // TODO More complex logic required here to handle this corner case for the sizes.
2395 if (!arch_supports_vector(cast_vopc, num_elem_for_cast, elem_bt_to, VecMaskNotUsed)) {
2396 log_if_needed(" ** not supported: arity=1 op=cast#%d/4 vlen1=%d etype2=%s ismask=%d",
2397 cast_vopc,
2398 num_elem_for_cast, type2name(elem_bt_to), is_mask);
2399 return false;
2400 }
2401
2402 op = gvn().transform(VectorCastNode::make(cast_vopc, op, elem_bt_to, num_elem_for_cast));
2403 // Now ensure that the destination gets properly resized to needed size.
2404 op = gvn().transform(new VectorReinterpretNode(op, op->bottom_type()->is_vect(), dst_type));
2405 } else if (num_elem_from > num_elem_to) {
2406 // Since number of elements from input is larger than output, simply reduce size of input
2407 // (we are supposed to drop top elements anyway).
2408 int num_elem_for_resize = num_elem_to;
2409
2410 // It is possible that arch does not support this intermediate vector size
2411 // TODO More complex logic required here to handle this corner case for the sizes.
2412 if (!arch_supports_vector(Op_VectorReinterpret,
2413 num_elem_for_resize,
2414 elem_bt_from,
2415 VecMaskNotUsed)) {
2416 log_if_needed(" ** not supported: arity=1 op=cast/5 vlen2=%d etype1=%s ismask=%d",
2417 num_elem_for_resize, type2name(elem_bt_from), is_mask);
2418 return false;
2419 }
2420
2421 const TypeVect* resize_type = TypeVect::make(elem_bt_from, num_elem_for_resize);
2422 op = gvn().transform(new VectorReinterpretNode(op, src_type, resize_type));
2423 op = gvn().transform(VectorCastNode::make(cast_vopc, op, elem_bt_to, num_elem_to));
2424 } else { // num_elem_from == num_elem_to
2425 if (is_mask) {
2426 // Make sure that cast for vector mask is implemented to particular type/size combination.
2427 if (!arch_supports_vector(Op_VectorMaskCast, num_elem_to, elem_bt_to, VecMaskNotUsed)) {
2428 log_if_needed(" ** not supported: arity=1 op=maskcast vlen2=%d etype2=%s ismask=%d",
2429 num_elem_to, type2name(elem_bt_to), is_mask);
2430 return false;
2431 }
2432 op = gvn().transform(new VectorMaskCastNode(op, dst_type));
2433 } else {
2434 // Since input and output number of elements match, and since we know this vector size is
2435 // supported, simply do a cast with no resize needed.
2436 op = gvn().transform(VectorCastNode::make(cast_vopc, op, elem_bt_to, num_elem_to));
2437 }
2438 }
2439 } else if (!Type::equals(src_type, dst_type)) {
2440 assert(!is_cast, "must be reinterpret");
2441 op = gvn().transform(new VectorReinterpretNode(op, src_type, dst_type));
2442 }
2443
2444 if (is_vector_shuffle(vbox_klass_to)) {
2445 op = partially_wrap_indexes(op, num_elem_to, elem_bt_to);
2446 }
2447
2448 const TypeInstPtr* vbox_type_to = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass_to);
2449 Node* vbox = box_vector(op, vbox_type_to, elem_bt_to, num_elem_to);
2450 set_result(vbox);
2451 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem_to * type2aelembytes(elem_bt_to))));
2452 return true;
2453 }
2454
2455 // public static
2456 // <V extends Vector<E>,
2457 // E>
2458 // V insert(Class<? extends V> vectorClass, Class<E> elementType, int vlen,
2459 // V vec, int ix, long val,
2460 // VecInsertOp<V> defaultImpl)
2461 bool LibraryCallKit::inline_vector_insert() {
2462 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
2463 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
2464 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
2465 const TypeInt* idx = gvn().type(argument(4))->isa_int();
2466
2467 if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr || idx == nullptr) {
2468 return false; // dead code
2469 }
2470 if (vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con() || !idx->is_con()) {
2471 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s idx=%s",
2472 NodeClassNames[argument(0)->Opcode()],
2473 NodeClassNames[argument(1)->Opcode()],
2474 NodeClassNames[argument(2)->Opcode()],
2475 NodeClassNames[argument(4)->Opcode()]);
2476 return false; // not enough info for intrinsification
2477 }
2478 if (!is_klass_initialized(vector_klass)) {
2479 log_if_needed(" ** klass argument not initialized");
2480 return false;
2481 }
2482 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2483 if (!elem_type->is_primitive_type()) {
2484 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2485 return false; // should be primitive type
2486 }
2487 BasicType elem_bt = elem_type->basic_type();
2488 int num_elem = vlen->get_con();
2489 if (!arch_supports_vector(Op_VectorInsert, num_elem, elem_bt, VecMaskNotUsed)) {
2490 log_if_needed(" ** not supported: arity=1 op=insert vlen=%d etype=%s ismask=no",
2491 num_elem, type2name(elem_bt));
2492 return false; // not supported
2493 }
2494
2495 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2496 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2497
2498 Node* opd = unbox_vector(argument(3), vbox_type, elem_bt, num_elem);
2499 if (opd == nullptr) {
2500 return false;
2501 }
2502
2503 Node* insert_val = argument(5);
2504 assert(gvn().type(insert_val)->isa_long() != nullptr, "expected to be long");
2505
2506 // Convert insert value back to its appropriate type.
2507 switch (elem_bt) {
2508 case T_BYTE:
2509 insert_val = gvn().transform(new ConvL2INode(insert_val));
2510 insert_val = gvn().transform(new CastIINode(insert_val, TypeInt::BYTE));
2511 break;
2512 case T_SHORT:
2513 insert_val = gvn().transform(new ConvL2INode(insert_val));
2514 insert_val = gvn().transform(new CastIINode(insert_val, TypeInt::SHORT));
2515 break;
2516 case T_INT:
2517 insert_val = gvn().transform(new ConvL2INode(insert_val));
2518 break;
2519 case T_FLOAT:
2520 insert_val = gvn().transform(new ConvL2INode(insert_val));
2521 insert_val = gvn().transform(new MoveI2FNode(insert_val));
2522 break;
2523 case T_DOUBLE:
2524 insert_val = gvn().transform(new MoveL2DNode(insert_val));
2525 break;
2526 case T_LONG:
2527 // no conversion needed
2528 break;
2529 default: fatal("%s", type2name(elem_bt)); break;
2530 }
2531
2532 Node* operation = gvn().transform(VectorInsertNode::make(opd, insert_val, idx->get_con(), gvn()));
2533
2534 Node* vbox = box_vector(operation, vbox_type, elem_bt, num_elem);
2535 set_result(vbox);
2536 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2537 return true;
2538 }
2539
2540 // public static
2541 // <VM extends VectorPayload,
2542 // E>
2543 // long extract(Class<? extends VM> vClass, Class<E> eClass,
2544 // int length,
2545 // VM vm, int i,
2546 // VecExtractOp<VM> defaultImpl)
2547 bool LibraryCallKit::inline_vector_extract() {
2548 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
2549 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
2550 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
2551 const TypeInt* idx = gvn().type(argument(4))->isa_int();
2552
2553 if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr || idx == nullptr) {
2554 return false; // dead code
2555 }
2556 if (vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con() || !idx->is_con()) {
2557 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s",
2558 NodeClassNames[argument(0)->Opcode()],
2559 NodeClassNames[argument(1)->Opcode()],
2560 NodeClassNames[argument(2)->Opcode()]);
2561 return false; // not enough info for intrinsification
2562 }
2563 if (!is_klass_initialized(vector_klass)) {
2564 log_if_needed(" ** klass argument not initialized");
2565 return false;
2566 }
2567 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2568 if (!elem_type->is_primitive_type()) {
2569 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2570 return false; // should be primitive type
2571 }
2572 BasicType elem_bt = elem_type->basic_type();
2573 int num_elem = vlen->get_con();
2574
2575 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2576 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2577
2578 Node* opd = nullptr;
2579
2580 if (is_vector_mask(vbox_klass)) {
2581 // vbox_klass is mask. This is used for VectorMask.laneIsSet(int).
2582
2583 Node* pos = argument(4); // can be variable
2584 if (arch_supports_vector(Op_ExtractUB, num_elem, elem_bt, VecMaskUseAll)) {
2585 // Transform mask to vector with type of boolean and utilize ExtractUB node.
2586 opd = unbox_vector(argument(3), vbox_type, elem_bt, num_elem);
2587 if (opd == nullptr) {
2588 return false;
2589 }
2590 opd = gvn().transform(VectorStoreMaskNode::make(gvn(), opd, elem_bt, num_elem));
2591 opd = gvn().transform(new ExtractUBNode(opd, pos));
2592 opd = gvn().transform(new ConvI2LNode(opd));
2593 } else if (arch_supports_vector(Op_VectorMaskToLong, num_elem, elem_bt, VecMaskUseLoad)) {
2594 opd = unbox_vector(argument(3), vbox_type, elem_bt, num_elem);
2595 if (opd == nullptr) {
2596 return false;
2597 }
2598 // VectorMaskToLongNode requires the input is either a mask or a vector with BOOLEAN type.
2599 if (opd->bottom_type()->isa_vectmask() == nullptr) {
2600 opd = gvn().transform(VectorStoreMaskNode::make(gvn(), opd, elem_bt, num_elem));
2601 }
2602 // ((toLong() >>> pos) & 1L
2603 opd = gvn().transform(new VectorMaskToLongNode(opd, TypeLong::LONG));
2604 opd = gvn().transform(new URShiftLNode(opd, pos));
2605 opd = gvn().transform(new AndLNode(opd, gvn().makecon(TypeLong::ONE)));
2606 } else {
2607 log_if_needed(" ** Rejected mask extraction because architecture does not support it");
2608 return false; // not supported
2609 }
2610 } else {
2611 // vbox_klass is vector. This is used for Vector.lane(int).
2612 if (!idx->is_con()) {
2613 log_if_needed(" ** missing constant: idx=%s", NodeClassNames[argument(4)->Opcode()]);
2614 return false; // not enough info for intrinsification
2615 }
2616
2617 int vopc = ExtractNode::opcode(elem_bt);
2618 if (!arch_supports_vector(vopc, num_elem, elem_bt, VecMaskNotUsed)) {
2619 log_if_needed(" ** not supported: arity=1 op=extract vlen=%d etype=%s ismask=no",
2620 num_elem, type2name(elem_bt));
2621 return false; // not supported
2622 }
2623
2624 opd = unbox_vector(argument(3), vbox_type, elem_bt, num_elem);
2625 if (opd == nullptr) {
2626 return false;
2627 }
2628 ConINode* idx_con = gvn().intcon(idx->get_con())->as_ConI();
2629
2630 opd = gvn().transform(ExtractNode::make(opd, idx_con, elem_bt));
2631 switch (elem_bt) {
2632 case T_BYTE:
2633 case T_SHORT:
2634 case T_INT: {
2635 opd = gvn().transform(new ConvI2LNode(opd));
2636 break;
2637 }
2638 case T_FLOAT: {
2639 opd = gvn().transform(new MoveF2INode(opd));
2640 opd = gvn().transform(new ConvI2LNode(opd));
2641 break;
2642 }
2643 case T_DOUBLE: {
2644 opd = gvn().transform(new MoveD2LNode(opd));
2645 break;
2646 }
2647 case T_LONG: {
2648 // no conversion needed
2649 break;
2650 }
2651 default: fatal("%s", type2name(elem_bt));
2652 }
2653 }
2654 set_result(opd);
2655 return true;
2656 }
2657
2658 // public static
2659 // <V extends Vector<E>,
2660 // M extends VectorMask<E>,
2661 // E>
2662 // V compressExpandOp(int opr,
2663 // Class<? extends V> vClass, Class<? extends M> mClass, Class<E> eClass,
2664 // int length, V v, M m,
2665 // CompressExpandOperation<V, M> defaultImpl)
2666 bool LibraryCallKit::inline_vector_compress_expand() {
2667 const TypeInt* opr = gvn().type(argument(0))->isa_int();
2668 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
2669 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
2670 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
2671 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
2672
2673 if (opr == nullptr || !opr->is_con() ||
2674 vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
2675 mask_klass == nullptr || mask_klass->const_oop() == nullptr ||
2676 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
2677 vlen == nullptr || !vlen->is_con()) {
2678 log_if_needed(" ** missing constant: opr=%s vclass=%s mclass=%s etype=%s vlen=%s",
2679 NodeClassNames[argument(0)->Opcode()],
2680 NodeClassNames[argument(1)->Opcode()],
2681 NodeClassNames[argument(2)->Opcode()],
2682 NodeClassNames[argument(3)->Opcode()],
2683 NodeClassNames[argument(4)->Opcode()]);
2684 return false; // not enough info for intrinsification
2685 }
2686
2687 if (!is_klass_initialized(vector_klass) || !is_klass_initialized(mask_klass)) {
2688 log_if_needed(" ** klass argument not initialized");
2689 return false;
2690 }
2691
2692 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2693 if (!elem_type->is_primitive_type()) {
2694 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2695 return false; // should be primitive type
2696 }
2697
2698 int num_elem = vlen->get_con();
2699 BasicType elem_bt = elem_type->basic_type();
2700 int opc = VectorSupport::vop2ideal(opr->get_con(), elem_bt);
2701
2702 if (!arch_supports_vector(opc, num_elem, elem_bt, VecMaskUseLoad)) {
2703 log_if_needed(" ** not supported: opc=%d vlen=%d etype=%s ismask=useload",
2704 opc, num_elem, type2name(elem_bt));
2705 return false; // not supported
2706 }
2707
2708 Node* opd1 = nullptr;
2709 const TypeInstPtr* vbox_type = nullptr;
2710 if (opc != Op_CompressM) {
2711 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2712 vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2713 opd1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
2714 if (opd1 == nullptr) {
2715 log_if_needed(" ** unbox failed vector=%s",
2716 NodeClassNames[argument(5)->Opcode()]);
2717 return false;
2718 }
2719 }
2720
2721 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
2722 assert(is_vector_mask(mbox_klass), "argument(6) should be a mask class");
2723 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
2724
2725 Node* mask = unbox_vector(argument(6), mbox_type, elem_bt, num_elem);
2726 if (mask == nullptr) {
2727 log_if_needed(" ** unbox failed mask=%s",
2728 NodeClassNames[argument(6)->Opcode()]);
2729 return false;
2730 }
2731
2732 const TypeVect* vt = TypeVect::make(elem_bt, num_elem, opc == Op_CompressM);
2733 Node* operation = gvn().transform(VectorNode::make(opc, opd1, mask, vt));
2734
2735 // Wrap it up in VectorBox to keep object type information.
2736 const TypeInstPtr* box_type = opc == Op_CompressM ? mbox_type : vbox_type;
2737 Node* vbox = box_vector(operation, box_type, elem_bt, num_elem);
2738 set_result(vbox);
2739 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2740 return true;
2741 }
2742
2743 // public static
2744 // <V extends Vector<E>,
2745 // E,
2746 // S extends VectorSpecies<E>>
2747 // V indexVector(Class<? extends V> vClass, Class<E> eClass,
2748 // int length,
2749 // V v, int step, S s,
2750 // IndexOperation<V, S> defaultImpl)
2751 bool LibraryCallKit::inline_index_vector() {
2752 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
2753 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
2754 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
2755
2756 if (vector_klass == nullptr || vector_klass->const_oop() == nullptr ||
2757 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
2758 vlen == nullptr || !vlen->is_con() ) {
2759 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s",
2760 NodeClassNames[argument(0)->Opcode()],
2761 NodeClassNames[argument(1)->Opcode()],
2762 NodeClassNames[argument(2)->Opcode()]);
2763 return false; // not enough info for intrinsification
2764 }
2765
2766 if (!is_klass_initialized(vector_klass)) {
2767 log_if_needed(" ** klass argument not initialized");
2768 return false;
2769 }
2770
2771 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2772 if (!elem_type->is_primitive_type()) {
2773 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2774 return false; // should be primitive type
2775 }
2776
2777 int num_elem = vlen->get_con();
2778 BasicType elem_bt = elem_type->basic_type();
2779
2780 // Check whether the iota index generation op is supported by the current hardware
2781 if (!arch_supports_vector(Op_VectorLoadConst, num_elem, elem_bt, VecMaskNotUsed)) {
2782 log_if_needed(" ** not supported: vlen=%d etype=%s", num_elem, type2name(elem_bt));
2783 return false; // not supported
2784 }
2785
2786 int mul_op = VectorSupport::vop2ideal(VectorSupport::VECTOR_OP_MUL, elem_bt);
2787 int vmul_op = VectorNode::opcode(mul_op, elem_bt);
2788 bool needs_mul = true;
2789 Node* scale = argument(4);
2790 const TypeInt* scale_type = gvn().type(scale)->isa_int();
2791 // Multiply is not needed if the scale is a constant "1".
2792 if (scale_type && scale_type->is_con() && scale_type->get_con() == 1) {
2793 needs_mul = false;
2794 } else {
2795 // Check whether the vector multiply op is supported by the current hardware
2796 if (!arch_supports_vector(vmul_op, num_elem, elem_bt, VecMaskNotUsed)) {
2797 log_if_needed(" ** not supported: vlen=%d etype=%s", num_elem, type2name(elem_bt));
2798 return false; // not supported
2799 }
2800
2801 // Check whether the scalar cast op is supported by the current hardware
2802 if (is_floating_point_type(elem_bt) || elem_bt == T_LONG) {
2803 int cast_op = elem_bt == T_LONG ? Op_ConvI2L :
2804 elem_bt == T_FLOAT? Op_ConvI2F : Op_ConvI2D;
2805 if (!Matcher::match_rule_supported(cast_op)) {
2806 log_if_needed(" ** Rejected op (%s) because architecture does not support it",
2807 NodeClassNames[cast_op]);
2808 return false; // not supported
2809 }
2810 }
2811 }
2812
2813 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2814 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2815 Node* opd = unbox_vector(argument(3), vbox_type, elem_bt, num_elem);
2816 if (opd == nullptr) {
2817 log_if_needed(" ** unbox failed vector=%s",
2818 NodeClassNames[argument(3)->Opcode()]);
2819 return false;
2820 }
2821
2822 int add_op = VectorSupport::vop2ideal(VectorSupport::VECTOR_OP_ADD, elem_bt);
2823 int vadd_op = VectorNode::opcode(add_op, elem_bt);
2824 bool needs_add = true;
2825 // The addition is not needed if all the element values of "opd" are zero
2826 if (VectorNode::is_all_zeros_vector(opd)) {
2827 needs_add = false;
2828 } else {
2829 // Check whether the vector addition op is supported by the current hardware
2830 if (!arch_supports_vector(vadd_op, num_elem, elem_bt, VecMaskNotUsed)) {
2831 log_if_needed(" ** not supported: vlen=%d etype=%s", num_elem, type2name(elem_bt));
2832 return false; // not supported
2833 }
2834 }
2835
2836 // Compute the iota indice vector
2837 const TypeVect* vt = TypeVect::make(elem_bt, num_elem);
2838 Node* index = gvn().transform(new VectorLoadConstNode(gvn().makecon(TypeInt::ZERO), vt));
2839
2840 // Broadcast the "scale" to a vector, and multiply the "scale" with iota indice vector.
2841 if (needs_mul) {
2842 switch (elem_bt) {
2843 case T_BOOLEAN: // fall-through
2844 case T_BYTE: // fall-through
2845 case T_SHORT: // fall-through
2846 case T_CHAR: // fall-through
2847 case T_INT: {
2848 // no conversion needed
2849 break;
2850 }
2851 case T_LONG: {
2852 scale = gvn().transform(new ConvI2LNode(scale));
2853 break;
2854 }
2855 case T_FLOAT: {
2856 scale = gvn().transform(new ConvI2FNode(scale));
2857 break;
2858 }
2859 case T_DOUBLE: {
2860 scale = gvn().transform(new ConvI2DNode(scale));
2861 break;
2862 }
2863 default: fatal("%s", type2name(elem_bt));
2864 }
2865 scale = gvn().transform(VectorNode::scalar2vector(scale, num_elem, Type::get_const_basic_type(elem_bt)));
2866 index = gvn().transform(VectorNode::make(vmul_op, index, scale, vt));
2867 }
2868
2869 // Add "opd" if addition is needed.
2870 if (needs_add) {
2871 index = gvn().transform(VectorNode::make(vadd_op, opd, index, vt));
2872 }
2873 Node* vbox = box_vector(index, vbox_type, elem_bt, num_elem);
2874 set_result(vbox);
2875 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2876 return true;
2877 }
2878
2879 // public static
2880 // <E,
2881 // M extends VectorMask<E>>
2882 // M indexPartiallyInUpperRange(Class<? extends M> mClass, Class<E> eClass, int length,
2883 // long offset, long limit,
2884 // IndexPartiallyInUpperRangeOperation<E, M> defaultImpl)
2885 bool LibraryCallKit::inline_index_partially_in_upper_range() {
2886 const TypeInstPtr* mask_klass = gvn().type(argument(0))->isa_instptr();
2887 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
2888 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
2889
2890 if (mask_klass == nullptr || mask_klass->const_oop() == nullptr ||
2891 elem_klass == nullptr || elem_klass->const_oop() == nullptr ||
2892 vlen == nullptr || !vlen->is_con()) {
2893 log_if_needed(" ** missing constant: mclass=%s etype=%s vlen=%s",
2894 NodeClassNames[argument(0)->Opcode()],
2895 NodeClassNames[argument(1)->Opcode()],
2896 NodeClassNames[argument(2)->Opcode()]);
2897 return false; // not enough info for intrinsification
2898 }
2899
2900 if (!is_klass_initialized(mask_klass)) {
2901 log_if_needed(" ** klass argument not initialized");
2902 return false;
2903 }
2904
2905 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2906 if (!elem_type->is_primitive_type()) {
2907 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2908 return false; // should be primitive type
2909 }
2910
2911 int num_elem = vlen->get_con();
2912 BasicType elem_bt = elem_type->basic_type();
2913
2914 // Check whether the necessary ops are supported by current hardware.
2915 bool supports_mask_gen = arch_supports_vector(Op_VectorMaskGen, num_elem, elem_bt, VecMaskUseStore);
2916 if (!supports_mask_gen) {
2917 if (!arch_supports_vector(Op_VectorLoadConst, num_elem, elem_bt, VecMaskNotUsed) ||
2918 !arch_supports_vector(Op_Replicate, num_elem, elem_bt, VecMaskNotUsed) ||
2919 !arch_supports_vector(Op_VectorMaskCmp, num_elem, elem_bt, VecMaskUseStore)) {
2920 log_if_needed(" ** not supported: vlen=%d etype=%s", num_elem, type2name(elem_bt));
2921 return false; // not supported
2922 }
2923
2924 // Check whether the scalar cast operation is supported by current hardware.
2925 if (elem_bt != T_LONG) {
2926 int cast_op = is_integral_type(elem_bt) ? Op_ConvL2I
2927 : (elem_bt == T_FLOAT ? Op_ConvL2F : Op_ConvL2D);
2928 if (!Matcher::match_rule_supported(cast_op)) {
2929 log_if_needed(" ** Rejected op (%s) because architecture does not support it",
2930 NodeClassNames[cast_op]);
2931 return false; // not supported
2932 }
2933 }
2934 }
2935
2936 Node* offset = argument(3);
2937 Node* limit = argument(5);
2938 if (offset == nullptr || limit == nullptr) {
2939 log_if_needed(" ** offset or limit argument is null");
2940 return false; // not supported
2941 }
2942
2943 ciKlass* box_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
2944 assert(is_vector_mask(box_klass), "argument(0) should be a mask class");
2945 const TypeInstPtr* box_type = TypeInstPtr::make_exact(TypePtr::NotNull, box_klass);
2946
2947 // We assume "offset > 0 && limit >= offset && limit - offset < num_elem".
2948 // So directly get indexLimit with "indexLimit = limit - offset".
2949 Node* indexLimit = gvn().transform(new SubLNode(limit, offset));
2950 Node* mask = nullptr;
2951 if (supports_mask_gen) {
2952 mask = gvn().transform(VectorMaskGenNode::make(indexLimit, elem_bt, num_elem));
2953 } else {
2954 // Generate the vector mask based on "mask = iota < indexLimit".
2955 // Broadcast "indexLimit" to a vector.
2956 switch (elem_bt) {
2957 case T_BOOLEAN: // fall-through
2958 case T_BYTE: // fall-through
2959 case T_SHORT: // fall-through
2960 case T_CHAR: // fall-through
2961 case T_INT: {
2962 indexLimit = gvn().transform(new ConvL2INode(indexLimit));
2963 break;
2964 }
2965 case T_DOUBLE: {
2966 indexLimit = gvn().transform(new ConvL2DNode(indexLimit));
2967 break;
2968 }
2969 case T_FLOAT: {
2970 indexLimit = gvn().transform(new ConvL2FNode(indexLimit));
2971 break;
2972 }
2973 case T_LONG: {
2974 // no conversion needed
2975 break;
2976 }
2977 default: fatal("%s", type2name(elem_bt));
2978 }
2979 indexLimit = gvn().transform(VectorNode::scalar2vector(indexLimit, num_elem, Type::get_const_basic_type(elem_bt)));
2980
2981 // Load the "iota" vector.
2982 const TypeVect* vt = TypeVect::make(elem_bt, num_elem);
2983 Node* iota = gvn().transform(new VectorLoadConstNode(gvn().makecon(TypeInt::ZERO), vt));
2984
2985 // Compute the vector mask with "mask = iota < indexLimit".
2986 ConINode* pred_node = (ConINode*)gvn().makecon(TypeInt::make(BoolTest::lt));
2987 const TypeVect* vmask_type = TypeVect::makemask(elem_bt, num_elem);
2988 mask = gvn().transform(new VectorMaskCmpNode(BoolTest::lt, iota, indexLimit, pred_node, vmask_type));
2989 }
2990 Node* vbox = box_vector(mask, box_type, elem_bt, num_elem);
2991 set_result(vbox);
2992 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2993 return true;
2994 }
2995
2996 #undef non_product_log_if_needed
2997 #undef log_if_needed