1 /*
2 * Copyright (c) 2020, 2024, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "ci/ciSymbols.hpp"
27 #include "classfile/vmSymbols.hpp"
28 #include "opto/library_call.hpp"
29 #include "opto/runtime.hpp"
30 #include "opto/vectornode.hpp"
31 #include "prims/vectorSupport.hpp"
32 #include "runtime/stubRoutines.hpp"
33
34 #ifdef ASSERT
35 static bool is_vector(ciKlass* klass) {
36 return klass->is_subclass_of(ciEnv::current()->vector_VectorPayload_klass());
37 }
38
39 static bool check_vbox(const TypeInstPtr* vbox_type) {
40 assert(vbox_type->klass_is_exact(), "");
41
42 ciInstanceKlass* ik = vbox_type->instance_klass();
43 assert(is_vector(ik), "not a vector");
44
45 ciField* fd1 = ik->get_field_by_name(ciSymbols::ETYPE_name(), ciSymbols::class_signature(), /* is_static */ true);
46 assert(fd1 != nullptr, "element type info is missing");
47
48 ciConstant val1 = fd1->constant_value();
49 BasicType elem_bt = val1.as_object()->as_instance()->java_mirror_type()->basic_type();
50 assert(is_java_primitive(elem_bt), "element type info is missing");
51
52 ciField* fd2 = ik->get_field_by_name(ciSymbols::VLENGTH_name(), ciSymbols::int_signature(), /* is_static */ true);
53 assert(fd2 != nullptr, "vector length info is missing");
54
55 ciConstant val2 = fd2->constant_value();
56 assert(val2.as_int() > 0, "vector length info is missing");
57
58 return true;
59 }
60 #endif
61
62 #define log_if_needed(...) \
63 if (C->print_intrinsics()) { \
64 tty->print_cr(__VA_ARGS__); \
65 }
66
67 #ifndef PRODUCT
68 #define non_product_log_if_needed(...) log_if_needed(__VA_ARGS__)
69 #else
70 #define non_product_log_if_needed(...)
71 #endif
72
73 static bool is_vector_mask(ciKlass* klass) {
74 return klass->is_subclass_of(ciEnv::current()->vector_VectorMask_klass());
75 }
76
77 static bool is_vector_shuffle(ciKlass* klass) {
78 return klass->is_subclass_of(ciEnv::current()->vector_VectorShuffle_klass());
79 }
80
81 bool LibraryCallKit::arch_supports_vector_rotate(int opc, int num_elem, BasicType elem_bt,
82 VectorMaskUseType mask_use_type, bool has_scalar_args) {
83 bool is_supported = true;
84
85 // has_scalar_args flag is true only for non-constant scalar shift count,
86 // since in this case shift needs to be broadcasted.
87 if (!Matcher::match_rule_supported_vector(opc, num_elem, elem_bt) ||
88 (has_scalar_args && !arch_supports_vector(Op_Replicate, num_elem, elem_bt, VecMaskNotUsed))) {
89 is_supported = false;
90 }
91
92 if (is_supported) {
93 // Check if mask unboxing is supported, this is a two step process which first loads the contents
94 // of boolean array into vector followed by either lane expansion to match the lane size of masked
95 // vector operation or populate the predicate register.
96 if ((mask_use_type & VecMaskUseLoad) != 0) {
97 if (!Matcher::match_rule_supported_vector(Op_VectorLoadMask, num_elem, elem_bt) ||
98 !Matcher::match_rule_supported_vector(Op_LoadVector, num_elem, T_BOOLEAN)) {
99 non_product_log_if_needed(" ** Rejected vector mask loading (%s,%s,%d) because architecture does not support it",
100 NodeClassNames[Op_VectorLoadMask], type2name(elem_bt), num_elem);
101 return false;
102 }
103 }
104
105 if ((mask_use_type & VecMaskUsePred) != 0) {
106 if (!Matcher::has_predicated_vectors() ||
107 !Matcher::match_rule_supported_vector_masked(opc, num_elem, elem_bt)) {
108 non_product_log_if_needed("Rejected vector mask predicate using (%s,%s,%d) because architecture does not support it",
109 NodeClassNames[opc], type2name(elem_bt), num_elem);
110 return false;
111 }
112 }
113 }
114
115 int lshiftopc, rshiftopc;
116 switch(elem_bt) {
117 case T_BYTE:
118 lshiftopc = Op_LShiftI;
119 rshiftopc = Op_URShiftB;
120 break;
121 case T_SHORT:
122 lshiftopc = Op_LShiftI;
123 rshiftopc = Op_URShiftS;
124 break;
125 case T_INT:
126 lshiftopc = Op_LShiftI;
127 rshiftopc = Op_URShiftI;
128 break;
129 case T_LONG:
130 lshiftopc = Op_LShiftL;
131 rshiftopc = Op_URShiftL;
132 break;
133 default: fatal("Unexpected type: %s", type2name(elem_bt));
134 }
135 int lshiftvopc = VectorNode::opcode(lshiftopc, elem_bt);
136 int rshiftvopc = VectorNode::opcode(rshiftopc, elem_bt);
137 if (!is_supported &&
138 arch_supports_vector(lshiftvopc, num_elem, elem_bt, VecMaskNotUsed, has_scalar_args) &&
139 arch_supports_vector(rshiftvopc, num_elem, elem_bt, VecMaskNotUsed, has_scalar_args) &&
140 arch_supports_vector(Op_OrV, num_elem, elem_bt, VecMaskNotUsed)) {
141 is_supported = true;
142 }
143 return is_supported;
144 }
145
146 Node* GraphKit::box_vector(Node* vector, const TypeInstPtr* vbox_type, BasicType elem_bt, int num_elem, bool deoptimize_on_exception) {
147 assert(EnableVectorSupport, "");
148
149 PreserveReexecuteState preexecs(this);
150 jvms()->set_should_reexecute(true);
151
152 VectorBoxAllocateNode* alloc = new VectorBoxAllocateNode(C, vbox_type);
153 set_edges_for_java_call(alloc, /*must_throw=*/false, /*separate_io_proj=*/true);
154 make_slow_call_ex(alloc, env()->Throwable_klass(), /*separate_io_proj=*/true, deoptimize_on_exception);
155 set_i_o(gvn().transform( new ProjNode(alloc, TypeFunc::I_O) ));
156 set_all_memory(gvn().transform( new ProjNode(alloc, TypeFunc::Memory) ));
157 Node* ret = gvn().transform(new ProjNode(alloc, TypeFunc::Parms));
158
159 assert(check_vbox(vbox_type), "");
160 const TypeVect* vt = TypeVect::make(elem_bt, num_elem, is_vector_mask(vbox_type->instance_klass()));
161 VectorBoxNode* vbox = new VectorBoxNode(C, ret, vector, vbox_type, vt);
162 return gvn().transform(vbox);
163 }
164
165 Node* GraphKit::unbox_vector(Node* v, const TypeInstPtr* vbox_type, BasicType elem_bt, int num_elem, bool shuffle_to_vector) {
166 assert(EnableVectorSupport, "");
167 const TypeInstPtr* vbox_type_v = gvn().type(v)->isa_instptr();
168 if (vbox_type_v == nullptr || vbox_type->instance_klass() != vbox_type_v->instance_klass()) {
169 return nullptr; // arguments don't agree on vector shapes
170 }
171 if (vbox_type_v->maybe_null()) {
172 return nullptr; // no nulls are allowed
173 }
174 assert(check_vbox(vbox_type), "");
175 const TypeVect* vt = TypeVect::make(elem_bt, num_elem, is_vector_mask(vbox_type->instance_klass()));
176 Node* unbox = gvn().transform(new VectorUnboxNode(C, vt, v, merged_memory(), shuffle_to_vector));
177 return unbox;
178 }
179
180 Node* GraphKit::vector_shift_count(Node* cnt, int shift_op, BasicType bt, int num_elem) {
181 assert(bt == T_INT || bt == T_LONG || bt == T_SHORT || bt == T_BYTE, "byte, short, long and int are supported");
182 juint mask = (type2aelembytes(bt) * BitsPerByte - 1);
183 Node* nmask = gvn().transform(ConNode::make(TypeInt::make(mask)));
184 Node* mcnt = gvn().transform(new AndINode(cnt, nmask));
185 return gvn().transform(VectorNode::shift_count(shift_op, mcnt, num_elem, bt));
186 }
187
188 bool LibraryCallKit::arch_supports_vector(int sopc, int num_elem, BasicType type, VectorMaskUseType mask_use_type, bool has_scalar_args) {
189 // Check that the operation is valid.
190 if (sopc <= 0) {
191 non_product_log_if_needed(" ** Rejected intrinsification because no valid vector op could be extracted");
192 return false;
193 }
194
195 if (VectorNode::is_vector_rotate(sopc)) {
196 if(!arch_supports_vector_rotate(sopc, num_elem, type, mask_use_type, has_scalar_args)) {
197 non_product_log_if_needed(" ** Rejected vector op (%s,%s,%d) because architecture does not support variable vector shifts",
198 NodeClassNames[sopc], type2name(type), num_elem);
199 return false;
200 }
201 } else if (VectorNode::is_vector_integral_negate(sopc)) {
202 if (!VectorNode::is_vector_integral_negate_supported(sopc, num_elem, type, false)) {
203 non_product_log_if_needed(" ** Rejected vector op (%s,%s,%d) because architecture does not support integral vector negate",
204 NodeClassNames[sopc], type2name(type), num_elem);
205 return false;
206 }
207 } else {
208 // Check that architecture supports this op-size-type combination.
209 if (!Matcher::match_rule_supported_vector(sopc, num_elem, type)) {
210 non_product_log_if_needed(" ** Rejected vector op (%s,%s,%d) because architecture does not support it",
211 NodeClassNames[sopc], type2name(type), num_elem);
212 return false;
213 } else {
214 assert(Matcher::match_rule_supported(sopc), "must be supported");
215 }
216 }
217
218 if (num_elem == 1) {
219 if (mask_use_type != VecMaskNotUsed) {
220 non_product_log_if_needed(" ** Rejected vector mask op (%s,%s,%d) because architecture does not support it",
221 NodeClassNames[sopc], type2name(type), num_elem);
222 return false;
223 }
224
225 if (sopc != 0) {
226 if (sopc != Op_LoadVector && sopc != Op_StoreVector) {
227 non_product_log_if_needed(" ** Not a svml call or load/store vector op (%s,%s,%d)",
228 NodeClassNames[sopc], type2name(type), num_elem);
229 return false;
230 }
231 }
232 }
233
234 if (!has_scalar_args && VectorNode::is_vector_shift(sopc) &&
235 Matcher::supports_vector_variable_shifts() == false) {
236 log_if_needed(" ** Rejected vector op (%s,%s,%d) because architecture does not support variable vector shifts",
237 NodeClassNames[sopc], type2name(type), num_elem);
238 return false;
239 }
240
241 // Check if mask unboxing is supported, this is a two step process which first loads the contents
242 // of boolean array into vector followed by either lane expansion to match the lane size of masked
243 // vector operation or populate the predicate register.
244 if ((mask_use_type & VecMaskUseLoad) != 0) {
245 if (!Matcher::match_rule_supported_vector(Op_VectorLoadMask, num_elem, type) ||
246 !Matcher::match_rule_supported_vector(Op_LoadVector, num_elem, T_BOOLEAN)) {
247 non_product_log_if_needed(" ** Rejected vector mask loading (%s,%s,%d) because architecture does not support it",
248 NodeClassNames[Op_VectorLoadMask], type2name(type), num_elem);
249 return false;
250 }
251 }
252
253 // Check if mask boxing is supported, this is a two step process which first stores the contents
254 // of mask vector / predicate register into a boolean vector followed by vector store operation to
255 // transfer the contents to underlined storage of mask boxes which is a boolean array.
256 if ((mask_use_type & VecMaskUseStore) != 0) {
257 if (!Matcher::match_rule_supported_vector(Op_VectorStoreMask, num_elem, type) ||
258 !Matcher::match_rule_supported_vector(Op_StoreVector, num_elem, T_BOOLEAN)) {
259 non_product_log_if_needed("Rejected vector mask storing (%s,%s,%d) because architecture does not support it",
260 NodeClassNames[Op_VectorStoreMask], type2name(type), num_elem);
261 return false;
262 }
263 }
264
265 if ((mask_use_type & VecMaskUsePred) != 0) {
266 bool is_supported = false;
267 if (Matcher::has_predicated_vectors()) {
268 if (VectorNode::is_vector_integral_negate(sopc)) {
269 is_supported = VectorNode::is_vector_integral_negate_supported(sopc, num_elem, type, true);
270 } else {
271 is_supported = Matcher::match_rule_supported_vector_masked(sopc, num_elem, type);
272 }
273 }
274 is_supported |= Matcher::supports_vector_predicate_op_emulation(sopc, num_elem, type);
275
276 if (!is_supported) {
277 non_product_log_if_needed("Rejected vector mask predicate using (%s,%s,%d) because architecture does not support it",
278 NodeClassNames[sopc], type2name(type), num_elem);
279 return false;
280 }
281 }
282
283 return true;
284 }
285
286 static bool is_klass_initialized(const TypeInstPtr* vec_klass) {
287 if (vec_klass->const_oop() == nullptr) {
288 return false; // uninitialized or some kind of unsafe access
289 }
290 assert(vec_klass->const_oop()->as_instance()->java_lang_Class_klass() != nullptr, "klass instance expected");
291 ciInstanceKlass* klass = vec_klass->const_oop()->as_instance()->java_lang_Class_klass()->as_instance_klass();
292 return klass->is_initialized();
293 }
294
295 // public static
296 // <V extends Vector<E>,
297 // M extends VectorMask<E>,
298 // E>
299 // V unaryOp(int oprId, Class<? extends V> vmClass, Class<? extends M> maskClass, Class<E> elementType,
300 // int length, V v, M m,
301 // UnaryOperation<V, M> defaultImpl)
302 //
303 // public static
304 // <V,
305 // M extends VectorMask<E>,
306 // E>
307 // V binaryOp(int oprId, Class<? extends V> vmClass, Class<? extends M> maskClass, Class<E> elementType,
308 // int length, V v1, V v2, M m,
309 // BinaryOperation<V, M> defaultImpl)
310 //
311 // public static
312 // <V extends Vector<E>,
313 // M extends VectorMask<E>,
314 // E>
315 // V ternaryOp(int oprId, Class<? extends V> vmClass, Class<? extends M> maskClass, Class<E> elementType,
316 // int length, V v1, V v2, V v3, M m,
317 // TernaryOperation<V, M> defaultImpl)
318 //
319 bool LibraryCallKit::inline_vector_nary_operation(int n) {
320 const TypeInt* opr = gvn().type(argument(0))->isa_int();
321 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
322 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
323 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
324 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
325
326 if (opr == nullptr || vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr ||
327 !opr->is_con() || vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
328 log_if_needed(" ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
329 NodeClassNames[argument(0)->Opcode()],
330 NodeClassNames[argument(1)->Opcode()],
331 NodeClassNames[argument(3)->Opcode()],
332 NodeClassNames[argument(4)->Opcode()]);
333 return false; // not enough info for intrinsification
334 }
335
336 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
337 if (!elem_type->is_primitive_type()) {
338 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
339 return false; // should be primitive type
340 }
341 if (!is_klass_initialized(vector_klass)) {
342 log_if_needed(" ** klass argument not initialized");
343 return false;
344 }
345
346 // "argument(n + 5)" should be the mask object. We assume it is "null" when no mask
347 // is used to control this operation.
348 const Type* vmask_type = gvn().type(argument(n + 5));
349 bool is_masked_op = vmask_type != TypePtr::NULL_PTR;
350 if (is_masked_op) {
351 if (mask_klass == nullptr || mask_klass->const_oop() == nullptr) {
352 log_if_needed(" ** missing constant: maskclass=%s", NodeClassNames[argument(2)->Opcode()]);
353 return false; // not enough info for intrinsification
354 }
355
356 if (!is_klass_initialized(mask_klass)) {
357 log_if_needed(" ** mask klass argument not initialized");
358 return false;
359 }
360
361 if (vmask_type->maybe_null()) {
362 log_if_needed(" ** null mask values are not allowed for masked op");
363 return false;
364 }
365 }
366
367 BasicType elem_bt = elem_type->basic_type();
368 int num_elem = vlen->get_con();
369 int opc = VectorSupport::vop2ideal(opr->get_con(), elem_bt);
370 int sopc = VectorNode::opcode(opc, elem_bt);
371 if ((opc != Op_CallLeafVector) && (sopc == 0)) {
372 log_if_needed(" ** operation not supported: opc=%s bt=%s", NodeClassNames[opc], type2name(elem_bt));
373 return false; // operation not supported
374 }
375 if (num_elem == 1) {
376 if (opc != Op_CallLeafVector || elem_bt != T_DOUBLE) {
377 log_if_needed(" ** not a svml call: arity=%d opc=%d vlen=%d etype=%s",
378 n, opc, num_elem, type2name(elem_bt));
379 return false;
380 }
381 }
382 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
383 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
384
385 if (is_vector_mask(vbox_klass)) {
386 assert(!is_masked_op, "mask operations do not need mask to control");
387 }
388
389 if (opc == Op_CallLeafVector) {
390 if (!UseVectorStubs) {
391 log_if_needed(" ** vector stubs support is disabled");
392 return false;
393 }
394 if (!Matcher::supports_vector_calling_convention()) {
395 log_if_needed(" ** no vector calling conventions supported");
396 return false;
397 }
398 if (!Matcher::vector_size_supported(elem_bt, num_elem)) {
399 log_if_needed(" ** vector size (vlen=%d, etype=%s) is not supported",
400 num_elem, type2name(elem_bt));
401 return false;
402 }
403 }
404
405 // When using mask, mask use type needs to be VecMaskUseLoad.
406 VectorMaskUseType mask_use_type = is_vector_mask(vbox_klass) ? VecMaskUseAll
407 : is_masked_op ? VecMaskUseLoad : VecMaskNotUsed;
408 if ((sopc != 0) && !arch_supports_vector(sopc, num_elem, elem_bt, mask_use_type)) {
409 log_if_needed(" ** not supported: arity=%d opc=%d vlen=%d etype=%s ismask=%d is_masked_op=%d",
410 n, sopc, num_elem, type2name(elem_bt),
411 is_vector_mask(vbox_klass) ? 1 : 0, is_masked_op ? 1 : 0);
412 return false; // not supported
413 }
414
415 // Return true if current platform has implemented the masked operation with predicate feature.
416 bool use_predicate = is_masked_op && sopc != 0 && arch_supports_vector(sopc, num_elem, elem_bt, VecMaskUsePred);
417 if (is_masked_op && !use_predicate && !arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad)) {
418 log_if_needed(" ** not supported: arity=%d opc=%d vlen=%d etype=%s ismask=0 is_masked_op=1",
419 n, sopc, num_elem, type2name(elem_bt));
420 return false;
421 }
422
423 Node* opd1 = nullptr; Node* opd2 = nullptr; Node* opd3 = nullptr;
424 switch (n) {
425 case 3: {
426 opd3 = unbox_vector(argument(7), vbox_type, elem_bt, num_elem);
427 if (opd3 == nullptr) {
428 log_if_needed(" ** unbox failed v3=%s",
429 NodeClassNames[argument(7)->Opcode()]);
430 return false;
431 }
432 // fall-through
433 }
434 case 2: {
435 opd2 = unbox_vector(argument(6), vbox_type, elem_bt, num_elem);
436 if (opd2 == nullptr) {
437 log_if_needed(" ** unbox failed v2=%s",
438 NodeClassNames[argument(6)->Opcode()]);
439 return false;
440 }
441 // fall-through
442 }
443 case 1: {
444 opd1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
445 if (opd1 == nullptr) {
446 log_if_needed(" ** unbox failed v1=%s",
447 NodeClassNames[argument(5)->Opcode()]);
448 return false;
449 }
450 break;
451 }
452 default: fatal("unsupported arity: %d", n);
453 }
454
455 Node* mask = nullptr;
456 if (is_masked_op) {
457 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
458 assert(is_vector_mask(mbox_klass), "argument(2) should be a mask class");
459 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
460 mask = unbox_vector(argument(n + 5), mbox_type, elem_bt, num_elem);
461 if (mask == nullptr) {
462 log_if_needed(" ** unbox failed mask=%s",
463 NodeClassNames[argument(n + 5)->Opcode()]);
464 return false;
465 }
466 }
467
468 Node* operation = nullptr;
469 if (opc == Op_CallLeafVector) {
470 assert(UseVectorStubs, "sanity");
471 operation = gen_call_to_svml(opr->get_con(), elem_bt, num_elem, opd1, opd2);
472 if (operation == nullptr) {
473 log_if_needed(" ** svml call failed for %s_%s_%d",
474 (elem_bt == T_FLOAT)?"float":"double",
475 VectorSupport::svmlname[opr->get_con() - VectorSupport::VECTOR_OP_SVML_START],
476 num_elem * type2aelembytes(elem_bt));
477 return false;
478 }
479 } else {
480 const TypeVect* vt = TypeVect::make(elem_bt, num_elem, is_vector_mask(vbox_klass));
481 switch (n) {
482 case 1:
483 case 2: {
484 operation = VectorNode::make(sopc, opd1, opd2, vt, is_vector_mask(vbox_klass), VectorNode::is_shift_opcode(opc));
485 break;
486 }
487 case 3: {
488 operation = VectorNode::make(sopc, opd1, opd2, opd3, vt);
489 break;
490 }
491 default: fatal("unsupported arity: %d", n);
492 }
493 }
494
495 if (is_masked_op && mask != nullptr) {
496 if (use_predicate) {
497 operation->add_req(mask);
498 operation->add_flag(Node::Flag_is_predicated_vector);
499 } else {
500 operation->add_flag(Node::Flag_is_predicated_using_blend);
501 operation = gvn().transform(operation);
502 operation = new VectorBlendNode(opd1, operation, mask);
503 }
504 }
505 operation = gvn().transform(operation);
506
507 // Wrap it up in VectorBox to keep object type information.
508 Node* vbox = box_vector(operation, vbox_type, elem_bt, num_elem);
509 set_result(vbox);
510 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
511 return true;
512 }
513
514 // <Sh extends VectorShuffle<E>, E>
515 // Sh ShuffleIota(Class<?> E, Class<?> shuffleClass, Vector.Species<E> s, int length,
516 // int start, int step, int wrap, ShuffleIotaOperation<Sh, E> defaultImpl)
517 bool LibraryCallKit::inline_vector_shuffle_iota() {
518 const TypeInstPtr* shuffle_klass = gvn().type(argument(1))->isa_instptr();
519 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
520 const TypeInt* start_val = gvn().type(argument(4))->isa_int();
521 const TypeInt* step_val = gvn().type(argument(5))->isa_int();
522 const TypeInt* wrap = gvn().type(argument(6))->isa_int();
523
524 if (shuffle_klass == nullptr || shuffle_klass->const_oop() == nullptr ||
525 vlen == nullptr || !vlen->is_con() || start_val == nullptr || step_val == nullptr ||
526 wrap == nullptr || !wrap->is_con()) {
527 return false; // not enough info for intrinsification
528 }
529
530 if (!is_klass_initialized(shuffle_klass)) {
531 log_if_needed(" ** klass argument not initialized");
532 return false;
533 }
534
535 int do_wrap = wrap->get_con();
536 int num_elem = vlen->get_con();
537 BasicType elem_bt = T_BYTE;
538
539 bool effective_indices_in_range = false;
540 if (start_val->is_con() && step_val->is_con()) {
541 int effective_min_index = start_val->get_con();
542 int effective_max_index = start_val->get_con() + step_val->get_con() * (num_elem - 1);
543 effective_indices_in_range = effective_max_index >= effective_min_index && effective_min_index >= -128 && effective_max_index <= 127;
544 }
545
546 if (!do_wrap && !effective_indices_in_range) {
547 // Disable instrinsification for unwrapped shuffle iota if start/step
548 // values are non-constant OR if intermediate result overflows byte value range.
549 return false;
550 }
551
552 if (!arch_supports_vector(Op_AddVB, num_elem, elem_bt, VecMaskNotUsed) ||
553 !arch_supports_vector(Op_AndV, num_elem, elem_bt, VecMaskNotUsed) ||
554 !arch_supports_vector(Op_VectorLoadConst, num_elem, elem_bt, VecMaskNotUsed) ||
555 !arch_supports_vector(Op_Replicate, num_elem, elem_bt, VecMaskNotUsed)) {
556 return false;
557 }
558
559 if (!do_wrap &&
560 (!arch_supports_vector(Op_SubVB, num_elem, elem_bt, VecMaskNotUsed) ||
561 !arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskNotUsed) ||
562 !arch_supports_vector(Op_VectorMaskCmp, num_elem, elem_bt, VecMaskNotUsed))) {
563 return false;
564 }
565
566 bool step_multiply = !step_val->is_con() || !is_power_of_2(step_val->get_con());
567 if ((step_multiply && !arch_supports_vector(Op_MulVB, num_elem, elem_bt, VecMaskNotUsed)) ||
568 (!step_multiply && !arch_supports_vector(Op_LShiftVB, num_elem, elem_bt, VecMaskNotUsed))) {
569 return false;
570 }
571
572 const Type * type_bt = Type::get_const_basic_type(elem_bt);
573 const TypeVect * vt = TypeVect::make(type_bt, num_elem);
574
575 Node* res = gvn().transform(new VectorLoadConstNode(gvn().makecon(TypeInt::ZERO), vt));
576
577 Node* start = argument(4);
578 Node* step = argument(5);
579
580 if (step_multiply) {
581 Node* bcast_step = gvn().transform(VectorNode::scalar2vector(step, num_elem, type_bt));
582 res = gvn().transform(VectorNode::make(Op_MulVB, res, bcast_step, vt));
583 } else if (step_val->get_con() > 1) {
584 Node* cnt = gvn().makecon(TypeInt::make(log2i_exact(step_val->get_con())));
585 Node* shift_cnt = vector_shift_count(cnt, Op_LShiftI, elem_bt, num_elem);
586 res = gvn().transform(VectorNode::make(Op_LShiftVB, res, shift_cnt, vt));
587 }
588
589 if (!start_val->is_con() || start_val->get_con() != 0) {
590 Node* bcast_start = gvn().transform(VectorNode::scalar2vector(start, num_elem, type_bt));
591 res = gvn().transform(VectorNode::make(Op_AddVB, res, bcast_start, vt));
592 }
593
594 Node * mod_val = gvn().makecon(TypeInt::make(num_elem-1));
595 Node * bcast_mod = gvn().transform(VectorNode::scalar2vector(mod_val, num_elem, type_bt));
596
597 if (do_wrap) {
598 // Wrap the indices greater than lane count.
599 res = gvn().transform(VectorNode::make(Op_AndV, res, bcast_mod, vt));
600 } else {
601 ConINode* pred_node = (ConINode*)gvn().makecon(TypeInt::make(BoolTest::ugt));
602 Node * lane_cnt = gvn().makecon(TypeInt::make(num_elem));
603 Node * bcast_lane_cnt = gvn().transform(VectorNode::scalar2vector(lane_cnt, num_elem, type_bt));
604 const TypeVect* vmask_type = TypeVect::makemask(elem_bt, num_elem);
605 Node* mask = gvn().transform(new VectorMaskCmpNode(BoolTest::ugt, bcast_lane_cnt, res, pred_node, vmask_type));
606
607 // Make the indices greater than lane count as -ve values to match the java side implementation.
608 res = gvn().transform(VectorNode::make(Op_AndV, res, bcast_mod, vt));
609 Node * biased_val = gvn().transform(VectorNode::make(Op_SubVB, res, bcast_lane_cnt, vt));
610 res = gvn().transform(new VectorBlendNode(biased_val, res, mask));
611 }
612
613 ciKlass* sbox_klass = shuffle_klass->const_oop()->as_instance()->java_lang_Class_klass();
614 const TypeInstPtr* shuffle_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, sbox_klass);
615
616 // Wrap it up in VectorBox to keep object type information.
617 res = box_vector(res, shuffle_box_type, elem_bt, num_elem);
618 set_result(res);
619 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
620 return true;
621 }
622
623 // <E, M>
624 // long maskReductionCoerced(int oper, Class<? extends M> maskClass, Class<?> elemClass,
625 // int length, M m, VectorMaskOp<M> defaultImpl)
626 bool LibraryCallKit::inline_vector_mask_operation() {
627 const TypeInt* oper = gvn().type(argument(0))->isa_int();
628 const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
629 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
630 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
631 Node* mask = argument(4);
632
633 if (mask_klass == nullptr || elem_klass == nullptr || mask->is_top() || vlen == nullptr) {
634 return false; // dead code
635 }
636
637 if (!is_klass_initialized(mask_klass)) {
638 log_if_needed(" ** klass argument not initialized");
639 return false;
640 }
641
642 int num_elem = vlen->get_con();
643 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
644 BasicType elem_bt = elem_type->basic_type();
645
646 int mopc = VectorSupport::vop2ideal(oper->get_con(), elem_bt);
647 if (!arch_supports_vector(mopc, num_elem, elem_bt, VecMaskUseLoad)) {
648 log_if_needed(" ** not supported: arity=1 op=cast#%d/3 vlen2=%d etype2=%s",
649 mopc, num_elem, type2name(elem_bt));
650 return false; // not supported
651 }
652
653 const Type* elem_ty = Type::get_const_basic_type(elem_bt);
654 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
655 const TypeInstPtr* mask_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
656 Node* mask_vec = unbox_vector(mask, mask_box_type, elem_bt, num_elem, true);
657 if (mask_vec == nullptr) {
658 log_if_needed(" ** unbox failed mask=%s",
659 NodeClassNames[argument(4)->Opcode()]);
660 return false;
661 }
662
663 if (mask_vec->bottom_type()->isa_vectmask() == nullptr) {
664 mask_vec = gvn().transform(VectorStoreMaskNode::make(gvn(), mask_vec, elem_bt, num_elem));
665 }
666 const Type* maskoper_ty = mopc == Op_VectorMaskToLong ? (const Type*)TypeLong::LONG : (const Type*)TypeInt::INT;
667 Node* maskoper = gvn().transform(VectorMaskOpNode::make(mask_vec, maskoper_ty, mopc));
668 if (mopc != Op_VectorMaskToLong) {
669 maskoper = ConvI2L(maskoper);
670 }
671 set_result(maskoper);
672
673 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
674 return true;
675 }
676
677 // public static
678 // <V,
679 // Sh extends VectorShuffle<E>,
680 // E>
681 // V shuffleToVector(Class<? extends Vector<E>> vclass, Class<E> elementType,
682 // Class<? extends Sh> shuffleClass, Sh s, int length,
683 // ShuffleToVectorOperation<V, Sh, E> defaultImpl)
684 bool LibraryCallKit::inline_vector_shuffle_to_vector() {
685 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
686 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
687 const TypeInstPtr* shuffle_klass = gvn().type(argument(2))->isa_instptr();
688 Node* shuffle = argument(3);
689 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
690
691 if (vector_klass == nullptr || elem_klass == nullptr || shuffle_klass == nullptr || shuffle->is_top() || vlen == nullptr) {
692 return false; // dead code
693 }
694 if (!vlen->is_con() || vector_klass->const_oop() == nullptr || shuffle_klass->const_oop() == nullptr) {
695 return false; // not enough info for intrinsification
696 }
697 if (!is_klass_initialized(shuffle_klass) || !is_klass_initialized(vector_klass) ) {
698 log_if_needed(" ** klass argument not initialized");
699 return false;
700 }
701
702 int num_elem = vlen->get_con();
703 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
704 BasicType elem_bt = elem_type->basic_type();
705
706 if (num_elem < 4) {
707 return false;
708 }
709
710 int cast_vopc = VectorCastNode::opcode(-1, T_BYTE); // from shuffle of type T_BYTE
711 // Make sure that cast is implemented to particular type/size combination.
712 if (!arch_supports_vector(cast_vopc, num_elem, elem_bt, VecMaskNotUsed)) {
713 log_if_needed(" ** not supported: arity=1 op=cast#%d/3 vlen2=%d etype2=%s",
714 cast_vopc, num_elem, type2name(elem_bt));
715 return false;
716 }
717
718 ciKlass* sbox_klass = shuffle_klass->const_oop()->as_instance()->java_lang_Class_klass();
719 const TypeInstPtr* shuffle_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, sbox_klass);
720
721 // Unbox shuffle with true flag to indicate its load shuffle to vector
722 // shuffle is a byte array
723 Node* shuffle_vec = unbox_vector(shuffle, shuffle_box_type, T_BYTE, num_elem, true);
724
725 // cast byte to target element type
726 shuffle_vec = gvn().transform(VectorCastNode::make(cast_vopc, shuffle_vec, elem_bt, num_elem));
727
728 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
729 const TypeInstPtr* vec_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
730
731 // Box vector
732 Node* res = box_vector(shuffle_vec, vec_box_type, elem_bt, num_elem);
733 set_result(res);
734 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
735 return true;
736 }
737
738 // public static
739 // <M,
740 // S extends VectorSpecies<E>,
741 // E>
742 // M fromBitsCoerced(Class<? extends M> vmClass, Class<E> elementType, int length,
743 // long bits, int mode, S s,
744 // BroadcastOperation<M, E, S> defaultImpl)
745 bool LibraryCallKit::inline_vector_frombits_coerced() {
746 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
747 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
748 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
749 const TypeLong* bits_type = gvn().type(argument(3))->isa_long();
750 // Mode argument determines the mode of operation it can take following values:-
751 // MODE_BROADCAST for vector Vector.broadcast and VectorMask.maskAll operations.
752 // MODE_BITS_COERCED_LONG_TO_MASK for VectorMask.fromLong operation.
753 const TypeInt* mode = gvn().type(argument(5))->isa_int();
754
755 if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr || mode == nullptr ||
756 bits_type == nullptr || vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr ||
757 !vlen->is_con() || !mode->is_con()) {
758 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s bitwise=%s",
759 NodeClassNames[argument(0)->Opcode()],
760 NodeClassNames[argument(1)->Opcode()],
761 NodeClassNames[argument(2)->Opcode()],
762 NodeClassNames[argument(5)->Opcode()]);
763 return false; // not enough info for intrinsification
764 }
765
766 if (!is_klass_initialized(vector_klass)) {
767 log_if_needed(" ** klass argument not initialized");
768 return false;
769 }
770 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
771 if (!elem_type->is_primitive_type()) {
772 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
773 return false; // should be primitive type
774 }
775 BasicType elem_bt = elem_type->basic_type();
776 int num_elem = vlen->get_con();
777 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
778 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
779
780 bool is_mask = is_vector_mask(vbox_klass);
781 int bcast_mode = mode->get_con();
782 VectorMaskUseType checkFlags = (VectorMaskUseType)(is_mask ? VecMaskUseAll : VecMaskNotUsed);
783 int opc = bcast_mode == VectorSupport::MODE_BITS_COERCED_LONG_TO_MASK ? Op_VectorLongToMask : Op_Replicate;
784
785 if (!arch_supports_vector(opc, num_elem, elem_bt, checkFlags, true /*has_scalar_args*/)) {
786 log_if_needed(" ** not supported: arity=0 op=broadcast vlen=%d etype=%s ismask=%d bcast_mode=%d",
787 num_elem, type2name(elem_bt),
788 is_mask ? 1 : 0,
789 bcast_mode);
790 return false; // not supported
791 }
792
793 Node* broadcast = nullptr;
794 Node* bits = argument(3);
795 Node* elem = bits;
796
797 if (opc == Op_VectorLongToMask) {
798 const TypeVect* vt = TypeVect::makemask(elem_bt, num_elem);
799 if (vt->isa_vectmask()) {
800 broadcast = gvn().transform(new VectorLongToMaskNode(elem, vt));
801 } else {
802 const TypeVect* mvt = TypeVect::make(T_BOOLEAN, num_elem);
803 broadcast = gvn().transform(new VectorLongToMaskNode(elem, mvt));
804 broadcast = gvn().transform(new VectorLoadMaskNode(broadcast, vt));
805 }
806 } else {
807 switch (elem_bt) {
808 case T_BOOLEAN: // fall-through
809 case T_BYTE: // fall-through
810 case T_SHORT: // fall-through
811 case T_CHAR: // fall-through
812 case T_INT: {
813 elem = gvn().transform(new ConvL2INode(bits));
814 break;
815 }
816 case T_DOUBLE: {
817 elem = gvn().transform(new MoveL2DNode(bits));
818 break;
819 }
820 case T_FLOAT: {
821 bits = gvn().transform(new ConvL2INode(bits));
822 elem = gvn().transform(new MoveI2FNode(bits));
823 break;
824 }
825 case T_LONG: {
826 // no conversion needed
827 break;
828 }
829 default: fatal("%s", type2name(elem_bt));
830 }
831 broadcast = VectorNode::scalar2vector(elem, num_elem, Type::get_const_basic_type(elem_bt), is_mask);
832 broadcast = gvn().transform(broadcast);
833 }
834
835 Node* box = box_vector(broadcast, vbox_type, elem_bt, num_elem);
836 set_result(box);
837 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
838 return true;
839 }
840
841 static bool elem_consistent_with_arr(BasicType elem_bt, const TypeAryPtr* arr_type, bool mismatched_ms) {
842 assert(arr_type != nullptr, "unexpected");
843 BasicType arr_elem_bt = arr_type->elem()->array_element_basic_type();
844 if (elem_bt == arr_elem_bt) {
845 return true;
846 } else if (elem_bt == T_SHORT && arr_elem_bt == T_CHAR) {
847 // Load/store of short vector from/to char[] is supported
848 return true;
849 } else if (elem_bt == T_BYTE && arr_elem_bt == T_BOOLEAN) {
850 // Load/store of byte vector from/to boolean[] is supported
851 return true;
852 } else {
853 return mismatched_ms;
854 }
855 }
856
857 // public static
858 // <C,
859 // VM extends VectorPayload,
860 // E,
861 // S extends VectorSpecies<E>>
862 // VM load(Class<? extends VM> vmClass, Class<E> eClass,
863 // int length,
864 // Object base, long offset, // Unsafe addressing
865 // boolean fromSegment,
866 // C container, long index, S s, // Arguments for default implementation
867 // LoadOperation<C, VM, S> defaultImpl) {
868 // public static
869 // <C,
870 // V extends VectorPayload>
871 // void store(Class<?> vClass, Class<?> eClass,
872 // int length,
873 // Object base, long offset, // Unsafe addressing
874 // boolean fromSegment,
875 // V v, C container, long index, // Arguments for default implementation
876 // StoreVectorOperation<C, V> defaultImpl) {
877 bool LibraryCallKit::inline_vector_mem_operation(bool is_store) {
878 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
879 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
880 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
881 const TypeInt* from_ms = gvn().type(argument(6))->isa_int();
882
883 if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr || !from_ms->is_con() ||
884 vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
885 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s from_ms=%s",
886 NodeClassNames[argument(0)->Opcode()],
887 NodeClassNames[argument(1)->Opcode()],
888 NodeClassNames[argument(2)->Opcode()],
889 NodeClassNames[argument(6)->Opcode()]);
890 return false; // not enough info for intrinsification
891 }
892 if (!is_klass_initialized(vector_klass)) {
893 log_if_needed(" ** klass argument not initialized");
894 return false;
895 }
896
897 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
898 if (!elem_type->is_primitive_type()) {
899 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
900 return false; // should be primitive type
901 }
902 BasicType elem_bt = elem_type->basic_type();
903 int num_elem = vlen->get_con();
904
905 // TODO When mask usage is supported, VecMaskNotUsed needs to be VecMaskUseLoad.
906 if (!arch_supports_vector(is_store ? Op_StoreVector : Op_LoadVector, num_elem, elem_bt, VecMaskNotUsed)) {
907 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d etype=%s ismask=no",
908 is_store, is_store ? "store" : "load",
909 num_elem, type2name(elem_bt));
910 return false; // not supported
911 }
912
913 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
914 bool is_mask = is_vector_mask(vbox_klass);
915
916 Node* base = argument(3);
917 Node* offset = ConvL2X(argument(4));
918
919 // Save state and restore on bailout
920 uint old_sp = sp();
921 SafePointNode* old_map = clone_map();
922
923 Node* addr = make_unsafe_address(base, offset, (is_mask ? T_BOOLEAN : elem_bt), true);
924
925 // The memory barrier checks are based on ones for unsafe access.
926 // This is not 1-1 implementation.
927 const Type *const base_type = gvn().type(base);
928
929 const TypePtr *addr_type = gvn().type(addr)->isa_ptr();
930 const TypeAryPtr* arr_type = addr_type->isa_aryptr();
931
932 const bool in_native = TypePtr::NULL_PTR == base_type; // base always null
933 const bool in_heap = !TypePtr::NULL_PTR->higher_equal(base_type); // base never null
934
935 const bool is_mixed_access = !in_heap && !in_native;
936
937 const bool is_mismatched_access = in_heap && (addr_type->isa_aryptr() == nullptr);
938
939 const bool needs_cpu_membar = is_mixed_access || is_mismatched_access;
940
941 // For non-masked mismatched memory segment vector read/write accesses, intrinsification can continue
942 // with unknown backing storage type and compiler can skip inserting explicit reinterpretation IR after
943 // loading from or before storing to backing storage which is mandatory for semantic correctness of
944 // big-endian memory layout.
945 bool mismatched_ms = LITTLE_ENDIAN_ONLY(false)
946 BIG_ENDIAN_ONLY(from_ms->get_con() && !is_mask && arr_type != nullptr &&
947 arr_type->elem()->array_element_basic_type() != elem_bt);
948 BasicType mem_elem_bt = mismatched_ms ? arr_type->elem()->array_element_basic_type() : elem_bt;
949 if (!is_java_primitive(mem_elem_bt)) {
950 log_if_needed(" ** non-primitive array element type");
951 return false;
952 }
953 int mem_num_elem = mismatched_ms ? (num_elem * type2aelembytes(elem_bt)) / type2aelembytes(mem_elem_bt) : num_elem;
954 if (arr_type != nullptr && !is_mask && !elem_consistent_with_arr(elem_bt, arr_type, mismatched_ms)) {
955 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d etype=%s atype=%s ismask=no",
956 is_store, is_store ? "store" : "load",
957 num_elem, type2name(elem_bt), type2name(arr_type->elem()->array_element_basic_type()));
958 set_map(old_map);
959 set_sp(old_sp);
960 return false;
961 }
962
963 // In case of mismatched memory segment accesses, we need to double check that the source type memory operations are supported by backend.
964 if (mismatched_ms) {
965 if (is_store) {
966 if (!arch_supports_vector(Op_StoreVector, num_elem, elem_bt, VecMaskNotUsed)
967 || !arch_supports_vector(Op_VectorReinterpret, mem_num_elem, mem_elem_bt, VecMaskNotUsed)) {
968 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d*8 etype=%s/8 ismask=no",
969 is_store, "store",
970 num_elem, type2name(elem_bt));
971 set_map(old_map);
972 set_sp(old_sp);
973 return false; // not supported
974 }
975 } else {
976 if (!arch_supports_vector(Op_LoadVector, mem_num_elem, mem_elem_bt, VecMaskNotUsed)
977 || !arch_supports_vector(Op_VectorReinterpret, num_elem, elem_bt, VecMaskNotUsed)) {
978 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d*8 etype=%s/8 ismask=no",
979 is_store, "load",
980 mem_num_elem, type2name(mem_elem_bt));
981 set_map(old_map);
982 set_sp(old_sp);
983 return false; // not supported
984 }
985 }
986 }
987 if (is_mask) {
988 if (!is_store) {
989 if (!arch_supports_vector(Op_LoadVector, num_elem, elem_bt, VecMaskUseLoad)) {
990 set_map(old_map);
991 set_sp(old_sp);
992 return false; // not supported
993 }
994 } else {
995 if (!arch_supports_vector(Op_StoreVector, num_elem, elem_bt, VecMaskUseStore)) {
996 set_map(old_map);
997 set_sp(old_sp);
998 return false; // not supported
999 }
1000 }
1001 }
1002
1003 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1004
1005 if (needs_cpu_membar) {
1006 insert_mem_bar(Op_MemBarCPUOrder);
1007 }
1008
1009 if (is_store) {
1010 Node* val = unbox_vector(argument(7), vbox_type, elem_bt, num_elem);
1011 if (val == nullptr) {
1012 set_map(old_map);
1013 set_sp(old_sp);
1014 return false; // operand unboxing failed
1015 }
1016 set_all_memory(reset_memory());
1017
1018 // In case the store needs to happen to byte array, reinterpret the incoming vector to byte vector.
1019 int store_num_elem = num_elem;
1020 if (mismatched_ms) {
1021 store_num_elem = mem_num_elem;
1022 const TypeVect* to_vect_type = TypeVect::make(mem_elem_bt, store_num_elem);
1023 val = gvn().transform(new VectorReinterpretNode(val, val->bottom_type()->is_vect(), to_vect_type));
1024 }
1025 if (is_mask) {
1026 val = gvn().transform(VectorStoreMaskNode::make(gvn(), val, elem_bt, num_elem));
1027 }
1028 Node* vstore = gvn().transform(StoreVectorNode::make(0, control(), memory(addr), addr, addr_type, val, store_num_elem));
1029 set_memory(vstore, addr_type);
1030 } else {
1031 // When using byte array, we need to load as byte then reinterpret the value. Otherwise, do a simple vector load.
1032 Node* vload = nullptr;
1033 if (mismatched_ms) {
1034 vload = gvn().transform(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, mem_num_elem, mem_elem_bt));
1035 const TypeVect* to_vect_type = TypeVect::make(elem_bt, num_elem);
1036 vload = gvn().transform(new VectorReinterpretNode(vload, vload->bottom_type()->is_vect(), to_vect_type));
1037 } else {
1038 // Special handle for masks
1039 if (is_mask) {
1040 vload = gvn().transform(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, num_elem, T_BOOLEAN));
1041 vload = gvn().transform(new VectorLoadMaskNode(vload, TypeVect::makemask(elem_bt, num_elem)));
1042 } else {
1043 vload = gvn().transform(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, num_elem, elem_bt));
1044 }
1045 }
1046 Node* box = box_vector(vload, vbox_type, elem_bt, num_elem);
1047 set_result(box);
1048 }
1049
1050 destruct_map_clone(old_map);
1051
1052 if (needs_cpu_membar) {
1053 insert_mem_bar(Op_MemBarCPUOrder);
1054 }
1055
1056 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1057 return true;
1058 }
1059
1060 // public static
1061 // <C,
1062 // V extends Vector<?>,
1063 // E,
1064 // S extends VectorSpecies<E>,
1065 // M extends VectorMask<E>>
1066 // V loadMasked(Class<? extends V> vClass, Class<M> mClass, Class<E> eClass,
1067 // int length, Object base, long offset, // Unsafe addressing
1068 // boolean fromSegment,
1069 // M m, int offsetInRange,
1070 // C container, long index, S s, // Arguments for default implementation
1071 // LoadVectorMaskedOperation<C, V, S, M> defaultImpl) {
1072 // public static
1073 // <C,
1074 // V extends Vector<E>,
1075 // M extends VectorMask<E>,
1076 // E>
1077 // void storeMasked(Class<? extends V> vClass, Class<M> mClass, Class<E> eClass,
1078 // int length,
1079 // Object base, long offset, // Unsafe addressing
1080 // boolean fromSegment,
1081 // V v, M m, C container, long index, // Arguments for default implementation
1082 // StoreVectorMaskedOperation<C, V, M> defaultImpl) {
1083
1084 bool LibraryCallKit::inline_vector_mem_masked_operation(bool is_store) {
1085 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
1086 const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
1087 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
1088 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
1089 const TypeInt* from_ms = gvn().type(argument(7))->isa_int();
1090
1091 if (vector_klass == nullptr || mask_klass == nullptr || elem_klass == nullptr || vlen == nullptr ||
1092 vector_klass->const_oop() == nullptr || mask_klass->const_oop() == nullptr || from_ms == nullptr ||
1093 elem_klass->const_oop() == nullptr || !vlen->is_con() || !from_ms->is_con()) {
1094 log_if_needed(" ** missing constant: vclass=%s mclass=%s etype=%s vlen=%s from_ms=%s",
1095 NodeClassNames[argument(0)->Opcode()],
1096 NodeClassNames[argument(1)->Opcode()],
1097 NodeClassNames[argument(2)->Opcode()],
1098 NodeClassNames[argument(3)->Opcode()],
1099 NodeClassNames[argument(7)->Opcode()]);
1100 return false; // not enough info for intrinsification
1101 }
1102 if (!is_klass_initialized(vector_klass)) {
1103 log_if_needed(" ** klass argument not initialized");
1104 return false;
1105 }
1106
1107 if (!is_klass_initialized(mask_klass)) {
1108 log_if_needed(" ** mask klass argument not initialized");
1109 return false;
1110 }
1111
1112 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1113 if (!elem_type->is_primitive_type()) {
1114 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1115 return false; // should be primitive type
1116 }
1117
1118 BasicType elem_bt = elem_type->basic_type();
1119 int num_elem = vlen->get_con();
1120
1121 Node* base = argument(4);
1122 Node* offset = ConvL2X(argument(5));
1123
1124 // Save state and restore on bailout
1125 uint old_sp = sp();
1126 SafePointNode* old_map = clone_map();
1127
1128 Node* addr = make_unsafe_address(base, offset, elem_bt, true);
1129 const TypePtr *addr_type = gvn().type(addr)->isa_ptr();
1130 const TypeAryPtr* arr_type = addr_type->isa_aryptr();
1131
1132 bool mismatched_ms = from_ms->get_con() && arr_type != nullptr && arr_type->elem()->array_element_basic_type() != elem_bt;
1133 BIG_ENDIAN_ONLY(if (mismatched_ms) return false;)
1134 // If there is no consistency between array and vector element types, it must be special byte array case
1135 if (arr_type != nullptr && !elem_consistent_with_arr(elem_bt, arr_type, mismatched_ms)) {
1136 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d etype=%s atype=%s",
1137 is_store, is_store ? "storeMasked" : "loadMasked",
1138 num_elem, type2name(elem_bt), type2name(arr_type->elem()->array_element_basic_type()));
1139 set_map(old_map);
1140 set_sp(old_sp);
1141 return false;
1142 }
1143
1144 int mem_num_elem = mismatched_ms ? num_elem * type2aelembytes(elem_bt) : num_elem;
1145 BasicType mem_elem_bt = mismatched_ms ? T_BYTE : elem_bt;
1146 bool supports_predicate = arch_supports_vector(is_store ? Op_StoreVectorMasked : Op_LoadVectorMasked,
1147 mem_num_elem, mem_elem_bt, VecMaskUseLoad);
1148
1149 // If current arch does not support the predicated operations, we have to bail
1150 // out when current case uses the predicate feature.
1151 if (!supports_predicate) {
1152 bool needs_predicate = false;
1153 if (is_store) {
1154 // Masked vector store always uses the predicated store.
1155 needs_predicate = true;
1156 } else {
1157 // Masked vector load with IOOBE always uses the predicated load.
1158 const TypeInt* offset_in_range = gvn().type(argument(9))->isa_int();
1159 if (!offset_in_range->is_con()) {
1160 log_if_needed(" ** missing constant: offsetInRange=%s",
1161 NodeClassNames[argument(8)->Opcode()]);
1162 set_map(old_map);
1163 set_sp(old_sp);
1164 return false;
1165 }
1166 needs_predicate = (offset_in_range->get_con() == 0);
1167 }
1168
1169 if (needs_predicate) {
1170 log_if_needed(" ** not supported: op=%s vlen=%d etype=%s mismatched_ms=%d",
1171 is_store ? "storeMasked" : "loadMasked",
1172 num_elem, type2name(elem_bt), mismatched_ms ? 1 : 0);
1173 set_map(old_map);
1174 set_sp(old_sp);
1175 return false;
1176 }
1177 }
1178
1179 // This only happens for masked vector load. If predicate is not supported, then check whether
1180 // the normal vector load and blend operations are supported by backend.
1181 if (!supports_predicate && (!arch_supports_vector(Op_LoadVector, mem_num_elem, mem_elem_bt, VecMaskNotUsed) ||
1182 !arch_supports_vector(Op_VectorBlend, mem_num_elem, mem_elem_bt, VecMaskUseLoad))) {
1183 log_if_needed(" ** not supported: op=loadMasked vlen=%d etype=%s mismatched_ms=%d",
1184 num_elem, type2name(elem_bt), mismatched_ms ? 1 : 0);
1185 set_map(old_map);
1186 set_sp(old_sp);
1187 return false;
1188 }
1189
1190 // Since we are using byte array, we need to double check that the vector reinterpret operation
1191 // with byte type is supported by backend.
1192 if (mismatched_ms) {
1193 if (!arch_supports_vector(Op_VectorReinterpret, mem_num_elem, T_BYTE, VecMaskNotUsed)) {
1194 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d etype=%s mismatched_ms=1",
1195 is_store, is_store ? "storeMasked" : "loadMasked",
1196 num_elem, type2name(elem_bt));
1197 set_map(old_map);
1198 set_sp(old_sp);
1199 return false;
1200 }
1201 }
1202
1203 // Since it needs to unbox the mask, we need to double check that the related load operations
1204 // for mask are supported by backend.
1205 if (!arch_supports_vector(Op_LoadVector, num_elem, elem_bt, VecMaskUseLoad)) {
1206 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d etype=%s",
1207 is_store, is_store ? "storeMasked" : "loadMasked",
1208 num_elem, type2name(elem_bt));
1209 set_map(old_map);
1210 set_sp(old_sp);
1211 return false;
1212 }
1213
1214 // Can base be null? Otherwise, always on-heap access.
1215 bool can_access_non_heap = TypePtr::NULL_PTR->higher_equal(gvn().type(base));
1216 if (can_access_non_heap) {
1217 insert_mem_bar(Op_MemBarCPUOrder);
1218 }
1219
1220 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1221 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1222 assert(!is_vector_mask(vbox_klass) && is_vector_mask(mbox_klass), "Invalid class type");
1223 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1224 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1225
1226 Node* mask = unbox_vector(is_store ? argument(9) : argument(8), mbox_type, elem_bt, num_elem);
1227 if (mask == nullptr) {
1228 log_if_needed(" ** unbox failed mask=%s",
1229 is_store ? NodeClassNames[argument(9)->Opcode()]
1230 : NodeClassNames[argument(8)->Opcode()]);
1231 set_map(old_map);
1232 set_sp(old_sp);
1233 return false;
1234 }
1235
1236 if (is_store) {
1237 Node* val = unbox_vector(argument(8), vbox_type, elem_bt, num_elem);
1238 if (val == nullptr) {
1239 log_if_needed(" ** unbox failed vector=%s",
1240 NodeClassNames[argument(8)->Opcode()]);
1241 set_map(old_map);
1242 set_sp(old_sp);
1243 return false; // operand unboxing failed
1244 }
1245 set_all_memory(reset_memory());
1246
1247 if (mismatched_ms) {
1248 // Reinterpret the incoming vector to byte vector.
1249 const TypeVect* to_vect_type = TypeVect::make(mem_elem_bt, mem_num_elem);
1250 val = gvn().transform(new VectorReinterpretNode(val, val->bottom_type()->is_vect(), to_vect_type));
1251 // Reinterpret the vector mask to byte type.
1252 const TypeVect* from_mask_type = TypeVect::makemask(elem_bt, num_elem);
1253 const TypeVect* to_mask_type = TypeVect::makemask(mem_elem_bt, mem_num_elem);
1254 mask = gvn().transform(new VectorReinterpretNode(mask, from_mask_type, to_mask_type));
1255 }
1256 Node* vstore = gvn().transform(new StoreVectorMaskedNode(control(), memory(addr), addr, val, addr_type, mask));
1257 set_memory(vstore, addr_type);
1258 } else {
1259 Node* vload = nullptr;
1260
1261 if (mismatched_ms) {
1262 // Reinterpret the vector mask to byte type.
1263 const TypeVect* from_mask_type = TypeVect::makemask(elem_bt, num_elem);
1264 const TypeVect* to_mask_type = TypeVect::makemask(mem_elem_bt, mem_num_elem);
1265 mask = gvn().transform(new VectorReinterpretNode(mask, from_mask_type, to_mask_type));
1266 }
1267
1268 if (supports_predicate) {
1269 // Generate masked load vector node if predicate feature is supported.
1270 const TypeVect* vt = TypeVect::make(mem_elem_bt, mem_num_elem);
1271 vload = gvn().transform(new LoadVectorMaskedNode(control(), memory(addr), addr, addr_type, vt, mask));
1272 } else {
1273 // Use the vector blend to implement the masked load vector. The biased elements are zeros.
1274 Node* zero = gvn().transform(gvn().zerocon(mem_elem_bt));
1275 zero = gvn().transform(VectorNode::scalar2vector(zero, mem_num_elem, Type::get_const_basic_type(mem_elem_bt)));
1276 vload = gvn().transform(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, mem_num_elem, mem_elem_bt));
1277 vload = gvn().transform(new VectorBlendNode(zero, vload, mask));
1278 }
1279
1280 if (mismatched_ms) {
1281 const TypeVect* to_vect_type = TypeVect::make(elem_bt, num_elem);
1282 vload = gvn().transform(new VectorReinterpretNode(vload, vload->bottom_type()->is_vect(), to_vect_type));
1283 }
1284
1285 Node* box = box_vector(vload, vbox_type, elem_bt, num_elem);
1286 set_result(box);
1287 }
1288
1289 destruct_map_clone(old_map);
1290
1291 if (can_access_non_heap) {
1292 insert_mem_bar(Op_MemBarCPUOrder);
1293 }
1294
1295 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1296 return true;
1297 }
1298
1299 // <C,
1300 // V extends Vector<?>,
1301 // W extends Vector<Integer>,
1302 // S extends VectorSpecies<E>,
1303 // M extends VectorMask<E>,
1304 // E>
1305 // V loadWithMap(Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType, int length,
1306 // Class<? extends Vector<Integer>> vectorIndexClass,
1307 // Object base, long offset, // Unsafe addressing
1308 // W index_vector, M m,
1309 // C container, int index, int[] indexMap, int indexM, S s, // Arguments for default implementation
1310 // LoadVectorOperationWithMap<C, V, E, S, M> defaultImpl)
1311 //
1312 // <C,
1313 // V extends Vector<E>,
1314 // W extends Vector<Integer>,
1315 // M extends VectorMask<E>,
1316 // E>
1317 // void storeWithMap(Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType,
1318 // int length, Class<? extends Vector<Integer>> vectorIndexClass, Object base, long offset, // Unsafe addressing
1319 // W index_vector, V v, M m,
1320 // C container, int index, int[] indexMap, int indexM, // Arguments for default implementation
1321 // StoreVectorOperationWithMap<C, V, M, E> defaultImpl)
1322 //
1323 bool LibraryCallKit::inline_vector_gather_scatter(bool is_scatter) {
1324 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
1325 const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
1326 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
1327 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
1328 const TypeInstPtr* vector_idx_klass = gvn().type(argument(4))->isa_instptr();
1329
1330 if (vector_klass == nullptr || elem_klass == nullptr || vector_idx_klass == nullptr || vlen == nullptr ||
1331 vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || vector_idx_klass->const_oop() == nullptr || !vlen->is_con()) {
1332 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s viclass=%s",
1333 NodeClassNames[argument(0)->Opcode()],
1334 NodeClassNames[argument(2)->Opcode()],
1335 NodeClassNames[argument(3)->Opcode()],
1336 NodeClassNames[argument(4)->Opcode()]);
1337 return false; // not enough info for intrinsification
1338 }
1339
1340 if (!is_klass_initialized(vector_klass) || !is_klass_initialized(vector_idx_klass)) {
1341 log_if_needed(" ** klass argument not initialized");
1342 return false;
1343 }
1344
1345 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1346 if (!elem_type->is_primitive_type()) {
1347 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1348 return false; // should be primitive type
1349 }
1350
1351 BasicType elem_bt = elem_type->basic_type();
1352 int num_elem = vlen->get_con();
1353
1354 const Type* vmask_type = gvn().type(is_scatter ? argument(10) : argument(9));
1355 bool is_masked_op = vmask_type != TypePtr::NULL_PTR;
1356 if (is_masked_op) {
1357 if (mask_klass == nullptr || mask_klass->const_oop() == nullptr) {
1358 log_if_needed(" ** missing constant: maskclass=%s", NodeClassNames[argument(1)->Opcode()]);
1359 return false; // not enough info for intrinsification
1360 }
1361
1362 if (!is_klass_initialized(mask_klass)) {
1363 log_if_needed(" ** mask klass argument not initialized");
1364 return false;
1365 }
1366
1367 if (vmask_type->maybe_null()) {
1368 log_if_needed(" ** null mask values are not allowed for masked op");
1369 return false;
1370 }
1371
1372 // Check whether the predicated gather/scatter node is supported by architecture.
1373 VectorMaskUseType mask = (VectorMaskUseType) (VecMaskUseLoad | VecMaskUsePred);
1374 if (!arch_supports_vector(is_scatter ? Op_StoreVectorScatterMasked : Op_LoadVectorGatherMasked, num_elem, elem_bt, mask)) {
1375 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d etype=%s is_masked_op=1",
1376 is_scatter, is_scatter ? "scatterMasked" : "gatherMasked",
1377 num_elem, type2name(elem_bt));
1378 return false; // not supported
1379 }
1380 } else {
1381 // Check whether the normal gather/scatter node is supported for non-masked operation.
1382 if (!arch_supports_vector(is_scatter ? Op_StoreVectorScatter : Op_LoadVectorGather, num_elem, elem_bt, VecMaskNotUsed)) {
1383 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d etype=%s is_masked_op=0",
1384 is_scatter, is_scatter ? "scatter" : "gather",
1385 num_elem, type2name(elem_bt));
1386 return false; // not supported
1387 }
1388 }
1389
1390 // Check that the vector holding indices is supported by architecture
1391 // For sub-word gathers expander receive index array.
1392 if (!is_subword_type(elem_bt) && !arch_supports_vector(Op_LoadVector, num_elem, T_INT, VecMaskNotUsed)) {
1393 log_if_needed(" ** not supported: arity=%d op=%s/loadindex vlen=%d etype=int is_masked_op=%d",
1394 is_scatter, is_scatter ? "scatter" : "gather",
1395 num_elem, is_masked_op ? 1 : 0);
1396 return false; // not supported
1397 }
1398
1399 Node* base = argument(5);
1400 Node* offset = ConvL2X(argument(6));
1401
1402 // Save state and restore on bailout
1403 uint old_sp = sp();
1404 SafePointNode* old_map = clone_map();
1405
1406 Node* addr = make_unsafe_address(base, offset, elem_bt, true);
1407
1408 const TypePtr *addr_type = gvn().type(addr)->isa_ptr();
1409 const TypeAryPtr* arr_type = addr_type->isa_aryptr();
1410
1411 // The array must be consistent with vector type
1412 if (arr_type == nullptr || (arr_type != nullptr && !elem_consistent_with_arr(elem_bt, arr_type, false))) {
1413 log_if_needed(" ** not supported: arity=%d op=%s vlen=%d etype=%s atype=%s ismask=no",
1414 is_scatter, is_scatter ? "scatter" : "gather",
1415 num_elem, type2name(elem_bt), type2name(arr_type->elem()->array_element_basic_type()));
1416 set_map(old_map);
1417 set_sp(old_sp);
1418 return false;
1419 }
1420
1421 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1422 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1423 ciKlass* vbox_idx_klass = vector_idx_klass->const_oop()->as_instance()->java_lang_Class_klass();
1424 if (vbox_idx_klass == nullptr) {
1425 set_map(old_map);
1426 set_sp(old_sp);
1427 return false;
1428 }
1429
1430 Node* index_vect = nullptr;
1431 const TypeInstPtr* vbox_idx_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_idx_klass);
1432 if (!is_subword_type(elem_bt)) {
1433 index_vect = unbox_vector(argument(8), vbox_idx_type, T_INT, num_elem);
1434 if (index_vect == nullptr) {
1435 set_map(old_map);
1436 set_sp(old_sp);
1437 return false;
1438 }
1439 }
1440
1441 Node* mask = nullptr;
1442 if (is_masked_op) {
1443 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1444 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1445 mask = unbox_vector(is_scatter ? argument(10) : argument(9), mbox_type, elem_bt, num_elem);
1446 if (mask == nullptr) {
1447 log_if_needed(" ** unbox failed mask=%s",
1448 is_scatter ? NodeClassNames[argument(10)->Opcode()]
1449 : NodeClassNames[argument(9)->Opcode()]);
1450 set_map(old_map);
1451 set_sp(old_sp);
1452 return false;
1453 }
1454 }
1455
1456 const TypeVect* vector_type = TypeVect::make(elem_bt, num_elem);
1457 if (is_scatter) {
1458 Node* val = unbox_vector(argument(9), vbox_type, elem_bt, num_elem);
1459 if (val == nullptr) {
1460 set_map(old_map);
1461 set_sp(old_sp);
1462 return false; // operand unboxing failed
1463 }
1464 set_all_memory(reset_memory());
1465
1466 Node* vstore = nullptr;
1467 if (mask != nullptr) {
1468 vstore = gvn().transform(new StoreVectorScatterMaskedNode(control(), memory(addr), addr, addr_type, val, index_vect, mask));
1469 } else {
1470 vstore = gvn().transform(new StoreVectorScatterNode(control(), memory(addr), addr, addr_type, val, index_vect));
1471 }
1472 set_memory(vstore, addr_type);
1473 } else {
1474 Node* vload = nullptr;
1475 Node* index = argument(11);
1476 Node* indexMap = argument(12);
1477 Node* indexM = argument(13);
1478 if (mask != nullptr) {
1479 if (is_subword_type(elem_bt)) {
1480 Node* index_arr_base = array_element_address(indexMap, indexM, T_INT);
1481 vload = gvn().transform(new LoadVectorGatherMaskedNode(control(), memory(addr), addr, addr_type, vector_type, index_arr_base, mask, index));
1482 } else {
1483 vload = gvn().transform(new LoadVectorGatherMaskedNode(control(), memory(addr), addr, addr_type, vector_type, index_vect, mask));
1484 }
1485 } else {
1486 if (is_subword_type(elem_bt)) {
1487 Node* index_arr_base = array_element_address(indexMap, indexM, T_INT);
1488 vload = gvn().transform(new LoadVectorGatherNode(control(), memory(addr), addr, addr_type, vector_type, index_arr_base, index));
1489 } else {
1490 vload = gvn().transform(new LoadVectorGatherNode(control(), memory(addr), addr, addr_type, vector_type, index_vect));
1491 }
1492 }
1493 Node* box = box_vector(vload, vbox_type, elem_bt, num_elem);
1494 set_result(box);
1495 }
1496
1497 destruct_map_clone(old_map);
1498
1499 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1500 return true;
1501 }
1502
1503 // public static
1504 // <V extends Vector<E>,
1505 // M extends VectorMask<E>,
1506 // E>
1507 // long reductionCoerced(int oprId, Class<? extends V> vectorClass, Class<? extends M> maskClass,
1508 // Class<E> elementType, int length, V v, M m,
1509 // ReductionOperation<V, M> defaultImpl)
1510 bool LibraryCallKit::inline_vector_reduction() {
1511 const TypeInt* opr = gvn().type(argument(0))->isa_int();
1512 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
1513 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
1514 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
1515 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
1516
1517 if (opr == nullptr || vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr ||
1518 !opr->is_con() || vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
1519 log_if_needed(" ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
1520 NodeClassNames[argument(0)->Opcode()],
1521 NodeClassNames[argument(1)->Opcode()],
1522 NodeClassNames[argument(3)->Opcode()],
1523 NodeClassNames[argument(4)->Opcode()]);
1524 return false; // not enough info for intrinsification
1525 }
1526 if (!is_klass_initialized(vector_klass)) {
1527 log_if_needed(" ** klass argument not initialized");
1528 return false;
1529 }
1530 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1531 if (!elem_type->is_primitive_type()) {
1532 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1533 return false; // should be primitive type
1534 }
1535
1536 const Type* vmask_type = gvn().type(argument(6));
1537 bool is_masked_op = vmask_type != TypePtr::NULL_PTR;
1538 if (is_masked_op) {
1539 if (mask_klass == nullptr || mask_klass->const_oop() == nullptr) {
1540 log_if_needed(" ** missing constant: maskclass=%s", NodeClassNames[argument(2)->Opcode()]);
1541 return false; // not enough info for intrinsification
1542 }
1543
1544 if (!is_klass_initialized(mask_klass)) {
1545 log_if_needed(" ** mask klass argument not initialized");
1546 return false;
1547 }
1548
1549 if (vmask_type->maybe_null()) {
1550 log_if_needed(" ** null mask values are not allowed for masked op");
1551 return false;
1552 }
1553 }
1554
1555 BasicType elem_bt = elem_type->basic_type();
1556 int num_elem = vlen->get_con();
1557 int opc = VectorSupport::vop2ideal(opr->get_con(), elem_bt);
1558 int sopc = ReductionNode::opcode(opc, elem_bt);
1559
1560 // When using mask, mask use type needs to be VecMaskUseLoad.
1561 if (!arch_supports_vector(sopc, num_elem, elem_bt, is_masked_op ? VecMaskUseLoad : VecMaskNotUsed)) {
1562 log_if_needed(" ** not supported: arity=1 op=%d/reduce vlen=%d etype=%s is_masked_op=%d",
1563 sopc, num_elem, type2name(elem_bt), is_masked_op ? 1 : 0);
1564 return false;
1565 }
1566
1567 // Return true if current platform has implemented the masked operation with predicate feature.
1568 bool use_predicate = is_masked_op && arch_supports_vector(sopc, num_elem, elem_bt, VecMaskUsePred);
1569 if (is_masked_op && !use_predicate && !arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad)) {
1570 log_if_needed(" ** not supported: arity=1 op=%d/reduce vlen=%d etype=%s is_masked_op=1",
1571 sopc, num_elem, type2name(elem_bt));
1572 return false;
1573 }
1574
1575 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1576 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1577
1578 Node* opd = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
1579 if (opd == nullptr) {
1580 return false; // operand unboxing failed
1581 }
1582
1583 Node* mask = nullptr;
1584 if (is_masked_op) {
1585 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1586 assert(is_vector_mask(mbox_klass), "argument(2) should be a mask class");
1587 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1588 mask = unbox_vector(argument(6), mbox_type, elem_bt, num_elem);
1589 if (mask == nullptr) {
1590 log_if_needed(" ** unbox failed mask=%s",
1591 NodeClassNames[argument(6)->Opcode()]);
1592 return false;
1593 }
1594 }
1595
1596 Node* init = ReductionNode::make_identity_con_scalar(gvn(), opc, elem_bt);
1597 Node* value = nullptr;
1598 if (mask == nullptr) {
1599 assert(!is_masked_op, "Masked op needs the mask value never null");
1600 value = ReductionNode::make(opc, nullptr, init, opd, elem_bt);
1601 } else {
1602 if (use_predicate) {
1603 value = ReductionNode::make(opc, nullptr, init, opd, elem_bt);
1604 value->add_req(mask);
1605 value->add_flag(Node::Flag_is_predicated_vector);
1606 } else {
1607 Node* reduce_identity = gvn().transform(VectorNode::scalar2vector(init, num_elem, Type::get_const_basic_type(elem_bt)));
1608 value = gvn().transform(new VectorBlendNode(reduce_identity, opd, mask));
1609 value = ReductionNode::make(opc, nullptr, init, value, elem_bt);
1610 }
1611 }
1612 value = gvn().transform(value);
1613
1614 Node* bits = nullptr;
1615 switch (elem_bt) {
1616 case T_BYTE:
1617 case T_SHORT:
1618 case T_INT: {
1619 bits = gvn().transform(new ConvI2LNode(value));
1620 break;
1621 }
1622 case T_FLOAT: {
1623 value = gvn().transform(new MoveF2INode(value));
1624 bits = gvn().transform(new ConvI2LNode(value));
1625 break;
1626 }
1627 case T_DOUBLE: {
1628 bits = gvn().transform(new MoveD2LNode(value));
1629 break;
1630 }
1631 case T_LONG: {
1632 bits = value; // no conversion needed
1633 break;
1634 }
1635 default: fatal("%s", type2name(elem_bt));
1636 }
1637 set_result(bits);
1638 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1639 return true;
1640 }
1641
1642 // public static <V> boolean test(int cond, Class<?> vectorClass, Class<?> elementType, int vlen,
1643 // V v1, V v2,
1644 // BiFunction<V, V, Boolean> defaultImpl)
1645 //
1646 bool LibraryCallKit::inline_vector_test() {
1647 const TypeInt* cond = gvn().type(argument(0))->isa_int();
1648 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
1649 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
1650 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
1651
1652 if (cond == nullptr || vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr ||
1653 !cond->is_con() || vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
1654 log_if_needed(" ** missing constant: cond=%s vclass=%s etype=%s vlen=%s",
1655 NodeClassNames[argument(0)->Opcode()],
1656 NodeClassNames[argument(1)->Opcode()],
1657 NodeClassNames[argument(2)->Opcode()],
1658 NodeClassNames[argument(3)->Opcode()]);
1659 return false; // not enough info for intrinsification
1660 }
1661 if (!is_klass_initialized(vector_klass)) {
1662 log_if_needed(" ** klass argument not initialized");
1663 return false;
1664 }
1665 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1666 if (!elem_type->is_primitive_type()) {
1667 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1668 return false; // should be primitive type
1669 }
1670 BasicType elem_bt = elem_type->basic_type();
1671 int num_elem = vlen->get_con();
1672 BoolTest::mask booltest = (BoolTest::mask)cond->get_con();
1673 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1674 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1675
1676 if (!arch_supports_vector(Op_VectorTest, num_elem, elem_bt, is_vector_mask(vbox_klass) ? VecMaskUseLoad : VecMaskNotUsed)) {
1677 log_if_needed(" ** not supported: arity=2 op=test/%d vlen=%d etype=%s ismask=%d",
1678 cond->get_con(), num_elem, type2name(elem_bt),
1679 is_vector_mask(vbox_klass));
1680 return false;
1681 }
1682
1683 Node* opd1 = unbox_vector(argument(4), vbox_type, elem_bt, num_elem);
1684 Node* opd2;
1685 if (Matcher::vectortest_needs_second_argument(booltest == BoolTest::overflow,
1686 opd1->bottom_type()->isa_vectmask())) {
1687 opd2 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
1688 } else {
1689 opd2 = opd1;
1690 }
1691 if (opd1 == nullptr || opd2 == nullptr) {
1692 return false; // operand unboxing failed
1693 }
1694
1695 Node* cmp = gvn().transform(new VectorTestNode(opd1, opd2, booltest));
1696 BoolTest::mask test = Matcher::vectortest_mask(booltest == BoolTest::overflow,
1697 opd1->bottom_type()->isa_vectmask(), num_elem);
1698 Node* bol = gvn().transform(new BoolNode(cmp, test));
1699 Node* res = gvn().transform(new CMoveINode(bol, gvn().intcon(0), gvn().intcon(1), TypeInt::BOOL));
1700
1701 set_result(res);
1702 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1703 return true;
1704 }
1705
1706 // public static
1707 // <V extends Vector<E>,
1708 // M extends VectorMask<E>,
1709 // E>
1710 // V blend(Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType, int vlen,
1711 // V v1, V v2, M m,
1712 // VectorBlendOp<V, M, E> defaultImpl)
1713 bool LibraryCallKit::inline_vector_blend() {
1714 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
1715 const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
1716 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
1717 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
1718
1719 if (mask_klass == nullptr || vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr) {
1720 return false; // dead code
1721 }
1722 if (mask_klass->const_oop() == nullptr || vector_klass->const_oop() == nullptr ||
1723 elem_klass->const_oop() == nullptr || !vlen->is_con()) {
1724 log_if_needed(" ** missing constant: vclass=%s mclass=%s etype=%s vlen=%s",
1725 NodeClassNames[argument(0)->Opcode()],
1726 NodeClassNames[argument(1)->Opcode()],
1727 NodeClassNames[argument(2)->Opcode()],
1728 NodeClassNames[argument(3)->Opcode()]);
1729 return false; // not enough info for intrinsification
1730 }
1731 if (!is_klass_initialized(vector_klass) || !is_klass_initialized(mask_klass)) {
1732 log_if_needed(" ** klass argument not initialized");
1733 return false;
1734 }
1735 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1736 if (!elem_type->is_primitive_type()) {
1737 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1738 return false; // should be primitive type
1739 }
1740 BasicType elem_bt = elem_type->basic_type();
1741 BasicType mask_bt = elem_bt;
1742 int num_elem = vlen->get_con();
1743
1744 if (!arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad)) {
1745 log_if_needed(" ** not supported: arity=2 op=blend vlen=%d etype=%s ismask=useload",
1746 num_elem, type2name(elem_bt));
1747 return false; // not supported
1748 }
1749 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1750 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1751
1752 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1753 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1754
1755 Node* v1 = unbox_vector(argument(4), vbox_type, elem_bt, num_elem);
1756 Node* v2 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
1757 Node* mask = unbox_vector(argument(6), mbox_type, mask_bt, num_elem);
1758
1759 if (v1 == nullptr || v2 == nullptr || mask == nullptr) {
1760 return false; // operand unboxing failed
1761 }
1762
1763 Node* blend = gvn().transform(new VectorBlendNode(v1, v2, mask));
1764
1765 Node* box = box_vector(blend, vbox_type, elem_bt, num_elem);
1766 set_result(box);
1767 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1768 return true;
1769 }
1770
1771 // public static
1772 // <V extends Vector<E>,
1773 // M extends VectorMask<E>,
1774 // E>
1775 // M compare(int cond, Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType, int vlen,
1776 // V v1, V v2, M m,
1777 // VectorCompareOp<V,M> defaultImpl)
1778 bool LibraryCallKit::inline_vector_compare() {
1779 const TypeInt* cond = gvn().type(argument(0))->isa_int();
1780 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
1781 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
1782 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
1783 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
1784
1785 if (cond == nullptr || vector_klass == nullptr || mask_klass == nullptr || elem_klass == nullptr || vlen == nullptr) {
1786 return false; // dead code
1787 }
1788 if (!cond->is_con() || vector_klass->const_oop() == nullptr || mask_klass->const_oop() == nullptr ||
1789 elem_klass->const_oop() == nullptr || !vlen->is_con()) {
1790 log_if_needed(" ** missing constant: cond=%s vclass=%s mclass=%s etype=%s vlen=%s",
1791 NodeClassNames[argument(0)->Opcode()],
1792 NodeClassNames[argument(1)->Opcode()],
1793 NodeClassNames[argument(2)->Opcode()],
1794 NodeClassNames[argument(3)->Opcode()],
1795 NodeClassNames[argument(4)->Opcode()]);
1796 return false; // not enough info for intrinsification
1797 }
1798 if (!is_klass_initialized(vector_klass) || !is_klass_initialized(mask_klass)) {
1799 log_if_needed(" ** klass argument not initialized");
1800 return false;
1801 }
1802 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1803 if (!elem_type->is_primitive_type()) {
1804 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1805 return false; // should be primitive type
1806 }
1807
1808 int num_elem = vlen->get_con();
1809 BasicType elem_bt = elem_type->basic_type();
1810 BasicType mask_bt = elem_bt;
1811
1812 if ((cond->get_con() & BoolTest::unsigned_compare) != 0) {
1813 if (!Matcher::supports_vector_comparison_unsigned(num_elem, elem_bt)) {
1814 log_if_needed(" ** not supported: unsigned comparison op=comp/%d vlen=%d etype=%s ismask=usestore",
1815 cond->get_con() & (BoolTest::unsigned_compare - 1), num_elem, type2name(elem_bt));
1816 return false;
1817 }
1818 }
1819
1820 if (!arch_supports_vector(Op_VectorMaskCmp, num_elem, elem_bt, VecMaskUseStore)) {
1821 log_if_needed(" ** not supported: arity=2 op=comp/%d vlen=%d etype=%s ismask=usestore",
1822 cond->get_con(), num_elem, type2name(elem_bt));
1823 return false;
1824 }
1825
1826 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1827 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1828
1829 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1830 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1831
1832 Node* v1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
1833 Node* v2 = unbox_vector(argument(6), vbox_type, elem_bt, num_elem);
1834
1835 bool is_masked_op = argument(7)->bottom_type() != TypePtr::NULL_PTR;
1836 Node* mask = is_masked_op ? unbox_vector(argument(7), mbox_type, elem_bt, num_elem) : nullptr;
1837 if (is_masked_op && mask == nullptr) {
1838 log_if_needed(" ** not supported: mask = null arity=2 op=comp/%d vlen=%d etype=%s ismask=usestore is_masked_op=1",
1839 cond->get_con(), num_elem, type2name(elem_bt));
1840 return false;
1841 }
1842
1843 bool use_predicate = is_masked_op && arch_supports_vector(Op_VectorMaskCmp, num_elem, elem_bt, VecMaskUsePred);
1844 if (is_masked_op && !use_predicate && !arch_supports_vector(Op_AndV, num_elem, elem_bt, VecMaskUseLoad)) {
1845 log_if_needed(" ** not supported: arity=2 op=comp/%d vlen=%d etype=%s ismask=usestore is_masked_op=1",
1846 cond->get_con(), num_elem, type2name(elem_bt));
1847 return false;
1848 }
1849
1850 if (v1 == nullptr || v2 == nullptr) {
1851 return false; // operand unboxing failed
1852 }
1853 BoolTest::mask pred = (BoolTest::mask)cond->get_con();
1854 ConINode* pred_node = (ConINode*)gvn().makecon(cond);
1855
1856 const TypeVect* vmask_type = TypeVect::makemask(mask_bt, num_elem);
1857 Node* operation = new VectorMaskCmpNode(pred, v1, v2, pred_node, vmask_type);
1858
1859 if (is_masked_op) {
1860 if (use_predicate) {
1861 operation->add_req(mask);
1862 operation->add_flag(Node::Flag_is_predicated_vector);
1863 } else {
1864 operation = gvn().transform(operation);
1865 operation = VectorNode::make(Op_AndV, operation, mask, vmask_type);
1866 }
1867 }
1868
1869 operation = gvn().transform(operation);
1870
1871 Node* box = box_vector(operation, mbox_type, mask_bt, num_elem);
1872 set_result(box);
1873 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1874 return true;
1875 }
1876
1877 // public static
1878 // <V extends Vector<E>,
1879 // Sh extends VectorShuffle<E>,
1880 // M extends VectorMask<E>,
1881 // E>
1882 // V rearrangeOp(Class<? extends V> vectorClass, Class<Sh> shuffleClass, Class<M> maskClass, Class<E> elementType, int vlen,
1883 // V v1, Sh sh, M m,
1884 // VectorRearrangeOp<V, Sh, M, E> defaultImpl)
1885 bool LibraryCallKit::inline_vector_rearrange() {
1886 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
1887 const TypeInstPtr* shuffle_klass = gvn().type(argument(1))->isa_instptr();
1888 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
1889 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
1890 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
1891
1892 if (vector_klass == nullptr || shuffle_klass == nullptr || elem_klass == nullptr || vlen == nullptr) {
1893 return false; // dead code
1894 }
1895 if (shuffle_klass->const_oop() == nullptr ||
1896 vector_klass->const_oop() == nullptr ||
1897 elem_klass->const_oop() == nullptr ||
1898 !vlen->is_con()) {
1899 log_if_needed(" ** missing constant: vclass=%s sclass=%s etype=%s vlen=%s",
1900 NodeClassNames[argument(0)->Opcode()],
1901 NodeClassNames[argument(1)->Opcode()],
1902 NodeClassNames[argument(3)->Opcode()],
1903 NodeClassNames[argument(4)->Opcode()]);
1904 return false; // not enough info for intrinsification
1905 }
1906 if (!is_klass_initialized(vector_klass) ||
1907 !is_klass_initialized(shuffle_klass)) {
1908 log_if_needed(" ** klass argument not initialized");
1909 return false;
1910 }
1911 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1912 if (!elem_type->is_primitive_type()) {
1913 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
1914 return false; // should be primitive type
1915 }
1916 BasicType elem_bt = elem_type->basic_type();
1917 BasicType shuffle_bt = elem_bt;
1918 int num_elem = vlen->get_con();
1919
1920 if (!arch_supports_vector(Op_VectorLoadShuffle, num_elem, elem_bt, VecMaskNotUsed)) {
1921 log_if_needed(" ** not supported: arity=0 op=load/shuffle vlen=%d etype=%s ismask=no",
1922 num_elem, type2name(elem_bt));
1923 return false; // not supported
1924 }
1925
1926 bool is_masked_op = argument(7)->bottom_type() != TypePtr::NULL_PTR;
1927 bool use_predicate = is_masked_op;
1928 if (is_masked_op &&
1929 (mask_klass == nullptr ||
1930 mask_klass->const_oop() == nullptr ||
1931 !is_klass_initialized(mask_klass))) {
1932 log_if_needed(" ** mask_klass argument not initialized");
1933 }
1934 VectorMaskUseType checkFlags = (VectorMaskUseType)(is_masked_op ? (VecMaskUseLoad | VecMaskUsePred) : VecMaskNotUsed);
1935 if (!arch_supports_vector(Op_VectorRearrange, num_elem, elem_bt, checkFlags)) {
1936 use_predicate = false;
1937 if(!is_masked_op ||
1938 (!arch_supports_vector(Op_VectorRearrange, num_elem, elem_bt, VecMaskNotUsed) ||
1939 !arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad) ||
1940 !arch_supports_vector(Op_Replicate, num_elem, elem_bt, VecMaskNotUsed))) {
1941 log_if_needed(" ** not supported: arity=2 op=shuffle/rearrange vlen=%d etype=%s ismask=no",
1942 num_elem, type2name(elem_bt));
1943 return false; // not supported
1944 }
1945 }
1946 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1947 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1948
1949 ciKlass* shbox_klass = shuffle_klass->const_oop()->as_instance()->java_lang_Class_klass();
1950 const TypeInstPtr* shbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, shbox_klass);
1951
1952 Node* v1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
1953 Node* shuffle = unbox_vector(argument(6), shbox_type, shuffle_bt, num_elem);
1954
1955 if (v1 == nullptr || shuffle == nullptr) {
1956 return false; // operand unboxing failed
1957 }
1958
1959 Node* mask = nullptr;
1960 if (is_masked_op) {
1961 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1962 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1963 mask = unbox_vector(argument(7), mbox_type, elem_bt, num_elem);
1964 if (mask == nullptr) {
1965 log_if_needed(" ** not supported: arity=3 op=shuffle/rearrange vlen=%d etype=%s ismask=useload is_masked_op=1",
1966 num_elem, type2name(elem_bt));
1967 return false;
1968 }
1969 }
1970
1971 Node* rearrange = new VectorRearrangeNode(v1, shuffle);
1972 if (is_masked_op) {
1973 if (use_predicate) {
1974 rearrange->add_req(mask);
1975 rearrange->add_flag(Node::Flag_is_predicated_vector);
1976 } else {
1977 const TypeVect* vt = v1->bottom_type()->is_vect();
1978 rearrange = gvn().transform(rearrange);
1979 Node* zero = gvn().makecon(Type::get_zero_type(elem_bt));
1980 Node* zerovec = gvn().transform(VectorNode::scalar2vector(zero, num_elem, Type::get_const_basic_type(elem_bt)));
1981 rearrange = new VectorBlendNode(zerovec, rearrange, mask);
1982 }
1983 }
1984 rearrange = gvn().transform(rearrange);
1985
1986 Node* box = box_vector(rearrange, vbox_type, elem_bt, num_elem);
1987 set_result(box);
1988 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1989 return true;
1990 }
1991
1992 static address get_svml_address(int vop, int bits, BasicType bt, char* name_ptr, int name_len) {
1993 address addr = nullptr;
1994 assert(UseVectorStubs, "sanity");
1995 assert(name_ptr != nullptr, "unexpected");
1996 assert((vop >= VectorSupport::VECTOR_OP_SVML_START) && (vop <= VectorSupport::VECTOR_OP_SVML_END), "unexpected");
1997 int op = vop - VectorSupport::VECTOR_OP_SVML_START;
1998
1999 switch(bits) {
2000 case 64: //fallthough
2001 case 128: //fallthough
2002 case 256: //fallthough
2003 case 512:
2004 if (bt == T_FLOAT) {
2005 snprintf(name_ptr, name_len, "vector_%s_float%d", VectorSupport::svmlname[op], bits);
2006 addr = StubRoutines::_vector_f_math[exact_log2(bits/64)][op];
2007 } else {
2008 assert(bt == T_DOUBLE, "must be FP type only");
2009 snprintf(name_ptr, name_len, "vector_%s_double%d", VectorSupport::svmlname[op], bits);
2010 addr = StubRoutines::_vector_d_math[exact_log2(bits/64)][op];
2011 }
2012 break;
2013 default:
2014 snprintf(name_ptr, name_len, "invalid");
2015 addr = nullptr;
2016 Unimplemented();
2017 break;
2018 }
2019
2020 return addr;
2021 }
2022
2023 Node* LibraryCallKit::gen_call_to_svml(int vector_api_op_id, BasicType bt, int num_elem, Node* opd1, Node* opd2) {
2024 assert(UseVectorStubs, "sanity");
2025 assert(vector_api_op_id >= VectorSupport::VECTOR_OP_SVML_START && vector_api_op_id <= VectorSupport::VECTOR_OP_SVML_END, "need valid op id");
2026 assert(opd1 != nullptr, "must not be null");
2027 const TypeVect* vt = TypeVect::make(bt, num_elem);
2028 const TypeFunc* call_type = OptoRuntime::Math_Vector_Vector_Type(opd2 != nullptr ? 2 : 1, vt, vt);
2029 char name[100] = "";
2030
2031 // Get address for svml method.
2032 address addr = get_svml_address(vector_api_op_id, vt->length_in_bytes() * BitsPerByte, bt, name, 100);
2033
2034 if (addr == nullptr) {
2035 return nullptr;
2036 }
2037
2038 assert(name[0] != '\0', "name must not be null");
2039 Node* operation = make_runtime_call(RC_VECTOR,
2040 call_type,
2041 addr,
2042 name,
2043 TypePtr::BOTTOM,
2044 opd1,
2045 opd2);
2046 return gvn().transform(new ProjNode(gvn().transform(operation), TypeFunc::Parms));
2047 }
2048
2049 // public static
2050 // <V extends Vector<E>,
2051 // M extends VectorMask<E>,
2052 // E>
2053 // V broadcastInt(int opr, Class<? extends V> vectorClass, Class<? extends M> maskClass,
2054 // Class<E> elementType, int length,
2055 // V v, int n, M m,
2056 // VectorBroadcastIntOp<V, M> defaultImpl)
2057 bool LibraryCallKit::inline_vector_broadcast_int() {
2058 const TypeInt* opr = gvn().type(argument(0))->isa_int();
2059 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
2060 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
2061 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
2062 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
2063
2064 if (opr == nullptr || vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr) {
2065 return false; // dead code
2066 }
2067 if (!opr->is_con() || vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
2068 log_if_needed(" ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
2069 NodeClassNames[argument(0)->Opcode()],
2070 NodeClassNames[argument(1)->Opcode()],
2071 NodeClassNames[argument(3)->Opcode()],
2072 NodeClassNames[argument(4)->Opcode()]);
2073 return false; // not enough info for intrinsification
2074 }
2075 if (!is_klass_initialized(vector_klass)) {
2076 log_if_needed(" ** klass argument not initialized");
2077 return false;
2078 }
2079
2080 const Type* vmask_type = gvn().type(argument(7));
2081 bool is_masked_op = vmask_type != TypePtr::NULL_PTR;
2082 if (is_masked_op) {
2083 if (mask_klass == nullptr || mask_klass->const_oop() == nullptr) {
2084 log_if_needed(" ** missing constant: maskclass=%s", NodeClassNames[argument(2)->Opcode()]);
2085 return false; // not enough info for intrinsification
2086 }
2087
2088 if (!is_klass_initialized(mask_klass)) {
2089 log_if_needed(" ** mask klass argument not initialized");
2090 return false;
2091 }
2092
2093 if (vmask_type->maybe_null()) {
2094 log_if_needed(" ** null mask values are not allowed for masked op");
2095 return false;
2096 }
2097 }
2098
2099 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2100 if (!elem_type->is_primitive_type()) {
2101 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2102 return false; // should be primitive type
2103 }
2104
2105 int num_elem = vlen->get_con();
2106 BasicType elem_bt = elem_type->basic_type();
2107 int opc = VectorSupport::vop2ideal(opr->get_con(), elem_bt);
2108
2109 bool is_shift = VectorNode::is_shift_opcode(opc);
2110 bool is_rotate = VectorNode::is_rotate_opcode(opc);
2111
2112 if (opc == 0 || (!is_shift && !is_rotate)) {
2113 log_if_needed(" ** operation not supported: op=%d bt=%s", opr->get_con(), type2name(elem_bt));
2114 return false; // operation not supported
2115 }
2116
2117 int sopc = VectorNode::opcode(opc, elem_bt);
2118 if (sopc == 0) {
2119 log_if_needed(" ** operation not supported: opc=%s bt=%s", NodeClassNames[opc], type2name(elem_bt));
2120 return false; // operation not supported
2121 }
2122
2123 Node* cnt = argument(6);
2124 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2125 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2126 const TypeInt* cnt_type = cnt->bottom_type()->isa_int();
2127
2128 // If CPU supports vector constant rotate instructions pass it directly
2129 bool is_const_rotate = is_rotate && cnt_type && cnt_type->is_con() &&
2130 Matcher::supports_vector_constant_rotates(cnt_type->get_con());
2131 bool has_scalar_args = is_rotate ? !is_const_rotate : true;
2132
2133 VectorMaskUseType checkFlags = (VectorMaskUseType)(is_masked_op ? (VecMaskUseLoad | VecMaskUsePred) : VecMaskNotUsed);
2134 bool use_predicate = is_masked_op;
2135
2136 if (!arch_supports_vector(sopc, num_elem, elem_bt, checkFlags, has_scalar_args)) {
2137 use_predicate = false;
2138 if (!is_masked_op ||
2139 (!arch_supports_vector(sopc, num_elem, elem_bt, VecMaskNotUsed, has_scalar_args) ||
2140 !arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad))) {
2141
2142 log_if_needed(" ** not supported: arity=0 op=int/%d vlen=%d etype=%s is_masked_op=%d",
2143 sopc, num_elem, type2name(elem_bt), is_masked_op ? 1 : 0);
2144 return false; // not supported
2145 }
2146 }
2147
2148 Node* opd1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
2149 Node* opd2 = nullptr;
2150 if (is_shift) {
2151 opd2 = vector_shift_count(cnt, opc, elem_bt, num_elem);
2152 } else {
2153 assert(is_rotate, "unexpected operation");
2154 if (!is_const_rotate) {
2155 const Type * type_bt = Type::get_const_basic_type(elem_bt);
2156 cnt = elem_bt == T_LONG ? gvn().transform(new ConvI2LNode(cnt)) : cnt;
2157 opd2 = gvn().transform(VectorNode::scalar2vector(cnt, num_elem, type_bt));
2158 } else {
2159 // Constant shift value.
2160 opd2 = cnt;
2161 }
2162 }
2163
2164 if (opd1 == nullptr || opd2 == nullptr) {
2165 return false;
2166 }
2167
2168 Node* mask = nullptr;
2169 if (is_masked_op) {
2170 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
2171 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
2172 mask = unbox_vector(argument(7), mbox_type, elem_bt, num_elem);
2173 if (mask == nullptr) {
2174 log_if_needed(" ** unbox failed mask=%s", NodeClassNames[argument(7)->Opcode()]);
2175 return false;
2176 }
2177 }
2178
2179 Node* operation = VectorNode::make(opc, opd1, opd2, num_elem, elem_bt);
2180 if (is_masked_op && mask != nullptr) {
2181 if (use_predicate) {
2182 operation->add_req(mask);
2183 operation->add_flag(Node::Flag_is_predicated_vector);
2184 } else {
2185 operation = gvn().transform(operation);
2186 operation = new VectorBlendNode(opd1, operation, mask);
2187 }
2188 }
2189 operation = gvn().transform(operation);
2190 Node* vbox = box_vector(operation, vbox_type, elem_bt, num_elem);
2191 set_result(vbox);
2192 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2193 return true;
2194 }
2195
2196 // public static <VOUT extends VectorPayload,
2197 // VIN extends VectorPayload,
2198 // S extends VectorSpecies>
2199 // VOUT convert(int oprId,
2200 // Class<?> fromVectorClass, Class<?> fromElementType, int fromVLen,
2201 // Class<?> toVectorClass, Class<?> toElementType, int toVLen,
2202 // VIN v, S s,
2203 // VectorConvertOp<VOUT, VIN, S> defaultImpl)
2204 //
2205 bool LibraryCallKit::inline_vector_convert() {
2206 const TypeInt* opr = gvn().type(argument(0))->isa_int();
2207
2208 const TypeInstPtr* vector_klass_from = gvn().type(argument(1))->isa_instptr();
2209 const TypeInstPtr* elem_klass_from = gvn().type(argument(2))->isa_instptr();
2210 const TypeInt* vlen_from = gvn().type(argument(3))->isa_int();
2211
2212 const TypeInstPtr* vector_klass_to = gvn().type(argument(4))->isa_instptr();
2213 const TypeInstPtr* elem_klass_to = gvn().type(argument(5))->isa_instptr();
2214 const TypeInt* vlen_to = gvn().type(argument(6))->isa_int();
2215
2216 if (opr == nullptr ||
2217 vector_klass_from == nullptr || elem_klass_from == nullptr || vlen_from == nullptr ||
2218 vector_klass_to == nullptr || elem_klass_to == nullptr || vlen_to == nullptr) {
2219 return false; // dead code
2220 }
2221 if (!opr->is_con() ||
2222 vector_klass_from->const_oop() == nullptr || elem_klass_from->const_oop() == nullptr || !vlen_from->is_con() ||
2223 vector_klass_to->const_oop() == nullptr || elem_klass_to->const_oop() == nullptr || !vlen_to->is_con()) {
2224 log_if_needed(" ** missing constant: opr=%s vclass_from=%s etype_from=%s vlen_from=%s vclass_to=%s etype_to=%s vlen_to=%s",
2225 NodeClassNames[argument(0)->Opcode()],
2226 NodeClassNames[argument(1)->Opcode()],
2227 NodeClassNames[argument(2)->Opcode()],
2228 NodeClassNames[argument(3)->Opcode()],
2229 NodeClassNames[argument(4)->Opcode()],
2230 NodeClassNames[argument(5)->Opcode()],
2231 NodeClassNames[argument(6)->Opcode()]);
2232 return false; // not enough info for intrinsification
2233 }
2234 if (!is_klass_initialized(vector_klass_from) || !is_klass_initialized(vector_klass_to)) {
2235 log_if_needed(" ** klass argument not initialized");
2236 return false;
2237 }
2238
2239 assert(opr->get_con() == VectorSupport::VECTOR_OP_CAST ||
2240 opr->get_con() == VectorSupport::VECTOR_OP_UCAST ||
2241 opr->get_con() == VectorSupport::VECTOR_OP_REINTERPRET, "wrong opcode");
2242 bool is_cast = (opr->get_con() == VectorSupport::VECTOR_OP_CAST || opr->get_con() == VectorSupport::VECTOR_OP_UCAST);
2243 bool is_ucast = (opr->get_con() == VectorSupport::VECTOR_OP_UCAST);
2244
2245 ciKlass* vbox_klass_from = vector_klass_from->const_oop()->as_instance()->java_lang_Class_klass();
2246 ciKlass* vbox_klass_to = vector_klass_to->const_oop()->as_instance()->java_lang_Class_klass();
2247 if (is_vector_shuffle(vbox_klass_from)) {
2248 return false; // vector shuffles aren't supported
2249 }
2250 bool is_mask = is_vector_mask(vbox_klass_from);
2251
2252 ciType* elem_type_from = elem_klass_from->const_oop()->as_instance()->java_mirror_type();
2253 if (!elem_type_from->is_primitive_type()) {
2254 return false; // should be primitive type
2255 }
2256 BasicType elem_bt_from = elem_type_from->basic_type();
2257 ciType* elem_type_to = elem_klass_to->const_oop()->as_instance()->java_mirror_type();
2258 if (!elem_type_to->is_primitive_type()) {
2259 return false; // should be primitive type
2260 }
2261 BasicType elem_bt_to = elem_type_to->basic_type();
2262
2263 int num_elem_from = vlen_from->get_con();
2264 int num_elem_to = vlen_to->get_con();
2265
2266 // Check whether we can unbox to appropriate size. Even with casting, checking for reinterpret is needed
2267 // since we may need to change size.
2268 if (!arch_supports_vector(Op_VectorReinterpret,
2269 num_elem_from,
2270 elem_bt_from,
2271 is_mask ? VecMaskUseAll : VecMaskNotUsed)) {
2272 log_if_needed(" ** not supported: arity=1 op=%s/1 vlen1=%d etype1=%s ismask=%d",
2273 is_cast ? "cast" : "reinterpret",
2274 num_elem_from, type2name(elem_bt_from), is_mask);
2275 return false;
2276 }
2277
2278 // Check whether we can support resizing/reinterpreting to the new size.
2279 if (!arch_supports_vector(Op_VectorReinterpret,
2280 num_elem_to,
2281 elem_bt_to,
2282 is_mask ? VecMaskUseAll : VecMaskNotUsed)) {
2283 log_if_needed(" ** not supported: arity=1 op=%s/2 vlen2=%d etype2=%s ismask=%d",
2284 is_cast ? "cast" : "reinterpret",
2285 num_elem_to, type2name(elem_bt_to), is_mask);
2286 return false;
2287 }
2288
2289 // At this point, we know that both input and output vector registers are supported
2290 // by the architecture. Next check if the casted type is simply to same type - which means
2291 // that it is actually a resize and not a cast.
2292 if (is_cast && elem_bt_from == elem_bt_to) {
2293 is_cast = false;
2294 }
2295
2296 const TypeInstPtr* vbox_type_from = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass_from);
2297
2298 Node* opd1 = unbox_vector(argument(7), vbox_type_from, elem_bt_from, num_elem_from);
2299 if (opd1 == nullptr) {
2300 return false;
2301 }
2302
2303 const TypeVect* src_type = TypeVect::make(elem_bt_from, num_elem_from, is_mask);
2304 const TypeVect* dst_type = TypeVect::make(elem_bt_to, num_elem_to, is_mask);
2305
2306 // Safety check to prevent casting if source mask is of type vector
2307 // and destination mask of type predicate vector and vice-versa.
2308 // From X86 standpoint, this case will only arise over KNL target,
2309 // where certain masks (depending on the species) are either propagated
2310 // through a vector or predicate register.
2311 if (is_mask &&
2312 ((src_type->isa_vectmask() == nullptr && dst_type->isa_vectmask()) ||
2313 (dst_type->isa_vectmask() == nullptr && src_type->isa_vectmask()))) {
2314 return false;
2315 }
2316
2317 Node* op = opd1;
2318 if (is_cast) {
2319 assert(!is_mask || num_elem_from == num_elem_to, "vector mask cast needs the same elem num");
2320 int cast_vopc = VectorCastNode::opcode(-1, elem_bt_from, !is_ucast);
2321
2322 // Make sure that vector cast is implemented to particular type/size combination if it is
2323 // not a mask casting.
2324 if (!is_mask && !arch_supports_vector(cast_vopc, num_elem_to, elem_bt_to, VecMaskNotUsed)) {
2325 log_if_needed(" ** not supported: arity=1 op=cast#%d/3 vlen2=%d etype2=%s ismask=%d",
2326 cast_vopc, num_elem_to, type2name(elem_bt_to), is_mask);
2327 return false;
2328 }
2329
2330 if (num_elem_from < num_elem_to) {
2331 // Since input and output number of elements are not consistent, we need to make sure we
2332 // properly size. Thus, first make a cast that retains the number of elements from source.
2333 int num_elem_for_cast = num_elem_from;
2334
2335 // It is possible that arch does not support this intermediate vector size
2336 // TODO More complex logic required here to handle this corner case for the sizes.
2337 if (!arch_supports_vector(cast_vopc, num_elem_for_cast, elem_bt_to, VecMaskNotUsed)) {
2338 log_if_needed(" ** not supported: arity=1 op=cast#%d/4 vlen1=%d etype2=%s ismask=%d",
2339 cast_vopc,
2340 num_elem_for_cast, type2name(elem_bt_to), is_mask);
2341 return false;
2342 }
2343
2344 op = gvn().transform(VectorCastNode::make(cast_vopc, op, elem_bt_to, num_elem_for_cast));
2345 // Now ensure that the destination gets properly resized to needed size.
2346 op = gvn().transform(new VectorReinterpretNode(op, op->bottom_type()->is_vect(), dst_type));
2347 } else if (num_elem_from > num_elem_to) {
2348 // Since number of elements from input is larger than output, simply reduce size of input
2349 // (we are supposed to drop top elements anyway).
2350 int num_elem_for_resize = num_elem_to;
2351
2352 // It is possible that arch does not support this intermediate vector size
2353 // TODO More complex logic required here to handle this corner case for the sizes.
2354 if (!arch_supports_vector(Op_VectorReinterpret,
2355 num_elem_for_resize,
2356 elem_bt_from,
2357 VecMaskNotUsed)) {
2358 log_if_needed(" ** not supported: arity=1 op=cast/5 vlen2=%d etype1=%s ismask=%d",
2359 num_elem_for_resize, type2name(elem_bt_from), is_mask);
2360 return false;
2361 }
2362
2363 const TypeVect* resize_type = TypeVect::make(elem_bt_from, num_elem_for_resize);
2364 op = gvn().transform(new VectorReinterpretNode(op, src_type, resize_type));
2365 op = gvn().transform(VectorCastNode::make(cast_vopc, op, elem_bt_to, num_elem_to));
2366 } else { // num_elem_from == num_elem_to
2367 if (is_mask) {
2368 // Make sure that cast for vector mask is implemented to particular type/size combination.
2369 if (!arch_supports_vector(Op_VectorMaskCast, num_elem_to, elem_bt_to, VecMaskNotUsed)) {
2370 log_if_needed(" ** not supported: arity=1 op=maskcast vlen2=%d etype2=%s ismask=%d",
2371 num_elem_to, type2name(elem_bt_to), is_mask);
2372 return false;
2373 }
2374 op = gvn().transform(new VectorMaskCastNode(op, dst_type));
2375 } else {
2376 // Since input and output number of elements match, and since we know this vector size is
2377 // supported, simply do a cast with no resize needed.
2378 op = gvn().transform(VectorCastNode::make(cast_vopc, op, elem_bt_to, num_elem_to));
2379 }
2380 }
2381 } else if (!Type::equals(src_type, dst_type)) {
2382 assert(!is_cast, "must be reinterpret");
2383 op = gvn().transform(new VectorReinterpretNode(op, src_type, dst_type));
2384 }
2385
2386 const TypeInstPtr* vbox_type_to = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass_to);
2387 Node* vbox = box_vector(op, vbox_type_to, elem_bt_to, num_elem_to);
2388 set_result(vbox);
2389 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem_to * type2aelembytes(elem_bt_to))));
2390 return true;
2391 }
2392
2393 // public static
2394 // <V extends Vector<E>,
2395 // E>
2396 // V insert(Class<? extends V> vectorClass, Class<E> elementType, int vlen,
2397 // V vec, int ix, long val,
2398 // VecInsertOp<V> defaultImpl)
2399 bool LibraryCallKit::inline_vector_insert() {
2400 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
2401 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
2402 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
2403 const TypeInt* idx = gvn().type(argument(4))->isa_int();
2404
2405 if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr || idx == nullptr) {
2406 return false; // dead code
2407 }
2408 if (vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con() || !idx->is_con()) {
2409 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s idx=%s",
2410 NodeClassNames[argument(0)->Opcode()],
2411 NodeClassNames[argument(1)->Opcode()],
2412 NodeClassNames[argument(2)->Opcode()],
2413 NodeClassNames[argument(4)->Opcode()]);
2414 return false; // not enough info for intrinsification
2415 }
2416 if (!is_klass_initialized(vector_klass)) {
2417 log_if_needed(" ** klass argument not initialized");
2418 return false;
2419 }
2420 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2421 if (!elem_type->is_primitive_type()) {
2422 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2423 return false; // should be primitive type
2424 }
2425 BasicType elem_bt = elem_type->basic_type();
2426 int num_elem = vlen->get_con();
2427 if (!arch_supports_vector(Op_VectorInsert, num_elem, elem_bt, VecMaskNotUsed)) {
2428 log_if_needed(" ** not supported: arity=1 op=insert vlen=%d etype=%s ismask=no",
2429 num_elem, type2name(elem_bt));
2430 return false; // not supported
2431 }
2432
2433 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2434 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2435
2436 Node* opd = unbox_vector(argument(3), vbox_type, elem_bt, num_elem);
2437 if (opd == nullptr) {
2438 return false;
2439 }
2440
2441 Node* insert_val = argument(5);
2442 assert(gvn().type(insert_val)->isa_long() != nullptr, "expected to be long");
2443
2444 // Convert insert value back to its appropriate type.
2445 switch (elem_bt) {
2446 case T_BYTE:
2447 insert_val = gvn().transform(new ConvL2INode(insert_val));
2448 insert_val = gvn().transform(new CastIINode(insert_val, TypeInt::BYTE));
2449 break;
2450 case T_SHORT:
2451 insert_val = gvn().transform(new ConvL2INode(insert_val));
2452 insert_val = gvn().transform(new CastIINode(insert_val, TypeInt::SHORT));
2453 break;
2454 case T_INT:
2455 insert_val = gvn().transform(new ConvL2INode(insert_val));
2456 break;
2457 case T_FLOAT:
2458 insert_val = gvn().transform(new ConvL2INode(insert_val));
2459 insert_val = gvn().transform(new MoveI2FNode(insert_val));
2460 break;
2461 case T_DOUBLE:
2462 insert_val = gvn().transform(new MoveL2DNode(insert_val));
2463 break;
2464 case T_LONG:
2465 // no conversion needed
2466 break;
2467 default: fatal("%s", type2name(elem_bt)); break;
2468 }
2469
2470 Node* operation = gvn().transform(VectorInsertNode::make(opd, insert_val, idx->get_con(), gvn()));
2471
2472 Node* vbox = box_vector(operation, vbox_type, elem_bt, num_elem);
2473 set_result(vbox);
2474 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2475 return true;
2476 }
2477
2478 // public static
2479 // <VM extends VectorPayload,
2480 // E>
2481 // long extract(Class<? extends VM> vClass, Class<E> eClass,
2482 // int length,
2483 // VM vm, int i,
2484 // VecExtractOp<VM> defaultImpl)
2485 bool LibraryCallKit::inline_vector_extract() {
2486 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
2487 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
2488 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
2489 const TypeInt* idx = gvn().type(argument(4))->isa_int();
2490
2491 if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr || idx == nullptr) {
2492 return false; // dead code
2493 }
2494 if (vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
2495 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s",
2496 NodeClassNames[argument(0)->Opcode()],
2497 NodeClassNames[argument(1)->Opcode()],
2498 NodeClassNames[argument(2)->Opcode()]);
2499 return false; // not enough info for intrinsification
2500 }
2501 if (!is_klass_initialized(vector_klass)) {
2502 log_if_needed(" ** klass argument not initialized");
2503 return false;
2504 }
2505 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2506 if (!elem_type->is_primitive_type()) {
2507 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2508 return false; // should be primitive type
2509 }
2510 BasicType elem_bt = elem_type->basic_type();
2511 int num_elem = vlen->get_con();
2512
2513 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2514 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2515
2516 Node* opd = nullptr;
2517
2518 if (is_vector_mask(vbox_klass)) {
2519 // vbox_klass is mask. This is used for VectorMask.laneIsSet(int).
2520
2521 Node* pos = argument(4); // can be variable
2522 if (arch_supports_vector(Op_ExtractUB, num_elem, elem_bt, VecMaskUseAll)) {
2523 // Transform mask to vector with type of boolean and utilize ExtractUB node.
2524 opd = unbox_vector(argument(3), vbox_type, elem_bt, num_elem);
2525 if (opd == nullptr) {
2526 return false;
2527 }
2528 opd = gvn().transform(VectorStoreMaskNode::make(gvn(), opd, elem_bt, num_elem));
2529 opd = gvn().transform(new ExtractUBNode(opd, pos));
2530 opd = gvn().transform(new ConvI2LNode(opd));
2531 } else if (arch_supports_vector(Op_VectorMaskToLong, num_elem, elem_bt, VecMaskUseLoad)) {
2532 opd = unbox_vector(argument(3), vbox_type, elem_bt, num_elem);
2533 if (opd == nullptr) {
2534 return false;
2535 }
2536 // VectorMaskToLongNode requires the input is either a mask or a vector with BOOLEAN type.
2537 if (opd->bottom_type()->isa_vectmask() == nullptr) {
2538 opd = gvn().transform(VectorStoreMaskNode::make(gvn(), opd, elem_bt, num_elem));
2539 }
2540 // ((toLong() >>> pos) & 1L
2541 opd = gvn().transform(new VectorMaskToLongNode(opd, TypeLong::LONG));
2542 opd = gvn().transform(new URShiftLNode(opd, pos));
2543 opd = gvn().transform(new AndLNode(opd, gvn().makecon(TypeLong::ONE)));
2544 } else {
2545 log_if_needed(" ** Rejected mask extraction because architecture does not support it");
2546 return false; // not supported
2547 }
2548 } else {
2549 // vbox_klass is vector. This is used for Vector.lane(int).
2550 if (!idx->is_con()) {
2551 log_if_needed(" ** missing constant: idx=%s", NodeClassNames[argument(4)->Opcode()]);
2552 return false; // not enough info for intrinsification
2553 }
2554
2555 int vopc = ExtractNode::opcode(elem_bt);
2556 if (!arch_supports_vector(vopc, num_elem, elem_bt, VecMaskNotUsed)) {
2557 log_if_needed(" ** not supported: arity=1 op=extract vlen=%d etype=%s ismask=no",
2558 num_elem, type2name(elem_bt));
2559 return false; // not supported
2560 }
2561
2562 opd = unbox_vector(argument(3), vbox_type, elem_bt, num_elem);
2563 if (opd == nullptr) {
2564 return false;
2565 }
2566 ConINode* idx_con = gvn().intcon(idx->get_con())->as_ConI();
2567
2568 opd = gvn().transform(ExtractNode::make(opd, idx_con, elem_bt));
2569 switch (elem_bt) {
2570 case T_BYTE:
2571 case T_SHORT:
2572 case T_INT: {
2573 opd = gvn().transform(new ConvI2LNode(opd));
2574 break;
2575 }
2576 case T_FLOAT: {
2577 opd = gvn().transform(new MoveF2INode(opd));
2578 opd = gvn().transform(new ConvI2LNode(opd));
2579 break;
2580 }
2581 case T_DOUBLE: {
2582 opd = gvn().transform(new MoveD2LNode(opd));
2583 break;
2584 }
2585 case T_LONG: {
2586 // no conversion needed
2587 break;
2588 }
2589 default: fatal("%s", type2name(elem_bt));
2590 }
2591 }
2592 set_result(opd);
2593 return true;
2594 }
2595
2596 // public static
2597 // <V extends Vector<E>,
2598 // M extends VectorMask<E>,
2599 // E>
2600 // V compressExpandOp(int opr,
2601 // Class<? extends V> vClass, Class<? extends M> mClass, Class<E> eClass,
2602 // int length, V v, M m,
2603 // CompressExpandOperation<V, M> defaultImpl)
2604 bool LibraryCallKit::inline_vector_compress_expand() {
2605 const TypeInt* opr = gvn().type(argument(0))->isa_int();
2606 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
2607 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
2608 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
2609 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
2610
2611 if (vector_klass == nullptr || elem_klass == nullptr || mask_klass == nullptr || vlen == nullptr ||
2612 vector_klass->const_oop() == nullptr || mask_klass->const_oop() == nullptr ||
2613 elem_klass->const_oop() == nullptr || !vlen->is_con() || !opr->is_con()) {
2614 log_if_needed(" ** missing constant: opr=%s vclass=%s mclass=%s etype=%s vlen=%s",
2615 NodeClassNames[argument(0)->Opcode()],
2616 NodeClassNames[argument(1)->Opcode()],
2617 NodeClassNames[argument(2)->Opcode()],
2618 NodeClassNames[argument(3)->Opcode()],
2619 NodeClassNames[argument(4)->Opcode()]);
2620 return false; // not enough info for intrinsification
2621 }
2622
2623 if (!is_klass_initialized(vector_klass) || !is_klass_initialized(mask_klass)) {
2624 log_if_needed(" ** klass argument not initialized");
2625 return false;
2626 }
2627
2628 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2629 if (!elem_type->is_primitive_type()) {
2630 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2631 return false; // should be primitive type
2632 }
2633
2634 int num_elem = vlen->get_con();
2635 BasicType elem_bt = elem_type->basic_type();
2636 int opc = VectorSupport::vop2ideal(opr->get_con(), elem_bt);
2637
2638 if (!arch_supports_vector(opc, num_elem, elem_bt, VecMaskUseLoad)) {
2639 log_if_needed(" ** not supported: opc=%d vlen=%d etype=%s ismask=useload",
2640 opc, num_elem, type2name(elem_bt));
2641 return false; // not supported
2642 }
2643
2644 Node* opd1 = nullptr;
2645 const TypeInstPtr* vbox_type = nullptr;
2646 if (opc != Op_CompressM) {
2647 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2648 vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2649 opd1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
2650 if (opd1 == nullptr) {
2651 log_if_needed(" ** unbox failed vector=%s",
2652 NodeClassNames[argument(5)->Opcode()]);
2653 return false;
2654 }
2655 }
2656
2657 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
2658 assert(is_vector_mask(mbox_klass), "argument(6) should be a mask class");
2659 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
2660
2661 Node* mask = unbox_vector(argument(6), mbox_type, elem_bt, num_elem);
2662 if (mask == nullptr) {
2663 log_if_needed(" ** unbox failed mask=%s",
2664 NodeClassNames[argument(6)->Opcode()]);
2665 return false;
2666 }
2667
2668 const TypeVect* vt = TypeVect::make(elem_bt, num_elem, opc == Op_CompressM);
2669 Node* operation = gvn().transform(VectorNode::make(opc, opd1, mask, vt));
2670
2671 // Wrap it up in VectorBox to keep object type information.
2672 const TypeInstPtr* box_type = opc == Op_CompressM ? mbox_type : vbox_type;
2673 Node* vbox = box_vector(operation, box_type, elem_bt, num_elem);
2674 set_result(vbox);
2675 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2676 return true;
2677 }
2678
2679 // public static
2680 // <V extends Vector<E>,
2681 // E,
2682 // S extends VectorSpecies<E>>
2683 // V indexVector(Class<? extends V> vClass, Class<E> eClass,
2684 // int length,
2685 // V v, int step, S s,
2686 // IndexOperation<V, S> defaultImpl)
2687 bool LibraryCallKit::inline_index_vector() {
2688 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
2689 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
2690 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
2691
2692 if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr ||
2693 vector_klass->const_oop() == nullptr || !vlen->is_con() ||
2694 elem_klass->const_oop() == nullptr) {
2695 log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s",
2696 NodeClassNames[argument(0)->Opcode()],
2697 NodeClassNames[argument(1)->Opcode()],
2698 NodeClassNames[argument(2)->Opcode()]);
2699 return false; // not enough info for intrinsification
2700 }
2701
2702 if (!is_klass_initialized(vector_klass)) {
2703 log_if_needed(" ** klass argument not initialized");
2704 return false;
2705 }
2706
2707 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2708 if (!elem_type->is_primitive_type()) {
2709 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2710 return false; // should be primitive type
2711 }
2712
2713 int num_elem = vlen->get_con();
2714 BasicType elem_bt = elem_type->basic_type();
2715
2716 // Check whether the iota index generation op is supported by the current hardware
2717 if (!arch_supports_vector(Op_VectorLoadConst, num_elem, elem_bt, VecMaskNotUsed)) {
2718 log_if_needed(" ** not supported: vlen=%d etype=%s", num_elem, type2name(elem_bt));
2719 return false; // not supported
2720 }
2721
2722 int mul_op = VectorSupport::vop2ideal(VectorSupport::VECTOR_OP_MUL, elem_bt);
2723 int vmul_op = VectorNode::opcode(mul_op, elem_bt);
2724 bool needs_mul = true;
2725 Node* scale = argument(4);
2726 const TypeInt* scale_type = gvn().type(scale)->isa_int();
2727 // Multiply is not needed if the scale is a constant "1".
2728 if (scale_type && scale_type->is_con() && scale_type->get_con() == 1) {
2729 needs_mul = false;
2730 } else {
2731 // Check whether the vector multiply op is supported by the current hardware
2732 if (!arch_supports_vector(vmul_op, num_elem, elem_bt, VecMaskNotUsed)) {
2733 log_if_needed(" ** not supported: vlen=%d etype=%s", num_elem, type2name(elem_bt));
2734 return false; // not supported
2735 }
2736
2737 // Check whether the scalar cast op is supported by the current hardware
2738 if (is_floating_point_type(elem_bt) || elem_bt == T_LONG) {
2739 int cast_op = elem_bt == T_LONG ? Op_ConvI2L :
2740 elem_bt == T_FLOAT? Op_ConvI2F : Op_ConvI2D;
2741 if (!Matcher::match_rule_supported(cast_op)) {
2742 log_if_needed(" ** Rejected op (%s) because architecture does not support it",
2743 NodeClassNames[cast_op]);
2744 return false; // not supported
2745 }
2746 }
2747 }
2748
2749 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2750 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2751 Node* opd = unbox_vector(argument(3), vbox_type, elem_bt, num_elem);
2752 if (opd == nullptr) {
2753 log_if_needed(" ** unbox failed vector=%s",
2754 NodeClassNames[argument(3)->Opcode()]);
2755 return false;
2756 }
2757
2758 int add_op = VectorSupport::vop2ideal(VectorSupport::VECTOR_OP_ADD, elem_bt);
2759 int vadd_op = VectorNode::opcode(add_op, elem_bt);
2760 bool needs_add = true;
2761 // The addition is not needed if all the element values of "opd" are zero
2762 if (VectorNode::is_all_zeros_vector(opd)) {
2763 needs_add = false;
2764 } else {
2765 // Check whether the vector addition op is supported by the current hardware
2766 if (!arch_supports_vector(vadd_op, num_elem, elem_bt, VecMaskNotUsed)) {
2767 log_if_needed(" ** not supported: vlen=%d etype=%s", num_elem, type2name(elem_bt));
2768 return false; // not supported
2769 }
2770 }
2771
2772 // Compute the iota indice vector
2773 const TypeVect* vt = TypeVect::make(elem_bt, num_elem);
2774 Node* index = gvn().transform(new VectorLoadConstNode(gvn().makecon(TypeInt::ZERO), vt));
2775
2776 // Broadcast the "scale" to a vector, and multiply the "scale" with iota indice vector.
2777 if (needs_mul) {
2778 switch (elem_bt) {
2779 case T_BOOLEAN: // fall-through
2780 case T_BYTE: // fall-through
2781 case T_SHORT: // fall-through
2782 case T_CHAR: // fall-through
2783 case T_INT: {
2784 // no conversion needed
2785 break;
2786 }
2787 case T_LONG: {
2788 scale = gvn().transform(new ConvI2LNode(scale));
2789 break;
2790 }
2791 case T_FLOAT: {
2792 scale = gvn().transform(new ConvI2FNode(scale));
2793 break;
2794 }
2795 case T_DOUBLE: {
2796 scale = gvn().transform(new ConvI2DNode(scale));
2797 break;
2798 }
2799 default: fatal("%s", type2name(elem_bt));
2800 }
2801 scale = gvn().transform(VectorNode::scalar2vector(scale, num_elem, Type::get_const_basic_type(elem_bt)));
2802 index = gvn().transform(VectorNode::make(vmul_op, index, scale, vt));
2803 }
2804
2805 // Add "opd" if addition is needed.
2806 if (needs_add) {
2807 index = gvn().transform(VectorNode::make(vadd_op, opd, index, vt));
2808 }
2809 Node* vbox = box_vector(index, vbox_type, elem_bt, num_elem);
2810 set_result(vbox);
2811 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2812 return true;
2813 }
2814
2815 // public static
2816 // <E,
2817 // M extends VectorMask<E>>
2818 // M indexPartiallyInUpperRange(Class<? extends M> mClass, Class<E> eClass, int length,
2819 // long offset, long limit,
2820 // IndexPartiallyInUpperRangeOperation<E, M> defaultImpl)
2821 bool LibraryCallKit::inline_index_partially_in_upper_range() {
2822 const TypeInstPtr* mask_klass = gvn().type(argument(0))->isa_instptr();
2823 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
2824 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
2825
2826 if (mask_klass == nullptr || elem_klass == nullptr || vlen == nullptr ||
2827 mask_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr || !vlen->is_con()) {
2828 log_if_needed(" ** missing constant: mclass=%s etype=%s vlen=%s",
2829 NodeClassNames[argument(0)->Opcode()],
2830 NodeClassNames[argument(1)->Opcode()],
2831 NodeClassNames[argument(2)->Opcode()]);
2832 return false; // not enough info for intrinsification
2833 }
2834
2835 if (!is_klass_initialized(mask_klass)) {
2836 log_if_needed(" ** klass argument not initialized");
2837 return false;
2838 }
2839
2840 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2841 if (!elem_type->is_primitive_type()) {
2842 log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
2843 return false; // should be primitive type
2844 }
2845
2846 int num_elem = vlen->get_con();
2847 BasicType elem_bt = elem_type->basic_type();
2848
2849 // Check whether the necessary ops are supported by current hardware.
2850 bool supports_mask_gen = arch_supports_vector(Op_VectorMaskGen, num_elem, elem_bt, VecMaskUseStore);
2851 if (!supports_mask_gen) {
2852 if (!arch_supports_vector(Op_VectorLoadConst, num_elem, elem_bt, VecMaskNotUsed) ||
2853 !arch_supports_vector(Op_Replicate, num_elem, elem_bt, VecMaskNotUsed) ||
2854 !arch_supports_vector(Op_VectorMaskCmp, num_elem, elem_bt, VecMaskUseStore)) {
2855 log_if_needed(" ** not supported: vlen=%d etype=%s", num_elem, type2name(elem_bt));
2856 return false; // not supported
2857 }
2858
2859 // Check whether the scalar cast operation is supported by current hardware.
2860 if (elem_bt != T_LONG) {
2861 int cast_op = is_integral_type(elem_bt) ? Op_ConvL2I
2862 : (elem_bt == T_FLOAT ? Op_ConvL2F : Op_ConvL2D);
2863 if (!Matcher::match_rule_supported(cast_op)) {
2864 log_if_needed(" ** Rejected op (%s) because architecture does not support it",
2865 NodeClassNames[cast_op]);
2866 return false; // not supported
2867 }
2868 }
2869 }
2870
2871 Node* offset = argument(3);
2872 Node* limit = argument(5);
2873 if (offset == nullptr || limit == nullptr) {
2874 log_if_needed(" ** offset or limit argument is null");
2875 return false; // not supported
2876 }
2877
2878 ciKlass* box_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
2879 assert(is_vector_mask(box_klass), "argument(0) should be a mask class");
2880 const TypeInstPtr* box_type = TypeInstPtr::make_exact(TypePtr::NotNull, box_klass);
2881
2882 // We assume "offset > 0 && limit >= offset && limit - offset < num_elem".
2883 // So directly get indexLimit with "indexLimit = limit - offset".
2884 Node* indexLimit = gvn().transform(new SubLNode(limit, offset));
2885 Node* mask = nullptr;
2886 if (supports_mask_gen) {
2887 mask = gvn().transform(VectorMaskGenNode::make(indexLimit, elem_bt, num_elem));
2888 } else {
2889 // Generate the vector mask based on "mask = iota < indexLimit".
2890 // Broadcast "indexLimit" to a vector.
2891 switch (elem_bt) {
2892 case T_BOOLEAN: // fall-through
2893 case T_BYTE: // fall-through
2894 case T_SHORT: // fall-through
2895 case T_CHAR: // fall-through
2896 case T_INT: {
2897 indexLimit = gvn().transform(new ConvL2INode(indexLimit));
2898 break;
2899 }
2900 case T_DOUBLE: {
2901 indexLimit = gvn().transform(new ConvL2DNode(indexLimit));
2902 break;
2903 }
2904 case T_FLOAT: {
2905 indexLimit = gvn().transform(new ConvL2FNode(indexLimit));
2906 break;
2907 }
2908 case T_LONG: {
2909 // no conversion needed
2910 break;
2911 }
2912 default: fatal("%s", type2name(elem_bt));
2913 }
2914 indexLimit = gvn().transform(VectorNode::scalar2vector(indexLimit, num_elem, Type::get_const_basic_type(elem_bt)));
2915
2916 // Load the "iota" vector.
2917 const TypeVect* vt = TypeVect::make(elem_bt, num_elem);
2918 Node* iota = gvn().transform(new VectorLoadConstNode(gvn().makecon(TypeInt::ZERO), vt));
2919
2920 // Compute the vector mask with "mask = iota < indexLimit".
2921 ConINode* pred_node = (ConINode*)gvn().makecon(TypeInt::make(BoolTest::lt));
2922 const TypeVect* vmask_type = TypeVect::makemask(elem_bt, num_elem);
2923 mask = gvn().transform(new VectorMaskCmpNode(BoolTest::lt, iota, indexLimit, pred_node, vmask_type));
2924 }
2925 Node* vbox = box_vector(mask, box_type, elem_bt, num_elem);
2926 set_result(vbox);
2927 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2928 return true;
2929 }
2930
2931 #undef non_product_log_if_needed
2932 #undef log_if_needed