1 /*
2 * Copyright (c) 2020, 2022, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "ci/ciSymbols.hpp"
27 #include "classfile/vmSymbols.hpp"
28 #include "opto/library_call.hpp"
29 #include "opto/runtime.hpp"
30 #include "opto/vectornode.hpp"
31 #include "prims/vectorSupport.hpp"
32 #include "runtime/stubRoutines.hpp"
33
34 #ifdef ASSERT
35 static bool is_vector(ciKlass* klass) {
36 return klass->is_subclass_of(ciEnv::current()->vector_VectorPayload_klass());
37 }
38
39 static bool check_vbox(const TypeInstPtr* vbox_type) {
40 assert(vbox_type->klass_is_exact(), "");
41
42 ciInstanceKlass* ik = vbox_type->klass()->as_instance_klass();
43 assert(is_vector(ik), "not a vector");
44
45 ciField* fd1 = ik->get_field_by_name(ciSymbols::ETYPE_name(), ciSymbols::class_signature(), /* is_static */ true);
46 assert(fd1 != NULL, "element type info is missing");
47
48 ciConstant val1 = fd1->constant_value();
49 BasicType elem_bt = val1.as_object()->as_instance()->java_mirror_type()->basic_type();
50 assert(is_java_primitive(elem_bt), "element type info is missing");
51
52 ciField* fd2 = ik->get_field_by_name(ciSymbols::VLENGTH_name(), ciSymbols::int_signature(), /* is_static */ true);
53 assert(fd2 != NULL, "vector length info is missing");
54
55 ciConstant val2 = fd2->constant_value();
56 assert(val2.as_int() > 0, "vector length info is missing");
57
58 return true;
59 }
60 #endif
61
62 static bool is_vector_mask(ciKlass* klass) {
63 return klass->is_subclass_of(ciEnv::current()->vector_VectorMask_klass());
64 }
65
66 static bool is_vector_shuffle(ciKlass* klass) {
67 return klass->is_subclass_of(ciEnv::current()->vector_VectorShuffle_klass());
68 }
69
70 bool LibraryCallKit::arch_supports_vector_rotate(int opc, int num_elem, BasicType elem_bt,
71 VectorMaskUseType mask_use_type, bool has_scalar_args) {
72 bool is_supported = true;
73
74 // has_scalar_args flag is true only for non-constant scalar shift count,
75 // since in this case shift needs to be broadcasted.
76 if (!Matcher::match_rule_supported_vector(opc, num_elem, elem_bt) ||
77 (has_scalar_args &&
78 !arch_supports_vector(VectorNode::replicate_opcode(elem_bt), num_elem, elem_bt, VecMaskNotUsed))) {
79 is_supported = false;
80 }
81
82 if (is_supported) {
83 // Check whether mask unboxing is supported.
84 if ((mask_use_type & VecMaskUseLoad) != 0) {
85 if (!Matcher::match_rule_supported_vector(Op_VectorLoadMask, num_elem, elem_bt)) {
86 #ifndef PRODUCT
87 if (C->print_intrinsics()) {
88 tty->print_cr(" ** Rejected vector mask loading (%s,%s,%d) because architecture does not support it",
89 NodeClassNames[Op_VectorLoadMask], type2name(elem_bt), num_elem);
90 }
91 #endif
92 return false;
93 }
94 }
95
96 if ((mask_use_type & VecMaskUsePred) != 0) {
97 if (!Matcher::has_predicated_vectors() ||
98 !Matcher::match_rule_supported_vector_masked(opc, num_elem, elem_bt)) {
99 #ifndef PRODUCT
100 if (C->print_intrinsics()) {
101 tty->print_cr("Rejected vector mask predicate using (%s,%s,%d) because architecture does not support it",
102 NodeClassNames[opc], type2name(elem_bt), num_elem);
103 }
104 #endif
105 return false;
106 }
107 }
108 }
109
110 int lshiftopc, rshiftopc;
111 switch(elem_bt) {
112 case T_BYTE:
113 lshiftopc = Op_LShiftI;
114 rshiftopc = Op_URShiftB;
115 break;
116 case T_SHORT:
117 lshiftopc = Op_LShiftI;
118 rshiftopc = Op_URShiftS;
119 break;
120 case T_INT:
121 lshiftopc = Op_LShiftI;
122 rshiftopc = Op_URShiftI;
123 break;
124 case T_LONG:
125 lshiftopc = Op_LShiftL;
126 rshiftopc = Op_URShiftL;
127 break;
128 default:
129 assert(false, "Unexpected type");
130 }
131 int lshiftvopc = VectorNode::opcode(lshiftopc, elem_bt);
132 int rshiftvopc = VectorNode::opcode(rshiftopc, elem_bt);
133 if (!is_supported &&
134 arch_supports_vector(lshiftvopc, num_elem, elem_bt, VecMaskNotUsed, has_scalar_args) &&
135 arch_supports_vector(rshiftvopc, num_elem, elem_bt, VecMaskNotUsed, has_scalar_args) &&
136 arch_supports_vector(Op_OrV, num_elem, elem_bt, VecMaskNotUsed)) {
137 is_supported = true;
138 }
139 return is_supported;
140 }
141
142 Node* GraphKit::box_vector(Node* vector, const TypeInstPtr* vbox_type, BasicType elem_bt, int num_elem, bool deoptimize_on_exception) {
143 assert(EnableVectorSupport, "");
144
145 PreserveReexecuteState preexecs(this);
146 jvms()->set_should_reexecute(true);
147
148 VectorBoxAllocateNode* alloc = new VectorBoxAllocateNode(C, vbox_type);
149 set_edges_for_java_call(alloc, /*must_throw=*/false, /*separate_io_proj=*/true);
150 make_slow_call_ex(alloc, env()->Throwable_klass(), /*separate_io_proj=*/true, deoptimize_on_exception);
151 set_i_o(gvn().transform( new ProjNode(alloc, TypeFunc::I_O) ));
152 set_all_memory(gvn().transform( new ProjNode(alloc, TypeFunc::Memory) ));
153 Node* ret = gvn().transform(new ProjNode(alloc, TypeFunc::Parms));
154
155 assert(check_vbox(vbox_type), "");
156 const TypeVect* vt = TypeVect::make(elem_bt, num_elem, is_vector_mask(vbox_type->klass()));
157 VectorBoxNode* vbox = new VectorBoxNode(C, ret, vector, vbox_type, vt);
158 return gvn().transform(vbox);
159 }
160
161 Node* GraphKit::unbox_vector(Node* v, const TypeInstPtr* vbox_type, BasicType elem_bt, int num_elem, bool shuffle_to_vector) {
162 assert(EnableVectorSupport, "");
163 const TypeInstPtr* vbox_type_v = gvn().type(v)->is_instptr();
164 if (vbox_type->klass() != vbox_type_v->klass()) {
165 return NULL; // arguments don't agree on vector shapes
166 }
167 if (vbox_type_v->maybe_null()) {
168 return NULL; // no nulls are allowed
169 }
170 assert(check_vbox(vbox_type), "");
171 const TypeVect* vt = TypeVect::make(elem_bt, num_elem, is_vector_mask(vbox_type->klass()));
172 Node* unbox = gvn().transform(new VectorUnboxNode(C, vt, v, merged_memory(), shuffle_to_vector));
173 return unbox;
174 }
175
176 Node* GraphKit::vector_shift_count(Node* cnt, int shift_op, BasicType bt, int num_elem) {
177 assert(bt == T_INT || bt == T_LONG || bt == T_SHORT || bt == T_BYTE, "byte, short, long and int are supported");
178 juint mask = (type2aelembytes(bt) * BitsPerByte - 1);
179 Node* nmask = gvn().transform(ConNode::make(TypeInt::make(mask)));
180 Node* mcnt = gvn().transform(new AndINode(cnt, nmask));
181 return gvn().transform(VectorNode::shift_count(shift_op, mcnt, num_elem, bt));
182 }
183
184 bool LibraryCallKit::arch_supports_vector(int sopc, int num_elem, BasicType type, VectorMaskUseType mask_use_type, bool has_scalar_args) {
185 // Check that the operation is valid.
186 if (sopc <= 0) {
187 #ifndef PRODUCT
188 if (C->print_intrinsics()) {
189 tty->print_cr(" ** Rejected intrinsification because no valid vector op could be extracted");
190 }
191 #endif
192 return false;
193 }
194
195 if (VectorNode::is_vector_rotate(sopc)) {
196 if(!arch_supports_vector_rotate(sopc, num_elem, type, mask_use_type, has_scalar_args)) {
197 #ifndef PRODUCT
198 if (C->print_intrinsics()) {
199 tty->print_cr(" ** Rejected vector op (%s,%s,%d) because architecture does not support variable vector shifts",
200 NodeClassNames[sopc], type2name(type), num_elem);
201 }
202 #endif
203 return false;
204 }
205 } else if (VectorNode::is_vector_integral_negate(sopc)) {
206 if (!VectorNode::is_vector_integral_negate_supported(sopc, num_elem, type, false)) {
207 #ifndef PRODUCT
208 if (C->print_intrinsics()) {
209 tty->print_cr(" ** Rejected vector op (%s,%s,%d) because architecture does not support integral vector negate",
210 NodeClassNames[sopc], type2name(type), num_elem);
211 }
212 #endif
213 return false;
214 }
215 } else {
216 // Check that architecture supports this op-size-type combination.
217 if (!Matcher::match_rule_supported_vector(sopc, num_elem, type)) {
218 #ifndef PRODUCT
219 if (C->print_intrinsics()) {
220 tty->print_cr(" ** Rejected vector op (%s,%s,%d) because architecture does not support it",
221 NodeClassNames[sopc], type2name(type), num_elem);
222 }
223 #endif
224 return false;
225 } else {
226 assert(Matcher::match_rule_supported(sopc), "must be supported");
227 }
228 }
229
230 if (num_elem == 1) {
231 if (mask_use_type != VecMaskNotUsed) {
232 #ifndef PRODUCT
233 if (C->print_intrinsics()) {
234 tty->print_cr(" ** Rejected vector mask op (%s,%s,%d) because architecture does not support it",
235 NodeClassNames[sopc], type2name(type), num_elem);
236 }
237 #endif
238 return false;
239 }
240
241 if (sopc != 0) {
242 if (sopc != Op_LoadVector && sopc != Op_StoreVector) {
243 #ifndef PRODUCT
244 if (C->print_intrinsics()) {
245 tty->print_cr(" ** Not a svml call or load/store vector op (%s,%s,%d)",
246 NodeClassNames[sopc], type2name(type), num_elem);
247 }
248 #endif
249 return false;
250 }
251 }
252 }
253
254 if (!has_scalar_args && VectorNode::is_vector_shift(sopc) &&
255 Matcher::supports_vector_variable_shifts() == false) {
256 if (C->print_intrinsics()) {
257 tty->print_cr(" ** Rejected vector op (%s,%s,%d) because architecture does not support variable vector shifts",
258 NodeClassNames[sopc], type2name(type), num_elem);
259 }
260 return false;
261 }
262
263 // Check whether mask unboxing is supported.
264 if ((mask_use_type & VecMaskUseLoad) != 0) {
265 if (!Matcher::match_rule_supported_vector(Op_VectorLoadMask, num_elem, type)) {
266 #ifndef PRODUCT
267 if (C->print_intrinsics()) {
268 tty->print_cr(" ** Rejected vector mask loading (%s,%s,%d) because architecture does not support it",
269 NodeClassNames[Op_VectorLoadMask], type2name(type), num_elem);
270 }
271 #endif
272 return false;
273 }
274 }
275
276 // Check whether mask boxing is supported.
277 if ((mask_use_type & VecMaskUseStore) != 0) {
278 if (!Matcher::match_rule_supported_vector(Op_VectorStoreMask, num_elem, type)) {
279 #ifndef PRODUCT
280 if (C->print_intrinsics()) {
281 tty->print_cr("Rejected vector mask storing (%s,%s,%d) because architecture does not support it",
282 NodeClassNames[Op_VectorStoreMask], type2name(type), num_elem);
283 }
284 #endif
285 return false;
286 }
287 }
288
289 if ((mask_use_type & VecMaskUsePred) != 0) {
290 bool is_supported = false;
291 if (Matcher::has_predicated_vectors()) {
292 if (VectorNode::is_vector_integral_negate(sopc)) {
293 is_supported = VectorNode::is_vector_integral_negate_supported(sopc, num_elem, type, true);
294 } else {
295 is_supported = Matcher::match_rule_supported_vector_masked(sopc, num_elem, type);
296 }
297 }
298
299 if (!is_supported) {
300 #ifndef PRODUCT
301 if (C->print_intrinsics()) {
302 tty->print_cr("Rejected vector mask predicate using (%s,%s,%d) because architecture does not support it",
303 NodeClassNames[sopc], type2name(type), num_elem);
304 }
305 #endif
306 return false;
307 }
308 }
309
310 return true;
311 }
312
313 static bool is_klass_initialized(const TypeInstPtr* vec_klass) {
314 if (vec_klass->const_oop() == NULL) {
315 return false; // uninitialized or some kind of unsafe access
316 }
317 assert(vec_klass->const_oop()->as_instance()->java_lang_Class_klass() != NULL, "klass instance expected");
318 ciInstanceKlass* klass = vec_klass->const_oop()->as_instance()->java_lang_Class_klass()->as_instance_klass();
319 return klass->is_initialized();
320 }
321
322 // public static
323 // <V extends Vector<E>,
324 // M extends VectorMask<E>,
325 // E>
326 // V unaryOp(int oprId, Class<? extends V> vmClass, Class<? extends M> maskClass, Class<E> elementType,
327 // int length, V v, M m,
328 // UnaryOperation<V, M> defaultImpl)
329 //
330 // public static
331 // <V,
332 // M extends VectorMask<E>,
333 // E>
334 // V binaryOp(int oprId, Class<? extends V> vmClass, Class<? extends M> maskClass, Class<E> elementType,
335 // int length, V v1, V v2, M m,
336 // BinaryOperation<V, M> defaultImpl)
337 //
338 // public static
339 // <V extends Vector<E>,
340 // M extends VectorMask<E>,
341 // E>
342 // V ternaryOp(int oprId, Class<? extends V> vmClass, Class<? extends M> maskClass, Class<E> elementType,
343 // int length, V v1, V v2, V v3, M m,
344 // TernaryOperation<V, M> defaultImpl)
345 //
346 bool LibraryCallKit::inline_vector_nary_operation(int n) {
347 const TypeInt* opr = gvn().type(argument(0))->isa_int();
348 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
349 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
350 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
351 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
352
353 if (opr == NULL || vector_klass == NULL || elem_klass == NULL || vlen == NULL ||
354 !opr->is_con() || vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || !vlen->is_con()) {
355 if (C->print_intrinsics()) {
356 tty->print_cr(" ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
357 NodeClassNames[argument(0)->Opcode()],
358 NodeClassNames[argument(1)->Opcode()],
359 NodeClassNames[argument(3)->Opcode()],
360 NodeClassNames[argument(4)->Opcode()]);
361 }
362 return false; // not enough info for intrinsification
363 }
364
365 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
366 if (!elem_type->is_primitive_type()) {
367 if (C->print_intrinsics()) {
368 tty->print_cr(" ** not a primitive bt=%d", elem_type->basic_type());
369 }
370 return false; // should be primitive type
371 }
372 if (!is_klass_initialized(vector_klass)) {
373 if (C->print_intrinsics()) {
374 tty->print_cr(" ** klass argument not initialized");
375 }
376 return false;
377 }
378
379 // "argument(n + 5)" should be the mask object. We assume it is "null" when no mask
380 // is used to control this operation.
381 const Type* vmask_type = gvn().type(argument(n + 5));
382 bool is_masked_op = vmask_type != TypePtr::NULL_PTR;
383 if (is_masked_op) {
384 if (mask_klass == NULL || mask_klass->const_oop() == NULL) {
385 if (C->print_intrinsics()) {
386 tty->print_cr(" ** missing constant: maskclass=%s", NodeClassNames[argument(2)->Opcode()]);
387 }
388 return false; // not enough info for intrinsification
389 }
390
391 if (!is_klass_initialized(mask_klass)) {
392 if (C->print_intrinsics()) {
393 tty->print_cr(" ** mask klass argument not initialized");
394 }
395 return false;
396 }
397
398 if (vmask_type->maybe_null()) {
399 if (C->print_intrinsics()) {
400 tty->print_cr(" ** null mask values are not allowed for masked op");
401 }
402 return false;
403 }
404 }
405
406 BasicType elem_bt = elem_type->basic_type();
407 int num_elem = vlen->get_con();
408 int opc = VectorSupport::vop2ideal(opr->get_con(), elem_bt);
409 int sopc = VectorNode::opcode(opc, elem_bt);
410 if ((opc != Op_CallLeafVector) && (sopc == 0)) {
411 if (C->print_intrinsics()) {
412 tty->print_cr(" ** operation not supported: opc=%s bt=%s", NodeClassNames[opc], type2name(elem_bt));
413 }
414 return false; // operation not supported
415 }
416 if (num_elem == 1) {
417 if (opc != Op_CallLeafVector || elem_bt != T_DOUBLE) {
418 if (C->print_intrinsics()) {
419 tty->print_cr(" ** not a svml call: arity=%d opc=%d vlen=%d etype=%s",
420 n, opc, num_elem, type2name(elem_bt));
421 }
422 return false;
423 }
424 }
425 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
426 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
427
428 if (is_vector_mask(vbox_klass)) {
429 assert(!is_masked_op, "mask operations do not need mask to control");
430 }
431
432 if (opc == Op_CallLeafVector) {
433 if (!UseVectorStubs) {
434 if (C->print_intrinsics()) {
435 tty->print_cr(" ** vector stubs support is disabled");
436 }
437 return false;
438 }
439 if (!Matcher::supports_vector_calling_convention()) {
440 if (C->print_intrinsics()) {
441 tty->print_cr(" ** no vector calling conventions supported");
442 }
443 return false;
444 }
445 if (!Matcher::vector_size_supported(elem_bt, num_elem)) {
446 if (C->print_intrinsics()) {
447 tty->print_cr(" ** vector size (vlen=%d, etype=%s) is not supported",
448 num_elem, type2name(elem_bt));
449 }
450 return false;
451 }
452 }
453
454 // When using mask, mask use type needs to be VecMaskUseLoad.
455 VectorMaskUseType mask_use_type = is_vector_mask(vbox_klass) ? VecMaskUseAll
456 : is_masked_op ? VecMaskUseLoad : VecMaskNotUsed;
457 if ((sopc != 0) && !arch_supports_vector(sopc, num_elem, elem_bt, mask_use_type)) {
458 if (C->print_intrinsics()) {
459 tty->print_cr(" ** not supported: arity=%d opc=%d vlen=%d etype=%s ismask=%d is_masked_op=%d",
460 n, sopc, num_elem, type2name(elem_bt),
461 is_vector_mask(vbox_klass) ? 1 : 0, is_masked_op ? 1 : 0);
462 }
463 return false; // not supported
464 }
465
466 // Return true if current platform has implemented the masked operation with predicate feature.
467 bool use_predicate = is_masked_op && sopc != 0 && arch_supports_vector(sopc, num_elem, elem_bt, VecMaskUsePred);
468 if (is_masked_op && !use_predicate && !arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad)) {
469 if (C->print_intrinsics()) {
470 tty->print_cr(" ** not supported: arity=%d opc=%d vlen=%d etype=%s ismask=0 is_masked_op=1",
471 n, sopc, num_elem, type2name(elem_bt));
472 }
473 return false;
474 }
475
476 Node* opd1 = NULL; Node* opd2 = NULL; Node* opd3 = NULL;
477 switch (n) {
478 case 3: {
479 opd3 = unbox_vector(argument(7), vbox_type, elem_bt, num_elem);
480 if (opd3 == NULL) {
481 if (C->print_intrinsics()) {
482 tty->print_cr(" ** unbox failed v3=%s",
483 NodeClassNames[argument(7)->Opcode()]);
484 }
485 return false;
486 }
487 // fall-through
488 }
489 case 2: {
490 opd2 = unbox_vector(argument(6), vbox_type, elem_bt, num_elem);
491 if (opd2 == NULL) {
492 if (C->print_intrinsics()) {
493 tty->print_cr(" ** unbox failed v2=%s",
494 NodeClassNames[argument(6)->Opcode()]);
495 }
496 return false;
497 }
498 // fall-through
499 }
500 case 1: {
501 opd1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
502 if (opd1 == NULL) {
503 if (C->print_intrinsics()) {
504 tty->print_cr(" ** unbox failed v1=%s",
505 NodeClassNames[argument(5)->Opcode()]);
506 }
507 return false;
508 }
509 break;
510 }
511 default: fatal("unsupported arity: %d", n);
512 }
513
514 Node* mask = NULL;
515 if (is_masked_op) {
516 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
517 assert(is_vector_mask(mbox_klass), "argument(2) should be a mask class");
518 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
519 mask = unbox_vector(argument(n + 5), mbox_type, elem_bt, num_elem);
520 if (mask == NULL) {
521 if (C->print_intrinsics()) {
522 tty->print_cr(" ** unbox failed mask=%s",
523 NodeClassNames[argument(n + 5)->Opcode()]);
524 }
525 return false;
526 }
527 }
528
529 Node* operation = NULL;
530 if (opc == Op_CallLeafVector) {
531 assert(UseVectorStubs, "sanity");
532 operation = gen_call_to_svml(opr->get_con(), elem_bt, num_elem, opd1, opd2);
533 if (operation == NULL) {
534 if (C->print_intrinsics()) {
535 tty->print_cr(" ** svml call failed for %s_%s_%d",
536 (elem_bt == T_FLOAT)?"float":"double",
537 VectorSupport::svmlname[opr->get_con() - VectorSupport::VECTOR_OP_SVML_START],
538 num_elem * type2aelembytes(elem_bt));
539 }
540 return false;
541 }
542 } else {
543 const TypeVect* vt = TypeVect::make(elem_bt, num_elem, is_vector_mask(vbox_klass));
544 switch (n) {
545 case 1:
546 case 2: {
547 operation = VectorNode::make(sopc, opd1, opd2, vt, is_vector_mask(vbox_klass), VectorNode::is_shift_opcode(opc));
548 break;
549 }
550 case 3: {
551 operation = VectorNode::make(sopc, opd1, opd2, opd3, vt);
552 break;
553 }
554 default: fatal("unsupported arity: %d", n);
555 }
556 }
557
558 if (is_masked_op && mask != NULL) {
559 if (use_predicate) {
560 operation->add_req(mask);
561 operation->add_flag(Node::Flag_is_predicated_vector);
562 } else {
563 operation = gvn().transform(operation);
564 operation = new VectorBlendNode(opd1, operation, mask);
565 }
566 }
567 operation = gvn().transform(operation);
568
569 // Wrap it up in VectorBox to keep object type information.
570 Node* vbox = box_vector(operation, vbox_type, elem_bt, num_elem);
571 set_result(vbox);
572 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
573 return true;
574 }
575
576 // <Sh extends VectorShuffle<E>, E>
577 // Sh ShuffleIota(Class<?> E, Class<?> shuffleClass, Vector.Species<E> s, int length,
578 // int start, int step, int wrap, ShuffleIotaOperation<Sh, E> defaultImpl)
579 bool LibraryCallKit::inline_vector_shuffle_iota() {
580 const TypeInstPtr* shuffle_klass = gvn().type(argument(1))->isa_instptr();
581 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
582 const TypeInt* start_val = gvn().type(argument(4))->isa_int();
583 const TypeInt* step_val = gvn().type(argument(5))->isa_int();
584 const TypeInt* wrap = gvn().type(argument(6))->isa_int();
585
586 Node* start = argument(4);
587 Node* step = argument(5);
588
589 if (shuffle_klass == NULL || vlen == NULL || start_val == NULL || step_val == NULL || wrap == NULL) {
590 return false; // dead code
591 }
592 if (!vlen->is_con() || !is_power_of_2(vlen->get_con()) ||
593 shuffle_klass->const_oop() == NULL || !wrap->is_con()) {
594 return false; // not enough info for intrinsification
595 }
596 if (!is_klass_initialized(shuffle_klass)) {
597 if (C->print_intrinsics()) {
598 tty->print_cr(" ** klass argument not initialized");
599 }
600 return false;
601 }
602
603 int do_wrap = wrap->get_con();
604 int num_elem = vlen->get_con();
605 BasicType elem_bt = T_BYTE;
606
607 if (!arch_supports_vector(VectorNode::replicate_opcode(elem_bt), num_elem, elem_bt, VecMaskNotUsed)) {
608 return false;
609 }
610 if (!arch_supports_vector(Op_AddVB, num_elem, elem_bt, VecMaskNotUsed)) {
611 return false;
612 }
613 if (!arch_supports_vector(Op_AndV, num_elem, elem_bt, VecMaskNotUsed)) {
614 return false;
615 }
616 if (!arch_supports_vector(Op_VectorLoadConst, num_elem, elem_bt, VecMaskNotUsed)) {
617 return false;
618 }
619 if (!arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad)) {
620 return false;
621 }
622 if (!arch_supports_vector(Op_VectorMaskCmp, num_elem, elem_bt, VecMaskUseStore)) {
623 return false;
624 }
625
626 const Type * type_bt = Type::get_const_basic_type(elem_bt);
627 const TypeVect * vt = TypeVect::make(type_bt, num_elem);
628
629 Node* res = gvn().transform(new VectorLoadConstNode(gvn().makecon(TypeInt::ZERO), vt));
630
631 if(!step_val->is_con() || !is_power_of_2(step_val->get_con())) {
632 Node* bcast_step = gvn().transform(VectorNode::scalar2vector(step, num_elem, type_bt));
633 res = gvn().transform(VectorNode::make(Op_MulI, res, bcast_step, num_elem, elem_bt));
634 } else if (step_val->get_con() > 1) {
635 Node* cnt = gvn().makecon(TypeInt::make(log2i_exact(step_val->get_con())));
636 Node* shift_cnt = vector_shift_count(cnt, Op_LShiftI, elem_bt, num_elem);
637 res = gvn().transform(VectorNode::make(Op_LShiftVB, res, shift_cnt, vt));
638 }
639
640 if (!start_val->is_con() || start_val->get_con() != 0) {
641 Node* bcast_start = gvn().transform(VectorNode::scalar2vector(start, num_elem, type_bt));
642 res = gvn().transform(VectorNode::make(Op_AddI, res, bcast_start, num_elem, elem_bt));
643 }
644
645 Node * mod_val = gvn().makecon(TypeInt::make(num_elem-1));
646 Node * bcast_mod = gvn().transform(VectorNode::scalar2vector(mod_val, num_elem, type_bt));
647 if(do_wrap) {
648 // Wrap the indices greater than lane count.
649 res = gvn().transform(VectorNode::make(Op_AndI, res, bcast_mod, num_elem, elem_bt));
650 } else {
651 ConINode* pred_node = (ConINode*)gvn().makecon(TypeInt::make(BoolTest::ge));
652 Node * lane_cnt = gvn().makecon(TypeInt::make(num_elem));
653 Node * bcast_lane_cnt = gvn().transform(VectorNode::scalar2vector(lane_cnt, num_elem, type_bt));
654 const TypeVect* vmask_type = TypeVect::makemask(elem_bt, num_elem);
655 Node* mask = gvn().transform(new VectorMaskCmpNode(BoolTest::ge, bcast_lane_cnt, res, pred_node, vmask_type));
656
657 // Make the indices greater than lane count as -ve values. This matches the java side implementation.
658 res = gvn().transform(VectorNode::make(Op_AndI, res, bcast_mod, num_elem, elem_bt));
659 Node * biased_val = gvn().transform(VectorNode::make(Op_SubI, res, bcast_lane_cnt, num_elem, elem_bt));
660 res = gvn().transform(new VectorBlendNode(biased_val, res, mask));
661 }
662
663 ciKlass* sbox_klass = shuffle_klass->const_oop()->as_instance()->java_lang_Class_klass();
664 const TypeInstPtr* shuffle_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, sbox_klass);
665
666 // Wrap it up in VectorBox to keep object type information.
667 res = box_vector(res, shuffle_box_type, elem_bt, num_elem);
668 set_result(res);
669 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
670 return true;
671 }
672
673 // <E, M>
674 // long maskReductionCoerced(int oper, Class<? extends M> maskClass, Class<?> elemClass,
675 // int length, M m, VectorMaskOp<M> defaultImpl)
676 bool LibraryCallKit::inline_vector_mask_operation() {
677 const TypeInt* oper = gvn().type(argument(0))->isa_int();
678 const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
679 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
680 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
681 Node* mask = argument(4);
682
683 if (mask_klass == NULL || elem_klass == NULL || mask->is_top() || vlen == NULL) {
684 return false; // dead code
685 }
686
687 if (!is_klass_initialized(mask_klass)) {
688 if (C->print_intrinsics()) {
689 tty->print_cr(" ** klass argument not initialized");
690 }
691 return false;
692 }
693
694 int num_elem = vlen->get_con();
695 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
696 BasicType elem_bt = elem_type->basic_type();
697
698 if (!arch_supports_vector(Op_LoadVector, num_elem, T_BOOLEAN, VecMaskNotUsed)) {
699 if (C->print_intrinsics()) {
700 tty->print_cr(" ** not supported: arity=1 op=cast#%d/3 vlen2=%d etype2=%s",
701 Op_LoadVector, num_elem, type2name(T_BOOLEAN));
702 }
703 return false; // not supported
704 }
705
706 int mopc = VectorSupport::vop2ideal(oper->get_con(), elem_bt);
707 if (!arch_supports_vector(mopc, num_elem, elem_bt, VecMaskNotUsed)) {
708 if (C->print_intrinsics()) {
709 tty->print_cr(" ** not supported: arity=1 op=cast#%d/3 vlen2=%d etype2=%s",
710 mopc, num_elem, type2name(elem_bt));
711 }
712 return false; // not supported
713 }
714
715 const Type* elem_ty = Type::get_const_basic_type(elem_bt);
716 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
717 const TypeInstPtr* mask_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
718 Node* mask_vec = unbox_vector(mask, mask_box_type, elem_bt, num_elem, true);
719 if (mask_vec == NULL) {
720 if (C->print_intrinsics()) {
721 tty->print_cr(" ** unbox failed mask=%s",
722 NodeClassNames[argument(4)->Opcode()]);
723 }
724 return false;
725 }
726
727 if (mask_vec->bottom_type()->isa_vectmask() == NULL) {
728 mask_vec = gvn().transform(VectorStoreMaskNode::make(gvn(), mask_vec, elem_bt, num_elem));
729 }
730 const Type* maskoper_ty = mopc == Op_VectorMaskToLong ? (const Type*)TypeLong::LONG : (const Type*)TypeInt::INT;
731 Node* maskoper = gvn().transform(VectorMaskOpNode::make(mask_vec, maskoper_ty, mopc));
732 if (mopc != Op_VectorMaskToLong) {
733 maskoper = ConvI2L(maskoper);
734 }
735 set_result(maskoper);
736
737 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
738 return true;
739 }
740
741 // public static
742 // <V,
743 // Sh extends VectorShuffle<E>,
744 // E>
745 // V shuffleToVector(Class<? extends Vector<E>> vclass, Class<E> elementType,
746 // Class<? extends Sh> shuffleClass, Sh s, int length,
747 // ShuffleToVectorOperation<V, Sh, E> defaultImpl)
748 bool LibraryCallKit::inline_vector_shuffle_to_vector() {
749 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
750 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
751 const TypeInstPtr* shuffle_klass = gvn().type(argument(2))->isa_instptr();
752 Node* shuffle = argument(3);
753 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
754
755 if (vector_klass == NULL || elem_klass == NULL || shuffle_klass == NULL || shuffle->is_top() || vlen == NULL) {
756 return false; // dead code
757 }
758 if (!vlen->is_con() || vector_klass->const_oop() == NULL || shuffle_klass->const_oop() == NULL) {
759 return false; // not enough info for intrinsification
760 }
761 if (!is_klass_initialized(shuffle_klass) || !is_klass_initialized(vector_klass) ) {
762 if (C->print_intrinsics()) {
763 tty->print_cr(" ** klass argument not initialized");
764 }
765 return false;
766 }
767
768 int num_elem = vlen->get_con();
769 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
770 BasicType elem_bt = elem_type->basic_type();
771
772 if (num_elem < 4) {
773 return false;
774 }
775
776 int cast_vopc = VectorCastNode::opcode(T_BYTE); // from shuffle of type T_BYTE
777 // Make sure that cast is implemented to particular type/size combination.
778 if (!arch_supports_vector(cast_vopc, num_elem, elem_bt, VecMaskNotUsed)) {
779 if (C->print_intrinsics()) {
780 tty->print_cr(" ** not supported: arity=1 op=cast#%d/3 vlen2=%d etype2=%s",
781 cast_vopc, num_elem, type2name(elem_bt));
782 }
783 return false;
784 }
785
786 ciKlass* sbox_klass = shuffle_klass->const_oop()->as_instance()->java_lang_Class_klass();
787 const TypeInstPtr* shuffle_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, sbox_klass);
788
789 // Unbox shuffle with true flag to indicate its load shuffle to vector
790 // shuffle is a byte array
791 Node* shuffle_vec = unbox_vector(shuffle, shuffle_box_type, T_BYTE, num_elem, true);
792
793 // cast byte to target element type
794 shuffle_vec = gvn().transform(VectorCastNode::make(cast_vopc, shuffle_vec, elem_bt, num_elem));
795
796 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
797 const TypeInstPtr* vec_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
798
799 // Box vector
800 Node* res = box_vector(shuffle_vec, vec_box_type, elem_bt, num_elem);
801 set_result(res);
802 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
803 return true;
804 }
805
806 // public static
807 // <M,
808 // S extends VectorSpecies<E>,
809 // E>
810 // M fromBitsCoerced(Class<? extends M> vmClass, Class<E> elementType, int length,
811 // long bits, int mode, S s,
812 // BroadcastOperation<M, E, S> defaultImpl)
813 bool LibraryCallKit::inline_vector_frombits_coerced() {
814 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
815 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
816 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
817 const TypeLong* bits_type = gvn().type(argument(3))->isa_long();
818 // Mode argument determines the mode of operation it can take following values:-
819 // MODE_BROADCAST for vector Vector.broadcast and VectorMask.maskAll operations.
820 // MODE_BITS_COERCED_LONG_TO_MASK for VectorMask.fromLong operation.
821 const TypeInt* mode = gvn().type(argument(5))->isa_int();
822
823 if (vector_klass == NULL || elem_klass == NULL || vlen == NULL || mode == NULL ||
824 bits_type == NULL || vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL ||
825 !vlen->is_con() || !mode->is_con()) {
826 if (C->print_intrinsics()) {
827 tty->print_cr(" ** missing constant: vclass=%s etype=%s vlen=%s bitwise=%s",
828 NodeClassNames[argument(0)->Opcode()],
829 NodeClassNames[argument(1)->Opcode()],
830 NodeClassNames[argument(2)->Opcode()],
831 NodeClassNames[argument(5)->Opcode()]);
832 }
833 return false; // not enough info for intrinsification
834 }
835
836 if (!is_klass_initialized(vector_klass)) {
837 if (C->print_intrinsics()) {
838 tty->print_cr(" ** klass argument not initialized");
839 }
840 return false;
841 }
842 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
843 if (!elem_type->is_primitive_type()) {
844 if (C->print_intrinsics()) {
845 tty->print_cr(" ** not a primitive bt=%d", elem_type->basic_type());
846 }
847 return false; // should be primitive type
848 }
849 BasicType elem_bt = elem_type->basic_type();
850 int num_elem = vlen->get_con();
851 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
852 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
853
854 bool is_mask = is_vector_mask(vbox_klass);
855 int bcast_mode = mode->get_con();
856 VectorMaskUseType checkFlags = (VectorMaskUseType)(is_mask ? VecMaskUseAll : VecMaskNotUsed);
857 int opc = bcast_mode == VectorSupport::MODE_BITS_COERCED_LONG_TO_MASK ? Op_VectorLongToMask : VectorNode::replicate_opcode(elem_bt);
858
859 if (!arch_supports_vector(opc, num_elem, elem_bt, checkFlags, true /*has_scalar_args*/)) {
860 if (C->print_intrinsics()) {
861 tty->print_cr(" ** not supported: arity=0 op=broadcast vlen=%d etype=%s ismask=%d bcast_mode=%d",
862 num_elem, type2name(elem_bt),
863 is_mask ? 1 : 0,
864 bcast_mode);
865 }
866 return false; // not supported
867 }
868
869 Node* broadcast = NULL;
870 Node* bits = argument(3);
871 Node* elem = bits;
872
873 if (opc == Op_VectorLongToMask) {
874 const TypeVect* vt = TypeVect::makemask(elem_bt, num_elem);
875 if (vt->isa_vectmask()) {
876 broadcast = gvn().transform(new VectorLongToMaskNode(elem, vt));
877 } else {
878 const TypeVect* mvt = TypeVect::make(T_BOOLEAN, num_elem);
879 broadcast = gvn().transform(new VectorLongToMaskNode(elem, mvt));
880 broadcast = gvn().transform(new VectorLoadMaskNode(broadcast, vt));
881 }
882 } else {
883 switch (elem_bt) {
884 case T_BOOLEAN: // fall-through
885 case T_BYTE: // fall-through
886 case T_SHORT: // fall-through
887 case T_CHAR: // fall-through
888 case T_INT: {
889 elem = gvn().transform(new ConvL2INode(bits));
890 break;
891 }
892 case T_DOUBLE: {
893 elem = gvn().transform(new MoveL2DNode(bits));
894 break;
895 }
896 case T_FLOAT: {
897 bits = gvn().transform(new ConvL2INode(bits));
898 elem = gvn().transform(new MoveI2FNode(bits));
899 break;
900 }
901 case T_LONG: {
902 // no conversion needed
903 break;
904 }
905 default: fatal("%s", type2name(elem_bt));
906 }
907 broadcast = VectorNode::scalar2vector(elem, num_elem, Type::get_const_basic_type(elem_bt), is_mask);
908 broadcast = gvn().transform(broadcast);
909 }
910
911 Node* box = box_vector(broadcast, vbox_type, elem_bt, num_elem);
912 set_result(box);
913 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
914 return true;
915 }
916
917 static bool elem_consistent_with_arr(BasicType elem_bt, const TypeAryPtr* arr_type) {
918 assert(arr_type != NULL, "unexpected");
919 BasicType arr_elem_bt = arr_type->elem()->array_element_basic_type();
920 if (elem_bt == arr_elem_bt) {
921 return true;
922 } else if (elem_bt == T_SHORT && arr_elem_bt == T_CHAR) {
923 // Load/store of short vector from/to char[] is supported
924 return true;
925 } else if (elem_bt == T_BYTE && arr_elem_bt == T_BOOLEAN) {
926 // Load/store of byte vector from/to boolean[] is supported
927 return true;
928 } else {
929 return false;
930 }
931 }
932
933 // public static
934 // <C,
935 // VM,
936 // E,
937 // S extends VectorSpecies<E>>
938 // VM load(Class<? extends VM> vmClass, Class<E> elementType, int length,
939 // Object base, long offset, // Unsafe addressing
940 // C container, int index, S s, // Arguments for default implementation
941 // LoadOperation<C, VM, E, S> defaultImpl)
942 //
943 // public static
944 // <C,
945 // V extends Vector<?>>
946 // void store(Class<?> vectorClass, Class<?> elementType, int length,
947 // Object base, long offset, // Unsafe addressing
948 // V v,
949 // C container, int index, // Arguments for default implementation
950 // StoreVectorOperation<C, V> defaultImpl)
951
952 bool LibraryCallKit::inline_vector_mem_operation(bool is_store) {
953 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
954 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
955 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
956
957 if (vector_klass == NULL || elem_klass == NULL || vlen == NULL ||
958 vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || !vlen->is_con()) {
959 if (C->print_intrinsics()) {
960 tty->print_cr(" ** missing constant: vclass=%s etype=%s vlen=%s",
961 NodeClassNames[argument(0)->Opcode()],
962 NodeClassNames[argument(1)->Opcode()],
963 NodeClassNames[argument(2)->Opcode()]);
964 }
965 return false; // not enough info for intrinsification
966 }
967 if (!is_klass_initialized(vector_klass)) {
968 if (C->print_intrinsics()) {
969 tty->print_cr(" ** klass argument not initialized");
970 }
971 return false;
972 }
973
974 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
975 if (!elem_type->is_primitive_type()) {
976 if (C->print_intrinsics()) {
977 tty->print_cr(" ** not a primitive bt=%d", elem_type->basic_type());
978 }
979 return false; // should be primitive type
980 }
981 BasicType elem_bt = elem_type->basic_type();
982 int num_elem = vlen->get_con();
983
984 // TODO When mask usage is supported, VecMaskNotUsed needs to be VecMaskUseLoad.
985 if (!arch_supports_vector(is_store ? Op_StoreVector : Op_LoadVector, num_elem, elem_bt, VecMaskNotUsed)) {
986 if (C->print_intrinsics()) {
987 tty->print_cr(" ** not supported: arity=%d op=%s vlen=%d etype=%s ismask=no",
988 is_store, is_store ? "store" : "load",
989 num_elem, type2name(elem_bt));
990 }
991 return false; // not supported
992 }
993
994 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
995 bool is_mask = is_vector_mask(vbox_klass);
996
997 Node* base = argument(3);
998 Node* offset = ConvL2X(argument(4));
999
1000 // Save state and restore on bailout
1001 uint old_sp = sp();
1002 SafePointNode* old_map = clone_map();
1003
1004 Node* addr = make_unsafe_address(base, offset, (is_mask ? T_BOOLEAN : elem_bt), true);
1005
1006 // The memory barrier checks are based on ones for unsafe access.
1007 // This is not 1-1 implementation.
1008 const Type *const base_type = gvn().type(base);
1009
1010 const TypePtr *addr_type = gvn().type(addr)->isa_ptr();
1011 const TypeAryPtr* arr_type = addr_type->isa_aryptr();
1012
1013 const bool in_native = TypePtr::NULL_PTR == base_type; // base always null
1014 const bool in_heap = !TypePtr::NULL_PTR->higher_equal(base_type); // base never null
1015
1016 const bool is_mixed_access = !in_heap && !in_native;
1017
1018 const bool is_mismatched_access = in_heap && (addr_type->isa_aryptr() == NULL);
1019
1020 const bool needs_cpu_membar = is_mixed_access || is_mismatched_access;
1021
1022 // Now handle special case where load/store happens from/to byte array but element type is not byte.
1023 bool using_byte_array = arr_type != NULL && arr_type->elem()->array_element_basic_type() == T_BYTE && elem_bt != T_BYTE;
1024 // Handle loading masks.
1025 // If there is no consistency between array and vector element types, it must be special byte array case or loading masks
1026 if (arr_type != NULL && !using_byte_array && !is_mask && !elem_consistent_with_arr(elem_bt, arr_type)) {
1027 if (C->print_intrinsics()) {
1028 tty->print_cr(" ** not supported: arity=%d op=%s vlen=%d etype=%s atype=%s ismask=no",
1029 is_store, is_store ? "store" : "load",
1030 num_elem, type2name(elem_bt), type2name(arr_type->elem()->array_element_basic_type()));
1031 }
1032 set_map(old_map);
1033 set_sp(old_sp);
1034 return false;
1035 }
1036 // Since we are using byte array, we need to double check that the byte operations are supported by backend.
1037 if (using_byte_array) {
1038 int byte_num_elem = num_elem * type2aelembytes(elem_bt);
1039 if (!arch_supports_vector(is_store ? Op_StoreVector : Op_LoadVector, byte_num_elem, T_BYTE, VecMaskNotUsed)
1040 || !arch_supports_vector(Op_VectorReinterpret, byte_num_elem, T_BYTE, VecMaskNotUsed)) {
1041 if (C->print_intrinsics()) {
1042 tty->print_cr(" ** not supported: arity=%d op=%s vlen=%d*8 etype=%s/8 ismask=no",
1043 is_store, is_store ? "store" : "load",
1044 byte_num_elem, type2name(elem_bt));
1045 }
1046 set_map(old_map);
1047 set_sp(old_sp);
1048 return false; // not supported
1049 }
1050 }
1051 if (is_mask) {
1052 if (!arch_supports_vector(Op_LoadVector, num_elem, T_BOOLEAN, VecMaskNotUsed)) {
1053 if (C->print_intrinsics()) {
1054 tty->print_cr(" ** not supported: arity=%d op=%s/mask vlen=%d etype=bit ismask=no",
1055 is_store, is_store ? "store" : "load",
1056 num_elem);
1057 }
1058 set_map(old_map);
1059 set_sp(old_sp);
1060 return false; // not supported
1061 }
1062 if (!is_store) {
1063 if (!arch_supports_vector(Op_LoadVector, num_elem, elem_bt, VecMaskUseLoad)) {
1064 set_map(old_map);
1065 set_sp(old_sp);
1066 return false; // not supported
1067 }
1068 } else {
1069 if (!arch_supports_vector(Op_StoreVector, num_elem, elem_bt, VecMaskUseStore)) {
1070 set_map(old_map);
1071 set_sp(old_sp);
1072 return false; // not supported
1073 }
1074 }
1075 }
1076
1077 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1078
1079 if (needs_cpu_membar) {
1080 insert_mem_bar(Op_MemBarCPUOrder);
1081 }
1082
1083 if (is_store) {
1084 Node* val = unbox_vector(argument(6), vbox_type, elem_bt, num_elem);
1085 if (val == NULL) {
1086 set_map(old_map);
1087 set_sp(old_sp);
1088 return false; // operand unboxing failed
1089 }
1090 set_all_memory(reset_memory());
1091
1092 // In case the store needs to happen to byte array, reinterpret the incoming vector to byte vector.
1093 int store_num_elem = num_elem;
1094 if (using_byte_array) {
1095 store_num_elem = num_elem * type2aelembytes(elem_bt);
1096 const TypeVect* to_vect_type = TypeVect::make(T_BYTE, store_num_elem);
1097 val = gvn().transform(new VectorReinterpretNode(val, val->bottom_type()->is_vect(), to_vect_type));
1098 }
1099
1100 Node* vstore = gvn().transform(StoreVectorNode::make(0, control(), memory(addr), addr, addr_type, val, store_num_elem));
1101 set_memory(vstore, addr_type);
1102 } else {
1103 // When using byte array, we need to load as byte then reinterpret the value. Otherwise, do a simple vector load.
1104 Node* vload = NULL;
1105 if (using_byte_array) {
1106 int load_num_elem = num_elem * type2aelembytes(elem_bt);
1107 vload = gvn().transform(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, load_num_elem, T_BYTE));
1108 const TypeVect* to_vect_type = TypeVect::make(elem_bt, num_elem);
1109 vload = gvn().transform(new VectorReinterpretNode(vload, vload->bottom_type()->is_vect(), to_vect_type));
1110 } else {
1111 // Special handle for masks
1112 if (is_mask) {
1113 vload = gvn().transform(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, num_elem, T_BOOLEAN));
1114 vload = gvn().transform(new VectorLoadMaskNode(vload, TypeVect::makemask(elem_bt, num_elem)));
1115 } else {
1116 vload = gvn().transform(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, num_elem, elem_bt));
1117 }
1118 }
1119 Node* box = box_vector(vload, vbox_type, elem_bt, num_elem);
1120 set_result(box);
1121 }
1122
1123 old_map->destruct(&_gvn);
1124
1125 if (needs_cpu_membar) {
1126 insert_mem_bar(Op_MemBarCPUOrder);
1127 }
1128
1129 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1130 return true;
1131 }
1132
1133 // public static
1134 // <C,
1135 // V extends Vector<?>,
1136 // E,
1137 // S extends VectorSpecies<E>,
1138 // M extends VectorMask<E>>
1139 // V loadMasked(Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType,
1140 // int length, Object base, long offset, M m,
1141 // C container, int index, S s, // Arguments for default implementation
1142 // LoadVectorMaskedOperation<C, V, S, M> defaultImpl) {
1143 //
1144 // public static
1145 // <C,
1146 // V extends Vector<E>,
1147 // M extends VectorMask<E>,
1148 // E>
1149 // void storeMasked(Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType,
1150 // int length, Object base, long offset,
1151 // V v, M m,
1152 // C container, int index, // Arguments for default implementation
1153 // StoreVectorMaskedOperation<C, V, M, E> defaultImpl) {
1154 //
1155 bool LibraryCallKit::inline_vector_mem_masked_operation(bool is_store) {
1156 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
1157 const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
1158 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
1159 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
1160
1161 if (vector_klass == NULL || mask_klass == NULL || elem_klass == NULL || vlen == NULL ||
1162 vector_klass->const_oop() == NULL || mask_klass->const_oop() == NULL ||
1163 elem_klass->const_oop() == NULL || !vlen->is_con()) {
1164 if (C->print_intrinsics()) {
1165 tty->print_cr(" ** missing constant: vclass=%s mclass=%s etype=%s vlen=%s",
1166 NodeClassNames[argument(0)->Opcode()],
1167 NodeClassNames[argument(1)->Opcode()],
1168 NodeClassNames[argument(2)->Opcode()],
1169 NodeClassNames[argument(3)->Opcode()]);
1170 }
1171 return false; // not enough info for intrinsification
1172 }
1173 if (!is_klass_initialized(vector_klass)) {
1174 if (C->print_intrinsics()) {
1175 tty->print_cr(" ** klass argument not initialized");
1176 }
1177 return false;
1178 }
1179
1180 if (!is_klass_initialized(mask_klass)) {
1181 if (C->print_intrinsics()) {
1182 tty->print_cr(" ** mask klass argument not initialized");
1183 }
1184 return false;
1185 }
1186
1187 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1188 if (!elem_type->is_primitive_type()) {
1189 if (C->print_intrinsics()) {
1190 tty->print_cr(" ** not a primitive bt=%d", elem_type->basic_type());
1191 }
1192 return false; // should be primitive type
1193 }
1194
1195 BasicType elem_bt = elem_type->basic_type();
1196 int num_elem = vlen->get_con();
1197
1198 Node* base = argument(4);
1199 Node* offset = ConvL2X(argument(5));
1200
1201 // Save state and restore on bailout
1202 uint old_sp = sp();
1203 SafePointNode* old_map = clone_map();
1204
1205 Node* addr = make_unsafe_address(base, offset, elem_bt, true);
1206 const TypePtr *addr_type = gvn().type(addr)->isa_ptr();
1207 const TypeAryPtr* arr_type = addr_type->isa_aryptr();
1208
1209 // Now handle special case where load/store happens from/to byte array but element type is not byte.
1210 bool using_byte_array = arr_type != NULL && arr_type->elem()->array_element_basic_type() == T_BYTE && elem_bt != T_BYTE;
1211 // If there is no consistency between array and vector element types, it must be special byte array case
1212 if (arr_type != NULL && !using_byte_array && !elem_consistent_with_arr(elem_bt, arr_type)) {
1213 if (C->print_intrinsics()) {
1214 tty->print_cr(" ** not supported: arity=%d op=%s vlen=%d etype=%s atype=%s",
1215 is_store, is_store ? "storeMasked" : "loadMasked",
1216 num_elem, type2name(elem_bt), type2name(arr_type->elem()->array_element_basic_type()));
1217 }
1218 set_map(old_map);
1219 set_sp(old_sp);
1220 return false;
1221 }
1222
1223 int mem_num_elem = using_byte_array ? num_elem * type2aelembytes(elem_bt) : num_elem;
1224 BasicType mem_elem_bt = using_byte_array ? T_BYTE : elem_bt;
1225 bool use_predicate = arch_supports_vector(is_store ? Op_StoreVectorMasked : Op_LoadVectorMasked,
1226 mem_num_elem, mem_elem_bt,
1227 (VectorMaskUseType) (VecMaskUseLoad | VecMaskUsePred));
1228 // Masked vector store operation needs the architecture predicate feature. We need to check
1229 // whether the predicated vector operation is supported by backend.
1230 if (is_store && !use_predicate) {
1231 if (C->print_intrinsics()) {
1232 tty->print_cr(" ** not supported: op=storeMasked vlen=%d etype=%s using_byte_array=%d",
1233 num_elem, type2name(elem_bt), using_byte_array ? 1 : 0);
1234 }
1235 set_map(old_map);
1236 set_sp(old_sp);
1237 return false;
1238 }
1239
1240 // This only happens for masked vector load. If predicate is not supported, then check whether
1241 // the normal vector load and blend operations are supported by backend.
1242 if (!use_predicate && (!arch_supports_vector(Op_LoadVector, mem_num_elem, mem_elem_bt, VecMaskNotUsed) ||
1243 !arch_supports_vector(Op_VectorBlend, mem_num_elem, mem_elem_bt, VecMaskUseLoad))) {
1244 if (C->print_intrinsics()) {
1245 tty->print_cr(" ** not supported: op=loadMasked vlen=%d etype=%s using_byte_array=%d",
1246 num_elem, type2name(elem_bt), using_byte_array ? 1 : 0);
1247 }
1248 set_map(old_map);
1249 set_sp(old_sp);
1250 return false;
1251 }
1252
1253 // Since we are using byte array, we need to double check that the vector reinterpret operation
1254 // with byte type is supported by backend.
1255 if (using_byte_array) {
1256 if (!arch_supports_vector(Op_VectorReinterpret, mem_num_elem, T_BYTE, VecMaskNotUsed)) {
1257 if (C->print_intrinsics()) {
1258 tty->print_cr(" ** not supported: arity=%d op=%s vlen=%d etype=%s using_byte_array=1",
1259 is_store, is_store ? "storeMasked" : "loadMasked",
1260 num_elem, type2name(elem_bt));
1261 }
1262 set_map(old_map);
1263 set_sp(old_sp);
1264 return false;
1265 }
1266 }
1267
1268 // Since it needs to unbox the mask, we need to double check that the related load operations
1269 // for mask are supported by backend.
1270 if (!arch_supports_vector(Op_LoadVector, num_elem, elem_bt, VecMaskUseLoad)) {
1271 if (C->print_intrinsics()) {
1272 tty->print_cr(" ** not supported: arity=%d op=%s vlen=%d etype=%s",
1273 is_store, is_store ? "storeMasked" : "loadMasked",
1274 num_elem, type2name(elem_bt));
1275 }
1276 set_map(old_map);
1277 set_sp(old_sp);
1278 return false;
1279 }
1280
1281 // Can base be NULL? Otherwise, always on-heap access.
1282 bool can_access_non_heap = TypePtr::NULL_PTR->higher_equal(gvn().type(base));
1283 if (can_access_non_heap) {
1284 insert_mem_bar(Op_MemBarCPUOrder);
1285 }
1286
1287 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1288 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1289 assert(!is_vector_mask(vbox_klass) && is_vector_mask(mbox_klass), "Invalid class type");
1290 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1291 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1292
1293 Node* mask = unbox_vector(is_store ? argument(8) : argument(7), mbox_type, elem_bt, num_elem);
1294 if (mask == NULL) {
1295 if (C->print_intrinsics()) {
1296 tty->print_cr(" ** unbox failed mask=%s",
1297 is_store ? NodeClassNames[argument(8)->Opcode()]
1298 : NodeClassNames[argument(7)->Opcode()]);
1299 }
1300 set_map(old_map);
1301 set_sp(old_sp);
1302 return false;
1303 }
1304
1305 if (is_store) {
1306 Node* val = unbox_vector(argument(7), vbox_type, elem_bt, num_elem);
1307 if (val == NULL) {
1308 if (C->print_intrinsics()) {
1309 tty->print_cr(" ** unbox failed vector=%s",
1310 NodeClassNames[argument(7)->Opcode()]);
1311 }
1312 set_map(old_map);
1313 set_sp(old_sp);
1314 return false; // operand unboxing failed
1315 }
1316 set_all_memory(reset_memory());
1317
1318 if (using_byte_array) {
1319 // Reinterpret the incoming vector to byte vector.
1320 const TypeVect* to_vect_type = TypeVect::make(mem_elem_bt, mem_num_elem);
1321 val = gvn().transform(new VectorReinterpretNode(val, val->bottom_type()->is_vect(), to_vect_type));
1322 // Reinterpret the vector mask to byte type.
1323 const TypeVect* from_mask_type = TypeVect::makemask(elem_bt, num_elem);
1324 const TypeVect* to_mask_type = TypeVect::makemask(mem_elem_bt, mem_num_elem);
1325 mask = gvn().transform(new VectorReinterpretNode(mask, from_mask_type, to_mask_type));
1326 }
1327 Node* vstore = gvn().transform(new StoreVectorMaskedNode(control(), memory(addr), addr, val, addr_type, mask));
1328 set_memory(vstore, addr_type);
1329 } else {
1330 Node* vload = NULL;
1331
1332 if (using_byte_array) {
1333 // Reinterpret the vector mask to byte type.
1334 const TypeVect* from_mask_type = TypeVect::makemask(elem_bt, num_elem);
1335 const TypeVect* to_mask_type = TypeVect::makemask(mem_elem_bt, mem_num_elem);
1336 mask = gvn().transform(new VectorReinterpretNode(mask, from_mask_type, to_mask_type));
1337 }
1338
1339 if (use_predicate) {
1340 // Generate masked load vector node if predicate feature is supported.
1341 const TypeVect* vt = TypeVect::make(mem_elem_bt, mem_num_elem);
1342 vload = gvn().transform(new LoadVectorMaskedNode(control(), memory(addr), addr, addr_type, vt, mask));
1343 } else {
1344 // Use the vector blend to implement the masked load vector. The biased elements are zeros.
1345 Node* zero = gvn().transform(gvn().zerocon(mem_elem_bt));
1346 zero = gvn().transform(VectorNode::scalar2vector(zero, mem_num_elem, Type::get_const_basic_type(mem_elem_bt)));
1347 vload = gvn().transform(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, mem_num_elem, mem_elem_bt));
1348 vload = gvn().transform(new VectorBlendNode(zero, vload, mask));
1349 }
1350
1351 if (using_byte_array) {
1352 const TypeVect* to_vect_type = TypeVect::make(elem_bt, num_elem);
1353 vload = gvn().transform(new VectorReinterpretNode(vload, vload->bottom_type()->is_vect(), to_vect_type));
1354 }
1355
1356 Node* box = box_vector(vload, vbox_type, elem_bt, num_elem);
1357 set_result(box);
1358 }
1359
1360 old_map->destruct(&_gvn);
1361
1362 if (can_access_non_heap) {
1363 insert_mem_bar(Op_MemBarCPUOrder);
1364 }
1365
1366 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1367 return true;
1368 }
1369
1370 // <C,
1371 // V extends Vector<?>,
1372 // W extends Vector<Integer>,
1373 // S extends VectorSpecies<E>,
1374 // M extends VectorMask<E>,
1375 // E>
1376 // V loadWithMap(Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType, int length,
1377 // Class<? extends Vector<Integer>> vectorIndexClass,
1378 // Object base, long offset, // Unsafe addressing
1379 // W index_vector, M m,
1380 // C container, int index, int[] indexMap, int indexM, S s, // Arguments for default implementation
1381 // LoadVectorOperationWithMap<C, V, E, S, M> defaultImpl)
1382 //
1383 // <C,
1384 // V extends Vector<E>,
1385 // W extends Vector<Integer>,
1386 // M extends VectorMask<E>,
1387 // E>
1388 // void storeWithMap(Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType,
1389 // int length, Class<? extends Vector<Integer>> vectorIndexClass, Object base, long offset, // Unsafe addressing
1390 // W index_vector, V v, M m,
1391 // C container, int index, int[] indexMap, int indexM, // Arguments for default implementation
1392 // StoreVectorOperationWithMap<C, V, M, E> defaultImpl)
1393 //
1394 bool LibraryCallKit::inline_vector_gather_scatter(bool is_scatter) {
1395 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
1396 const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
1397 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
1398 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
1399 const TypeInstPtr* vector_idx_klass = gvn().type(argument(4))->isa_instptr();
1400
1401 if (vector_klass == NULL || elem_klass == NULL || vector_idx_klass == NULL || vlen == NULL ||
1402 vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || vector_idx_klass->const_oop() == NULL || !vlen->is_con()) {
1403 if (C->print_intrinsics()) {
1404 tty->print_cr(" ** missing constant: vclass=%s etype=%s vlen=%s viclass=%s",
1405 NodeClassNames[argument(0)->Opcode()],
1406 NodeClassNames[argument(2)->Opcode()],
1407 NodeClassNames[argument(3)->Opcode()],
1408 NodeClassNames[argument(4)->Opcode()]);
1409 }
1410 return false; // not enough info for intrinsification
1411 }
1412
1413 if (!is_klass_initialized(vector_klass) || !is_klass_initialized(vector_idx_klass)) {
1414 if (C->print_intrinsics()) {
1415 tty->print_cr(" ** klass argument not initialized");
1416 }
1417 return false;
1418 }
1419
1420 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1421 if (!elem_type->is_primitive_type()) {
1422 if (C->print_intrinsics()) {
1423 tty->print_cr(" ** not a primitive bt=%d", elem_type->basic_type());
1424 }
1425 return false; // should be primitive type
1426 }
1427
1428 BasicType elem_bt = elem_type->basic_type();
1429 int num_elem = vlen->get_con();
1430
1431 const Type* vmask_type = gvn().type(is_scatter ? argument(10) : argument(9));
1432 bool is_masked_op = vmask_type != TypePtr::NULL_PTR;
1433 if (is_masked_op) {
1434 if (mask_klass == NULL || mask_klass->const_oop() == NULL) {
1435 if (C->print_intrinsics()) {
1436 tty->print_cr(" ** missing constant: maskclass=%s", NodeClassNames[argument(1)->Opcode()]);
1437 }
1438 return false; // not enough info for intrinsification
1439 }
1440
1441 if (!is_klass_initialized(mask_klass)) {
1442 if (C->print_intrinsics()) {
1443 tty->print_cr(" ** mask klass argument not initialized");
1444 }
1445 return false;
1446 }
1447
1448 if (vmask_type->maybe_null()) {
1449 if (C->print_intrinsics()) {
1450 tty->print_cr(" ** null mask values are not allowed for masked op");
1451 }
1452 return false;
1453 }
1454
1455 // Check whether the predicated gather/scatter node is supported by architecture.
1456 if (!arch_supports_vector(is_scatter ? Op_StoreVectorScatterMasked : Op_LoadVectorGatherMasked, num_elem, elem_bt,
1457 (VectorMaskUseType) (VecMaskUseLoad | VecMaskUsePred))) {
1458 if (C->print_intrinsics()) {
1459 tty->print_cr(" ** not supported: arity=%d op=%s vlen=%d etype=%s is_masked_op=1",
1460 is_scatter, is_scatter ? "scatterMasked" : "gatherMasked",
1461 num_elem, type2name(elem_bt));
1462 }
1463 return false; // not supported
1464 }
1465 } else {
1466 // Check whether the normal gather/scatter node is supported for non-masked operation.
1467 if (!arch_supports_vector(is_scatter ? Op_StoreVectorScatter : Op_LoadVectorGather, num_elem, elem_bt, VecMaskNotUsed)) {
1468 if (C->print_intrinsics()) {
1469 tty->print_cr(" ** not supported: arity=%d op=%s vlen=%d etype=%s is_masked_op=0",
1470 is_scatter, is_scatter ? "scatter" : "gather",
1471 num_elem, type2name(elem_bt));
1472 }
1473 return false; // not supported
1474 }
1475 }
1476
1477 // Check that the vector holding indices is supported by architecture
1478 if (!arch_supports_vector(Op_LoadVector, num_elem, T_INT, VecMaskNotUsed)) {
1479 if (C->print_intrinsics()) {
1480 tty->print_cr(" ** not supported: arity=%d op=%s/loadindex vlen=%d etype=int is_masked_op=%d",
1481 is_scatter, is_scatter ? "scatter" : "gather",
1482 num_elem, is_masked_op ? 1 : 0);
1483 }
1484 return false; // not supported
1485 }
1486
1487 Node* base = argument(5);
1488 Node* offset = ConvL2X(argument(6));
1489
1490 // Save state and restore on bailout
1491 uint old_sp = sp();
1492 SafePointNode* old_map = clone_map();
1493
1494 Node* addr = make_unsafe_address(base, offset, elem_bt, true);
1495
1496 const TypePtr *addr_type = gvn().type(addr)->isa_ptr();
1497 const TypeAryPtr* arr_type = addr_type->isa_aryptr();
1498
1499 // The array must be consistent with vector type
1500 if (arr_type == NULL || (arr_type != NULL && !elem_consistent_with_arr(elem_bt, arr_type))) {
1501 if (C->print_intrinsics()) {
1502 tty->print_cr(" ** not supported: arity=%d op=%s vlen=%d etype=%s atype=%s ismask=no",
1503 is_scatter, is_scatter ? "scatter" : "gather",
1504 num_elem, type2name(elem_bt), type2name(arr_type->elem()->array_element_basic_type()));
1505 }
1506 set_map(old_map);
1507 set_sp(old_sp);
1508 return false;
1509 }
1510
1511 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1512 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1513 ciKlass* vbox_idx_klass = vector_idx_klass->const_oop()->as_instance()->java_lang_Class_klass();
1514 if (vbox_idx_klass == NULL) {
1515 set_map(old_map);
1516 set_sp(old_sp);
1517 return false;
1518 }
1519
1520 const TypeInstPtr* vbox_idx_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_idx_klass);
1521 Node* index_vect = unbox_vector(argument(8), vbox_idx_type, T_INT, num_elem);
1522 if (index_vect == NULL) {
1523 set_map(old_map);
1524 set_sp(old_sp);
1525 return false;
1526 }
1527
1528 Node* mask = NULL;
1529 if (is_masked_op) {
1530 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1531 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1532 mask = unbox_vector(is_scatter ? argument(10) : argument(9), mbox_type, elem_bt, num_elem);
1533 if (mask == NULL) {
1534 if (C->print_intrinsics()) {
1535 tty->print_cr(" ** unbox failed mask=%s",
1536 is_scatter ? NodeClassNames[argument(10)->Opcode()]
1537 : NodeClassNames[argument(9)->Opcode()]);
1538 }
1539 set_map(old_map);
1540 set_sp(old_sp);
1541 return false;
1542 }
1543 }
1544
1545 const TypeVect* vector_type = TypeVect::make(elem_bt, num_elem);
1546 if (is_scatter) {
1547 Node* val = unbox_vector(argument(9), vbox_type, elem_bt, num_elem);
1548 if (val == NULL) {
1549 set_map(old_map);
1550 set_sp(old_sp);
1551 return false; // operand unboxing failed
1552 }
1553 set_all_memory(reset_memory());
1554
1555 Node* vstore = NULL;
1556 if (mask != NULL) {
1557 vstore = gvn().transform(new StoreVectorScatterMaskedNode(control(), memory(addr), addr, addr_type, val, index_vect, mask));
1558 } else {
1559 vstore = gvn().transform(new StoreVectorScatterNode(control(), memory(addr), addr, addr_type, val, index_vect));
1560 }
1561 set_memory(vstore, addr_type);
1562 } else {
1563 Node* vload = NULL;
1564 if (mask != NULL) {
1565 vload = gvn().transform(new LoadVectorGatherMaskedNode(control(), memory(addr), addr, addr_type, vector_type, index_vect, mask));
1566 } else {
1567 vload = gvn().transform(new LoadVectorGatherNode(control(), memory(addr), addr, addr_type, vector_type, index_vect));
1568 }
1569 Node* box = box_vector(vload, vbox_type, elem_bt, num_elem);
1570 set_result(box);
1571 }
1572
1573 old_map->destruct(&_gvn);
1574
1575 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1576 return true;
1577 }
1578
1579 // public static
1580 // <V extends Vector<E>,
1581 // M extends VectorMask<E>,
1582 // E>
1583 // long reductionCoerced(int oprId, Class<? extends V> vectorClass, Class<? extends M> maskClass,
1584 // Class<E> elementType, int length, V v, M m,
1585 // ReductionOperation<V, M> defaultImpl)
1586 bool LibraryCallKit::inline_vector_reduction() {
1587 const TypeInt* opr = gvn().type(argument(0))->isa_int();
1588 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
1589 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
1590 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
1591 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
1592
1593 if (opr == NULL || vector_klass == NULL || elem_klass == NULL || vlen == NULL ||
1594 !opr->is_con() || vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || !vlen->is_con()) {
1595 if (C->print_intrinsics()) {
1596 tty->print_cr(" ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
1597 NodeClassNames[argument(0)->Opcode()],
1598 NodeClassNames[argument(1)->Opcode()],
1599 NodeClassNames[argument(3)->Opcode()],
1600 NodeClassNames[argument(4)->Opcode()]);
1601 }
1602 return false; // not enough info for intrinsification
1603 }
1604 if (!is_klass_initialized(vector_klass)) {
1605 if (C->print_intrinsics()) {
1606 tty->print_cr(" ** klass argument not initialized");
1607 }
1608 return false;
1609 }
1610 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1611 if (!elem_type->is_primitive_type()) {
1612 if (C->print_intrinsics()) {
1613 tty->print_cr(" ** not a primitive bt=%d", elem_type->basic_type());
1614 }
1615 return false; // should be primitive type
1616 }
1617
1618 const Type* vmask_type = gvn().type(argument(6));
1619 bool is_masked_op = vmask_type != TypePtr::NULL_PTR;
1620 if (is_masked_op) {
1621 if (mask_klass == NULL || mask_klass->const_oop() == NULL) {
1622 if (C->print_intrinsics()) {
1623 tty->print_cr(" ** missing constant: maskclass=%s", NodeClassNames[argument(2)->Opcode()]);
1624 }
1625 return false; // not enough info for intrinsification
1626 }
1627
1628 if (!is_klass_initialized(mask_klass)) {
1629 if (C->print_intrinsics()) {
1630 tty->print_cr(" ** mask klass argument not initialized");
1631 }
1632 return false;
1633 }
1634
1635 if (vmask_type->maybe_null()) {
1636 if (C->print_intrinsics()) {
1637 tty->print_cr(" ** null mask values are not allowed for masked op");
1638 }
1639 return false;
1640 }
1641 }
1642
1643 BasicType elem_bt = elem_type->basic_type();
1644 int num_elem = vlen->get_con();
1645 int opc = VectorSupport::vop2ideal(opr->get_con(), elem_bt);
1646 int sopc = ReductionNode::opcode(opc, elem_bt);
1647
1648 // When using mask, mask use type needs to be VecMaskUseLoad.
1649 if (!arch_supports_vector(sopc, num_elem, elem_bt, is_masked_op ? VecMaskUseLoad : VecMaskNotUsed)) {
1650 if (C->print_intrinsics()) {
1651 tty->print_cr(" ** not supported: arity=1 op=%d/reduce vlen=%d etype=%s is_masked_op=%d",
1652 sopc, num_elem, type2name(elem_bt), is_masked_op ? 1 : 0);
1653 }
1654 return false;
1655 }
1656
1657 // Return true if current platform has implemented the masked operation with predicate feature.
1658 bool use_predicate = is_masked_op && arch_supports_vector(sopc, num_elem, elem_bt, VecMaskUsePred);
1659 if (is_masked_op && !use_predicate && !arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad)) {
1660 if (C->print_intrinsics()) {
1661 tty->print_cr(" ** not supported: arity=1 op=%d/reduce vlen=%d etype=%s is_masked_op=1",
1662 sopc, num_elem, type2name(elem_bt));
1663 }
1664 return false;
1665 }
1666
1667 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1668 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1669
1670 Node* opd = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
1671 if (opd == NULL) {
1672 return false; // operand unboxing failed
1673 }
1674
1675 Node* mask = NULL;
1676 if (is_masked_op) {
1677 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1678 assert(is_vector_mask(mbox_klass), "argument(2) should be a mask class");
1679 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1680 mask = unbox_vector(argument(6), mbox_type, elem_bt, num_elem);
1681 if (mask == NULL) {
1682 if (C->print_intrinsics()) {
1683 tty->print_cr(" ** unbox failed mask=%s",
1684 NodeClassNames[argument(6)->Opcode()]);
1685 }
1686 return false;
1687 }
1688 }
1689
1690 Node* init = ReductionNode::make_reduction_input(gvn(), opc, elem_bt);
1691 Node* value = NULL;
1692 if (mask == NULL) {
1693 assert(!is_masked_op, "Masked op needs the mask value never null");
1694 value = ReductionNode::make(opc, NULL, init, opd, elem_bt);
1695 } else {
1696 if (use_predicate) {
1697 value = ReductionNode::make(opc, NULL, init, opd, elem_bt);
1698 value->add_req(mask);
1699 value->add_flag(Node::Flag_is_predicated_vector);
1700 } else {
1701 Node* reduce_identity = gvn().transform(VectorNode::scalar2vector(init, num_elem, Type::get_const_basic_type(elem_bt)));
1702 value = gvn().transform(new VectorBlendNode(reduce_identity, opd, mask));
1703 value = ReductionNode::make(opc, NULL, init, value, elem_bt);
1704 }
1705 }
1706 value = gvn().transform(value);
1707
1708 Node* bits = NULL;
1709 switch (elem_bt) {
1710 case T_BYTE:
1711 case T_SHORT:
1712 case T_INT: {
1713 bits = gvn().transform(new ConvI2LNode(value));
1714 break;
1715 }
1716 case T_FLOAT: {
1717 value = gvn().transform(new MoveF2INode(value));
1718 bits = gvn().transform(new ConvI2LNode(value));
1719 break;
1720 }
1721 case T_DOUBLE: {
1722 bits = gvn().transform(new MoveD2LNode(value));
1723 break;
1724 }
1725 case T_LONG: {
1726 bits = value; // no conversion needed
1727 break;
1728 }
1729 default: fatal("%s", type2name(elem_bt));
1730 }
1731 set_result(bits);
1732 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1733 return true;
1734 }
1735
1736 // public static <V> boolean test(int cond, Class<?> vectorClass, Class<?> elementType, int vlen,
1737 // V v1, V v2,
1738 // BiFunction<V, V, Boolean> defaultImpl)
1739 //
1740 bool LibraryCallKit::inline_vector_test() {
1741 const TypeInt* cond = gvn().type(argument(0))->isa_int();
1742 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
1743 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
1744 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
1745
1746 if (cond == NULL || vector_klass == NULL || elem_klass == NULL || vlen == NULL ||
1747 !cond->is_con() || vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || !vlen->is_con()) {
1748 if (C->print_intrinsics()) {
1749 tty->print_cr(" ** missing constant: cond=%s vclass=%s etype=%s vlen=%s",
1750 NodeClassNames[argument(0)->Opcode()],
1751 NodeClassNames[argument(1)->Opcode()],
1752 NodeClassNames[argument(2)->Opcode()],
1753 NodeClassNames[argument(3)->Opcode()]);
1754 }
1755 return false; // not enough info for intrinsification
1756 }
1757 if (!is_klass_initialized(vector_klass)) {
1758 if (C->print_intrinsics()) {
1759 tty->print_cr(" ** klass argument not initialized");
1760 }
1761 return false;
1762 }
1763 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1764 if (!elem_type->is_primitive_type()) {
1765 if (C->print_intrinsics()) {
1766 tty->print_cr(" ** not a primitive bt=%d", elem_type->basic_type());
1767 }
1768 return false; // should be primitive type
1769 }
1770 BasicType elem_bt = elem_type->basic_type();
1771 int num_elem = vlen->get_con();
1772 BoolTest::mask booltest = (BoolTest::mask)cond->get_con();
1773 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1774 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1775
1776 if (!arch_supports_vector(Op_VectorTest, num_elem, elem_bt, is_vector_mask(vbox_klass) ? VecMaskUseLoad : VecMaskNotUsed)) {
1777 if (C->print_intrinsics()) {
1778 tty->print_cr(" ** not supported: arity=2 op=test/%d vlen=%d etype=%s ismask=%d",
1779 cond->get_con(), num_elem, type2name(elem_bt),
1780 is_vector_mask(vbox_klass));
1781 }
1782 return false;
1783 }
1784
1785 Node* opd1 = unbox_vector(argument(4), vbox_type, elem_bt, num_elem);
1786 Node* opd2 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
1787 if (opd1 == NULL || opd2 == NULL) {
1788 return false; // operand unboxing failed
1789 }
1790 Node* test = new VectorTestNode(opd1, opd2, booltest);
1791 test = gvn().transform(test);
1792
1793 set_result(test);
1794 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1795 return true;
1796 }
1797
1798 // public static
1799 // <V extends Vector<E>,
1800 // M extends VectorMask<E>,
1801 // E>
1802 // V blend(Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType, int vlen,
1803 // V v1, V v2, M m,
1804 // VectorBlendOp<V, M, E> defaultImpl)
1805 bool LibraryCallKit::inline_vector_blend() {
1806 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
1807 const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
1808 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
1809 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
1810
1811 if (mask_klass == NULL || vector_klass == NULL || elem_klass == NULL || vlen == NULL) {
1812 return false; // dead code
1813 }
1814 if (mask_klass->const_oop() == NULL || vector_klass->const_oop() == NULL ||
1815 elem_klass->const_oop() == NULL || !vlen->is_con()) {
1816 if (C->print_intrinsics()) {
1817 tty->print_cr(" ** missing constant: vclass=%s mclass=%s etype=%s vlen=%s",
1818 NodeClassNames[argument(0)->Opcode()],
1819 NodeClassNames[argument(1)->Opcode()],
1820 NodeClassNames[argument(2)->Opcode()],
1821 NodeClassNames[argument(3)->Opcode()]);
1822 }
1823 return false; // not enough info for intrinsification
1824 }
1825 if (!is_klass_initialized(vector_klass) || !is_klass_initialized(mask_klass)) {
1826 if (C->print_intrinsics()) {
1827 tty->print_cr(" ** klass argument not initialized");
1828 }
1829 return false;
1830 }
1831 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1832 if (!elem_type->is_primitive_type()) {
1833 if (C->print_intrinsics()) {
1834 tty->print_cr(" ** not a primitive bt=%d", elem_type->basic_type());
1835 }
1836 return false; // should be primitive type
1837 }
1838 BasicType elem_bt = elem_type->basic_type();
1839 BasicType mask_bt = elem_bt;
1840 int num_elem = vlen->get_con();
1841
1842 if (!arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad)) {
1843 if (C->print_intrinsics()) {
1844 tty->print_cr(" ** not supported: arity=2 op=blend vlen=%d etype=%s ismask=useload",
1845 num_elem, type2name(elem_bt));
1846 }
1847 return false; // not supported
1848 }
1849 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1850 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1851
1852 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1853 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1854
1855 Node* v1 = unbox_vector(argument(4), vbox_type, elem_bt, num_elem);
1856 Node* v2 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
1857 Node* mask = unbox_vector(argument(6), mbox_type, mask_bt, num_elem);
1858
1859 if (v1 == NULL || v2 == NULL || mask == NULL) {
1860 return false; // operand unboxing failed
1861 }
1862
1863 Node* blend = gvn().transform(new VectorBlendNode(v1, v2, mask));
1864
1865 Node* box = box_vector(blend, vbox_type, elem_bt, num_elem);
1866 set_result(box);
1867 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1868 return true;
1869 }
1870
1871 // public static
1872 // <V extends Vector<E>,
1873 // M extends VectorMask<E>,
1874 // E>
1875 // M compare(int cond, Class<? extends V> vectorClass, Class<M> maskClass, Class<E> elementType, int vlen,
1876 // V v1, V v2, M m,
1877 // VectorCompareOp<V,M> defaultImpl)
1878 bool LibraryCallKit::inline_vector_compare() {
1879 const TypeInt* cond = gvn().type(argument(0))->isa_int();
1880 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
1881 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
1882 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
1883 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
1884
1885 if (cond == NULL || vector_klass == NULL || mask_klass == NULL || elem_klass == NULL || vlen == NULL) {
1886 return false; // dead code
1887 }
1888 if (!cond->is_con() || vector_klass->const_oop() == NULL || mask_klass->const_oop() == NULL ||
1889 elem_klass->const_oop() == NULL || !vlen->is_con()) {
1890 if (C->print_intrinsics()) {
1891 tty->print_cr(" ** missing constant: cond=%s vclass=%s mclass=%s etype=%s vlen=%s",
1892 NodeClassNames[argument(0)->Opcode()],
1893 NodeClassNames[argument(1)->Opcode()],
1894 NodeClassNames[argument(2)->Opcode()],
1895 NodeClassNames[argument(3)->Opcode()],
1896 NodeClassNames[argument(4)->Opcode()]);
1897 }
1898 return false; // not enough info for intrinsification
1899 }
1900 if (!is_klass_initialized(vector_klass) || !is_klass_initialized(mask_klass)) {
1901 if (C->print_intrinsics()) {
1902 tty->print_cr(" ** klass argument not initialized");
1903 }
1904 return false;
1905 }
1906 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1907 if (!elem_type->is_primitive_type()) {
1908 if (C->print_intrinsics()) {
1909 tty->print_cr(" ** not a primitive bt=%d", elem_type->basic_type());
1910 }
1911 return false; // should be primitive type
1912 }
1913
1914 int num_elem = vlen->get_con();
1915 BasicType elem_bt = elem_type->basic_type();
1916 BasicType mask_bt = elem_bt;
1917
1918 if ((cond->get_con() & BoolTest::unsigned_compare) != 0) {
1919 if (!Matcher::supports_vector_comparison_unsigned(num_elem, elem_bt)) {
1920 if (C->print_intrinsics()) {
1921 tty->print_cr(" ** not supported: unsigned comparison op=comp/%d vlen=%d etype=%s ismask=usestore",
1922 cond->get_con() & (BoolTest::unsigned_compare - 1), num_elem, type2name(elem_bt));
1923 }
1924 return false;
1925 }
1926 }
1927
1928 if (!arch_supports_vector(Op_VectorMaskCmp, num_elem, elem_bt, VecMaskUseStore)) {
1929 if (C->print_intrinsics()) {
1930 tty->print_cr(" ** not supported: arity=2 op=comp/%d vlen=%d etype=%s ismask=usestore",
1931 cond->get_con(), num_elem, type2name(elem_bt));
1932 }
1933 return false;
1934 }
1935
1936 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1937 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1938
1939 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1940 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1941
1942 Node* v1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
1943 Node* v2 = unbox_vector(argument(6), vbox_type, elem_bt, num_elem);
1944
1945 bool is_masked_op = argument(7)->bottom_type() != TypePtr::NULL_PTR;
1946 Node* mask = is_masked_op ? unbox_vector(argument(7), mbox_type, elem_bt, num_elem) : NULL;
1947 if (is_masked_op && mask == NULL) {
1948 if (C->print_intrinsics()) {
1949 tty->print_cr(" ** not supported: mask = null arity=2 op=comp/%d vlen=%d etype=%s ismask=usestore is_masked_op=1",
1950 cond->get_con(), num_elem, type2name(elem_bt));
1951 }
1952 return false;
1953 }
1954
1955 bool use_predicate = is_masked_op && arch_supports_vector(Op_VectorMaskCmp, num_elem, elem_bt, VecMaskUsePred);
1956 if (is_masked_op && !use_predicate && !arch_supports_vector(Op_AndV, num_elem, elem_bt, VecMaskUseLoad)) {
1957 if (C->print_intrinsics()) {
1958 tty->print_cr(" ** not supported: arity=2 op=comp/%d vlen=%d etype=%s ismask=usestore is_masked_op=1",
1959 cond->get_con(), num_elem, type2name(elem_bt));
1960 }
1961 return false;
1962 }
1963
1964 if (v1 == NULL || v2 == NULL) {
1965 return false; // operand unboxing failed
1966 }
1967 BoolTest::mask pred = (BoolTest::mask)cond->get_con();
1968 ConINode* pred_node = (ConINode*)gvn().makecon(cond);
1969
1970 const TypeVect* vmask_type = TypeVect::makemask(mask_bt, num_elem);
1971 Node* operation = new VectorMaskCmpNode(pred, v1, v2, pred_node, vmask_type);
1972
1973 if (is_masked_op) {
1974 if (use_predicate) {
1975 operation->add_req(mask);
1976 operation->add_flag(Node::Flag_is_predicated_vector);
1977 } else {
1978 operation = gvn().transform(operation);
1979 operation = VectorNode::make(Op_AndV, operation, mask, vmask_type);
1980 }
1981 }
1982
1983 operation = gvn().transform(operation);
1984
1985 Node* box = box_vector(operation, mbox_type, mask_bt, num_elem);
1986 set_result(box);
1987 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1988 return true;
1989 }
1990
1991 // public static
1992 // <V extends Vector<E>,
1993 // Sh extends VectorShuffle<E>,
1994 // M extends VectorMask<E>,
1995 // E>
1996 // V rearrangeOp(Class<? extends V> vectorClass, Class<Sh> shuffleClass, Class<M> maskClass, Class<E> elementType, int vlen,
1997 // V v1, Sh sh, M m,
1998 // VectorRearrangeOp<V, Sh, M, E> defaultImpl)
1999 bool LibraryCallKit::inline_vector_rearrange() {
2000 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
2001 const TypeInstPtr* shuffle_klass = gvn().type(argument(1))->isa_instptr();
2002 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
2003 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
2004 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
2005
2006 if (vector_klass == NULL || shuffle_klass == NULL || elem_klass == NULL || vlen == NULL) {
2007 return false; // dead code
2008 }
2009 if (shuffle_klass->const_oop() == NULL ||
2010 vector_klass->const_oop() == NULL ||
2011 elem_klass->const_oop() == NULL ||
2012 !vlen->is_con()) {
2013 if (C->print_intrinsics()) {
2014 tty->print_cr(" ** missing constant: vclass=%s sclass=%s etype=%s vlen=%s",
2015 NodeClassNames[argument(0)->Opcode()],
2016 NodeClassNames[argument(1)->Opcode()],
2017 NodeClassNames[argument(3)->Opcode()],
2018 NodeClassNames[argument(4)->Opcode()]);
2019 }
2020 return false; // not enough info for intrinsification
2021 }
2022 if (!is_klass_initialized(vector_klass) ||
2023 !is_klass_initialized(shuffle_klass)) {
2024 if (C->print_intrinsics()) {
2025 tty->print_cr(" ** klass argument not initialized");
2026 }
2027 return false;
2028 }
2029 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2030 if (!elem_type->is_primitive_type()) {
2031 if (C->print_intrinsics()) {
2032 tty->print_cr(" ** not a primitive bt=%d", elem_type->basic_type());
2033 }
2034 return false; // should be primitive type
2035 }
2036 BasicType elem_bt = elem_type->basic_type();
2037 BasicType shuffle_bt = elem_bt;
2038 int num_elem = vlen->get_con();
2039
2040 if (!arch_supports_vector(Op_VectorLoadShuffle, num_elem, elem_bt, VecMaskNotUsed)) {
2041 if (C->print_intrinsics()) {
2042 tty->print_cr(" ** not supported: arity=0 op=load/shuffle vlen=%d etype=%s ismask=no",
2043 num_elem, type2name(elem_bt));
2044 }
2045 return false; // not supported
2046 }
2047
2048 bool is_masked_op = argument(7)->bottom_type() != TypePtr::NULL_PTR;
2049 bool use_predicate = is_masked_op;
2050 if (is_masked_op &&
2051 (mask_klass == NULL ||
2052 mask_klass->const_oop() == NULL ||
2053 !is_klass_initialized(mask_klass))) {
2054 if (C->print_intrinsics()) {
2055 tty->print_cr(" ** mask_klass argument not initialized");
2056 }
2057 }
2058 VectorMaskUseType checkFlags = (VectorMaskUseType)(is_masked_op ? (VecMaskUseLoad | VecMaskUsePred) : VecMaskNotUsed);
2059 if (!arch_supports_vector(Op_VectorRearrange, num_elem, elem_bt, checkFlags)) {
2060 use_predicate = false;
2061 if(!is_masked_op ||
2062 (!arch_supports_vector(Op_VectorRearrange, num_elem, elem_bt, VecMaskNotUsed) ||
2063 !arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad) ||
2064 !arch_supports_vector(VectorNode::replicate_opcode(elem_bt), num_elem, elem_bt, VecMaskNotUsed))) {
2065 if (C->print_intrinsics()) {
2066 tty->print_cr(" ** not supported: arity=2 op=shuffle/rearrange vlen=%d etype=%s ismask=no",
2067 num_elem, type2name(elem_bt));
2068 }
2069 return false; // not supported
2070 }
2071 }
2072 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2073 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2074
2075 ciKlass* shbox_klass = shuffle_klass->const_oop()->as_instance()->java_lang_Class_klass();
2076 const TypeInstPtr* shbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, shbox_klass);
2077
2078 Node* v1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
2079 Node* shuffle = unbox_vector(argument(6), shbox_type, shuffle_bt, num_elem);
2080
2081 if (v1 == NULL || shuffle == NULL) {
2082 return false; // operand unboxing failed
2083 }
2084
2085 Node* mask = NULL;
2086 if (is_masked_op) {
2087 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
2088 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
2089 mask = unbox_vector(argument(7), mbox_type, elem_bt, num_elem);
2090 if (mask == NULL) {
2091 if (C->print_intrinsics()) {
2092 tty->print_cr(" ** not supported: arity=3 op=shuffle/rearrange vlen=%d etype=%s ismask=useload is_masked_op=1",
2093 num_elem, type2name(elem_bt));
2094 }
2095 return false;
2096 }
2097 }
2098
2099 Node* rearrange = new VectorRearrangeNode(v1, shuffle);
2100 if (is_masked_op) {
2101 if (use_predicate) {
2102 rearrange->add_req(mask);
2103 rearrange->add_flag(Node::Flag_is_predicated_vector);
2104 } else {
2105 const TypeVect* vt = v1->bottom_type()->is_vect();
2106 rearrange = gvn().transform(rearrange);
2107 Node* zero = gvn().makecon(Type::get_zero_type(elem_bt));
2108 Node* zerovec = gvn().transform(VectorNode::scalar2vector(zero, num_elem, Type::get_const_basic_type(elem_bt)));
2109 rearrange = new VectorBlendNode(zerovec, rearrange, mask);
2110 }
2111 }
2112 rearrange = gvn().transform(rearrange);
2113
2114 Node* box = box_vector(rearrange, vbox_type, elem_bt, num_elem);
2115 set_result(box);
2116 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2117 return true;
2118 }
2119
2120 static address get_svml_address(int vop, int bits, BasicType bt, char* name_ptr, int name_len) {
2121 address addr = NULL;
2122 assert(UseVectorStubs, "sanity");
2123 assert(name_ptr != NULL, "unexpected");
2124 assert((vop >= VectorSupport::VECTOR_OP_SVML_START) && (vop <= VectorSupport::VECTOR_OP_SVML_END), "unexpected");
2125 int op = vop - VectorSupport::VECTOR_OP_SVML_START;
2126
2127 switch(bits) {
2128 case 64: //fallthough
2129 case 128: //fallthough
2130 case 256: //fallthough
2131 case 512:
2132 if (bt == T_FLOAT) {
2133 snprintf(name_ptr, name_len, "vector_%s_float%d", VectorSupport::svmlname[op], bits);
2134 addr = StubRoutines::_vector_f_math[exact_log2(bits/64)][op];
2135 } else {
2136 assert(bt == T_DOUBLE, "must be FP type only");
2137 snprintf(name_ptr, name_len, "vector_%s_double%d", VectorSupport::svmlname[op], bits);
2138 addr = StubRoutines::_vector_d_math[exact_log2(bits/64)][op];
2139 }
2140 break;
2141 default:
2142 snprintf(name_ptr, name_len, "invalid");
2143 addr = NULL;
2144 Unimplemented();
2145 break;
2146 }
2147
2148 return addr;
2149 }
2150
2151 Node* LibraryCallKit::gen_call_to_svml(int vector_api_op_id, BasicType bt, int num_elem, Node* opd1, Node* opd2) {
2152 assert(UseVectorStubs, "sanity");
2153 assert(vector_api_op_id >= VectorSupport::VECTOR_OP_SVML_START && vector_api_op_id <= VectorSupport::VECTOR_OP_SVML_END, "need valid op id");
2154 assert(opd1 != NULL, "must not be null");
2155 const TypeVect* vt = TypeVect::make(bt, num_elem);
2156 const TypeFunc* call_type = OptoRuntime::Math_Vector_Vector_Type(opd2 != NULL ? 2 : 1, vt, vt);
2157 char name[100] = "";
2158
2159 // Get address for svml method.
2160 address addr = get_svml_address(vector_api_op_id, vt->length_in_bytes() * BitsPerByte, bt, name, 100);
2161
2162 if (addr == NULL) {
2163 return NULL;
2164 }
2165
2166 assert(name != NULL, "name must not be null");
2167 Node* operation = make_runtime_call(RC_VECTOR,
2168 call_type,
2169 addr,
2170 name,
2171 TypePtr::BOTTOM,
2172 opd1,
2173 opd2);
2174 return gvn().transform(new ProjNode(gvn().transform(operation), TypeFunc::Parms));
2175 }
2176
2177 // public static
2178 // <V extends Vector<E>,
2179 // M extends VectorMask<E>,
2180 // E>
2181 // V broadcastInt(int opr, Class<? extends V> vectorClass, Class<? extends M> maskClass,
2182 // Class<E> elementType, int length,
2183 // V v, int n, M m,
2184 // VectorBroadcastIntOp<V, M> defaultImpl)
2185 bool LibraryCallKit::inline_vector_broadcast_int() {
2186 const TypeInt* opr = gvn().type(argument(0))->isa_int();
2187 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
2188 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
2189 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
2190 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
2191
2192 if (opr == NULL || vector_klass == NULL || elem_klass == NULL || vlen == NULL) {
2193 return false; // dead code
2194 }
2195 if (!opr->is_con() || vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || !vlen->is_con()) {
2196 if (C->print_intrinsics()) {
2197 tty->print_cr(" ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
2198 NodeClassNames[argument(0)->Opcode()],
2199 NodeClassNames[argument(1)->Opcode()],
2200 NodeClassNames[argument(3)->Opcode()],
2201 NodeClassNames[argument(4)->Opcode()]);
2202 }
2203 return false; // not enough info for intrinsification
2204 }
2205 if (!is_klass_initialized(vector_klass)) {
2206 if (C->print_intrinsics()) {
2207 tty->print_cr(" ** klass argument not initialized");
2208 }
2209 return false;
2210 }
2211
2212 const Type* vmask_type = gvn().type(argument(7));
2213 bool is_masked_op = vmask_type != TypePtr::NULL_PTR;
2214 if (is_masked_op) {
2215 if (mask_klass == NULL || mask_klass->const_oop() == NULL) {
2216 if (C->print_intrinsics()) {
2217 tty->print_cr(" ** missing constant: maskclass=%s", NodeClassNames[argument(2)->Opcode()]);
2218 }
2219 return false; // not enough info for intrinsification
2220 }
2221
2222 if (!is_klass_initialized(mask_klass)) {
2223 if (C->print_intrinsics()) {
2224 tty->print_cr(" ** mask klass argument not initialized");
2225 }
2226 return false;
2227 }
2228
2229 if (vmask_type->maybe_null()) {
2230 if (C->print_intrinsics()) {
2231 tty->print_cr(" ** null mask values are not allowed for masked op");
2232 }
2233 return false;
2234 }
2235 }
2236
2237 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2238 if (!elem_type->is_primitive_type()) {
2239 if (C->print_intrinsics()) {
2240 tty->print_cr(" ** not a primitive bt=%d", elem_type->basic_type());
2241 }
2242 return false; // should be primitive type
2243 }
2244
2245 int num_elem = vlen->get_con();
2246 BasicType elem_bt = elem_type->basic_type();
2247 int opc = VectorSupport::vop2ideal(opr->get_con(), elem_bt);
2248
2249 bool is_shift = VectorNode::is_shift_opcode(opc);
2250 bool is_rotate = VectorNode::is_rotate_opcode(opc);
2251
2252 if (opc == 0 || (!is_shift && !is_rotate)) {
2253 if (C->print_intrinsics()) {
2254 tty->print_cr(" ** operation not supported: op=%d bt=%s", opr->get_con(), type2name(elem_bt));
2255 }
2256 return false; // operation not supported
2257 }
2258
2259 int sopc = VectorNode::opcode(opc, elem_bt);
2260 if (sopc == 0) {
2261 if (C->print_intrinsics()) {
2262 tty->print_cr(" ** operation not supported: opc=%s bt=%s", NodeClassNames[opc], type2name(elem_bt));
2263 }
2264 return false; // operation not supported
2265 }
2266
2267 Node* cnt = argument(6);
2268 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2269 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2270 const TypeInt* cnt_type = cnt->bottom_type()->isa_int();
2271
2272 // If CPU supports vector constant rotate instructions pass it directly
2273 bool is_const_rotate = is_rotate && cnt_type && cnt_type->is_con() &&
2274 Matcher::supports_vector_constant_rotates(cnt_type->get_con());
2275 bool has_scalar_args = is_rotate ? !is_const_rotate : true;
2276
2277 VectorMaskUseType checkFlags = (VectorMaskUseType)(is_masked_op ? (VecMaskUseLoad | VecMaskUsePred) : VecMaskNotUsed);
2278 bool use_predicate = is_masked_op;
2279
2280 if (!arch_supports_vector(sopc, num_elem, elem_bt, checkFlags, has_scalar_args)) {
2281 use_predicate = false;
2282 if (!is_masked_op ||
2283 (!arch_supports_vector(sopc, num_elem, elem_bt, VecMaskNotUsed, has_scalar_args) ||
2284 !arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad))) {
2285
2286 if (C->print_intrinsics()) {
2287 tty->print_cr(" ** not supported: arity=0 op=int/%d vlen=%d etype=%s is_masked_op=%d",
2288 sopc, num_elem, type2name(elem_bt), is_masked_op ? 1 : 0);
2289 }
2290 return false; // not supported
2291 }
2292 }
2293
2294 Node* opd1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
2295 Node* opd2 = NULL;
2296 if (is_shift) {
2297 opd2 = vector_shift_count(cnt, opc, elem_bt, num_elem);
2298 } else {
2299 assert(is_rotate, "unexpected operation");
2300 if (!is_const_rotate) {
2301 const Type * type_bt = Type::get_const_basic_type(elem_bt);
2302 cnt = elem_bt == T_LONG ? gvn().transform(new ConvI2LNode(cnt)) : cnt;
2303 opd2 = gvn().transform(VectorNode::scalar2vector(cnt, num_elem, type_bt));
2304 } else {
2305 // Constant shift value.
2306 opd2 = cnt;
2307 }
2308 }
2309
2310 if (opd1 == NULL || opd2 == NULL) {
2311 return false;
2312 }
2313
2314 Node* mask = NULL;
2315 if (is_masked_op) {
2316 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
2317 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
2318 mask = unbox_vector(argument(7), mbox_type, elem_bt, num_elem);
2319 if (mask == NULL) {
2320 if (C->print_intrinsics()) {
2321 tty->print_cr(" ** unbox failed mask=%s", NodeClassNames[argument(7)->Opcode()]);
2322 }
2323 return false;
2324 }
2325 }
2326
2327 Node* operation = VectorNode::make(opc, opd1, opd2, num_elem, elem_bt);
2328 if (is_masked_op && mask != NULL) {
2329 if (use_predicate) {
2330 operation->add_req(mask);
2331 operation->add_flag(Node::Flag_is_predicated_vector);
2332 } else {
2333 operation = gvn().transform(operation);
2334 operation = new VectorBlendNode(opd1, operation, mask);
2335 }
2336 }
2337 operation = gvn().transform(operation);
2338 Node* vbox = box_vector(operation, vbox_type, elem_bt, num_elem);
2339 set_result(vbox);
2340 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2341 return true;
2342 }
2343
2344 // public static <VOUT extends VectorPayload,
2345 // VIN extends VectorPayload,
2346 // S extends VectorSpecies>
2347 // VOUT convert(int oprId,
2348 // Class<?> fromVectorClass, Class<?> fromElementType, int fromVLen,
2349 // Class<?> toVectorClass, Class<?> toElementType, int toVLen,
2350 // VIN v, S s,
2351 // VectorConvertOp<VOUT, VIN, S> defaultImpl)
2352 //
2353 bool LibraryCallKit::inline_vector_convert() {
2354 const TypeInt* opr = gvn().type(argument(0))->isa_int();
2355
2356 const TypeInstPtr* vector_klass_from = gvn().type(argument(1))->isa_instptr();
2357 const TypeInstPtr* elem_klass_from = gvn().type(argument(2))->isa_instptr();
2358 const TypeInt* vlen_from = gvn().type(argument(3))->isa_int();
2359
2360 const TypeInstPtr* vector_klass_to = gvn().type(argument(4))->isa_instptr();
2361 const TypeInstPtr* elem_klass_to = gvn().type(argument(5))->isa_instptr();
2362 const TypeInt* vlen_to = gvn().type(argument(6))->isa_int();
2363
2364 if (opr == NULL ||
2365 vector_klass_from == NULL || elem_klass_from == NULL || vlen_from == NULL ||
2366 vector_klass_to == NULL || elem_klass_to == NULL || vlen_to == NULL) {
2367 return false; // dead code
2368 }
2369 if (!opr->is_con() ||
2370 vector_klass_from->const_oop() == NULL || elem_klass_from->const_oop() == NULL || !vlen_from->is_con() ||
2371 vector_klass_to->const_oop() == NULL || elem_klass_to->const_oop() == NULL || !vlen_to->is_con()) {
2372 if (C->print_intrinsics()) {
2373 tty->print_cr(" ** missing constant: opr=%s vclass_from=%s etype_from=%s vlen_from=%s vclass_to=%s etype_to=%s vlen_to=%s",
2374 NodeClassNames[argument(0)->Opcode()],
2375 NodeClassNames[argument(1)->Opcode()],
2376 NodeClassNames[argument(2)->Opcode()],
2377 NodeClassNames[argument(3)->Opcode()],
2378 NodeClassNames[argument(4)->Opcode()],
2379 NodeClassNames[argument(5)->Opcode()],
2380 NodeClassNames[argument(6)->Opcode()]);
2381 }
2382 return false; // not enough info for intrinsification
2383 }
2384 if (!is_klass_initialized(vector_klass_from) || !is_klass_initialized(vector_klass_to)) {
2385 if (C->print_intrinsics()) {
2386 tty->print_cr(" ** klass argument not initialized");
2387 }
2388 return false;
2389 }
2390
2391 assert(opr->get_con() == VectorSupport::VECTOR_OP_CAST ||
2392 opr->get_con() == VectorSupport::VECTOR_OP_UCAST ||
2393 opr->get_con() == VectorSupport::VECTOR_OP_REINTERPRET, "wrong opcode");
2394 bool is_cast = (opr->get_con() == VectorSupport::VECTOR_OP_CAST || opr->get_con() == VectorSupport::VECTOR_OP_UCAST);
2395 bool is_ucast = (opr->get_con() == VectorSupport::VECTOR_OP_UCAST);
2396
2397 ciKlass* vbox_klass_from = vector_klass_from->const_oop()->as_instance()->java_lang_Class_klass();
2398 ciKlass* vbox_klass_to = vector_klass_to->const_oop()->as_instance()->java_lang_Class_klass();
2399 if (is_vector_shuffle(vbox_klass_from)) {
2400 return false; // vector shuffles aren't supported
2401 }
2402 bool is_mask = is_vector_mask(vbox_klass_from);
2403
2404 ciType* elem_type_from = elem_klass_from->const_oop()->as_instance()->java_mirror_type();
2405 if (!elem_type_from->is_primitive_type()) {
2406 return false; // should be primitive type
2407 }
2408 BasicType elem_bt_from = elem_type_from->basic_type();
2409 ciType* elem_type_to = elem_klass_to->const_oop()->as_instance()->java_mirror_type();
2410 if (!elem_type_to->is_primitive_type()) {
2411 return false; // should be primitive type
2412 }
2413 BasicType elem_bt_to = elem_type_to->basic_type();
2414
2415 int num_elem_from = vlen_from->get_con();
2416 int num_elem_to = vlen_to->get_con();
2417
2418 // Check whether we can unbox to appropriate size. Even with casting, checking for reinterpret is needed
2419 // since we may need to change size.
2420 if (!arch_supports_vector(Op_VectorReinterpret,
2421 num_elem_from,
2422 elem_bt_from,
2423 is_mask ? VecMaskUseAll : VecMaskNotUsed)) {
2424 if (C->print_intrinsics()) {
2425 tty->print_cr(" ** not supported: arity=1 op=%s/1 vlen1=%d etype1=%s ismask=%d",
2426 is_cast ? "cast" : "reinterpret",
2427 num_elem_from, type2name(elem_bt_from), is_mask);
2428 }
2429 return false;
2430 }
2431
2432 // Check whether we can support resizing/reinterpreting to the new size.
2433 if (!arch_supports_vector(Op_VectorReinterpret,
2434 num_elem_to,
2435 elem_bt_to,
2436 is_mask ? VecMaskUseAll : VecMaskNotUsed)) {
2437 if (C->print_intrinsics()) {
2438 tty->print_cr(" ** not supported: arity=1 op=%s/2 vlen2=%d etype2=%s ismask=%d",
2439 is_cast ? "cast" : "reinterpret",
2440 num_elem_to, type2name(elem_bt_to), is_mask);
2441 }
2442 return false;
2443 }
2444
2445 // At this point, we know that both input and output vector registers are supported
2446 // by the architecture. Next check if the casted type is simply to same type - which means
2447 // that it is actually a resize and not a cast.
2448 if (is_cast && elem_bt_from == elem_bt_to) {
2449 is_cast = false;
2450 }
2451
2452 const TypeInstPtr* vbox_type_from = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass_from);
2453
2454 Node* opd1 = unbox_vector(argument(7), vbox_type_from, elem_bt_from, num_elem_from);
2455 if (opd1 == NULL) {
2456 return false;
2457 }
2458
2459 const TypeVect* src_type = TypeVect::make(elem_bt_from, num_elem_from, is_mask);
2460 const TypeVect* dst_type = TypeVect::make(elem_bt_to, num_elem_to, is_mask);
2461
2462 // Safety check to prevent casting if source mask is of type vector
2463 // and destination mask of type predicate vector and vice-versa.
2464 // From X86 standpoint, this case will only arise over KNL target,
2465 // where certain masks (depending on the species) are either propagated
2466 // through a vector or predicate register.
2467 if (is_mask &&
2468 ((src_type->isa_vectmask() == NULL && dst_type->isa_vectmask()) ||
2469 (dst_type->isa_vectmask() == NULL && src_type->isa_vectmask()))) {
2470 return false;
2471 }
2472
2473 Node* op = opd1;
2474 if (is_cast) {
2475 BasicType new_elem_bt_to = elem_bt_to;
2476 BasicType new_elem_bt_from = elem_bt_from;
2477 if (is_mask && is_floating_point_type(elem_bt_from)) {
2478 new_elem_bt_from = elem_bt_from == T_FLOAT ? T_INT : T_LONG;
2479 }
2480 int cast_vopc = VectorCastNode::opcode(new_elem_bt_from, !is_ucast);
2481 // Make sure that cast is implemented to particular type/size combination.
2482 if (!arch_supports_vector(cast_vopc, num_elem_to, elem_bt_to, VecMaskNotUsed)) {
2483 if (C->print_intrinsics()) {
2484 tty->print_cr(" ** not supported: arity=1 op=cast#%d/3 vlen2=%d etype2=%s ismask=%d",
2485 cast_vopc,
2486 num_elem_to, type2name(elem_bt_to), is_mask);
2487 }
2488 return false;
2489 }
2490
2491 if (num_elem_from < num_elem_to) {
2492 // Since input and output number of elements are not consistent, we need to make sure we
2493 // properly size. Thus, first make a cast that retains the number of elements from source.
2494 int num_elem_for_cast = num_elem_from;
2495
2496 // It is possible that arch does not support this intermediate vector size
2497 // TODO More complex logic required here to handle this corner case for the sizes.
2498 if (!arch_supports_vector(cast_vopc, num_elem_for_cast, elem_bt_to, VecMaskNotUsed)) {
2499 if (C->print_intrinsics()) {
2500 tty->print_cr(" ** not supported: arity=1 op=cast#%d/4 vlen1=%d etype2=%s ismask=%d",
2501 cast_vopc,
2502 num_elem_for_cast, type2name(elem_bt_to), is_mask);
2503 }
2504 return false;
2505 }
2506
2507 op = gvn().transform(VectorCastNode::make(cast_vopc, op, elem_bt_to, num_elem_for_cast));
2508 // Now ensure that the destination gets properly resized to needed size.
2509 op = gvn().transform(new VectorReinterpretNode(op, op->bottom_type()->is_vect(), dst_type));
2510 } else if (num_elem_from > num_elem_to) {
2511 // Since number elements from input is larger than output, simply reduce size of input (we are supposed to
2512 // drop top elements anyway).
2513 int num_elem_for_resize = num_elem_to;
2514
2515 // It is possible that arch does not support this intermediate vector size
2516 // TODO More complex logic required here to handle this corner case for the sizes.
2517 if (!arch_supports_vector(Op_VectorReinterpret,
2518 num_elem_for_resize,
2519 elem_bt_from,
2520 VecMaskNotUsed)) {
2521 if (C->print_intrinsics()) {
2522 tty->print_cr(" ** not supported: arity=1 op=cast/5 vlen2=%d etype1=%s ismask=%d",
2523 num_elem_for_resize, type2name(elem_bt_from), is_mask);
2524 }
2525 return false;
2526 }
2527
2528 op = gvn().transform(new VectorReinterpretNode(op,
2529 src_type,
2530 TypeVect::make(elem_bt_from,
2531 num_elem_for_resize)));
2532 op = gvn().transform(VectorCastNode::make(cast_vopc, op, elem_bt_to, num_elem_to));
2533 } else {
2534 if (is_mask) {
2535 if ((dst_type->isa_vectmask() && src_type->isa_vectmask()) ||
2536 (type2aelembytes(elem_bt_from) == type2aelembytes(elem_bt_to))) {
2537 op = gvn().transform(new VectorMaskCastNode(op, dst_type));
2538 } else {
2539 op = VectorMaskCastNode::makeCastNode(&gvn(), op, dst_type);
2540 }
2541 } else {
2542 // Since input and output number of elements match, and since we know this vector size is
2543 // supported, simply do a cast with no resize needed.
2544 op = gvn().transform(VectorCastNode::make(cast_vopc, op, elem_bt_to, num_elem_to));
2545 }
2546 }
2547 } else if (Type::cmp(src_type, dst_type) != 0) {
2548 assert(!is_cast, "must be reinterpret");
2549 op = gvn().transform(new VectorReinterpretNode(op, src_type, dst_type));
2550 }
2551
2552 const TypeInstPtr* vbox_type_to = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass_to);
2553 Node* vbox = box_vector(op, vbox_type_to, elem_bt_to, num_elem_to);
2554 set_result(vbox);
2555 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem_to * type2aelembytes(elem_bt_to))));
2556 return true;
2557 }
2558
2559 // public static
2560 // <V extends Vector<E>,
2561 // E>
2562 // V insert(Class<? extends V> vectorClass, Class<E> elementType, int vlen,
2563 // V vec, int ix, long val,
2564 // VecInsertOp<V> defaultImpl)
2565 bool LibraryCallKit::inline_vector_insert() {
2566 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
2567 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
2568 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
2569 const TypeInt* idx = gvn().type(argument(4))->isa_int();
2570
2571 if (vector_klass == NULL || elem_klass == NULL || vlen == NULL || idx == NULL) {
2572 return false; // dead code
2573 }
2574 if (vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || !vlen->is_con() || !idx->is_con()) {
2575 if (C->print_intrinsics()) {
2576 tty->print_cr(" ** missing constant: vclass=%s etype=%s vlen=%s idx=%s",
2577 NodeClassNames[argument(0)->Opcode()],
2578 NodeClassNames[argument(1)->Opcode()],
2579 NodeClassNames[argument(2)->Opcode()],
2580 NodeClassNames[argument(4)->Opcode()]);
2581 }
2582 return false; // not enough info for intrinsification
2583 }
2584 if (!is_klass_initialized(vector_klass)) {
2585 if (C->print_intrinsics()) {
2586 tty->print_cr(" ** klass argument not initialized");
2587 }
2588 return false;
2589 }
2590 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2591 if (!elem_type->is_primitive_type()) {
2592 if (C->print_intrinsics()) {
2593 tty->print_cr(" ** not a primitive bt=%d", elem_type->basic_type());
2594 }
2595 return false; // should be primitive type
2596 }
2597 BasicType elem_bt = elem_type->basic_type();
2598 int num_elem = vlen->get_con();
2599 if (!arch_supports_vector(Op_VectorInsert, num_elem, elem_bt, VecMaskNotUsed)) {
2600 if (C->print_intrinsics()) {
2601 tty->print_cr(" ** not supported: arity=1 op=insert vlen=%d etype=%s ismask=no",
2602 num_elem, type2name(elem_bt));
2603 }
2604 return false; // not supported
2605 }
2606
2607 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2608 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2609
2610 Node* opd = unbox_vector(argument(3), vbox_type, elem_bt, num_elem);
2611 if (opd == NULL) {
2612 return false;
2613 }
2614
2615 Node* insert_val = argument(5);
2616 assert(gvn().type(insert_val)->isa_long() != NULL, "expected to be long");
2617
2618 // Convert insert value back to its appropriate type.
2619 switch (elem_bt) {
2620 case T_BYTE:
2621 insert_val = gvn().transform(new ConvL2INode(insert_val));
2622 insert_val = gvn().transform(new CastIINode(insert_val, TypeInt::BYTE));
2623 break;
2624 case T_SHORT:
2625 insert_val = gvn().transform(new ConvL2INode(insert_val));
2626 insert_val = gvn().transform(new CastIINode(insert_val, TypeInt::SHORT));
2627 break;
2628 case T_INT:
2629 insert_val = gvn().transform(new ConvL2INode(insert_val));
2630 break;
2631 case T_FLOAT:
2632 insert_val = gvn().transform(new ConvL2INode(insert_val));
2633 insert_val = gvn().transform(new MoveI2FNode(insert_val));
2634 break;
2635 case T_DOUBLE:
2636 insert_val = gvn().transform(new MoveL2DNode(insert_val));
2637 break;
2638 case T_LONG:
2639 // no conversion needed
2640 break;
2641 default: fatal("%s", type2name(elem_bt)); break;
2642 }
2643
2644 Node* operation = gvn().transform(VectorInsertNode::make(opd, insert_val, idx->get_con()));
2645
2646 Node* vbox = box_vector(operation, vbox_type, elem_bt, num_elem);
2647 set_result(vbox);
2648 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
2649 return true;
2650 }
2651
2652 // public static
2653 // <V extends Vector<E>,
2654 // E>
2655 // long extract(Class<? extends V> vectorClass, Class<E> elementType, int vlen,
2656 // V vec, int ix,
2657 // VecExtractOp<V> defaultImpl)
2658 bool LibraryCallKit::inline_vector_extract() {
2659 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
2660 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
2661 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
2662 const TypeInt* idx = gvn().type(argument(4))->isa_int();
2663
2664 if (vector_klass == NULL || elem_klass == NULL || vlen == NULL || idx == NULL) {
2665 return false; // dead code
2666 }
2667 if (vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || !vlen->is_con() || !idx->is_con()) {
2668 if (C->print_intrinsics()) {
2669 tty->print_cr(" ** missing constant: vclass=%s etype=%s vlen=%s idx=%s",
2670 NodeClassNames[argument(0)->Opcode()],
2671 NodeClassNames[argument(1)->Opcode()],
2672 NodeClassNames[argument(2)->Opcode()],
2673 NodeClassNames[argument(4)->Opcode()]);
2674 }
2675 return false; // not enough info for intrinsification
2676 }
2677 if (!is_klass_initialized(vector_klass)) {
2678 if (C->print_intrinsics()) {
2679 tty->print_cr(" ** klass argument not initialized");
2680 }
2681 return false;
2682 }
2683 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
2684 if (!elem_type->is_primitive_type()) {
2685 if (C->print_intrinsics()) {
2686 tty->print_cr(" ** not a primitive bt=%d", elem_type->basic_type());
2687 }
2688 return false; // should be primitive type
2689 }
2690 BasicType elem_bt = elem_type->basic_type();
2691 int num_elem = vlen->get_con();
2692 int vopc = ExtractNode::opcode(elem_bt);
2693 if (!arch_supports_vector(vopc, num_elem, elem_bt, VecMaskNotUsed)) {
2694 if (C->print_intrinsics()) {
2695 tty->print_cr(" ** not supported: arity=1 op=extract vlen=%d etype=%s ismask=no",
2696 num_elem, type2name(elem_bt));
2697 }
2698 return false; // not supported
2699 }
2700
2701 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
2702 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
2703
2704 Node* opd = unbox_vector(argument(3), vbox_type, elem_bt, num_elem);
2705 if (opd == NULL) {
2706 return false;
2707 }
2708
2709 Node* operation = gvn().transform(ExtractNode::make(opd, idx->get_con(), elem_bt));
2710
2711 Node* bits = NULL;
2712 switch (elem_bt) {
2713 case T_BYTE:
2714 case T_SHORT:
2715 case T_INT: {
2716 bits = gvn().transform(new ConvI2LNode(operation));
2717 break;
2718 }
2719 case T_FLOAT: {
2720 bits = gvn().transform(new MoveF2INode(operation));
2721 bits = gvn().transform(new ConvI2LNode(bits));
2722 break;
2723 }
2724 case T_DOUBLE: {
2725 bits = gvn().transform(new MoveD2LNode(operation));
2726 break;
2727 }
2728 case T_LONG: {
2729 bits = operation; // no conversion needed
2730 break;
2731 }
2732 default: fatal("%s", type2name(elem_bt));
2733 }
2734
2735 set_result(bits);
2736 return true;
2737 }
2738
--- EOF ---