1 /*
2 * Copyright (c) 2017, 2021, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation. Oracle designates this
8 * particular file as subject to the "Classpath" exception as provided
9 * by Oracle in the LICENSE file that accompanied this code.
10 *
11 * This code is distributed in the hope that it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 * version 2 for more details (a copy is included in the LICENSE file that
15 * accompanied this code).
16 *
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 */
25 package jdk.incubator.vector;
26
27 import java.nio.ByteBuffer;
28 import java.util.Arrays;
29 import java.util.Objects;
30 import java.util.function.IntUnaryOperator;
31
32 import jdk.internal.vm.annotation.ForceInline;
33 import jdk.internal.vm.vector.VectorSupport;
34
35 import static jdk.internal.vm.vector.VectorSupport.*;
36
37 import static jdk.incubator.vector.VectorOperators.*;
38
39 #warn This file is preprocessed before being compiled
40
41 @SuppressWarnings("cast") // warning: redundant cast
42 final class $vectortype$ extends $abstractvectortype$ {
43 static final $Type$Species VSPECIES =
44 ($Type$Species) $Type$Vector.SPECIES_$BITS$;
45
46 static final VectorShape VSHAPE =
47 VSPECIES.vectorShape();
48
49 static final Class<$vectortype$> VCLASS = $vectortype$.class;
50
51 static final int VSIZE = VSPECIES.vectorBitSize();
52
53 static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM
54
55 static final Class<$Boxtype$> ETYPE = $type$.class; // used by the JVM
56
57 $vectortype$($type$[] v) {
58 super(v);
59 }
60
61 // For compatibility as $vectortype$::new,
62 // stored into species.vectorFactory.
63 $vectortype$(Object v) {
64 this(($type$[]) v);
65 }
66
67 static final $vectortype$ ZERO = new $vectortype$(new $type$[VLENGTH]);
68 static final $vectortype$ IOTA = new $vectortype$(VSPECIES.iotaArray());
69
70 static {
71 // Warm up a few species caches.
72 // If we do this too much we will
73 // get NPEs from bootstrap circularity.
74 VSPECIES.dummyVector();
75 VSPECIES.withLanes(LaneType.BYTE);
76 }
77
78 // Specialized extractors
79
80 @ForceInline
81 final @Override
82 public $Type$Species vspecies() {
83 // ISSUE: This should probably be a @Stable
84 // field inside AbstractVector, rather than
85 // a megamorphic method.
86 return VSPECIES;
87 }
88
89 @ForceInline
90 @Override
91 public final Class<$Boxtype$> elementType() { return $type$.class; }
92
93 @ForceInline
94 @Override
95 public final int elementSize() { return $Boxtype$.SIZE; }
96
97 @ForceInline
98 @Override
99 public final VectorShape shape() { return VSHAPE; }
100
101 @ForceInline
102 @Override
103 public final int length() { return VLENGTH; }
104
105 @ForceInline
106 @Override
107 public final int bitSize() { return VSIZE; }
108
109 @ForceInline
110 @Override
111 public final int byteSize() { return VSIZE / Byte.SIZE; }
112
113 /*package-private*/
114 @ForceInline
115 final @Override
116 $type$[] vec() {
117 return ($type$[])getPayload();
118 }
119
120 // Virtualized constructors
121
122 @Override
123 @ForceInline
124 public final $vectortype$ broadcast($type$ e) {
125 return ($vectortype$) super.broadcastTemplate(e); // specialize
126 }
127
128 #if[!long]
129 @Override
130 @ForceInline
131 public final $vectortype$ broadcast(long e) {
132 return ($vectortype$) super.broadcastTemplate(e); // specialize
133 }
134 #end[!long]
135
136 @Override
137 @ForceInline
138 $masktype$ maskFromArray(boolean[] bits) {
139 return new $masktype$(bits);
140 }
141
142 @Override
143 @ForceInline
144 $shuffletype$ iotaShuffle() { return $shuffletype$.IOTA; }
145
146 @ForceInline
147 $shuffletype$ iotaShuffle(int start, int step, boolean wrap) {
148 if (wrap) {
149 return ($shuffletype$)VectorSupport.shuffleIota(ETYPE, $shuffletype$.class, VSPECIES, VLENGTH, start, step, 1,
150 (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i*lstep + lstart, l))));
151 } else {
152 return ($shuffletype$)VectorSupport.shuffleIota(ETYPE, $shuffletype$.class, VSPECIES, VLENGTH, start, step, 0,
153 (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (i*lstep + lstart)));
154 }
155 }
156
157 @Override
158 @ForceInline
159 $shuffletype$ shuffleFromBytes(byte[] reorder) { return new $shuffletype$(reorder); }
160
161 @Override
162 @ForceInline
163 $shuffletype$ shuffleFromArray(int[] indexes, int i) { return new $shuffletype$(indexes, i); }
164
165 @Override
166 @ForceInline
167 $shuffletype$ shuffleFromOp(IntUnaryOperator fn) { return new $shuffletype$(fn); }
168
169 // Make a vector of the same species but the given elements:
170 @ForceInline
171 final @Override
172 $vectortype$ vectorFactory($type$[] vec) {
173 return new $vectortype$(vec);
174 }
175
176 @ForceInline
177 final @Override
178 Byte$bits$Vector asByteVectorRaw() {
179 return (Byte$bits$Vector) super.asByteVectorRawTemplate(); // specialize
180 }
181
182 @ForceInline
183 final @Override
184 AbstractVector<?> asVectorRaw(LaneType laneType) {
185 return super.asVectorRawTemplate(laneType); // specialize
186 }
187
188 // Unary operator
189
190 @ForceInline
191 final @Override
192 $vectortype$ uOp(FUnOp f) {
193 return ($vectortype$) super.uOpTemplate(f); // specialize
194 }
195
196 @ForceInline
197 final @Override
198 $vectortype$ uOp(VectorMask<$Boxtype$> m, FUnOp f) {
199 return ($vectortype$)
200 super.uOpTemplate(($masktype$)m, f); // specialize
201 }
202
203 // Binary operator
204
205 @ForceInline
206 final @Override
207 $vectortype$ bOp(Vector<$Boxtype$> v, FBinOp f) {
208 return ($vectortype$) super.bOpTemplate(($vectortype$)v, f); // specialize
209 }
210
211 @ForceInline
212 final @Override
213 $vectortype$ bOp(Vector<$Boxtype$> v,
214 VectorMask<$Boxtype$> m, FBinOp f) {
215 return ($vectortype$)
216 super.bOpTemplate(($vectortype$)v, ($masktype$)m,
217 f); // specialize
218 }
219
220 // Ternary operator
221
222 @ForceInline
223 final @Override
224 $vectortype$ tOp(Vector<$Boxtype$> v1, Vector<$Boxtype$> v2, FTriOp f) {
225 return ($vectortype$)
226 super.tOpTemplate(($vectortype$)v1, ($vectortype$)v2,
227 f); // specialize
228 }
229
230 @ForceInline
231 final @Override
232 $vectortype$ tOp(Vector<$Boxtype$> v1, Vector<$Boxtype$> v2,
233 VectorMask<$Boxtype$> m, FTriOp f) {
234 return ($vectortype$)
235 super.tOpTemplate(($vectortype$)v1, ($vectortype$)v2,
236 ($masktype$)m, f); // specialize
237 }
238
239 @ForceInline
240 final @Override
241 $type$ rOp($type$ v, VectorMask<$Boxtype$> m, FBinOp f) {
242 return super.rOpTemplate(v, m, f); // specialize
243 }
244
245 @Override
246 @ForceInline
247 public final <F>
248 Vector<F> convertShape(VectorOperators.Conversion<$Boxtype$,F> conv,
249 VectorSpecies<F> rsp, int part) {
250 return super.convertShapeTemplate(conv, rsp, part); // specialize
251 }
252
253 @Override
254 @ForceInline
255 public final <F>
256 Vector<F> reinterpretShape(VectorSpecies<F> toSpecies, int part) {
257 return super.reinterpretShapeTemplate(toSpecies, part); // specialize
258 }
259
260 // Specialized algebraic operations:
261
262 // The following definition forces a specialized version of this
263 // crucial method into the v-table of this class. A call to add()
264 // will inline to a call to lanewise(ADD,), at which point the JIT
265 // intrinsic will have the opcode of ADD, plus all the metadata
266 // for this particular class, enabling it to generate precise
267 // code.
268 //
269 // There is probably no benefit to the JIT to specialize the
270 // masked or broadcast versions of the lanewise method.
271
272 @Override
273 @ForceInline
274 public $vectortype$ lanewise(Unary op) {
275 return ($vectortype$) super.lanewiseTemplate(op); // specialize
276 }
277
278 @Override
279 @ForceInline
280 public $vectortype$ lanewise(Unary op, VectorMask<$Boxtype$> m) {
281 return ($vectortype$) super.lanewiseTemplate(op, $masktype$.class, ($masktype$) m); // specialize
282 }
283
284 @Override
285 @ForceInline
286 public $vectortype$ lanewise(Binary op, Vector<$Boxtype$> v) {
287 return ($vectortype$) super.lanewiseTemplate(op, v); // specialize
288 }
289
290 @Override
291 @ForceInline
292 public $vectortype$ lanewise(Binary op, Vector<$Boxtype$> v, VectorMask<$Boxtype$> m) {
293 return ($vectortype$) super.lanewiseTemplate(op, $masktype$.class, v, ($masktype$) m); // specialize
294 }
295
296 #if[!FP]
297 /*package-private*/
298 @Override
299 @ForceInline $vectortype$
300 lanewiseShift(VectorOperators.Binary op, int e) {
301 return ($vectortype$) super.lanewiseShiftTemplate(op, e); // specialize
302 }
303
304 /*package-private*/
305 @Override
306 @ForceInline $vectortype$
307 lanewiseShift(VectorOperators.Binary op, int e, VectorMask<$Boxtype$> m) {
308 return ($vectortype$) super.lanewiseShiftTemplate(op, $masktype$.class, e, ($masktype$) m); // specialize
309 }
310 #end[!FP]
311
312 /*package-private*/
313 @Override
314 @ForceInline
315 public final
316 $vectortype$
317 lanewise(Ternary op, Vector<$Boxtype$> v1, Vector<$Boxtype$> v2) {
318 return ($vectortype$) super.lanewiseTemplate(op, v1, v2); // specialize
319 }
320
321 @Override
322 @ForceInline
323 public final
324 $vectortype$
325 lanewise(Ternary op, Vector<$Boxtype$> v1, Vector<$Boxtype$> v2, VectorMask<$Boxtype$> m) {
326 return ($vectortype$) super.lanewiseTemplate(op, $masktype$.class, v1, v2, ($masktype$) m); // specialize
327 }
328
329 @Override
330 @ForceInline
331 public final
332 $vectortype$ addIndex(int scale) {
333 return ($vectortype$) super.addIndexTemplate(scale); // specialize
334 }
335
336 // Type specific horizontal reductions
337
338 @Override
339 @ForceInline
340 public final $type$ reduceLanes(VectorOperators.Associative op) {
341 return super.reduceLanesTemplate(op); // specialized
342 }
343
344 @Override
345 @ForceInline
346 public final $type$ reduceLanes(VectorOperators.Associative op,
347 VectorMask<$Boxtype$> m) {
348 return super.reduceLanesTemplate(op, $masktype$.class, ($masktype$) m); // specialized
349 }
350
351 @Override
352 @ForceInline
353 public final long reduceLanesToLong(VectorOperators.Associative op) {
354 return (long) super.reduceLanesTemplate(op); // specialized
355 }
356
357 @Override
358 @ForceInline
359 public final long reduceLanesToLong(VectorOperators.Associative op,
360 VectorMask<$Boxtype$> m) {
361 return (long) super.reduceLanesTemplate(op, $masktype$.class, ($masktype$) m); // specialized
362 }
363
364 @ForceInline
365 public VectorShuffle<$Boxtype$> toShuffle() {
366 return super.toShuffleTemplate($shuffletype$.class); // specialize
367 }
368
369 // Specialized unary testing
370
371 @Override
372 @ForceInline
373 public final $masktype$ test(Test op) {
374 return super.testTemplate($masktype$.class, op); // specialize
375 }
376
377 // Specialized comparisons
378
379 @Override
380 @ForceInline
381 public final $masktype$ compare(Comparison op, Vector<$Boxtype$> v) {
382 return super.compareTemplate($masktype$.class, op, v); // specialize
383 }
384
385 @Override
386 @ForceInline
387 public final $masktype$ compare(Comparison op, $type$ s) {
388 return super.compareTemplate($masktype$.class, op, s); // specialize
389 }
390
391 #if[!long]
392 @Override
393 @ForceInline
394 public final $masktype$ compare(Comparison op, long s) {
395 return super.compareTemplate($masktype$.class, op, s); // specialize
396 }
397 #end[!long]
398
399 @Override
400 @ForceInline
401 public final $masktype$ compare(Comparison op, Vector<$Boxtype$> v, VectorMask<$Boxtype$> m) {
402 return super.compareTemplate($masktype$.class, op, v, ($masktype$) m);
403 }
404
405
406 @Override
407 @ForceInline
408 public $vectortype$ blend(Vector<$Boxtype$> v, VectorMask<$Boxtype$> m) {
409 return ($vectortype$)
410 super.blendTemplate($masktype$.class,
411 ($vectortype$) v,
412 ($masktype$) m); // specialize
413 }
414
415 @Override
416 @ForceInline
417 public $vectortype$ slice(int origin, Vector<$Boxtype$> v) {
418 return ($vectortype$) super.sliceTemplate(origin, v); // specialize
419 }
420
421 @Override
422 @ForceInline
423 public $vectortype$ slice(int origin) {
424 return ($vectortype$) super.sliceTemplate(origin); // specialize
425 }
426
427 @Override
428 @ForceInline
429 public $vectortype$ unslice(int origin, Vector<$Boxtype$> w, int part) {
430 return ($vectortype$) super.unsliceTemplate(origin, w, part); // specialize
431 }
432
433 @Override
434 @ForceInline
435 public $vectortype$ unslice(int origin, Vector<$Boxtype$> w, int part, VectorMask<$Boxtype$> m) {
436 return ($vectortype$)
437 super.unsliceTemplate($masktype$.class,
438 origin, w, part,
439 ($masktype$) m); // specialize
440 }
441
442 @Override
443 @ForceInline
444 public $vectortype$ unslice(int origin) {
445 return ($vectortype$) super.unsliceTemplate(origin); // specialize
446 }
447
448 @Override
449 @ForceInline
450 public $vectortype$ rearrange(VectorShuffle<$Boxtype$> s) {
451 return ($vectortype$)
452 super.rearrangeTemplate($shuffletype$.class,
453 ($shuffletype$) s); // specialize
454 }
455
456 @Override
457 @ForceInline
458 public $vectortype$ rearrange(VectorShuffle<$Boxtype$> shuffle,
459 VectorMask<$Boxtype$> m) {
460 return ($vectortype$)
461 super.rearrangeTemplate($shuffletype$.class,
462 $masktype$.class,
463 ($shuffletype$) shuffle,
464 ($masktype$) m); // specialize
465 }
466
467 @Override
468 @ForceInline
469 public $vectortype$ rearrange(VectorShuffle<$Boxtype$> s,
470 Vector<$Boxtype$> v) {
471 return ($vectortype$)
472 super.rearrangeTemplate($shuffletype$.class,
473 ($shuffletype$) s,
474 ($vectortype$) v); // specialize
475 }
476
477 @Override
478 @ForceInline
479 public $vectortype$ selectFrom(Vector<$Boxtype$> v) {
480 return ($vectortype$)
481 super.selectFromTemplate(($vectortype$) v); // specialize
482 }
483
484 @Override
485 @ForceInline
486 public $vectortype$ selectFrom(Vector<$Boxtype$> v,
487 VectorMask<$Boxtype$> m) {
488 return ($vectortype$)
489 super.selectFromTemplate(($vectortype$) v,
490 ($masktype$) m); // specialize
491 }
492
493
494 #if[FP]
495 @ForceInline
496 @Override
497 public $type$ lane(int i) {
498 #if[!Max]
499 $bitstype$ bits;
500 switch(i) {
501 case 0: bits = laneHelper(0); break;
502 #if[!1L]
503 case 1: bits = laneHelper(1); break;
504 #if[!2L]
505 case 2: bits = laneHelper(2); break;
506 case 3: bits = laneHelper(3); break;
507 #if[!4L]
508 case 4: bits = laneHelper(4); break;
509 case 5: bits = laneHelper(5); break;
510 case 6: bits = laneHelper(6); break;
511 case 7: bits = laneHelper(7); break;
512 #if[!8L]
513 case 8: bits = laneHelper(8); break;
514 case 9: bits = laneHelper(9); break;
515 case 10: bits = laneHelper(10); break;
516 case 11: bits = laneHelper(11); break;
517 case 12: bits = laneHelper(12); break;
518 case 13: bits = laneHelper(13); break;
519 case 14: bits = laneHelper(14); break;
520 case 15: bits = laneHelper(15); break;
521 #end[!8L]
522 #end[!4L]
523 #end[!2L]
524 #end[!1L]
525 default: throw new IllegalArgumentException("Index " + i + " must be zero or positive, and less than " + VLENGTH);
526 }
527 #else[!Max]
528 if (i < 0 || i >= VLENGTH) {
529 throw new IllegalArgumentException("Index " + i + " must be zero or positive, and less than " + VLENGTH);
530 }
531 $bitstype$ bits = laneHelper(i);
532 #end[!Max]
533 return $Type$.$bitstype$BitsTo$Fptype$(bits);
534 }
535
536 public $bitstype$ laneHelper(int i) {
537 return ($bitstype$) VectorSupport.extract(
538 VCLASS, ETYPE, VLENGTH,
539 this, i,
540 (vec, ix) -> {
541 $type$[] vecarr = vec.vec();
542 return (long)$Type$.$type$To$Bitstype$Bits(vecarr[ix]);
543 });
544 }
545
546 @ForceInline
547 @Override
548 public $vectortype$ withLane(int i, $type$ e) {
549 #if[!Max]
550 switch(i) {
551 case 0: return withLaneHelper(0, e);
552 #if[!1L]
553 case 1: return withLaneHelper(1, e);
554 #if[!2L]
555 case 2: return withLaneHelper(2, e);
556 case 3: return withLaneHelper(3, e);
557 #if[!4L]
558 case 4: return withLaneHelper(4, e);
559 case 5: return withLaneHelper(5, e);
560 case 6: return withLaneHelper(6, e);
561 case 7: return withLaneHelper(7, e);
562 #if[!8L]
563 case 8: return withLaneHelper(8, e);
564 case 9: return withLaneHelper(9, e);
565 case 10: return withLaneHelper(10, e);
566 case 11: return withLaneHelper(11, e);
567 case 12: return withLaneHelper(12, e);
568 case 13: return withLaneHelper(13, e);
569 case 14: return withLaneHelper(14, e);
570 case 15: return withLaneHelper(15, e);
571 #end[!8L]
572 #end[!4L]
573 #end[!2L]
574 #end[!1L]
575 default: throw new IllegalArgumentException("Index " + i + " must be zero or positive, and less than " + VLENGTH);
576 }
577 #else[!Max]
578 if (i < 0 || i >= VLENGTH) {
579 throw new IllegalArgumentException("Index " + i + " must be zero or positive, and less than " + VLENGTH);
580 }
581 return withLaneHelper(i, e);
582 #end[!Max]
583 }
584
585 public $vectortype$ withLaneHelper(int i, $type$ e) {
586 return VectorSupport.insert(
587 VCLASS, ETYPE, VLENGTH,
588 this, i, (long)$Type$.$type$To$Bitstype$Bits(e),
589 (v, ix, bits) -> {
590 $type$[] res = v.vec().clone();
591 res[ix] = $Type$.$bitstype$BitsTo$Type$(($bitstype$)bits);
592 return v.vectorFactory(res);
593 });
594 }
595 #else[FP]
596 @ForceInline
597 @Override
598 public $type$ lane(int i) {
599 #if[!Max]
600 switch(i) {
601 case 0: return laneHelper(0);
602 #if[!1L]
603 case 1: return laneHelper(1);
604 #if[!2L]
605 case 2: return laneHelper(2);
606 case 3: return laneHelper(3);
607 #if[!4L]
608 case 4: return laneHelper(4);
609 case 5: return laneHelper(5);
610 case 6: return laneHelper(6);
611 case 7: return laneHelper(7);
612 #if[!8L]
613 case 8: return laneHelper(8);
614 case 9: return laneHelper(9);
615 case 10: return laneHelper(10);
616 case 11: return laneHelper(11);
617 case 12: return laneHelper(12);
618 case 13: return laneHelper(13);
619 case 14: return laneHelper(14);
620 case 15: return laneHelper(15);
621 #if[!16L]
622 case 16: return laneHelper(16);
623 case 17: return laneHelper(17);
624 case 18: return laneHelper(18);
625 case 19: return laneHelper(19);
626 case 20: return laneHelper(20);
627 case 21: return laneHelper(21);
628 case 22: return laneHelper(22);
629 case 23: return laneHelper(23);
630 case 24: return laneHelper(24);
631 case 25: return laneHelper(25);
632 case 26: return laneHelper(26);
633 case 27: return laneHelper(27);
634 case 28: return laneHelper(28);
635 case 29: return laneHelper(29);
636 case 30: return laneHelper(30);
637 case 31: return laneHelper(31);
638 #if[!32L]
639 case 32: return laneHelper(32);
640 case 33: return laneHelper(33);
641 case 34: return laneHelper(34);
642 case 35: return laneHelper(35);
643 case 36: return laneHelper(36);
644 case 37: return laneHelper(37);
645 case 38: return laneHelper(38);
646 case 39: return laneHelper(39);
647 case 40: return laneHelper(40);
648 case 41: return laneHelper(41);
649 case 42: return laneHelper(42);
650 case 43: return laneHelper(43);
651 case 44: return laneHelper(44);
652 case 45: return laneHelper(45);
653 case 46: return laneHelper(46);
654 case 47: return laneHelper(47);
655 case 48: return laneHelper(48);
656 case 49: return laneHelper(49);
657 case 50: return laneHelper(50);
658 case 51: return laneHelper(51);
659 case 52: return laneHelper(52);
660 case 53: return laneHelper(53);
661 case 54: return laneHelper(54);
662 case 55: return laneHelper(55);
663 case 56: return laneHelper(56);
664 case 57: return laneHelper(57);
665 case 58: return laneHelper(58);
666 case 59: return laneHelper(59);
667 case 60: return laneHelper(60);
668 case 61: return laneHelper(61);
669 case 62: return laneHelper(62);
670 case 63: return laneHelper(63);
671 #end[!32L]
672 #end[!16L]
673 #end[!8L]
674 #end[!4L]
675 #end[!2L]
676 #end[!1L]
677 default: throw new IllegalArgumentException("Index " + i + " must be zero or positive, and less than " + VLENGTH);
678 }
679 #else[!Max]
680 if (i < 0 || i >= VLENGTH) {
681 throw new IllegalArgumentException("Index " + i + " must be zero or positive, and less than " + VLENGTH);
682 }
683 return laneHelper(i);
684 #end[!Max]
685 }
686
687 public $type$ laneHelper(int i) {
688 return ($type$) VectorSupport.extract(
689 VCLASS, ETYPE, VLENGTH,
690 this, i,
691 (vec, ix) -> {
692 $type$[] vecarr = vec.vec();
693 return (long)vecarr[ix];
694 });
695 }
696
697 @ForceInline
698 @Override
699 public $vectortype$ withLane(int i, $type$ e) {
700 #if[!Max]
701 switch (i) {
702 case 0: return withLaneHelper(0, e);
703 #if[!1L]
704 case 1: return withLaneHelper(1, e);
705 #if[!2L]
706 case 2: return withLaneHelper(2, e);
707 case 3: return withLaneHelper(3, e);
708 #if[!4L]
709 case 4: return withLaneHelper(4, e);
710 case 5: return withLaneHelper(5, e);
711 case 6: return withLaneHelper(6, e);
712 case 7: return withLaneHelper(7, e);
713 #if[!8L]
714 case 8: return withLaneHelper(8, e);
715 case 9: return withLaneHelper(9, e);
716 case 10: return withLaneHelper(10, e);
717 case 11: return withLaneHelper(11, e);
718 case 12: return withLaneHelper(12, e);
719 case 13: return withLaneHelper(13, e);
720 case 14: return withLaneHelper(14, e);
721 case 15: return withLaneHelper(15, e);
722 #if[!16L]
723 case 16: return withLaneHelper(16, e);
724 case 17: return withLaneHelper(17, e);
725 case 18: return withLaneHelper(18, e);
726 case 19: return withLaneHelper(19, e);
727 case 20: return withLaneHelper(20, e);
728 case 21: return withLaneHelper(21, e);
729 case 22: return withLaneHelper(22, e);
730 case 23: return withLaneHelper(23, e);
731 case 24: return withLaneHelper(24, e);
732 case 25: return withLaneHelper(25, e);
733 case 26: return withLaneHelper(26, e);
734 case 27: return withLaneHelper(27, e);
735 case 28: return withLaneHelper(28, e);
736 case 29: return withLaneHelper(29, e);
737 case 30: return withLaneHelper(30, e);
738 case 31: return withLaneHelper(31, e);
739 #if[!32L]
740 case 32: return withLaneHelper(32, e);
741 case 33: return withLaneHelper(33, e);
742 case 34: return withLaneHelper(34, e);
743 case 35: return withLaneHelper(35, e);
744 case 36: return withLaneHelper(36, e);
745 case 37: return withLaneHelper(37, e);
746 case 38: return withLaneHelper(38, e);
747 case 39: return withLaneHelper(39, e);
748 case 40: return withLaneHelper(40, e);
749 case 41: return withLaneHelper(41, e);
750 case 42: return withLaneHelper(42, e);
751 case 43: return withLaneHelper(43, e);
752 case 44: return withLaneHelper(44, e);
753 case 45: return withLaneHelper(45, e);
754 case 46: return withLaneHelper(46, e);
755 case 47: return withLaneHelper(47, e);
756 case 48: return withLaneHelper(48, e);
757 case 49: return withLaneHelper(49, e);
758 case 50: return withLaneHelper(50, e);
759 case 51: return withLaneHelper(51, e);
760 case 52: return withLaneHelper(52, e);
761 case 53: return withLaneHelper(53, e);
762 case 54: return withLaneHelper(54, e);
763 case 55: return withLaneHelper(55, e);
764 case 56: return withLaneHelper(56, e);
765 case 57: return withLaneHelper(57, e);
766 case 58: return withLaneHelper(58, e);
767 case 59: return withLaneHelper(59, e);
768 case 60: return withLaneHelper(60, e);
769 case 61: return withLaneHelper(61, e);
770 case 62: return withLaneHelper(62, e);
771 case 63: return withLaneHelper(63, e);
772 #end[!32L]
773 #end[!16L]
774 #end[!8L]
775 #end[!4L]
776 #end[!2L]
777 #end[!1L]
778 default: throw new IllegalArgumentException("Index " + i + " must be zero or positive, and less than " + VLENGTH);
779 }
780 #else[!Max]
781 if (i < 0 || i >= VLENGTH) {
782 throw new IllegalArgumentException("Index " + i + " must be zero or positive, and less than " + VLENGTH);
783 }
784 return withLaneHelper(i, e);
785 #end[!Max]
786 }
787
788 public $vectortype$ withLaneHelper(int i, $type$ e) {
789 return VectorSupport.insert(
790 VCLASS, ETYPE, VLENGTH,
791 this, i, (long)e,
792 (v, ix, bits) -> {
793 $type$[] res = v.vec().clone();
794 res[ix] = ($type$)bits;
795 return v.vectorFactory(res);
796 });
797 }
798 #end[FP]
799
800 // Mask
801
802 static final class $masktype$ extends AbstractMask<$Boxtype$> {
803 static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM
804 static final Class<$Boxtype$> ETYPE = $type$.class; // used by the JVM
805
806 $masktype$(boolean[] bits) {
807 this(bits, 0);
808 }
809
810 $masktype$(boolean[] bits, int offset) {
811 super(prepare(bits, offset));
812 }
813
814 $masktype$(boolean val) {
815 super(prepare(val));
816 }
817
818 private static boolean[] prepare(boolean[] bits, int offset) {
819 boolean[] newBits = new boolean[VSPECIES.laneCount()];
820 for (int i = 0; i < newBits.length; i++) {
821 newBits[i] = bits[offset + i];
822 }
823 return newBits;
824 }
825
826 private static boolean[] prepare(boolean val) {
827 boolean[] bits = new boolean[VSPECIES.laneCount()];
828 Arrays.fill(bits, val);
829 return bits;
830 }
831
832 @ForceInline
833 final @Override
834 public $Type$Species vspecies() {
835 // ISSUE: This should probably be a @Stable
836 // field inside AbstractMask, rather than
837 // a megamorphic method.
838 return VSPECIES;
839 }
840
841 @ForceInline
842 boolean[] getBits() {
843 return (boolean[])getPayload();
844 }
845
846 @Override
847 $masktype$ uOp(MUnOp f) {
848 boolean[] res = new boolean[vspecies().laneCount()];
849 boolean[] bits = getBits();
850 for (int i = 0; i < res.length; i++) {
851 res[i] = f.apply(i, bits[i]);
852 }
853 return new $masktype$(res);
854 }
855
856 @Override
857 $masktype$ bOp(VectorMask<$Boxtype$> m, MBinOp f) {
858 boolean[] res = new boolean[vspecies().laneCount()];
859 boolean[] bits = getBits();
860 boolean[] mbits = (($masktype$)m).getBits();
861 for (int i = 0; i < res.length; i++) {
862 res[i] = f.apply(i, bits[i], mbits[i]);
863 }
864 return new $masktype$(res);
865 }
866
867 @ForceInline
868 @Override
869 public final
870 $vectortype$ toVector() {
871 return ($vectortype$) super.toVectorTemplate(); // specialize
872 }
873
874 /**
875 * Helper function for lane-wise mask conversions.
876 * This function kicks in after intrinsic failure.
877 */
878 @ForceInline
879 private final <E>
880 VectorMask<E> defaultMaskCast(AbstractSpecies<E> dsp) {
881 if (length() != dsp.laneCount())
882 throw new IllegalArgumentException("VectorMask length and species length differ");
883 boolean[] maskArray = toArray();
884 return dsp.maskFactory(maskArray).check(dsp);
885 }
886
887 @Override
888 @ForceInline
889 public <E> VectorMask<E> cast(VectorSpecies<E> dsp) {
890 AbstractSpecies<E> species = (AbstractSpecies<E>) dsp;
891 if (length() != species.laneCount())
892 throw new IllegalArgumentException("VectorMask length and species length differ");
893
894 return VectorSupport.convert(VectorSupport.VECTOR_OP_CAST,
895 this.getClass(), ETYPE, VLENGTH,
896 species.maskType(), species.elementType(), VLENGTH,
897 this, species,
898 (m, s) -> s.maskFactory(m.toArray()).check(s));
899 }
900
901 @Override
902 @ForceInline
903 public $masktype$ eq(VectorMask<$Boxtype$> mask) {
904 Objects.requireNonNull(mask);
905 $masktype$ m = ($masktype$)mask;
906 return xor(m.not());
907 }
908
909 // Unary operations
910
911 @Override
912 @ForceInline
913 public $masktype$ not() {
914 return xor(maskAll(true));
915 }
916
917 // Binary operations
918
919 @Override
920 @ForceInline
921 public $masktype$ and(VectorMask<$Boxtype$> mask) {
922 Objects.requireNonNull(mask);
923 $masktype$ m = ($masktype$)mask;
924 return VectorSupport.binaryOp(VECTOR_OP_AND, $masktype$.class, null, $bitstype$.class, VLENGTH,
925 this, m, null,
926 (m1, m2, vm) -> m1.bOp(m2, (i, a, b) -> a & b));
927 }
928
929 @Override
930 @ForceInline
931 public $masktype$ or(VectorMask<$Boxtype$> mask) {
932 Objects.requireNonNull(mask);
933 $masktype$ m = ($masktype$)mask;
934 return VectorSupport.binaryOp(VECTOR_OP_OR, $masktype$.class, null, $bitstype$.class, VLENGTH,
935 this, m, null,
936 (m1, m2, vm) -> m1.bOp(m2, (i, a, b) -> a | b));
937 }
938
939 @ForceInline
940 /* package-private */
941 $masktype$ xor(VectorMask<$Boxtype$> mask) {
942 Objects.requireNonNull(mask);
943 $masktype$ m = ($masktype$)mask;
944 return VectorSupport.binaryOp(VECTOR_OP_XOR, $masktype$.class, null, $bitstype$.class, VLENGTH,
945 this, m, null,
946 (m1, m2, vm) -> m1.bOp(m2, (i, a, b) -> a ^ b));
947 }
948
949 // Mask Query operations
950
951 @Override
952 @ForceInline
953 public int trueCount() {
954 return (int) VectorSupport.maskReductionCoerced(VECTOR_OP_MASK_TRUECOUNT, $masktype$.class, $bitstype$.class, VLENGTH, this,
955 (m) -> trueCountHelper(m.getBits()));
956 }
957
958 @Override
959 @ForceInline
960 public int firstTrue() {
961 return (int) VectorSupport.maskReductionCoerced(VECTOR_OP_MASK_FIRSTTRUE, $masktype$.class, $bitstype$.class, VLENGTH, this,
962 (m) -> firstTrueHelper(m.getBits()));
963 }
964
965 @Override
966 @ForceInline
967 public int lastTrue() {
968 return (int) VectorSupport.maskReductionCoerced(VECTOR_OP_MASK_LASTTRUE, $masktype$.class, $bitstype$.class, VLENGTH, this,
969 (m) -> lastTrueHelper(m.getBits()));
970 }
971
972 @Override
973 @ForceInline
974 public long toLong() {
975 if (length() > Long.SIZE) {
976 throw new UnsupportedOperationException("too many lanes for one long");
977 }
978 return VectorSupport.maskReductionCoerced(VECTOR_OP_MASK_TOLONG, $masktype$.class, $bitstype$.class, VLENGTH, this,
979 (m) -> toLongHelper(m.getBits()));
980 }
981
982 // Reductions
983
984 @Override
985 @ForceInline
986 public boolean anyTrue() {
987 return VectorSupport.test(BT_ne, $masktype$.class, $bitstype$.class, VLENGTH,
988 this, vspecies().maskAll(true),
989 (m, __) -> anyTrueHelper((($masktype$)m).getBits()));
990 }
991
992 @Override
993 @ForceInline
994 public boolean allTrue() {
995 return VectorSupport.test(BT_overflow, $masktype$.class, $bitstype$.class, VLENGTH,
996 this, vspecies().maskAll(true),
997 (m, __) -> allTrueHelper((($masktype$)m).getBits()));
998 }
999
1000 @ForceInline
1001 /*package-private*/
1002 static $masktype$ maskAll(boolean bit) {
1003 return VectorSupport.broadcastCoerced($masktype$.class, $bitstype$.class, VLENGTH,
1004 (bit ? -1 : 0), null,
1005 (v, __) -> (v != 0 ? TRUE_MASK : FALSE_MASK));
1006 }
1007 private static final $masktype$ TRUE_MASK = new $masktype$(true);
1008 private static final $masktype$ FALSE_MASK = new $masktype$(false);
1009
1010 #if[intAndMax]
1011
1012 static boolean[] maskLowerHalf() {
1013 boolean[] a = new boolean[VLENGTH];
1014 int len = a.length >> 1;
1015 for (int i = 0; i < len; i++) {
1016 a[i] = true;
1017 }
1018 return a;
1019 }
1020
1021 #end[intAndMax]
1022 #if[intAndMax]
1023 static final IntMaxMask LOWER_HALF_TRUE_MASK = new IntMaxMask(maskLowerHalf());
1024 #end[intAndMax]
1025 }
1026
1027 // Shuffle
1028
1029 static final class $shuffletype$ extends AbstractShuffle<$Boxtype$> {
1030 static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM
1031 static final Class<$Boxtype$> ETYPE = $type$.class; // used by the JVM
1032
1033 $shuffletype$(byte[] reorder) {
1034 super(VLENGTH, reorder);
1035 }
1036
1037 public $shuffletype$(int[] reorder) {
1038 super(VLENGTH, reorder);
1039 }
1040
1041 public $shuffletype$(int[] reorder, int i) {
1042 super(VLENGTH, reorder, i);
1043 }
1044
1045 public $shuffletype$(IntUnaryOperator fn) {
1046 super(VLENGTH, fn);
1047 }
1048
1049 @Override
1050 public $Type$Species vspecies() {
1051 return VSPECIES;
1052 }
1053
1054 static {
1055 // There must be enough bits in the shuffle lanes to encode
1056 // VLENGTH valid indexes and VLENGTH exceptional ones.
1057 assert(VLENGTH < Byte.MAX_VALUE);
1058 assert(Byte.MIN_VALUE <= -VLENGTH);
1059 }
1060 static final $shuffletype$ IOTA = new $shuffletype$(IDENTITY);
1061
1062 @Override
1063 @ForceInline
1064 public $vectortype$ toVector() {
1065 return VectorSupport.shuffleToVector(VCLASS, ETYPE, $shuffletype$.class, this, VLENGTH,
1066 (s) -> (($vectortype$)(((AbstractShuffle<$Boxtype$>)(s)).toVectorTemplate())));
1067 }
1068
1069 @Override
1070 @ForceInline
1071 public <F> VectorShuffle<F> cast(VectorSpecies<F> s) {
1072 AbstractSpecies<F> species = (AbstractSpecies<F>) s;
1073 if (length() != species.laneCount())
1074 throw new IllegalArgumentException("VectorShuffle length and species length differ");
1075 int[] shuffleArray = toArray();
1076 return s.shuffleFromArray(shuffleArray, 0).check(s);
1077 }
1078
1079 @ForceInline
1080 @Override
1081 public $shuffletype$ rearrange(VectorShuffle<$Boxtype$> shuffle) {
1082 $shuffletype$ s = ($shuffletype$) shuffle;
1083 byte[] reorder1 = reorder();
1084 byte[] reorder2 = s.reorder();
1085 byte[] r = new byte[reorder1.length];
1086 for (int i = 0; i < reorder1.length; i++) {
1087 int ssi = reorder2[i];
1088 r[i] = reorder1[ssi]; // throws on exceptional index
1089 }
1090 return new $shuffletype$(r);
1091 }
1092 }
1093
1094 // ================================================
1095
1096 // Specialized low-level memory operations.
1097
1098 @ForceInline
1099 @Override
1100 final
1101 $abstractvectortype$ fromArray0($type$[] a, int offset) {
1102 return super.fromArray0Template(a, offset); // specialize
1103 }
1104
1105 @ForceInline
1106 @Override
1107 final
1108 $abstractvectortype$ fromArray0($type$[] a, int offset, VectorMask<$Boxtype$> m) {
1109 return super.fromArray0Template($masktype$.class, a, offset, ($masktype$) m); // specialize
1110 }
1111
1112 #if[!byteOrShort]
1113 @ForceInline
1114 @Override
1115 final
1116 $abstractvectortype$ fromArray0($type$[] a, int offset, int[] indexMap, int mapOffset, VectorMask<$Boxtype$> m) {
1117 return super.fromArray0Template($masktype$.class, a, offset, indexMap, mapOffset, ($masktype$) m);
1118 }
1119 #end[!byteOrShort]
1120
1121 #if[short]
1122 @ForceInline
1123 @Override
1124 final
1125 $abstractvectortype$ fromCharArray0(char[] a, int offset) {
1126 return super.fromCharArray0Template(a, offset); // specialize
1127 }
1128
1129 @ForceInline
1130 @Override
1131 final
1132 $abstractvectortype$ fromCharArray0(char[] a, int offset, VectorMask<$Boxtype$> m) {
1133 return super.fromCharArray0Template($masktype$.class, a, offset, ($masktype$) m); // specialize
1134 }
1135 #end[short]
1136
1137 #if[byte]
1138 @ForceInline
1139 @Override
1140 final
1141 $abstractvectortype$ fromBooleanArray0(boolean[] a, int offset) {
1142 return super.fromBooleanArray0Template(a, offset); // specialize
1143 }
1144
1145 @ForceInline
1146 @Override
1147 final
1148 $abstractvectortype$ fromBooleanArray0(boolean[] a, int offset, VectorMask<$Boxtype$> m) {
1149 return super.fromBooleanArray0Template($masktype$.class, a, offset, ($masktype$) m); // specialize
1150 }
1151 #end[byte]
1152
1153 @ForceInline
1154 @Override
1155 final
1156 $abstractvectortype$ fromByteArray0(byte[] a, int offset) {
1157 return super.fromByteArray0Template(a, offset); // specialize
1158 }
1159
1160 @ForceInline
1161 @Override
1162 final
1163 $abstractvectortype$ fromByteArray0(byte[] a, int offset, VectorMask<$Boxtype$> m) {
1164 return super.fromByteArray0Template($masktype$.class, a, offset, ($masktype$) m); // specialize
1165 }
1166
1167 @ForceInline
1168 @Override
1169 final
1170 $abstractvectortype$ fromByteBuffer0(ByteBuffer bb, int offset) {
1171 return super.fromByteBuffer0Template(bb, offset); // specialize
1172 }
1173
1174 @ForceInline
1175 @Override
1176 final
1177 $abstractvectortype$ fromByteBuffer0(ByteBuffer bb, int offset, VectorMask<$Boxtype$> m) {
1178 return super.fromByteBuffer0Template($masktype$.class, bb, offset, ($masktype$) m); // specialize
1179 }
1180
1181 @ForceInline
1182 @Override
1183 final
1184 void intoArray0($type$[] a, int offset) {
1185 super.intoArray0Template(a, offset); // specialize
1186 }
1187
1188 @ForceInline
1189 @Override
1190 final
1191 void intoArray0($type$[] a, int offset, VectorMask<$Boxtype$> m) {
1192 super.intoArray0Template($masktype$.class, a, offset, ($masktype$) m);
1193 }
1194
1195 #if[!byteOrShort]
1196 @ForceInline
1197 @Override
1198 final
1199 void intoArray0($type$[] a, int offset, int[] indexMap, int mapOffset, VectorMask<$Boxtype$> m) {
1200 super.intoArray0Template($masktype$.class, a, offset, indexMap, mapOffset, ($masktype$) m);
1201 }
1202 #end[!byteOrShort]
1203
1204 #if[byte]
1205 @ForceInline
1206 @Override
1207 final
1208 void intoBooleanArray0(boolean[] a, int offset, VectorMask<$Boxtype$> m) {
1209 super.intoBooleanArray0Template($masktype$.class, a, offset, ($masktype$) m);
1210 }
1211 #end[byte]
1212
1213 @ForceInline
1214 @Override
1215 final
1216 void intoByteArray0(byte[] a, int offset) {
1217 super.intoByteArray0Template(a, offset); // specialize
1218 }
1219
1220 @ForceInline
1221 @Override
1222 final
1223 void intoByteArray0(byte[] a, int offset, VectorMask<$Boxtype$> m) {
1224 super.intoByteArray0Template($masktype$.class, a, offset, ($masktype$) m); // specialize
1225 }
1226
1227 @ForceInline
1228 @Override
1229 final
1230 void intoByteBuffer0(ByteBuffer bb, int offset, VectorMask<$Boxtype$> m) {
1231 super.intoByteBuffer0Template($masktype$.class, bb, offset, ($masktype$) m);
1232 }
1233
1234 #if[short]
1235 @ForceInline
1236 @Override
1237 final
1238 void intoCharArray0(char[] a, int offset, VectorMask<$Boxtype$> m) {
1239 super.intoCharArray0Template($masktype$.class, a, offset, ($masktype$) m);
1240 }
1241 #end[short]
1242
1243 // End of specialized low-level memory operations.
1244
1245 // ================================================
1246
1247 }