1 /*
2 * Copyright (c) 2017, 2022, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation. Oracle designates this
8 * particular file as subject to the "Classpath" exception as provided
9 * by Oracle in the LICENSE file that accompanied this code.
10 *
11 * This code is distributed in the hope that it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 * version 2 for more details (a copy is included in the LICENSE file that
15 * accompanied this code).
16 *
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 */
25 package jdk.incubator.vector;
26
27 import java.util.Arrays;
28 import java.util.Objects;
29 import java.util.function.IntUnaryOperator;
30
31 import jdk.incubator.foreign.MemorySegment;
32 import jdk.internal.vm.annotation.ForceInline;
33 import jdk.internal.vm.vector.VectorSupport;
34
35 import static jdk.internal.vm.vector.VectorSupport.*;
36
37 import static jdk.incubator.vector.VectorOperators.*;
38
39 #warn This file is preprocessed before being compiled
40
41 @SuppressWarnings("cast") // warning: redundant cast
42 final class $vectortype$ extends $abstractvectortype$ {
43 static final $Type$Species VSPECIES =
44 ($Type$Species) $Type$Vector.SPECIES_$BITS$;
45
46 static final VectorShape VSHAPE =
47 VSPECIES.vectorShape();
48
49 static final Class<$vectortype$> VCLASS = $vectortype$.class;
50
51 static final int VSIZE = VSPECIES.vectorBitSize();
52
53 static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM
54
55 static final Class<$Boxtype$> ETYPE = $type$.class; // used by the JVM
56
57 $vectortype$($type$[] v) {
58 super(v);
59 }
60
61 // For compatibility as $vectortype$::new,
62 // stored into species.vectorFactory.
63 $vectortype$(Object v) {
64 this(($type$[]) v);
65 }
66
67 static final $vectortype$ ZERO = new $vectortype$(new $type$[VLENGTH]);
68 static final $vectortype$ IOTA = new $vectortype$(VSPECIES.iotaArray());
69
70 static {
71 // Warm up a few species caches.
72 // If we do this too much we will
73 // get NPEs from bootstrap circularity.
74 VSPECIES.dummyVector();
75 VSPECIES.withLanes(LaneType.BYTE);
76 }
77
78 // Specialized extractors
79
80 @ForceInline
81 final @Override
82 public $Type$Species vspecies() {
83 // ISSUE: This should probably be a @Stable
84 // field inside AbstractVector, rather than
85 // a megamorphic method.
86 return VSPECIES;
87 }
88
89 @ForceInline
90 @Override
91 public final Class<$Boxtype$> elementType() { return $type$.class; }
92
93 @ForceInline
94 @Override
95 public final int elementSize() { return $Boxtype$.SIZE; }
96
97 @ForceInline
98 @Override
99 public final VectorShape shape() { return VSHAPE; }
100
101 @ForceInline
102 @Override
103 public final int length() { return VLENGTH; }
104
105 @ForceInline
106 @Override
107 public final int bitSize() { return VSIZE; }
108
109 @ForceInline
110 @Override
111 public final int byteSize() { return VSIZE / Byte.SIZE; }
112
113 /*package-private*/
114 @ForceInline
115 final @Override
116 $type$[] vec() {
117 return ($type$[])getPayload();
118 }
119
120 // Virtualized constructors
121
122 @Override
123 @ForceInline
124 public final $vectortype$ broadcast($type$ e) {
125 return ($vectortype$) super.broadcastTemplate(e); // specialize
126 }
127
128 #if[!long]
129 @Override
130 @ForceInline
131 public final $vectortype$ broadcast(long e) {
132 return ($vectortype$) super.broadcastTemplate(e); // specialize
133 }
134 #end[!long]
135
136 @Override
137 @ForceInline
138 $masktype$ maskFromArray(boolean[] bits) {
139 return new $masktype$(bits);
140 }
141
142 @Override
143 @ForceInline
144 $shuffletype$ iotaShuffle() { return $shuffletype$.IOTA; }
145
146 @ForceInline
147 $shuffletype$ iotaShuffle(int start, int step, boolean wrap) {
148 if (wrap) {
149 return ($shuffletype$)VectorSupport.shuffleIota(ETYPE, $shuffletype$.class, VSPECIES, VLENGTH, start, step, 1,
150 (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i*lstep + lstart, l))));
151 } else {
152 return ($shuffletype$)VectorSupport.shuffleIota(ETYPE, $shuffletype$.class, VSPECIES, VLENGTH, start, step, 0,
153 (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (i*lstep + lstart)));
154 }
155 }
156
157 @Override
158 @ForceInline
159 $shuffletype$ shuffleFromBytes(byte[] reorder) { return new $shuffletype$(reorder); }
160
161 @Override
162 @ForceInline
163 $shuffletype$ shuffleFromArray(int[] indexes, int i) { return new $shuffletype$(indexes, i); }
164
165 @Override
166 @ForceInline
167 $shuffletype$ shuffleFromOp(IntUnaryOperator fn) { return new $shuffletype$(fn); }
168
169 // Make a vector of the same species but the given elements:
170 @ForceInline
171 final @Override
172 $vectortype$ vectorFactory($type$[] vec) {
173 return new $vectortype$(vec);
174 }
175
176 @ForceInline
177 final @Override
178 Byte$bits$Vector asByteVectorRaw() {
179 return (Byte$bits$Vector) super.asByteVectorRawTemplate(); // specialize
180 }
181
182 @ForceInline
183 final @Override
184 AbstractVector<?> asVectorRaw(LaneType laneType) {
185 return super.asVectorRawTemplate(laneType); // specialize
186 }
187
188 // Unary operator
189
190 @ForceInline
191 final @Override
192 $vectortype$ uOp(FUnOp f) {
193 return ($vectortype$) super.uOpTemplate(f); // specialize
194 }
195
196 @ForceInline
197 final @Override
198 $vectortype$ uOp(VectorMask<$Boxtype$> m, FUnOp f) {
199 return ($vectortype$)
200 super.uOpTemplate(($masktype$)m, f); // specialize
201 }
202
203 // Binary operator
204
205 @ForceInline
206 final @Override
207 $vectortype$ bOp(Vector<$Boxtype$> v, FBinOp f) {
208 return ($vectortype$) super.bOpTemplate(($vectortype$)v, f); // specialize
209 }
210
211 @ForceInline
212 final @Override
213 $vectortype$ bOp(Vector<$Boxtype$> v,
214 VectorMask<$Boxtype$> m, FBinOp f) {
215 return ($vectortype$)
216 super.bOpTemplate(($vectortype$)v, ($masktype$)m,
217 f); // specialize
218 }
219
220 // Ternary operator
221
222 @ForceInline
223 final @Override
224 $vectortype$ tOp(Vector<$Boxtype$> v1, Vector<$Boxtype$> v2, FTriOp f) {
225 return ($vectortype$)
226 super.tOpTemplate(($vectortype$)v1, ($vectortype$)v2,
227 f); // specialize
228 }
229
230 @ForceInline
231 final @Override
232 $vectortype$ tOp(Vector<$Boxtype$> v1, Vector<$Boxtype$> v2,
233 VectorMask<$Boxtype$> m, FTriOp f) {
234 return ($vectortype$)
235 super.tOpTemplate(($vectortype$)v1, ($vectortype$)v2,
236 ($masktype$)m, f); // specialize
237 }
238
239 @ForceInline
240 final @Override
241 $type$ rOp($type$ v, VectorMask<$Boxtype$> m, FBinOp f) {
242 return super.rOpTemplate(v, m, f); // specialize
243 }
244
245 @Override
246 @ForceInline
247 public final <F>
248 Vector<F> convertShape(VectorOperators.Conversion<$Boxtype$,F> conv,
249 VectorSpecies<F> rsp, int part) {
250 return super.convertShapeTemplate(conv, rsp, part); // specialize
251 }
252
253 @Override
254 @ForceInline
255 public final <F>
256 Vector<F> reinterpretShape(VectorSpecies<F> toSpecies, int part) {
257 return super.reinterpretShapeTemplate(toSpecies, part); // specialize
258 }
259
260 // Specialized algebraic operations:
261
262 // The following definition forces a specialized version of this
263 // crucial method into the v-table of this class. A call to add()
264 // will inline to a call to lanewise(ADD,), at which point the JIT
265 // intrinsic will have the opcode of ADD, plus all the metadata
266 // for this particular class, enabling it to generate precise
267 // code.
268 //
269 // There is probably no benefit to the JIT to specialize the
270 // masked or broadcast versions of the lanewise method.
271
272 @Override
273 @ForceInline
274 public $vectortype$ lanewise(Unary op) {
275 return ($vectortype$) super.lanewiseTemplate(op); // specialize
276 }
277
278 @Override
279 @ForceInline
280 public $vectortype$ lanewise(Unary op, VectorMask<$Boxtype$> m) {
281 return ($vectortype$) super.lanewiseTemplate(op, $masktype$.class, ($masktype$) m); // specialize
282 }
283
284 @Override
285 @ForceInline
286 public $vectortype$ lanewise(Binary op, Vector<$Boxtype$> v) {
287 return ($vectortype$) super.lanewiseTemplate(op, v); // specialize
288 }
289
290 @Override
291 @ForceInline
292 public $vectortype$ lanewise(Binary op, Vector<$Boxtype$> v, VectorMask<$Boxtype$> m) {
293 return ($vectortype$) super.lanewiseTemplate(op, $masktype$.class, v, ($masktype$) m); // specialize
294 }
295
296 #if[!FP]
297 /*package-private*/
298 @Override
299 @ForceInline $vectortype$
300 lanewiseShift(VectorOperators.Binary op, int e) {
301 return ($vectortype$) super.lanewiseShiftTemplate(op, e); // specialize
302 }
303
304 /*package-private*/
305 @Override
306 @ForceInline $vectortype$
307 lanewiseShift(VectorOperators.Binary op, int e, VectorMask<$Boxtype$> m) {
308 return ($vectortype$) super.lanewiseShiftTemplate(op, $masktype$.class, e, ($masktype$) m); // specialize
309 }
310 #end[!FP]
311
312 /*package-private*/
313 @Override
314 @ForceInline
315 public final
316 $vectortype$
317 lanewise(Ternary op, Vector<$Boxtype$> v1, Vector<$Boxtype$> v2) {
318 return ($vectortype$) super.lanewiseTemplate(op, v1, v2); // specialize
319 }
320
321 @Override
322 @ForceInline
323 public final
324 $vectortype$
325 lanewise(Ternary op, Vector<$Boxtype$> v1, Vector<$Boxtype$> v2, VectorMask<$Boxtype$> m) {
326 return ($vectortype$) super.lanewiseTemplate(op, $masktype$.class, v1, v2, ($masktype$) m); // specialize
327 }
328
329 @Override
330 @ForceInline
331 public final
332 $vectortype$ addIndex(int scale) {
333 return ($vectortype$) super.addIndexTemplate(scale); // specialize
334 }
335
336 // Type specific horizontal reductions
337
338 @Override
339 @ForceInline
340 public final $type$ reduceLanes(VectorOperators.Associative op) {
341 return super.reduceLanesTemplate(op); // specialized
342 }
343
344 @Override
345 @ForceInline
346 public final $type$ reduceLanes(VectorOperators.Associative op,
347 VectorMask<$Boxtype$> m) {
348 return super.reduceLanesTemplate(op, $masktype$.class, ($masktype$) m); // specialized
349 }
350
351 @Override
352 @ForceInline
353 public final long reduceLanesToLong(VectorOperators.Associative op) {
354 return (long) super.reduceLanesTemplate(op); // specialized
355 }
356
357 @Override
358 @ForceInline
359 public final long reduceLanesToLong(VectorOperators.Associative op,
360 VectorMask<$Boxtype$> m) {
361 return (long) super.reduceLanesTemplate(op, $masktype$.class, ($masktype$) m); // specialized
362 }
363
364 @ForceInline
365 public VectorShuffle<$Boxtype$> toShuffle() {
366 return super.toShuffleTemplate($shuffletype$.class); // specialize
367 }
368
369 // Specialized unary testing
370
371 @Override
372 @ForceInline
373 public final $masktype$ test(Test op) {
374 return super.testTemplate($masktype$.class, op); // specialize
375 }
376
377 @Override
378 @ForceInline
379 public final $masktype$ test(Test op, VectorMask<$Boxtype$> m) {
380 return super.testTemplate($masktype$.class, op, ($masktype$) m); // specialize
381 }
382
383 // Specialized comparisons
384
385 @Override
386 @ForceInline
387 public final $masktype$ compare(Comparison op, Vector<$Boxtype$> v) {
388 return super.compareTemplate($masktype$.class, op, v); // specialize
389 }
390
391 @Override
392 @ForceInline
393 public final $masktype$ compare(Comparison op, $type$ s) {
394 return super.compareTemplate($masktype$.class, op, s); // specialize
395 }
396
397 #if[!long]
398 @Override
399 @ForceInline
400 public final $masktype$ compare(Comparison op, long s) {
401 return super.compareTemplate($masktype$.class, op, s); // specialize
402 }
403 #end[!long]
404
405 @Override
406 @ForceInline
407 public final $masktype$ compare(Comparison op, Vector<$Boxtype$> v, VectorMask<$Boxtype$> m) {
408 return super.compareTemplate($masktype$.class, op, v, ($masktype$) m);
409 }
410
411
412 @Override
413 @ForceInline
414 public $vectortype$ blend(Vector<$Boxtype$> v, VectorMask<$Boxtype$> m) {
415 return ($vectortype$)
416 super.blendTemplate($masktype$.class,
417 ($vectortype$) v,
418 ($masktype$) m); // specialize
419 }
420
421 @Override
422 @ForceInline
423 public $vectortype$ slice(int origin, Vector<$Boxtype$> v) {
424 return ($vectortype$) super.sliceTemplate(origin, v); // specialize
425 }
426
427 @Override
428 @ForceInline
429 public $vectortype$ slice(int origin) {
430 return ($vectortype$) super.sliceTemplate(origin); // specialize
431 }
432
433 @Override
434 @ForceInline
435 public $vectortype$ unslice(int origin, Vector<$Boxtype$> w, int part) {
436 return ($vectortype$) super.unsliceTemplate(origin, w, part); // specialize
437 }
438
439 @Override
440 @ForceInline
441 public $vectortype$ unslice(int origin, Vector<$Boxtype$> w, int part, VectorMask<$Boxtype$> m) {
442 return ($vectortype$)
443 super.unsliceTemplate($masktype$.class,
444 origin, w, part,
445 ($masktype$) m); // specialize
446 }
447
448 @Override
449 @ForceInline
450 public $vectortype$ unslice(int origin) {
451 return ($vectortype$) super.unsliceTemplate(origin); // specialize
452 }
453
454 @Override
455 @ForceInline
456 public $vectortype$ rearrange(VectorShuffle<$Boxtype$> s) {
457 return ($vectortype$)
458 super.rearrangeTemplate($shuffletype$.class,
459 ($shuffletype$) s); // specialize
460 }
461
462 @Override
463 @ForceInline
464 public $vectortype$ rearrange(VectorShuffle<$Boxtype$> shuffle,
465 VectorMask<$Boxtype$> m) {
466 return ($vectortype$)
467 super.rearrangeTemplate($shuffletype$.class,
468 $masktype$.class,
469 ($shuffletype$) shuffle,
470 ($masktype$) m); // specialize
471 }
472
473 @Override
474 @ForceInline
475 public $vectortype$ rearrange(VectorShuffle<$Boxtype$> s,
476 Vector<$Boxtype$> v) {
477 return ($vectortype$)
478 super.rearrangeTemplate($shuffletype$.class,
479 ($shuffletype$) s,
480 ($vectortype$) v); // specialize
481 }
482
483 @Override
484 @ForceInline
485 public $vectortype$ compress(VectorMask<$Boxtype$> m) {
486 return ($vectortype$)
487 super.compressTemplate($masktype$.class,
488 ($masktype$) m); // specialize
489 }
490
491 @Override
492 @ForceInline
493 public $vectortype$ expand(VectorMask<$Boxtype$> m) {
494 return ($vectortype$)
495 super.expandTemplate($masktype$.class,
496 ($masktype$) m); // specialize
497 }
498
499 @Override
500 @ForceInline
501 public $vectortype$ selectFrom(Vector<$Boxtype$> v) {
502 return ($vectortype$)
503 super.selectFromTemplate(($vectortype$) v); // specialize
504 }
505
506 @Override
507 @ForceInline
508 public $vectortype$ selectFrom(Vector<$Boxtype$> v,
509 VectorMask<$Boxtype$> m) {
510 return ($vectortype$)
511 super.selectFromTemplate(($vectortype$) v,
512 ($masktype$) m); // specialize
513 }
514
515
516 #if[FP]
517 @ForceInline
518 @Override
519 public $type$ lane(int i) {
520 #if[!Max]
521 $bitstype$ bits;
522 switch(i) {
523 case 0: bits = laneHelper(0); break;
524 #if[!1L]
525 case 1: bits = laneHelper(1); break;
526 #if[!2L]
527 case 2: bits = laneHelper(2); break;
528 case 3: bits = laneHelper(3); break;
529 #if[!4L]
530 case 4: bits = laneHelper(4); break;
531 case 5: bits = laneHelper(5); break;
532 case 6: bits = laneHelper(6); break;
533 case 7: bits = laneHelper(7); break;
534 #if[!8L]
535 case 8: bits = laneHelper(8); break;
536 case 9: bits = laneHelper(9); break;
537 case 10: bits = laneHelper(10); break;
538 case 11: bits = laneHelper(11); break;
539 case 12: bits = laneHelper(12); break;
540 case 13: bits = laneHelper(13); break;
541 case 14: bits = laneHelper(14); break;
542 case 15: bits = laneHelper(15); break;
543 #end[!8L]
544 #end[!4L]
545 #end[!2L]
546 #end[!1L]
547 default: throw new IllegalArgumentException("Index " + i + " must be zero or positive, and less than " + VLENGTH);
548 }
549 #else[!Max]
550 if (i < 0 || i >= VLENGTH) {
551 throw new IllegalArgumentException("Index " + i + " must be zero or positive, and less than " + VLENGTH);
552 }
553 $bitstype$ bits = laneHelper(i);
554 #end[!Max]
555 return $Type$.$bitstype$BitsTo$Fptype$(bits);
556 }
557
558 public $bitstype$ laneHelper(int i) {
559 return ($bitstype$) VectorSupport.extract(
560 VCLASS, ETYPE, VLENGTH,
561 this, i,
562 (vec, ix) -> {
563 $type$[] vecarr = vec.vec();
564 return (long)$Type$.$type$To$Bitstype$Bits(vecarr[ix]);
565 });
566 }
567
568 @ForceInline
569 @Override
570 public $vectortype$ withLane(int i, $type$ e) {
571 #if[!Max]
572 switch(i) {
573 case 0: return withLaneHelper(0, e);
574 #if[!1L]
575 case 1: return withLaneHelper(1, e);
576 #if[!2L]
577 case 2: return withLaneHelper(2, e);
578 case 3: return withLaneHelper(3, e);
579 #if[!4L]
580 case 4: return withLaneHelper(4, e);
581 case 5: return withLaneHelper(5, e);
582 case 6: return withLaneHelper(6, e);
583 case 7: return withLaneHelper(7, e);
584 #if[!8L]
585 case 8: return withLaneHelper(8, e);
586 case 9: return withLaneHelper(9, e);
587 case 10: return withLaneHelper(10, e);
588 case 11: return withLaneHelper(11, e);
589 case 12: return withLaneHelper(12, e);
590 case 13: return withLaneHelper(13, e);
591 case 14: return withLaneHelper(14, e);
592 case 15: return withLaneHelper(15, e);
593 #end[!8L]
594 #end[!4L]
595 #end[!2L]
596 #end[!1L]
597 default: throw new IllegalArgumentException("Index " + i + " must be zero or positive, and less than " + VLENGTH);
598 }
599 #else[!Max]
600 if (i < 0 || i >= VLENGTH) {
601 throw new IllegalArgumentException("Index " + i + " must be zero or positive, and less than " + VLENGTH);
602 }
603 return withLaneHelper(i, e);
604 #end[!Max]
605 }
606
607 public $vectortype$ withLaneHelper(int i, $type$ e) {
608 return VectorSupport.insert(
609 VCLASS, ETYPE, VLENGTH,
610 this, i, (long)$Type$.$type$To$Bitstype$Bits(e),
611 (v, ix, bits) -> {
612 $type$[] res = v.vec().clone();
613 res[ix] = $Type$.$bitstype$BitsTo$Type$(($bitstype$)bits);
614 return v.vectorFactory(res);
615 });
616 }
617 #else[FP]
618 @ForceInline
619 @Override
620 public $type$ lane(int i) {
621 #if[!Max]
622 switch(i) {
623 case 0: return laneHelper(0);
624 #if[!1L]
625 case 1: return laneHelper(1);
626 #if[!2L]
627 case 2: return laneHelper(2);
628 case 3: return laneHelper(3);
629 #if[!4L]
630 case 4: return laneHelper(4);
631 case 5: return laneHelper(5);
632 case 6: return laneHelper(6);
633 case 7: return laneHelper(7);
634 #if[!8L]
635 case 8: return laneHelper(8);
636 case 9: return laneHelper(9);
637 case 10: return laneHelper(10);
638 case 11: return laneHelper(11);
639 case 12: return laneHelper(12);
640 case 13: return laneHelper(13);
641 case 14: return laneHelper(14);
642 case 15: return laneHelper(15);
643 #if[!16L]
644 case 16: return laneHelper(16);
645 case 17: return laneHelper(17);
646 case 18: return laneHelper(18);
647 case 19: return laneHelper(19);
648 case 20: return laneHelper(20);
649 case 21: return laneHelper(21);
650 case 22: return laneHelper(22);
651 case 23: return laneHelper(23);
652 case 24: return laneHelper(24);
653 case 25: return laneHelper(25);
654 case 26: return laneHelper(26);
655 case 27: return laneHelper(27);
656 case 28: return laneHelper(28);
657 case 29: return laneHelper(29);
658 case 30: return laneHelper(30);
659 case 31: return laneHelper(31);
660 #if[!32L]
661 case 32: return laneHelper(32);
662 case 33: return laneHelper(33);
663 case 34: return laneHelper(34);
664 case 35: return laneHelper(35);
665 case 36: return laneHelper(36);
666 case 37: return laneHelper(37);
667 case 38: return laneHelper(38);
668 case 39: return laneHelper(39);
669 case 40: return laneHelper(40);
670 case 41: return laneHelper(41);
671 case 42: return laneHelper(42);
672 case 43: return laneHelper(43);
673 case 44: return laneHelper(44);
674 case 45: return laneHelper(45);
675 case 46: return laneHelper(46);
676 case 47: return laneHelper(47);
677 case 48: return laneHelper(48);
678 case 49: return laneHelper(49);
679 case 50: return laneHelper(50);
680 case 51: return laneHelper(51);
681 case 52: return laneHelper(52);
682 case 53: return laneHelper(53);
683 case 54: return laneHelper(54);
684 case 55: return laneHelper(55);
685 case 56: return laneHelper(56);
686 case 57: return laneHelper(57);
687 case 58: return laneHelper(58);
688 case 59: return laneHelper(59);
689 case 60: return laneHelper(60);
690 case 61: return laneHelper(61);
691 case 62: return laneHelper(62);
692 case 63: return laneHelper(63);
693 #end[!32L]
694 #end[!16L]
695 #end[!8L]
696 #end[!4L]
697 #end[!2L]
698 #end[!1L]
699 default: throw new IllegalArgumentException("Index " + i + " must be zero or positive, and less than " + VLENGTH);
700 }
701 #else[!Max]
702 if (i < 0 || i >= VLENGTH) {
703 throw new IllegalArgumentException("Index " + i + " must be zero or positive, and less than " + VLENGTH);
704 }
705 return laneHelper(i);
706 #end[!Max]
707 }
708
709 public $type$ laneHelper(int i) {
710 return ($type$) VectorSupport.extract(
711 VCLASS, ETYPE, VLENGTH,
712 this, i,
713 (vec, ix) -> {
714 $type$[] vecarr = vec.vec();
715 return (long)vecarr[ix];
716 });
717 }
718
719 @ForceInline
720 @Override
721 public $vectortype$ withLane(int i, $type$ e) {
722 #if[!Max]
723 switch (i) {
724 case 0: return withLaneHelper(0, e);
725 #if[!1L]
726 case 1: return withLaneHelper(1, e);
727 #if[!2L]
728 case 2: return withLaneHelper(2, e);
729 case 3: return withLaneHelper(3, e);
730 #if[!4L]
731 case 4: return withLaneHelper(4, e);
732 case 5: return withLaneHelper(5, e);
733 case 6: return withLaneHelper(6, e);
734 case 7: return withLaneHelper(7, e);
735 #if[!8L]
736 case 8: return withLaneHelper(8, e);
737 case 9: return withLaneHelper(9, e);
738 case 10: return withLaneHelper(10, e);
739 case 11: return withLaneHelper(11, e);
740 case 12: return withLaneHelper(12, e);
741 case 13: return withLaneHelper(13, e);
742 case 14: return withLaneHelper(14, e);
743 case 15: return withLaneHelper(15, e);
744 #if[!16L]
745 case 16: return withLaneHelper(16, e);
746 case 17: return withLaneHelper(17, e);
747 case 18: return withLaneHelper(18, e);
748 case 19: return withLaneHelper(19, e);
749 case 20: return withLaneHelper(20, e);
750 case 21: return withLaneHelper(21, e);
751 case 22: return withLaneHelper(22, e);
752 case 23: return withLaneHelper(23, e);
753 case 24: return withLaneHelper(24, e);
754 case 25: return withLaneHelper(25, e);
755 case 26: return withLaneHelper(26, e);
756 case 27: return withLaneHelper(27, e);
757 case 28: return withLaneHelper(28, e);
758 case 29: return withLaneHelper(29, e);
759 case 30: return withLaneHelper(30, e);
760 case 31: return withLaneHelper(31, e);
761 #if[!32L]
762 case 32: return withLaneHelper(32, e);
763 case 33: return withLaneHelper(33, e);
764 case 34: return withLaneHelper(34, e);
765 case 35: return withLaneHelper(35, e);
766 case 36: return withLaneHelper(36, e);
767 case 37: return withLaneHelper(37, e);
768 case 38: return withLaneHelper(38, e);
769 case 39: return withLaneHelper(39, e);
770 case 40: return withLaneHelper(40, e);
771 case 41: return withLaneHelper(41, e);
772 case 42: return withLaneHelper(42, e);
773 case 43: return withLaneHelper(43, e);
774 case 44: return withLaneHelper(44, e);
775 case 45: return withLaneHelper(45, e);
776 case 46: return withLaneHelper(46, e);
777 case 47: return withLaneHelper(47, e);
778 case 48: return withLaneHelper(48, e);
779 case 49: return withLaneHelper(49, e);
780 case 50: return withLaneHelper(50, e);
781 case 51: return withLaneHelper(51, e);
782 case 52: return withLaneHelper(52, e);
783 case 53: return withLaneHelper(53, e);
784 case 54: return withLaneHelper(54, e);
785 case 55: return withLaneHelper(55, e);
786 case 56: return withLaneHelper(56, e);
787 case 57: return withLaneHelper(57, e);
788 case 58: return withLaneHelper(58, e);
789 case 59: return withLaneHelper(59, e);
790 case 60: return withLaneHelper(60, e);
791 case 61: return withLaneHelper(61, e);
792 case 62: return withLaneHelper(62, e);
793 case 63: return withLaneHelper(63, e);
794 #end[!32L]
795 #end[!16L]
796 #end[!8L]
797 #end[!4L]
798 #end[!2L]
799 #end[!1L]
800 default: throw new IllegalArgumentException("Index " + i + " must be zero or positive, and less than " + VLENGTH);
801 }
802 #else[!Max]
803 if (i < 0 || i >= VLENGTH) {
804 throw new IllegalArgumentException("Index " + i + " must be zero or positive, and less than " + VLENGTH);
805 }
806 return withLaneHelper(i, e);
807 #end[!Max]
808 }
809
810 public $vectortype$ withLaneHelper(int i, $type$ e) {
811 return VectorSupport.insert(
812 VCLASS, ETYPE, VLENGTH,
813 this, i, (long)e,
814 (v, ix, bits) -> {
815 $type$[] res = v.vec().clone();
816 res[ix] = ($type$)bits;
817 return v.vectorFactory(res);
818 });
819 }
820 #end[FP]
821
822 // Mask
823
824 static final class $masktype$ extends AbstractMask<$Boxtype$> {
825 static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM
826 static final Class<$Boxtype$> ETYPE = $type$.class; // used by the JVM
827
828 $masktype$(boolean[] bits) {
829 this(bits, 0);
830 }
831
832 $masktype$(boolean[] bits, int offset) {
833 super(prepare(bits, offset));
834 }
835
836 $masktype$(boolean val) {
837 super(prepare(val));
838 }
839
840 private static boolean[] prepare(boolean[] bits, int offset) {
841 boolean[] newBits = new boolean[VSPECIES.laneCount()];
842 for (int i = 0; i < newBits.length; i++) {
843 newBits[i] = bits[offset + i];
844 }
845 return newBits;
846 }
847
848 private static boolean[] prepare(boolean val) {
849 boolean[] bits = new boolean[VSPECIES.laneCount()];
850 Arrays.fill(bits, val);
851 return bits;
852 }
853
854 @ForceInline
855 final @Override
856 public $Type$Species vspecies() {
857 // ISSUE: This should probably be a @Stable
858 // field inside AbstractMask, rather than
859 // a megamorphic method.
860 return VSPECIES;
861 }
862
863 @ForceInline
864 boolean[] getBits() {
865 return (boolean[])getPayload();
866 }
867
868 @Override
869 $masktype$ uOp(MUnOp f) {
870 boolean[] res = new boolean[vspecies().laneCount()];
871 boolean[] bits = getBits();
872 for (int i = 0; i < res.length; i++) {
873 res[i] = f.apply(i, bits[i]);
874 }
875 return new $masktype$(res);
876 }
877
878 @Override
879 $masktype$ bOp(VectorMask<$Boxtype$> m, MBinOp f) {
880 boolean[] res = new boolean[vspecies().laneCount()];
881 boolean[] bits = getBits();
882 boolean[] mbits = (($masktype$)m).getBits();
883 for (int i = 0; i < res.length; i++) {
884 res[i] = f.apply(i, bits[i], mbits[i]);
885 }
886 return new $masktype$(res);
887 }
888
889 @ForceInline
890 @Override
891 public final
892 $vectortype$ toVector() {
893 return ($vectortype$) super.toVectorTemplate(); // specialize
894 }
895
896 /**
897 * Helper function for lane-wise mask conversions.
898 * This function kicks in after intrinsic failure.
899 */
900 @ForceInline
901 private final <E>
902 VectorMask<E> defaultMaskCast(AbstractSpecies<E> dsp) {
903 if (length() != dsp.laneCount())
904 throw new IllegalArgumentException("VectorMask length and species length differ");
905 boolean[] maskArray = toArray();
906 return dsp.maskFactory(maskArray).check(dsp);
907 }
908
909 @Override
910 @ForceInline
911 public <E> VectorMask<E> cast(VectorSpecies<E> dsp) {
912 AbstractSpecies<E> species = (AbstractSpecies<E>) dsp;
913 if (length() != species.laneCount())
914 throw new IllegalArgumentException("VectorMask length and species length differ");
915
916 return VectorSupport.convert(VectorSupport.VECTOR_OP_CAST,
917 this.getClass(), ETYPE, VLENGTH,
918 species.maskType(), species.elementType(), VLENGTH,
919 this, species,
920 (m, s) -> s.maskFactory(m.toArray()).check(s));
921 }
922
923 @Override
924 @ForceInline
925 public $masktype$ eq(VectorMask<$Boxtype$> mask) {
926 Objects.requireNonNull(mask);
927 $masktype$ m = ($masktype$)mask;
928 return xor(m.not());
929 }
930
931 // Unary operations
932
933 @Override
934 @ForceInline
935 public $masktype$ not() {
936 return xor(maskAll(true));
937 }
938
939 @Override
940 @ForceInline
941 public $masktype$ compress() {
942 return ($masktype$)VectorSupport.comExpOp(VectorSupport.VECTOR_OP_MASK_COMPRESS,
943 $vectortype$.class, $masktype$.class, ETYPE, VLENGTH, null, this,
944 (v1, m1) -> VSPECIES.iota().compare(VectorOperators.LT, m1.trueCount()));
945 }
946
947
948 // Binary operations
949
950 @Override
951 @ForceInline
952 public $masktype$ and(VectorMask<$Boxtype$> mask) {
953 Objects.requireNonNull(mask);
954 $masktype$ m = ($masktype$)mask;
955 return VectorSupport.binaryOp(VECTOR_OP_AND, $masktype$.class, null, $bitstype$.class, VLENGTH,
956 this, m, null,
957 (m1, m2, vm) -> m1.bOp(m2, (i, a, b) -> a & b));
958 }
959
960 @Override
961 @ForceInline
962 public $masktype$ or(VectorMask<$Boxtype$> mask) {
963 Objects.requireNonNull(mask);
964 $masktype$ m = ($masktype$)mask;
965 return VectorSupport.binaryOp(VECTOR_OP_OR, $masktype$.class, null, $bitstype$.class, VLENGTH,
966 this, m, null,
967 (m1, m2, vm) -> m1.bOp(m2, (i, a, b) -> a | b));
968 }
969
970 @ForceInline
971 /* package-private */
972 $masktype$ xor(VectorMask<$Boxtype$> mask) {
973 Objects.requireNonNull(mask);
974 $masktype$ m = ($masktype$)mask;
975 return VectorSupport.binaryOp(VECTOR_OP_XOR, $masktype$.class, null, $bitstype$.class, VLENGTH,
976 this, m, null,
977 (m1, m2, vm) -> m1.bOp(m2, (i, a, b) -> a ^ b));
978 }
979
980 // Mask Query operations
981
982 @Override
983 @ForceInline
984 public int trueCount() {
985 return (int) VectorSupport.maskReductionCoerced(VECTOR_OP_MASK_TRUECOUNT, $masktype$.class, $bitstype$.class, VLENGTH, this,
986 (m) -> trueCountHelper(m.getBits()));
987 }
988
989 @Override
990 @ForceInline
991 public int firstTrue() {
992 return (int) VectorSupport.maskReductionCoerced(VECTOR_OP_MASK_FIRSTTRUE, $masktype$.class, $bitstype$.class, VLENGTH, this,
993 (m) -> firstTrueHelper(m.getBits()));
994 }
995
996 @Override
997 @ForceInline
998 public int lastTrue() {
999 return (int) VectorSupport.maskReductionCoerced(VECTOR_OP_MASK_LASTTRUE, $masktype$.class, $bitstype$.class, VLENGTH, this,
1000 (m) -> lastTrueHelper(m.getBits()));
1001 }
1002
1003 @Override
1004 @ForceInline
1005 public long toLong() {
1006 if (length() > Long.SIZE) {
1007 throw new UnsupportedOperationException("too many lanes for one long");
1008 }
1009 return VectorSupport.maskReductionCoerced(VECTOR_OP_MASK_TOLONG, $masktype$.class, $bitstype$.class, VLENGTH, this,
1010 (m) -> toLongHelper(m.getBits()));
1011 }
1012
1013 // Reductions
1014
1015 @Override
1016 @ForceInline
1017 public boolean anyTrue() {
1018 return VectorSupport.test(BT_ne, $masktype$.class, $bitstype$.class, VLENGTH,
1019 this, vspecies().maskAll(true),
1020 (m, __) -> anyTrueHelper((($masktype$)m).getBits()));
1021 }
1022
1023 @Override
1024 @ForceInline
1025 public boolean allTrue() {
1026 return VectorSupport.test(BT_overflow, $masktype$.class, $bitstype$.class, VLENGTH,
1027 this, vspecies().maskAll(true),
1028 (m, __) -> allTrueHelper((($masktype$)m).getBits()));
1029 }
1030
1031 @ForceInline
1032 /*package-private*/
1033 static $masktype$ maskAll(boolean bit) {
1034 return VectorSupport.fromBitsCoerced($masktype$.class, $bitstype$.class, VLENGTH,
1035 (bit ? -1 : 0), MODE_BROADCAST, null,
1036 (v, __) -> (v != 0 ? TRUE_MASK : FALSE_MASK));
1037 }
1038 private static final $masktype$ TRUE_MASK = new $masktype$(true);
1039 private static final $masktype$ FALSE_MASK = new $masktype$(false);
1040
1041 #if[intAndMax]
1042
1043 static boolean[] maskLowerHalf() {
1044 boolean[] a = new boolean[VLENGTH];
1045 int len = a.length >> 1;
1046 for (int i = 0; i < len; i++) {
1047 a[i] = true;
1048 }
1049 return a;
1050 }
1051
1052 #end[intAndMax]
1053 #if[intAndMax]
1054 static final IntMaxMask LOWER_HALF_TRUE_MASK = new IntMaxMask(maskLowerHalf());
1055 #end[intAndMax]
1056 }
1057
1058 // Shuffle
1059
1060 static final class $shuffletype$ extends AbstractShuffle<$Boxtype$> {
1061 static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM
1062 static final Class<$Boxtype$> ETYPE = $type$.class; // used by the JVM
1063
1064 $shuffletype$(byte[] reorder) {
1065 super(VLENGTH, reorder);
1066 }
1067
1068 public $shuffletype$(int[] reorder) {
1069 super(VLENGTH, reorder);
1070 }
1071
1072 public $shuffletype$(int[] reorder, int i) {
1073 super(VLENGTH, reorder, i);
1074 }
1075
1076 public $shuffletype$(IntUnaryOperator fn) {
1077 super(VLENGTH, fn);
1078 }
1079
1080 @Override
1081 public $Type$Species vspecies() {
1082 return VSPECIES;
1083 }
1084
1085 static {
1086 // There must be enough bits in the shuffle lanes to encode
1087 // VLENGTH valid indexes and VLENGTH exceptional ones.
1088 assert(VLENGTH < Byte.MAX_VALUE);
1089 assert(Byte.MIN_VALUE <= -VLENGTH);
1090 }
1091 static final $shuffletype$ IOTA = new $shuffletype$(IDENTITY);
1092
1093 @Override
1094 @ForceInline
1095 public $vectortype$ toVector() {
1096 return VectorSupport.shuffleToVector(VCLASS, ETYPE, $shuffletype$.class, this, VLENGTH,
1097 (s) -> (($vectortype$)(((AbstractShuffle<$Boxtype$>)(s)).toVectorTemplate())));
1098 }
1099
1100 @Override
1101 @ForceInline
1102 public <F> VectorShuffle<F> cast(VectorSpecies<F> s) {
1103 AbstractSpecies<F> species = (AbstractSpecies<F>) s;
1104 if (length() != species.laneCount())
1105 throw new IllegalArgumentException("VectorShuffle length and species length differ");
1106 int[] shuffleArray = toArray();
1107 return s.shuffleFromArray(shuffleArray, 0).check(s);
1108 }
1109
1110 @ForceInline
1111 @Override
1112 public $shuffletype$ rearrange(VectorShuffle<$Boxtype$> shuffle) {
1113 $shuffletype$ s = ($shuffletype$) shuffle;
1114 byte[] reorder1 = reorder();
1115 byte[] reorder2 = s.reorder();
1116 byte[] r = new byte[reorder1.length];
1117 for (int i = 0; i < reorder1.length; i++) {
1118 int ssi = reorder2[i];
1119 r[i] = reorder1[ssi]; // throws on exceptional index
1120 }
1121 return new $shuffletype$(r);
1122 }
1123 }
1124
1125 // ================================================
1126
1127 // Specialized low-level memory operations.
1128
1129 @ForceInline
1130 @Override
1131 final
1132 $abstractvectortype$ fromArray0($type$[] a, int offset) {
1133 return super.fromArray0Template(a, offset); // specialize
1134 }
1135
1136 @ForceInline
1137 @Override
1138 final
1139 $abstractvectortype$ fromArray0($type$[] a, int offset, VectorMask<$Boxtype$> m) {
1140 return super.fromArray0Template($masktype$.class, a, offset, ($masktype$) m); // specialize
1141 }
1142
1143 #if[!byteOrShort]
1144 @ForceInline
1145 @Override
1146 final
1147 $abstractvectortype$ fromArray0($type$[] a, int offset, int[] indexMap, int mapOffset, VectorMask<$Boxtype$> m) {
1148 return super.fromArray0Template($masktype$.class, a, offset, indexMap, mapOffset, ($masktype$) m);
1149 }
1150 #end[!byteOrShort]
1151
1152 #if[short]
1153 @ForceInline
1154 @Override
1155 final
1156 $abstractvectortype$ fromCharArray0(char[] a, int offset) {
1157 return super.fromCharArray0Template(a, offset); // specialize
1158 }
1159
1160 @ForceInline
1161 @Override
1162 final
1163 $abstractvectortype$ fromCharArray0(char[] a, int offset, VectorMask<$Boxtype$> m) {
1164 return super.fromCharArray0Template($masktype$.class, a, offset, ($masktype$) m); // specialize
1165 }
1166 #end[short]
1167
1168 #if[byte]
1169 @ForceInline
1170 @Override
1171 final
1172 $abstractvectortype$ fromBooleanArray0(boolean[] a, int offset) {
1173 return super.fromBooleanArray0Template(a, offset); // specialize
1174 }
1175
1176 @ForceInline
1177 @Override
1178 final
1179 $abstractvectortype$ fromBooleanArray0(boolean[] a, int offset, VectorMask<$Boxtype$> m) {
1180 return super.fromBooleanArray0Template($masktype$.class, a, offset, ($masktype$) m); // specialize
1181 }
1182 #end[byte]
1183
1184 @ForceInline
1185 @Override
1186 final
1187 $abstractvectortype$ fromMemorySegment0(MemorySegment ms, long offset) {
1188 return super.fromMemorySegment0Template(ms, offset); // specialize
1189 }
1190
1191 @ForceInline
1192 @Override
1193 final
1194 $abstractvectortype$ fromMemorySegment0(MemorySegment ms, long offset, VectorMask<$Boxtype$> m) {
1195 return super.fromMemorySegment0Template($masktype$.class, ms, offset, ($masktype$) m); // specialize
1196 }
1197
1198 @ForceInline
1199 @Override
1200 final
1201 void intoArray0($type$[] a, int offset) {
1202 super.intoArray0Template(a, offset); // specialize
1203 }
1204
1205 @ForceInline
1206 @Override
1207 final
1208 void intoArray0($type$[] a, int offset, VectorMask<$Boxtype$> m) {
1209 super.intoArray0Template($masktype$.class, a, offset, ($masktype$) m);
1210 }
1211
1212 #if[!byteOrShort]
1213 @ForceInline
1214 @Override
1215 final
1216 void intoArray0($type$[] a, int offset, int[] indexMap, int mapOffset, VectorMask<$Boxtype$> m) {
1217 super.intoArray0Template($masktype$.class, a, offset, indexMap, mapOffset, ($masktype$) m);
1218 }
1219 #end[!byteOrShort]
1220
1221 #if[byte]
1222 @ForceInline
1223 @Override
1224 final
1225 void intoBooleanArray0(boolean[] a, int offset, VectorMask<$Boxtype$> m) {
1226 super.intoBooleanArray0Template($masktype$.class, a, offset, ($masktype$) m);
1227 }
1228 #end[byte]
1229
1230 @ForceInline
1231 @Override
1232 final
1233 void intoMemorySegment0(MemorySegment ms, long offset, VectorMask<$Boxtype$> m) {
1234 super.intoMemorySegment0Template($masktype$.class, ms, offset, ($masktype$) m);
1235 }
1236
1237 #if[short]
1238 @ForceInline
1239 @Override
1240 final
1241 void intoCharArray0(char[] a, int offset, VectorMask<$Boxtype$> m) {
1242 super.intoCharArray0Template($masktype$.class, a, offset, ($masktype$) m);
1243 }
1244 #end[short]
1245
1246 // End of specialized low-level memory operations.
1247
1248 // ================================================
1249
1250 }