1 /*
   2  * Copyright (c) 2017, 2022, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.  Oracle designates this
   8  * particular file as subject to the "Classpath" exception as provided
   9  * by Oracle in the LICENSE file that accompanied this code.
  10  *
  11  * This code is distributed in the hope that it will be useful, but WITHOUT
  12  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  14  * version 2 for more details (a copy is included in the LICENSE file that
  15  * accompanied this code).
  16  *
  17  * You should have received a copy of the GNU General Public License version
  18  * 2 along with this work; if not, write to the Free Software Foundation,
  19  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  20  *
  21  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  22  * or visit www.oracle.com if you need additional information or have any
  23  * questions.
  24  */
  25 package jdk.incubator.vector;
  26 
  27 import java.util.Arrays;
  28 import java.util.Objects;
  29 import java.util.function.IntUnaryOperator;
  30 
  31 import jdk.incubator.foreign.MemorySegment;
  32 import jdk.internal.vm.annotation.ForceInline;
  33 import jdk.internal.vm.vector.VectorSupport;
  34 
  35 import static jdk.internal.vm.vector.VectorSupport.*;
  36 
  37 import static jdk.incubator.vector.VectorOperators.*;
  38 
  39 #warn This file is preprocessed before being compiled
  40 
  41 @SuppressWarnings("cast")  // warning: redundant cast
  42 final class $vectortype$ extends $abstractvectortype$ {
  43     static final $Type$Species VSPECIES =
  44         ($Type$Species) $Type$Vector.SPECIES_$BITS$;
  45 
  46     static final VectorShape VSHAPE =
  47         VSPECIES.vectorShape();
  48 
  49     static final Class<$vectortype$> VCLASS = $vectortype$.class;
  50 
  51     static final int VSIZE = VSPECIES.vectorBitSize();
  52 
  53     static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM
  54 
  55     static final Class<$Boxtype$> ETYPE = $type$.class; // used by the JVM
  56 
  57     $vectortype$($type$[] v) {
  58         super(v);
  59     }
  60 
  61     // For compatibility as $vectortype$::new,
  62     // stored into species.vectorFactory.
  63     $vectortype$(Object v) {
  64         this(($type$[]) v);
  65     }
  66 
  67     static final $vectortype$ ZERO = new $vectortype$(new $type$[VLENGTH]);
  68     static final $vectortype$ IOTA = new $vectortype$(VSPECIES.iotaArray());
  69 
  70     static {
  71         // Warm up a few species caches.
  72         // If we do this too much we will
  73         // get NPEs from bootstrap circularity.
  74         VSPECIES.dummyVector();
  75         VSPECIES.withLanes(LaneType.BYTE);
  76     }
  77 
  78     // Specialized extractors
  79 
  80     @ForceInline
  81     final @Override
  82     public $Type$Species vspecies() {
  83         // ISSUE:  This should probably be a @Stable
  84         // field inside AbstractVector, rather than
  85         // a megamorphic method.
  86         return VSPECIES;
  87     }
  88 
  89     @ForceInline
  90     @Override
  91     public final Class<$Boxtype$> elementType() { return $type$.class; }
  92 
  93     @ForceInline
  94     @Override
  95     public final int elementSize() { return $Boxtype$.SIZE; }
  96 
  97     @ForceInline
  98     @Override
  99     public final VectorShape shape() { return VSHAPE; }
 100 
 101     @ForceInline
 102     @Override
 103     public final int length() { return VLENGTH; }
 104 
 105     @ForceInline
 106     @Override
 107     public final int bitSize() { return VSIZE; }
 108 
 109     @ForceInline
 110     @Override
 111     public final int byteSize() { return VSIZE / Byte.SIZE; }
 112 
 113     /*package-private*/
 114     @ForceInline
 115     final @Override
 116     $type$[] vec() {
 117         return ($type$[])getPayload();
 118     }
 119 
 120     // Virtualized constructors
 121 
 122     @Override
 123     @ForceInline
 124     public final $vectortype$ broadcast($type$ e) {
 125         return ($vectortype$) super.broadcastTemplate(e);  // specialize
 126     }
 127 
 128 #if[!long]
 129     @Override
 130     @ForceInline
 131     public final $vectortype$ broadcast(long e) {
 132         return ($vectortype$) super.broadcastTemplate(e);  // specialize
 133     }
 134 #end[!long]
 135 
 136     @Override
 137     @ForceInline
 138     $masktype$ maskFromArray(boolean[] bits) {
 139         return new $masktype$(bits);
 140     }
 141 
 142     @Override
 143     @ForceInline
 144     $shuffletype$ iotaShuffle() { return $shuffletype$.IOTA; }
 145 
 146     @ForceInline
 147     $shuffletype$ iotaShuffle(int start, int step, boolean wrap) {
 148       if (wrap) {
 149         return ($shuffletype$)VectorSupport.shuffleIota(ETYPE, $shuffletype$.class, VSPECIES, VLENGTH, start, step, 1,
 150                 (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i*lstep + lstart, l))));
 151       } else {
 152         return ($shuffletype$)VectorSupport.shuffleIota(ETYPE, $shuffletype$.class, VSPECIES, VLENGTH, start, step, 0,
 153                 (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (i*lstep + lstart)));
 154       }
 155     }
 156 
 157     @Override
 158     @ForceInline
 159     $shuffletype$ shuffleFromBytes(byte[] reorder) { return new $shuffletype$(reorder); }
 160 
 161     @Override
 162     @ForceInline
 163     $shuffletype$ shuffleFromArray(int[] indexes, int i) { return new $shuffletype$(indexes, i); }
 164 
 165     @Override
 166     @ForceInline
 167     $shuffletype$ shuffleFromOp(IntUnaryOperator fn) { return new $shuffletype$(fn); }
 168 
 169     // Make a vector of the same species but the given elements:
 170     @ForceInline
 171     final @Override
 172     $vectortype$ vectorFactory($type$[] vec) {
 173         return new $vectortype$(vec);
 174     }
 175 
 176     @ForceInline
 177     final @Override
 178     Byte$bits$Vector asByteVectorRaw() {
 179         return (Byte$bits$Vector) super.asByteVectorRawTemplate();  // specialize
 180     }
 181 
 182     @ForceInline
 183     final @Override
 184     AbstractVector<?> asVectorRaw(LaneType laneType) {
 185         return super.asVectorRawTemplate(laneType);  // specialize
 186     }
 187 
 188     // Unary operator
 189 
 190     @ForceInline
 191     final @Override
 192     $vectortype$ uOp(FUnOp f) {
 193         return ($vectortype$) super.uOpTemplate(f);  // specialize
 194     }
 195 
 196     @ForceInline
 197     final @Override
 198     $vectortype$ uOp(VectorMask<$Boxtype$> m, FUnOp f) {
 199         return ($vectortype$)
 200             super.uOpTemplate(($masktype$)m, f);  // specialize
 201     }
 202 
 203     // Binary operator
 204 
 205     @ForceInline
 206     final @Override
 207     $vectortype$ bOp(Vector<$Boxtype$> v, FBinOp f) {
 208         return ($vectortype$) super.bOpTemplate(($vectortype$)v, f);  // specialize
 209     }
 210 
 211     @ForceInline
 212     final @Override
 213     $vectortype$ bOp(Vector<$Boxtype$> v,
 214                      VectorMask<$Boxtype$> m, FBinOp f) {
 215         return ($vectortype$)
 216             super.bOpTemplate(($vectortype$)v, ($masktype$)m,
 217                               f);  // specialize
 218     }
 219 
 220     // Ternary operator
 221 
 222     @ForceInline
 223     final @Override
 224     $vectortype$ tOp(Vector<$Boxtype$> v1, Vector<$Boxtype$> v2, FTriOp f) {
 225         return ($vectortype$)
 226             super.tOpTemplate(($vectortype$)v1, ($vectortype$)v2,
 227                               f);  // specialize
 228     }
 229 
 230     @ForceInline
 231     final @Override
 232     $vectortype$ tOp(Vector<$Boxtype$> v1, Vector<$Boxtype$> v2,
 233                      VectorMask<$Boxtype$> m, FTriOp f) {
 234         return ($vectortype$)
 235             super.tOpTemplate(($vectortype$)v1, ($vectortype$)v2,
 236                               ($masktype$)m, f);  // specialize
 237     }
 238 
 239     @ForceInline
 240     final @Override
 241     $type$ rOp($type$ v, VectorMask<$Boxtype$> m, FBinOp f) {
 242         return super.rOpTemplate(v, m, f);  // specialize
 243     }
 244 
 245     @Override
 246     @ForceInline
 247     public final <F>
 248     Vector<F> convertShape(VectorOperators.Conversion<$Boxtype$,F> conv,
 249                            VectorSpecies<F> rsp, int part) {
 250         return super.convertShapeTemplate(conv, rsp, part);  // specialize
 251     }
 252 
 253     @Override
 254     @ForceInline
 255     public final <F>
 256     Vector<F> reinterpretShape(VectorSpecies<F> toSpecies, int part) {
 257         return super.reinterpretShapeTemplate(toSpecies, part);  // specialize
 258     }
 259 
 260     // Specialized algebraic operations:
 261 
 262     // The following definition forces a specialized version of this
 263     // crucial method into the v-table of this class.  A call to add()
 264     // will inline to a call to lanewise(ADD,), at which point the JIT
 265     // intrinsic will have the opcode of ADD, plus all the metadata
 266     // for this particular class, enabling it to generate precise
 267     // code.
 268     //
 269     // There is probably no benefit to the JIT to specialize the
 270     // masked or broadcast versions of the lanewise method.
 271 
 272     @Override
 273     @ForceInline
 274     public $vectortype$ lanewise(Unary op) {
 275         return ($vectortype$) super.lanewiseTemplate(op);  // specialize
 276     }
 277 
 278     @Override
 279     @ForceInline
 280     public $vectortype$ lanewise(Unary op, VectorMask<$Boxtype$> m) {
 281         return ($vectortype$) super.lanewiseTemplate(op, $masktype$.class, ($masktype$) m);  // specialize
 282     }
 283 
 284     @Override
 285     @ForceInline
 286     public $vectortype$ lanewise(Binary op, Vector<$Boxtype$> v) {
 287         return ($vectortype$) super.lanewiseTemplate(op, v);  // specialize
 288     }
 289 
 290     @Override
 291     @ForceInline
 292     public $vectortype$ lanewise(Binary op, Vector<$Boxtype$> v, VectorMask<$Boxtype$> m) {
 293         return ($vectortype$) super.lanewiseTemplate(op, $masktype$.class, v, ($masktype$) m);  // specialize
 294     }
 295 
 296 #if[!FP]
 297     /*package-private*/
 298     @Override
 299     @ForceInline $vectortype$
 300     lanewiseShift(VectorOperators.Binary op, int e) {
 301         return ($vectortype$) super.lanewiseShiftTemplate(op, e);  // specialize
 302     }
 303 
 304     /*package-private*/
 305     @Override
 306     @ForceInline $vectortype$
 307     lanewiseShift(VectorOperators.Binary op, int e, VectorMask<$Boxtype$> m) {
 308         return ($vectortype$) super.lanewiseShiftTemplate(op, $masktype$.class, e, ($masktype$) m);  // specialize
 309     }
 310 #end[!FP]
 311 
 312     /*package-private*/
 313     @Override
 314     @ForceInline
 315     public final
 316     $vectortype$
 317     lanewise(Ternary op, Vector<$Boxtype$> v1, Vector<$Boxtype$> v2) {
 318         return ($vectortype$) super.lanewiseTemplate(op, v1, v2);  // specialize
 319     }
 320 
 321     @Override
 322     @ForceInline
 323     public final
 324     $vectortype$
 325     lanewise(Ternary op, Vector<$Boxtype$> v1, Vector<$Boxtype$> v2, VectorMask<$Boxtype$> m) {
 326         return ($vectortype$) super.lanewiseTemplate(op, $masktype$.class, v1, v2, ($masktype$) m);  // specialize
 327     }
 328 
 329     @Override
 330     @ForceInline
 331     public final
 332     $vectortype$ addIndex(int scale) {
 333         return ($vectortype$) super.addIndexTemplate(scale);  // specialize
 334     }
 335 
 336     // Type specific horizontal reductions
 337 
 338     @Override
 339     @ForceInline
 340     public final $type$ reduceLanes(VectorOperators.Associative op) {
 341         return super.reduceLanesTemplate(op);  // specialized
 342     }
 343 
 344     @Override
 345     @ForceInline
 346     public final $type$ reduceLanes(VectorOperators.Associative op,
 347                                     VectorMask<$Boxtype$> m) {
 348         return super.reduceLanesTemplate(op, $masktype$.class, ($masktype$) m);  // specialized
 349     }
 350 
 351     @Override
 352     @ForceInline
 353     public final long reduceLanesToLong(VectorOperators.Associative op) {
 354         return (long) super.reduceLanesTemplate(op);  // specialized
 355     }
 356 
 357     @Override
 358     @ForceInline
 359     public final long reduceLanesToLong(VectorOperators.Associative op,
 360                                         VectorMask<$Boxtype$> m) {
 361         return (long) super.reduceLanesTemplate(op, $masktype$.class, ($masktype$) m);  // specialized
 362     }
 363 
 364     @ForceInline
 365     public VectorShuffle<$Boxtype$> toShuffle() {
 366         return super.toShuffleTemplate($shuffletype$.class); // specialize
 367     }
 368 
 369     // Specialized unary testing
 370 
 371     @Override
 372     @ForceInline
 373     public final $masktype$ test(Test op) {
 374         return super.testTemplate($masktype$.class, op);  // specialize
 375     }
 376 
 377     @Override
 378     @ForceInline
 379     public final $masktype$ test(Test op, VectorMask<$Boxtype$> m) {
 380         return super.testTemplate($masktype$.class, op, ($masktype$) m);  // specialize
 381     }
 382 
 383     // Specialized comparisons
 384 
 385     @Override
 386     @ForceInline
 387     public final $masktype$ compare(Comparison op, Vector<$Boxtype$> v) {
 388         return super.compareTemplate($masktype$.class, op, v);  // specialize
 389     }
 390 
 391     @Override
 392     @ForceInline
 393     public final $masktype$ compare(Comparison op, $type$ s) {
 394         return super.compareTemplate($masktype$.class, op, s);  // specialize
 395     }
 396 
 397 #if[!long]
 398     @Override
 399     @ForceInline
 400     public final $masktype$ compare(Comparison op, long s) {
 401         return super.compareTemplate($masktype$.class, op, s);  // specialize
 402     }
 403 #end[!long]
 404 
 405     @Override
 406     @ForceInline
 407     public final $masktype$ compare(Comparison op, Vector<$Boxtype$> v, VectorMask<$Boxtype$> m) {
 408         return super.compareTemplate($masktype$.class, op, v, ($masktype$) m);
 409     }
 410 
 411 
 412     @Override
 413     @ForceInline
 414     public $vectortype$ blend(Vector<$Boxtype$> v, VectorMask<$Boxtype$> m) {
 415         return ($vectortype$)
 416             super.blendTemplate($masktype$.class,
 417                                 ($vectortype$) v,
 418                                 ($masktype$) m);  // specialize
 419     }
 420 
 421     @Override
 422     @ForceInline
 423     public $vectortype$ slice(int origin, Vector<$Boxtype$> v) {
 424         return ($vectortype$) super.sliceTemplate(origin, v);  // specialize
 425     }
 426 
 427     @Override
 428     @ForceInline
 429     public $vectortype$ slice(int origin) {
 430         return ($vectortype$) super.sliceTemplate(origin);  // specialize
 431     }
 432 
 433     @Override
 434     @ForceInline
 435     public $vectortype$ unslice(int origin, Vector<$Boxtype$> w, int part) {
 436         return ($vectortype$) super.unsliceTemplate(origin, w, part);  // specialize
 437     }
 438 
 439     @Override
 440     @ForceInline
 441     public $vectortype$ unslice(int origin, Vector<$Boxtype$> w, int part, VectorMask<$Boxtype$> m) {
 442         return ($vectortype$)
 443             super.unsliceTemplate($masktype$.class,
 444                                   origin, w, part,
 445                                   ($masktype$) m);  // specialize
 446     }
 447 
 448     @Override
 449     @ForceInline
 450     public $vectortype$ unslice(int origin) {
 451         return ($vectortype$) super.unsliceTemplate(origin);  // specialize
 452     }
 453 
 454     @Override
 455     @ForceInline
 456     public $vectortype$ rearrange(VectorShuffle<$Boxtype$> s) {
 457         return ($vectortype$)
 458             super.rearrangeTemplate($shuffletype$.class,
 459                                     ($shuffletype$) s);  // specialize
 460     }
 461 
 462     @Override
 463     @ForceInline
 464     public $vectortype$ rearrange(VectorShuffle<$Boxtype$> shuffle,
 465                                   VectorMask<$Boxtype$> m) {
 466         return ($vectortype$)
 467             super.rearrangeTemplate($shuffletype$.class,
 468                                     $masktype$.class,
 469                                     ($shuffletype$) shuffle,
 470                                     ($masktype$) m);  // specialize
 471     }
 472 
 473     @Override
 474     @ForceInline
 475     public $vectortype$ rearrange(VectorShuffle<$Boxtype$> s,
 476                                   Vector<$Boxtype$> v) {
 477         return ($vectortype$)
 478             super.rearrangeTemplate($shuffletype$.class,
 479                                     ($shuffletype$) s,
 480                                     ($vectortype$) v);  // specialize
 481     }
 482 
 483     @Override
 484     @ForceInline
 485     public $vectortype$ compress(VectorMask<$Boxtype$> m) {
 486         return ($vectortype$)
 487             super.compressTemplate($masktype$.class,
 488                                    ($masktype$) m);  // specialize
 489     }
 490 
 491     @Override
 492     @ForceInline
 493     public $vectortype$ expand(VectorMask<$Boxtype$> m) {
 494         return ($vectortype$)
 495             super.expandTemplate($masktype$.class,
 496                                    ($masktype$) m);  // specialize
 497     }
 498 
 499     @Override
 500     @ForceInline
 501     public $vectortype$ selectFrom(Vector<$Boxtype$> v) {
 502         return ($vectortype$)
 503             super.selectFromTemplate(($vectortype$) v);  // specialize
 504     }
 505 
 506     @Override
 507     @ForceInline
 508     public $vectortype$ selectFrom(Vector<$Boxtype$> v,
 509                                    VectorMask<$Boxtype$> m) {
 510         return ($vectortype$)
 511             super.selectFromTemplate(($vectortype$) v,
 512                                      ($masktype$) m);  // specialize
 513     }
 514 
 515 
 516 #if[FP]
 517     @ForceInline
 518     @Override
 519     public $type$ lane(int i) {
 520 #if[!Max]
 521         $bitstype$ bits;
 522         switch(i) {
 523             case 0: bits = laneHelper(0); break;
 524 #if[!1L]
 525             case 1: bits = laneHelper(1); break;
 526 #if[!2L]
 527             case 2: bits = laneHelper(2); break;
 528             case 3: bits = laneHelper(3); break;
 529 #if[!4L]
 530             case 4: bits = laneHelper(4); break;
 531             case 5: bits = laneHelper(5); break;
 532             case 6: bits = laneHelper(6); break;
 533             case 7: bits = laneHelper(7); break;
 534 #if[!8L]
 535             case 8: bits = laneHelper(8); break;
 536             case 9: bits = laneHelper(9); break;
 537             case 10: bits = laneHelper(10); break;
 538             case 11: bits = laneHelper(11); break;
 539             case 12: bits = laneHelper(12); break;
 540             case 13: bits = laneHelper(13); break;
 541             case 14: bits = laneHelper(14); break;
 542             case 15: bits = laneHelper(15); break;
 543 #end[!8L]
 544 #end[!4L]
 545 #end[!2L]
 546 #end[!1L]
 547             default: throw new IllegalArgumentException("Index " + i + " must be zero or positive, and less than " + VLENGTH);
 548         }
 549 #else[!Max]
 550         if (i < 0 || i >= VLENGTH) {
 551             throw new IllegalArgumentException("Index " + i + " must be zero or positive, and less than " + VLENGTH);
 552         }
 553         $bitstype$ bits = laneHelper(i);
 554 #end[!Max]
 555         return $Type$.$bitstype$BitsTo$Fptype$(bits);
 556     }
 557 
 558     public $bitstype$ laneHelper(int i) {
 559         return ($bitstype$) VectorSupport.extract(
 560                      VCLASS, ETYPE, VLENGTH,
 561                      this, i,
 562                      (vec, ix) -> {
 563                      $type$[] vecarr = vec.vec();
 564                      return (long)$Type$.$type$To$Bitstype$Bits(vecarr[ix]);
 565                      });
 566     }
 567 
 568     @ForceInline
 569     @Override
 570     public $vectortype$ withLane(int i, $type$ e) {
 571 #if[!Max]
 572         switch(i) {
 573             case 0: return withLaneHelper(0, e);
 574 #if[!1L]
 575             case 1: return withLaneHelper(1, e);
 576 #if[!2L]
 577             case 2: return withLaneHelper(2, e);
 578             case 3: return withLaneHelper(3, e);
 579 #if[!4L]
 580             case 4: return withLaneHelper(4, e);
 581             case 5: return withLaneHelper(5, e);
 582             case 6: return withLaneHelper(6, e);
 583             case 7: return withLaneHelper(7, e);
 584 #if[!8L]
 585             case 8: return withLaneHelper(8, e);
 586             case 9: return withLaneHelper(9, e);
 587             case 10: return withLaneHelper(10, e);
 588             case 11: return withLaneHelper(11, e);
 589             case 12: return withLaneHelper(12, e);
 590             case 13: return withLaneHelper(13, e);
 591             case 14: return withLaneHelper(14, e);
 592             case 15: return withLaneHelper(15, e);
 593 #end[!8L]
 594 #end[!4L]
 595 #end[!2L]
 596 #end[!1L]
 597             default: throw new IllegalArgumentException("Index " + i + " must be zero or positive, and less than " + VLENGTH);
 598         }
 599 #else[!Max]
 600         if (i < 0 || i >= VLENGTH) {
 601             throw new IllegalArgumentException("Index " + i + " must be zero or positive, and less than " + VLENGTH);
 602         }
 603         return withLaneHelper(i, e);
 604 #end[!Max]
 605     }
 606 
 607     public $vectortype$ withLaneHelper(int i, $type$ e) {
 608         return VectorSupport.insert(
 609                                 VCLASS, ETYPE, VLENGTH,
 610                                 this, i, (long)$Type$.$type$To$Bitstype$Bits(e),
 611                                 (v, ix, bits) -> {
 612                                     $type$[] res = v.vec().clone();
 613                                     res[ix] = $Type$.$bitstype$BitsTo$Type$(($bitstype$)bits);
 614                                     return v.vectorFactory(res);
 615                                 });
 616     }
 617 #else[FP]
 618     @ForceInline
 619     @Override
 620     public $type$ lane(int i) {
 621 #if[!Max]
 622         switch(i) {
 623             case 0: return laneHelper(0);
 624 #if[!1L]
 625             case 1: return laneHelper(1);
 626 #if[!2L]
 627             case 2: return laneHelper(2);
 628             case 3: return laneHelper(3);
 629 #if[!4L]
 630             case 4: return laneHelper(4);
 631             case 5: return laneHelper(5);
 632             case 6: return laneHelper(6);
 633             case 7: return laneHelper(7);
 634 #if[!8L]
 635             case 8: return laneHelper(8);
 636             case 9: return laneHelper(9);
 637             case 10: return laneHelper(10);
 638             case 11: return laneHelper(11);
 639             case 12: return laneHelper(12);
 640             case 13: return laneHelper(13);
 641             case 14: return laneHelper(14);
 642             case 15: return laneHelper(15);
 643 #if[!16L]
 644             case 16: return laneHelper(16);
 645             case 17: return laneHelper(17);
 646             case 18: return laneHelper(18);
 647             case 19: return laneHelper(19);
 648             case 20: return laneHelper(20);
 649             case 21: return laneHelper(21);
 650             case 22: return laneHelper(22);
 651             case 23: return laneHelper(23);
 652             case 24: return laneHelper(24);
 653             case 25: return laneHelper(25);
 654             case 26: return laneHelper(26);
 655             case 27: return laneHelper(27);
 656             case 28: return laneHelper(28);
 657             case 29: return laneHelper(29);
 658             case 30: return laneHelper(30);
 659             case 31: return laneHelper(31);
 660 #if[!32L]
 661             case 32: return laneHelper(32);
 662             case 33: return laneHelper(33);
 663             case 34: return laneHelper(34);
 664             case 35: return laneHelper(35);
 665             case 36: return laneHelper(36);
 666             case 37: return laneHelper(37);
 667             case 38: return laneHelper(38);
 668             case 39: return laneHelper(39);
 669             case 40: return laneHelper(40);
 670             case 41: return laneHelper(41);
 671             case 42: return laneHelper(42);
 672             case 43: return laneHelper(43);
 673             case 44: return laneHelper(44);
 674             case 45: return laneHelper(45);
 675             case 46: return laneHelper(46);
 676             case 47: return laneHelper(47);
 677             case 48: return laneHelper(48);
 678             case 49: return laneHelper(49);
 679             case 50: return laneHelper(50);
 680             case 51: return laneHelper(51);
 681             case 52: return laneHelper(52);
 682             case 53: return laneHelper(53);
 683             case 54: return laneHelper(54);
 684             case 55: return laneHelper(55);
 685             case 56: return laneHelper(56);
 686             case 57: return laneHelper(57);
 687             case 58: return laneHelper(58);
 688             case 59: return laneHelper(59);
 689             case 60: return laneHelper(60);
 690             case 61: return laneHelper(61);
 691             case 62: return laneHelper(62);
 692             case 63: return laneHelper(63);
 693 #end[!32L]
 694 #end[!16L]
 695 #end[!8L]
 696 #end[!4L]
 697 #end[!2L]
 698 #end[!1L]
 699             default: throw new IllegalArgumentException("Index " + i + " must be zero or positive, and less than " + VLENGTH);
 700         }
 701 #else[!Max]
 702         if (i < 0 || i >= VLENGTH) {
 703             throw new IllegalArgumentException("Index " + i + " must be zero or positive, and less than " + VLENGTH);
 704         }
 705         return laneHelper(i);
 706 #end[!Max]
 707     }
 708 
 709     public $type$ laneHelper(int i) {
 710         return ($type$) VectorSupport.extract(
 711                                 VCLASS, ETYPE, VLENGTH,
 712                                 this, i,
 713                                 (vec, ix) -> {
 714                                     $type$[] vecarr = vec.vec();
 715                                     return (long)vecarr[ix];
 716                                 });
 717     }
 718 
 719     @ForceInline
 720     @Override
 721     public $vectortype$ withLane(int i, $type$ e) {
 722 #if[!Max]
 723         switch (i) {
 724             case 0: return withLaneHelper(0, e);
 725 #if[!1L]
 726             case 1: return withLaneHelper(1, e);
 727 #if[!2L]
 728             case 2: return withLaneHelper(2, e);
 729             case 3: return withLaneHelper(3, e);
 730 #if[!4L]
 731             case 4: return withLaneHelper(4, e);
 732             case 5: return withLaneHelper(5, e);
 733             case 6: return withLaneHelper(6, e);
 734             case 7: return withLaneHelper(7, e);
 735 #if[!8L]
 736             case 8: return withLaneHelper(8, e);
 737             case 9: return withLaneHelper(9, e);
 738             case 10: return withLaneHelper(10, e);
 739             case 11: return withLaneHelper(11, e);
 740             case 12: return withLaneHelper(12, e);
 741             case 13: return withLaneHelper(13, e);
 742             case 14: return withLaneHelper(14, e);
 743             case 15: return withLaneHelper(15, e);
 744 #if[!16L]
 745             case 16: return withLaneHelper(16, e);
 746             case 17: return withLaneHelper(17, e);
 747             case 18: return withLaneHelper(18, e);
 748             case 19: return withLaneHelper(19, e);
 749             case 20: return withLaneHelper(20, e);
 750             case 21: return withLaneHelper(21, e);
 751             case 22: return withLaneHelper(22, e);
 752             case 23: return withLaneHelper(23, e);
 753             case 24: return withLaneHelper(24, e);
 754             case 25: return withLaneHelper(25, e);
 755             case 26: return withLaneHelper(26, e);
 756             case 27: return withLaneHelper(27, e);
 757             case 28: return withLaneHelper(28, e);
 758             case 29: return withLaneHelper(29, e);
 759             case 30: return withLaneHelper(30, e);
 760             case 31: return withLaneHelper(31, e);
 761 #if[!32L]
 762             case 32: return withLaneHelper(32, e);
 763             case 33: return withLaneHelper(33, e);
 764             case 34: return withLaneHelper(34, e);
 765             case 35: return withLaneHelper(35, e);
 766             case 36: return withLaneHelper(36, e);
 767             case 37: return withLaneHelper(37, e);
 768             case 38: return withLaneHelper(38, e);
 769             case 39: return withLaneHelper(39, e);
 770             case 40: return withLaneHelper(40, e);
 771             case 41: return withLaneHelper(41, e);
 772             case 42: return withLaneHelper(42, e);
 773             case 43: return withLaneHelper(43, e);
 774             case 44: return withLaneHelper(44, e);
 775             case 45: return withLaneHelper(45, e);
 776             case 46: return withLaneHelper(46, e);
 777             case 47: return withLaneHelper(47, e);
 778             case 48: return withLaneHelper(48, e);
 779             case 49: return withLaneHelper(49, e);
 780             case 50: return withLaneHelper(50, e);
 781             case 51: return withLaneHelper(51, e);
 782             case 52: return withLaneHelper(52, e);
 783             case 53: return withLaneHelper(53, e);
 784             case 54: return withLaneHelper(54, e);
 785             case 55: return withLaneHelper(55, e);
 786             case 56: return withLaneHelper(56, e);
 787             case 57: return withLaneHelper(57, e);
 788             case 58: return withLaneHelper(58, e);
 789             case 59: return withLaneHelper(59, e);
 790             case 60: return withLaneHelper(60, e);
 791             case 61: return withLaneHelper(61, e);
 792             case 62: return withLaneHelper(62, e);
 793             case 63: return withLaneHelper(63, e);
 794 #end[!32L]
 795 #end[!16L]
 796 #end[!8L]
 797 #end[!4L]
 798 #end[!2L]
 799 #end[!1L]
 800             default: throw new IllegalArgumentException("Index " + i + " must be zero or positive, and less than " + VLENGTH);
 801         }
 802 #else[!Max]
 803         if (i < 0 || i >= VLENGTH) {
 804             throw new IllegalArgumentException("Index " + i + " must be zero or positive, and less than " + VLENGTH);
 805         }
 806         return withLaneHelper(i, e);
 807 #end[!Max]
 808     }
 809 
 810     public $vectortype$ withLaneHelper(int i, $type$ e) {
 811         return VectorSupport.insert(
 812                                 VCLASS, ETYPE, VLENGTH,
 813                                 this, i, (long)e,
 814                                 (v, ix, bits) -> {
 815                                     $type$[] res = v.vec().clone();
 816                                     res[ix] = ($type$)bits;
 817                                     return v.vectorFactory(res);
 818                                 });
 819     }
 820 #end[FP]
 821 
 822     // Mask
 823 
 824     static final class $masktype$ extends AbstractMask<$Boxtype$> {
 825         static final int VLENGTH = VSPECIES.laneCount();    // used by the JVM
 826         static final Class<$Boxtype$> ETYPE = $type$.class; // used by the JVM
 827 
 828         $masktype$(boolean[] bits) {
 829             this(bits, 0);
 830         }
 831 
 832         $masktype$(boolean[] bits, int offset) {
 833             super(prepare(bits, offset));
 834         }
 835 
 836         $masktype$(boolean val) {
 837             super(prepare(val));
 838         }
 839 
 840         private static boolean[] prepare(boolean[] bits, int offset) {
 841             boolean[] newBits = new boolean[VSPECIES.laneCount()];
 842             for (int i = 0; i < newBits.length; i++) {
 843                 newBits[i] = bits[offset + i];
 844             }
 845             return newBits;
 846         }
 847 
 848         private static boolean[] prepare(boolean val) {
 849             boolean[] bits = new boolean[VSPECIES.laneCount()];
 850             Arrays.fill(bits, val);
 851             return bits;
 852         }
 853 
 854         @ForceInline
 855         final @Override
 856         public $Type$Species vspecies() {
 857             // ISSUE:  This should probably be a @Stable
 858             // field inside AbstractMask, rather than
 859             // a megamorphic method.
 860             return VSPECIES;
 861         }
 862 
 863         @ForceInline
 864         boolean[] getBits() {
 865             return (boolean[])getPayload();
 866         }
 867 
 868         @Override
 869         $masktype$ uOp(MUnOp f) {
 870             boolean[] res = new boolean[vspecies().laneCount()];
 871             boolean[] bits = getBits();
 872             for (int i = 0; i < res.length; i++) {
 873                 res[i] = f.apply(i, bits[i]);
 874             }
 875             return new $masktype$(res);
 876         }
 877 
 878         @Override
 879         $masktype$ bOp(VectorMask<$Boxtype$> m, MBinOp f) {
 880             boolean[] res = new boolean[vspecies().laneCount()];
 881             boolean[] bits = getBits();
 882             boolean[] mbits = (($masktype$)m).getBits();
 883             for (int i = 0; i < res.length; i++) {
 884                 res[i] = f.apply(i, bits[i], mbits[i]);
 885             }
 886             return new $masktype$(res);
 887         }
 888 
 889         @ForceInline
 890         @Override
 891         public final
 892         $vectortype$ toVector() {
 893             return ($vectortype$) super.toVectorTemplate();  // specialize
 894         }
 895 
 896         /**
 897          * Helper function for lane-wise mask conversions.
 898          * This function kicks in after intrinsic failure.
 899          */
 900         @ForceInline
 901         private final <E>
 902         VectorMask<E> defaultMaskCast(AbstractSpecies<E> dsp) {
 903             if (length() != dsp.laneCount())
 904                 throw new IllegalArgumentException("VectorMask length and species length differ");
 905             boolean[] maskArray = toArray();
 906             return  dsp.maskFactory(maskArray).check(dsp);
 907         }
 908 
 909         @Override
 910         @ForceInline
 911         public <E> VectorMask<E> cast(VectorSpecies<E> dsp) {
 912             AbstractSpecies<E> species = (AbstractSpecies<E>) dsp;
 913             if (length() != species.laneCount())
 914                 throw new IllegalArgumentException("VectorMask length and species length differ");
 915 
 916             return VectorSupport.convert(VectorSupport.VECTOR_OP_CAST,
 917                 this.getClass(), ETYPE, VLENGTH,
 918                 species.maskType(), species.elementType(), VLENGTH,
 919                 this, species,
 920                 (m, s) -> s.maskFactory(m.toArray()).check(s));
 921         }
 922 
 923         @Override
 924         @ForceInline
 925         public $masktype$ eq(VectorMask<$Boxtype$> mask) {
 926             Objects.requireNonNull(mask);
 927             $masktype$ m = ($masktype$)mask;
 928             return xor(m.not());
 929         }
 930 
 931         // Unary operations
 932 
 933         @Override
 934         @ForceInline
 935         public $masktype$ not() {
 936             return xor(maskAll(true));
 937         }
 938 
 939         @Override
 940         @ForceInline
 941         public $masktype$ compress() {
 942             return ($masktype$)VectorSupport.comExpOp(VectorSupport.VECTOR_OP_MASK_COMPRESS,
 943                 $vectortype$.class, $masktype$.class, ETYPE, VLENGTH, null, this,
 944                 (v1, m1) -> VSPECIES.iota().compare(VectorOperators.LT, m1.trueCount()));
 945         }
 946 
 947 
 948         // Binary operations
 949 
 950         @Override
 951         @ForceInline
 952         public $masktype$ and(VectorMask<$Boxtype$> mask) {
 953             Objects.requireNonNull(mask);
 954             $masktype$ m = ($masktype$)mask;
 955             return VectorSupport.binaryOp(VECTOR_OP_AND, $masktype$.class, null, $bitstype$.class, VLENGTH,
 956                                           this, m, null,
 957                                           (m1, m2, vm) -> m1.bOp(m2, (i, a, b) -> a & b));
 958         }
 959 
 960         @Override
 961         @ForceInline
 962         public $masktype$ or(VectorMask<$Boxtype$> mask) {
 963             Objects.requireNonNull(mask);
 964             $masktype$ m = ($masktype$)mask;
 965             return VectorSupport.binaryOp(VECTOR_OP_OR, $masktype$.class, null, $bitstype$.class, VLENGTH,
 966                                           this, m, null,
 967                                           (m1, m2, vm) -> m1.bOp(m2, (i, a, b) -> a | b));
 968         }
 969 
 970         @ForceInline
 971         /* package-private */
 972         $masktype$ xor(VectorMask<$Boxtype$> mask) {
 973             Objects.requireNonNull(mask);
 974             $masktype$ m = ($masktype$)mask;
 975             return VectorSupport.binaryOp(VECTOR_OP_XOR, $masktype$.class, null, $bitstype$.class, VLENGTH,
 976                                           this, m, null,
 977                                           (m1, m2, vm) -> m1.bOp(m2, (i, a, b) -> a ^ b));
 978         }
 979 
 980         // Mask Query operations
 981 
 982         @Override
 983         @ForceInline
 984         public int trueCount() {
 985             return (int) VectorSupport.maskReductionCoerced(VECTOR_OP_MASK_TRUECOUNT, $masktype$.class, $bitstype$.class, VLENGTH, this,
 986                                                       (m) -> trueCountHelper(m.getBits()));
 987         }
 988 
 989         @Override
 990         @ForceInline
 991         public int firstTrue() {
 992             return (int) VectorSupport.maskReductionCoerced(VECTOR_OP_MASK_FIRSTTRUE, $masktype$.class, $bitstype$.class, VLENGTH, this,
 993                                                       (m) -> firstTrueHelper(m.getBits()));
 994         }
 995 
 996         @Override
 997         @ForceInline
 998         public int lastTrue() {
 999             return (int) VectorSupport.maskReductionCoerced(VECTOR_OP_MASK_LASTTRUE, $masktype$.class, $bitstype$.class, VLENGTH, this,
1000                                                       (m) -> lastTrueHelper(m.getBits()));
1001         }
1002 
1003         @Override
1004         @ForceInline
1005         public long toLong() {
1006             if (length() > Long.SIZE) {
1007                 throw new UnsupportedOperationException("too many lanes for one long");
1008             }
1009             return VectorSupport.maskReductionCoerced(VECTOR_OP_MASK_TOLONG, $masktype$.class, $bitstype$.class, VLENGTH, this,
1010                                                       (m) -> toLongHelper(m.getBits()));
1011         }
1012 
1013         // Reductions
1014 
1015         @Override
1016         @ForceInline
1017         public boolean anyTrue() {
1018             return VectorSupport.test(BT_ne, $masktype$.class, $bitstype$.class, VLENGTH,
1019                                          this, vspecies().maskAll(true),
1020                                          (m, __) -> anyTrueHelper((($masktype$)m).getBits()));
1021         }
1022 
1023         @Override
1024         @ForceInline
1025         public boolean allTrue() {
1026             return VectorSupport.test(BT_overflow, $masktype$.class, $bitstype$.class, VLENGTH,
1027                                          this, vspecies().maskAll(true),
1028                                          (m, __) -> allTrueHelper((($masktype$)m).getBits()));
1029         }
1030 
1031         @ForceInline
1032         /*package-private*/
1033         static $masktype$ maskAll(boolean bit) {
1034             return VectorSupport.fromBitsCoerced($masktype$.class, $bitstype$.class, VLENGTH,
1035                                                  (bit ? -1 : 0), MODE_BROADCAST, null,
1036                                                  (v, __) -> (v != 0 ? TRUE_MASK : FALSE_MASK));
1037         }
1038         private static final $masktype$  TRUE_MASK = new $masktype$(true);
1039         private static final $masktype$ FALSE_MASK = new $masktype$(false);
1040 
1041 #if[intAndMax]
1042 
1043         static boolean[] maskLowerHalf() {
1044             boolean[] a = new boolean[VLENGTH];
1045             int len = a.length >> 1;
1046             for (int i = 0; i < len; i++) {
1047                 a[i] = true;
1048             }
1049             return a;
1050         }
1051 
1052 #end[intAndMax]
1053 #if[intAndMax]
1054         static final IntMaxMask LOWER_HALF_TRUE_MASK = new IntMaxMask(maskLowerHalf());
1055 #end[intAndMax]
1056     }
1057 
1058     // Shuffle
1059 
1060     static final class $shuffletype$ extends AbstractShuffle<$Boxtype$> {
1061         static final int VLENGTH = VSPECIES.laneCount();    // used by the JVM
1062         static final Class<$Boxtype$> ETYPE = $type$.class; // used by the JVM
1063 
1064         $shuffletype$(byte[] reorder) {
1065             super(VLENGTH, reorder);
1066         }
1067 
1068         public $shuffletype$(int[] reorder) {
1069             super(VLENGTH, reorder);
1070         }
1071 
1072         public $shuffletype$(int[] reorder, int i) {
1073             super(VLENGTH, reorder, i);
1074         }
1075 
1076         public $shuffletype$(IntUnaryOperator fn) {
1077             super(VLENGTH, fn);
1078         }
1079 
1080         @Override
1081         public $Type$Species vspecies() {
1082             return VSPECIES;
1083         }
1084 
1085         static {
1086             // There must be enough bits in the shuffle lanes to encode
1087             // VLENGTH valid indexes and VLENGTH exceptional ones.
1088             assert(VLENGTH < Byte.MAX_VALUE);
1089             assert(Byte.MIN_VALUE <= -VLENGTH);
1090         }
1091         static final $shuffletype$ IOTA = new $shuffletype$(IDENTITY);
1092 
1093         @Override
1094         @ForceInline
1095         public $vectortype$ toVector() {
1096             return VectorSupport.shuffleToVector(VCLASS, ETYPE, $shuffletype$.class, this, VLENGTH,
1097                                                     (s) -> (($vectortype$)(((AbstractShuffle<$Boxtype$>)(s)).toVectorTemplate())));
1098         }
1099 
1100         @Override
1101         @ForceInline
1102         public <F> VectorShuffle<F> cast(VectorSpecies<F> s) {
1103             AbstractSpecies<F> species = (AbstractSpecies<F>) s;
1104             if (length() != species.laneCount())
1105                 throw new IllegalArgumentException("VectorShuffle length and species length differ");
1106             int[] shuffleArray = toArray();
1107             return s.shuffleFromArray(shuffleArray, 0).check(s);
1108         }
1109 
1110         @ForceInline
1111         @Override
1112         public $shuffletype$ rearrange(VectorShuffle<$Boxtype$> shuffle) {
1113             $shuffletype$ s = ($shuffletype$) shuffle;
1114             byte[] reorder1 = reorder();
1115             byte[] reorder2 = s.reorder();
1116             byte[] r = new byte[reorder1.length];
1117             for (int i = 0; i < reorder1.length; i++) {
1118                 int ssi = reorder2[i];
1119                 r[i] = reorder1[ssi];  // throws on exceptional index
1120             }
1121             return new $shuffletype$(r);
1122         }
1123     }
1124 
1125     // ================================================
1126 
1127     // Specialized low-level memory operations.
1128 
1129     @ForceInline
1130     @Override
1131     final
1132     $abstractvectortype$ fromArray0($type$[] a, int offset) {
1133         return super.fromArray0Template(a, offset);  // specialize
1134     }
1135 
1136     @ForceInline
1137     @Override
1138     final
1139     $abstractvectortype$ fromArray0($type$[] a, int offset, VectorMask<$Boxtype$> m) {
1140         return super.fromArray0Template($masktype$.class, a, offset, ($masktype$) m);  // specialize
1141     }
1142 
1143 #if[!byteOrShort]
1144     @ForceInline
1145     @Override
1146     final
1147     $abstractvectortype$ fromArray0($type$[] a, int offset, int[] indexMap, int mapOffset, VectorMask<$Boxtype$> m) {
1148         return super.fromArray0Template($masktype$.class, a, offset, indexMap, mapOffset, ($masktype$) m);
1149     }
1150 #end[!byteOrShort]
1151 
1152 #if[short]
1153     @ForceInline
1154     @Override
1155     final
1156     $abstractvectortype$ fromCharArray0(char[] a, int offset) {
1157         return super.fromCharArray0Template(a, offset);  // specialize
1158     }
1159 
1160     @ForceInline
1161     @Override
1162     final
1163     $abstractvectortype$ fromCharArray0(char[] a, int offset, VectorMask<$Boxtype$> m) {
1164         return super.fromCharArray0Template($masktype$.class, a, offset, ($masktype$) m);  // specialize
1165     }
1166 #end[short]
1167 
1168 #if[byte]
1169     @ForceInline
1170     @Override
1171     final
1172     $abstractvectortype$ fromBooleanArray0(boolean[] a, int offset) {
1173         return super.fromBooleanArray0Template(a, offset);  // specialize
1174     }
1175 
1176     @ForceInline
1177     @Override
1178     final
1179     $abstractvectortype$ fromBooleanArray0(boolean[] a, int offset, VectorMask<$Boxtype$> m) {
1180         return super.fromBooleanArray0Template($masktype$.class, a, offset, ($masktype$) m);  // specialize
1181     }
1182 #end[byte]
1183 
1184     @ForceInline
1185     @Override
1186     final
1187     $abstractvectortype$ fromMemorySegment0(MemorySegment ms, long offset) {
1188         return super.fromMemorySegment0Template(ms, offset);  // specialize
1189     }
1190 
1191     @ForceInline
1192     @Override
1193     final
1194     $abstractvectortype$ fromMemorySegment0(MemorySegment ms, long offset, VectorMask<$Boxtype$> m) {
1195         return super.fromMemorySegment0Template($masktype$.class, ms, offset, ($masktype$) m);  // specialize
1196     }
1197 
1198     @ForceInline
1199     @Override
1200     final
1201     void intoArray0($type$[] a, int offset) {
1202         super.intoArray0Template(a, offset);  // specialize
1203     }
1204 
1205     @ForceInline
1206     @Override
1207     final
1208     void intoArray0($type$[] a, int offset, VectorMask<$Boxtype$> m) {
1209         super.intoArray0Template($masktype$.class, a, offset, ($masktype$) m);
1210     }
1211 
1212 #if[!byteOrShort]
1213     @ForceInline
1214     @Override
1215     final
1216     void intoArray0($type$[] a, int offset, int[] indexMap, int mapOffset, VectorMask<$Boxtype$> m) {
1217         super.intoArray0Template($masktype$.class, a, offset, indexMap, mapOffset, ($masktype$) m);
1218     }
1219 #end[!byteOrShort]
1220 
1221 #if[byte]
1222     @ForceInline
1223     @Override
1224     final
1225     void intoBooleanArray0(boolean[] a, int offset, VectorMask<$Boxtype$> m) {
1226         super.intoBooleanArray0Template($masktype$.class, a, offset, ($masktype$) m);
1227     }
1228 #end[byte]
1229 
1230     @ForceInline
1231     @Override
1232     final
1233     void intoMemorySegment0(MemorySegment ms, long offset, VectorMask<$Boxtype$> m) {
1234         super.intoMemorySegment0Template($masktype$.class, ms, offset, ($masktype$) m);
1235     }
1236 
1237 #if[short]
1238     @ForceInline
1239     @Override
1240     final
1241     void intoCharArray0(char[] a, int offset, VectorMask<$Boxtype$> m) {
1242         super.intoCharArray0Template($masktype$.class, a, offset, ($masktype$) m);
1243     }
1244 #end[short]
1245 
1246     // End of specialized low-level memory operations.
1247 
1248     // ================================================
1249 
1250 }