1 /*
   2  * Copyright (c) 2017, 2021, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.  Oracle designates this
   8  * particular file as subject to the "Classpath" exception as provided
   9  * by Oracle in the LICENSE file that accompanied this code.
  10  *
  11  * This code is distributed in the hope that it will be useful, but WITHOUT
  12  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  14  * version 2 for more details (a copy is included in the LICENSE file that
  15  * accompanied this code).
  16  *
  17  * You should have received a copy of the GNU General Public License version
  18  * 2 along with this work; if not, write to the Free Software Foundation,
  19  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  20  *
  21  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  22  * or visit www.oracle.com if you need additional information or have any
  23  * questions.
  24  */
  25 package jdk.incubator.vector;
  26 
  27 import java.nio.ByteBuffer;
  28 import java.util.Arrays;
  29 import java.util.Objects;
  30 import java.util.function.IntUnaryOperator;
  31 
  32 import jdk.internal.vm.annotation.ForceInline;
  33 import jdk.internal.vm.vector.VectorSupport;
  34 
  35 import static jdk.internal.vm.vector.VectorSupport.*;
  36 
  37 import static jdk.incubator.vector.VectorOperators.*;
  38 
  39 #warn This file is preprocessed before being compiled
  40 
  41 @SuppressWarnings("cast")  // warning: redundant cast
  42 final class $vectortype$ extends $abstractvectortype$ {
  43     static final $Type$Species VSPECIES =
  44         ($Type$Species) $Type$Vector.SPECIES_$BITS$;
  45 
  46     static final VectorShape VSHAPE =
  47         VSPECIES.vectorShape();
  48 
  49     static final Class<$vectortype$> VCLASS = $vectortype$.class;
  50 
  51     static final int VSIZE = VSPECIES.vectorBitSize();
  52 
  53     static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM
  54 
  55     static final Class<$Boxtype$> ETYPE = $type$.class; // used by the JVM
  56 
  57     $vectortype$($type$[] v) {
  58         super(v);
  59     }
  60 
  61     // For compatibility as $vectortype$::new,
  62     // stored into species.vectorFactory.
  63     $vectortype$(Object v) {
  64         this(($type$[]) v);
  65     }
  66 
  67     static final $vectortype$ ZERO = new $vectortype$(new $type$[VLENGTH]);
  68     static final $vectortype$ IOTA = new $vectortype$(VSPECIES.iotaArray());
  69 
  70     static {
  71         // Warm up a few species caches.
  72         // If we do this too much we will
  73         // get NPEs from bootstrap circularity.
  74         VSPECIES.dummyVector();
  75         VSPECIES.withLanes(LaneType.BYTE);
  76     }
  77 
  78     // Specialized extractors
  79 
  80     @ForceInline
  81     final @Override
  82     public $Type$Species vspecies() {
  83         // ISSUE:  This should probably be a @Stable
  84         // field inside AbstractVector, rather than
  85         // a megamorphic method.
  86         return VSPECIES;
  87     }
  88 
  89     @ForceInline
  90     @Override
  91     public final Class<$Boxtype$> elementType() { return $type$.class; }
  92 
  93     @ForceInline
  94     @Override
  95     public final int elementSize() { return $Boxtype$.SIZE; }
  96 
  97     @ForceInline
  98     @Override
  99     public final VectorShape shape() { return VSHAPE; }
 100 
 101     @ForceInline
 102     @Override
 103     public final int length() { return VLENGTH; }
 104 
 105     @ForceInline
 106     @Override
 107     public final int bitSize() { return VSIZE; }
 108 
 109     @ForceInline
 110     @Override
 111     public final int byteSize() { return VSIZE / Byte.SIZE; }
 112 
 113     /*package-private*/
 114     @ForceInline
 115     final @Override
 116     $type$[] vec() {
 117         return ($type$[])getPayload();
 118     }
 119 
 120     // Virtualized constructors
 121 
 122     @Override
 123     @ForceInline
 124     public final $vectortype$ broadcast($type$ e) {
 125         return ($vectortype$) super.broadcastTemplate(e);  // specialize
 126     }
 127 
 128 #if[!long]
 129     @Override
 130     @ForceInline
 131     public final $vectortype$ broadcast(long e) {
 132         return ($vectortype$) super.broadcastTemplate(e);  // specialize
 133     }
 134 #end[!long]
 135 
 136     @Override
 137     @ForceInline
 138     $masktype$ maskFromArray(boolean[] bits) {
 139         return new $masktype$(bits);
 140     }
 141 
 142     @Override
 143     @ForceInline
 144     $shuffletype$ iotaShuffle() { return $shuffletype$.IOTA; }
 145 
 146     @ForceInline
 147     $shuffletype$ iotaShuffle(int start, int step, boolean wrap) {
 148       if (wrap) {
 149         return ($shuffletype$)VectorSupport.shuffleIota(ETYPE, $shuffletype$.class, VSPECIES, VLENGTH, start, step, 1,
 150                 (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i*lstep + lstart, l))));
 151       } else {
 152         return ($shuffletype$)VectorSupport.shuffleIota(ETYPE, $shuffletype$.class, VSPECIES, VLENGTH, start, step, 0,
 153                 (l, lstart, lstep, s) -> s.shuffleFromOp(i -> (i*lstep + lstart)));
 154       }
 155     }
 156 
 157     @Override
 158     @ForceInline
 159     $shuffletype$ shuffleFromBytes(byte[] reorder) { return new $shuffletype$(reorder); }
 160 
 161     @Override
 162     @ForceInline
 163     $shuffletype$ shuffleFromArray(int[] indexes, int i) { return new $shuffletype$(indexes, i); }
 164 
 165     @Override
 166     @ForceInline
 167     $shuffletype$ shuffleFromOp(IntUnaryOperator fn) { return new $shuffletype$(fn); }
 168 
 169     // Make a vector of the same species but the given elements:
 170     @ForceInline
 171     final @Override
 172     $vectortype$ vectorFactory($type$[] vec) {
 173         return new $vectortype$(vec);
 174     }
 175 
 176     @ForceInline
 177     final @Override
 178     Byte$bits$Vector asByteVectorRaw() {
 179         return (Byte$bits$Vector) super.asByteVectorRawTemplate();  // specialize
 180     }
 181 
 182     @ForceInline
 183     final @Override
 184     AbstractVector<?> asVectorRaw(LaneType laneType) {
 185         return super.asVectorRawTemplate(laneType);  // specialize
 186     }
 187 
 188     // Unary operator
 189 
 190     @ForceInline
 191     final @Override
 192     $vectortype$ uOp(FUnOp f) {
 193         return ($vectortype$) super.uOpTemplate(f);  // specialize
 194     }
 195 
 196     @ForceInline
 197     final @Override
 198     $vectortype$ uOp(VectorMask<$Boxtype$> m, FUnOp f) {
 199         return ($vectortype$)
 200             super.uOpTemplate(($masktype$)m, f);  // specialize
 201     }
 202 
 203     // Binary operator
 204 
 205     @ForceInline
 206     final @Override
 207     $vectortype$ bOp(Vector<$Boxtype$> v, FBinOp f) {
 208         return ($vectortype$) super.bOpTemplate(($vectortype$)v, f);  // specialize
 209     }
 210 
 211     @ForceInline
 212     final @Override
 213     $vectortype$ bOp(Vector<$Boxtype$> v,
 214                      VectorMask<$Boxtype$> m, FBinOp f) {
 215         return ($vectortype$)
 216             super.bOpTemplate(($vectortype$)v, ($masktype$)m,
 217                               f);  // specialize
 218     }
 219 
 220     // Ternary operator
 221 
 222     @ForceInline
 223     final @Override
 224     $vectortype$ tOp(Vector<$Boxtype$> v1, Vector<$Boxtype$> v2, FTriOp f) {
 225         return ($vectortype$)
 226             super.tOpTemplate(($vectortype$)v1, ($vectortype$)v2,
 227                               f);  // specialize
 228     }
 229 
 230     @ForceInline
 231     final @Override
 232     $vectortype$ tOp(Vector<$Boxtype$> v1, Vector<$Boxtype$> v2,
 233                      VectorMask<$Boxtype$> m, FTriOp f) {
 234         return ($vectortype$)
 235             super.tOpTemplate(($vectortype$)v1, ($vectortype$)v2,
 236                               ($masktype$)m, f);  // specialize
 237     }
 238 
 239     @ForceInline
 240     final @Override
 241     $type$ rOp($type$ v, FBinOp f) {
 242         return super.rOpTemplate(v, f);  // specialize
 243     }
 244 
 245     @Override
 246     @ForceInline
 247     public final <F>
 248     Vector<F> convertShape(VectorOperators.Conversion<$Boxtype$,F> conv,
 249                            VectorSpecies<F> rsp, int part) {
 250         return super.convertShapeTemplate(conv, rsp, part);  // specialize
 251     }
 252 
 253     @Override
 254     @ForceInline
 255     public final <F>
 256     Vector<F> reinterpretShape(VectorSpecies<F> toSpecies, int part) {
 257         return super.reinterpretShapeTemplate(toSpecies, part);  // specialize
 258     }
 259 
 260     // Specialized algebraic operations:
 261 
 262     // The following definition forces a specialized version of this
 263     // crucial method into the v-table of this class.  A call to add()
 264     // will inline to a call to lanewise(ADD,), at which point the JIT
 265     // intrinsic will have the opcode of ADD, plus all the metadata
 266     // for this particular class, enabling it to generate precise
 267     // code.
 268     //
 269     // There is probably no benefit to the JIT to specialize the
 270     // masked or broadcast versions of the lanewise method.
 271 
 272     @Override
 273     @ForceInline
 274     public $vectortype$ lanewise(Unary op) {
 275         return ($vectortype$) super.lanewiseTemplate(op);  // specialize
 276     }
 277 
 278     @Override
 279     @ForceInline
 280     public $vectortype$ lanewise(Binary op, Vector<$Boxtype$> v) {
 281         return ($vectortype$) super.lanewiseTemplate(op, v);  // specialize
 282     }
 283 
 284 #if[!FP]
 285     /*package-private*/
 286     @Override
 287     @ForceInline $vectortype$
 288     lanewiseShift(VectorOperators.Binary op, int e) {
 289         return ($vectortype$) super.lanewiseShiftTemplate(op, e);  // specialize
 290     }
 291 #end[!FP]
 292 
 293     /*package-private*/
 294     @Override
 295     @ForceInline
 296     public final
 297     $vectortype$
 298     lanewise(VectorOperators.Ternary op, Vector<$Boxtype$> v1, Vector<$Boxtype$> v2) {
 299         return ($vectortype$) super.lanewiseTemplate(op, v1, v2);  // specialize
 300     }
 301 
 302     @Override
 303     @ForceInline
 304     public final
 305     $vectortype$ addIndex(int scale) {
 306         return ($vectortype$) super.addIndexTemplate(scale);  // specialize
 307     }
 308 
 309     // Type specific horizontal reductions
 310 
 311     @Override
 312     @ForceInline
 313     public final $type$ reduceLanes(VectorOperators.Associative op) {
 314         return super.reduceLanesTemplate(op);  // specialized
 315     }
 316 
 317     @Override
 318     @ForceInline
 319     public final $type$ reduceLanes(VectorOperators.Associative op,
 320                                     VectorMask<$Boxtype$> m) {
 321         return super.reduceLanesTemplate(op, m);  // specialized
 322     }
 323 
 324     @Override
 325     @ForceInline
 326     public final long reduceLanesToLong(VectorOperators.Associative op) {
 327         return (long) super.reduceLanesTemplate(op);  // specialized
 328     }
 329 
 330     @Override
 331     @ForceInline
 332     public final long reduceLanesToLong(VectorOperators.Associative op,
 333                                         VectorMask<$Boxtype$> m) {
 334         return (long) super.reduceLanesTemplate(op, m);  // specialized
 335     }
 336 
 337     @ForceInline
 338     public VectorShuffle<$Boxtype$> toShuffle() {
 339         return super.toShuffleTemplate($shuffletype$.class); // specialize
 340     }
 341 
 342     // Specialized unary testing
 343 
 344     @Override
 345     @ForceInline
 346     public final $masktype$ test(Test op) {
 347         return super.testTemplate($masktype$.class, op);  // specialize
 348     }
 349 
 350     // Specialized comparisons
 351 
 352     @Override
 353     @ForceInline
 354     public final $masktype$ compare(Comparison op, Vector<$Boxtype$> v) {
 355         return super.compareTemplate($masktype$.class, op, v);  // specialize
 356     }
 357 
 358     @Override
 359     @ForceInline
 360     public final $masktype$ compare(Comparison op, $type$ s) {
 361         return super.compareTemplate($masktype$.class, op, s);  // specialize
 362     }
 363 
 364 #if[!long]
 365     @Override
 366     @ForceInline
 367     public final $masktype$ compare(Comparison op, long s) {
 368         return super.compareTemplate($masktype$.class, op, s);  // specialize
 369     }
 370 #end[!long]
 371 
 372     @Override
 373     @ForceInline
 374     public $vectortype$ blend(Vector<$Boxtype$> v, VectorMask<$Boxtype$> m) {
 375         return ($vectortype$)
 376             super.blendTemplate($masktype$.class,
 377                                 ($vectortype$) v,
 378                                 ($masktype$) m);  // specialize
 379     }
 380 
 381     @Override
 382     @ForceInline
 383     public $vectortype$ slice(int origin, Vector<$Boxtype$> v) {
 384         return ($vectortype$) super.sliceTemplate(origin, v);  // specialize
 385     }
 386 
 387     @Override
 388     @ForceInline
 389     public $vectortype$ slice(int origin) {
 390         return ($vectortype$) super.sliceTemplate(origin);  // specialize
 391     }
 392 
 393     @Override
 394     @ForceInline
 395     public $vectortype$ unslice(int origin, Vector<$Boxtype$> w, int part) {
 396         return ($vectortype$) super.unsliceTemplate(origin, w, part);  // specialize
 397     }
 398 
 399     @Override
 400     @ForceInline
 401     public $vectortype$ unslice(int origin, Vector<$Boxtype$> w, int part, VectorMask<$Boxtype$> m) {
 402         return ($vectortype$)
 403             super.unsliceTemplate($masktype$.class,
 404                                   origin, w, part,
 405                                   ($masktype$) m);  // specialize
 406     }
 407 
 408     @Override
 409     @ForceInline
 410     public $vectortype$ unslice(int origin) {
 411         return ($vectortype$) super.unsliceTemplate(origin);  // specialize
 412     }
 413 
 414     @Override
 415     @ForceInline
 416     public $vectortype$ rearrange(VectorShuffle<$Boxtype$> s) {
 417         return ($vectortype$)
 418             super.rearrangeTemplate($shuffletype$.class,
 419                                     ($shuffletype$) s);  // specialize
 420     }
 421 
 422     @Override
 423     @ForceInline
 424     public $vectortype$ rearrange(VectorShuffle<$Boxtype$> shuffle,
 425                                   VectorMask<$Boxtype$> m) {
 426         return ($vectortype$)
 427             super.rearrangeTemplate($shuffletype$.class,
 428                                     ($shuffletype$) shuffle,
 429                                     ($masktype$) m);  // specialize
 430     }
 431 
 432     @Override
 433     @ForceInline
 434     public $vectortype$ rearrange(VectorShuffle<$Boxtype$> s,
 435                                   Vector<$Boxtype$> v) {
 436         return ($vectortype$)
 437             super.rearrangeTemplate($shuffletype$.class,
 438                                     ($shuffletype$) s,
 439                                     ($vectortype$) v);  // specialize
 440     }
 441 
 442     @Override
 443     @ForceInline
 444     public $vectortype$ selectFrom(Vector<$Boxtype$> v) {
 445         return ($vectortype$)
 446             super.selectFromTemplate(($vectortype$) v);  // specialize
 447     }
 448 
 449     @Override
 450     @ForceInline
 451     public $vectortype$ selectFrom(Vector<$Boxtype$> v,
 452                                    VectorMask<$Boxtype$> m) {
 453         return ($vectortype$)
 454             super.selectFromTemplate(($vectortype$) v,
 455                                      ($masktype$) m);  // specialize
 456     }
 457 
 458 
 459 #if[FP]
 460     @ForceInline
 461     @Override
 462     public $type$ lane(int i) {
 463 #if[!Max]
 464         $bitstype$ bits;
 465         switch(i) {
 466             case 0: bits = laneHelper(0); break;
 467 #if[!1L]
 468             case 1: bits = laneHelper(1); break;
 469 #if[!2L]
 470             case 2: bits = laneHelper(2); break;
 471             case 3: bits = laneHelper(3); break;
 472 #if[!4L]
 473             case 4: bits = laneHelper(4); break;
 474             case 5: bits = laneHelper(5); break;
 475             case 6: bits = laneHelper(6); break;
 476             case 7: bits = laneHelper(7); break;
 477 #if[!8L]
 478             case 8: bits = laneHelper(8); break;
 479             case 9: bits = laneHelper(9); break;
 480             case 10: bits = laneHelper(10); break;
 481             case 11: bits = laneHelper(11); break;
 482             case 12: bits = laneHelper(12); break;
 483             case 13: bits = laneHelper(13); break;
 484             case 14: bits = laneHelper(14); break;
 485             case 15: bits = laneHelper(15); break;
 486 #end[!8L]
 487 #end[!4L]
 488 #end[!2L]
 489 #end[!1L]
 490             default: throw new IllegalArgumentException("Index " + i + " must be zero or positive, and less than " + VLENGTH);
 491         }
 492 #else[!Max]
 493         if (i < 0 || i >= VLENGTH) {
 494             throw new IllegalArgumentException("Index " + i + " must be zero or positive, and less than " + VLENGTH);
 495         }
 496         $bitstype$ bits = laneHelper(i);
 497 #end[!Max]
 498         return $Type$.$bitstype$BitsTo$Fptype$(bits);
 499     }
 500 
 501     public $bitstype$ laneHelper(int i) {
 502         return ($bitstype$) VectorSupport.extract(
 503                      VCLASS, ETYPE, VLENGTH,
 504                      this, i,
 505                      (vec, ix) -> {
 506                      $type$[] vecarr = vec.vec();
 507                      return (long)$Type$.$type$To$Bitstype$Bits(vecarr[ix]);
 508                      });
 509     }
 510 
 511     @ForceInline
 512     @Override
 513     public $vectortype$ withLane(int i, $type$ e) {
 514 #if[!Max]
 515         switch(i) {
 516             case 0: return withLaneHelper(0, e);
 517 #if[!1L]
 518             case 1: return withLaneHelper(1, e);
 519 #if[!2L]
 520             case 2: return withLaneHelper(2, e);
 521             case 3: return withLaneHelper(3, e);
 522 #if[!4L]
 523             case 4: return withLaneHelper(4, e);
 524             case 5: return withLaneHelper(5, e);
 525             case 6: return withLaneHelper(6, e);
 526             case 7: return withLaneHelper(7, e);
 527 #if[!8L]
 528             case 8: return withLaneHelper(8, e);
 529             case 9: return withLaneHelper(9, e);
 530             case 10: return withLaneHelper(10, e);
 531             case 11: return withLaneHelper(11, e);
 532             case 12: return withLaneHelper(12, e);
 533             case 13: return withLaneHelper(13, e);
 534             case 14: return withLaneHelper(14, e);
 535             case 15: return withLaneHelper(15, e);
 536 #end[!8L]
 537 #end[!4L]
 538 #end[!2L]
 539 #end[!1L]
 540             default: throw new IllegalArgumentException("Index " + i + " must be zero or positive, and less than " + VLENGTH);
 541         }
 542 #else[!Max]
 543         if (i < 0 || i >= VLENGTH) {
 544             throw new IllegalArgumentException("Index " + i + " must be zero or positive, and less than " + VLENGTH);
 545         }
 546         return withLaneHelper(i, e);
 547 #end[!Max]
 548     }
 549 
 550     public $vectortype$ withLaneHelper(int i, $type$ e) {
 551         return VectorSupport.insert(
 552                                 VCLASS, ETYPE, VLENGTH,
 553                                 this, i, (long)$Type$.$type$To$Bitstype$Bits(e),
 554                                 (v, ix, bits) -> {
 555                                     $type$[] res = v.vec().clone();
 556                                     res[ix] = $Type$.$bitstype$BitsTo$Type$(($bitstype$)bits);
 557                                     return v.vectorFactory(res);
 558                                 });
 559     }
 560 #else[FP]
 561     @ForceInline
 562     @Override
 563     public $type$ lane(int i) {
 564 #if[!Max]
 565         switch(i) {
 566             case 0: return laneHelper(0);
 567 #if[!1L]
 568             case 1: return laneHelper(1);
 569 #if[!2L]
 570             case 2: return laneHelper(2);
 571             case 3: return laneHelper(3);
 572 #if[!4L]
 573             case 4: return laneHelper(4);
 574             case 5: return laneHelper(5);
 575             case 6: return laneHelper(6);
 576             case 7: return laneHelper(7);
 577 #if[!8L]
 578             case 8: return laneHelper(8);
 579             case 9: return laneHelper(9);
 580             case 10: return laneHelper(10);
 581             case 11: return laneHelper(11);
 582             case 12: return laneHelper(12);
 583             case 13: return laneHelper(13);
 584             case 14: return laneHelper(14);
 585             case 15: return laneHelper(15);
 586 #if[!16L]
 587             case 16: return laneHelper(16);
 588             case 17: return laneHelper(17);
 589             case 18: return laneHelper(18);
 590             case 19: return laneHelper(19);
 591             case 20: return laneHelper(20);
 592             case 21: return laneHelper(21);
 593             case 22: return laneHelper(22);
 594             case 23: return laneHelper(23);
 595             case 24: return laneHelper(24);
 596             case 25: return laneHelper(25);
 597             case 26: return laneHelper(26);
 598             case 27: return laneHelper(27);
 599             case 28: return laneHelper(28);
 600             case 29: return laneHelper(29);
 601             case 30: return laneHelper(30);
 602             case 31: return laneHelper(31);
 603 #if[!32L]
 604             case 32: return laneHelper(32);
 605             case 33: return laneHelper(33);
 606             case 34: return laneHelper(34);
 607             case 35: return laneHelper(35);
 608             case 36: return laneHelper(36);
 609             case 37: return laneHelper(37);
 610             case 38: return laneHelper(38);
 611             case 39: return laneHelper(39);
 612             case 40: return laneHelper(40);
 613             case 41: return laneHelper(41);
 614             case 42: return laneHelper(42);
 615             case 43: return laneHelper(43);
 616             case 44: return laneHelper(44);
 617             case 45: return laneHelper(45);
 618             case 46: return laneHelper(46);
 619             case 47: return laneHelper(47);
 620             case 48: return laneHelper(48);
 621             case 49: return laneHelper(49);
 622             case 50: return laneHelper(50);
 623             case 51: return laneHelper(51);
 624             case 52: return laneHelper(52);
 625             case 53: return laneHelper(53);
 626             case 54: return laneHelper(54);
 627             case 55: return laneHelper(55);
 628             case 56: return laneHelper(56);
 629             case 57: return laneHelper(57);
 630             case 58: return laneHelper(58);
 631             case 59: return laneHelper(59);
 632             case 60: return laneHelper(60);
 633             case 61: return laneHelper(61);
 634             case 62: return laneHelper(62);
 635             case 63: return laneHelper(63);
 636 #end[!32L]
 637 #end[!16L]
 638 #end[!8L]
 639 #end[!4L]
 640 #end[!2L]
 641 #end[!1L]
 642             default: throw new IllegalArgumentException("Index " + i + " must be zero or positive, and less than " + VLENGTH);
 643         }
 644 #else[!Max]
 645         if (i < 0 || i >= VLENGTH) {
 646             throw new IllegalArgumentException("Index " + i + " must be zero or positive, and less than " + VLENGTH);
 647         }
 648         return laneHelper(i);
 649 #end[!Max]
 650     }
 651 
 652     public $type$ laneHelper(int i) {
 653         return ($type$) VectorSupport.extract(
 654                                 VCLASS, ETYPE, VLENGTH,
 655                                 this, i,
 656                                 (vec, ix) -> {
 657                                     $type$[] vecarr = vec.vec();
 658                                     return (long)vecarr[ix];
 659                                 });
 660     }
 661 
 662     @ForceInline
 663     @Override
 664     public $vectortype$ withLane(int i, $type$ e) {
 665 #if[!Max]
 666         switch (i) {
 667             case 0: return withLaneHelper(0, e);
 668 #if[!1L]
 669             case 1: return withLaneHelper(1, e);
 670 #if[!2L]
 671             case 2: return withLaneHelper(2, e);
 672             case 3: return withLaneHelper(3, e);
 673 #if[!4L]
 674             case 4: return withLaneHelper(4, e);
 675             case 5: return withLaneHelper(5, e);
 676             case 6: return withLaneHelper(6, e);
 677             case 7: return withLaneHelper(7, e);
 678 #if[!8L]
 679             case 8: return withLaneHelper(8, e);
 680             case 9: return withLaneHelper(9, e);
 681             case 10: return withLaneHelper(10, e);
 682             case 11: return withLaneHelper(11, e);
 683             case 12: return withLaneHelper(12, e);
 684             case 13: return withLaneHelper(13, e);
 685             case 14: return withLaneHelper(14, e);
 686             case 15: return withLaneHelper(15, e);
 687 #if[!16L]
 688             case 16: return withLaneHelper(16, e);
 689             case 17: return withLaneHelper(17, e);
 690             case 18: return withLaneHelper(18, e);
 691             case 19: return withLaneHelper(19, e);
 692             case 20: return withLaneHelper(20, e);
 693             case 21: return withLaneHelper(21, e);
 694             case 22: return withLaneHelper(22, e);
 695             case 23: return withLaneHelper(23, e);
 696             case 24: return withLaneHelper(24, e);
 697             case 25: return withLaneHelper(25, e);
 698             case 26: return withLaneHelper(26, e);
 699             case 27: return withLaneHelper(27, e);
 700             case 28: return withLaneHelper(28, e);
 701             case 29: return withLaneHelper(29, e);
 702             case 30: return withLaneHelper(30, e);
 703             case 31: return withLaneHelper(31, e);
 704 #if[!32L]
 705             case 32: return withLaneHelper(32, e);
 706             case 33: return withLaneHelper(33, e);
 707             case 34: return withLaneHelper(34, e);
 708             case 35: return withLaneHelper(35, e);
 709             case 36: return withLaneHelper(36, e);
 710             case 37: return withLaneHelper(37, e);
 711             case 38: return withLaneHelper(38, e);
 712             case 39: return withLaneHelper(39, e);
 713             case 40: return withLaneHelper(40, e);
 714             case 41: return withLaneHelper(41, e);
 715             case 42: return withLaneHelper(42, e);
 716             case 43: return withLaneHelper(43, e);
 717             case 44: return withLaneHelper(44, e);
 718             case 45: return withLaneHelper(45, e);
 719             case 46: return withLaneHelper(46, e);
 720             case 47: return withLaneHelper(47, e);
 721             case 48: return withLaneHelper(48, e);
 722             case 49: return withLaneHelper(49, e);
 723             case 50: return withLaneHelper(50, e);
 724             case 51: return withLaneHelper(51, e);
 725             case 52: return withLaneHelper(52, e);
 726             case 53: return withLaneHelper(53, e);
 727             case 54: return withLaneHelper(54, e);
 728             case 55: return withLaneHelper(55, e);
 729             case 56: return withLaneHelper(56, e);
 730             case 57: return withLaneHelper(57, e);
 731             case 58: return withLaneHelper(58, e);
 732             case 59: return withLaneHelper(59, e);
 733             case 60: return withLaneHelper(60, e);
 734             case 61: return withLaneHelper(61, e);
 735             case 62: return withLaneHelper(62, e);
 736             case 63: return withLaneHelper(63, e);
 737 #end[!32L]
 738 #end[!16L]
 739 #end[!8L]
 740 #end[!4L]
 741 #end[!2L]
 742 #end[!1L]
 743             default: throw new IllegalArgumentException("Index " + i + " must be zero or positive, and less than " + VLENGTH);
 744         }
 745 #else[!Max]
 746         if (i < 0 || i >= VLENGTH) {
 747             throw new IllegalArgumentException("Index " + i + " must be zero or positive, and less than " + VLENGTH);
 748         }
 749         return withLaneHelper(i, e);
 750 #end[!Max]
 751     }
 752 
 753     public $vectortype$ withLaneHelper(int i, $type$ e) {
 754         return VectorSupport.insert(
 755                                 VCLASS, ETYPE, VLENGTH,
 756                                 this, i, (long)e,
 757                                 (v, ix, bits) -> {
 758                                     $type$[] res = v.vec().clone();
 759                                     res[ix] = ($type$)bits;
 760                                     return v.vectorFactory(res);
 761                                 });
 762     }
 763 #end[FP]
 764 
 765     // Mask
 766 
 767     static final class $masktype$ extends AbstractMask<$Boxtype$> {
 768         static final int VLENGTH = VSPECIES.laneCount();    // used by the JVM
 769         static final Class<$Boxtype$> ETYPE = $type$.class; // used by the JVM
 770 
 771         $masktype$(boolean[] bits) {
 772             this(bits, 0);
 773         }
 774 
 775         $masktype$(boolean[] bits, int offset) {
 776             super(prepare(bits, offset));
 777         }
 778 
 779         $masktype$(boolean val) {
 780             super(prepare(val));
 781         }
 782 
 783         private static boolean[] prepare(boolean[] bits, int offset) {
 784             boolean[] newBits = new boolean[VSPECIES.laneCount()];
 785             for (int i = 0; i < newBits.length; i++) {
 786                 newBits[i] = bits[offset + i];
 787             }
 788             return newBits;
 789         }
 790 
 791         private static boolean[] prepare(boolean val) {
 792             boolean[] bits = new boolean[VSPECIES.laneCount()];
 793             Arrays.fill(bits, val);
 794             return bits;
 795         }
 796 
 797         @ForceInline
 798         final @Override
 799         public $Type$Species vspecies() {
 800             // ISSUE:  This should probably be a @Stable
 801             // field inside AbstractMask, rather than
 802             // a megamorphic method.
 803             return VSPECIES;
 804         }
 805 
 806         @ForceInline
 807         boolean[] getBits() {
 808             return (boolean[])getPayload();
 809         }
 810 
 811         @Override
 812         $masktype$ uOp(MUnOp f) {
 813             boolean[] res = new boolean[vspecies().laneCount()];
 814             boolean[] bits = getBits();
 815             for (int i = 0; i < res.length; i++) {
 816                 res[i] = f.apply(i, bits[i]);
 817             }
 818             return new $masktype$(res);
 819         }
 820 
 821         @Override
 822         $masktype$ bOp(VectorMask<$Boxtype$> m, MBinOp f) {
 823             boolean[] res = new boolean[vspecies().laneCount()];
 824             boolean[] bits = getBits();
 825             boolean[] mbits = (($masktype$)m).getBits();
 826             for (int i = 0; i < res.length; i++) {
 827                 res[i] = f.apply(i, bits[i], mbits[i]);
 828             }
 829             return new $masktype$(res);
 830         }
 831 
 832         @ForceInline
 833         @Override
 834         public final
 835         $vectortype$ toVector() {
 836             return ($vectortype$) super.toVectorTemplate();  // specialize
 837         }
 838 
 839         /**
 840          * Helper function for lane-wise mask conversions.
 841          * This function kicks in after intrinsic failure.
 842          */
 843         @ForceInline
 844         private final <E>
 845         VectorMask<E> defaultMaskCast(AbstractSpecies<E> dsp) {
 846             if (length() != dsp.laneCount())
 847                 throw new IllegalArgumentException("VectorMask length and species length differ");
 848             boolean[] maskArray = toArray();
 849             return  dsp.maskFactory(maskArray).check(dsp);
 850         }
 851 
 852         @Override
 853         @ForceInline
 854         public <E> VectorMask<E> cast(VectorSpecies<E> dsp) {
 855             AbstractSpecies<E> species = (AbstractSpecies<E>) dsp;
 856             if (length() != species.laneCount())
 857                 throw new IllegalArgumentException("VectorMask length and species length differ");
 858             if (VSIZE == species.vectorBitSize()) {
 859                 Class<?> dtype = species.elementType();
 860                 Class<?> dmtype = species.maskType();
 861                 return VectorSupport.convert(VectorSupport.VECTOR_OP_REINTERPRET,
 862                     this.getClass(), ETYPE, VLENGTH,
 863                     dmtype, dtype, VLENGTH,
 864                     this, species,
 865                     $Type$$bits$Mask::defaultMaskCast);
 866             }
 867             return this.defaultMaskCast(species);
 868         }
 869 
 870         @Override
 871         @ForceInline
 872         public $masktype$ eq(VectorMask<$Boxtype$> mask) {
 873             Objects.requireNonNull(mask);
 874             $masktype$ m = ($masktype$)mask;
 875             return xor(m.not());
 876         }
 877 
 878         // Unary operations
 879 
 880         @Override
 881         @ForceInline
 882         public $masktype$ not() {
 883             return xor(maskAll(true));
 884         }
 885 
 886         // Binary operations
 887 
 888         @Override
 889         @ForceInline
 890         public $masktype$ and(VectorMask<$Boxtype$> mask) {
 891             Objects.requireNonNull(mask);
 892             $masktype$ m = ($masktype$)mask;
 893             return VectorSupport.binaryOp(VECTOR_OP_AND, $masktype$.class, $bitstype$.class, VLENGTH,
 894                                              this, m,
 895                                              (m1, m2) -> m1.bOp(m2, (i, a, b) -> a & b));
 896         }
 897 
 898         @Override
 899         @ForceInline
 900         public $masktype$ or(VectorMask<$Boxtype$> mask) {
 901             Objects.requireNonNull(mask);
 902             $masktype$ m = ($masktype$)mask;
 903             return VectorSupport.binaryOp(VECTOR_OP_OR, $masktype$.class, $bitstype$.class, VLENGTH,
 904                                              this, m,
 905                                              (m1, m2) -> m1.bOp(m2, (i, a, b) -> a | b));
 906         }
 907 
 908         @ForceInline
 909         /* package-private */
 910         $masktype$ xor(VectorMask<$Boxtype$> mask) {
 911             Objects.requireNonNull(mask);
 912             $masktype$ m = ($masktype$)mask;
 913             return VectorSupport.binaryOp(VECTOR_OP_XOR, $masktype$.class, $bitstype$.class, VLENGTH,
 914                                           this, m,
 915                                           (m1, m2) -> m1.bOp(m2, (i, a, b) -> a ^ b));
 916         }
 917 
 918         // Mask Query operations
 919 
 920         @Override
 921         @ForceInline
 922         public int trueCount() {
 923             return VectorSupport.maskReductionCoerced(VECTOR_OP_MASK_TRUECOUNT, $masktype$.class, $bitstype$.class, VLENGTH, this,
 924                                                       (m) -> trueCountHelper((($masktype$)m).getBits()));
 925         }
 926 
 927         @Override
 928         @ForceInline
 929         public int firstTrue() {
 930             return VectorSupport.maskReductionCoerced(VECTOR_OP_MASK_FIRSTTRUE, $masktype$.class, $bitstype$.class, VLENGTH, this,
 931                                                       (m) -> firstTrueHelper((($masktype$)m).getBits()));
 932         }
 933 
 934         @Override
 935         @ForceInline
 936         public int lastTrue() {
 937             return VectorSupport.maskReductionCoerced(VECTOR_OP_MASK_LASTTRUE, $masktype$.class, $bitstype$.class, VLENGTH, this,
 938                                                       (m) -> lastTrueHelper((($masktype$)m).getBits()));
 939         }
 940 
 941         // Reductions
 942 
 943         @Override
 944         @ForceInline
 945         public boolean anyTrue() {
 946             return VectorSupport.test(BT_ne, $masktype$.class, $bitstype$.class, VLENGTH,
 947                                          this, vspecies().maskAll(true),
 948                                          (m, __) -> anyTrueHelper((($masktype$)m).getBits()));
 949         }
 950 
 951         @Override
 952         @ForceInline
 953         public boolean allTrue() {
 954             return VectorSupport.test(BT_overflow, $masktype$.class, $bitstype$.class, VLENGTH,
 955                                          this, vspecies().maskAll(true),
 956                                          (m, __) -> allTrueHelper((($masktype$)m).getBits()));
 957         }
 958 
 959         @ForceInline
 960         /*package-private*/
 961         static $masktype$ maskAll(boolean bit) {
 962             return VectorSupport.broadcastCoerced($masktype$.class, $bitstype$.class, VLENGTH,
 963                                                   (bit ? -1 : 0), null,
 964                                                   (v, __) -> (v != 0 ? TRUE_MASK : FALSE_MASK));
 965         }
 966         private static final $masktype$  TRUE_MASK = new $masktype$(true);
 967         private static final $masktype$ FALSE_MASK = new $masktype$(false);
 968 
 969 #if[intAndMax]
 970 
 971         static boolean[] maskLowerHalf() {
 972             boolean[] a = new boolean[VLENGTH];
 973             int len = a.length >> 1;
 974             for (int i = 0; i < len; i++) {
 975                 a[i] = true;
 976             }
 977             return a;
 978         }
 979 
 980 #end[intAndMax]
 981 #if[intAndMax]
 982         static final IntMaxMask LOWER_HALF_TRUE_MASK = new IntMaxMask(maskLowerHalf());
 983 #end[intAndMax]
 984     }
 985 
 986     // Shuffle
 987 
 988     static final class $shuffletype$ extends AbstractShuffle<$Boxtype$> {
 989         static final int VLENGTH = VSPECIES.laneCount();    // used by the JVM
 990         static final Class<$Boxtype$> ETYPE = $type$.class; // used by the JVM
 991 
 992         $shuffletype$(byte[] reorder) {
 993             super(VLENGTH, reorder);
 994         }
 995 
 996         public $shuffletype$(int[] reorder) {
 997             super(VLENGTH, reorder);
 998         }
 999 
1000         public $shuffletype$(int[] reorder, int i) {
1001             super(VLENGTH, reorder, i);
1002         }
1003 
1004         public $shuffletype$(IntUnaryOperator fn) {
1005             super(VLENGTH, fn);
1006         }
1007 
1008         @Override
1009         public $Type$Species vspecies() {
1010             return VSPECIES;
1011         }
1012 
1013         static {
1014             // There must be enough bits in the shuffle lanes to encode
1015             // VLENGTH valid indexes and VLENGTH exceptional ones.
1016             assert(VLENGTH < Byte.MAX_VALUE);
1017             assert(Byte.MIN_VALUE <= -VLENGTH);
1018         }
1019         static final $shuffletype$ IOTA = new $shuffletype$(IDENTITY);
1020 
1021         @Override
1022         @ForceInline
1023         public $vectortype$ toVector() {
1024             return VectorSupport.shuffleToVector(VCLASS, ETYPE, $shuffletype$.class, this, VLENGTH,
1025                                                     (s) -> (($vectortype$)(((AbstractShuffle<$Boxtype$>)(s)).toVectorTemplate())));
1026         }
1027 
1028         @Override
1029         @ForceInline
1030         public <F> VectorShuffle<F> cast(VectorSpecies<F> s) {
1031             AbstractSpecies<F> species = (AbstractSpecies<F>) s;
1032             if (length() != species.laneCount())
1033                 throw new IllegalArgumentException("VectorShuffle length and species length differ");
1034             int[] shuffleArray = toArray();
1035             return s.shuffleFromArray(shuffleArray, 0).check(s);
1036         }
1037 
1038         @ForceInline
1039         @Override
1040         public $shuffletype$ rearrange(VectorShuffle<$Boxtype$> shuffle) {
1041             $shuffletype$ s = ($shuffletype$) shuffle;
1042             byte[] reorder1 = reorder();
1043             byte[] reorder2 = s.reorder();
1044             byte[] r = new byte[reorder1.length];
1045             for (int i = 0; i < reorder1.length; i++) {
1046                 int ssi = reorder2[i];
1047                 r[i] = reorder1[ssi];  // throws on exceptional index
1048             }
1049             return new $shuffletype$(r);
1050         }
1051     }
1052 
1053     // ================================================
1054 
1055     // Specialized low-level memory operations.
1056 
1057     @ForceInline
1058     @Override
1059     final
1060     $abstractvectortype$ fromArray0($type$[] a, int offset) {
1061         return super.fromArray0Template(a, offset);  // specialize
1062     }
1063 
1064 #if[short]
1065     @ForceInline
1066     @Override
1067     final
1068     $abstractvectortype$ fromCharArray0(char[] a, int offset) {
1069         return super.fromCharArray0Template(a, offset);  // specialize
1070     }
1071 #end[short]
1072 
1073 #if[byte]
1074     @ForceInline
1075     @Override
1076     final
1077     $abstractvectortype$ fromBooleanArray0(boolean[] a, int offset) {
1078         return super.fromBooleanArray0Template(a, offset);  // specialize
1079     }
1080 #end[byte]
1081 
1082     @ForceInline
1083     @Override
1084     final
1085     $abstractvectortype$ fromByteArray0(byte[] a, int offset) {
1086         return super.fromByteArray0Template(a, offset);  // specialize
1087     }
1088 
1089     @ForceInline
1090     @Override
1091     final
1092     $abstractvectortype$ fromByteBuffer0(ByteBuffer bb, int offset) {
1093         return super.fromByteBuffer0Template(bb, offset);  // specialize
1094     }
1095 
1096     @ForceInline
1097     @Override
1098     final
1099     void intoArray0($type$[] a, int offset) {
1100         super.intoArray0Template(a, offset);  // specialize
1101     }
1102 
1103     @ForceInline
1104     @Override
1105     final
1106     void intoByteArray0(byte[] a, int offset) {
1107         super.intoByteArray0Template(a, offset);  // specialize
1108     }
1109 
1110     // End of specialized low-level memory operations.
1111 
1112     // ================================================
1113 
1114 }